[chore] Remove logrus from circleci, filesystem, gitlab, and s3 sources (#1089)

* [chore] Remove logrus from circleci, filesystem, gitlab, and s3 sources

* Address comments

pkg/engine/circleci.go

@@ -29,6 +29,10 @@ func (e *Engine) ScanCircleCI(ctx context.Context, token string) error {
 	}
 
 	circleSource := circleci.Source{}
+	ctx = context.WithValues(ctx,
+		"source_type", circleSource.Type().String(),
+		"source_name", "Circle CI",
+	)
 	err = circleSource.Init(ctx, "trufflehog - Circle CI", 0, int64(sourcespb.SourceType_SOURCE_TYPE_CIRCLECI), true, &conn, runtime.NumCPU())
 	if err != nil {
 		return errors.WrapPrefix(err, "failed to init Circle CI source", 0)

pkg/engine/filesystem.go

@@ -27,6 +27,11 @@ func (e *Engine) ScanFileSystem(ctx context.Context, c sources.Config) error {
 	}
 
 	fileSystemSource := filesystem.Source{}
+
+	ctx = context.WithValues(ctx,
+		"source_type", fileSystemSource.Type().String(),
+		"source_name", "filesystem",
+	)
 	err = fileSystemSource.Init(ctx, "trufflehog - filesystem", 0, int64(sourcespb.SourceType_SOURCE_TYPE_FILESYSTEM), true, &conn, runtime.NumCPU())
 	if err != nil {
 		return errors.WrapPrefix(err, "could not init filesystem source", 0)

pkg/engine/git.go

@@ -54,6 +54,10 @@ func (e *Engine) ScanGit(ctx context.Context, c sources.Config) error {
 			}
 		})
 
+	ctx = context.WithValues(ctx,
+		"source_type", sourcespb.SourceType_SOURCE_TYPE_GIT.String(),
+		"source_name", "git",
+	)
 	e.sourcesWg.Add(1)
 	go func() {
 		defer common.RecoverWithExit(ctx)

pkg/engine/github.go

@@ -38,8 +38,8 @@ func (e *Engine) ScanGitHub(ctx context.Context, c sources.Config) error {
 		return err
 	}
 	ctx = context.WithValues(ctx,
-		"source_type", source.Type(),
+		"source_type", source.Type().String(),
-		"name", "trufflehog - github",
+		"source_name", "github",
 	)
 	err = source.Init(ctx, "trufflehog - github", 0, 0, false, &conn, c.Concurrency)
 	if err != nil {

pkg/engine/gitlab.go

@@ -53,6 +53,10 @@ func (e *Engine) ScanGitLab(ctx context.Context, c sources.Config) error {
 	}
 
 	gitlabSource := gitlab.Source{}
+	ctx = context.WithValues(ctx,
+		"source_type", gitlabSource.Type().String(),
+		"source_name", "gitlab",
+	)
 	err = gitlabSource.Init(ctx, "trufflehog - gitlab", 0, int64(sourcespb.SourceType_SOURCE_TYPE_GITLAB), true, &conn, runtime.NumCPU())
 	if err != nil {
 		return errors.WrapPrefix(err, "could not init GitLab source", 0)

pkg/engine/s3.go

@@ -46,6 +46,10 @@ func (e *Engine) ScanS3(ctx context.Context, c sources.Config) error {
 	}
 
 	s3Source := s3.Source{}
+	ctx = context.WithValues(ctx,
+		"source_type", s3Source.Type().String(),
+		"source_name", "s3",
+	)
 	err = s3Source.Init(ctx, "trufflehog - s3", 0, int64(sourcespb.SourceType_SOURCE_TYPE_S3), true, &conn, runtime.NumCPU())
 	if err != nil {
 		return errors.WrapPrefix(err, "failed to init S3 source", 0)

pkg/engine/syslog.go

@@ -42,6 +42,10 @@ func (e *Engine) ScanSyslog(ctx context.Context, c sources.Config) error {
 		return errors.WrapPrefix(err, "error unmarshalling connection", 0)
 	}
 	source := syslog.Source{}
+	ctx = context.WithValues(ctx,
+		"source_type", source.Type().String(),
+		"source_name", "syslog",
+	)
 	err = source.Init(ctx, "trufflehog - syslog", 0, 0, false, &conn, c.Concurrency)
 	source.InjectConnection(connection)
 	if err != nil {

pkg/sources/circleci/circleci.go

@@ -9,7 +9,6 @@ import (
 	"sync/atomic"
 
 	"github.com/go-errors/errors"
-	log "github.com/sirupsen/logrus"
 	"golang.org/x/sync/errgroup"
 	"google.golang.org/protobuf/proto"
 	"google.golang.org/protobuf/types/known/anypb"
@@ -111,14 +110,14 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
 			}
 
 			atomic.AddUint64(&scanned, 1)
-			log.Debugf("scanned %d/%d projects", scanned, len(projects))
+			ctx.Logger().V(2).Info(fmt.Sprintf("scanned %d/%d projects", scanned, len(projects)))
 			return nil
 		})
 	}
 
 	_ = s.jobPool.Wait()
 	if scanErrs.Count() > 0 {
-		log.Debugf("encountered %d errors while scanning; errors: %v", scanErrs.Count(), scanErrs)
+		ctx.Logger().V(2).Info("encountered errors while scanning", "count", scanErrs.Count(), "errors", scanErrs)
 	}
 
 	return nil

pkg/sources/filesystem/filesystem.go

@@ -9,7 +9,7 @@ import (
 
 	diskbufferreader "github.com/bill-rich/disk-buffer-reader"
 	"github.com/go-errors/errors"
-	log "github.com/sirupsen/logrus"
+	"github.com/go-logr/logr"
 	"google.golang.org/protobuf/proto"
 	"google.golang.org/protobuf/types/known/anypb"
 
@@ -35,7 +35,7 @@ type Source struct {
 	jobId    int64
 	verify   bool
 	paths    []string
-	log      *log.Entry
+	log      logr.Logger
 	filter   *common.Filter
 	sources.Progress
 }
@@ -59,7 +59,7 @@ func (s *Source) JobID() int64 {
 
 // Init returns an initialized Filesystem source.
 func (s *Source) Init(aCtx context.Context, name string, jobId, sourceId int64, verify bool, connection *anypb.Any, _ int) error {
-	s.log = log.WithField("source", s.Type()).WithField("name", name)
+	s.log = aCtx.Logger()
 
 	s.name = name
 	s.sourceId = sourceId
@@ -98,10 +98,11 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
 			}
 
 			path := filepath.Join(cleanPath, relativePath)
+			logger := ctx.Logger().WithValues("file", path)
 
 			fileStat, err := os.Stat(path)
 			if err != nil {
-				log.WithError(err).Warnf("unable to stat file: %s", path)
+				logger.Error(err, "unable to stat file")
 				return nil
 			}
 			if !fileStat.Mode().IsRegular() {
@@ -114,15 +115,15 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
 
 			inputFile, err := os.Open(path)
 			if err != nil {
-				log.Warn(err)
+				logger.V(2).Info("error opening file", "error", err)
 				return nil
 			}
 			defer inputFile.Close()
-			log.WithField("file_path", path).Trace("scanning file")
+			logger.V(5).Info("scanning file")
 
 			reReader, err := diskbufferreader.New(inputFile)
 			if err != nil {
-				log.WithError(err).Error("Could not create re-readable reader.")
+				logger.Error(err, "Could not create re-readable reader.")
 			}
 			defer reReader.Close()
 

pkg/sources/filesystem/filesystem_test.go

@@ -5,7 +5,6 @@ import (
 	"time"
 
 	"github.com/kylelemons/godebug/pretty"
-	log "github.com/sirupsen/logrus"
 	"google.golang.org/protobuf/types/known/anypb"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
@@ -51,8 +50,6 @@ func TestSource_Scan(t *testing.T) {
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			s := Source{}
-			log.SetLevel(log.DebugLevel)
-			log.SetFormatter(&log.TextFormatter{ForceColors: true})
 
 			conn, err := anypb.New(tt.init.connection)
 			if err != nil {

pkg/sources/gitlab/gitlab.go

@@ -23,7 +23,6 @@ import (
 	"github.com/go-errors/errors"
 	gogit "github.com/go-git/go-git/v5"
 	"github.com/gobwas/glob"
-	log "github.com/sirupsen/logrus"
 	"github.com/xanzy/go-gitlab"
 	"golang.org/x/exp/slices"
 	"google.golang.org/protobuf/proto"
@@ -69,7 +68,6 @@ func (s *Source) JobID() int64 {
 
 // Init returns an initialized Gitlab source.
 func (s *Source) Init(_ context.Context, name string, jobId, sourceId int64, verify bool, connection *anypb.Any, concurrency int) error {
-
 	s.name = name
 	s.sourceId = sourceId
 	s.jobId = jobId
@@ -143,7 +141,7 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
 	// Get repo within target.
 	repos, errs := s.getRepos()
 	for _, repoErr := range errs {
-		log.WithError(repoErr).Warn("error getting repo")
+		ctx.Logger().Info("error getting repo", "error", repoErr)
 	}
 
 	// End early if we had errors getting specified repos but none were validated.
@@ -153,13 +151,13 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
 
 	// Get all repos if not specified.
 	if len(repos) == 0 {
-		projects, err := s.getAllProjects(apiClient)
+		projects, err := s.getAllProjects(ctx, apiClient)
 		if err != nil {
 			return fmt.Errorf("error getting all projects: %v", err)
 		}
 		// Turn projects into URLs for Git cloner.
 		for _, prj := range projects {
-			if s.ignoreRepo(prj.PathWithNamespace) {
+			if s.ignoreRepo(ctx, prj.PathWithNamespace) {
 				continue
 			}
 
@@ -231,7 +229,7 @@ func (s *Source) basicAuthSuccessful(apiClient *gitlab.Client) bool {
 	return false
 }
 
-func (s *Source) getAllProjects(apiClient *gitlab.Client) ([]*gitlab.Project, error) {
+func (s *Source) getAllProjects(ctx context.Context, apiClient *gitlab.Client) ([]*gitlab.Project, error) {
 	// Projects without repo will get user projects, groups projects, and subgroup projects.
 	user, _, err := apiClient.Users.CurrentUser()
 
@@ -289,7 +287,10 @@ func (s *Source) getAllProjects(apiClient *gitlab.Client) ([]*gitlab.Project, er
 		for {
 			grpPrjs, res, err := apiClient.Groups.ListGroupProjects(group.ID, listGroupProjectOptions)
 			if err != nil {
-				log.WithError(err).WithField("group", group.FullPath).Warn("received error on listing group projects, you probably don't have permissions to do that")
+				ctx.Logger().Info("received error on listing group projects, you probably don't have permissions to do that",
+					"group", group.FullPath,
+					"error", err,
+				)
 				break
 			}
 			for _, prj := range grpPrjs {
@@ -305,7 +306,7 @@ func (s *Source) getAllProjects(apiClient *gitlab.Client) ([]*gitlab.Project, er
 	for _, project := range projects {
 		projectNamesWithNamespace = append(projectNamesWithNamespace, project.NameWithNamespace)
 	}
-	log.WithField("projects", strings.Join(projectNamesWithNamespace, ", ")).Debugf("Enumerated %d GitLab projects", len(projects))
+	ctx.Logger().V(2).Info("Enumerated GitLab projects", "count", len(projects), "projects", projectNamesWithNamespace)
 
 	var projectList []*gitlab.Project
 	for _, project := range projects {
@@ -334,7 +335,6 @@ func (s *Source) getRepos() ([]string, []error) {
 }
 
 func (s *Source) scanRepos(ctx context.Context, chunksChan chan *sources.Chunk) error {
-
 	// If there is resume information available, limit this scan to only the repos that still need scanning.
 	reposToScan, progressIndexOffset := sources.FilterReposToResume(s.repos, s.GetProgress().EncodedResumeInfo)
 	s.repos = reposToScan
@@ -343,15 +343,16 @@ func (s *Source) scanRepos(ctx context.Context, chunksChan chan *sources.Chunk)
 	for i, repo := range s.repos {
 		i, repoURL := i, repo
 		s.jobPool.Go(func() error {
+			logger := ctx.Logger().WithValues("repo", repoURL)
 			if common.IsDone(ctx) {
 				// We are returning nil instead of the scanErrors slice here because
 				// we don't want to mark this scan as errored if we cancelled it.
-				log.Debugf("Skipping repo %s because context was cancelled", repoURL)
+				logger.V(2).Info("Skipping repo because context was cancelled")
 				return nil
 			}
 
 			if len(repoURL) == 0 {
-				log.Debugf("Skipping empty repo %s", repoURL)
+				logger.V(2).Info("Skipping empty repo")
 				return nil
 			}
 
@@ -383,35 +384,35 @@ func (s *Source) scanRepos(ctx context.Context, chunksChan chan *sources.Chunk)
 				return nil
 			}
 
-			log.Debugf("Starting to scan repo %d/%d: %s", i+1, len(s.repos), repo)
+			logger.V(2).Info(fmt.Sprintf("Starting to scan repo %d/%d", i+1, len(s.repos)))
 			if err = s.git.ScanRepo(ctx, repo, path, s.scanOptions, chunksChan); err != nil {
 				scanErrs.Add(err)
 				return nil
 			}
 
-			log.Debugf("Completed scanning repo %d/%d: %s", i+1, len(s.repos), repo)
+			logger.V(2).Info(fmt.Sprintf("Completed scanning repo %d/%d", i+1, len(s.repos)))
 			return nil
 		})
 	}
 
 	_ = s.jobPool.Wait()
 	if scanErrs.Count() > 0 {
-		log.Debugf("encountered %d errors while scanning; errors: %v", scanErrs.Count(), scanErrs)
+		ctx.Logger().V(2).Info("encountered errors while scanning", "count", scanErrs.Count(), "errors", scanErrs)
 	}
 	s.SetProgressComplete(len(s.repos), len(s.repos), "Completed Gitlab scan", "")
 
 	return nil
 }
 
-func (s *Source) ignoreRepo(r string) bool {
+func (s *Source) ignoreRepo(ctx context.Context, r string) bool {
 	for _, ignore := range s.ignoreRepos {
 		g, err := glob.Compile(ignore)
 		if err != nil {
-			log.WithError(err).Errorf("could not compile ignore repo glob %s", ignore)
+			ctx.Logger().Error(err, "could not compile ignore repo glob", "glob", ignore)
 			continue
 		}
 		if g.Match(r) {
-			log.Debugf("Ignoring repo %s", r)
+			ctx.Logger().V(2).Info("Ignoring repo", "repo", r)
 			return true
 		}
 	}

pkg/sources/gitlab/gitlab_test.go

@@ -2,12 +2,10 @@ package gitlab
 
 import (
 	"fmt"
-	"io"
 	"reflect"
 	"testing"
 
 	"github.com/kylelemons/godebug/pretty"
-	log "github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/assert"
 	"golang.org/x/sync/errgroup"
 	"google.golang.org/protobuf/types/known/anypb"
@@ -24,8 +22,6 @@ func TestSource_Scan(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
-	log.SetLevel(log.DebugLevel)
-	log.SetFormatter(&log.TextFormatter{ForceColors: true})
 	secret, err := common.GetTestSecret(ctx)
 	if err != nil {
 		t.Fatal(fmt.Errorf("failed to access secret: %v", err))
@@ -142,7 +138,6 @@ func TestSource_Scan(t *testing.T) {
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			s := Source{}
-			log.SetLevel(log.DebugLevel)
 
 			conn, err := anypb.New(tt.init.connection)
 			if err != nil {
@@ -200,7 +195,6 @@ func Test_setProgressCompleteWithRepo_resumeInfo(t *testing.T) {
 		},
 	}
 
-	log.SetOutput(io.Discard)
 	s := &Source{repos: []string{}}
 
 	for _, tt := range tests {
@@ -248,8 +242,6 @@ func Test_setProgressCompleteWithRepo_Progress(t *testing.T) {
 		},
 	}
 
-	log.SetOutput(io.Discard)
-
 	for _, tt := range tests {
 		s := &Source{
 			repos: tt.repos,

pkg/sources/resume.go

@@ -2,8 +2,6 @@ package sources
 
 import (
 	"strings"
-
-	"github.com/sirupsen/logrus"
 )
 
 // RemoveRepoFromResumeInfo removes the repoURL from the resume info.
@@ -18,8 +16,7 @@ func RemoveRepoFromResumeInfo(resumeRepos []string, repoURL string) []string {
 
 	if index == -1 {
 		// We should never be able to be here. But if we are, it means the resume info never had the repo added.
-		// So log the error and do nothing.
+		// So do nothing.
-		logrus.Errorf("repoURL (%q) not found in list of encode resume info: %v", repoURL, resumeRepos)
 		return resumeRepos
 	}
 

pkg/sources/resume_test.go

@@ -1,11 +1,8 @@
 package sources
 
 import (
-	"io"
 	"reflect"
 	"testing"
-
-	"github.com/sirupsen/logrus"
 )
 
 func TestRemoveRepoFromResumeInfo(t *testing.T) {
@@ -31,9 +28,6 @@ func TestRemoveRepoFromResumeInfo(t *testing.T) {
 		},
 	}
 
-	// Prevent the error message from showing up.
-	logrus.SetOutput(io.Discard)
-
 	for _, tt := range tests {
 		gotResumeInfoSlice := RemoveRepoFromResumeInfo(tt.startingResumeInfoSlice, tt.repoURL)
 		if !reflect.DeepEqual(gotResumeInfoSlice, tt.wantResumeInfoSlice) {
@@ -57,9 +51,6 @@ func TestEncodeResumeInfo(t *testing.T) {
 		},
 	}
 
-	logger := logrus.New()
-	logger.Out = io.Discard
-
 	for _, tt := range tests {
 		gotEncodedResumeInfo := EncodeResumeInfo(tt.startingResumeInfoSlice)
 		if gotEncodedResumeInfo != tt.wantEncodedResumeInfo {

pkg/sources/s3/s3_test.go

@@ -9,7 +9,6 @@ import (
 	"time"
 
 	"github.com/kylelemons/godebug/pretty"
-	log "github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/assert"
 	"google.golang.org/protobuf/types/known/anypb"
 
@@ -62,14 +61,11 @@ func TestSource_Chunks(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			log.SetFormatter(&log.TextFormatter{ForceColors: true})
 			ctx, cancel := context.WithTimeout(context.Background(), time.Second*30)
 			var cancelOnce sync.Once
 			defer cancelOnce.Do(cancel)
 
 			s := Source{}
-			log.SetLevel(log.DebugLevel)
-
 			conn, err := anypb.New(tt.init.connection)
 			if err != nil {
 				t.Fatal(err)