From d317ddb51a9db96fb6495da55dfe045727bda31b Mon Sep 17 00:00:00 2001 From: Miccah Date: Fri, 10 Feb 2023 11:02:55 -0600 Subject: [PATCH] [chore] Remove logrus from circleci, filesystem, gitlab, and s3 sources (#1089) * [chore] Remove logrus from circleci, filesystem, gitlab, and s3 sources * Address comments --- pkg/engine/circleci.go | 4 +++ pkg/engine/filesystem.go | 5 ++++ pkg/engine/git.go | 4 +++ pkg/engine/github.go | 4 +-- pkg/engine/gitlab.go | 4 +++ pkg/engine/s3.go | 4 +++ pkg/engine/syslog.go | 4 +++ pkg/sources/circleci/circleci.go | 5 ++-- pkg/sources/filesystem/filesystem.go | 15 +++++----- pkg/sources/filesystem/filesystem_test.go | 3 -- pkg/sources/gitlab/gitlab.go | 35 ++++++++++++----------- pkg/sources/gitlab/gitlab_test.go | 8 ------ pkg/sources/resume.go | 5 +--- pkg/sources/resume_test.go | 9 ------ pkg/sources/s3/s3_test.go | 4 --- 15 files changed, 56 insertions(+), 57 deletions(-) diff --git a/pkg/engine/circleci.go b/pkg/engine/circleci.go index 8d30c0e1f..44f00cac6 100644 --- a/pkg/engine/circleci.go +++ b/pkg/engine/circleci.go @@ -29,6 +29,10 @@ func (e *Engine) ScanCircleCI(ctx context.Context, token string) error { } circleSource := circleci.Source{} + ctx = context.WithValues(ctx, + "source_type", circleSource.Type().String(), + "source_name", "Circle CI", + ) err = circleSource.Init(ctx, "trufflehog - Circle CI", 0, int64(sourcespb.SourceType_SOURCE_TYPE_CIRCLECI), true, &conn, runtime.NumCPU()) if err != nil { return errors.WrapPrefix(err, "failed to init Circle CI source", 0) diff --git a/pkg/engine/filesystem.go b/pkg/engine/filesystem.go index 59ec8cf9d..61527bf8b 100644 --- a/pkg/engine/filesystem.go +++ b/pkg/engine/filesystem.go @@ -27,6 +27,11 @@ func (e *Engine) ScanFileSystem(ctx context.Context, c sources.Config) error { } fileSystemSource := filesystem.Source{} + + ctx = context.WithValues(ctx, + "source_type", fileSystemSource.Type().String(), + "source_name", "filesystem", + ) err = fileSystemSource.Init(ctx, "trufflehog - filesystem", 0, int64(sourcespb.SourceType_SOURCE_TYPE_FILESYSTEM), true, &conn, runtime.NumCPU()) if err != nil { return errors.WrapPrefix(err, "could not init filesystem source", 0) diff --git a/pkg/engine/git.go b/pkg/engine/git.go index 59cb4f1a4..edc3b02d9 100644 --- a/pkg/engine/git.go +++ b/pkg/engine/git.go @@ -54,6 +54,10 @@ func (e *Engine) ScanGit(ctx context.Context, c sources.Config) error { } }) + ctx = context.WithValues(ctx, + "source_type", sourcespb.SourceType_SOURCE_TYPE_GIT.String(), + "source_name", "git", + ) e.sourcesWg.Add(1) go func() { defer common.RecoverWithExit(ctx) diff --git a/pkg/engine/github.go b/pkg/engine/github.go index ba3ec543f..927415df6 100644 --- a/pkg/engine/github.go +++ b/pkg/engine/github.go @@ -38,8 +38,8 @@ func (e *Engine) ScanGitHub(ctx context.Context, c sources.Config) error { return err } ctx = context.WithValues(ctx, - "source_type", source.Type(), - "name", "trufflehog - github", + "source_type", source.Type().String(), + "source_name", "github", ) err = source.Init(ctx, "trufflehog - github", 0, 0, false, &conn, c.Concurrency) if err != nil { diff --git a/pkg/engine/gitlab.go b/pkg/engine/gitlab.go index 3ae377085..c0257c8a5 100644 --- a/pkg/engine/gitlab.go +++ b/pkg/engine/gitlab.go @@ -53,6 +53,10 @@ func (e *Engine) ScanGitLab(ctx context.Context, c sources.Config) error { } gitlabSource := gitlab.Source{} + ctx = context.WithValues(ctx, + "source_type", gitlabSource.Type().String(), + "source_name", "gitlab", + ) err = gitlabSource.Init(ctx, "trufflehog - gitlab", 0, int64(sourcespb.SourceType_SOURCE_TYPE_GITLAB), true, &conn, runtime.NumCPU()) if err != nil { return errors.WrapPrefix(err, "could not init GitLab source", 0) diff --git a/pkg/engine/s3.go b/pkg/engine/s3.go index 88329a0a4..5ba708769 100644 --- a/pkg/engine/s3.go +++ b/pkg/engine/s3.go @@ -46,6 +46,10 @@ func (e *Engine) ScanS3(ctx context.Context, c sources.Config) error { } s3Source := s3.Source{} + ctx = context.WithValues(ctx, + "source_type", s3Source.Type().String(), + "source_name", "s3", + ) err = s3Source.Init(ctx, "trufflehog - s3", 0, int64(sourcespb.SourceType_SOURCE_TYPE_S3), true, &conn, runtime.NumCPU()) if err != nil { return errors.WrapPrefix(err, "failed to init S3 source", 0) diff --git a/pkg/engine/syslog.go b/pkg/engine/syslog.go index 2868b5818..d12ba4705 100644 --- a/pkg/engine/syslog.go +++ b/pkg/engine/syslog.go @@ -42,6 +42,10 @@ func (e *Engine) ScanSyslog(ctx context.Context, c sources.Config) error { return errors.WrapPrefix(err, "error unmarshalling connection", 0) } source := syslog.Source{} + ctx = context.WithValues(ctx, + "source_type", source.Type().String(), + "source_name", "syslog", + ) err = source.Init(ctx, "trufflehog - syslog", 0, 0, false, &conn, c.Concurrency) source.InjectConnection(connection) if err != nil { diff --git a/pkg/sources/circleci/circleci.go b/pkg/sources/circleci/circleci.go index bdf22d4c1..f6d95e50c 100644 --- a/pkg/sources/circleci/circleci.go +++ b/pkg/sources/circleci/circleci.go @@ -9,7 +9,6 @@ import ( "sync/atomic" "github.com/go-errors/errors" - log "github.com/sirupsen/logrus" "golang.org/x/sync/errgroup" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/anypb" @@ -111,14 +110,14 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err } atomic.AddUint64(&scanned, 1) - log.Debugf("scanned %d/%d projects", scanned, len(projects)) + ctx.Logger().V(2).Info(fmt.Sprintf("scanned %d/%d projects", scanned, len(projects))) return nil }) } _ = s.jobPool.Wait() if scanErrs.Count() > 0 { - log.Debugf("encountered %d errors while scanning; errors: %v", scanErrs.Count(), scanErrs) + ctx.Logger().V(2).Info("encountered errors while scanning", "count", scanErrs.Count(), "errors", scanErrs) } return nil diff --git a/pkg/sources/filesystem/filesystem.go b/pkg/sources/filesystem/filesystem.go index 3dda10887..7562e59d8 100644 --- a/pkg/sources/filesystem/filesystem.go +++ b/pkg/sources/filesystem/filesystem.go @@ -9,7 +9,7 @@ import ( diskbufferreader "github.com/bill-rich/disk-buffer-reader" "github.com/go-errors/errors" - log "github.com/sirupsen/logrus" + "github.com/go-logr/logr" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/anypb" @@ -35,7 +35,7 @@ type Source struct { jobId int64 verify bool paths []string - log *log.Entry + log logr.Logger filter *common.Filter sources.Progress } @@ -59,7 +59,7 @@ func (s *Source) JobID() int64 { // Init returns an initialized Filesystem source. func (s *Source) Init(aCtx context.Context, name string, jobId, sourceId int64, verify bool, connection *anypb.Any, _ int) error { - s.log = log.WithField("source", s.Type()).WithField("name", name) + s.log = aCtx.Logger() s.name = name s.sourceId = sourceId @@ -98,10 +98,11 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err } path := filepath.Join(cleanPath, relativePath) + logger := ctx.Logger().WithValues("file", path) fileStat, err := os.Stat(path) if err != nil { - log.WithError(err).Warnf("unable to stat file: %s", path) + logger.Error(err, "unable to stat file") return nil } if !fileStat.Mode().IsRegular() { @@ -114,15 +115,15 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err inputFile, err := os.Open(path) if err != nil { - log.Warn(err) + logger.V(2).Info("error opening file", "error", err) return nil } defer inputFile.Close() - log.WithField("file_path", path).Trace("scanning file") + logger.V(5).Info("scanning file") reReader, err := diskbufferreader.New(inputFile) if err != nil { - log.WithError(err).Error("Could not create re-readable reader.") + logger.Error(err, "Could not create re-readable reader.") } defer reReader.Close() diff --git a/pkg/sources/filesystem/filesystem_test.go b/pkg/sources/filesystem/filesystem_test.go index a3e07525c..46fe85491 100644 --- a/pkg/sources/filesystem/filesystem_test.go +++ b/pkg/sources/filesystem/filesystem_test.go @@ -5,7 +5,6 @@ import ( "time" "github.com/kylelemons/godebug/pretty" - log "github.com/sirupsen/logrus" "google.golang.org/protobuf/types/known/anypb" "github.com/trufflesecurity/trufflehog/v3/pkg/context" @@ -51,8 +50,6 @@ func TestSource_Scan(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { s := Source{} - log.SetLevel(log.DebugLevel) - log.SetFormatter(&log.TextFormatter{ForceColors: true}) conn, err := anypb.New(tt.init.connection) if err != nil { diff --git a/pkg/sources/gitlab/gitlab.go b/pkg/sources/gitlab/gitlab.go index 73a1283c0..73ddcf0e4 100644 --- a/pkg/sources/gitlab/gitlab.go +++ b/pkg/sources/gitlab/gitlab.go @@ -23,7 +23,6 @@ import ( "github.com/go-errors/errors" gogit "github.com/go-git/go-git/v5" "github.com/gobwas/glob" - log "github.com/sirupsen/logrus" "github.com/xanzy/go-gitlab" "golang.org/x/exp/slices" "google.golang.org/protobuf/proto" @@ -69,7 +68,6 @@ func (s *Source) JobID() int64 { // Init returns an initialized Gitlab source. func (s *Source) Init(_ context.Context, name string, jobId, sourceId int64, verify bool, connection *anypb.Any, concurrency int) error { - s.name = name s.sourceId = sourceId s.jobId = jobId @@ -143,7 +141,7 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err // Get repo within target. repos, errs := s.getRepos() for _, repoErr := range errs { - log.WithError(repoErr).Warn("error getting repo") + ctx.Logger().Info("error getting repo", "error", repoErr) } // End early if we had errors getting specified repos but none were validated. @@ -153,13 +151,13 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err // Get all repos if not specified. if len(repos) == 0 { - projects, err := s.getAllProjects(apiClient) + projects, err := s.getAllProjects(ctx, apiClient) if err != nil { return fmt.Errorf("error getting all projects: %v", err) } // Turn projects into URLs for Git cloner. for _, prj := range projects { - if s.ignoreRepo(prj.PathWithNamespace) { + if s.ignoreRepo(ctx, prj.PathWithNamespace) { continue } @@ -231,7 +229,7 @@ func (s *Source) basicAuthSuccessful(apiClient *gitlab.Client) bool { return false } -func (s *Source) getAllProjects(apiClient *gitlab.Client) ([]*gitlab.Project, error) { +func (s *Source) getAllProjects(ctx context.Context, apiClient *gitlab.Client) ([]*gitlab.Project, error) { // Projects without repo will get user projects, groups projects, and subgroup projects. user, _, err := apiClient.Users.CurrentUser() @@ -289,7 +287,10 @@ func (s *Source) getAllProjects(apiClient *gitlab.Client) ([]*gitlab.Project, er for { grpPrjs, res, err := apiClient.Groups.ListGroupProjects(group.ID, listGroupProjectOptions) if err != nil { - log.WithError(err).WithField("group", group.FullPath).Warn("received error on listing group projects, you probably don't have permissions to do that") + ctx.Logger().Info("received error on listing group projects, you probably don't have permissions to do that", + "group", group.FullPath, + "error", err, + ) break } for _, prj := range grpPrjs { @@ -305,7 +306,7 @@ func (s *Source) getAllProjects(apiClient *gitlab.Client) ([]*gitlab.Project, er for _, project := range projects { projectNamesWithNamespace = append(projectNamesWithNamespace, project.NameWithNamespace) } - log.WithField("projects", strings.Join(projectNamesWithNamespace, ", ")).Debugf("Enumerated %d GitLab projects", len(projects)) + ctx.Logger().V(2).Info("Enumerated GitLab projects", "count", len(projects), "projects", projectNamesWithNamespace) var projectList []*gitlab.Project for _, project := range projects { @@ -334,7 +335,6 @@ func (s *Source) getRepos() ([]string, []error) { } func (s *Source) scanRepos(ctx context.Context, chunksChan chan *sources.Chunk) error { - // If there is resume information available, limit this scan to only the repos that still need scanning. reposToScan, progressIndexOffset := sources.FilterReposToResume(s.repos, s.GetProgress().EncodedResumeInfo) s.repos = reposToScan @@ -343,15 +343,16 @@ func (s *Source) scanRepos(ctx context.Context, chunksChan chan *sources.Chunk) for i, repo := range s.repos { i, repoURL := i, repo s.jobPool.Go(func() error { + logger := ctx.Logger().WithValues("repo", repoURL) if common.IsDone(ctx) { // We are returning nil instead of the scanErrors slice here because // we don't want to mark this scan as errored if we cancelled it. - log.Debugf("Skipping repo %s because context was cancelled", repoURL) + logger.V(2).Info("Skipping repo because context was cancelled") return nil } if len(repoURL) == 0 { - log.Debugf("Skipping empty repo %s", repoURL) + logger.V(2).Info("Skipping empty repo") return nil } @@ -383,35 +384,35 @@ func (s *Source) scanRepos(ctx context.Context, chunksChan chan *sources.Chunk) return nil } - log.Debugf("Starting to scan repo %d/%d: %s", i+1, len(s.repos), repo) + logger.V(2).Info(fmt.Sprintf("Starting to scan repo %d/%d", i+1, len(s.repos))) if err = s.git.ScanRepo(ctx, repo, path, s.scanOptions, chunksChan); err != nil { scanErrs.Add(err) return nil } - log.Debugf("Completed scanning repo %d/%d: %s", i+1, len(s.repos), repo) + logger.V(2).Info(fmt.Sprintf("Completed scanning repo %d/%d", i+1, len(s.repos))) return nil }) } _ = s.jobPool.Wait() if scanErrs.Count() > 0 { - log.Debugf("encountered %d errors while scanning; errors: %v", scanErrs.Count(), scanErrs) + ctx.Logger().V(2).Info("encountered errors while scanning", "count", scanErrs.Count(), "errors", scanErrs) } s.SetProgressComplete(len(s.repos), len(s.repos), "Completed Gitlab scan", "") return nil } -func (s *Source) ignoreRepo(r string) bool { +func (s *Source) ignoreRepo(ctx context.Context, r string) bool { for _, ignore := range s.ignoreRepos { g, err := glob.Compile(ignore) if err != nil { - log.WithError(err).Errorf("could not compile ignore repo glob %s", ignore) + ctx.Logger().Error(err, "could not compile ignore repo glob", "glob", ignore) continue } if g.Match(r) { - log.Debugf("Ignoring repo %s", r) + ctx.Logger().V(2).Info("Ignoring repo", "repo", r) return true } } diff --git a/pkg/sources/gitlab/gitlab_test.go b/pkg/sources/gitlab/gitlab_test.go index 3b194a65a..68088c522 100644 --- a/pkg/sources/gitlab/gitlab_test.go +++ b/pkg/sources/gitlab/gitlab_test.go @@ -2,12 +2,10 @@ package gitlab import ( "fmt" - "io" "reflect" "testing" "github.com/kylelemons/godebug/pretty" - log "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "golang.org/x/sync/errgroup" "google.golang.org/protobuf/types/known/anypb" @@ -24,8 +22,6 @@ func TestSource_Scan(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - log.SetLevel(log.DebugLevel) - log.SetFormatter(&log.TextFormatter{ForceColors: true}) secret, err := common.GetTestSecret(ctx) if err != nil { t.Fatal(fmt.Errorf("failed to access secret: %v", err)) @@ -142,7 +138,6 @@ func TestSource_Scan(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { s := Source{} - log.SetLevel(log.DebugLevel) conn, err := anypb.New(tt.init.connection) if err != nil { @@ -200,7 +195,6 @@ func Test_setProgressCompleteWithRepo_resumeInfo(t *testing.T) { }, } - log.SetOutput(io.Discard) s := &Source{repos: []string{}} for _, tt := range tests { @@ -248,8 +242,6 @@ func Test_setProgressCompleteWithRepo_Progress(t *testing.T) { }, } - log.SetOutput(io.Discard) - for _, tt := range tests { s := &Source{ repos: tt.repos, diff --git a/pkg/sources/resume.go b/pkg/sources/resume.go index d085b5d69..4969a5750 100644 --- a/pkg/sources/resume.go +++ b/pkg/sources/resume.go @@ -2,8 +2,6 @@ package sources import ( "strings" - - "github.com/sirupsen/logrus" ) // RemoveRepoFromResumeInfo removes the repoURL from the resume info. @@ -18,8 +16,7 @@ func RemoveRepoFromResumeInfo(resumeRepos []string, repoURL string) []string { if index == -1 { // We should never be able to be here. But if we are, it means the resume info never had the repo added. - // So log the error and do nothing. - logrus.Errorf("repoURL (%q) not found in list of encode resume info: %v", repoURL, resumeRepos) + // So do nothing. return resumeRepos } diff --git a/pkg/sources/resume_test.go b/pkg/sources/resume_test.go index 56bda955d..6a89c05f7 100644 --- a/pkg/sources/resume_test.go +++ b/pkg/sources/resume_test.go @@ -1,11 +1,8 @@ package sources import ( - "io" "reflect" "testing" - - "github.com/sirupsen/logrus" ) func TestRemoveRepoFromResumeInfo(t *testing.T) { @@ -31,9 +28,6 @@ func TestRemoveRepoFromResumeInfo(t *testing.T) { }, } - // Prevent the error message from showing up. - logrus.SetOutput(io.Discard) - for _, tt := range tests { gotResumeInfoSlice := RemoveRepoFromResumeInfo(tt.startingResumeInfoSlice, tt.repoURL) if !reflect.DeepEqual(gotResumeInfoSlice, tt.wantResumeInfoSlice) { @@ -57,9 +51,6 @@ func TestEncodeResumeInfo(t *testing.T) { }, } - logger := logrus.New() - logger.Out = io.Discard - for _, tt := range tests { gotEncodedResumeInfo := EncodeResumeInfo(tt.startingResumeInfoSlice) if gotEncodedResumeInfo != tt.wantEncodedResumeInfo { diff --git a/pkg/sources/s3/s3_test.go b/pkg/sources/s3/s3_test.go index e1f962543..5d8ce03e2 100644 --- a/pkg/sources/s3/s3_test.go +++ b/pkg/sources/s3/s3_test.go @@ -9,7 +9,6 @@ import ( "time" "github.com/kylelemons/godebug/pretty" - log "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "google.golang.org/protobuf/types/known/anypb" @@ -62,14 +61,11 @@ func TestSource_Chunks(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - log.SetFormatter(&log.TextFormatter{ForceColors: true}) ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) var cancelOnce sync.Once defer cancelOnce.Do(cancel) s := Source{} - log.SetLevel(log.DebugLevel) - conn, err := anypb.New(tt.init.connection) if err != nil { t.Fatal(err)