trufflesecurity/trufflehog
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[chore] Remove logrus from circleci, filesystem, gitlab, and s3 sources (#1089)

Browse Source 
* [chore] Remove logrus from circleci, filesystem, gitlab, and s3 sources

* Address comments
This commit is contained in:
Miccah
2023-02-10 11:02:55 -06:00
committed by GitHub
parent 29be679370
commit d317ddb51a
15 changed files with 56 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/engine/circleci.go
+4
View File
@@ -29,6 +29,10 @@ func (e *Engine) ScanCircleCI(ctx context.Context, token string) error {
}
circleSource := circleci.Source{}
ctx = context.WithValues(ctx,
"source_type", circleSource.Type().String(),
"source_name", "Circle CI",
)
err = circleSource.Init(ctx, "trufflehog - Circle CI", 0, int64(sourcespb.SourceType_SOURCE_TYPE_CIRCLECI), true, &conn, runtime.NumCPU())
if err != nil {
return errors.WrapPrefix(err, "failed to init Circle CI source", 0)
pkg/engine/filesystem.go
+5
View File
@@ -27,6 +27,11 @@ func (e *Engine) ScanFileSystem(ctx context.Context, c sources.Config) error {
}
fileSystemSource := filesystem.Source{}
ctx = context.WithValues(ctx,
"source_type", fileSystemSource.Type().String(),
"source_name", "filesystem",
)
err = fileSystemSource.Init(ctx, "trufflehog - filesystem", 0, int64(sourcespb.SourceType_SOURCE_TYPE_FILESYSTEM), true, &conn, runtime.NumCPU())
if err != nil {
return errors.WrapPrefix(err, "could not init filesystem source", 0)
pkg/engine/git.go
+4
View File
@@ -54,6 +54,10 @@ func (e *Engine) ScanGit(ctx context.Context, c sources.Config) error {
}
})
ctx = context.WithValues(ctx,
"source_type", sourcespb.SourceType_SOURCE_TYPE_GIT.String(),
"source_name", "git",
)
e.sourcesWg.Add(1)
go func() {
defer common.RecoverWithExit(ctx)
pkg/engine/github.go
+2 -2
View File
@@ -38,8 +38,8 @@ func (e *Engine) ScanGitHub(ctx context.Context, c sources.Config) error {
return err
}
ctx = context.WithValues(ctx,
"source_type", source.Type(),
"name", "trufflehog - github",
"source_type", source.Type().String(),
"source_name", "github",
)
err = source.Init(ctx, "trufflehog - github", 0, 0, false, &conn, c.Concurrency)
if err != nil {
pkg/engine/gitlab.go
+4
View File
@@ -53,6 +53,10 @@ func (e *Engine) ScanGitLab(ctx context.Context, c sources.Config) error {
}
gitlabSource := gitlab.Source{}
ctx = context.WithValues(ctx,
"source_type", gitlabSource.Type().String(),
"source_name", "gitlab",
)
err = gitlabSource.Init(ctx, "trufflehog - gitlab", 0, int64(sourcespb.SourceType_SOURCE_TYPE_GITLAB), true, &conn, runtime.NumCPU())
if err != nil {
return errors.WrapPrefix(err, "could not init GitLab source", 0)
pkg/engine/s3.go
+4
View File
@@ -46,6 +46,10 @@ func (e *Engine) ScanS3(ctx context.Context, c sources.Config) error {
}
s3Source := s3.Source{}
ctx = context.WithValues(ctx,
"source_type", s3Source.Type().String(),
"source_name", "s3",
)
err = s3Source.Init(ctx, "trufflehog - s3", 0, int64(sourcespb.SourceType_SOURCE_TYPE_S3), true, &conn, runtime.NumCPU())
if err != nil {
return errors.WrapPrefix(err, "failed to init S3 source", 0)
pkg/engine/syslog.go
+4
View File
@@ -42,6 +42,10 @@ func (e *Engine) ScanSyslog(ctx context.Context, c sources.Config) error {
return errors.WrapPrefix(err, "error unmarshalling connection", 0)
}
source := syslog.Source{}
ctx = context.WithValues(ctx,
"source_type", source.Type().String(),
"source_name", "syslog",
)
err = source.Init(ctx, "trufflehog - syslog", 0, 0, false, &conn, c.Concurrency)
source.InjectConnection(connection)
if err != nil {
pkg/sources/circleci/circleci.go
+2 -3
View File
@@ -9,7 +9,6 @@ import (
"sync/atomic"
"github.com/go-errors/errors"
log "github.com/sirupsen/logrus"
"golang.org/x/sync/errgroup"
"google.golang.org/protobuf/proto"
"google.golang.org/protobuf/types/known/anypb"
@@ -111,14 +110,14 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
}
atomic.AddUint64(&scanned, 1)
log.Debugf("scanned %d/%d projects", scanned, len(projects))
ctx.Logger().V(2).Info(fmt.Sprintf("scanned %d/%d projects", scanned, len(projects)))
return nil
})
}
_ = s.jobPool.Wait()
if scanErrs.Count() > 0 {
log.Debugf("encountered %d errors while scanning; errors: %v", scanErrs.Count(), scanErrs)
ctx.Logger().V(2).Info("encountered errors while scanning", "count", scanErrs.Count(), "errors", scanErrs)
}
return nil
pkg/sources/filesystem/filesystem.go
+8 -7
View File
@@ -9,7 +9,7 @@ import (
diskbufferreader "github.com/bill-rich/disk-buffer-reader"
"github.com/go-errors/errors"
log "github.com/sirupsen/logrus"
"github.com/go-logr/logr"
"google.golang.org/protobuf/proto"
"google.golang.org/protobuf/types/known/anypb"
@@ -35,7 +35,7 @@ type Source struct {
jobId int64
verify bool
paths []string
log *log.Entry
log logr.Logger
filter *common.Filter
sources.Progress
}
@@ -59,7 +59,7 @@ func (s *Source) JobID() int64 {
// Init returns an initialized Filesystem source.
func (s *Source) Init(aCtx context.Context, name string, jobId, sourceId int64, verify bool, connection *anypb.Any, _ int) error {
s.log = log.WithField("source", s.Type()).WithField("name", name)
s.log = aCtx.Logger()
s.name = name
s.sourceId = sourceId
@@ -98,10 +98,11 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
}
path := filepath.Join(cleanPath, relativePath)
logger := ctx.Logger().WithValues("file", path)
fileStat, err := os.Stat(path)
if err != nil {
log.WithError(err).Warnf("unable to stat file: %s", path)
logger.Error(err, "unable to stat file")
return nil
}
if !fileStat.Mode().IsRegular() {
@@ -114,15 +115,15 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
inputFile, err := os.Open(path)
if err != nil {
log.Warn(err)
logger.V(2).Info("error opening file", "error", err)
return nil
}
defer inputFile.Close()
log.WithField("file_path", path).Trace("scanning file")
logger.V(5).Info("scanning file")
reReader, err := diskbufferreader.New(inputFile)
if err != nil {
log.WithError(err).Error("Could not create re-readable reader.")
logger.Error(err, "Could not create re-readable reader.")
}
defer reReader.Close()
pkg/sources/filesystem/filesystem_test.go
-3
View File
@@ -5,7 +5,6 @@ import (
"time"
"github.com/kylelemons/godebug/pretty"
log "github.com/sirupsen/logrus"
"google.golang.org/protobuf/types/known/anypb"
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
@@ -51,8 +50,6 @@ func TestSource_Scan(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
s := Source{}
log.SetLevel(log.DebugLevel)
log.SetFormatter(&log.TextFormatter{ForceColors: true})
conn, err := anypb.New(tt.init.connection)
if err != nil {
pkg/sources/gitlab/gitlab.go
+18 -17
View File
@@ -23,7 +23,6 @@ import (
"github.com/go-errors/errors"
gogit "github.com/go-git/go-git/v5"
"github.com/gobwas/glob"
log "github.com/sirupsen/logrus"
"github.com/xanzy/go-gitlab"
"golang.org/x/exp/slices"
"google.golang.org/protobuf/proto"
@@ -69,7 +68,6 @@ func (s *Source) JobID() int64 {
// Init returns an initialized Gitlab source.
func (s *Source) Init(_ context.Context, name string, jobId, sourceId int64, verify bool, connection *anypb.Any, concurrency int) error {
s.name = name
s.sourceId = sourceId
s.jobId = jobId
@@ -143,7 +141,7 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
// Get repo within target.
repos, errs := s.getRepos()
for _, repoErr := range errs {
log.WithError(repoErr).Warn("error getting repo")
ctx.Logger().Info("error getting repo", "error", repoErr)
}
// End early if we had errors getting specified repos but none were validated.
@@ -153,13 +151,13 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
// Get all repos if not specified.
if len(repos) == 0 {
projects, err := s.getAllProjects(apiClient)
projects, err := s.getAllProjects(ctx, apiClient)
if err != nil {
return fmt.Errorf("error getting all projects: %v", err)
}
// Turn projects into URLs for Git cloner.
for _, prj := range projects {
if s.ignoreRepo(prj.PathWithNamespace) {
if s.ignoreRepo(ctx, prj.PathWithNamespace) {
continue
}
@@ -231,7 +229,7 @@ func (s *Source) basicAuthSuccessful(apiClient *gitlab.Client) bool {
return false
}
func (s *Source) getAllProjects(apiClient *gitlab.Client) ([]*gitlab.Project, error) {
func (s *Source) getAllProjects(ctx context.Context, apiClient *gitlab.Client) ([]*gitlab.Project, error) {
// Projects without repo will get user projects, groups projects, and subgroup projects.
user, _, err := apiClient.Users.CurrentUser()
@@ -289,7 +287,10 @@ func (s *Source) getAllProjects(apiClient *gitlab.Client) ([]*gitlab.Project, er
for {
grpPrjs, res, err := apiClient.Groups.ListGroupProjects(group.ID, listGroupProjectOptions)
if err != nil {
log.WithError(err).WithField("group", group.FullPath).Warn("received error on listing group projects, you probably don't have permissions to do that")
ctx.Logger().Info("received error on listing group projects, you probably don't have permissions to do that",
"group", group.FullPath,
"error", err,
)
break
}
for _, prj := range grpPrjs {
@@ -305,7 +306,7 @@ func (s *Source) getAllProjects(apiClient *gitlab.Client) ([]*gitlab.Project, er
for _, project := range projects {
projectNamesWithNamespace = append(projectNamesWithNamespace, project.NameWithNamespace)
}
log.WithField("projects", strings.Join(projectNamesWithNamespace, ", ")).Debugf("Enumerated %d GitLab projects", len(projects))
ctx.Logger().V(2).Info("Enumerated GitLab projects", "count", len(projects), "projects", projectNamesWithNamespace)
var projectList []*gitlab.Project
for _, project := range projects {
@@ -334,7 +335,6 @@ func (s *Source) getRepos() ([]string, []error) {
}
func (s *Source) scanRepos(ctx context.Context, chunksChan chan *sources.Chunk) error {
// If there is resume information available, limit this scan to only the repos that still need scanning.
reposToScan, progressIndexOffset := sources.FilterReposToResume(s.repos, s.GetProgress().EncodedResumeInfo)
s.repos = reposToScan
@@ -343,15 +343,16 @@ func (s *Source) scanRepos(ctx context.Context, chunksChan chan *sources.Chunk)
for i, repo := range s.repos {
i, repoURL := i, repo
s.jobPool.Go(func() error {
logger := ctx.Logger().WithValues("repo", repoURL)
if common.IsDone(ctx) {
// We are returning nil instead of the scanErrors slice here because
// we don't want to mark this scan as errored if we cancelled it.
log.Debugf("Skipping repo %s because context was cancelled", repoURL)
logger.V(2).Info("Skipping repo because context was cancelled")
return nil
}
if len(repoURL) == 0 {
log.Debugf("Skipping empty repo %s", repoURL)
logger.V(2).Info("Skipping empty repo")
return nil
}
@@ -383,35 +384,35 @@ func (s *Source) scanRepos(ctx context.Context, chunksChan chan *sources.Chunk)
return nil
}
log.Debugf("Starting to scan repo %d/%d: %s", i+1, len(s.repos), repo)
logger.V(2).Info(fmt.Sprintf("Starting to scan repo %d/%d", i+1, len(s.repos)))
if err = s.git.ScanRepo(ctx, repo, path, s.scanOptions, chunksChan); err != nil {
scanErrs.Add(err)
return nil
}
log.Debugf("Completed scanning repo %d/%d: %s", i+1, len(s.repos), repo)
logger.V(2).Info(fmt.Sprintf("Completed scanning repo %d/%d", i+1, len(s.repos)))
return nil
})
}
_ = s.jobPool.Wait()
if scanErrs.Count() > 0 {
log.Debugf("encountered %d errors while scanning; errors: %v", scanErrs.Count(), scanErrs)
ctx.Logger().V(2).Info("encountered errors while scanning", "count", scanErrs.Count(), "errors", scanErrs)
}
s.SetProgressComplete(len(s.repos), len(s.repos), "Completed Gitlab scan", "")
return nil
}
func (s *Source) ignoreRepo(r string) bool {
func (s *Source) ignoreRepo(ctx context.Context, r string) bool {
for _, ignore := range s.ignoreRepos {
g, err := glob.Compile(ignore)
if err != nil {
log.WithError(err).Errorf("could not compile ignore repo glob %s", ignore)
ctx.Logger().Error(err, "could not compile ignore repo glob", "glob", ignore)
continue
}
if g.Match(r) {
log.Debugf("Ignoring repo %s", r)
ctx.Logger().V(2).Info("Ignoring repo", "repo", r)
return true
}
}
pkg/sources/gitlab/gitlab_test.go
-8
View File
@@ -2,12 +2,10 @@ package gitlab
import (
"fmt"
"io"
"reflect"
"testing"
"github.com/kylelemons/godebug/pretty"
log "github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert"
"golang.org/x/sync/errgroup"
"google.golang.org/protobuf/types/known/anypb"
@@ -24,8 +22,6 @@ func TestSource_Scan(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
log.SetLevel(log.DebugLevel)
log.SetFormatter(&log.TextFormatter{ForceColors: true})
secret, err := common.GetTestSecret(ctx)
if err != nil {
t.Fatal(fmt.Errorf("failed to access secret: %v", err))
@@ -142,7 +138,6 @@ func TestSource_Scan(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
s := Source{}
log.SetLevel(log.DebugLevel)
conn, err := anypb.New(tt.init.connection)
if err != nil {
@@ -200,7 +195,6 @@ func Test_setProgressCompleteWithRepo_resumeInfo(t *testing.T) {
},
}
log.SetOutput(io.Discard)
s := &Source{repos: []string{}}
for _, tt := range tests {
@@ -248,8 +242,6 @@ func Test_setProgressCompleteWithRepo_Progress(t *testing.T) {
},
}
log.SetOutput(io.Discard)
for _, tt := range tests {
s := &Source{
repos: tt.repos,
pkg/sources/resume.go
+1 -4
View File
@@ -2,8 +2,6 @@ package sources
import (
"strings"
"github.com/sirupsen/logrus"
)
// RemoveRepoFromResumeInfo removes the repoURL from the resume info.
@@ -18,8 +16,7 @@ func RemoveRepoFromResumeInfo(resumeRepos []string, repoURL string) []string {
if index == -1 {
// We should never be able to be here. But if we are, it means the resume info never had the repo added.
// So log the error and do nothing.
logrus.Errorf("repoURL (%q) not found in list of encode resume info: %v", repoURL, resumeRepos)
// So do nothing.
return resumeRepos
}
pkg/sources/resume_test.go
-9
View File
@@ -1,11 +1,8 @@
package sources
import (
"io"
"reflect"
"testing"
"github.com/sirupsen/logrus"
)
func TestRemoveRepoFromResumeInfo(t *testing.T) {
@@ -31,9 +28,6 @@ func TestRemoveRepoFromResumeInfo(t *testing.T) {
},
}
// Prevent the error message from showing up.
logrus.SetOutput(io.Discard)
for _, tt := range tests {
gotResumeInfoSlice := RemoveRepoFromResumeInfo(tt.startingResumeInfoSlice, tt.repoURL)
if !reflect.DeepEqual(gotResumeInfoSlice, tt.wantResumeInfoSlice) {
@@ -57,9 +51,6 @@ func TestEncodeResumeInfo(t *testing.T) {
},
}
logger := logrus.New()
logger.Out = io.Discard
for _, tt := range tests {
gotEncodedResumeInfo := EncodeResumeInfo(tt.startingResumeInfoSlice)
if gotEncodedResumeInfo != tt.wantEncodedResumeInfo {
pkg/sources/s3/s3_test.go
-4
View File
@@ -9,7 +9,6 @@ import (
"time"
"github.com/kylelemons/godebug/pretty"
log "github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert"
"google.golang.org/protobuf/types/known/anypb"
@@ -62,14 +61,11 @@ func TestSource_Chunks(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
log.SetFormatter(&log.TextFormatter{ForceColors: true})
ctx, cancel := context.WithTimeout(context.Background(), time.Second*30)
var cancelOnce sync.Once
defer cancelOnce.Do(cancel)
s := Source{}
log.SetLevel(log.DebugLevel)
conn, err := anypb.New(tt.init.connection)
if err != nil {
t.Fatal(err)