diff --git a/go.mod b/go.mod index 5dbd09dd0..9efbf65b7 100644 --- a/go.mod +++ b/go.mod @@ -47,7 +47,6 @@ require ( github.com/xanzy/go-gitlab v0.73.1 go.uber.org/zap v1.22.0 golang.org/x/crypto v0.0.0-20211215165025-cf75a172585e - golang.org/x/net v0.0.0-20220805013720-a33c5aa5df48 golang.org/x/oauth2 v0.0.0-20220722155238-128564f6959c golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f google.golang.org/genproto v0.0.0-20220617124728-180714bec0ad @@ -114,6 +113,7 @@ require ( go.uber.org/atomic v1.7.0 // indirect go.uber.org/multierr v1.6.0 // indirect golang.org/x/mod v0.5.0 // indirect + golang.org/x/net v0.0.0-20220805013720-a33c5aa5df48 // indirect golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab // indirect golang.org/x/text v0.3.7 // indirect golang.org/x/time v0.0.0-20220722155302-e5dcc9cfc0b9 // indirect diff --git a/hack/snifftest/main.go b/hack/snifftest/main.go index 2de774af3..5f4082157 100644 --- a/hack/snifftest/main.go +++ b/hack/snifftest/main.go @@ -1,7 +1,6 @@ package main import ( - "context" "fmt" "os" "reflect" @@ -16,6 +15,7 @@ import ( "golang.org/x/sync/semaphore" "gopkg.in/alecthomas/kingpin.v2" + "github.com/trufflesecurity/trufflehog/v3/pkg/context" "github.com/trufflesecurity/trufflehog/v3/pkg/decoders" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/engine" diff --git a/main.go b/main.go index 4b7eeaba1..e8742c9ba 100644 --- a/main.go +++ b/main.go @@ -1,7 +1,6 @@ package main import ( - "context" "fmt" "log" "net/http" @@ -19,15 +18,15 @@ import ( "github.com/sirupsen/logrus" "gopkg.in/alecthomas/kingpin.v2" - "github.com/trufflesecurity/trufflehog/v3/pkg/sources" - "github.com/trufflesecurity/trufflehog/v3/pkg/updater" - "github.com/trufflesecurity/trufflehog/v3/pkg/version" - "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/context" "github.com/trufflesecurity/trufflehog/v3/pkg/decoders" "github.com/trufflesecurity/trufflehog/v3/pkg/engine" "github.com/trufflesecurity/trufflehog/v3/pkg/output" + "github.com/trufflesecurity/trufflehog/v3/pkg/sources" "github.com/trufflesecurity/trufflehog/v3/pkg/sources/git" + "github.com/trufflesecurity/trufflehog/v3/pkg/updater" + "github.com/trufflesecurity/trufflehog/v3/pkg/version" ) var ( @@ -264,7 +263,7 @@ func run(state overseer.State) { } } // asynchronously wait for scanning to finish and cleanup - go e.Finish() + go e.Finish(ctx) if !*jsonLegacy && !*jsonOut { fmt.Fprintf(os.Stderr, "🐷🔑🐷 TruffleHog. Unearth your secrets. 🐷🔑🐷\n\n") diff --git a/pkg/common/recover.go b/pkg/common/recover.go new file mode 100644 index 000000000..56888dd0f --- /dev/null +++ b/pkg/common/recover.go @@ -0,0 +1,27 @@ +package common + +import ( + "fmt" + "os" + "runtime/debug" + "time" + + "github.com/getsentry/sentry-go" + + "github.com/trufflesecurity/trufflehog/v3/pkg/context" +) + +// Recover handles panics and reports to Sentry before exiting. +func Recover(ctx context.Context) { + if err := recover(); err != nil { + panicStack := string(debug.Stack()) + if eventID := sentry.CurrentHub().Recover(err); eventID != nil { + ctx.Logger().Info("panic captured", "event_id", *eventID) + } + fmt.Fprint(os.Stderr, panicStack) + if !sentry.Flush(time.Second * 5) { + ctx.Logger().Info("sentry flush failed") + } + os.Exit(1) + } +} diff --git a/pkg/engine/engine.go b/pkg/engine/engine.go index 57d4e8a9f..4b579a9e1 100644 --- a/pkg/engine/engine.go +++ b/pkg/engine/engine.go @@ -2,7 +2,6 @@ package engine import ( "bytes" - "context" "runtime" "strings" "sync" @@ -11,6 +10,8 @@ import ( "github.com/sirupsen/logrus" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/context" "github.com/trufflesecurity/trufflehog/v3/pkg/decoders" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb" @@ -97,6 +98,7 @@ func Start(ctx context.Context, options ...EngineOption) *Engine { for i := 0; i < e.concurrency; i++ { e.workersWg.Add(1) go func() { + defer common.Recover(ctx) defer e.workersWg.Done() e.detectorWorker(ctx) }() @@ -108,7 +110,8 @@ func Start(ctx context.Context, options ...EngineOption) *Engine { // Finish waits for running sources to complete and workers to finish scanning // chunks before closing their respective channels. Once Finish is called, no // more sources may be scanned by the engine. -func (e *Engine) Finish() { +func (e *Engine) Finish(ctx context.Context) { + defer common.Recover(ctx) // wait for the sources to finish putting chunks onto the chunks channel e.sourcesWg.Wait() close(e.chunks) diff --git a/pkg/engine/filesystem.go b/pkg/engine/filesystem.go index 22eb45452..7823303cc 100644 --- a/pkg/engine/filesystem.go +++ b/pkg/engine/filesystem.go @@ -1,7 +1,6 @@ package engine import ( - "context" "runtime" "github.com/go-errors/errors" @@ -9,6 +8,8 @@ import ( "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/anypb" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/context" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb" "github.com/trufflesecurity/trufflehog/v3/pkg/sources" "github.com/trufflesecurity/trufflehog/v3/pkg/sources/filesystem" @@ -33,6 +34,7 @@ func (e *Engine) ScanFileSystem(ctx context.Context, c sources.Config) error { } e.sourcesWg.Add(1) go func() { + defer common.Recover(ctx) defer e.sourcesWg.Done() err := fileSystemSource.Chunks(ctx, e.ChunksChan()) if err != nil { diff --git a/pkg/engine/git.go b/pkg/engine/git.go index d4fed1e68..1638d4ff2 100644 --- a/pkg/engine/git.go +++ b/pkg/engine/git.go @@ -1,7 +1,6 @@ package engine import ( - "context" "fmt" "runtime" @@ -11,6 +10,8 @@ import ( "github.com/go-git/go-git/v5/plumbing/object" "github.com/sirupsen/logrus" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/context" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb" "github.com/trufflesecurity/trufflehog/v3/pkg/sources" @@ -106,6 +107,7 @@ func (e *Engine) ScanGit(ctx context.Context, c sources.Config) error { e.sourcesWg.Add(1) go func() { + defer common.Recover(ctx) defer e.sourcesWg.Done() err := gitSource.ScanRepo(ctx, repo, c.RepoPath, scanOptions, e.ChunksChan()) if err != nil { diff --git a/pkg/engine/git_test.go b/pkg/engine/git_test.go index 2d841f894..9803066c0 100644 --- a/pkg/engine/git_test.go +++ b/pkg/engine/git_test.go @@ -1,11 +1,11 @@ package engine import ( - "context" "os" "testing" "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/context" "github.com/trufflesecurity/trufflehog/v3/pkg/decoders" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb" "github.com/trufflesecurity/trufflehog/v3/pkg/sources" @@ -67,7 +67,7 @@ func TestGitEngine(t *testing.T) { if err := e.ScanGit(ctx, cfg); err != nil { return } - go e.Finish() + go e.Finish(ctx) resultCount := 0 for result := range e.ResultsChan() { switch meta := result.SourceMetadata.GetData().(type) { @@ -122,5 +122,5 @@ func BenchmarkGitEngine(b *testing.B) { return } } - e.Finish() + e.Finish(ctx) } diff --git a/pkg/engine/github.go b/pkg/engine/github.go index f8f8d8480..0c8f9d157 100644 --- a/pkg/engine/github.go +++ b/pkg/engine/github.go @@ -1,12 +1,12 @@ package engine import ( - "context" - "github.com/sirupsen/logrus" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/anypb" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/context" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb" "github.com/trufflesecurity/trufflehog/v3/pkg/sources" "github.com/trufflesecurity/trufflehog/v3/pkg/sources/github" @@ -43,6 +43,7 @@ func (e *Engine) ScanGitHub(ctx context.Context, c sources.Config) error { e.sourcesWg.Add(1) go func() { + defer common.Recover(ctx) defer e.sourcesWg.Done() err := source.Chunks(ctx, e.ChunksChan()) if err != nil { diff --git a/pkg/engine/gitlab.go b/pkg/engine/gitlab.go index 1ae0f6272..cf3fbfcfc 100644 --- a/pkg/engine/gitlab.go +++ b/pkg/engine/gitlab.go @@ -6,10 +6,11 @@ import ( "github.com/go-errors/errors" "github.com/sirupsen/logrus" - "golang.org/x/net/context" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/anypb" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/context" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb" "github.com/trufflesecurity/trufflehog/v3/pkg/sources" "github.com/trufflesecurity/trufflehog/v3/pkg/sources/gitlab" @@ -51,6 +52,7 @@ func (e *Engine) ScanGitLab(ctx context.Context, c sources.Config) error { e.sourcesWg.Add(1) go func() { + defer common.Recover(ctx) defer e.sourcesWg.Done() err := gitlabSource.Chunks(ctx, e.ChunksChan()) if err != nil { diff --git a/pkg/engine/s3.go b/pkg/engine/s3.go index 5295e2748..7c1df9f36 100644 --- a/pkg/engine/s3.go +++ b/pkg/engine/s3.go @@ -1,7 +1,6 @@ package engine import ( - "context" "fmt" "runtime" @@ -10,6 +9,8 @@ import ( "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/anypb" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/context" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/credentialspb" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb" "github.com/trufflesecurity/trufflehog/v3/pkg/sources" @@ -53,6 +54,7 @@ func (e *Engine) ScanS3(ctx context.Context, c sources.Config) error { e.sourcesWg.Add(1) go func() { + defer common.Recover(ctx) defer e.sourcesWg.Done() err := s3Source.Chunks(ctx, e.ChunksChan()) if err != nil { diff --git a/pkg/engine/syslog.go b/pkg/engine/syslog.go index 3b0ffbd5b..ff1754ab2 100644 --- a/pkg/engine/syslog.go +++ b/pkg/engine/syslog.go @@ -1,7 +1,6 @@ package engine import ( - "context" "os" "github.com/go-errors/errors" @@ -9,6 +8,8 @@ import ( "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/anypb" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/context" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb" "github.com/trufflesecurity/trufflehog/v3/pkg/sources" "github.com/trufflesecurity/trufflehog/v3/pkg/sources/syslog" @@ -51,6 +52,7 @@ func (e *Engine) ScanSyslog(ctx context.Context, c sources.Config) error { e.sourcesWg.Add(1) go func() { + defer common.Recover(ctx) defer e.sourcesWg.Done() err := source.Chunks(ctx, e.ChunksChan()) if err != nil { diff --git a/pkg/gitparse/gitparse.go b/pkg/gitparse/gitparse.go index 925da99ce..09ae120fc 100644 --- a/pkg/gitparse/gitparse.go +++ b/pkg/gitparse/gitparse.go @@ -11,6 +11,9 @@ import ( "time" log "github.com/sirupsen/logrus" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/context" ) // DateFormat is the standard date format for git. @@ -34,7 +37,7 @@ type Diff struct { } // RepoPath parses the output of the `git log` command for the `source` path. -func RepoPath(source string, head string) (chan Commit, error) { +func RepoPath(ctx context.Context, source string, head string) (chan Commit, error) { commitChan := make(chan Commit) args := []string{"-C", source, "log", "-p", "-U0", "--full-history", "--diff-filter=AM", "--date=format:%a %b %d %H:%M:%S %Y %z"} @@ -77,6 +80,7 @@ func RepoPath(source string, head string) (chan Commit, error) { }() go func() { + defer common.Recover(ctx) for { line, err := outReader.ReadBytes([]byte("\n")[0]) if err != nil && len(line) == 0 { diff --git a/pkg/sources/filesystem/filesystem.go b/pkg/sources/filesystem/filesystem.go index 22fbc81e3..c2030b030 100644 --- a/pkg/sources/filesystem/filesystem.go +++ b/pkg/sources/filesystem/filesystem.go @@ -1,7 +1,6 @@ package filesystem import ( - "context" "fmt" "io" "io/fs" @@ -15,6 +14,7 @@ import ( "google.golang.org/protobuf/types/known/anypb" "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/context" "github.com/trufflesecurity/trufflehog/v3/pkg/handlers" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb" diff --git a/pkg/sources/filesystem/filesystem_test.go b/pkg/sources/filesystem/filesystem_test.go index 21c39427b..a3e07525c 100644 --- a/pkg/sources/filesystem/filesystem_test.go +++ b/pkg/sources/filesystem/filesystem_test.go @@ -1,7 +1,6 @@ package filesystem import ( - "context" "testing" "time" @@ -9,6 +8,7 @@ import ( log "github.com/sirupsen/logrus" "google.golang.org/protobuf/types/known/anypb" + "github.com/trufflesecurity/trufflehog/v3/pkg/context" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb" "github.com/trufflesecurity/trufflehog/v3/pkg/sources" diff --git a/pkg/sources/git/git.go b/pkg/sources/git/git.go index 8b5d5bbfb..894eb61b4 100644 --- a/pkg/sources/git/git.go +++ b/pkg/sources/git/git.go @@ -2,7 +2,6 @@ package git import ( "bytes" - "context" "fmt" "io" "io/ioutil" @@ -27,6 +26,7 @@ import ( "google.golang.org/protobuf/types/known/anypb" "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/context" "github.com/trufflesecurity/trufflehog/v3/pkg/gitparse" "github.com/trufflesecurity/trufflehog/v3/pkg/handlers" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb" @@ -279,7 +279,7 @@ func GitCmdCheck() error { return nil } -func (s *Git) ScanCommits(repo *git.Repository, path string, scanOptions *ScanOptions, chunksChan chan *sources.Chunk) error { +func (s *Git) ScanCommits(ctx context.Context, repo *git.Repository, path string, scanOptions *ScanOptions, chunksChan chan *sources.Chunk) error { if err := GitCmdCheck(); err != nil { return err } @@ -287,7 +287,7 @@ func (s *Git) ScanCommits(repo *git.Repository, path string, scanOptions *ScanOp zerolog.SetGlobalLevel(zerolog.Disabled) } - commitChan, err := gitparse.RepoPath(path, scanOptions.HeadHash) + commitChan, err := gitparse.RepoPath(ctx, path, scanOptions.HeadHash) if err != nil { return err } @@ -412,9 +412,9 @@ func (s *Git) ScanUnstaged(repo *git.Repository, scanOptions *ScanOptions, chunk return nil } -func (s *Git) ScanRepo(_ context.Context, repo *git.Repository, repoPath string, scanOptions *ScanOptions, chunksChan chan *sources.Chunk) error { +func (s *Git) ScanRepo(ctx context.Context, repo *git.Repository, repoPath string, scanOptions *ScanOptions, chunksChan chan *sources.Chunk) error { start := time.Now().UnixNano() - if err := s.ScanCommits(repo, repoPath, scanOptions, chunksChan); err != nil { + if err := s.ScanCommits(ctx, repo, repoPath, scanOptions, chunksChan); err != nil { return err } if err := s.ScanUnstaged(repo, scanOptions, chunksChan); err != nil { @@ -591,9 +591,9 @@ func PrepareRepo(uriString string) (string, bool, error) { remotePath := uri.String() remote = true path, _, err = CloneRepoUsingSSH(remotePath) - if err != nil { - return path, remote, fmt.Errorf("failed to clone unauthenticated Git repo (%s): %s", remotePath, err) - } + if err != nil { + return path, remote, fmt.Errorf("failed to clone unauthenticated Git repo (%s): %s", remotePath, err) + } default: return "", remote, fmt.Errorf("unsupported Git URI: %s", uriString) } diff --git a/pkg/sources/git/git_test.go b/pkg/sources/git/git_test.go index 1169e209e..9b544bd97 100644 --- a/pkg/sources/git/git_test.go +++ b/pkg/sources/git/git_test.go @@ -2,19 +2,20 @@ package git import ( "bytes" - "context" "fmt" "strings" "testing" "github.com/kylelemons/godebug/pretty" log "github.com/sirupsen/logrus" + "google.golang.org/protobuf/types/known/anypb" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/context" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/credentialspb" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb" "github.com/trufflesecurity/trufflehog/v3/pkg/sources" - "google.golang.org/protobuf/types/known/anypb" ) func TestSource_Scan(t *testing.T) { diff --git a/pkg/sources/github/github.go b/pkg/sources/github/github.go index 457ba0915..77ce7336b 100644 --- a/pkg/sources/github/github.go +++ b/pkg/sources/github/github.go @@ -1,7 +1,6 @@ package github import ( - "context" "fmt" "net/http" "os" @@ -25,6 +24,7 @@ import ( "google.golang.org/protobuf/types/known/anypb" "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/context" "github.com/trufflesecurity/trufflehog/v3/pkg/giturl" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/credentialspb" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb" diff --git a/pkg/sources/github/github_integration_test.go b/pkg/sources/github/github_integration_test.go index a5ae67f69..a58d4341a 100644 --- a/pkg/sources/github/github_integration_test.go +++ b/pkg/sources/github/github_integration_test.go @@ -4,7 +4,6 @@ package github import ( - "context" "encoding/base64" "fmt" "os" @@ -19,6 +18,7 @@ import ( "google.golang.org/protobuf/types/known/anypb" "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/context" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/credentialspb" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb" diff --git a/pkg/sources/github/github_test.go b/pkg/sources/github/github_test.go index 0524f0d7e..b74d9b722 100644 --- a/pkg/sources/github/github_test.go +++ b/pkg/sources/github/github_test.go @@ -2,7 +2,6 @@ package github import ( "bytes" - "context" "crypto/rand" "crypto/rsa" "crypto/x509" @@ -21,6 +20,7 @@ import ( "google.golang.org/protobuf/types/known/anypb" "gopkg.in/h2non/gock.v1" + "github.com/trufflesecurity/trufflehog/v3/pkg/context" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/credentialspb" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb" ) diff --git a/pkg/sources/gitlab/gitlab.go b/pkg/sources/gitlab/gitlab.go index ef8878fb0..86355c73e 100644 --- a/pkg/sources/gitlab/gitlab.go +++ b/pkg/sources/gitlab/gitlab.go @@ -1,7 +1,6 @@ package gitlab import ( - "context" "fmt" "net/url" "os" @@ -11,6 +10,7 @@ import ( "sync" "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/context" "github.com/trufflesecurity/trufflehog/v3/pkg/giturl" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb" diff --git a/pkg/sources/gitlab/gitlab_test.go b/pkg/sources/gitlab/gitlab_test.go index 8aa373f27..fdec4cb1a 100644 --- a/pkg/sources/gitlab/gitlab_test.go +++ b/pkg/sources/gitlab/gitlab_test.go @@ -1,7 +1,6 @@ package gitlab import ( - "context" "fmt" "io" "reflect" @@ -12,6 +11,7 @@ import ( "google.golang.org/protobuf/types/known/anypb" "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/context" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/credentialspb" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb" "github.com/trufflesecurity/trufflehog/v3/pkg/sources" diff --git a/pkg/sources/s3/s3.go b/pkg/sources/s3/s3.go index c2b949a19..49ba0a354 100644 --- a/pkg/sources/s3/s3.go +++ b/pkg/sources/s3/s3.go @@ -1,7 +1,6 @@ package s3 import ( - "context" "fmt" "strings" "sync" @@ -15,15 +14,17 @@ import ( diskbufferreader "github.com/bill-rich/disk-buffer-reader" "github.com/go-errors/errors" log "github.com/sirupsen/logrus" + "golang.org/x/sync/semaphore" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/anypb" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/context" "github.com/trufflesecurity/trufflehog/v3/pkg/handlers" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb" "github.com/trufflesecurity/trufflehog/v3/pkg/sanitizer" "github.com/trufflesecurity/trufflehog/v3/pkg/sources" - "golang.org/x/sync/semaphore" - "google.golang.org/protobuf/proto" - "google.golang.org/protobuf/types/known/anypb" ) type Source struct { @@ -193,6 +194,7 @@ func (s *Source) pageChunker(ctx context.Context, client *s3.S3, chunksChan chan } wg.Add(1) go func(ctx context.Context, wg *sync.WaitGroup, sem *semaphore.Weighted, obj *s3.Object) { + defer common.Recover(ctx) defer sem.Release(1) defer wg.Done() //defer log.Debugf("DONE - %s", *obj.Key) diff --git a/pkg/sources/s3/s3_test.go b/pkg/sources/s3/s3_test.go index 5ce7cd2d3..6aa667ef9 100644 --- a/pkg/sources/s3/s3_test.go +++ b/pkg/sources/s3/s3_test.go @@ -1,7 +1,6 @@ package s3 import ( - "context" "encoding/base64" "fmt" "os" @@ -14,6 +13,7 @@ import ( "google.golang.org/protobuf/types/known/anypb" "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/context" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/credentialspb" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb" "github.com/trufflesecurity/trufflehog/v3/pkg/sources" diff --git a/pkg/sources/sources.go b/pkg/sources/sources.go index ea0d36c71..2cdf54063 100644 --- a/pkg/sources/sources.go +++ b/pkg/sources/sources.go @@ -1,12 +1,12 @@ package sources import ( - "context" "sync" "google.golang.org/protobuf/types/known/anypb" "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/context" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb" ) diff --git a/pkg/sources/syslog/syslog.go b/pkg/sources/syslog/syslog.go index 2731879a5..11b4784ba 100644 --- a/pkg/sources/syslog/syslog.go +++ b/pkg/sources/syslog/syslog.go @@ -1,10 +1,8 @@ package syslog import ( - "context" "crypto/tls" "fmt" - "github.com/trufflesecurity/trufflehog/v3/pkg/common" "io" "net" "runtime" @@ -19,6 +17,8 @@ import ( "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/anypb" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/context" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb" "github.com/trufflesecurity/trufflehog/v3/pkg/sources" @@ -210,6 +210,7 @@ func (s *Source) parseSyslogMetadata(input []byte, remote string) (*source_metad } func (s *Source) monitorConnection(ctx context.Context, conn net.Conn, chunksChan chan *sources.Chunk) { + defer common.Recover(ctx) for { if common.IsDone(ctx) { return