[THOG-709] - Recover from detector panics (#810)

pkg/common/recover.go

@@ -11,8 +11,22 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
 )
 
-// Recover handles panics and reports to Sentry before exiting.
+// Recover handles panics and reports to Sentry.
 func Recover(ctx context.Context) {
+	if err := recover(); err != nil {
+		panicStack := string(debug.Stack())
+		if eventID := sentry.CurrentHub().Recover(err); eventID != nil {
+			ctx.Logger().Info("panic captured", "event_id", *eventID)
+		}
+		fmt.Fprint(os.Stderr, panicStack)
+		if !sentry.Flush(time.Second * 5) {
+			ctx.Logger().Info("sentry flush failed")
+		}
+	}
+}
+
+// RecoverWithExit handles panics and reports to Sentry before exiting.
+func RecoverWithExit(ctx context.Context) {
 	if err := recover(); err != nil {
 		panicStack := string(debug.Stack())
 		if eventID := sentry.CurrentHub().Recover(err); eventID != nil {

pkg/engine/engine.go

@@ -98,7 +98,7 @@ func Start(ctx context.Context, options ...EngineOption) *Engine {
 	for i := 0; i < e.concurrency; i++ {
 		e.workersWg.Add(1)
 		go func() {
-			defer common.Recover(ctx)
+			defer common.RecoverWithExit(ctx)
 			defer e.workersWg.Done()
 			e.detectorWorker(ctx)
 		}()
@@ -111,7 +111,7 @@ func Start(ctx context.Context, options ...EngineOption) *Engine {
 // chunks before closing their respective channels. Once Finish is called, no
 // more sources may be scanned by the engine.
 func (e *Engine) Finish(ctx context.Context) {
-	defer common.Recover(ctx)
+	defer common.RecoverWithExit(ctx)
 	// wait for the sources to finish putting chunks onto the chunks channel
 	e.sourcesWg.Wait()
 	close(e.chunks)
@@ -163,7 +163,12 @@ func (e *Engine) detectorWorker(ctx context.Context) {
 	for chunk := range e.chunks {
 		fragStart, mdLine := fragmentFirstLine(chunk)
 		for _, decoder := range e.decoders {
-			decoded := decoder.FromChunk(chunk)
+			decoded := func(ctx context.Context) *sources.Chunk {
+				defer func(ctx context.Context) {
+					common.Recover(ctx)
+				}(ctx)
+				return decoder.FromChunk(chunk)
+			}(ctx)
 			if decoded == nil {
 				continue
 			}

pkg/engine/filesystem.go

@@ -34,7 +34,7 @@ func (e *Engine) ScanFileSystem(ctx context.Context, c sources.Config) error {
 	}
 	e.sourcesWg.Add(1)
 	go func() {
-		defer common.Recover(ctx)
+		defer common.RecoverWithExit(ctx)
 		defer e.sourcesWg.Done()
 		err := fileSystemSource.Chunks(ctx, e.ChunksChan())
 		if err != nil {

pkg/engine/git.go

@@ -107,7 +107,7 @@ func (e *Engine) ScanGit(ctx context.Context, c sources.Config) error {
 
 	e.sourcesWg.Add(1)
 	go func() {
-		defer common.Recover(ctx)
+		defer common.RecoverWithExit(ctx)
 		defer e.sourcesWg.Done()
 		err := gitSource.ScanRepo(ctx, repo, c.RepoPath, scanOptions, e.ChunksChan())
 		if err != nil {

pkg/engine/github.go

@@ -43,7 +43,7 @@ func (e *Engine) ScanGitHub(ctx context.Context, c sources.Config) error {
 
 	e.sourcesWg.Add(1)
 	go func() {
-		defer common.Recover(ctx)
+		defer common.RecoverWithExit(ctx)
 		defer e.sourcesWg.Done()
 		err := source.Chunks(ctx, e.ChunksChan())
 		if err != nil {

pkg/engine/gitlab.go

@@ -52,7 +52,7 @@ func (e *Engine) ScanGitLab(ctx context.Context, c sources.Config) error {
 
 	e.sourcesWg.Add(1)
 	go func() {
-		defer common.Recover(ctx)
+		defer common.RecoverWithExit(ctx)
 		defer e.sourcesWg.Done()
 		err := gitlabSource.Chunks(ctx, e.ChunksChan())
 		if err != nil {

pkg/engine/s3.go

@@ -54,7 +54,7 @@ func (e *Engine) ScanS3(ctx context.Context, c sources.Config) error {
 
 	e.sourcesWg.Add(1)
 	go func() {
-		defer common.Recover(ctx)
+		defer common.RecoverWithExit(ctx)
 		defer e.sourcesWg.Done()
 		err := s3Source.Chunks(ctx, e.ChunksChan())
 		if err != nil {

pkg/engine/syslog.go

@@ -52,7 +52,7 @@ func (e *Engine) ScanSyslog(ctx context.Context, c sources.Config) error {
 
 	e.sourcesWg.Add(1)
 	go func() {
-		defer common.Recover(ctx)
+		defer common.RecoverWithExit(ctx)
 		defer e.sourcesWg.Done()
 		err := source.Chunks(ctx, e.ChunksChan())
 		if err != nil {

pkg/gitparse/gitparse.go

@@ -142,7 +142,7 @@ func FromReader(ctx context.Context, stdOut io.Reader, commitChan chan Commit) {
 	var currentCommit *Commit
 	var currentDiff *Diff
 
-	defer common.Recover(ctx)
+	defer common.RecoverWithExit(ctx)
 	for {
 		line, err := outReader.ReadBytes([]byte("\n")[0])
 		if err != nil && len(line) == 0 {
@@ -313,7 +313,7 @@ func isMinusDiffLine(line []byte) bool {
 	return false
 }
 
-//  fmt.Println("ok")
+// fmt.Println("ok")
 func isContextDiffLine(line []byte) bool {
 	if len(line) >= 1 && bytes.Equal(line[:1], []byte(" ")) {
 		return true

pkg/sources/s3/s3.go

@@ -194,7 +194,7 @@ func (s *Source) pageChunker(ctx context.Context, client *s3.S3, chunksChan chan
 		}
 		wg.Add(1)
 		go func(ctx context.Context, wg *sync.WaitGroup, sem *semaphore.Weighted, obj *s3.Object) {
-			defer common.Recover(ctx)
+			defer common.RecoverWithExit(ctx)
 			defer sem.Release(1)
 			defer wg.Done()
 			// defer log.Debugf("DONE - %s", *obj.Key)

pkg/sources/syslog/syslog.go

@@ -210,7 +210,7 @@ func (s *Source) parseSyslogMetadata(input []byte, remote string) (*source_metad
 }
 
 func (s *Source) monitorConnection(ctx context.Context, conn net.Conn, chunksChan chan *sources.Chunk) {
-	defer common.Recover(ctx)
+	defer common.RecoverWithExit(ctx)
 	for {
 		if common.IsDone(ctx) {
 			return