Adding no-ignore flag to allow reporting of "ignored" secrets (#5297)

* Adding no-ignore flag to allow reporting of "ignored" secrets

* ran make man

* changed --no-ignore to --no-ignore-tag

README.md

@@ -406,6 +406,8 @@ aws s3 cp s3://example/gzipped/data.gz - | gunzip -c | trufflehog stdin
   - A verified result means TruffleHog confirmed the credential is valid by testing it against the service's API. For private keys, we've confirmed the key can be used live for SSH or SSL authentication. Check out our Driftwood blog post to learn more [Blog post](https://trufflesecurity.com/blog/driftwood-know-if-private-keys-are-sensitive/)
 - Is there an easy way to ignore specific secrets?
   - If the scanned source [supports line numbers](https://github.com/trufflesecurity/trufflehog/blob/d6375ba92172fd830abb4247cca15e3176448c5d/pkg/engine/engine.go#L358-L365), then you can add a `trufflehog:ignore` comment on the line containing the secret to ignore that secrets.
+- Can I find secrets with `trufflehog:ignore` included?
+  - Pass `--no-ignore-tag` to report results even when their line carries a `trufflehog:ignore` comment. This is useful for reviewing previously accepted findings.
 
 # :newspaper: What's new in v3?
 

docs/man/trufflehog.1

@@ -62,6 +62,9 @@ Only output first unverified result per chunk per detector if there are more tha
 \fB--filter-entropy=FILTER-ENTROPY\fR
 Filter unverified results with Shannon entropy. Start with 3.0.
 .TP
+\fB--no-ignore-tag\fR
+Report results even if the line has a 'trufflehog:ignore' comment.
+.TP
 \fB--max-decode-depth=5\fR
 Maximum depth of iterative decoding. Each decoder's output is fed back through all decoders, up to this limit. 1 = single pass, 2+ = chained decoding (e.g., base64 inside utf16).
 .TP

main.go

@@ -66,6 +66,7 @@ var (
 	allowVerificationOverlap   = cli.Flag("allow-verification-overlap", "Allow verification of similar credentials across detectors").Bool()
 	filterUnverified           = cli.Flag("filter-unverified", "Only output first unverified result per chunk per detector if there are more than one results.").Bool()
 	filterEntropy              = cli.Flag("filter-entropy", "Filter unverified results with Shannon entropy. Start with 3.0.").Float64()
+	noIgnoreTag                = cli.Flag("no-ignore-tag", "Report results even if the line has a 'trufflehog:ignore' comment.").Bool()
 	scanEntireChunk            = cli.Flag("scan-entire-chunk", "Scan the entire chunk for secrets.").Hidden().Default("false").Bool()
 	maxDecodeDepth             = cli.Flag("max-decode-depth", "Maximum depth of iterative decoding. Each decoder's output is fed back through all decoders, up to this limit. 1 = single pass, 2+ = chained decoding (e.g., base64 inside utf16).").Default("5").Int()
 	compareDetectionStrategies = cli.Flag("compare-detection-strategies", "Compare different detection strategies for matching spans").Hidden().Default("false").Bool()
@@ -660,6 +661,7 @@ func run(state overseer.State, logSync func() error) {
 		Dispatcher:               engine.NewPrinterDispatcher(printer),
 		FilterUnverified:         *filterUnverified,
 		FilterEntropy:            *filterEntropy,
+		NoIgnoreTag:              *noIgnoreTag,
 		VerificationOverlap:      *allowVerificationOverlap,
 		Results:                  parsedResults,
 		PrintAvgDetectorTime:     *printAvgDetectorTime,

pkg/engine/engine.go

@@ -126,6 +126,10 @@ type Config struct {
 	FilterUnverified      bool
 	ShouldScanEntireChunk bool
 
+	// NoIgnoreTag disables the "trufflehog:ignore" tag. If set to true, results are
+	// reported even when the line they were found on carries the tag.
+	NoIgnoreTag bool
+
 	Dispatcher ResultsDispatcher
 
 	// SourceManager orchestrates source enumeration and concurrent chunk production.
@@ -195,6 +199,8 @@ type Engine struct {
 	// By default, the engine will only scan a subset of the chunk if a detector matches the chunk.
 	// If this flag is set to true, the engine will scan the entire chunk.
 	scanEntireChunk bool
+	// noIgnoreTag disables the "trufflehog:ignore" tag, so tagged lines are still reported.
+	noIgnoreTag bool
 
 	// ahoCorasickHandler manages the Aho-Corasick trie and related keyword lookups.
 	AhoCorasickCore *ahocorasick.Core
@@ -259,6 +265,7 @@ func NewEngine(ctx context.Context, cfg *Config) (*Engine, error) {
 		verificationOverlap:                 cfg.VerificationOverlap,
 		sourceManager:                       cfg.SourceManager,
 		scanEntireChunk:                     cfg.ShouldScanEntireChunk,
+		noIgnoreTag:                         cfg.NoIgnoreTag,
 		detectorVerificationOverrides:       cfg.DetectorVerificationOverrides,
 		detectorWorkerMultiplier:            cfg.DetectorWorkerMultiplier,
 		notificationWorkerMultiplier:        cfg.NotificationWorkerMultiplier,
@@ -1265,7 +1272,8 @@ func (e *Engine) filterResults(
 }
 
 // processResult generates a detectors.ResultWithMetadata from the provided chunk and result and puts it on the results
-// channel, unless the result exists on a line with an ignore tag, in which case no result is generated.
+// channel, unless the result exists on a line with an ignore tag and --no-ignore-tag is not passed, in which case
+// no result is generated.
 func (e *Engine) processResult(
 	ctx context.Context,
 	res detectors.Result,
@@ -1290,7 +1298,7 @@ func (e *Engine) processResult(
 		}
 		chunk = copyChunk
 	}
-	if ignoreLinePresent {
+	if ignoreLinePresent && !e.noIgnoreTag {
 		resultsDropped.WithLabelValues("process_result", "ignore_line_tag", res.DetectorType.String()).Inc()
 		return
 	}

pkg/engine/engine_test.go

@@ -723,6 +723,39 @@ func TestProcessResult_IgnoreLinePresent_NothingGenerated(t *testing.T) {
 	assert.Empty(t, e.results)
 }
 
+func TestProcessResult_IgnoreLinePresentWithNoIgnoreTag_ResultGenerated(t *testing.T) {
+	// Arrange: Create an engine that does not honor ignore tags
+	e := Engine{results: make(chan detectors.ResultWithMetadata, 1), noIgnoreTag: true}
+
+	// Arrange: Create a Chunk
+	chunk := sources.Chunk{
+		Data: []byte("swordfish trufflehog:ignore"),
+		SourceMetadata: &source_metadatapb.MetaData{
+			Data: &source_metadatapb.MetaData_Git{
+				Git: &source_metadatapb.Git{
+					Line: 1,
+				},
+			},
+		},
+		SourceType: sourcespb.SourceType_SOURCE_TYPE_GIT,
+	}
+
+	// Arrange: Create a Result
+	result := detectors.Result{
+		Raw:      []byte("swordfish"),
+		Verified: true,
+	}
+
+	// Act
+	e.processResult(context.AddLogger(t.Context()), result, chunk, 0, "", nil)
+
+	// Assert that the result was reported anyway, with its line number still set
+	require.Len(t, e.results, 1)
+	r := <-e.results
+	assert.Equal(t, []byte("swordfish"), r.Raw)
+	assert.Equal(t, int64(1), r.SourceMetadata.GetGit().GetLine())
+}
+
 func TestProcessResult_AllFieldsCopied(t *testing.T) {
 	// Arrange: Create an engine
 	e := Engine{results: make(chan detectors.ResultWithMetadata, 1)}