feat: Increase Datadog token detection distance

Allow a larger character window between keywords and tokens to improve detection accuracy, especially with long URLs or comments.

Co-authored-by: dylan <[email protected]>

pkg/detectors/datadogtoken/datadogtoken.go

@@ -29,8 +29,10 @@ var (
 	client = common.SaneHttpClient()
 
 	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
-	appPat = regexp.MustCompile(detectors.PrefixRegex([]string{"datadog", "dd"}) + `\b([a-zA-Z-0-9]{40})\b`)
-	apiPat = regexp.MustCompile(detectors.PrefixRegex([]string{"datadog", "dd"}) + `\b([a-zA-Z-0-9]{32})\b`)
+	// Datadog keys are often documented next to long URLs; allow a larger span so the keyword isn't "pushed away"
+	// by comment/URL tails.
+	appPat = regexp.MustCompile(detectors.PrefixRegexWithMaxDistance([]string{"datadog", "dd"}, 200) + `\b([a-zA-Z-0-9]{40})\b`)
+	apiPat = regexp.MustCompile(detectors.PrefixRegexWithMaxDistance([]string{"datadog", "dd"}, 200) + `\b([a-zA-Z-0-9]{32})\b`)
 )
 
 type userServiceResponse struct {

pkg/detectors/datadogtoken/datadogtoken_test.go

@@ -38,6 +38,16 @@ func TestDataDogToken_Pattern(t *testing.T) {
 	d := Scanner{}
 	ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
 
+	longURLCommentBetweenKeywordAndKey := `
+class DATADOG {
+	// Use this link to find the API Key https://app.datadoghq.com/organization-settings/api-keys?filter=tray&id=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
+	static API_KEY = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa1";
+	// Use this link to find the APP KEY https://app.datadoghq.com/organization-settings/application-keys?filter=tray&id=bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb
+	static APP_KEY = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb2";
+}
+`
+	longURLCommentBetweenKeywordAndKeySecret := "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb2aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa1"
+
 	tests := []struct {
 		name  string
 		input string
@@ -48,6 +58,11 @@ func TestDataDogToken_Pattern(t *testing.T) {
 			input: validPattern,
 			want:  []string{secret},
 		},
+		{
+			name:  "keyword far due to long URL/comment tail",
+			input: longURLCommentBetweenKeywordAndKey,
+			want:  []string{longURLCommentBetweenKeywordAndKeySecret},
+		},
 	}
 
 	for _, test := range tests {

pkg/detectors/detectors.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"crypto/rand"
 	"errors"
+	"fmt"
 	"math/big"
 	"net/url"
 	"strings"
@@ -260,9 +261,22 @@ func CleanResults(results []Result) []Result {
 // 40 characters of the capturing group that follows.
 // This can help prevent false positives.
 func PrefixRegex(keywords []string) string {
+	return PrefixRegexWithMaxDistance(keywords, 40)
+}
+
+// PrefixRegexWithMaxDistance is like PrefixRegex, but allows callers to tune
+// how far the keyword may appear from the capturing group that follows.
+//
+// This is intentionally a byte/character window, not a "line" window. Some
+// detectors may need a larger span to tolerate long URLs/comments without
+// widening the window globally for every detector.
+func PrefixRegexWithMaxDistance(keywords []string, maxChars int) string {
+	if maxChars < 0 {
+		maxChars = 0
+	}
 	pre := `(?i:`
 	middle := strings.Join(keywords, "|")
-	post := `)(?:.|[\n\r]){0,40}?`
+	post := fmt.Sprintf(`)(?:.|[\n\r]){0,%d}?`, maxChars)
 	return pre + middle + post
 }