[INS-254] Datadog detector verification fix and endpoint configuration (#4616)

* [INS-233] Added support to verify token agains all datadog domains

* [INS-233] Added support to verify token agains all datadog domains

* Fixed cloud endpoint test

* Added precedence to endpoint selection userdefiner -> datafound -> default

* fixed one integration test

* [INS-240] add API key verification fallback when app key verification fails

* Resolved comments

* Fixed the tests according to new changes

* Removed apikey verification logic datadogtoken file

* Reverted the engine test changed earlier

* Resolved comment(s)

* added /api to endpoint

* removed unecessary print

* removed configuredEndpoint to simplify logic

* removed matching with /api suffix

* Fixed the failing integration test

* Update keys in AnalysisInfo map to use snake_case

* fixed bot comments

* fixed ssrf vulnerablity

pkg/detectors/datadogtoken/datadogtoken.go

@@ -29,8 +29,9 @@ var (
 	client = common.SaneHttpClient()
 
 	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
-	appPat = regexp.MustCompile(detectors.PrefixRegex([]string{"datadog", "dd"}) + `\b([a-zA-Z-0-9]{40})\b`)
+	appPat        = regexp.MustCompile(detectors.PrefixRegex([]string{"datadog", "dd"}) + `\b([a-zA-Z-0-9]{40})\b`)
-	apiPat = regexp.MustCompile(detectors.PrefixRegex([]string{"datadog", "dd"}) + `\b([a-zA-Z-0-9]{32})\b`)
+	apiPat        = regexp.MustCompile(detectors.PrefixRegex([]string{"datadog", "dd"}) + `\b([a-zA-Z-0-9]{32})\b`)
+	datadogURLPat = regexp.MustCompile(`\b(api(?:\.[a-z0-9-]+)?\.(?:datadoghq|ddog-gov)\.(com|eu))\b`)
 )
 
 type userServiceResponse struct {
@@ -95,7 +96,7 @@ func setOrganizationInfo(opt []*options, s1 *detectors.Result) {
 // Keywords are used for efficiently pre-filtering chunks.
 // Use identifiers in the secret preferably, or the provider name.
 func (s Scanner) Keywords() []string {
-	return []string{"datadog"}
+	return []string{"datadog", "ddog-gov"}
 }
 
 // FromData will find and optionally verify DatadogToken secrets in a given set of bytes.
@@ -105,12 +106,19 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 	appMatches := appPat.FindAllStringSubmatch(dataStr, -1)
 	apiMatches := apiPat.FindAllStringSubmatch(dataStr, -1)
 
+	var uniqueFoundUrls = make(map[string]struct{})
+	for _, matches := range datadogURLPat.FindAllStringSubmatch(dataStr, -1) {
+		uniqueFoundUrls["https://"+matches[1]] = struct{}{}
+	}
+	endpoints := make([]string, 0, len(uniqueFoundUrls))
+	for endpoint := range uniqueFoundUrls {
+		endpoints = append(endpoints, endpoint)
+	}
+
 	for _, apiMatch := range apiMatches {
 		resApiMatch := strings.TrimSpace(apiMatch[1])
-		appIncluded := false
 		for _, appMatch := range appMatches {
 			resAppMatch := strings.TrimSpace(appMatch[1])
-
 			s1 := detectors.Result{
 				DetectorType: detectorspb.DetectorType_DatadogToken,
 				Raw:          []byte(resAppMatch),
@@ -121,7 +129,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			}
 
 			if verify {
-				for _, baseURL := range s.Endpoints() {
+				for _, baseURL := range s.Endpoints(endpoints...) {
 					req, err := http.NewRequestWithContext(ctx, "GET", baseURL+"/api/v2/users", nil)
 					if err != nil {
 						continue
@@ -134,7 +142,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 						defer res.Body.Close()
 						if res.StatusCode >= 200 && res.StatusCode < 300 {
 							s1.Verified = true
-							s1.AnalysisInfo = map[string]string{"apiKey": resApiMatch, "appKey": resAppMatch}
+							s1.AnalysisInfo = map[string]string{"api_key": resApiMatch, "app_key": resAppMatch, "endpoint": baseURL}
 							var serviceResponse userServiceResponse
 							if err := json.NewDecoder(res.Body).Decode(&serviceResponse); err == nil {
 								// setup emails
@@ -146,38 +154,8 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 									setOrganizationInfo(serviceResponse.Included, &s1)
 								}
 							}
-						}
+							// break the loop once we've successfully validated the token against a baseURL
-					}
+							break
-				}
-			}
-			appIncluded = true
-			results = append(results, s1)
-		}
-
-		if !appIncluded {
-			s1 := detectors.Result{
-				DetectorType: detectorspb.DetectorType_DatadogToken,
-				Raw:          []byte(resApiMatch),
-				RawV2:        []byte(resApiMatch),
-				ExtraData: map[string]string{
-					"Type": "APIKeyOnly",
-				},
-			}
-
-			if verify {
-				for _, baseURL := range s.Endpoints() {
-					req, err := http.NewRequestWithContext(ctx, "GET", baseURL+"/api/v1/validate", nil)
-					if err != nil {
-						continue
-					}
-					req.Header.Add("Content-Type", "application/json")
-					req.Header.Add("DD-API-KEY", resApiMatch)
-					res, err := client.Do(req)
-					if err == nil {
-						defer res.Body.Close()
-						if res.StatusCode >= 200 && res.StatusCode < 300 {
-							s1.Verified = true
-							s1.AnalysisInfo = map[string]string{"apiKey": resApiMatch}
 						}
 					}
 				}

pkg/detectors/datadogtoken/datadogtoken_integration_test.go

@@ -26,6 +26,7 @@ func TestDatadogToken_FromChunk(t *testing.T) {
 	apiKey := testSecrets.MustGetField("DATADOGTOKEN_TOKEN")
 	appKey := testSecrets.MustGetField("DATADOGTOKEN_APPKEY")
 	inactiveAppKey := testSecrets.MustGetField("DATADOGTOKEN_INACTIVE")
+	endpoint := "https://api.us5.datadoghq.com"
 
 	type args struct {
 		ctx    context.Context
@@ -44,7 +45,7 @@ func TestDatadogToken_FromChunk(t *testing.T) {
 			s:    Scanner{},
 			args: args{
 				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a datadogtoken secret %s within datadog %s", appKey, apiKey)),
+				data:   []byte(fmt.Sprintf("You can find a datadogtoken secret %s within datadog %s and endpoint %s", appKey, apiKey, endpoint)),
 				verify: true,
 			},
 			want: []detectors.Result{
@@ -54,6 +55,11 @@ func TestDatadogToken_FromChunk(t *testing.T) {
 					ExtraData: map[string]string{
 						"Type": "Application+APIKey",
 					},
+					AnalysisInfo: map[string]string{
+						"api_key":  apiKey,
+						"app_key":  appKey,
+						"endpoint": endpoint,
+					},
 				},
 			},
 			wantErr: false,
@@ -77,25 +83,6 @@ func TestDatadogToken_FromChunk(t *testing.T) {
 			},
 			wantErr: false,
 		},
-		{
-			name: "api key found, verified",
-			s:    Scanner{},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a datadogtoken secret %s", apiKey)), // the secret would satisfy the regex but not pass validation
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_DatadogToken,
-					Verified:     true,
-					ExtraData: map[string]string{
-						"Type": "APIKeyOnly",
-					},
-				},
-			},
-			wantErr: false,
-		},
 		{
 			name: "not found",
 			s:    Scanner{},
@@ -115,6 +102,7 @@ func TestDatadogToken_FromChunk(t *testing.T) {
 			// use default cloud endpoint
 			s.UseCloudEndpoint(true)
 			s.SetCloudEndpoint(s.CloudEndpoint())
+			s.UseFoundEndpoints(true)
 
 			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
 			if (err != nil) != tt.wantErr {

pkg/detectors/datadogtoken/datadogtoken_test.go

@@ -10,85 +10,127 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
 )
 
-var (
+func TestDataDogToken_Pattern_WithValidAPIandAppKey(t *testing.T) {
-	validPattern = `
-		# Datadog Configuration File: config.yaml
-		database:
-			host: $DB_HOST
-			port: $DB_PORT
-			username: $DB_USERNAME
-			password: $DB_PASS  # IMPORTANT: Do not share this password publicly
-
-		api:
-			auth_type: "API-Key"
-			in: "Header"
-			dd_api_secret: "FKNwdbyfYTmGUm5DK3yHEuK-BBQf0fVG"
-			dd_app: "iHxNanzZ8vjrmbjXK7NJLrwpGw2czdSh90PKH6VL"
-			base_url: "https://api.example.com/v1/example"
-			response_code: 200
-
-		# Notes:
-		# - Remember to rotate the secret every 90 days.
-		# - The above credentials should only be used in a secure environment.
-	`
-	secret = "iHxNanzZ8vjrmbjXK7NJLrwpGw2czdSh90PKH6VLFKNwdbyfYTmGUm5DK3yHEuK-BBQf0fVG"
-)
-
-func TestDataDogToken_Pattern(t *testing.T) {
 	d := Scanner{}
 	ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
 
-	tests := []struct {
+	input := `
-		name  string
+			dd_api_secret: "FKNwdbyfYTmGUm5DK3yHEuK-BBQf0fVG"
-		input string
+			dd_app: "iHxNanzZ8vjrmbjXK7NJLrwpGw2czdSh90PKH6VL"
-		want  []string
+			base_url1: "https://api.us5.datadoghq.com"
-	}{
+			base_url2: "https://api.app.ddog-gov.com"
-		{
+	`
-			name:  "valid pattern",
+	want := []string{"iHxNanzZ8vjrmbjXK7NJLrwpGw2czdSh90PKH6VLFKNwdbyfYTmGUm5DK3yHEuK-BBQf0fVG"}
-			input: validPattern,
+	wantedResultType := "Application+APIKey"
-			want:  []string{secret},
+	matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(input))
-		},
+	if len(matchedDetectors) == 0 {
+		t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), input)
+		return
+	}
+	results, err := d.FromData(context.Background(), false, []byte(input))
+	if err != nil {
+		t.Errorf("error = %v", err)
+		return
+	}
+	if len(results) != len(want) {
+		if len(results) == 0 {
+			t.Errorf("did not receive result")
+		} else {
+			t.Errorf("expected %d results, only received %d", len(want), len(results))
+		}
+		return
 	}
 
-	for _, test := range tests {
+	actual := make(map[string]struct{}, len(results))
-		t.Run(test.name, func(t *testing.T) {
+	for _, r := range results {
-			matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+		if len(r.RawV2) > 0 {
-			if len(matchedDetectors) == 0 {
+			actual[string(r.RawV2)] = struct{}{}
-				t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
+		} else {
-				return
+			actual[string(r.Raw)] = struct{}{}
-			}
+		}
+		if r.ExtraData["Type"] != wantedResultType {
+			t.Errorf("expected result type %s, got %s", wantedResultType, r.ExtraData["Type"])
+		}
+	}
+	expected := make(map[string]struct{}, len(want))
+	for _, v := range want {
+		expected[v] = struct{}{}
+	}
 
-			results, err := d.FromData(context.Background(), false, []byte(test.input))
+	if diff := cmp.Diff(expected, actual); diff != "" {
-			if err != nil {
+		t.Errorf("%s diff: (-want +got)\n%s", "TestDataDogToken_Pattern_WithValidAPIandAppKey", diff)
-				t.Errorf("error = %v", err)
+	}
-				return
+}
-			}
 
-			if len(results) != len(test.want) {
+func TestDataDogToken_Pattern_WithAPIKeyOnly(t *testing.T) {
-				if len(results) == 0 {
+	d := Scanner{}
-					t.Errorf("did not receive result")
+	ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
-				} else {
-					t.Errorf("expected %d results, only received %d", len(test.want), len(results))
-				}
-				return
-			}
 
-			actual := make(map[string]struct{}, len(results))
+	input := `
-			for _, r := range results {
+			dd_api_secret: "FKNwdbyfYTmGUm5DK3yHEuK-BBQf0fVG"
-				if len(r.RawV2) > 0 {
+			base_url: "https://api.us5.datadoghq.com"
-					actual[string(r.RawV2)] = struct{}{}
+			response_code: 200
-				} else {
+	`
-					actual[string(r.Raw)] = struct{}{}
+	matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(input))
-				}
+	if len(matchedDetectors) == 0 {
-			}
+		t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), input)
-			expected := make(map[string]struct{}, len(test.want))
+		return
-			for _, v := range test.want {
+	}
-				expected[v] = struct{}{}
+	results, err := d.FromData(context.Background(), false, []byte(input))
-			}
+	if err != nil {
+		t.Errorf("error = %v", err)
+		return
+	}
 
-			if diff := cmp.Diff(expected, actual); diff != "" {
+	if len(results) != 0 {
-				t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+		t.Errorf("expected 0 results, received %d", len(results))
-			}
+	}
-		})
+}
+
+func TestDataDogToken_NoSecrets(t *testing.T) {
+	d := Scanner{}
+	ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
+
+	input := `
+			base_url1: "https://api.us5.datadoghq.com"
+			base_url2: "https://api.app.ddog-gov.com"
+	`
+	matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(input))
+	if len(matchedDetectors) == 0 {
+		t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), input)
+		return
+	}
+	results, err := d.FromData(context.Background(), false, []byte(input))
+	if err != nil {
+		t.Errorf("error = %v", err)
+		return
+	}
+	if len(results) != 0 {
+		t.Errorf("expected 0 results, received %d", len(results))
+	}
+}
+
+func TestDataDogToken_InvalidSecrets(t *testing.T) {
+	d := Scanner{}
+	ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
+
+	input := `
+			dd_api_secret: "@FKNwdbyfYTmGUm5DK3yHEuK"
+			dd_app: "iHxNanzZ8vjrmbjXK7NJLrwpGw2czdSh90PKH6VL"
+			base_url1: "https://api.us5.datadoghq.com"
+			base_url2: "https://api.app.ddog-gov.com"
+	`
+	matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(input))
+	if len(matchedDetectors) == 0 {
+		t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), input)
+		return
+	}
+	results, err := d.FromData(context.Background(), false, []byte(input))
+	if err != nil {
+		t.Errorf("error = %v", err)
+		return
+	}
+	if len(results) != 0 {
+		t.Errorf("expected 0 results, received %d", len(results))
 	}
 }