trufflesecurity/trufflehog
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[INS-254] Datadog detector verification fix and endpoint configuration (#4616)

Browse Source 
* [INS-233] Added support to verify token agains all datadog domains

* [INS-233] Added support to verify token agains all datadog domains

* Fixed cloud endpoint test

* Added precedence to endpoint selection userdefiner -> datafound -> default

* fixed one integration test

* [INS-240] add API key verification fallback when app key verification fails

* Resolved comments

* Fixed the tests according to new changes

* Removed apikey verification logic datadogtoken file

* Reverted the engine test changed earlier

* Resolved comment(s)

* added /api to endpoint

* removed unecessary print

* removed configuredEndpoint to simplify logic

* removed matching with /api suffix

* Fixed the failing integration test

* Update keys in AnalysisInfo map to use snake_case

* fixed bot comments

* fixed ssrf vulnerablity
This commit is contained in:
Muneeb Ullah Khan
2026-03-11 15:51:35 +05:00
committed by GitHub
parent bc31aa9770
commit 16d6dcf000
3 changed files with 138 additions and 130 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/detectors/datadogtoken/datadogtoken.go
+17 -39
View File
@@ -29,8 +29,9 @@ var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
appPat = regexp.MustCompile(detectors.PrefixRegex([]string{"datadog", "dd"}) + `\b([a-zA-Z-0-9]{40})\b`)
apiPat = regexp.MustCompile(detectors.PrefixRegex([]string{"datadog", "dd"}) + `\b([a-zA-Z-0-9]{32})\b`)
appPat = regexp.MustCompile(detectors.PrefixRegex([]string{"datadog", "dd"}) + `\b([a-zA-Z-0-9]{40})\b`)
apiPat = regexp.MustCompile(detectors.PrefixRegex([]string{"datadog", "dd"}) + `\b([a-zA-Z-0-9]{32})\b`)
datadogURLPat = regexp.MustCompile(`\b(api(?:\.[a-z0-9-]+)?\.(?:datadoghq|ddog-gov)\.(com|eu))\b`)
)
type userServiceResponse struct {
@@ -95,7 +96,7 @@ func setOrganizationInfo(opt []*options, s1 *detectors.Result) {
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"datadog"}
return []string{"datadog", "ddog-gov"}
}
// FromData will find and optionally verify DatadogToken secrets in a given set of bytes.
@@ -105,12 +106,19 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
appMatches := appPat.FindAllStringSubmatch(dataStr, -1)
apiMatches := apiPat.FindAllStringSubmatch(dataStr, -1)
var uniqueFoundUrls = make(map[string]struct{})
for _, matches := range datadogURLPat.FindAllStringSubmatch(dataStr, -1) {
uniqueFoundUrls["https://"+matches[1]] = struct{}{}
}
endpoints := make([]string, 0, len(uniqueFoundUrls))
for endpoint := range uniqueFoundUrls {
endpoints = append(endpoints, endpoint)
}
for _, apiMatch := range apiMatches {
resApiMatch := strings.TrimSpace(apiMatch[1])
appIncluded := false
for _, appMatch := range appMatches {
resAppMatch := strings.TrimSpace(appMatch[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_DatadogToken,
Raw: []byte(resAppMatch),
@@ -121,7 +129,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
}
if verify {
for _, baseURL := range s.Endpoints() {
for _, baseURL := range s.Endpoints(endpoints...) {
req, err := http.NewRequestWithContext(ctx, "GET", baseURL+"/api/v2/users", nil)
if err != nil {
continue
@@ -134,7 +142,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
s1.AnalysisInfo = map[string]string{"apiKey": resApiMatch, "appKey": resAppMatch}
s1.AnalysisInfo = map[string]string{"api_key": resApiMatch, "app_key": resAppMatch, "endpoint": baseURL}
var serviceResponse userServiceResponse
if err := json.NewDecoder(res.Body).Decode(&serviceResponse); err == nil {
// setup emails
@@ -146,38 +154,8 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
setOrganizationInfo(serviceResponse.Included, &s1)
}
}
}
}
}
}
appIncluded = true
results = append(results, s1)
}
if !appIncluded {
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_DatadogToken,
Raw: []byte(resApiMatch),
RawV2: []byte(resApiMatch),
ExtraData: map[string]string{
"Type": "APIKeyOnly",
},
}
if verify {
for _, baseURL := range s.Endpoints() {
req, err := http.NewRequestWithContext(ctx, "GET", baseURL+"/api/v1/validate", nil)
if err != nil {
continue
}
req.Header.Add("Content-Type", "application/json")
req.Header.Add("DD-API-KEY", resApiMatch)
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
s1.AnalysisInfo = map[string]string{"apiKey": resApiMatch}
// break the loop once we've successfully validated the token against a baseURL
break
}
}
}
pkg/detectors/datadogtoken/datadogtoken_integration_test.go
+8 -20
View File
@@ -26,6 +26,7 @@ func TestDatadogToken_FromChunk(t *testing.T) {
apiKey := testSecrets.MustGetField("DATADOGTOKEN_TOKEN")
appKey := testSecrets.MustGetField("DATADOGTOKEN_APPKEY")
inactiveAppKey := testSecrets.MustGetField("DATADOGTOKEN_INACTIVE")
endpoint := "https://api.us5.datadoghq.com"
type args struct {
ctx context.Context
@@ -44,7 +45,7 @@ func TestDatadogToken_FromChunk(t *testing.T) {
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a datadogtoken secret %s within datadog %s", appKey, apiKey)),
data: []byte(fmt.Sprintf("You can find a datadogtoken secret %s within datadog %s and endpoint %s", appKey, apiKey, endpoint)),
verify: true,
},
want: []detectors.Result{
@@ -54,6 +55,11 @@ func TestDatadogToken_FromChunk(t *testing.T) {
ExtraData: map[string]string{
"Type": "Application+APIKey",
},
AnalysisInfo: map[string]string{
"api_key": apiKey,
"app_key": appKey,
"endpoint": endpoint,
},
},
},
wantErr: false,
@@ -77,25 +83,6 @@ func TestDatadogToken_FromChunk(t *testing.T) {
},
wantErr: false,
},
{
name: "api key found, verified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a datadogtoken secret %s", apiKey)), // the secret would satisfy the regex but not pass validation
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_DatadogToken,
Verified: true,
ExtraData: map[string]string{
"Type": "APIKeyOnly",
},
},
},
wantErr: false,
},
{
name: "not found",
s: Scanner{},
@@ -115,6 +102,7 @@ func TestDatadogToken_FromChunk(t *testing.T) {
// use default cloud endpoint
s.UseCloudEndpoint(true)
s.SetCloudEndpoint(s.CloudEndpoint())
s.UseFoundEndpoints(true)
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
if (err != nil) != tt.wantErr {
pkg/detectors/datadogtoken/datadogtoken_test.go
+113 -71
View File
@@ -10,85 +10,127 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
var (
validPattern = `
# Datadog Configuration File: config.yaml
database:
host: $DB_HOST
port: $DB_PORT
username: $DB_USERNAME
password: $DB_PASS # IMPORTANT: Do not share this password publicly
api:
auth_type: "API-Key"
in: "Header"
dd_api_secret: "FKNwdbyfYTmGUm5DK3yHEuK-BBQf0fVG"
dd_app: "iHxNanzZ8vjrmbjXK7NJLrwpGw2czdSh90PKH6VL"
base_url: "https://api.example.com/v1/example"
response_code: 200
# Notes:
# - Remember to rotate the secret every 90 days.
# - The above credentials should only be used in a secure environment.
`
secret = "iHxNanzZ8vjrmbjXK7NJLrwpGw2czdSh90PKH6VLFKNwdbyfYTmGUm5DK3yHEuK-BBQf0fVG"
)
func TestDataDogToken_Pattern(t *testing.T) {
func TestDataDogToken_Pattern_WithValidAPIandAppKey(t *testing.T) {
d := Scanner{}
ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
name string
input string
want []string
}{
{
name: "valid pattern",
input: validPattern,
want: []string{secret},
},
input := `
dd_api_secret: "FKNwdbyfYTmGUm5DK3yHEuK-BBQf0fVG"
dd_app: "iHxNanzZ8vjrmbjXK7NJLrwpGw2czdSh90PKH6VL"
base_url1: "https://api.us5.datadoghq.com"
base_url2: "https://api.app.ddog-gov.com"
`
want := []string{"iHxNanzZ8vjrmbjXK7NJLrwpGw2czdSh90PKH6VLFKNwdbyfYTmGUm5DK3yHEuK-BBQf0fVG"}
wantedResultType := "Application+APIKey"
matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(input))
if len(matchedDetectors) == 0 {
t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), input)
return
}
results, err := d.FromData(context.Background(), false, []byte(input))
if err != nil {
t.Errorf("error = %v", err)
return
}
if len(results) != len(want) {
if len(results) == 0 {
t.Errorf("did not receive result")
} else {
t.Errorf("expected %d results, only received %d", len(want), len(results))
}
return
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
if len(matchedDetectors) == 0 {
t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
actual := make(map[string]struct{}, len(results))
for _, r := range results {
if len(r.RawV2) > 0 {
actual[string(r.RawV2)] = struct{}{}
} else {
actual[string(r.Raw)] = struct{}{}
}
if r.ExtraData["Type"] != wantedResultType {
t.Errorf("expected result type %s, got %s", wantedResultType, r.ExtraData["Type"])
}
}
expected := make(map[string]struct{}, len(want))
for _, v := range want {
expected[v] = struct{}{}
}
results, err := d.FromData(context.Background(), false, []byte(test.input))
if err != nil {
t.Errorf("error = %v", err)
return
}
if diff := cmp.Diff(expected, actual); diff != "" {
t.Errorf("%s diff: (-want +got)\n%s", "TestDataDogToken_Pattern_WithValidAPIandAppKey", diff)
}
}
if len(results) != len(test.want) {
if len(results) == 0 {
t.Errorf("did not receive result")
} else {
t.Errorf("expected %d results, only received %d", len(test.want), len(results))
}
return
}
func TestDataDogToken_Pattern_WithAPIKeyOnly(t *testing.T) {
d := Scanner{}
ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
actual := make(map[string]struct{}, len(results))
for _, r := range results {
if len(r.RawV2) > 0 {
actual[string(r.RawV2)] = struct{}{}
} else {
actual[string(r.Raw)] = struct{}{}
}
}
expected := make(map[string]struct{}, len(test.want))
for _, v := range test.want {
expected[v] = struct{}{}
}
input := `
dd_api_secret: "FKNwdbyfYTmGUm5DK3yHEuK-BBQf0fVG"
base_url: "https://api.us5.datadoghq.com"
response_code: 200
`
matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(input))
if len(matchedDetectors) == 0 {
t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), input)
return
}
results, err := d.FromData(context.Background(), false, []byte(input))
if err != nil {
t.Errorf("error = %v", err)
return
}
if diff := cmp.Diff(expected, actual); diff != "" {
t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
}
})
if len(results) != 0 {
t.Errorf("expected 0 results, received %d", len(results))
}
}
func TestDataDogToken_NoSecrets(t *testing.T) {
d := Scanner{}
ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
input := `
base_url1: "https://api.us5.datadoghq.com"
base_url2: "https://api.app.ddog-gov.com"
`
matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(input))
if len(matchedDetectors) == 0 {
t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), input)
return
}
results, err := d.FromData(context.Background(), false, []byte(input))
if err != nil {
t.Errorf("error = %v", err)
return
}
if len(results) != 0 {
t.Errorf("expected 0 results, received %d", len(results))
}
}
func TestDataDogToken_InvalidSecrets(t *testing.T) {
d := Scanner{}
ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
input := `
dd_api_secret: "@FKNwdbyfYTmGUm5DK3yHEuK"
dd_app: "iHxNanzZ8vjrmbjXK7NJLrwpGw2czdSh90PKH6VL"
base_url1: "https://api.us5.datadoghq.com"
base_url2: "https://api.app.ddog-gov.com"
`
matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(input))
if len(matchedDetectors) == 0 {
t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), input)
return
}
results, err := d.FromData(context.Background(), false, []byte(input))
if err != nil {
t.Errorf("error = %v", err)
return
}
if len(results) != 0 {
t.Errorf("expected 0 results, received %d", len(results))
}
}