diff --git a/main.go b/main.go index 01b36cfd7..9c098db6b 100644 --- a/main.go +++ b/main.go @@ -567,6 +567,7 @@ func run(state overseer.State, logSync func() error) { feature.NewRelicLicenseKeyDetectorEnabled.Store(true) feature.NewRelicBrowserKeyDetectorEnabled.Store(true) feature.NewRelicUserKeyDetectorEnabled.Store(true) + feature.NewRelicInsightsQueryKeyDetectorEnabled.Store(true) conf := &config.Config{} if *configFilename != "" { diff --git a/pkg/detectors/newrelicinsightsquerykey/newrelicinsightsquerykey.go b/pkg/detectors/newrelicinsightsquerykey/newrelicinsightsquerykey.go new file mode 100644 index 000000000..913e53af6 --- /dev/null +++ b/pkg/detectors/newrelicinsightsquerykey/newrelicinsightsquerykey.go @@ -0,0 +1,144 @@ +package newrelicinsightsquerykey + +import ( + "context" + "errors" + "fmt" + "io" + "net/http" + "strings" + + regexp "github.com/wasilibs/go-re2" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" +) + +type Scanner struct { + detectors.DefaultMultiPartCredentialProvider + + client *http.Client +} + +// Ensure the Scanner satisfies the interfaces at compile time. +var _ detectors.Detector = (*Scanner)(nil) + +var ( + defaultClient = common.SaneHttpClient() + keyPat = regexp.MustCompile(`\b(NRIQ-[a-zA-Z0-9-_]{25})`) + accountIDPat = regexp.MustCompile(detectors.PrefixRegex([]string{"relic", "account", "id"}) + `\b(\d{4,10})\b`) +) + +func (s Scanner) getClient() *http.Client { + if s.client != nil { + return s.client + } + + return defaultClient +} + +// Keywords are used for efficiently pre-filtering chunks. +func (s Scanner) Keywords() []string { return []string{"nriq-"} } + +func (s Scanner) Type() detector_typepb.DetectorType { + return detector_typepb.DetectorType_NewRelicInsightsQueryKey +} + +func (s Scanner) Description() string { + return "A New Relic Insights Query Key is a read-only API key used to execute NRQL queries against your account's event data via the legacy Insights Query API. It allows secure retrieval of analytics data without permitting any data ingestion or modification." +} + +func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) { + dataStr := string(data) + + keyMatches := keyPat.FindAllStringSubmatch(dataStr, -1) + accountIDMatches := accountIDPat.FindAllStringSubmatch(dataStr, -1) + uniqueAccountIDMatches := make(map[string]struct{}) + for _, match := range accountIDMatches { + uniqueAccountIDMatches[match[1]] = struct{}{} + } + + for _, keyMatch := range keyMatches { + for accountID := range uniqueAccountIDMatches { + keyResMatch := strings.TrimSpace(keyMatch[1]) + accountIDResMatch := strings.TrimSpace(accountID) + + s1 := detectors.Result{ + DetectorType: s.Type(), + Raw: []byte(keyResMatch), + RawV2: []byte(keyResMatch + accountIDResMatch), + Redacted: keyResMatch[:8] + "...", + SecretParts: map[string]string{ + "key": keyResMatch, + "account_id": accountIDResMatch, + }, + } + + if verify { + isVerified, extraData, verificationErr := s.verify(ctx, keyResMatch, accountIDResMatch) + s1.Verified = isVerified + s1.ExtraData = extraData + if extraData != nil { + s1.SecretParts["region"] = extraData["region"] + } + s1.SetVerificationError(verificationErr) + } + + results = append(results, s1) + } + + } + + return results, nil +} + +// verify checks if the provided key is valid by making a request to the New Relic Insights Query API. +// It checks both the US and EU endpoints before returning an error. +// Account ID is required to verify as the API endpoint is account-specific. +func (s Scanner) verify(ctx context.Context, key string, accountID string) (bool, map[string]string, error) { + regionUrls := map[string]string{ + "us": fmt.Sprintf("https://insights-api.newrelic.com/v1/accounts/%s/query?nrql=SELECT%%201", accountID), + "eu": fmt.Sprintf("https://insights-api.eu.newrelic.com/v1/accounts/%s/query?nrql=SELECT%%201", accountID), + } + errs := make([]error, 0, len(regionUrls)) + for region, regionUrl := range regionUrls { + verified, err := s.verifyRegion(ctx, key, regionUrl) + if err != nil { + errs = append(errs, fmt.Errorf("error verifying region %s: %w", region, err)) + continue + } + if verified { + return true, map[string]string{"region": region}, nil + } + } + return false, nil, errors.Join(errs...) +} + +func (s Scanner) verifyRegion(ctx context.Context, key, regionUrl string) (bool, error) { + req, err := http.NewRequestWithContext( + ctx, http.MethodGet, regionUrl, http.NoBody) + if err != nil { + return false, fmt.Errorf("error constructing request: %w", err) + } + req.Header.Set("X-Query-Key", key) + + client := s.getClient() + res, err := client.Do(req) + if err != nil { + return false, fmt.Errorf("error making request: %w", err) + } + defer func() { + _, _ = io.Copy(io.Discard, res.Body) + _ = res.Body.Close() + }() + + switch res.StatusCode { + case http.StatusOK: + return true, nil + case http.StatusUnauthorized: + return false, nil + default: + return false, fmt.Errorf("unexpected status code: %d", res.StatusCode) + } +} diff --git a/pkg/detectors/newrelicinsightsquerykey/newrelicinsightsquerykey_integration_test.go b/pkg/detectors/newrelicinsightsquerykey/newrelicinsightsquerykey_integration_test.go new file mode 100644 index 000000000..83f8845d1 --- /dev/null +++ b/pkg/detectors/newrelicinsightsquerykey/newrelicinsightsquerykey_integration_test.go @@ -0,0 +1,168 @@ +//go:build detectors +// +build detectors + +package newrelicinsightsquerykey + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/kylelemons/godebug/pretty" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" +) + +func TestNewRelicInsightsQueryKey_FromChunk(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors6") + if err != nil { + t.Fatalf("could not get test secrets from GCP: %s", err) + } + + key := testSecrets.MustGetField("NEW_RELIC_INSIGHTS_QUERY_KEY") + accountID := testSecrets.MustGetField("NEW_RELIC_INSIGHTS_ACCOUNT_ID") + keyEU := testSecrets.MustGetField("NEW_RELIC_INSIGHTS_QUERY_KEY_EU") + accountIDEU := testSecrets.MustGetField("NEW_RELIC_INSIGHTS_ACCOUNT_ID_EU") + keyInactive := "NRIQ-Xc_V8HruIZ271_l9FQm-_nJ8_" + + type args struct { + ctx context.Context + data []byte + verify bool + } + tests := []struct { + name string + s Scanner + args args + want []detectors.Result + wantErr bool + }{ + { + name: "found, verified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a new relic insights query key %s and account ID %s within", key, accountID)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detector_typepb.DetectorType_NewRelicInsightsQueryKey, + Verified: true, + ExtraData: map[string]string{ + "region": "us", + }, + SecretParts: map[string]string{ + "key": key, + "account_id": accountID, + "region": "us", + }, + }, + }, + wantErr: false, + }, + { + name: "found eu, verified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a new EU relic insights query key %s and account ID %s within", keyEU, accountIDEU)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detector_typepb.DetectorType_NewRelicInsightsQueryKey, + Verified: true, + ExtraData: map[string]string{ + "region": "eu", + }, + SecretParts: map[string]string{ + "key": keyEU, + "account_id": accountIDEU, + "region": "eu", + }, + }, + }, + wantErr: false, + }, + { + name: "found, unverified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a new relic insights query key %s and account ID %s within", keyInactive, accountID)), // the secret would satisfy the regex but not pass validation + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detector_typepb.DetectorType_NewRelicInsightsQueryKey, + Verified: false, + SecretParts: map[string]string{ + "key": keyInactive, + "account_id": accountID, + }, + }, + }, + wantErr: false, + }, + { + name: "not found", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte("You cannot find the secret within"), + verify: true, + }, + want: nil, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := Scanner{} + got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + if (err != nil) != tt.wantErr { + t.Errorf("NewRelicInsightsQueryKey.FromData() error = %v, wantErr %v", err, tt.wantErr) + return + } + for i := range got { + if len(got[i].Raw) == 0 { + t.Fatalf("no raw secret present: \n %+v", got[i]) + } + got[i].Raw = nil + if len(got[i].RawV2) == 0 { + t.Fatalf("no rawV2 secret present: \n %+v", got[i]) + } + got[i].RawV2 = nil + if len(got[i].Redacted) == 0 { + t.Fatalf("no redacted secret present: \n %+v", got[i]) + } + got[i].Redacted = "" + } + if diff := pretty.Compare(got, tt.want); diff != "" { + t.Errorf("NewRelicInsightsQueryKey.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + } + }) + } +} + +func BenchmarkFromData(benchmark *testing.B) { + ctx := context.Background() + s := Scanner{} + for name, data := range detectors.MustGetBenchmarkData() { + benchmark.Run(name, func(b *testing.B) { + b.ResetTimer() + for n := 0; n < b.N; n++ { + _, err := s.FromData(ctx, false, data) + if err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/pkg/detectors/newrelicinsightsquerykey/newrelicinsightsquerykey_test.go b/pkg/detectors/newrelicinsightsquerykey/newrelicinsightsquerykey_test.go new file mode 100644 index 000000000..817c70c30 --- /dev/null +++ b/pkg/detectors/newrelicinsightsquerykey/newrelicinsightsquerykey_test.go @@ -0,0 +1,81 @@ +package newrelicinsightsquerykey + +import ( + "context" + "fmt" + "testing" + + "github.com/google/go-cmp/cmp" + + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" +) + +var ( + validPattern = "NRIQ-Xc_V8HruIZ271_l9FQm-_nJ7_" + invalidPattern = "NRIQ-Xc_V8HruIZ271_l9FQm-_nJ7" + accountID = "7746934" +) + +func TestNewRelicInsightsQueryKey_Pattern(t *testing.T) { + d := Scanner{} + ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) + tests := []struct { + name string + input string + want []string + }{ + { + name: "valid pattern", + input: fmt.Sprintf("new relic insights query key = '%s' account ID = '%s'", validPattern, accountID), + want: []string{validPattern + accountID}, + }, + { + name: "invalid pattern", + input: fmt.Sprintf("new relic insights query key = '%s' account ID = '%s'", invalidPattern, accountID), + want: []string{}, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + if len(matchedDetectors) == 0 { + t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input) + return + } + + results, err := d.FromData(context.Background(), false, []byte(test.input)) + if err != nil { + t.Errorf("error = %v", err) + return + } + + if len(results) != len(test.want) { + if len(results) == 0 { + t.Errorf("did not receive result") + } else { + t.Errorf("expected %d results, only received %d", len(test.want), len(results)) + } + return + } + + actual := make(map[string]struct{}, len(results)) + for _, r := range results { + if len(r.RawV2) > 0 { + actual[string(r.RawV2)] = struct{}{} + } else { + actual[string(r.Raw)] = struct{}{} + } + } + expected := make(map[string]struct{}, len(test.want)) + for _, v := range test.want { + expected[v] = struct{}{} + } + + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) + } + }) + } +} diff --git a/pkg/engine/defaults/defaults.go b/pkg/engine/defaults/defaults.go index 07c3c84e9..84d169b00 100644 --- a/pkg/engine/defaults/defaults.go +++ b/pkg/engine/defaults/defaults.go @@ -512,6 +512,7 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/neutrinoapi" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/newrelicbrowserkey" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/newrelicinsightsinsertkey" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/newrelicinsightsquerykey" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/newreliclicensekey" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/newrelicpersonalapikey" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/newrelicuserkey" @@ -1428,6 +1429,7 @@ func buildDetectorList() []detectors.Detector { &neutrinoapi.Scanner{}, &newrelicbrowserkey.Scanner{}, &newrelicinsightsinsertkey.Scanner{}, + &newrelicinsightsquerykey.Scanner{}, &newreliclicensekey.Scanner{}, &newrelicpersonalapikey.Scanner{}, &newrelicuserkey.Scanner{}, @@ -1876,6 +1878,8 @@ func buildDetectorList() []detectors.Detector { return !feature.NewRelicBrowserKeyDetectorEnabled.Load() case *newrelicuserkey.Scanner: return !feature.NewRelicUserKeyDetectorEnabled.Load() + case *newrelicinsightsquerykey.Scanner: + return !feature.NewRelicInsightsQueryKeyDetectorEnabled.Load() default: return false } diff --git a/pkg/engine/defaults/defaults_test.go b/pkg/engine/defaults/defaults_test.go index 2b7a6b58a..2ff090b77 100644 --- a/pkg/engine/defaults/defaults_test.go +++ b/pkg/engine/defaults/defaults_test.go @@ -148,6 +148,7 @@ var excludedFromDefaultList = map[detector_typepb.DetectorType]struct{}{ detector_typepb.DetectorType_NewRelicLicenseKey: {}, detector_typepb.DetectorType_NewRelicBrowserKey: {}, detector_typepb.DetectorType_NewRelicUserKey: {}, + detector_typepb.DetectorType_NewRelicInsightsQueryKey: {}, // Reserved / special types. detector_typepb.DetectorType_CustomRegex: {}, // added dynamically via engine config, not via buildDetectorList() diff --git a/pkg/feature/feature.go b/pkg/feature/feature.go index 52c09009f..43b0af1dd 100644 --- a/pkg/feature/feature.go +++ b/pkg/feature/feature.go @@ -45,6 +45,7 @@ var ( NewRelicLicenseKeyDetectorEnabled atomic.Bool NewRelicBrowserKeyDetectorEnabled atomic.Bool NewRelicUserKeyDetectorEnabled atomic.Bool + NewRelicInsightsQueryKeyDetectorEnabled atomic.Bool ) type AtomicString struct { diff --git a/pkg/pb/detector_typepb/detector_type.pb.go b/pkg/pb/detector_typepb/detector_type.pb.go index 45df1ecfe..34dc1b5b5 100644 --- a/pkg/pb/detector_typepb/detector_type.pb.go +++ b/pkg/pb/detector_typepb/detector_type.pb.go @@ -1120,6 +1120,7 @@ const ( DetectorType_NewRelicLicenseKey DetectorType = 1064 DetectorType_NewRelicBrowserKey DetectorType = 1065 DetectorType_NewRelicUserKey DetectorType = 1066 + DetectorType_NewRelicInsightsQueryKey DetectorType = 1067 ) // Enum value maps for DetectorType. @@ -2188,6 +2189,7 @@ var ( 1064: "NewRelicLicenseKey", 1065: "NewRelicBrowserKey", 1066: "NewRelicUserKey", + 1067: "NewRelicInsightsQueryKey", } DetectorType_value = map[string]int32{ "Alibaba": 0, @@ -3253,6 +3255,7 @@ var ( "NewRelicLicenseKey": 1064, "NewRelicBrowserKey": 1065, "NewRelicUserKey": 1066, + "NewRelicInsightsQueryKey": 1067, } ) @@ -3288,7 +3291,7 @@ var File_detector_type_proto protoreflect.FileDescriptor var file_detector_type_proto_rawDesc = []byte{ 0x0a, 0x13, 0x64, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0d, 0x64, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x5f, - 0x74, 0x79, 0x70, 0x65, 0x2a, 0x91, 0x8b, 0x01, 0x0a, 0x0c, 0x44, 0x65, 0x74, 0x65, 0x63, 0x74, + 0x74, 0x79, 0x70, 0x65, 0x2a, 0xb0, 0x8b, 0x01, 0x0a, 0x0c, 0x44, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x41, 0x6c, 0x69, 0x62, 0x61, 0x62, 0x61, 0x10, 0x00, 0x12, 0x08, 0x0a, 0x04, 0x41, 0x4d, 0x51, 0x50, 0x10, 0x01, 0x12, 0x07, 0x0a, 0x03, 0x41, 0x57, 0x53, 0x10, 0x02, 0x12, 0x09, 0x0a, 0x05, 0x41, 0x7a, 0x75, 0x72, 0x65, 0x10, @@ -4401,12 +4404,14 @@ var file_detector_type_proto_rawDesc = []byte{ 0x65, 0x6e, 0x73, 0x65, 0x4b, 0x65, 0x79, 0x10, 0xa8, 0x08, 0x12, 0x17, 0x0a, 0x12, 0x4e, 0x65, 0x77, 0x52, 0x65, 0x6c, 0x69, 0x63, 0x42, 0x72, 0x6f, 0x77, 0x73, 0x65, 0x72, 0x4b, 0x65, 0x79, 0x10, 0xa9, 0x08, 0x12, 0x14, 0x0a, 0x0f, 0x4e, 0x65, 0x77, 0x52, 0x65, 0x6c, 0x69, 0x63, 0x55, - 0x73, 0x65, 0x72, 0x4b, 0x65, 0x79, 0x10, 0xaa, 0x08, 0x42, 0x41, 0x5a, 0x3f, 0x67, 0x69, 0x74, - 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x73, - 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x68, - 0x6f, 0x67, 0x2f, 0x76, 0x33, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x64, 0x65, 0x74, - 0x65, 0x63, 0x74, 0x6f, 0x72, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x33, + 0x73, 0x65, 0x72, 0x4b, 0x65, 0x79, 0x10, 0xaa, 0x08, 0x12, 0x1d, 0x0a, 0x18, 0x4e, 0x65, 0x77, + 0x52, 0x65, 0x6c, 0x69, 0x63, 0x49, 0x6e, 0x73, 0x69, 0x67, 0x68, 0x74, 0x73, 0x51, 0x75, 0x65, + 0x72, 0x79, 0x4b, 0x65, 0x79, 0x10, 0xab, 0x08, 0x42, 0x41, 0x5a, 0x3f, 0x67, 0x69, 0x74, 0x68, + 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x73, 0x65, + 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x68, 0x6f, + 0x67, 0x2f, 0x76, 0x33, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x64, 0x65, 0x74, 0x65, + 0x63, 0x74, 0x6f, 0x72, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x33, } var ( diff --git a/proto/detector_type.proto b/proto/detector_type.proto index ecaf753ff..92024c0f9 100644 --- a/proto/detector_type.proto +++ b/proto/detector_type.proto @@ -1068,4 +1068,5 @@ enum DetectorType { NewRelicLicenseKey = 1064; NewRelicBrowserKey = 1065; NewRelicUserKey = 1066; + NewRelicInsightsQueryKey = 1067; }