From 82df476e759c1448517fac6bfb3677685cdcd78a Mon Sep 17 00:00:00 2001 From: Mustansir Date: Fri, 31 Jul 2026 18:30:05 +0500 Subject: [PATCH] [INS-344] Add New Relic Mobile App Token Detector (#4810) * add new relic mobile app token detector * chore: regen protos, feature glag gating and add secretParts to detector result * fix: derive region from token captured * chore: regen protos --------- Co-authored-by: Muneeb Ullah Khan --- main.go | 1 + .../newrelicmobileapptoken.go | 123 +++++++++++++ ...newrelicmobileapptoken_integration_test.go | 163 ++++++++++++++++++ .../newrelicmobileapptoken_test.go | 86 +++++++++ pkg/engine/defaults/defaults.go | 4 + pkg/engine/defaults/defaults_test.go | 1 + pkg/feature/feature.go | 1 + pkg/pb/detector_typepb/detector_type.pb.go | 18 +- proto/detector_type.proto | 1 + 9 files changed, 391 insertions(+), 7 deletions(-) create mode 100644 pkg/detectors/newrelicmobileapptoken/newrelicmobileapptoken.go create mode 100644 pkg/detectors/newrelicmobileapptoken/newrelicmobileapptoken_integration_test.go create mode 100644 pkg/detectors/newrelicmobileapptoken/newrelicmobileapptoken_test.go diff --git a/main.go b/main.go index 9c098db6b..08e862068 100644 --- a/main.go +++ b/main.go @@ -568,6 +568,7 @@ func run(state overseer.State, logSync func() error) { feature.NewRelicBrowserKeyDetectorEnabled.Store(true) feature.NewRelicUserKeyDetectorEnabled.Store(true) feature.NewRelicInsightsQueryKeyDetectorEnabled.Store(true) + feature.NewRelicMobileAppTokenDetectorEnabled.Store(true) conf := &config.Config{} if *configFilename != "" { diff --git a/pkg/detectors/newrelicmobileapptoken/newrelicmobileapptoken.go b/pkg/detectors/newrelicmobileapptoken/newrelicmobileapptoken.go new file mode 100644 index 000000000..233e306ff --- /dev/null +++ b/pkg/detectors/newrelicmobileapptoken/newrelicmobileapptoken.go @@ -0,0 +1,123 @@ +package newrelicmobileapptoken + +import ( + "context" + "fmt" + "io" + "net/http" + "strings" + + regexp "github.com/wasilibs/go-re2" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" +) + +type Scanner struct { + client *http.Client +} + +// Ensure the Scanner satisfies the interfaces at compile time. +var _ detectors.Detector = (*Scanner)(nil) + +var ( + defaultClient = common.SaneHttpClient() + // US region keys start with AA, followed by a 40 characters hexadecimal string, end with "-NRMA" + // EU region keys start with eu01xx, followed by a 36 characters hexadecimal string, end with "-NRMA" + keyPat = regexp.MustCompile(`\b((AA[0-9a-f]{40}|eu01xx[0-9a-f]{36})-NRMA)\b`) +) + +func (s Scanner) getClient() *http.Client { + if s.client != nil { + return s.client + } + + return defaultClient +} + +// Keywords are used for efficiently pre-filtering chunks. +func (s Scanner) Keywords() []string { return []string{"-nrma"} } + +func (s Scanner) Type() detector_typepb.DetectorType { + return detector_typepb.DetectorType_NewRelicMobileAppToken +} + +func (s Scanner) Description() string { + return "A New Relic Mobile App Token is an authentication key used to send mobile application telemetry data (such as performance metrics, crashes, and events) from iOS and Android apps to New Relic for monitoring and analysis. It is specific to each mobile app and ensures secure data ingestion." +} + +func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) { + dataStr := string(data) + + matches := keyPat.FindAllStringSubmatch(dataStr, -1) + for _, match := range matches { + resMatch := strings.TrimSpace(match[1]) + + s1 := detectors.Result{ + DetectorType: s.Type(), + Raw: []byte(resMatch), + Redacted: resMatch[:8] + "...", + SecretParts: map[string]string{"key": resMatch}, + } + + if strings.HasPrefix(resMatch, "eu01xx") { + s1.SecretParts["region"] = "eu" + s1.ExtraData = map[string]string{"region": "eu"} + } else { + s1.SecretParts["region"] = "us" + s1.ExtraData = map[string]string{"region": "us"} + } + + if verify { + isVerified, verificationErr := s.verify(ctx, resMatch, s1.SecretParts["region"]) + s1.Verified = isVerified + s1.SetVerificationError(verificationErr) + } + + results = append(results, s1) + } + + return results, nil +} + +// verify checks if the provided key is valid by making a request to the New Relic Android Agent internal API. +// A POST request is made to the /mobile/v5/connect endpoint. If the response status code is 400, +// it indicates that the key is valid but the request is malformed (since we're not sending a proper payload), +// while a 401 status code indicates that the key is invalid. Any other status code is treated as an error. +// This API is not documented, and was discovered by digging into New Relic's Android agent SDK code: +// https://github.com/newrelic/newrelic-android-agent +func (s Scanner) verify(ctx context.Context, key, region string) (bool, error) { + host := "https://mobile-collector.newrelic.com" + + if region == "eu" { + // EU region keys have a different host + host = "https://mobile-collector.eu01.nr-data.net" + } + req, err := http.NewRequestWithContext( + ctx, http.MethodPost, host+"/mobile/v5/connect", http.NoBody) + if err != nil { + return false, fmt.Errorf("error constructing request: %w", err) + } + req.Header.Add("Content-Type", "application/json") + req.Header.Add("X-App-License-Key", key) + + client := s.getClient() + res, err := client.Do(req) + if err != nil { + return false, fmt.Errorf("error making request: %w", err) + } + defer func() { + _, _ = io.Copy(io.Discard, res.Body) + _ = res.Body.Close() + }() + + switch res.StatusCode { + case http.StatusBadRequest: + return true, nil + case http.StatusUnauthorized: + return false, nil + default: + return false, fmt.Errorf("unexpected status code: %d", res.StatusCode) + } +} diff --git a/pkg/detectors/newrelicmobileapptoken/newrelicmobileapptoken_integration_test.go b/pkg/detectors/newrelicmobileapptoken/newrelicmobileapptoken_integration_test.go new file mode 100644 index 000000000..17ad45605 --- /dev/null +++ b/pkg/detectors/newrelicmobileapptoken/newrelicmobileapptoken_integration_test.go @@ -0,0 +1,163 @@ +//go:build detectors +// +build detectors + +package newrelicmobileapptoken + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/kylelemons/godebug/pretty" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" +) + +func TestNewRelicMobileAppToken_FromChunk(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors6") + if err != nil { + t.Fatalf("could not get test secrets from GCP: %s", err) + } + + key := testSecrets.MustGetField("NEW_RELIC_MOBILE_APP_TOKEN") + keyEU := testSecrets.MustGetField("NEW_RELIC_MOBILE_APP_TOKEN_EU") + keyInactive := "AAcc7eb96551e8cd65818865695f35bb109455d623-NRMA" + + type args struct { + ctx context.Context + data []byte + verify bool + } + tests := []struct { + name string + s Scanner + args args + want []detectors.Result + wantErr bool + }{ + { + name: "found, verified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a new relic mobile app token %s within", key)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detector_typepb.DetectorType_NewRelicMobileAppToken, + Verified: true, + ExtraData: map[string]string{ + "region": "us", + }, + SecretParts: map[string]string{ + "key": key, + "region": "us", + }, + }, + }, + wantErr: false, + }, + { + name: "found eu, verified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a new EU relic mobile app token %s within", keyEU)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detector_typepb.DetectorType_NewRelicMobileAppToken, + Verified: true, + ExtraData: map[string]string{ + "region": "eu", + }, + SecretParts: map[string]string{ + "key": keyEU, + "region": "eu", + }, + }, + }, + wantErr: false, + }, + { + name: "found, unverified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a new relic mobile app token %s within", keyInactive)), // the secret would satisfy the regex but not pass validation + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detector_typepb.DetectorType_NewRelicMobileAppToken, + Verified: false, + ExtraData: map[string]string{ + "region": "us", + }, + SecretParts: map[string]string{ + "key": keyInactive, + "region": "us", + }, + }, + }, + wantErr: false, + }, + { + name: "not found", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte("You cannot find the secret within"), + verify: true, + }, + want: nil, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := Scanner{} + got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + if (err != nil) != tt.wantErr { + t.Errorf("NewRelicMobileAppToken.FromData() error = %v, wantErr %v", err, tt.wantErr) + return + } + for i := range got { + if len(got[i].Raw) == 0 { + t.Fatalf("no raw secret present: \n %+v", got[i]) + } + got[i].Raw = nil + if len(got[i].Redacted) == 0 { + t.Fatalf("no redacted secret present: \n %+v", got[i]) + } + got[i].Redacted = "" + } + if diff := pretty.Compare(got, tt.want); diff != "" { + t.Errorf("NewRelicMobileAppToken.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + } + }) + } +} + +func BenchmarkFromData(benchmark *testing.B) { + ctx := context.Background() + s := Scanner{} + for name, data := range detectors.MustGetBenchmarkData() { + benchmark.Run(name, func(b *testing.B) { + b.ResetTimer() + for n := 0; n < b.N; n++ { + _, err := s.FromData(ctx, false, data) + if err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/pkg/detectors/newrelicmobileapptoken/newrelicmobileapptoken_test.go b/pkg/detectors/newrelicmobileapptoken/newrelicmobileapptoken_test.go new file mode 100644 index 000000000..4edd199f2 --- /dev/null +++ b/pkg/detectors/newrelicmobileapptoken/newrelicmobileapptoken_test.go @@ -0,0 +1,86 @@ +package newrelicmobileapptoken + +import ( + "context" + "fmt" + "testing" + + "github.com/google/go-cmp/cmp" + + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" +) + +var ( + validPattern = "AAcc7eb96551e8cd65818865695f35bb109455d623-NRMA" + validPatternEU = "eu01xxbfd4a807e4099453ba160493119a126319cb-NRMA" + invalidPattern = "AAcc7eb96551e8cd65818865695f35bb109455d6-NRMA" +) + +func TestNewRelicMobileAppToken_Pattern(t *testing.T) { + d := Scanner{} + ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) + tests := []struct { + name string + input string + want []string + }{ + { + name: "valid pattern", + input: fmt.Sprintf("new relic mobile app token = '%s'", validPattern), + want: []string{validPattern}, + }, + { + name: "valid pattern EU", + input: fmt.Sprintf("new relic mobile app token EU = '%s'", validPatternEU), + want: []string{validPatternEU}, + }, + { + name: "invalid pattern", + input: fmt.Sprintf("new relic mobile app token = '%s'", invalidPattern), + want: []string{}, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + if len(matchedDetectors) == 0 { + t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input) + return + } + + results, err := d.FromData(context.Background(), false, []byte(test.input)) + if err != nil { + t.Errorf("error = %v", err) + return + } + + if len(results) != len(test.want) { + if len(results) == 0 { + t.Errorf("did not receive result") + } else { + t.Errorf("expected %d results, only received %d", len(test.want), len(results)) + } + return + } + + actual := make(map[string]struct{}, len(results)) + for _, r := range results { + if len(r.RawV2) > 0 { + actual[string(r.RawV2)] = struct{}{} + } else { + actual[string(r.Raw)] = struct{}{} + } + } + expected := make(map[string]struct{}, len(test.want)) + for _, v := range test.want { + expected[v] = struct{}{} + } + + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) + } + }) + } +} diff --git a/pkg/engine/defaults/defaults.go b/pkg/engine/defaults/defaults.go index 84d169b00..fc9ce0e22 100644 --- a/pkg/engine/defaults/defaults.go +++ b/pkg/engine/defaults/defaults.go @@ -514,6 +514,7 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/newrelicinsightsinsertkey" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/newrelicinsightsquerykey" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/newreliclicensekey" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/newrelicmobileapptoken" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/newrelicpersonalapikey" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/newrelicuserkey" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/newsapi" @@ -1431,6 +1432,7 @@ func buildDetectorList() []detectors.Detector { &newrelicinsightsinsertkey.Scanner{}, &newrelicinsightsquerykey.Scanner{}, &newreliclicensekey.Scanner{}, + &newrelicmobileapptoken.Scanner{}, &newrelicpersonalapikey.Scanner{}, &newrelicuserkey.Scanner{}, &newsapi.Scanner{}, @@ -1880,6 +1882,8 @@ func buildDetectorList() []detectors.Detector { return !feature.NewRelicUserKeyDetectorEnabled.Load() case *newrelicinsightsquerykey.Scanner: return !feature.NewRelicInsightsQueryKeyDetectorEnabled.Load() + case *newrelicmobileapptoken.Scanner: + return !feature.NewRelicMobileAppTokenDetectorEnabled.Load() default: return false } diff --git a/pkg/engine/defaults/defaults_test.go b/pkg/engine/defaults/defaults_test.go index 2ff090b77..b7365f4e5 100644 --- a/pkg/engine/defaults/defaults_test.go +++ b/pkg/engine/defaults/defaults_test.go @@ -149,6 +149,7 @@ var excludedFromDefaultList = map[detector_typepb.DetectorType]struct{}{ detector_typepb.DetectorType_NewRelicBrowserKey: {}, detector_typepb.DetectorType_NewRelicUserKey: {}, detector_typepb.DetectorType_NewRelicInsightsQueryKey: {}, + detector_typepb.DetectorType_NewRelicMobileAppToken: {}, // Reserved / special types. detector_typepb.DetectorType_CustomRegex: {}, // added dynamically via engine config, not via buildDetectorList() diff --git a/pkg/feature/feature.go b/pkg/feature/feature.go index 43b0af1dd..d77a9d33f 100644 --- a/pkg/feature/feature.go +++ b/pkg/feature/feature.go @@ -46,6 +46,7 @@ var ( NewRelicBrowserKeyDetectorEnabled atomic.Bool NewRelicUserKeyDetectorEnabled atomic.Bool NewRelicInsightsQueryKeyDetectorEnabled atomic.Bool + NewRelicMobileAppTokenDetectorEnabled atomic.Bool ) type AtomicString struct { diff --git a/pkg/pb/detector_typepb/detector_type.pb.go b/pkg/pb/detector_typepb/detector_type.pb.go index 34dc1b5b5..203cb11f1 100644 --- a/pkg/pb/detector_typepb/detector_type.pb.go +++ b/pkg/pb/detector_typepb/detector_type.pb.go @@ -1121,6 +1121,7 @@ const ( DetectorType_NewRelicBrowserKey DetectorType = 1065 DetectorType_NewRelicUserKey DetectorType = 1066 DetectorType_NewRelicInsightsQueryKey DetectorType = 1067 + DetectorType_NewRelicMobileAppToken DetectorType = 1068 ) // Enum value maps for DetectorType. @@ -2190,6 +2191,7 @@ var ( 1065: "NewRelicBrowserKey", 1066: "NewRelicUserKey", 1067: "NewRelicInsightsQueryKey", + 1068: "NewRelicMobileAppToken", } DetectorType_value = map[string]int32{ "Alibaba": 0, @@ -3256,6 +3258,7 @@ var ( "NewRelicBrowserKey": 1065, "NewRelicUserKey": 1066, "NewRelicInsightsQueryKey": 1067, + "NewRelicMobileAppToken": 1068, } ) @@ -3291,7 +3294,7 @@ var File_detector_type_proto protoreflect.FileDescriptor var file_detector_type_proto_rawDesc = []byte{ 0x0a, 0x13, 0x64, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0d, 0x64, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x5f, - 0x74, 0x79, 0x70, 0x65, 0x2a, 0xb0, 0x8b, 0x01, 0x0a, 0x0c, 0x44, 0x65, 0x74, 0x65, 0x63, 0x74, + 0x74, 0x79, 0x70, 0x65, 0x2a, 0xcd, 0x8b, 0x01, 0x0a, 0x0c, 0x44, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x41, 0x6c, 0x69, 0x62, 0x61, 0x62, 0x61, 0x10, 0x00, 0x12, 0x08, 0x0a, 0x04, 0x41, 0x4d, 0x51, 0x50, 0x10, 0x01, 0x12, 0x07, 0x0a, 0x03, 0x41, 0x57, 0x53, 0x10, 0x02, 0x12, 0x09, 0x0a, 0x05, 0x41, 0x7a, 0x75, 0x72, 0x65, 0x10, @@ -4406,12 +4409,13 @@ var file_detector_type_proto_rawDesc = []byte{ 0x10, 0xa9, 0x08, 0x12, 0x14, 0x0a, 0x0f, 0x4e, 0x65, 0x77, 0x52, 0x65, 0x6c, 0x69, 0x63, 0x55, 0x73, 0x65, 0x72, 0x4b, 0x65, 0x79, 0x10, 0xaa, 0x08, 0x12, 0x1d, 0x0a, 0x18, 0x4e, 0x65, 0x77, 0x52, 0x65, 0x6c, 0x69, 0x63, 0x49, 0x6e, 0x73, 0x69, 0x67, 0x68, 0x74, 0x73, 0x51, 0x75, 0x65, - 0x72, 0x79, 0x4b, 0x65, 0x79, 0x10, 0xab, 0x08, 0x42, 0x41, 0x5a, 0x3f, 0x67, 0x69, 0x74, 0x68, - 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x73, 0x65, - 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x68, 0x6f, - 0x67, 0x2f, 0x76, 0x33, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x64, 0x65, 0x74, 0x65, - 0x63, 0x74, 0x6f, 0x72, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x33, + 0x72, 0x79, 0x4b, 0x65, 0x79, 0x10, 0xab, 0x08, 0x12, 0x1b, 0x0a, 0x16, 0x4e, 0x65, 0x77, 0x52, + 0x65, 0x6c, 0x69, 0x63, 0x4d, 0x6f, 0x62, 0x69, 0x6c, 0x65, 0x41, 0x70, 0x70, 0x54, 0x6f, 0x6b, + 0x65, 0x6e, 0x10, 0xac, 0x08, 0x42, 0x41, 0x5a, 0x3f, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, + 0x63, 0x6f, 0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x73, 0x65, 0x63, 0x75, 0x72, + 0x69, 0x74, 0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x68, 0x6f, 0x67, 0x2f, 0x76, + 0x33, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x64, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, + 0x72, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/proto/detector_type.proto b/proto/detector_type.proto index 92024c0f9..65cfb0f2b 100644 --- a/proto/detector_type.proto +++ b/proto/detector_type.proto @@ -1069,4 +1069,5 @@ enum DetectorType { NewRelicBrowserKey = 1065; NewRelicUserKey = 1066; NewRelicInsightsQueryKey = 1067; + NewRelicMobileAppToken = 1068; }