[INS-345] Add New Relic Insights Query Key detector (#4781)

* add new relic insights query key detector

* embed DefaultMultiPartCredentialProvider

* remove unused params

* detector type fix after merge

* remove unnecessary check

* fix: add secretparts and gate detector in buildDetectorList

* regen protos

---------

Co-authored-by: Muneeb Ullah Khan <[email protected]>
This commit is contained in:
Mustansir
2026-07-30 18:47:02 +05:00
committed by GitHub
co-authored by Muneeb Ullah Khan
parent e6fc3489dd
commit f9c8139aae
9 changed files with 413 additions and 7 deletions
+1
View File
@@ -567,6 +567,7 @@ func run(state overseer.State, logSync func() error) {
feature.NewRelicLicenseKeyDetectorEnabled.Store(true)
feature.NewRelicBrowserKeyDetectorEnabled.Store(true)
feature.NewRelicUserKeyDetectorEnabled.Store(true)
feature.NewRelicInsightsQueryKeyDetectorEnabled.Store(true)
conf := &config.Config{}
if *configFilename != "" {
@@ -0,0 +1,144 @@
package newrelicinsightsquerykey
import (
"context"
"errors"
"fmt"
"io"
"net/http"
"strings"
regexp "github.com/wasilibs/go-re2"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb"
)
type Scanner struct {
detectors.DefaultMultiPartCredentialProvider
client *http.Client
}
// Ensure the Scanner satisfies the interfaces at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var (
defaultClient = common.SaneHttpClient()
keyPat = regexp.MustCompile(`\b(NRIQ-[a-zA-Z0-9-_]{25})`)
accountIDPat = regexp.MustCompile(detectors.PrefixRegex([]string{"relic", "account", "id"}) + `\b(\d{4,10})\b`)
)
func (s Scanner) getClient() *http.Client {
if s.client != nil {
return s.client
}
return defaultClient
}
// Keywords are used for efficiently pre-filtering chunks.
func (s Scanner) Keywords() []string { return []string{"nriq-"} }
func (s Scanner) Type() detector_typepb.DetectorType {
return detector_typepb.DetectorType_NewRelicInsightsQueryKey
}
func (s Scanner) Description() string {
return "A New Relic Insights Query Key is a read-only API key used to execute NRQL queries against your account's event data via the legacy Insights Query API. It allows secure retrieval of analytics data without permitting any data ingestion or modification."
}
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
keyMatches := keyPat.FindAllStringSubmatch(dataStr, -1)
accountIDMatches := accountIDPat.FindAllStringSubmatch(dataStr, -1)
uniqueAccountIDMatches := make(map[string]struct{})
for _, match := range accountIDMatches {
uniqueAccountIDMatches[match[1]] = struct{}{}
}
for _, keyMatch := range keyMatches {
for accountID := range uniqueAccountIDMatches {
keyResMatch := strings.TrimSpace(keyMatch[1])
accountIDResMatch := strings.TrimSpace(accountID)
s1 := detectors.Result{
DetectorType: s.Type(),
Raw: []byte(keyResMatch),
RawV2: []byte(keyResMatch + accountIDResMatch),
Redacted: keyResMatch[:8] + "...",
SecretParts: map[string]string{
"key": keyResMatch,
"account_id": accountIDResMatch,
},
}
if verify {
isVerified, extraData, verificationErr := s.verify(ctx, keyResMatch, accountIDResMatch)
s1.Verified = isVerified
s1.ExtraData = extraData
if extraData != nil {
s1.SecretParts["region"] = extraData["region"]
}
s1.SetVerificationError(verificationErr)
}
results = append(results, s1)
}
}
return results, nil
}
// verify checks if the provided key is valid by making a request to the New Relic Insights Query API.
// It checks both the US and EU endpoints before returning an error.
// Account ID is required to verify as the API endpoint is account-specific.
func (s Scanner) verify(ctx context.Context, key string, accountID string) (bool, map[string]string, error) {
regionUrls := map[string]string{
"us": fmt.Sprintf("https://insights-api.newrelic.com/v1/accounts/%s/query?nrql=SELECT%%201", accountID),
"eu": fmt.Sprintf("https://insights-api.eu.newrelic.com/v1/accounts/%s/query?nrql=SELECT%%201", accountID),
}
errs := make([]error, 0, len(regionUrls))
for region, regionUrl := range regionUrls {
verified, err := s.verifyRegion(ctx, key, regionUrl)
if err != nil {
errs = append(errs, fmt.Errorf("error verifying region %s: %w", region, err))
continue
}
if verified {
return true, map[string]string{"region": region}, nil
}
}
return false, nil, errors.Join(errs...)
}
func (s Scanner) verifyRegion(ctx context.Context, key, regionUrl string) (bool, error) {
req, err := http.NewRequestWithContext(
ctx, http.MethodGet, regionUrl, http.NoBody)
if err != nil {
return false, fmt.Errorf("error constructing request: %w", err)
}
req.Header.Set("X-Query-Key", key)
client := s.getClient()
res, err := client.Do(req)
if err != nil {
return false, fmt.Errorf("error making request: %w", err)
}
defer func() {
_, _ = io.Copy(io.Discard, res.Body)
_ = res.Body.Close()
}()
switch res.StatusCode {
case http.StatusOK:
return true, nil
case http.StatusUnauthorized:
return false, nil
default:
return false, fmt.Errorf("unexpected status code: %d", res.StatusCode)
}
}
@@ -0,0 +1,168 @@
//go:build detectors
// +build detectors
package newrelicinsightsquerykey
import (
"context"
"fmt"
"testing"
"time"
"github.com/kylelemons/godebug/pretty"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb"
)
func TestNewRelicInsightsQueryKey_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors6")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
key := testSecrets.MustGetField("NEW_RELIC_INSIGHTS_QUERY_KEY")
accountID := testSecrets.MustGetField("NEW_RELIC_INSIGHTS_ACCOUNT_ID")
keyEU := testSecrets.MustGetField("NEW_RELIC_INSIGHTS_QUERY_KEY_EU")
accountIDEU := testSecrets.MustGetField("NEW_RELIC_INSIGHTS_ACCOUNT_ID_EU")
keyInactive := "NRIQ-Xc_V8HruIZ271_l9FQm-_nJ8_"
type args struct {
ctx context.Context
data []byte
verify bool
}
tests := []struct {
name string
s Scanner
args args
want []detectors.Result
wantErr bool
}{
{
name: "found, verified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a new relic insights query key %s and account ID %s within", key, accountID)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detector_typepb.DetectorType_NewRelicInsightsQueryKey,
Verified: true,
ExtraData: map[string]string{
"region": "us",
},
SecretParts: map[string]string{
"key": key,
"account_id": accountID,
"region": "us",
},
},
},
wantErr: false,
},
{
name: "found eu, verified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a new EU relic insights query key %s and account ID %s within", keyEU, accountIDEU)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detector_typepb.DetectorType_NewRelicInsightsQueryKey,
Verified: true,
ExtraData: map[string]string{
"region": "eu",
},
SecretParts: map[string]string{
"key": keyEU,
"account_id": accountIDEU,
"region": "eu",
},
},
},
wantErr: false,
},
{
name: "found, unverified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a new relic insights query key %s and account ID %s within", keyInactive, accountID)), // the secret would satisfy the regex but not pass validation
verify: true,
},
want: []detectors.Result{
{
DetectorType: detector_typepb.DetectorType_NewRelicInsightsQueryKey,
Verified: false,
SecretParts: map[string]string{
"key": keyInactive,
"account_id": accountID,
},
},
},
wantErr: false,
},
{
name: "not found",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte("You cannot find the secret within"),
verify: true,
},
want: nil,
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
s := Scanner{}
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
if (err != nil) != tt.wantErr {
t.Errorf("NewRelicInsightsQueryKey.FromData() error = %v, wantErr %v", err, tt.wantErr)
return
}
for i := range got {
if len(got[i].Raw) == 0 {
t.Fatalf("no raw secret present: \n %+v", got[i])
}
got[i].Raw = nil
if len(got[i].RawV2) == 0 {
t.Fatalf("no rawV2 secret present: \n %+v", got[i])
}
got[i].RawV2 = nil
if len(got[i].Redacted) == 0 {
t.Fatalf("no redacted secret present: \n %+v", got[i])
}
got[i].Redacted = ""
}
if diff := pretty.Compare(got, tt.want); diff != "" {
t.Errorf("NewRelicInsightsQueryKey.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
}
})
}
}
func BenchmarkFromData(benchmark *testing.B) {
ctx := context.Background()
s := Scanner{}
for name, data := range detectors.MustGetBenchmarkData() {
benchmark.Run(name, func(b *testing.B) {
b.ResetTimer()
for n := 0; n < b.N; n++ {
_, err := s.FromData(ctx, false, data)
if err != nil {
b.Fatal(err)
}
}
})
}
}
@@ -0,0 +1,81 @@
package newrelicinsightsquerykey
import (
"context"
"fmt"
"testing"
"github.com/google/go-cmp/cmp"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
var (
validPattern = "NRIQ-Xc_V8HruIZ271_l9FQm-_nJ7_"
invalidPattern = "NRIQ-Xc_V8HruIZ271_l9FQm-_nJ7"
accountID = "7746934"
)
func TestNewRelicInsightsQueryKey_Pattern(t *testing.T) {
d := Scanner{}
ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
name string
input string
want []string
}{
{
name: "valid pattern",
input: fmt.Sprintf("new relic insights query key = '%s' account ID = '%s'", validPattern, accountID),
want: []string{validPattern + accountID},
},
{
name: "invalid pattern",
input: fmt.Sprintf("new relic insights query key = '%s' account ID = '%s'", invalidPattern, accountID),
want: []string{},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
if len(matchedDetectors) == 0 {
t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
results, err := d.FromData(context.Background(), false, []byte(test.input))
if err != nil {
t.Errorf("error = %v", err)
return
}
if len(results) != len(test.want) {
if len(results) == 0 {
t.Errorf("did not receive result")
} else {
t.Errorf("expected %d results, only received %d", len(test.want), len(results))
}
return
}
actual := make(map[string]struct{}, len(results))
for _, r := range results {
if len(r.RawV2) > 0 {
actual[string(r.RawV2)] = struct{}{}
} else {
actual[string(r.Raw)] = struct{}{}
}
}
expected := make(map[string]struct{}, len(test.want))
for _, v := range test.want {
expected[v] = struct{}{}
}
if diff := cmp.Diff(expected, actual); diff != "" {
t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
}
})
}
}
+4
View File
@@ -512,6 +512,7 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/neutrinoapi"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/newrelicbrowserkey"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/newrelicinsightsinsertkey"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/newrelicinsightsquerykey"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/newreliclicensekey"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/newrelicpersonalapikey"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/newrelicuserkey"
@@ -1428,6 +1429,7 @@ func buildDetectorList() []detectors.Detector {
&neutrinoapi.Scanner{},
&newrelicbrowserkey.Scanner{},
&newrelicinsightsinsertkey.Scanner{},
&newrelicinsightsquerykey.Scanner{},
&newreliclicensekey.Scanner{},
&newrelicpersonalapikey.Scanner{},
&newrelicuserkey.Scanner{},
@@ -1876,6 +1878,8 @@ func buildDetectorList() []detectors.Detector {
return !feature.NewRelicBrowserKeyDetectorEnabled.Load()
case *newrelicuserkey.Scanner:
return !feature.NewRelicUserKeyDetectorEnabled.Load()
case *newrelicinsightsquerykey.Scanner:
return !feature.NewRelicInsightsQueryKeyDetectorEnabled.Load()
default:
return false
}
+1
View File
@@ -148,6 +148,7 @@ var excludedFromDefaultList = map[detector_typepb.DetectorType]struct{}{
detector_typepb.DetectorType_NewRelicLicenseKey: {},
detector_typepb.DetectorType_NewRelicBrowserKey: {},
detector_typepb.DetectorType_NewRelicUserKey: {},
detector_typepb.DetectorType_NewRelicInsightsQueryKey: {},
// Reserved / special types.
detector_typepb.DetectorType_CustomRegex: {}, // added dynamically via engine config, not via buildDetectorList()
+1
View File
@@ -45,6 +45,7 @@ var (
NewRelicLicenseKeyDetectorEnabled atomic.Bool
NewRelicBrowserKeyDetectorEnabled atomic.Bool
NewRelicUserKeyDetectorEnabled atomic.Bool
NewRelicInsightsQueryKeyDetectorEnabled atomic.Bool
)
type AtomicString struct {
+12 -7
View File
@@ -1120,6 +1120,7 @@ const (
DetectorType_NewRelicLicenseKey DetectorType = 1064
DetectorType_NewRelicBrowserKey DetectorType = 1065
DetectorType_NewRelicUserKey DetectorType = 1066
DetectorType_NewRelicInsightsQueryKey DetectorType = 1067
)
// Enum value maps for DetectorType.
@@ -2188,6 +2189,7 @@ var (
1064: "NewRelicLicenseKey",
1065: "NewRelicBrowserKey",
1066: "NewRelicUserKey",
1067: "NewRelicInsightsQueryKey",
}
DetectorType_value = map[string]int32{
"Alibaba": 0,
@@ -3253,6 +3255,7 @@ var (
"NewRelicLicenseKey": 1064,
"NewRelicBrowserKey": 1065,
"NewRelicUserKey": 1066,
"NewRelicInsightsQueryKey": 1067,
}
)
@@ -3288,7 +3291,7 @@ var File_detector_type_proto protoreflect.FileDescriptor
var file_detector_type_proto_rawDesc = []byte{
0x0a, 0x13, 0x64, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x2e,
0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0d, 0x64, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x5f,
0x74, 0x79, 0x70, 0x65, 0x2a, 0x91, 0x8b, 0x01, 0x0a, 0x0c, 0x44, 0x65, 0x74, 0x65, 0x63, 0x74,
0x74, 0x79, 0x70, 0x65, 0x2a, 0xb0, 0x8b, 0x01, 0x0a, 0x0c, 0x44, 0x65, 0x74, 0x65, 0x63, 0x74,
0x6f, 0x72, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x41, 0x6c, 0x69, 0x62, 0x61, 0x62,
0x61, 0x10, 0x00, 0x12, 0x08, 0x0a, 0x04, 0x41, 0x4d, 0x51, 0x50, 0x10, 0x01, 0x12, 0x07, 0x0a,
0x03, 0x41, 0x57, 0x53, 0x10, 0x02, 0x12, 0x09, 0x0a, 0x05, 0x41, 0x7a, 0x75, 0x72, 0x65, 0x10,
@@ -4401,12 +4404,14 @@ var file_detector_type_proto_rawDesc = []byte{
0x65, 0x6e, 0x73, 0x65, 0x4b, 0x65, 0x79, 0x10, 0xa8, 0x08, 0x12, 0x17, 0x0a, 0x12, 0x4e, 0x65,
0x77, 0x52, 0x65, 0x6c, 0x69, 0x63, 0x42, 0x72, 0x6f, 0x77, 0x73, 0x65, 0x72, 0x4b, 0x65, 0x79,
0x10, 0xa9, 0x08, 0x12, 0x14, 0x0a, 0x0f, 0x4e, 0x65, 0x77, 0x52, 0x65, 0x6c, 0x69, 0x63, 0x55,
0x73, 0x65, 0x72, 0x4b, 0x65, 0x79, 0x10, 0xaa, 0x08, 0x42, 0x41, 0x5a, 0x3f, 0x67, 0x69, 0x74,
0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x73,
0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x68,
0x6f, 0x67, 0x2f, 0x76, 0x33, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x64, 0x65, 0x74,
0x65, 0x63, 0x74, 0x6f, 0x72, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72,
0x6f, 0x74, 0x6f, 0x33,
0x73, 0x65, 0x72, 0x4b, 0x65, 0x79, 0x10, 0xaa, 0x08, 0x12, 0x1d, 0x0a, 0x18, 0x4e, 0x65, 0x77,
0x52, 0x65, 0x6c, 0x69, 0x63, 0x49, 0x6e, 0x73, 0x69, 0x67, 0x68, 0x74, 0x73, 0x51, 0x75, 0x65,
0x72, 0x79, 0x4b, 0x65, 0x79, 0x10, 0xab, 0x08, 0x42, 0x41, 0x5a, 0x3f, 0x67, 0x69, 0x74, 0x68,
0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x73, 0x65,
0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x68, 0x6f,
0x67, 0x2f, 0x76, 0x33, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x64, 0x65, 0x74, 0x65,
0x63, 0x74, 0x6f, 0x72, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f,
0x74, 0x6f, 0x33,
}
var (
+1
View File
@@ -1068,4 +1068,5 @@ enum DetectorType {
NewRelicLicenseKey = 1064;
NewRelicBrowserKey = 1065;
NewRelicUserKey = 1066;
NewRelicInsightsQueryKey = 1067;
}