trufflesecurity/trufflehog
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[INS-332] Add New Relic Insights Insert key detector (#4778)

Browse Source 
* add new relic insights insert key detector

* add build tag to integration test

* check both region urls before returning error

* extract region verification logic to separate method

* detector type fix after merge

* add secret parts

* ran make protos

* gate behind feature flag
This commit is contained in:
Mustansir
2026-07-07 11:02:28 +05:00
committed by GitHub
parent 9174f5d1cb
commit 67339b2cb5
9 changed files with 414 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files
main.go
+1
View File
@@ -552,6 +552,7 @@ func run(state overseer.State, logSync func() error) {
feature.RedHatPyxisDetectorEnabled.Store(true)
feature.OctopusDeployDetectorEnabled.Store(true)
feature.OpenRouterDetectorEnabled.Store(true)
feature.NewRelicInsightsInsertKeyDetectorEnabled.Store(true)
conf := &config.Config{}
if *configFilename != "" {
pkg/detectors/newrelicinsightsinsertkey/newrelicinsightsinsertkey.go
+127
View File
@@ -0,0 +1,127 @@
package newrelicinsightsinsertkey
import (
"context"
"errors"
"fmt"
"io"
"net/http"
"strings"
regexp "github.com/wasilibs/go-re2"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb"
)
type Scanner struct {
client *http.Client
}
// Ensure the Scanner satisfies the interfaces at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var (
defaultClient = common.SaneHttpClient()
keyPat = regexp.MustCompile(`\b(NRII-[a-zA-Z0-9-_]{25})`)
)
func (s Scanner) getClient() *http.Client {
if s.client != nil {
return s.client
}
return defaultClient
}
// Keywords are used for efficiently pre-filtering chunks.
func (s Scanner) Keywords() []string { return []string{"nrii-"} }
func (s Scanner) Type() detector_typepb.DetectorType {
return detector_typepb.DetectorType_NewRelicInsightsInsertKey
}
func (s Scanner) Description() string {
return "A New Relic Insights Insert Key is an authentication token used to send event data (such as custom events, logs, and metrics) to New Relic Insights for analysis and visualization. It ensures secure data ingestion from your applications and services."
}
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
for _, match := range matches {
resMatch := strings.TrimSpace(match[1])
s1 := detectors.Result{
DetectorType: s.Type(),
Raw: []byte(resMatch),
Redacted: resMatch[:8] + "...",
SecretParts: map[string]string{
"key": resMatch,
},
}
if verify {
isVerified, extraData, verificationErr := s.verify(ctx, resMatch)
s1.Verified = isVerified
s1.ExtraData = extraData
s1.SetVerificationError(verificationErr)
}
results = append(results, s1)
}
return results, nil
}
// verify checks if the provided key is valid by making a request to the New Relic Insights API.
// It sends a POST request to the events endpoint. A valid key will result in a 200 OK response, while an invalid key will return a 403 Forbidden.
// Even though the response is 200, no data is actually published to New Relic since the request body is empty.
// https://docs.newrelic.com/docs/data-apis/ingest-apis/event-api/introduction-event-api/
func (s Scanner) verify(ctx context.Context, key string) (bool, map[string]string, error) {
regionUrls := map[string]string{
"us": "https://insights-collector.newrelic.com/v1/accounts/`nowaythiscanexist/events",
"eu": "https://insights-collector.eu01.nr-data.net/v1/accounts/`nowaythiscanexist/events",
}
errs := make([]error, 0, len(regionUrls))
for region, regionUrl := range regionUrls {
verified, err := s.verifyRegion(ctx, key, regionUrl)
if err != nil {
errs = append(errs, fmt.Errorf("error verifying region %s: %w", region, err))
continue
}
if verified {
return true, map[string]string{"region": region}, nil
}
}
return false, nil, errors.Join(errs...)
}
func (s Scanner) verifyRegion(ctx context.Context, key, regionUrl string) (bool, error) {
req, err := http.NewRequestWithContext(
ctx, http.MethodPost, regionUrl, http.NoBody)
if err != nil {
return false, fmt.Errorf("error constructing request: %w", err)
}
req.Header.Set("X-Insert-Key", key)
client := s.getClient()
res, err := client.Do(req)
if err != nil {
return false, fmt.Errorf("error making request: %w", err)
}
defer func() {
_, _ = io.Copy(io.Discard, res.Body)
_ = res.Body.Close()
}()
switch res.StatusCode {
case http.StatusOK:
return true, nil
case http.StatusForbidden:
return false, nil
default:
return false, fmt.Errorf("unexpected status code: %d", res.StatusCode)
}
}
pkg/detectors/newrelicinsightsinsertkey/newrelicinsightsinsertkey_integration_test.go
+152
View File
@@ -0,0 +1,152 @@
//go:build detectors
// +build detectors
package newrelicinsightsinsertkey
import (
"context"
"fmt"
"testing"
"time"
"github.com/kylelemons/godebug/pretty"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb"
)
func TestNewRelicInsightsInsertKey_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors6")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
key := testSecrets.MustGetField("NEW_RELIC_INSIGHTS_INSERT_KEY")
keyEU := testSecrets.MustGetField("NEW_RELIC_INSIGHTS_INSERT_KEY_EU")
keyInactive := "NRII-d-2Vf-L1w-8B9Y_--6x8-_QjA"
type args struct {
ctx context.Context
data []byte
verify bool
}
tests := []struct {
name string
s Scanner
args args
want []detectors.Result
wantErr bool
}{
{
name: "found, verified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a new relic insights insert key %s within", key)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detector_typepb.DetectorType_NewRelicInsightsInsertKey,
Verified: true,
ExtraData: map[string]string{
"region": "us",
},
},
},
wantErr: false,
},
{
name: "found eu, verified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a new EU relic insights insert key %s within", keyEU)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detector_typepb.DetectorType_NewRelicInsightsInsertKey,
Verified: true,
ExtraData: map[string]string{
"region": "eu",
},
},
},
wantErr: false,
},
{
name: "found, unverified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a new relic insights insert key %s within", keyInactive)), // the secret would satisfy the regex but not pass validation
verify: true,
},
want: []detectors.Result{
{
DetectorType: detector_typepb.DetectorType_NewRelicInsightsInsertKey,
Verified: false,
},
},
wantErr: false,
},
{
name: "not found",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte("You cannot find the secret within"),
verify: true,
},
want: nil,
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
s := Scanner{}
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
if (err != nil) != tt.wantErr {
t.Errorf("NewRelicInsightsInsertKey.FromData() error = %v, wantErr %v", err, tt.wantErr)
return
}
for i := range got {
if len(got[i].Raw) == 0 {
t.Fatalf("no raw secret present: \n %+v", got[i])
}
got[i].Raw = nil
if len(got[i].Redacted) == 0 {
t.Fatalf("no redacted secret present: \n %+v", got[i])
}
got[i].Redacted = ""
if len(got[i].SecretParts) == 0 {
t.Fatalf("no secret parts present: \n %+v", got[i])
}
got[i].SecretParts = nil
}
if diff := pretty.Compare(got, tt.want); diff != "" {
t.Errorf("NewRelicInsightsInsertKey.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
}
})
}
}
func BenchmarkFromData(benchmark *testing.B) {
ctx := context.Background()
s := Scanner{}
for name, data := range detectors.MustGetBenchmarkData() {
benchmark.Run(name, func(b *testing.B) {
b.ResetTimer()
for n := 0; n < b.N; n++ {
_, err := s.FromData(ctx, false, data)
if err != nil {
b.Fatal(err)
}
}
})
}
}
pkg/detectors/newrelicinsightsinsertkey/newrelicinsightsinsertkey_test.go
+80
View File
@@ -0,0 +1,80 @@
package newrelicinsightsinsertkey
import (
"context"
"fmt"
"testing"
"github.com/google/go-cmp/cmp"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
var (
validPattern = "NRII-d-2Vf-L1w-8B9Y_--6x8-_QjA"
invalidPattern = "NRII-d-2Vf-L1w-8B9Y_--6x8-_Qj"
)
func TestNewRelicInsightsInsertKey_Pattern(t *testing.T) {
d := Scanner{}
ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
name string
input string
want []string
}{
{
name: "valid pattern",
input: fmt.Sprintf("new relic insights insert key = '%s'", validPattern),
want: []string{validPattern},
},
{
name: "invalid pattern",
input: fmt.Sprintf("new relic insights insert key = '%s'", invalidPattern),
want: []string{},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
if len(matchedDetectors) == 0 {
t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
results, err := d.FromData(context.Background(), false, []byte(test.input))
if err != nil {
t.Errorf("error = %v", err)
return
}
if len(results) != len(test.want) {
if len(results) == 0 {
t.Errorf("did not receive result")
} else {
t.Errorf("expected %d results, only received %d", len(test.want), len(results))
}
return
}
actual := make(map[string]struct{}, len(results))
for _, r := range results {
if len(r.RawV2) > 0 {
actual[string(r.RawV2)] = struct{}{}
} else {
actual[string(r.Raw)] = struct{}{}
}
}
expected := make(map[string]struct{}, len(test.want))
for _, v := range test.want {
expected[v] = struct{}{}
}
if diff := cmp.Diff(expected, actual); diff != "" {
t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
}
})
}
}
pkg/engine/defaults/defaults.go
+4
View File
@@ -502,6 +502,7 @@ import (
netlifyv2 "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/netlify/v2"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/netsuite"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/neutrinoapi"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/newrelicinsightsinsertkey"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/newrelicpersonalapikey"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/newsapi"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/newscatcher"
@@ -1404,6 +1405,7 @@ func buildDetectorList() []detectors.Detector {
&netlifyv2.Scanner{},
&netsuite.Scanner{},
&neutrinoapi.Scanner{},
&newrelicinsightsinsertkey.Scanner{},
&newrelicpersonalapikey.Scanner{},
&newsapi.Scanner{},
&newscatcher.Scanner{},
@@ -1820,6 +1822,8 @@ func buildDetectorList() []detectors.Detector {
return !feature.OctopusDeployDetectorEnabled.Load()
case *openrouter.Scanner:
return !feature.OpenRouterDetectorEnabled.Load()
case *newrelicinsightsinsertkey.Scanner:
return !feature.NewRelicInsightsInsertKeyDetectorEnabled.Load()
default:
return false
}
pkg/engine/defaults/defaults_test.go
+15 -14
View File
@@ -125,20 +125,21 @@ var excludedFromDefaultList = map[detector_typepb.DetectorType]struct{}{
// Feature flag gated detectors
// These should be removed from this list when we remove the feature flag
detector_typepb.DetectorType_Cloudinary: {},
detector_typepb.DetectorType_DatadogApikey: {},
detector_typepb.DetectorType_Enigma: {},
detector_typepb.DetectorType_GitLabOauth2: {},
detector_typepb.DetectorType_Pinecone: {},
detector_typepb.DetectorType_TLy: {},
detector_typepb.DetectorType_Wit: {},
detector_typepb.DetectorType_Rev: {},
detector_typepb.DetectorType_User: {},
detector_typepb.DetectorType_BrainTrustApiKey: {},
detector_typepb.DetectorType_PgAnalyzeReadKey: {},
detector_typepb.DetectorType_RedHatPyxis: {},
detector_typepb.DetectorType_OctopusDeploy: {},
detector_typepb.DetectorType_OpenRouter: {},
detector_typepb.DetectorType_Cloudinary: {},
detector_typepb.DetectorType_DatadogApikey: {},
detector_typepb.DetectorType_Enigma: {},
detector_typepb.DetectorType_GitLabOauth2: {},
detector_typepb.DetectorType_Pinecone: {},
detector_typepb.DetectorType_TLy: {},
detector_typepb.DetectorType_Wit: {},
detector_typepb.DetectorType_Rev: {},
detector_typepb.DetectorType_User: {},
detector_typepb.DetectorType_BrainTrustApiKey: {},
detector_typepb.DetectorType_PgAnalyzeReadKey: {},
detector_typepb.DetectorType_RedHatPyxis: {},
detector_typepb.DetectorType_OctopusDeploy: {},
detector_typepb.DetectorType_OpenRouter: {},
detector_typepb.DetectorType_NewRelicInsightsInsertKey: {},
// Reserved / special types.
detector_typepb.DetectorType_CustomRegex: {}, // added dynamically via engine config, not via buildDetectorList()
pkg/feature/feature.go
+27 -26
View File
@@ -5,32 +5,33 @@ import (
)
var (
ForceSkipBinaries atomic.Bool
ForceSkipArchives atomic.Bool
GitCloneTimeoutDuration atomic.Int64
SkipAdditionalRefs atomic.Bool
EnableAPKHandler atomic.Bool
UserAgentSuffix AtomicString
UseSimplifiedGitlabEnumeration atomic.Bool
UseGitMirror atomic.Bool
GitlabProjectsPerPage atomic.Int64
UseGithubGraphQLAPI atomic.Bool // use github graphql api to fetch issues, pr's and comments
HTMLDecoderEnabled atomic.Bool
PineconeDetectorEnabled atomic.Bool
CloudinaryDetectorEnabled atomic.Bool
GitLabOAuthDetectorEnabled atomic.Bool
EnigmaDetectorEnabled atomic.Bool
DatadogApiKeyDetectorEnabled atomic.Bool
TlyDetectorEnabled atomic.Bool
WitDetectorEnabled atomic.Bool
RevDetectorEnabled atomic.Bool
UserDetectorEnabled atomic.Bool
BraintrustDetectorEnabled atomic.Bool
PgAnalyzeReadKeyDetectorEnabled atomic.Bool
RedHatPyxisDetectorEnabled atomic.Bool
OctopusDeployDetectorEnabled atomic.Bool
DropUnverifiedJWTResults atomic.Bool
OpenRouterDetectorEnabled atomic.Bool
ForceSkipBinaries atomic.Bool
ForceSkipArchives atomic.Bool
GitCloneTimeoutDuration atomic.Int64
SkipAdditionalRefs atomic.Bool
EnableAPKHandler atomic.Bool
UserAgentSuffix AtomicString
UseSimplifiedGitlabEnumeration atomic.Bool
UseGitMirror atomic.Bool
GitlabProjectsPerPage atomic.Int64
UseGithubGraphQLAPI atomic.Bool // use github graphql api to fetch issues, pr's and comments
HTMLDecoderEnabled atomic.Bool
PineconeDetectorEnabled atomic.Bool
CloudinaryDetectorEnabled atomic.Bool
GitLabOAuthDetectorEnabled atomic.Bool
EnigmaDetectorEnabled atomic.Bool
DatadogApiKeyDetectorEnabled atomic.Bool
TlyDetectorEnabled atomic.Bool
WitDetectorEnabled atomic.Bool
RevDetectorEnabled atomic.Bool
UserDetectorEnabled atomic.Bool
BraintrustDetectorEnabled atomic.Bool
PgAnalyzeReadKeyDetectorEnabled atomic.Bool
RedHatPyxisDetectorEnabled atomic.Bool
OctopusDeployDetectorEnabled atomic.Bool
DropUnverifiedJWTResults atomic.Bool
OpenRouterDetectorEnabled atomic.Bool
NewRelicInsightsInsertKeyDetectorEnabled atomic.Bool
)
type AtomicString struct {
pkg/pb/detector_typepb/detector_type.pb.go
+7 -2
View File
@@ -1111,6 +1111,7 @@ const (
DetectorType_RedHatPyxis DetectorType = 1055
DetectorType_OctopusDeploy DetectorType = 1056
DetectorType_OpenRouter DetectorType = 1057
DetectorType_NewRelicInsightsInsertKey DetectorType = 1058
)
// Enum value maps for DetectorType.
@@ -2170,6 +2171,7 @@ var (
1055: "RedHatPyxis",
1056: "OctopusDeploy",
1057: "OpenRouter",
1058: "NewRelicInsightsInsertKey",
}
DetectorType_value = map[string]int32{
"Alibaba": 0,
@@ -3226,6 +3228,7 @@ var (
"RedHatPyxis": 1055,
"OctopusDeploy": 1056,
"OpenRouter": 1057,
"NewRelicInsightsInsertKey": 1058,
}
)
@@ -3261,7 +3264,7 @@ var File_detector_type_proto protoreflect.FileDescriptor
var file_detector_type_proto_rawDesc = []byte{
0x0a, 0x13, 0x64, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x2e,
0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0d, 0x64, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x5f,
0x74, 0x79, 0x70, 0x65, 0x2a, 0xc7, 0x89, 0x01, 0x0a, 0x0c, 0x44, 0x65, 0x74, 0x65, 0x63, 0x74,
0x74, 0x79, 0x70, 0x65, 0x2a, 0xe7, 0x89, 0x01, 0x0a, 0x0c, 0x44, 0x65, 0x74, 0x65, 0x63, 0x74,
0x6f, 0x72, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x41, 0x6c, 0x69, 0x62, 0x61, 0x62,
0x61, 0x10, 0x00, 0x12, 0x08, 0x0a, 0x04, 0x41, 0x4d, 0x51, 0x50, 0x10, 0x01, 0x12, 0x07, 0x0a,
0x03, 0x41, 0x57, 0x53, 0x10, 0x02, 0x12, 0x09, 0x0a, 0x05, 0x41, 0x7a, 0x75, 0x72, 0x65, 0x10,
@@ -4361,7 +4364,9 @@ var file_detector_type_proto_rawDesc = []byte{
0x61, 0x64, 0x4b, 0x65, 0x79, 0x10, 0x9e, 0x08, 0x12, 0x10, 0x0a, 0x0b, 0x52, 0x65, 0x64, 0x48,
0x61, 0x74, 0x50, 0x79, 0x78, 0x69, 0x73, 0x10, 0x9f, 0x08, 0x12, 0x12, 0x0a, 0x0d, 0x4f, 0x63,
0x74, 0x6f, 0x70, 0x75, 0x73, 0x44, 0x65, 0x70, 0x6c, 0x6f, 0x79, 0x10, 0xa0, 0x08, 0x12, 0x0f,
0x0a, 0x0a, 0x4f, 0x70, 0x65, 0x6e, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x72, 0x10, 0xa1, 0x08, 0x42,
0x0a, 0x0a, 0x4f, 0x70, 0x65, 0x6e, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x72, 0x10, 0xa1, 0x08, 0x12,
0x1e, 0x0a, 0x19, 0x4e, 0x65, 0x77, 0x52, 0x65, 0x6c, 0x69, 0x63, 0x49, 0x6e, 0x73, 0x69, 0x67,
0x68, 0x74, 0x73, 0x49, 0x6e, 0x73, 0x65, 0x72, 0x74, 0x4b, 0x65, 0x79, 0x10, 0xa2, 0x08, 0x42,
0x41, 0x5a, 0x3f, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x74, 0x72,
0x75, 0x66, 0x66, 0x6c, 0x65, 0x73, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x2f, 0x74, 0x72,
0x75, 0x66, 0x66, 0x6c, 0x65, 0x68, 0x6f, 0x67, 0x2f, 0x76, 0x33, 0x2f, 0x70, 0x6b, 0x67, 0x2f,
proto/detector_type.proto
+1
View File
@@ -1059,4 +1059,5 @@ enum DetectorType {
RedHatPyxis = 1055;
OctopusDeploy = 1056;
OpenRouter = 1057;
NewRelicInsightsInsertKey = 1058;
}