From 6c979705861339b7e6ca6229545904769240d75f Mon Sep 17 00:00:00 2001 From: Mustansir Date: Tue, 7 Jul 2026 15:42:46 +0500 Subject: [PATCH] [INS-468] Add improved lob detector to defaults.go (#4971) * Add feature flags for new detectors * Slight rename, ensure the flags are false by default * Missed a name change * Turn on new detectors for OSS user * Instead of only adding flagged detectors when their flag is enabled, remove them when their flag is disabled * Thank you cursorbot * Excempt the flagged detectors from a test * add datadogapikey detector to defaults.go * gate detector behind feature flag * enable the flag on main * make detector list consistent in alphabetical order * add lob detector to defaults.go * tighten regex, refactor detector according to current practices, update verification endpoint * add environment to extra data * gate behind feature flag --------- Co-authored-by: Charlie Gunyon Co-authored-by: Charlie Gunyon --- main.go | 1 + pkg/detectors/lob/lob.go | 65 +++++++++++++++++------ pkg/detectors/lob/lob_integration_test.go | 10 ++++ pkg/detectors/lob/lob_test.go | 24 ++++----- pkg/engine/defaults/defaults.go | 4 ++ pkg/engine/defaults/defaults_test.go | 2 +- pkg/feature/feature.go | 1 + 7 files changed, 77 insertions(+), 30 deletions(-) diff --git a/main.go b/main.go index c6d63d5e4..86a6b1c65 100644 --- a/main.go +++ b/main.go @@ -556,6 +556,7 @@ func run(state overseer.State, logSync func() error) { feature.DuffelTokenDetectorEnabled.Store(true) feature.ShippoDetectorEnabled.Store(true) feature.IPInfoDetectorEnabled.Store(true) + feature.LobDetectorEnabled.Store(true) conf := &config.Config{} if *configFilename != "" { diff --git a/pkg/detectors/lob/lob.go b/pkg/detectors/lob/lob.go index a0902356f..7662ce29f 100644 --- a/pkg/detectors/lob/lob.go +++ b/pkg/detectors/lob/lob.go @@ -2,6 +2,7 @@ package lob import ( "context" + "fmt" "net/http" "strings" @@ -12,22 +13,31 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" ) -type Scanner struct{} +type Scanner struct { + client *http.Client +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) var ( - client = common.SaneHttpClient() + defaultClient = common.SaneHttpClient() // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. - keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"lob"}) + `\b([a-zA-Z0-9_]{40})\b`) + keyPat = regexp.MustCompile(`\b((live|test)_[a-zA-Z0-9_]{35})\b`) ) +func (s Scanner) getClient() *http.Client { + if s.client != nil { + return s.client + } + return defaultClient +} + // Keywords are used for efficiently pre-filtering chunks. // Use identifiers in the secret preferably, or the provider name. func (s Scanner) Keywords() []string { - return []string{"lob"} + return []string{"live_", "test_"} } // FromData will find and optionally verify Lob secrets in a given set of bytes. @@ -36,28 +46,25 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result matches := keyPat.FindAllStringSubmatch(dataStr, -1) + uniqueMatches := make(map[string]struct{}) for _, match := range matches { - resMatch := strings.TrimSpace(match[1]) + uniqueMatches[strings.TrimSpace(match[1])] = struct{}{} + } + for resMatch := range uniqueMatches { s1 := detectors.Result{ DetectorType: detector_typepb.DetectorType_Lob, Raw: []byte(resMatch), SecretParts: map[string]string{"key": resMatch}, + ExtraData: map[string]string{ + "environment": resMatch[:4], // live or test + }, } if verify { - req, err := http.NewRequestWithContext(ctx, "GET", "https://api.lob.com/v1/addresses", nil) - if err != nil { - continue - } - req.SetBasicAuth(resMatch, "") - res, err := client.Do(req) - if err == nil { - defer func() { _ = res.Body.Close() }() - if res.StatusCode >= 200 && res.StatusCode < 300 { - s1.Verified = true - } - } + verified, err := s.verify(ctx, resMatch) + s1.Verified = verified + s1.SetVerificationError(err) } results = append(results, s1) @@ -66,6 +73,30 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result return results, nil } +func (s Scanner) verify(ctx context.Context, key string) (bool, error) { + req, err := http.NewRequestWithContext(ctx, "POST", "https://api.lob.com/v1/us_verifications", nil) + if err != nil { + return false, err + } + req.SetBasicAuth(key, "") + client := s.getClient() + res, err := client.Do(req) + if err != nil { + return false, err + } + defer func() { _ = res.Body.Close() }() + switch res.StatusCode { + case http.StatusForbidden, http.StatusUnprocessableEntity: + // 403 indicates key is active but no billing method on file + // 422 indicates key is active but request body is invalid + return true, nil + case http.StatusUnauthorized: + return false, nil + default: + return false, fmt.Errorf("unexpected status code: %d", res.StatusCode) + } +} + func (s Scanner) Type() detector_typepb.DetectorType { return detector_typepb.DetectorType_Lob } diff --git a/pkg/detectors/lob/lob_integration_test.go b/pkg/detectors/lob/lob_integration_test.go index 97180296a..39204b4ae 100644 --- a/pkg/detectors/lob/lob_integration_test.go +++ b/pkg/detectors/lob/lob_integration_test.go @@ -50,6 +50,9 @@ func TestLob_FromChunk(t *testing.T) { { DetectorType: detector_typepb.DetectorType_Lob, Verified: true, + ExtraData: map[string]string{ + "environment": "live", + }, }, }, wantErr: false, @@ -66,6 +69,9 @@ func TestLob_FromChunk(t *testing.T) { { DetectorType: detector_typepb.DetectorType_Lob, Verified: false, + ExtraData: map[string]string{ + "environment": "live", + }, }, }, wantErr: false, @@ -95,6 +101,10 @@ func TestLob_FromChunk(t *testing.T) { t.Fatalf("no raw secret present: \n %+v", got[i]) } got[i].Raw = nil + if len(got[i].SecretParts) == 0 { + t.Fatalf("no secret parts present: \n %+v", got[i]) + } + got[i].SecretParts = nil } if diff := pretty.Compare(got, tt.want); diff != "" { t.Errorf("Lob.FromData() %s diff: (-got +want)\n%s", tt.name, diff) diff --git a/pkg/detectors/lob/lob_test.go b/pkg/detectors/lob/lob_test.go index 3483cb6f7..aea6564af 100644 --- a/pkg/detectors/lob/lob_test.go +++ b/pkg/detectors/lob/lob_test.go @@ -12,9 +12,9 @@ import ( ) var ( - validPattern = "XUTbwiuF1_qmP5BvXi4hMeDafM2VoNz5yMH__rI5" - invalidPattern = "XUTbwiuF1_qmP5BvXi4hMeDafM2VoNz5yMH__rI" - keyword = "lob" + validPattern = "live_0979969b3f6cc23ed67e9b650bfaf64f710" + validPatternTest = "test_0979969b3f6cc23ed67e9b650bfaf64f710" + invalidPattern = "live_0979969b3f6cc23ed67e9b650bfaf64f71" ) func TestLob_Pattern(t *testing.T) { @@ -26,23 +26,23 @@ func TestLob_Pattern(t *testing.T) { want []string }{ { - name: "valid pattern - with keyword lob", - input: fmt.Sprintf("%s token = '%s'", keyword, validPattern), + name: "valid live pattern", + input: fmt.Sprintf("token = '%s'", validPattern), want: []string{validPattern}, }, + { + name: "valid test pattern", + input: fmt.Sprintf("token = '%s'", validPatternTest), + want: []string{validPatternTest}, + }, { name: "valid pattern - ignore duplicate", - input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern), + input: fmt.Sprintf("token = '%s' | '%s'", validPattern, validPattern), want: []string{validPattern}, }, - { - name: "valid pattern - key out of prefix range", - input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern), - want: []string{}, - }, { name: "invalid pattern", - input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern), + input: fmt.Sprintf("'%s'", invalidPattern), want: []string{}, }, } diff --git a/pkg/engine/defaults/defaults.go b/pkg/engine/defaults/defaults.go index 8c2a6e88d..8014be10e 100644 --- a/pkg/engine/defaults/defaults.go +++ b/pkg/engine/defaults/defaults.go @@ -447,6 +447,7 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/liveagent" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/livestorm" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/loadmill" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/lob" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/locationiq" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/loggly" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/loginradius" @@ -1348,6 +1349,7 @@ func buildDetectorList() []detectors.Detector { &liveagent.Scanner{}, &livestorm.Scanner{}, &loadmill.Scanner{}, + &lob.Scanner{}, &locationiq.Scanner{}, &loggly.Scanner{}, &loginradius.Scanner{}, @@ -1836,6 +1838,8 @@ func buildDetectorList() []detectors.Detector { return !feature.ShippoDetectorEnabled.Load() case *ipinfo.Scanner: return !feature.IPInfoDetectorEnabled.Load() + case *lob.Scanner: + return !feature.LobDetectorEnabled.Load() default: return false } diff --git a/pkg/engine/defaults/defaults_test.go b/pkg/engine/defaults/defaults_test.go index 52fc8e1f7..0da5d8625 100644 --- a/pkg/engine/defaults/defaults_test.go +++ b/pkg/engine/defaults/defaults_test.go @@ -119,7 +119,6 @@ var excludedFromDefaultList = map[detector_typepb.DetectorType]struct{}{ // They are not added immediately out of caution for the impact on customers/users. // Remove each entry once its detector has been carefully added. detector_typepb.DetectorType_Guru: {}, - detector_typepb.DetectorType_Lob: {}, detector_typepb.DetectorType_Tru: {}, // Feature flag gated detectors @@ -142,6 +141,7 @@ var excludedFromDefaultList = map[detector_typepb.DetectorType]struct{}{ detector_typepb.DetectorType_DuffelToken: {}, detector_typepb.DetectorType_Shippo: {}, detector_typepb.DetectorType_IPInfo: {}, + detector_typepb.DetectorType_Lob: {}, // Reserved / special types. detector_typepb.DetectorType_CustomRegex: {}, // added dynamically via engine config, not via buildDetectorList() diff --git a/pkg/feature/feature.go b/pkg/feature/feature.go index 62f155632..f688a88ad 100644 --- a/pkg/feature/feature.go +++ b/pkg/feature/feature.go @@ -35,6 +35,7 @@ var ( DuffelTokenDetectorEnabled atomic.Bool ShippoDetectorEnabled atomic.Bool IPInfoDetectorEnabled atomic.Bool + LobDetectorEnabled atomic.Bool ) type AtomicString struct {