[INS-468] Add improved lob detector to defaults.go (#4971)

* Add feature flags for new detectors

* Slight rename, ensure the flags are false by default

* Missed a name change

* Turn on new detectors for OSS user

* Instead of only adding flagged detectors when their flag is enabled, remove them when their flag is disabled

* Thank you cursorbot

* Excempt the flagged detectors from a test

* add datadogapikey detector to defaults.go

* gate detector behind feature flag

* enable the flag on main

* make detector list consistent in alphabetical order

* add lob detector to defaults.go

* tighten regex, refactor detector according to current practices, update verification endpoint

* add environment to extra data

* gate behind feature flag

---------

Co-authored-by: Charlie Gunyon <[email protected]>
Co-authored-by: Charlie Gunyon <[email protected]>

main.go

@@ -556,6 +556,7 @@ func run(state overseer.State, logSync func() error) {
 	feature.DuffelTokenDetectorEnabled.Store(true)
 	feature.ShippoDetectorEnabled.Store(true)
 	feature.IPInfoDetectorEnabled.Store(true)
+	feature.LobDetectorEnabled.Store(true)
 
 	conf := &config.Config{}
 	if *configFilename != "" {

pkg/detectors/lob/lob.go

@@ -2,6 +2,7 @@ package lob
 
 import (
 	"context"
+	"fmt"
 	"net/http"
 	"strings"
 
@@ -12,22 +13,31 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb"
 )
 
-type Scanner struct{}
+type Scanner struct {
+	client *http.Client
+}
 
 // Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
-	client = common.SaneHttpClient()
+	defaultClient = common.SaneHttpClient()
 
 	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
-	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"lob"}) + `\b([a-zA-Z0-9_]{40})\b`)
+	keyPat = regexp.MustCompile(`\b((live|test)_[a-zA-Z0-9_]{35})\b`)
 )
 
+func (s Scanner) getClient() *http.Client {
+	if s.client != nil {
+		return s.client
+	}
+	return defaultClient
+}
+
 // Keywords are used for efficiently pre-filtering chunks.
 // Use identifiers in the secret preferably, or the provider name.
 func (s Scanner) Keywords() []string {
-	return []string{"lob"}
+	return []string{"live_", "test_"}
 }
 
 // FromData will find and optionally verify Lob secrets in a given set of bytes.
@@ -36,28 +46,25 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 
 	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
 
+	uniqueMatches := make(map[string]struct{})
 	for _, match := range matches {
-		resMatch := strings.TrimSpace(match[1])
+		uniqueMatches[strings.TrimSpace(match[1])] = struct{}{}
+	}
 
+	for resMatch := range uniqueMatches {
 		s1 := detectors.Result{
 			DetectorType: detector_typepb.DetectorType_Lob,
 			Raw:          []byte(resMatch),
 			SecretParts:  map[string]string{"key": resMatch},
+			ExtraData: map[string]string{
+				"environment": resMatch[:4], // live or test
+			},
 		}
 
 		if verify {
-			req, err := http.NewRequestWithContext(ctx, "GET", "https://api.lob.com/v1/addresses", nil)
+			verified, err := s.verify(ctx, resMatch)
-			if err != nil {
+			s1.Verified = verified
-				continue
+			s1.SetVerificationError(err)
-			}
-			req.SetBasicAuth(resMatch, "")
-			res, err := client.Do(req)
-			if err == nil {
-				defer func() { _ = res.Body.Close() }()
-				if res.StatusCode >= 200 && res.StatusCode < 300 {
-					s1.Verified = true
-				}
-			}
 		}
 
 		results = append(results, s1)
@@ -66,6 +73,30 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 	return results, nil
 }
 
+func (s Scanner) verify(ctx context.Context, key string) (bool, error) {
+	req, err := http.NewRequestWithContext(ctx, "POST", "https://api.lob.com/v1/us_verifications", nil)
+	if err != nil {
+		return false, err
+	}
+	req.SetBasicAuth(key, "")
+	client := s.getClient()
+	res, err := client.Do(req)
+	if err != nil {
+		return false, err
+	}
+	defer func() { _ = res.Body.Close() }()
+	switch res.StatusCode {
+	case http.StatusForbidden, http.StatusUnprocessableEntity:
+		// 403 indicates key is active but no billing method on file
+		// 422 indicates key is active but request body is invalid
+		return true, nil
+	case http.StatusUnauthorized:
+		return false, nil
+	default:
+		return false, fmt.Errorf("unexpected status code: %d", res.StatusCode)
+	}
+}
+
 func (s Scanner) Type() detector_typepb.DetectorType {
 	return detector_typepb.DetectorType_Lob
 }

pkg/detectors/lob/lob_integration_test.go

@@ -50,6 +50,9 @@ func TestLob_FromChunk(t *testing.T) {
 				{
 					DetectorType: detector_typepb.DetectorType_Lob,
 					Verified:     true,
+					ExtraData: map[string]string{
+						"environment": "live",
+					},
 				},
 			},
 			wantErr: false,
@@ -66,6 +69,9 @@ func TestLob_FromChunk(t *testing.T) {
 				{
 					DetectorType: detector_typepb.DetectorType_Lob,
 					Verified:     false,
+					ExtraData: map[string]string{
+						"environment": "live",
+					},
 				},
 			},
 			wantErr: false,
@@ -95,6 +101,10 @@ func TestLob_FromChunk(t *testing.T) {
 					t.Fatalf("no raw secret present: \n %+v", got[i])
 				}
 				got[i].Raw = nil
+				if len(got[i].SecretParts) == 0 {
+					t.Fatalf("no secret parts present: \n %+v", got[i])
+				}
+				got[i].SecretParts = nil
 			}
 			if diff := pretty.Compare(got, tt.want); diff != "" {
 				t.Errorf("Lob.FromData() %s diff: (-got +want)\n%s", tt.name, diff)

pkg/detectors/lob/lob_test.go

@@ -12,9 +12,9 @@ import (
 )
 
 var (
-	validPattern   = "XUTbwiuF1_qmP5BvXi4hMeDafM2VoNz5yMH__rI5"
+	validPattern     = "live_0979969b3f6cc23ed67e9b650bfaf64f710"
-	invalidPattern = "XUTbwiuF1_qmP5BvXi4hMeDafM2VoNz5yMH__rI"
+	validPatternTest = "test_0979969b3f6cc23ed67e9b650bfaf64f710"
-	keyword        = "lob"
+	invalidPattern   = "live_0979969b3f6cc23ed67e9b650bfaf64f71"
 )
 
 func TestLob_Pattern(t *testing.T) {
@@ -26,23 +26,23 @@ func TestLob_Pattern(t *testing.T) {
 		want  []string
 	}{
 		{
-			name:  "valid pattern - with keyword lob",
+			name:  "valid live pattern",
-			input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+			input: fmt.Sprintf("token = '%s'", validPattern),
 			want:  []string{validPattern},
 		},
+		{
+			name:  "valid test pattern",
+			input: fmt.Sprintf("token = '%s'", validPatternTest),
+			want:  []string{validPatternTest},
+		},
 		{
 			name:  "valid pattern - ignore duplicate",
-			input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
+			input: fmt.Sprintf("token = '%s' | '%s'", validPattern, validPattern),
 			want:  []string{validPattern},
 		},
-		{
-			name:  "valid pattern - key out of prefix range",
-			input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
-			want:  []string{},
-		},
 		{
 			name:  "invalid pattern",
-			input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+			input: fmt.Sprintf("'%s'", invalidPattern),
 			want:  []string{},
 		},
 	}

pkg/engine/defaults/defaults.go

@@ -447,6 +447,7 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/liveagent"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/livestorm"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/loadmill"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/lob"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/locationiq"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/loggly"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/loginradius"
@@ -1348,6 +1349,7 @@ func buildDetectorList() []detectors.Detector {
 		&liveagent.Scanner{},
 		&livestorm.Scanner{},
 		&loadmill.Scanner{},
+		&lob.Scanner{},
 		&locationiq.Scanner{},
 		&loggly.Scanner{},
 		&loginradius.Scanner{},
@@ -1836,6 +1838,8 @@ func buildDetectorList() []detectors.Detector {
 			return !feature.ShippoDetectorEnabled.Load()
 		case *ipinfo.Scanner:
 			return !feature.IPInfoDetectorEnabled.Load()
+		case *lob.Scanner:
+			return !feature.LobDetectorEnabled.Load()
 		default:
 			return false
 		}

pkg/engine/defaults/defaults_test.go

@@ -119,7 +119,6 @@ var excludedFromDefaultList = map[detector_typepb.DetectorType]struct{}{
 	// They are not added immediately out of caution for the impact on customers/users.
 	// Remove each entry once its detector has been carefully added.
 	detector_typepb.DetectorType_Guru: {},
-	detector_typepb.DetectorType_Lob:  {},
 	detector_typepb.DetectorType_Tru:  {},
 
 	// Feature flag gated detectors
@@ -142,6 +141,7 @@ var excludedFromDefaultList = map[detector_typepb.DetectorType]struct{}{
 	detector_typepb.DetectorType_DuffelToken:               {},
 	detector_typepb.DetectorType_Shippo:                    {},
 	detector_typepb.DetectorType_IPInfo:                    {},
+	detector_typepb.DetectorType_Lob:                       {},
 
 	// Reserved / special types.
 	detector_typepb.DetectorType_CustomRegex: {}, // added dynamically via engine config, not via buildDetectorList()

pkg/feature/feature.go

@@ -35,6 +35,7 @@ var (
 	DuffelTokenDetectorEnabled               atomic.Bool
 	ShippoDetectorEnabled                    atomic.Bool
 	IPInfoDetectorEnabled                    atomic.Bool
+	LobDetectorEnabled                       atomic.Bool
 )
 
 type AtomicString struct {