[INS-473] Add wit detector to defaults.go, gate it behind feat flag and update verification logic (#5008)

* Add wit detector to defaults.go, gate it behind feat flag and update verification logic

go.mod

@@ -253,6 +253,7 @@ require (
 	github.com/moby/sys/userns v0.1.0 // indirect
 	github.com/moby/term v0.5.2 // indirect
 	github.com/montanaflynn/stats v0.7.1 // indirect
+	github.com/morikuni/aec v1.0.0 // indirect
 	github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 // indirect
 	github.com/muesli/cancelreader v0.2.2 // indirect
 	github.com/muesli/termenv v0.16.0 // indirect
@@ -319,5 +320,6 @@ require (
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20251222181119-0a764e51fe1b // indirect
 	google.golang.org/grpc v1.79.3 // indirect
 	gopkg.in/warnings.v0 v0.1.2 // indirect
+	gotest.tools/v3 v3.5.2 // indirect
 	pault.ag/go/topsort v0.1.1 // indirect
 )

main.go

@@ -535,6 +535,7 @@ func run(state overseer.State, logSync func() error) {
 	feature.EnigmaDetectorEnabled.Store(true)
 	feature.DatadogApiKeyDetectorEnabled.Store(true)
 	feature.TlyDetectorEnabled.Store(true)
+	feature.WitDetectorEnabled.Store(true)
 
 	conf := &config.Config{}
 	if *configFilename != "" {

pkg/detectors/wit/wit.go

@@ -3,6 +3,7 @@ package wit
 import (
 	"context"
 	"fmt"
+	"io"
 	"net/http"
 	"strings"
 
@@ -32,39 +33,75 @@ func (s Scanner) Keywords() []string {
 }
 
 // FromData will find and optionally verify Wit secrets in a given set of bytes.
-func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
+func (s Scanner) FromData(
+	ctx context.Context,
+	verify bool,
+	data []byte,
+) (results []detectors.Result, err error) {
+
 	dataStr := string(data)
 
-	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+	uniqueKeys := make(map[string]struct{})
+	for _, match := range keyPat.FindAllStringSubmatch(dataStr, -1) {
+		uniqueKeys[strings.TrimSpace(match[1])] = struct{}{}
+	}
 
-	for _, match := range matches {
+	for key := range uniqueKeys {
-		resMatch := strings.TrimSpace(match[1])
+		result := detectors.Result{
-
-		s1 := detectors.Result{
 			DetectorType: detector_typepb.DetectorType_Wit,
-			Raw:          []byte(resMatch),
+			Raw:          []byte(key),
-			SecretParts:  map[string]string{"key": resMatch},
+			SecretParts: map[string]string{
+				"key": key,
+			},
 		}
 
 		if verify {
-			req, err := http.NewRequestWithContext(ctx, "GET", "https://api.wit.ai/message?q=saascndncdcdksCHDKSCVSDCasdasdVCSDVCSDAVHKCDCVHKSADVCKDVKCDSVHCSACVHJDSCVJHSADCVJHSAJ", nil)
+			verified, verificationErr := verifyWitKey(ctx, client, key)
-			if err != nil {
+			result.SetVerificationError(verificationErr, key)
-				continue
+			result.Verified = verified
-			}
-			req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))
-			res, err := client.Do(req)
-			if err == nil {
-				defer func() { _ = res.Body.Close() }()
-				if res.StatusCode >= 200 && res.StatusCode < 300 {
-					s1.Verified = true
-				}
-			}
 		}
 
-		results = append(results, s1)
+		results = append(results, result)
 	}
 
-	return results, nil
+	return
+}
+
+func verifyWitKey(
+	ctx context.Context,
+	client *http.Client,
+	key string,
+) (bool, error) {
+
+	req, err := http.NewRequestWithContext(
+		ctx,
+		http.MethodGet,
+		"https://api.wit.ai/apps?offset=1&limit=2",
+		http.NoBody,
+	)
+	if err != nil {
+		return false, err
+	}
+
+	req.Header.Set("Authorization", "Bearer "+key)
+
+	res, err := client.Do(req)
+	if err != nil {
+		return false, err
+	}
+	defer func() {
+		_, _ = io.Copy(io.Discard, res.Body)
+		_ = res.Body.Close()
+	}()
+
+	switch res.StatusCode {
+	case http.StatusOK:
+		return true, nil
+	case http.StatusUnauthorized, http.StatusForbidden:
+		return false, nil
+	default:
+		return false, fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
+	}
 }
 
 func (s Scanner) Type() detector_typepb.DetectorType {

pkg/detectors/wit/wit_integration_test.go

@@ -9,8 +9,8 @@ import (
 	"testing"
 	"time"
 
-	"github.com/kylelemons/godebug/pretty"
+	"github.com/google/go-cmp/cmp"
-
+	"github.com/google/go-cmp/cmp/cmpopts"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb"
@@ -96,7 +96,17 @@ func TestWit_FromChunk(t *testing.T) {
 				}
 				got[i].Raw = nil
 			}
-			if diff := pretty.Compare(got, tt.want); diff != "" {
+			ignoreOpts := cmpopts.IgnoreFields(
+				detectors.Result{},
+				"ExtraData",
+				"verificationError",
+				"primarySecret",
+				"SecretParts",
+				"chunkOffset",
+				"chunkOffsetSet",
+			)
+
+			if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
 				t.Errorf("Wit.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
 			}
 		})

pkg/engine/defaults/defaults.go

@@ -843,6 +843,7 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/weightsandbiases"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/whoxy"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/wistia"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/wit"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/wiz"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/worksnaps"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/workstack"
@@ -1744,6 +1745,7 @@ func buildDetectorList() []detectors.Detector {
 		// &wepay.Scanner{},
 		&whoxy.Scanner{},
 		&wistia.Scanner{},
+		&wit.Scanner{},
 		&wiz.Scanner{},
 		&worksnaps.Scanner{},
 		&workstack.Scanner{},
@@ -1788,6 +1790,8 @@ func buildDetectorList() []detectors.Detector {
 			return !feature.DatadogApiKeyDetectorEnabled.Load()
 		case *tly.Scanner:
 			return !feature.TlyDetectorEnabled.Load()
+		case *wit.Scanner:
+			return !feature.WitDetectorEnabled.Load()
 		default:
 			return false
 		}

pkg/engine/defaults/defaults_test.go

@@ -124,7 +124,6 @@ var excludedFromDefaultList = map[detector_typepb.DetectorType]struct{}{
 	detector_typepb.DetectorType_Rev:    {},
 	detector_typepb.DetectorType_Tru:    {},
 	detector_typepb.DetectorType_User:   {},
-	detector_typepb.DetectorType_Wit:    {},
 
 	// Feature flag gated detectors
 	// These should be removed from this list when we remove the feature flag
@@ -134,6 +133,7 @@ var excludedFromDefaultList = map[detector_typepb.DetectorType]struct{}{
 	detector_typepb.DetectorType_GitLabOauth2:  {},
 	detector_typepb.DetectorType_Pinecone:      {},
 	detector_typepb.DetectorType_TLy:           {},
+	detector_typepb.DetectorType_Wit:           {},
 
 	// Reserved / special types.
 	detector_typepb.DetectorType_CustomRegex: {}, // added dynamically via engine config, not via buildDetectorList()

pkg/feature/feature.go

@@ -22,6 +22,7 @@ var (
 	EnigmaDetectorEnabled          atomic.Bool
 	DatadogApiKeyDetectorEnabled   atomic.Bool
 	TlyDetectorEnabled             atomic.Bool
+	WitDetectorEnabled             atomic.Bool
 )
 
 type AtomicString struct {