diff --git a/go.mod b/go.mod index 7697cc375..5808ea579 100644 --- a/go.mod +++ b/go.mod @@ -253,6 +253,7 @@ require ( github.com/moby/sys/userns v0.1.0 // indirect github.com/moby/term v0.5.2 // indirect github.com/montanaflynn/stats v0.7.1 // indirect + github.com/morikuni/aec v1.0.0 // indirect github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 // indirect github.com/muesli/cancelreader v0.2.2 // indirect github.com/muesli/termenv v0.16.0 // indirect @@ -319,5 +320,6 @@ require ( google.golang.org/genproto/googleapis/rpc v0.0.0-20251222181119-0a764e51fe1b // indirect google.golang.org/grpc v1.79.3 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect + gotest.tools/v3 v3.5.2 // indirect pault.ag/go/topsort v0.1.1 // indirect ) diff --git a/main.go b/main.go index 56bbf9b85..24c4955ce 100644 --- a/main.go +++ b/main.go @@ -535,6 +535,7 @@ func run(state overseer.State, logSync func() error) { feature.EnigmaDetectorEnabled.Store(true) feature.DatadogApiKeyDetectorEnabled.Store(true) feature.TlyDetectorEnabled.Store(true) + feature.WitDetectorEnabled.Store(true) conf := &config.Config{} if *configFilename != "" { diff --git a/pkg/detectors/wit/wit.go b/pkg/detectors/wit/wit.go index a6a259d3d..b40a1627d 100644 --- a/pkg/detectors/wit/wit.go +++ b/pkg/detectors/wit/wit.go @@ -3,6 +3,7 @@ package wit import ( "context" "fmt" + "io" "net/http" "strings" @@ -32,39 +33,75 @@ func (s Scanner) Keywords() []string { } // FromData will find and optionally verify Wit secrets in a given set of bytes. -func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) { +func (s Scanner) FromData( + ctx context.Context, + verify bool, + data []byte, +) (results []detectors.Result, err error) { + dataStr := string(data) - matches := keyPat.FindAllStringSubmatch(dataStr, -1) + uniqueKeys := make(map[string]struct{}) + for _, match := range keyPat.FindAllStringSubmatch(dataStr, -1) { + uniqueKeys[strings.TrimSpace(match[1])] = struct{}{} + } - for _, match := range matches { - resMatch := strings.TrimSpace(match[1]) - - s1 := detectors.Result{ + for key := range uniqueKeys { + result := detectors.Result{ DetectorType: detector_typepb.DetectorType_Wit, - Raw: []byte(resMatch), - SecretParts: map[string]string{"key": resMatch}, + Raw: []byte(key), + SecretParts: map[string]string{ + "key": key, + }, } if verify { - req, err := http.NewRequestWithContext(ctx, "GET", "https://api.wit.ai/message?q=saascndncdcdksCHDKSCVSDCasdasdVCSDVCSDAVHKCDCVHKSADVCKDVKCDSVHCSACVHJDSCVJHSADCVJHSAJ", nil) - if err != nil { - continue - } - req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) - res, err := client.Do(req) - if err == nil { - defer func() { _ = res.Body.Close() }() - if res.StatusCode >= 200 && res.StatusCode < 300 { - s1.Verified = true - } - } + verified, verificationErr := verifyWitKey(ctx, client, key) + result.SetVerificationError(verificationErr, key) + result.Verified = verified } - results = append(results, s1) + results = append(results, result) } - return results, nil + return +} + +func verifyWitKey( + ctx context.Context, + client *http.Client, + key string, +) (bool, error) { + + req, err := http.NewRequestWithContext( + ctx, + http.MethodGet, + "https://api.wit.ai/apps?offset=1&limit=2", + http.NoBody, + ) + if err != nil { + return false, err + } + + req.Header.Set("Authorization", "Bearer "+key) + + res, err := client.Do(req) + if err != nil { + return false, err + } + defer func() { + _, _ = io.Copy(io.Discard, res.Body) + _ = res.Body.Close() + }() + + switch res.StatusCode { + case http.StatusOK: + return true, nil + case http.StatusUnauthorized, http.StatusForbidden: + return false, nil + default: + return false, fmt.Errorf("unexpected HTTP response status %d", res.StatusCode) + } } func (s Scanner) Type() detector_typepb.DetectorType { diff --git a/pkg/detectors/wit/wit_integration_test.go b/pkg/detectors/wit/wit_integration_test.go index 12a40027c..bf45af401 100644 --- a/pkg/detectors/wit/wit_integration_test.go +++ b/pkg/detectors/wit/wit_integration_test.go @@ -9,8 +9,8 @@ import ( "testing" "time" - "github.com/kylelemons/godebug/pretty" - + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" @@ -96,7 +96,17 @@ func TestWit_FromChunk(t *testing.T) { } got[i].Raw = nil } - if diff := pretty.Compare(got, tt.want); diff != "" { + ignoreOpts := cmpopts.IgnoreFields( + detectors.Result{}, + "ExtraData", + "verificationError", + "primarySecret", + "SecretParts", + "chunkOffset", + "chunkOffsetSet", + ) + + if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" { t.Errorf("Wit.FromData() %s diff: (-got +want)\n%s", tt.name, diff) } }) diff --git a/pkg/engine/defaults/defaults.go b/pkg/engine/defaults/defaults.go index c3de0f556..254279131 100644 --- a/pkg/engine/defaults/defaults.go +++ b/pkg/engine/defaults/defaults.go @@ -843,6 +843,7 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/weightsandbiases" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/whoxy" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/wistia" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/wit" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/wiz" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/worksnaps" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/workstack" @@ -1744,6 +1745,7 @@ func buildDetectorList() []detectors.Detector { // &wepay.Scanner{}, &whoxy.Scanner{}, &wistia.Scanner{}, + &wit.Scanner{}, &wiz.Scanner{}, &worksnaps.Scanner{}, &workstack.Scanner{}, @@ -1788,6 +1790,8 @@ func buildDetectorList() []detectors.Detector { return !feature.DatadogApiKeyDetectorEnabled.Load() case *tly.Scanner: return !feature.TlyDetectorEnabled.Load() + case *wit.Scanner: + return !feature.WitDetectorEnabled.Load() default: return false } diff --git a/pkg/engine/defaults/defaults_test.go b/pkg/engine/defaults/defaults_test.go index f52219ebb..47f21058f 100644 --- a/pkg/engine/defaults/defaults_test.go +++ b/pkg/engine/defaults/defaults_test.go @@ -124,7 +124,6 @@ var excludedFromDefaultList = map[detector_typepb.DetectorType]struct{}{ detector_typepb.DetectorType_Rev: {}, detector_typepb.DetectorType_Tru: {}, detector_typepb.DetectorType_User: {}, - detector_typepb.DetectorType_Wit: {}, // Feature flag gated detectors // These should be removed from this list when we remove the feature flag @@ -134,6 +133,7 @@ var excludedFromDefaultList = map[detector_typepb.DetectorType]struct{}{ detector_typepb.DetectorType_GitLabOauth2: {}, detector_typepb.DetectorType_Pinecone: {}, detector_typepb.DetectorType_TLy: {}, + detector_typepb.DetectorType_Wit: {}, // Reserved / special types. detector_typepb.DetectorType_CustomRegex: {}, // added dynamically via engine config, not via buildDetectorList() diff --git a/pkg/feature/feature.go b/pkg/feature/feature.go index 80271d4e8..51defe577 100644 --- a/pkg/feature/feature.go +++ b/pkg/feature/feature.go @@ -22,6 +22,7 @@ var ( EnigmaDetectorEnabled atomic.Bool DatadogApiKeyDetectorEnabled atomic.Bool TlyDetectorEnabled atomic.Bool + WitDetectorEnabled atomic.Bool ) type AtomicString struct {