diff --git a/pkg/detectors/cliengo/cliengo.go b/pkg/detectors/cliengo/cliengo.go index e9dbebab9..625f3d482 100644 --- a/pkg/detectors/cliengo/cliengo.go +++ b/pkg/detectors/cliengo/cliengo.go @@ -18,7 +18,7 @@ type Scanner struct{} var _ detectors.Detector = (*Scanner)(nil) var ( - client = common.SaneHttpClient() + client = detectors.NewClientWithDedup(common.SaneHttpClient()) // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"cliengo"}) + `\b([0-9a-f]{8}\-[0-9a-f]{4}\-[0-9a-f]{4}\-[0-9a-f]{4}\-[0-9a-f]{12})\b`) @@ -50,7 +50,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result if err != nil { continue } - res, err := client.Do(req) + res, err := detectors.DoWithDedup(client, detector_typepb.DetectorType_Cliengo, resMatch, req) if err == nil { defer res.Body.Close() if res.StatusCode >= 200 && res.StatusCode < 300 { diff --git a/pkg/detectors/clockify/clockify.go b/pkg/detectors/clockify/clockify.go index 103fe4e6e..b0e4f0e71 100644 --- a/pkg/detectors/clockify/clockify.go +++ b/pkg/detectors/clockify/clockify.go @@ -18,7 +18,7 @@ type Scanner struct{} var _ detectors.Detector = (*Scanner)(nil) var ( - client = common.SaneHttpClient() + client = detectors.NewClientWithDedup(common.SaneHttpClient()) // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"clockify"}) + `\b([a-zA-Z0-9]{48})\b`) @@ -52,7 +52,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } req.Header.Add("content-type", "application/json") req.Header.Add("X-Api-Key", resMatch) - res, err := client.Do(req) + res, err := detectors.DoWithDedup(client, detector_typepb.DetectorType_Clockify, resMatch, req) if err == nil { defer res.Body.Close() if res.StatusCode >= 200 && res.StatusCode < 300 { diff --git a/pkg/detectors/databox/databox.go b/pkg/detectors/databox/databox.go index 441c1b706..a2c4e2e3d 100644 --- a/pkg/detectors/databox/databox.go +++ b/pkg/detectors/databox/databox.go @@ -20,7 +20,7 @@ type Scanner struct{} var _ detectors.Detector = (*Scanner)(nil) var ( - client = common.SaneHttpClient() + client = detectors.NewClientWithDedup(common.SaneHttpClient()) // Make sure that your group is surrounded in boundary characters such as below to reduce false positives keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"databox"}) + common.BuildRegex(common.RegexPattern, "", 21)) @@ -65,7 +65,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Content-Type", "application/json") req.Header.Add("Accept", "application/vnd.databox.v2+json") req.Header.Add("Authorization", fmt.Sprintf("Basic %s", sEnc)) - res, err := client.Do(req) + res, err := detectors.DoWithDedup(client, detector_typepb.DetectorType_Databox, resMatch, req) if err == nil { defer res.Body.Close() if res.StatusCode >= 200 && res.StatusCode < 300 { diff --git a/pkg/detectors/detectors.go b/pkg/detectors/detectors.go index f15c2956e..fc781af1d 100644 --- a/pkg/detectors/detectors.go +++ b/pkg/detectors/detectors.go @@ -4,7 +4,9 @@ import ( "context" "crypto/rand" "errors" + "fmt" "math/big" + "net/http" "net/url" "strings" "unicode" @@ -335,3 +337,20 @@ func ParseURLAndStripPathAndParams(u string) (*url.URL, error) { parsedURL.RawQuery = "" return parsedURL, nil } + +type dedupKeyContextKey struct{} + +func withDedupKey(ctx context.Context, detType detector_typepb.DetectorType, credential string) context.Context { + key := fmt.Sprintf("%d:%s", int32(detType), credential) + return context.WithValue(ctx, dedupKeyContextKey{}, key) +} + +// DoWithDedup executes req through client, coalescing concurrent requests that share +// the same detector type and credential into a single network call via singleflight. +// The response body is fully buffered and replayed to every waiting caller. +// +// Use this instead of client.Do for all verification requests on a client created +// with NewClientWithDedup or WithDedup — it is the only way to activate deduplication. +func DoWithDedup(client *http.Client, detType detector_typepb.DetectorType, credential string, req *http.Request) (*http.Response, error) { + return client.Do(req.WithContext(withDedupKey(req.Context(), detType, credential))) +} diff --git a/pkg/detectors/finage/finage.go b/pkg/detectors/finage/finage.go index 067a4bd42..27b5a1038 100644 --- a/pkg/detectors/finage/finage.go +++ b/pkg/detectors/finage/finage.go @@ -19,7 +19,7 @@ type Scanner struct{} var _ detectors.Detector = (*Scanner)(nil) var ( - client = common.SaneHttpClient() + client = detectors.NewClientWithDedup(common.SaneHttpClient()) // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. keyPat = regexp.MustCompile(`\b(API_KEY[0-9A-Z]{32})\b`) @@ -52,7 +52,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result continue } req.Header.Add("Content-Type", "application/json") - res, err := client.Do(req) + res, err := detectors.DoWithDedup(client, detector_typepb.DetectorType_Finage, resMatch, req) if err == nil { defer res.Body.Close() if res.StatusCode >= 200 && res.StatusCode < 300 { diff --git a/pkg/detectors/geocode/geocode.go b/pkg/detectors/geocode/geocode.go index c2160584f..265d595d5 100644 --- a/pkg/detectors/geocode/geocode.go +++ b/pkg/detectors/geocode/geocode.go @@ -20,7 +20,7 @@ type Scanner struct{} var _ detectors.Detector = (*Scanner)(nil) var ( - client = common.SaneHttpClient() + client = detectors.NewClientWithDedup(common.SaneHttpClient()) // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"geocode"}) + `\b([a-z0-9]{28})\b`) @@ -52,7 +52,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result if err != nil { continue } - res, err := client.Do(req) + res, err := detectors.DoWithDedup(client, detector_typepb.DetectorType_Geocode, resMatch, req) if err == nil { bodyBytes, err := io.ReadAll(res.Body) if err == nil { diff --git a/pkg/detectors/gitter/gitter.go b/pkg/detectors/gitter/gitter.go index cd8253186..ca047ecea 100644 --- a/pkg/detectors/gitter/gitter.go +++ b/pkg/detectors/gitter/gitter.go @@ -19,7 +19,7 @@ type Scanner struct{} var _ detectors.Detector = (*Scanner)(nil) var ( - client = common.SaneHttpClient() + client = detectors.NewClientWithDedup(common.SaneHttpClient()) // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"gitter"}) + `\b([a-z0-9-]{40})\b`) @@ -52,7 +52,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result continue } req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) - res, err := client.Do(req) + res, err := detectors.DoWithDedup(client, detector_typepb.DetectorType_Gitter, resMatch, req) if err == nil { defer res.Body.Close() if res.StatusCode >= 200 && res.StatusCode < 300 { diff --git a/pkg/detectors/holidayapi/holidayapi.go b/pkg/detectors/holidayapi/holidayapi.go index e27734e5b..aa1405f25 100644 --- a/pkg/detectors/holidayapi/holidayapi.go +++ b/pkg/detectors/holidayapi/holidayapi.go @@ -19,7 +19,7 @@ type Scanner struct{} var _ detectors.Detector = (*Scanner)(nil) var ( - client = common.SaneHttpClient() + client = detectors.NewClientWithDedup(common.SaneHttpClient()) // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"holidayapi"}) + `\b([a-z0-9-]{36})\b`) @@ -51,7 +51,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result if err != nil { continue } - res, err := client.Do(req) + res, err := detectors.DoWithDedup(client, detector_typepb.DetectorType_HolidayAPI, resMatch, req) if err == nil { defer res.Body.Close() if res.StatusCode >= 200 && res.StatusCode < 300 { diff --git a/pkg/detectors/http.go b/pkg/detectors/http.go index 28f121e32..8624d8346 100644 --- a/pkg/detectors/http.go +++ b/pkg/detectors/http.go @@ -1,14 +1,19 @@ package detectors import ( + "bytes" "context" "errors" + "fmt" + "io" "net" "net/http" "slices" "sync" "time" + "golang.org/x/sync/singleflight" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/feature" ) @@ -175,3 +180,93 @@ func NewDetectorHttpClient(opts ...ClientOption) *http.Client { client.Transport = common.NewInstrumentedTransport(client.Transport) return client } + +// bufferedResponse holds a fully-read HTTP response so it can be replayed to +// every goroutine that was coalesced by singleflight. +type bufferedResponse struct { + statusCode int + header http.Header + body []byte +} + +// singleflightTransport is an http.RoundTripper that coalesces concurrent requests +// sharing the same deduplication key into a single network call. It is a no-op for +// requests whose context does not carry a dedup key. +type singleflightTransport struct { + base http.RoundTripper + group singleflight.Group +} + +func (t *singleflightTransport) RoundTrip(req *http.Request) (*http.Response, error) { + key, ok := req.Context().Value(dedupKeyContextKey{}).(string) + if !ok || key == "" { + return t.base.RoundTrip(req) + } + + // DoChan is used instead of Do so each caller can independently respect its + // own context cancellation without blocking on the shared in-flight call. + ch := t.group.DoChan(key, func() (any, error) { + // Detach the in-flight request from the first caller's cancellation so + // that one goroutine timing out doesn't abort the shared network call + // and propagate an error to all coalesced waiters. + // + // context.WithoutCancel also strips any deadline (e.g. from + // http.Client.Timeout), so we re-attach the original deadline if + // present. Without this the shared request has no timeout and a + // hanging server would leak the goroutine and pin the singleflight + // key indefinitely. + sharedCtx := context.WithoutCancel(req.Context()) + if deadline, ok := req.Context().Deadline(); ok { + var cancel context.CancelFunc + sharedCtx, cancel = context.WithDeadline(sharedCtx, deadline) + defer cancel() + } + sharedReq := req.WithContext(sharedCtx) + resp, err := t.base.RoundTrip(sharedReq) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + return &bufferedResponse{ + statusCode: resp.StatusCode, + header: resp.Header.Clone(), + body: body, + }, nil + }) + + select { + case result := <-ch: + if result.Err != nil { + return nil, result.Err + } + br := result.Val.(*bufferedResponse) + return &http.Response{ + StatusCode: br.statusCode, + Status: fmt.Sprintf("%d %s", br.statusCode, http.StatusText(br.statusCode)), + Header: br.header.Clone(), + Body: io.NopCloser(bytes.NewReader(br.body)), + }, nil + case <-req.Context().Done(): + return nil, req.Context().Err() + } +} + +// NewClientWithDedup wraps base with a transport that deduplicates concurrent +// verification requests sharing the same key. Detectors opt in per credential by +// calling WithDedupKey on the request context before client.Do — no other changes +// to request building or response reading are needed. +func NewClientWithDedup(base *http.Client) *http.Client { + clone := *base + transport := base.Transport + if transport == nil { + transport = http.DefaultTransport + } + clone.Transport = &singleflightTransport{base: transport} + return &clone +} diff --git a/pkg/detectors/http_test.go b/pkg/detectors/http_test.go index 7210a837f..242151963 100644 --- a/pkg/detectors/http_test.go +++ b/pkg/detectors/http_test.go @@ -2,11 +2,18 @@ package detectors import ( "context" + "fmt" "net" "net/http" + "net/http/httptest" + "sync" + "sync/atomic" "testing" + "time" "github.com/stretchr/testify/assert" + + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" ) func TestWithNoLocalIP(t *testing.T) { @@ -95,6 +102,213 @@ func TestWithNoLocalIP(t *testing.T) { }) } +// TestDoWithDedup_Singleflight verifies that concurrent DoWithDedup calls sharing the +// same detector type and credential are coalesced into one network call. Each request +// the server receives returns a distinct body, so all goroutines should observe the +// body from exactly one actual server-side request. +func TestDoWithDedup_Singleflight(t *testing.T) { + var requestCount int32 + + // The 20 ms sleep keeps the first request in-flight long enough for all + // goroutines to call DoWithDedup before the result is ready. + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + n := atomic.AddInt32(&requestCount, 1) + time.Sleep(20 * time.Millisecond) + fmt.Fprintf(w, `{"request":%d}`, n) + })) + defer server.Close() + + client := NewClientWithDedup(server.Client()) + + const goroutines = 5 + bodies := make([]string, goroutines) + statuses := make([]int, goroutines) + errs := make([]error, goroutines) + + var wg sync.WaitGroup + for i := range goroutines { + wg.Add(1) + go func(i int) { + defer wg.Done() + req, err := http.NewRequestWithContext(t.Context(), http.MethodGet, server.URL, http.NoBody) + if err != nil { + errs[i] = err + return + } + resp, err := DoWithDedup(client, detector_typepb.DetectorType_Meraki, "test-credential", req) + if err != nil { + errs[i] = err + return + } + defer resp.Body.Close() + var buf [512]byte + n, _ := resp.Body.Read(buf[:]) + bodies[i] = string(buf[:n]) + statuses[i] = resp.StatusCode + }(i) + } + wg.Wait() + + for _, err := range errs { + assert.NoError(t, err) + } + for _, s := range statuses { + assert.Equal(t, http.StatusOK, s) + } + assert.Equal(t, int32(1), atomic.LoadInt32(&requestCount), + "singleflight should coalesce all concurrent calls into one HTTP request") + for i := 1; i < goroutines; i++ { + assert.Equal(t, bodies[0], bodies[i]) + } +} + +// TestDoWithDedup_WaiterContextCancelled verifies that a waiter whose context is +// cancelled bails out with an error while other waiters still receive the response, +// and only one HTTP request is made. +func TestDoWithDedup_WaiterContextCancelled(t *testing.T) { + var requestCount int32 + // inFlight is closed by the server once it starts handling the request, + // giving us a reliable signal to cancel one waiter mid-flight. + inFlight := make(chan struct{}) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + atomic.AddInt32(&requestCount, 1) + close(inFlight) + time.Sleep(50 * time.Millisecond) + w.WriteHeader(http.StatusOK) + })) + defer server.Close() + + client := NewClientWithDedup(server.Client()) + + ctx1, cancel1 := context.WithCancel(context.Background()) + defer cancel1() + + type result struct { + status int + err error + } + results := make([]result, 3) + var wg sync.WaitGroup + + for i, ctx := range []context.Context{ctx1, context.Background(), context.Background()} { + wg.Add(1) + go func(i int, ctx context.Context) { + defer wg.Done() + req, _ := http.NewRequestWithContext(ctx, http.MethodGet, server.URL, http.NoBody) + resp, err := DoWithDedup(client, detector_typepb.DetectorType_Meraki, "cred", req) + if err != nil { + results[i] = result{err: err} + return + } + defer resp.Body.Close() + results[i] = result{status: resp.StatusCode} + }(i, ctx) + } + + <-inFlight + cancel1() + wg.Wait() + + assert.ErrorIs(t, results[0].err, context.Canceled, "cancelled waiter should get context error") + assert.NoError(t, results[1].err) + assert.Equal(t, http.StatusOK, results[1].status) + assert.NoError(t, results[2].err) + assert.Equal(t, http.StatusOK, results[2].status) + assert.Equal(t, int32(1), atomic.LoadInt32(&requestCount), "only one HTTP request should be made") +} + +// TestDoWithDedup_FirstCallerContextCancelled verifies that cancelling the first +// caller's context does not abort the shared in-flight HTTP call: the second caller +// should still receive a valid response, and only one HTTP request is made. +func TestDoWithDedup_FirstCallerContextCancelled(t *testing.T) { + var requestCount int32 + inFlight := make(chan struct{}) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + atomic.AddInt32(&requestCount, 1) + close(inFlight) + time.Sleep(50 * time.Millisecond) + w.WriteHeader(http.StatusOK) + })) + defer server.Close() + + client := NewClientWithDedup(server.Client()) + + ctx1, cancel1 := context.WithCancel(context.Background()) + defer cancel1() + + var firstErr error + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + req, _ := http.NewRequestWithContext(ctx1, http.MethodGet, server.URL, http.NoBody) + resp, err := DoWithDedup(client, detector_typepb.DetectorType_Meraki, "cred", req) + if err != nil { + firstErr = err + return + } + resp.Body.Close() + }() + + // Cancel the first caller once the server is processing, then immediately + // start a second caller that should coalesce into the still-running call. + <-inFlight + cancel1() + + var secondStatus int + var secondErr error + wg.Add(1) + go func() { + defer wg.Done() + req, _ := http.NewRequestWithContext(context.Background(), http.MethodGet, server.URL, http.NoBody) + resp, err := DoWithDedup(client, detector_typepb.DetectorType_Meraki, "cred", req) + if err != nil { + secondErr = err + return + } + defer resp.Body.Close() + secondStatus = resp.StatusCode + }() + + wg.Wait() + + assert.ErrorIs(t, firstErr, context.Canceled, "first caller should get context error") + assert.NoError(t, secondErr, "second caller should succeed despite first caller's cancellation") + assert.Equal(t, http.StatusOK, secondStatus) + assert.Equal(t, int32(1), atomic.LoadInt32(&requestCount), "only one HTTP request should be made") +} + +// TestDoWithDedup_DeadlinePreserved verifies that the client timeout still applies +// to the shared in-flight call after context.WithoutCancel strips cancellation. +// A hanging server must not cause an indefinite leak. +func TestDoWithDedup_DeadlinePreserved(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + select { + case <-time.After(10 * time.Second): + case <-r.Context().Done(): + } + })) + defer server.Close() + + base := server.Client() + base.Timeout = 75 * time.Millisecond + client := NewClientWithDedup(base) + + req, _ := http.NewRequestWithContext(context.Background(), http.MethodGet, server.URL, http.NoBody) + start := time.Now() + resp, err := DoWithDedup(client, detector_typepb.DetectorType_Meraki, "cred", req) + if err == nil { + defer resp.Body.Close() + } + + elapsed := time.Since(start) + + assert.Error(t, err, "request to hanging server should fail") + assert.Less(t, elapsed, time.Second, "timeout should be enforced by client deadline, not run indefinitely") +} + func TestIsLocalIP(t *testing.T) { testCases := []struct { name string diff --git a/pkg/detectors/ipinfo/ipinfo.go b/pkg/detectors/ipinfo/ipinfo.go index bb5c50b3b..ec3165606 100644 --- a/pkg/detectors/ipinfo/ipinfo.go +++ b/pkg/detectors/ipinfo/ipinfo.go @@ -21,7 +21,7 @@ type Scanner struct { var _ detectors.Detector = (*Scanner)(nil) var ( - defaultClient = common.SaneHttpClient() + defaultClient = detectors.NewClientWithDedup(common.SaneHttpClient()) // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"ipinfo"}) + `\b([a-f0-9]{14})\b`) ) @@ -56,7 +56,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result if err != nil { continue } - res, err := client.Do(req) + res, err := detectors.DoWithDedup(client, detector_typepb.DetectorType_IPInfo, resMatch, req) if err == nil { defer res.Body.Close() if res.StatusCode >= 200 && res.StatusCode < 300 { diff --git a/pkg/detectors/loadmill/loadmill.go b/pkg/detectors/loadmill/loadmill.go index c8a8b6ff4..826465711 100644 --- a/pkg/detectors/loadmill/loadmill.go +++ b/pkg/detectors/loadmill/loadmill.go @@ -18,7 +18,7 @@ type Scanner struct{} var _ detectors.Detector = (*Scanner)(nil) var ( - client = common.SaneHttpClient() + client = detectors.NewClientWithDedup(common.SaneHttpClient()) // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"loadmill"}) + `\b([0-9a-zA-Z]{40})\b`) @@ -52,7 +52,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } req.Header.Add("Accept", "application/vnd.loadmill+json; version=3") req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) - res, err := client.Do(req) + res, err := detectors.DoWithDedup(client, detector_typepb.DetectorType_Loadmill, resMatch, req) if err == nil { defer res.Body.Close() if res.StatusCode >= 200 && res.StatusCode < 300 { diff --git a/pkg/detectors/meraki/meraki.go b/pkg/detectors/meraki/meraki.go index 74c220e5b..43ffdb305 100644 --- a/pkg/detectors/meraki/meraki.go +++ b/pkg/detectors/meraki/meraki.go @@ -29,7 +29,7 @@ type merakiOrganizations struct { var _ detectors.Detector = (*Scanner)(nil) var ( - defaultClient = common.SaneHttpClient() + defaultClient = detectors.NewClientWithDedup(common.SaneHttpClient()) apiKey = regexp.MustCompile(detectors.PrefixRegex([]string{"meraki"}) + `([0-9a-f]{40})`) ) @@ -57,9 +57,7 @@ func (s Scanner) Description() string { func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) { dataStr := string(data) - // uniqueMatches will hold unique match values and ensure we only process unique matches found in the data string - var uniqueMatches = make(map[string]struct{}) - + uniqueMatches := make(map[string]struct{}) for _, match := range apiKey.FindAllStringSubmatch(dataStr, -1) { uniqueMatches[match[1]] = struct{}{} } @@ -107,11 +105,9 @@ func verifyMerakiApiKey(ctx context.Context, client *http.Client, match string) if err != nil { return nil, false, err } - - // set the required auth header req.Header.Set("X-Cisco-Meraki-API-Key", match) - resp, err := client.Do(req) + resp, err := detectors.DoWithDedup(client, detector_typepb.DetectorType_Meraki, match, req) if err != nil { return nil, false, err } @@ -122,12 +118,10 @@ func verifyMerakiApiKey(ctx context.Context, client *http.Client, match string) switch resp.StatusCode { case http.StatusOK: - // in case token is verified, capture the organization id's and name which are accessible via token. var organizations []merakiOrganizations if err = json.NewDecoder(resp.Body).Decode(&organizations); err != nil { return nil, false, err } - return organizations, true, nil case http.StatusUnauthorized: return nil, false, nil diff --git a/pkg/detectors/mixmax/mixmax.go b/pkg/detectors/mixmax/mixmax.go index de8e26e61..0322c59c5 100644 --- a/pkg/detectors/mixmax/mixmax.go +++ b/pkg/detectors/mixmax/mixmax.go @@ -17,7 +17,7 @@ type Scanner struct{} var _ detectors.Detector = (*Scanner)(nil) var ( - client = common.SaneHttpClient() + client = detectors.NewClientWithDedup(common.SaneHttpClient()) // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"mixmax"}) + `\b([a-zA-Z0-9_-]{36})\b`) @@ -51,7 +51,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } req.Header.Add("X-API-Token", resMatch) - res, err := client.Do(req) + res, err := detectors.DoWithDedup(client, detector_typepb.DetectorType_Mixmax, resMatch, req) if err == nil { defer res.Body.Close() if res.StatusCode >= 200 && res.StatusCode < 300 { diff --git a/pkg/detectors/opencagedata/opencagedata.go b/pkg/detectors/opencagedata/opencagedata.go index 2c61bb5a5..9ca1b56ac 100644 --- a/pkg/detectors/opencagedata/opencagedata.go +++ b/pkg/detectors/opencagedata/opencagedata.go @@ -18,7 +18,7 @@ type Scanner struct{} var _ detectors.Detector = (*Scanner)(nil) var ( - client = common.SaneHttpClient() + client = detectors.NewClientWithDedup(common.SaneHttpClient()) // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"opencagedata"}) + `\b([a-z0-9]{32})\b`) @@ -50,7 +50,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result if err != nil { continue } - res, err := client.Do(req) + res, err := detectors.DoWithDedup(client, detector_typepb.DetectorType_OpenCageData, resMatch, req) if err == nil { defer res.Body.Close() if res.StatusCode >= 200 && res.StatusCode < 300 { diff --git a/pkg/detectors/proxycrawl/proxycrawl.go b/pkg/detectors/proxycrawl/proxycrawl.go index 682a32d76..842c9f09a 100644 --- a/pkg/detectors/proxycrawl/proxycrawl.go +++ b/pkg/detectors/proxycrawl/proxycrawl.go @@ -19,7 +19,7 @@ type Scanner struct{} var _ detectors.Detector = (*Scanner)(nil) var ( - client = common.SaneHttpClient() + client = detectors.NewClientWithDedup(common.SaneHttpClient()) // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"proxycrawl"}) + `\b([a-zA-Z0-9_]{22})\b`) @@ -53,7 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result if err != nil { continue } - res, err := client.Do(req) + res, err := detectors.DoWithDedup(client, detector_typepb.DetectorType_ProxyCrawl, resMatch, req) if err == nil { defer res.Body.Close() if res.StatusCode >= 200 && res.StatusCode < 300 { diff --git a/pkg/detectors/tickettailor/tickettailor.go b/pkg/detectors/tickettailor/tickettailor.go index 5a4d74502..9ff38ede8 100644 --- a/pkg/detectors/tickettailor/tickettailor.go +++ b/pkg/detectors/tickettailor/tickettailor.go @@ -19,7 +19,7 @@ type Scanner struct{} var _ detectors.Detector = (*Scanner)(nil) var ( - client = common.SaneHttpClient() + client = detectors.NewClientWithDedup(common.SaneHttpClient()) // Make sure that your group is surrounded in boundary characters such as below to reduce false positives keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"tickettailor"}) + `\b(sk_[0-9]{4}_[0-9]{6}_[a-f0-9]{32})`) @@ -77,7 +77,7 @@ func verifyTicketTailor(ctx context.Context, client *http.Client, apiKey string) req.Header.Add("Accept", "application/json") // as per API docs we only need to use apiKey as username in basic auth and leave password as empty: https://developers.tickettailor.com/#authentication req.SetBasicAuth(apiKey, "") - resp, err := client.Do(req) + resp, err := detectors.DoWithDedup(client, detector_typepb.DetectorType_Tickettailor, apiKey, req) if err != nil { return false, nil } diff --git a/pkg/detectors/toggltrack/toggltrack.go b/pkg/detectors/toggltrack/toggltrack.go index d646f1af6..909533407 100644 --- a/pkg/detectors/toggltrack/toggltrack.go +++ b/pkg/detectors/toggltrack/toggltrack.go @@ -19,7 +19,7 @@ type Scanner struct{} var _ detectors.Detector = (*Scanner)(nil) var ( - client = common.SaneHttpClient() + client = detectors.NewClientWithDedup(common.SaneHttpClient()) // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"toggl"}) + `\b([0-9Aa-z]{32})\b`) @@ -55,7 +55,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result continue } req.Header.Add("Authorization", fmt.Sprintf("Basic %s", sEnc)) - res, err := client.Do(req) + res, err := detectors.DoWithDedup(client, detector_typepb.DetectorType_TogglTrack, resMatch, req) if err == nil { defer res.Body.Close() if res.StatusCode >= 200 && res.StatusCode < 300 { diff --git a/pkg/detectors/veriphone/veriphone.go b/pkg/detectors/veriphone/veriphone.go index 447437d87..e3a3ca6e2 100644 --- a/pkg/detectors/veriphone/veriphone.go +++ b/pkg/detectors/veriphone/veriphone.go @@ -18,7 +18,7 @@ type Scanner struct{} var _ detectors.Detector = (*Scanner)(nil) var ( - client = common.SaneHttpClient() + client = detectors.NewClientWithDedup(common.SaneHttpClient()) // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"veriphone"}) + `\b([0-9A-Z]{32})\b`) @@ -50,7 +50,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result if err != nil { continue } - res, err := client.Do(req) + res, err := detectors.DoWithDedup(client, detector_typepb.DetectorType_Veriphone, resMatch, req) if err == nil { defer res.Body.Close() if res.StatusCode >= 200 && res.StatusCode < 300 { diff --git a/pkg/output/plain.go b/pkg/output/plain.go index 04a9e216d..a1df13de7 100644 --- a/pkg/output/plain.go +++ b/pkg/output/plain.go @@ -57,9 +57,11 @@ func (p *PlainPrinter) Print(_ context.Context, r *detectors.ResultWithMetadata) yellowPrinter.Printf("Verification issue: %s\n", out.VerificationError) } } + if r.VerificationFromCache { - cyanPrinter.Print("(Verification info cached)\n") + cyanPrinter.Print("(🔍 Using cached verification)\n") } + printer.Printf("Detector Type: %s\n", out.DetectorType) printer.Printf("Decoder Type: %s\n", out.DecoderType) printer.Printf("Raw result: %s\n", whitePrinter.Sprint(out.Raw))