Deduplicate concurrent credential verification requests via singleflight (#4314)

* Cache verification info for reuse

* updated hashing

* tried fixing concurrent verification issue

* Added singleflight to avoid concurrent verification

* resolved linter

* Ok I tried something new for a much simpler detector

* some enhancements

* Added test case

* re-added tags

* Deduplicate concurrent credential verification requests via singleflight

* Enforce dedup key via DoWithDedup, remove public WithDedupKey

* remove unused func

* Remove dead io.Copy after io.ReadAll error in singleflight transport

* Preserve deadline on shared singleflight request after WithoutCancel

* Added test cases and moved existing ones to http_test.go

* fixed linter

pkg/detectors/cliengo/cliengo.go

@@ -18,7 +18,7 @@ type Scanner struct{}
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
-	client = common.SaneHttpClient()
+	client = detectors.NewClientWithDedup(common.SaneHttpClient())
 
 	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"cliengo"}) + `\b([0-9a-f]{8}\-[0-9a-f]{4}\-[0-9a-f]{4}\-[0-9a-f]{4}\-[0-9a-f]{12})\b`)
@@ -50,7 +50,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			if err != nil {
 				continue
 			}
-			res, err := client.Do(req)
+			res, err := detectors.DoWithDedup(client, detector_typepb.DetectorType_Cliengo, resMatch, req)
 			if err == nil {
 				defer res.Body.Close()
 				if res.StatusCode >= 200 && res.StatusCode < 300 {

pkg/detectors/clockify/clockify.go

@@ -18,7 +18,7 @@ type Scanner struct{}
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
-	client = common.SaneHttpClient()
+	client = detectors.NewClientWithDedup(common.SaneHttpClient())
 
 	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"clockify"}) + `\b([a-zA-Z0-9]{48})\b`)
@@ -52,7 +52,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			}
 			req.Header.Add("content-type", "application/json")
 			req.Header.Add("X-Api-Key", resMatch)
-			res, err := client.Do(req)
+			res, err := detectors.DoWithDedup(client, detector_typepb.DetectorType_Clockify, resMatch, req)
 			if err == nil {
 				defer res.Body.Close()
 				if res.StatusCode >= 200 && res.StatusCode < 300 {

pkg/detectors/databox/databox.go

@@ -20,7 +20,7 @@ type Scanner struct{}
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
-	client = common.SaneHttpClient()
+	client = detectors.NewClientWithDedup(common.SaneHttpClient())
 
 	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"databox"}) + common.BuildRegex(common.RegexPattern, "", 21))
@@ -65,7 +65,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			req.Header.Add("Content-Type", "application/json")
 			req.Header.Add("Accept", "application/vnd.databox.v2+json")
 			req.Header.Add("Authorization", fmt.Sprintf("Basic %s", sEnc))
-			res, err := client.Do(req)
+			res, err := detectors.DoWithDedup(client, detector_typepb.DetectorType_Databox, resMatch, req)
 			if err == nil {
 				defer res.Body.Close()
 				if res.StatusCode >= 200 && res.StatusCode < 300 {

pkg/detectors/detectors.go

@@ -4,7 +4,9 @@ import (
 	"context"
 	"crypto/rand"
 	"errors"
+	"fmt"
 	"math/big"
+	"net/http"
 	"net/url"
 	"strings"
 	"unicode"
@@ -335,3 +337,20 @@ func ParseURLAndStripPathAndParams(u string) (*url.URL, error) {
 	parsedURL.RawQuery = ""
 	return parsedURL, nil
 }
+
+type dedupKeyContextKey struct{}
+
+func withDedupKey(ctx context.Context, detType detector_typepb.DetectorType, credential string) context.Context {
+	key := fmt.Sprintf("%d:%s", int32(detType), credential)
+	return context.WithValue(ctx, dedupKeyContextKey{}, key)
+}
+
+// DoWithDedup executes req through client, coalescing concurrent requests that share
+// the same detector type and credential into a single network call via singleflight.
+// The response body is fully buffered and replayed to every waiting caller.
+//
+// Use this instead of client.Do for all verification requests on a client created
+// with NewClientWithDedup or WithDedup — it is the only way to activate deduplication.
+func DoWithDedup(client *http.Client, detType detector_typepb.DetectorType, credential string, req *http.Request) (*http.Response, error) {
+	return client.Do(req.WithContext(withDedupKey(req.Context(), detType, credential)))
+}

pkg/detectors/finage/finage.go

@@ -19,7 +19,7 @@ type Scanner struct{}
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
-	client = common.SaneHttpClient()
+	client = detectors.NewClientWithDedup(common.SaneHttpClient())
 
 	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat = regexp.MustCompile(`\b(API_KEY[0-9A-Z]{32})\b`)
@@ -52,7 +52,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				continue
 			}
 			req.Header.Add("Content-Type", "application/json")
-			res, err := client.Do(req)
+			res, err := detectors.DoWithDedup(client, detector_typepb.DetectorType_Finage, resMatch, req)
 			if err == nil {
 				defer res.Body.Close()
 				if res.StatusCode >= 200 && res.StatusCode < 300 {

pkg/detectors/geocode/geocode.go

@@ -20,7 +20,7 @@ type Scanner struct{}
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
-	client = common.SaneHttpClient()
+	client = detectors.NewClientWithDedup(common.SaneHttpClient())
 
 	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"geocode"}) + `\b([a-z0-9]{28})\b`)
@@ -52,7 +52,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			if err != nil {
 				continue
 			}
-			res, err := client.Do(req)
+			res, err := detectors.DoWithDedup(client, detector_typepb.DetectorType_Geocode, resMatch, req)
 			if err == nil {
 				bodyBytes, err := io.ReadAll(res.Body)
 				if err == nil {

pkg/detectors/gitter/gitter.go

@@ -19,7 +19,7 @@ type Scanner struct{}
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
-	client = common.SaneHttpClient()
+	client = detectors.NewClientWithDedup(common.SaneHttpClient())
 
 	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"gitter"}) + `\b([a-z0-9-]{40})\b`)
@@ -52,7 +52,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				continue
 			}
 			req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))
-			res, err := client.Do(req)
+			res, err := detectors.DoWithDedup(client, detector_typepb.DetectorType_Gitter, resMatch, req)
 			if err == nil {
 				defer res.Body.Close()
 				if res.StatusCode >= 200 && res.StatusCode < 300 {

pkg/detectors/holidayapi/holidayapi.go

@@ -19,7 +19,7 @@ type Scanner struct{}
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
-	client = common.SaneHttpClient()
+	client = detectors.NewClientWithDedup(common.SaneHttpClient())
 
 	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"holidayapi"}) + `\b([a-z0-9-]{36})\b`)
@@ -51,7 +51,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			if err != nil {
 				continue
 			}
-			res, err := client.Do(req)
+			res, err := detectors.DoWithDedup(client, detector_typepb.DetectorType_HolidayAPI, resMatch, req)
 			if err == nil {
 				defer res.Body.Close()
 				if res.StatusCode >= 200 && res.StatusCode < 300 {

pkg/detectors/http.go

@@ -1,14 +1,19 @@
 package detectors
 
 import (
+	"bytes"
 	"context"
 	"errors"
+	"fmt"
+	"io"
 	"net"
 	"net/http"
 	"slices"
 	"sync"
 	"time"
 
+	"golang.org/x/sync/singleflight"
+
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/feature"
 )
@@ -175,3 +180,93 @@ func NewDetectorHttpClient(opts ...ClientOption) *http.Client {
 	client.Transport = common.NewInstrumentedTransport(client.Transport)
 	return client
 }
+
+// bufferedResponse holds a fully-read HTTP response so it can be replayed to
+// every goroutine that was coalesced by singleflight.
+type bufferedResponse struct {
+	statusCode int
+	header     http.Header
+	body       []byte
+}
+
+// singleflightTransport is an http.RoundTripper that coalesces concurrent requests
+// sharing the same deduplication key into a single network call. It is a no-op for
+// requests whose context does not carry a dedup key.
+type singleflightTransport struct {
+	base  http.RoundTripper
+	group singleflight.Group
+}
+
+func (t *singleflightTransport) RoundTrip(req *http.Request) (*http.Response, error) {
+	key, ok := req.Context().Value(dedupKeyContextKey{}).(string)
+	if !ok || key == "" {
+		return t.base.RoundTrip(req)
+	}
+
+	// DoChan is used instead of Do so each caller can independently respect its
+	// own context cancellation without blocking on the shared in-flight call.
+	ch := t.group.DoChan(key, func() (any, error) {
+		// Detach the in-flight request from the first caller's cancellation so
+		// that one goroutine timing out doesn't abort the shared network call
+		// and propagate an error to all coalesced waiters.
+		//
+		// context.WithoutCancel also strips any deadline (e.g. from
+		// http.Client.Timeout), so we re-attach the original deadline if
+		// present. Without this the shared request has no timeout and a
+		// hanging server would leak the goroutine and pin the singleflight
+		// key indefinitely.
+		sharedCtx := context.WithoutCancel(req.Context())
+		if deadline, ok := req.Context().Deadline(); ok {
+			var cancel context.CancelFunc
+			sharedCtx, cancel = context.WithDeadline(sharedCtx, deadline)
+			defer cancel()
+		}
+		sharedReq := req.WithContext(sharedCtx)
+		resp, err := t.base.RoundTrip(sharedReq)
+		if err != nil {
+			return nil, err
+		}
+		defer resp.Body.Close()
+
+		body, err := io.ReadAll(resp.Body)
+		if err != nil {
+			return nil, err
+		}
+
+		return &bufferedResponse{
+			statusCode: resp.StatusCode,
+			header:     resp.Header.Clone(),
+			body:       body,
+		}, nil
+	})
+
+	select {
+	case result := <-ch:
+		if result.Err != nil {
+			return nil, result.Err
+		}
+		br := result.Val.(*bufferedResponse)
+		return &http.Response{
+			StatusCode: br.statusCode,
+			Status:     fmt.Sprintf("%d %s", br.statusCode, http.StatusText(br.statusCode)),
+			Header:     br.header.Clone(),
+			Body:       io.NopCloser(bytes.NewReader(br.body)),
+		}, nil
+	case <-req.Context().Done():
+		return nil, req.Context().Err()
+	}
+}
+
+// NewClientWithDedup wraps base with a transport that deduplicates concurrent
+// verification requests sharing the same key. Detectors opt in per credential by
+// calling WithDedupKey on the request context before client.Do — no other changes
+// to request building or response reading are needed.
+func NewClientWithDedup(base *http.Client) *http.Client {
+	clone := *base
+	transport := base.Transport
+	if transport == nil {
+		transport = http.DefaultTransport
+	}
+	clone.Transport = &singleflightTransport{base: transport}
+	return &clone
+}

pkg/detectors/http_test.go

@@ -2,11 +2,18 @@ package detectors
 
 import (
 	"context"
+	"fmt"
 	"net"
 	"net/http"
+	"net/http/httptest"
+	"sync"
+	"sync/atomic"
 	"testing"
+	"time"
 
 	"github.com/stretchr/testify/assert"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb"
 )
 
 func TestWithNoLocalIP(t *testing.T) {
@@ -95,6 +102,213 @@ func TestWithNoLocalIP(t *testing.T) {
 	})
 }
 
+// TestDoWithDedup_Singleflight verifies that concurrent DoWithDedup calls sharing the
+// same detector type and credential are coalesced into one network call. Each request
+// the server receives returns a distinct body, so all goroutines should observe the
+// body from exactly one actual server-side request.
+func TestDoWithDedup_Singleflight(t *testing.T) {
+	var requestCount int32
+
+	// The 20 ms sleep keeps the first request in-flight long enough for all
+	// goroutines to call DoWithDedup before the result is ready.
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		n := atomic.AddInt32(&requestCount, 1)
+		time.Sleep(20 * time.Millisecond)
+		fmt.Fprintf(w, `{"request":%d}`, n)
+	}))
+	defer server.Close()
+
+	client := NewClientWithDedup(server.Client())
+
+	const goroutines = 5
+	bodies := make([]string, goroutines)
+	statuses := make([]int, goroutines)
+	errs := make([]error, goroutines)
+
+	var wg sync.WaitGroup
+	for i := range goroutines {
+		wg.Add(1)
+		go func(i int) {
+			defer wg.Done()
+			req, err := http.NewRequestWithContext(t.Context(), http.MethodGet, server.URL, http.NoBody)
+			if err != nil {
+				errs[i] = err
+				return
+			}
+			resp, err := DoWithDedup(client, detector_typepb.DetectorType_Meraki, "test-credential", req)
+			if err != nil {
+				errs[i] = err
+				return
+			}
+			defer resp.Body.Close()
+			var buf [512]byte
+			n, _ := resp.Body.Read(buf[:])
+			bodies[i] = string(buf[:n])
+			statuses[i] = resp.StatusCode
+		}(i)
+	}
+	wg.Wait()
+
+	for _, err := range errs {
+		assert.NoError(t, err)
+	}
+	for _, s := range statuses {
+		assert.Equal(t, http.StatusOK, s)
+	}
+	assert.Equal(t, int32(1), atomic.LoadInt32(&requestCount),
+		"singleflight should coalesce all concurrent calls into one HTTP request")
+	for i := 1; i < goroutines; i++ {
+		assert.Equal(t, bodies[0], bodies[i])
+	}
+}
+
+// TestDoWithDedup_WaiterContextCancelled verifies that a waiter whose context is
+// cancelled bails out with an error while other waiters still receive the response,
+// and only one HTTP request is made.
+func TestDoWithDedup_WaiterContextCancelled(t *testing.T) {
+	var requestCount int32
+	// inFlight is closed by the server once it starts handling the request,
+	// giving us a reliable signal to cancel one waiter mid-flight.
+	inFlight := make(chan struct{})
+
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		atomic.AddInt32(&requestCount, 1)
+		close(inFlight)
+		time.Sleep(50 * time.Millisecond)
+		w.WriteHeader(http.StatusOK)
+	}))
+	defer server.Close()
+
+	client := NewClientWithDedup(server.Client())
+
+	ctx1, cancel1 := context.WithCancel(context.Background())
+	defer cancel1()
+
+	type result struct {
+		status int
+		err    error
+	}
+	results := make([]result, 3)
+	var wg sync.WaitGroup
+
+	for i, ctx := range []context.Context{ctx1, context.Background(), context.Background()} {
+		wg.Add(1)
+		go func(i int, ctx context.Context) {
+			defer wg.Done()
+			req, _ := http.NewRequestWithContext(ctx, http.MethodGet, server.URL, http.NoBody)
+			resp, err := DoWithDedup(client, detector_typepb.DetectorType_Meraki, "cred", req)
+			if err != nil {
+				results[i] = result{err: err}
+				return
+			}
+			defer resp.Body.Close()
+			results[i] = result{status: resp.StatusCode}
+		}(i, ctx)
+	}
+
+	<-inFlight
+	cancel1()
+	wg.Wait()
+
+	assert.ErrorIs(t, results[0].err, context.Canceled, "cancelled waiter should get context error")
+	assert.NoError(t, results[1].err)
+	assert.Equal(t, http.StatusOK, results[1].status)
+	assert.NoError(t, results[2].err)
+	assert.Equal(t, http.StatusOK, results[2].status)
+	assert.Equal(t, int32(1), atomic.LoadInt32(&requestCount), "only one HTTP request should be made")
+}
+
+// TestDoWithDedup_FirstCallerContextCancelled verifies that cancelling the first
+// caller's context does not abort the shared in-flight HTTP call: the second caller
+// should still receive a valid response, and only one HTTP request is made.
+func TestDoWithDedup_FirstCallerContextCancelled(t *testing.T) {
+	var requestCount int32
+	inFlight := make(chan struct{})
+
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		atomic.AddInt32(&requestCount, 1)
+		close(inFlight)
+		time.Sleep(50 * time.Millisecond)
+		w.WriteHeader(http.StatusOK)
+	}))
+	defer server.Close()
+
+	client := NewClientWithDedup(server.Client())
+
+	ctx1, cancel1 := context.WithCancel(context.Background())
+	defer cancel1()
+
+	var firstErr error
+	var wg sync.WaitGroup
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		req, _ := http.NewRequestWithContext(ctx1, http.MethodGet, server.URL, http.NoBody)
+		resp, err := DoWithDedup(client, detector_typepb.DetectorType_Meraki, "cred", req)
+		if err != nil {
+			firstErr = err
+			return
+		}
+		resp.Body.Close()
+	}()
+
+	// Cancel the first caller once the server is processing, then immediately
+	// start a second caller that should coalesce into the still-running call.
+	<-inFlight
+	cancel1()
+
+	var secondStatus int
+	var secondErr error
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		req, _ := http.NewRequestWithContext(context.Background(), http.MethodGet, server.URL, http.NoBody)
+		resp, err := DoWithDedup(client, detector_typepb.DetectorType_Meraki, "cred", req)
+		if err != nil {
+			secondErr = err
+			return
+		}
+		defer resp.Body.Close()
+		secondStatus = resp.StatusCode
+	}()
+
+	wg.Wait()
+
+	assert.ErrorIs(t, firstErr, context.Canceled, "first caller should get context error")
+	assert.NoError(t, secondErr, "second caller should succeed despite first caller's cancellation")
+	assert.Equal(t, http.StatusOK, secondStatus)
+	assert.Equal(t, int32(1), atomic.LoadInt32(&requestCount), "only one HTTP request should be made")
+}
+
+// TestDoWithDedup_DeadlinePreserved verifies that the client timeout still applies
+// to the shared in-flight call after context.WithoutCancel strips cancellation.
+// A hanging server must not cause an indefinite leak.
+func TestDoWithDedup_DeadlinePreserved(t *testing.T) {
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		select {
+		case <-time.After(10 * time.Second):
+		case <-r.Context().Done():
+		}
+	}))
+	defer server.Close()
+
+	base := server.Client()
+	base.Timeout = 75 * time.Millisecond
+	client := NewClientWithDedup(base)
+
+	req, _ := http.NewRequestWithContext(context.Background(), http.MethodGet, server.URL, http.NoBody)
+	start := time.Now()
+	resp, err := DoWithDedup(client, detector_typepb.DetectorType_Meraki, "cred", req)
+	if err == nil {
+		defer resp.Body.Close()
+	}
+
+	elapsed := time.Since(start)
+
+	assert.Error(t, err, "request to hanging server should fail")
+	assert.Less(t, elapsed, time.Second, "timeout should be enforced by client deadline, not run indefinitely")
+}
+
 func TestIsLocalIP(t *testing.T) {
 	testCases := []struct {
 		name     string

pkg/detectors/ipinfo/ipinfo.go

@@ -21,7 +21,7 @@ type Scanner struct {
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
-	defaultClient = common.SaneHttpClient()
+	defaultClient = detectors.NewClientWithDedup(common.SaneHttpClient())
 	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"ipinfo"}) + `\b([a-f0-9]{14})\b`)
 )
@@ -56,7 +56,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			if err != nil {
 				continue
 			}
-			res, err := client.Do(req)
+			res, err := detectors.DoWithDedup(client, detector_typepb.DetectorType_IPInfo, resMatch, req)
 			if err == nil {
 				defer res.Body.Close()
 				if res.StatusCode >= 200 && res.StatusCode < 300 {

pkg/detectors/loadmill/loadmill.go

@@ -18,7 +18,7 @@ type Scanner struct{}
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
-	client = common.SaneHttpClient()
+	client = detectors.NewClientWithDedup(common.SaneHttpClient())
 
 	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"loadmill"}) + `\b([0-9a-zA-Z]{40})\b`)
@@ -52,7 +52,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			}
 			req.Header.Add("Accept", "application/vnd.loadmill+json; version=3")
 			req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))
-			res, err := client.Do(req)
+			res, err := detectors.DoWithDedup(client, detector_typepb.DetectorType_Loadmill, resMatch, req)
 			if err == nil {
 				defer res.Body.Close()
 				if res.StatusCode >= 200 && res.StatusCode < 300 {

pkg/detectors/meraki/meraki.go

@@ -29,7 +29,7 @@ type merakiOrganizations struct {
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
-	defaultClient = common.SaneHttpClient()
+	defaultClient = detectors.NewClientWithDedup(common.SaneHttpClient())
 
 	apiKey = regexp.MustCompile(detectors.PrefixRegex([]string{"meraki"}) + `([0-9a-f]{40})`)
 )
@@ -57,9 +57,7 @@ func (s Scanner) Description() string {
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
 	dataStr := string(data)
 
-	// uniqueMatches will hold unique match values and ensure we only process unique matches found in the data string
-	var uniqueMatches = make(map[string]struct{})
-
+	uniqueMatches := make(map[string]struct{})
 	for _, match := range apiKey.FindAllStringSubmatch(dataStr, -1) {
 		uniqueMatches[match[1]] = struct{}{}
 	}
@@ -107,11 +105,9 @@ func verifyMerakiApiKey(ctx context.Context, client *http.Client, match string)
 	if err != nil {
 		return nil, false, err
 	}
-
-	// set the required auth header
 	req.Header.Set("X-Cisco-Meraki-API-Key", match)
 
-	resp, err := client.Do(req)
+	resp, err := detectors.DoWithDedup(client, detector_typepb.DetectorType_Meraki, match, req)
 	if err != nil {
 		return nil, false, err
 	}
@@ -122,12 +118,10 @@ func verifyMerakiApiKey(ctx context.Context, client *http.Client, match string)
 
 	switch resp.StatusCode {
 	case http.StatusOK:
-		// in case token is verified, capture the organization id's and name which are accessible via token.
 		var organizations []merakiOrganizations
 		if err = json.NewDecoder(resp.Body).Decode(&organizations); err != nil {
 			return nil, false, err
 		}
-
 		return organizations, true, nil
 	case http.StatusUnauthorized:
 		return nil, false, nil

pkg/detectors/mixmax/mixmax.go

@@ -17,7 +17,7 @@ type Scanner struct{}
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
-	client = common.SaneHttpClient()
+	client = detectors.NewClientWithDedup(common.SaneHttpClient())
 
 	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"mixmax"}) + `\b([a-zA-Z0-9_-]{36})\b`)
@@ -51,7 +51,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			}
 
 			req.Header.Add("X-API-Token", resMatch)
-			res, err := client.Do(req)
+			res, err := detectors.DoWithDedup(client, detector_typepb.DetectorType_Mixmax, resMatch, req)
 			if err == nil {
 				defer res.Body.Close()
 				if res.StatusCode >= 200 && res.StatusCode < 300 {

pkg/detectors/opencagedata/opencagedata.go

@@ -18,7 +18,7 @@ type Scanner struct{}
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
-	client = common.SaneHttpClient()
+	client = detectors.NewClientWithDedup(common.SaneHttpClient())
 
 	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"opencagedata"}) + `\b([a-z0-9]{32})\b`)
@@ -50,7 +50,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			if err != nil {
 				continue
 			}
-			res, err := client.Do(req)
+			res, err := detectors.DoWithDedup(client, detector_typepb.DetectorType_OpenCageData, resMatch, req)
 			if err == nil {
 				defer res.Body.Close()
 				if res.StatusCode >= 200 && res.StatusCode < 300 {

pkg/detectors/proxycrawl/proxycrawl.go

@@ -19,7 +19,7 @@ type Scanner struct{}
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
-	client = common.SaneHttpClient()
+	client = detectors.NewClientWithDedup(common.SaneHttpClient())
 
 	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"proxycrawl"}) + `\b([a-zA-Z0-9_]{22})\b`)
@@ -53,7 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			if err != nil {
 				continue
 			}
-			res, err := client.Do(req)
+			res, err := detectors.DoWithDedup(client, detector_typepb.DetectorType_ProxyCrawl, resMatch, req)
 			if err == nil {
 				defer res.Body.Close()
 				if res.StatusCode >= 200 && res.StatusCode < 300 {

pkg/detectors/tickettailor/tickettailor.go

@@ -19,7 +19,7 @@ type Scanner struct{}
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
-	client = common.SaneHttpClient()
+	client = detectors.NewClientWithDedup(common.SaneHttpClient())
 
 	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"tickettailor"}) + `\b(sk_[0-9]{4}_[0-9]{6}_[a-f0-9]{32})`)
@@ -77,7 +77,7 @@ func verifyTicketTailor(ctx context.Context, client *http.Client, apiKey string)
 	req.Header.Add("Accept", "application/json")
 	// as per API docs we only need to use apiKey as username in basic auth and leave password as empty: https://developers.tickettailor.com/#authentication
 	req.SetBasicAuth(apiKey, "")
-	resp, err := client.Do(req)
+	resp, err := detectors.DoWithDedup(client, detector_typepb.DetectorType_Tickettailor, apiKey, req)
 	if err != nil {
 		return false, nil
 	}

pkg/detectors/toggltrack/toggltrack.go

@@ -19,7 +19,7 @@ type Scanner struct{}
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
-	client = common.SaneHttpClient()
+	client = detectors.NewClientWithDedup(common.SaneHttpClient())
 
 	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"toggl"}) + `\b([0-9Aa-z]{32})\b`)
@@ -55,7 +55,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				continue
 			}
 			req.Header.Add("Authorization", fmt.Sprintf("Basic %s", sEnc))
-			res, err := client.Do(req)
+			res, err := detectors.DoWithDedup(client, detector_typepb.DetectorType_TogglTrack, resMatch, req)
 			if err == nil {
 				defer res.Body.Close()
 				if res.StatusCode >= 200 && res.StatusCode < 300 {

pkg/detectors/veriphone/veriphone.go

@@ -18,7 +18,7 @@ type Scanner struct{}
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
-	client = common.SaneHttpClient()
+	client = detectors.NewClientWithDedup(common.SaneHttpClient())
 
 	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"veriphone"}) + `\b([0-9A-Z]{32})\b`)
@@ -50,7 +50,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			if err != nil {
 				continue
 			}
-			res, err := client.Do(req)
+			res, err := detectors.DoWithDedup(client, detector_typepb.DetectorType_Veriphone, resMatch, req)
 			if err == nil {
 				defer res.Body.Close()
 				if res.StatusCode >= 200 && res.StatusCode < 300 {

pkg/output/plain.go

@@ -57,9 +57,11 @@ func (p *PlainPrinter) Print(_ context.Context, r *detectors.ResultWithMetadata)
 			yellowPrinter.Printf("Verification issue: %s\n", out.VerificationError)
 		}
 	}
+
 	if r.VerificationFromCache {
-		cyanPrinter.Print("(Verification info cached)\n")
+		cyanPrinter.Print("(🔍 Using cached verification)\n")
 	}
+
 	printer.Printf("Detector Type: %s\n", out.DetectorType)
 	printer.Printf("Decoder Type: %s\n", out.DecoderType)
 	printer.Printf("Raw result: %s\n", whitePrinter.Sprint(out.Raw))