added and updated pattern tests for detectors starting from p to q (#3710)

pkg/detectors/packagecloud/packagecloud_integration_test.go

@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package packagecloud
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/kylelemons/godebug/pretty"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestPackageCloud_FromChunk(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	defer cancel()
+	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+	if err != nil {
+		t.Fatalf("could not get test secrets from GCP: %s", err)
+	}
+	secret := testSecrets.MustGetField("PACKAGECLOUD")
+	inactiveSecret := testSecrets.MustGetField("PACKAGECLOUD_INACTIVE")
+
+	type args struct {
+		ctx    context.Context
+		data   []byte
+		verify bool
+	}
+	tests := []struct {
+		name    string
+		s       Scanner
+		args    args
+		want    []detectors.Result
+		wantErr bool
+	}{
+		{
+			name: "found, verified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a packagecloud secret %s within", secret)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_PackageCloud,
+					Verified:     true,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "found, unverified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a packagecloud secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_PackageCloud,
+					Verified:     false,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "not found",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte("You cannot find the secret within"),
+				verify: true,
+			},
+			want:    nil,
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			s := Scanner{}
+			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("PackageCloud.FromData() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			for i := range got {
+				if len(got[i].Raw) == 0 {
+					t.Fatalf("no raw secret present: \n %+v", got[i])
+				}
+				got[i].Raw = nil
+			}
+			if diff := pretty.Compare(got, tt.want); diff != "" {
+				t.Errorf("PackageCloud.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+			}
+		})
+	}
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+	ctx := context.Background()
+	s := Scanner{}
+	for name, data := range detectors.MustGetBenchmarkData() {
+		benchmark.Run(name, func(b *testing.B) {
+			b.ResetTimer()
+			for n := 0; n < b.N; n++ {
+				_, err := s.FromData(ctx, false, data)
+				if err != nil {
+					b.Fatal(err)
+				}
+			}
+		})
+	}
+}

pkg/detectors/packagecloud/packagecloud_test.go

@@ -1,119 +1,90 @@
-//go:build detectors
-// +build detectors
-
 package packagecloud
 
 import (
 	"context"
 	"fmt"
 	"testing"
-	"time"
 
-	"github.com/kylelemons/godebug/pretty"
+	"github.com/google/go-cmp/cmp"
+
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
-
+	"github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 )
 
-func TestPackageCloud_FromChunk(t *testing.T) {
+var (
-	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	validPattern   = "3a4c163d0c145e7346b53f6d9be8e0a18058a5b7f03e2b41"
-	defer cancel()
+	invalidPattern = "3a4c163d0c145e7346b53f6d?be8e0a18058a5b7f03e2b41"
-	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+	keyword        = "packagecloud"
-	if err != nil {
+)
-		t.Fatalf("could not get test secrets from GCP: %s", err)
-	}
-	secret := testSecrets.MustGetField("PACKAGECLOUD")
-	inactiveSecret := testSecrets.MustGetField("PACKAGECLOUD_INACTIVE")
 
-	type args struct {
+func TestPackageCloud_Pattern(t *testing.T) {
-		ctx    context.Context
+	d := Scanner{}
-		data   []byte
+	ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
-		verify bool
-	}
 	tests := []struct {
-		name    string
+		name  string
-		s       Scanner
+		input string
-		args    args
+		want  []string
-		want    []detectors.Result
-		wantErr bool
 	}{
 		{
-			name: "found, verified",
+			name:  "valid pattern - with keyword packagecloud",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
-			args: args{
+			want:  []string{validPattern},
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a packagecloud secret %s within", secret)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_PackageCloud,
-					Verified:     true,
-				},
-			},
-			wantErr: false,
 		},
 		{
-			name: "found, unverified",
+			name:  "valid pattern - ignore duplicate",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
-			args: args{
+			want:  []string{validPattern},
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a packagecloud secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_PackageCloud,
-					Verified:     false,
-				},
-			},
-			wantErr: false,
 		},
 		{
-			name: "not found",
+			name:  "valid pattern - key out of prefix range",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
-			args: args{
+			want:  []string{},
-				ctx:    context.Background(),
+		},
-				data:   []byte("You cannot find the secret within"),
+		{
-				verify: true,
+			name:  "invalid pattern",
-			},
+			input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
-			want:    nil,
+			want:  []string{},
-			wantErr: false,
 		},
 	}
-	for _, tt := range tests {
+
-		t.Run(tt.name, func(t *testing.T) {
+	for _, test := range tests {
-			s := Scanner{}
+		t.Run(test.name, func(t *testing.T) {
-			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
-			if (err != nil) != tt.wantErr {
+			if len(matchedDetectors) == 0 {
-				t.Errorf("PackageCloud.FromData() error = %v, wantErr %v", err, tt.wantErr)
+				t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
 				return
 			}
-			for i := range got {
-				if len(got[i].Raw) == 0 {
-					t.Fatalf("no raw secret present: \n %+v", got[i])
-				}
-				got[i].Raw = nil
-			}
-			if diff := pretty.Compare(got, tt.want); diff != "" {
-				t.Errorf("PackageCloud.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
-			}
-		})
-	}
-}
 
-func BenchmarkFromData(benchmark *testing.B) {
+			results, err := d.FromData(context.Background(), false, []byte(test.input))
-	ctx := context.Background()
+			if err != nil {
-	s := Scanner{}
+				t.Errorf("error = %v", err)
-	for name, data := range detectors.MustGetBenchmarkData() {
+				return
-		benchmark.Run(name, func(b *testing.B) {
+			}
-			b.ResetTimer()
+
-			for n := 0; n < b.N; n++ {
+			if len(results) != len(test.want) {
-				_, err := s.FromData(ctx, false, data)
+				if len(results) == 0 {
-				if err != nil {
+					t.Errorf("did not receive result")
-					b.Fatal(err)
+				} else {
+					t.Errorf("expected %d results, only received %d", len(test.want), len(results))
 				}
+				return
+			}
+
+			actual := make(map[string]struct{}, len(results))
+			for _, r := range results {
+				if len(r.RawV2) > 0 {
+					actual[string(r.RawV2)] = struct{}{}
+				} else {
+					actual[string(r.Raw)] = struct{}{}
+				}
+			}
+			expected := make(map[string]struct{}, len(test.want))
+			for _, v := range test.want {
+				expected[v] = struct{}{}
+			}
+
+			if diff := cmp.Diff(expected, actual); diff != "" {
+				t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
 			}
 		})
 	}

pkg/detectors/pagerdutyapikey/pagerdutyapikey_integration_test.go

@@ -0,0 +1,157 @@
+//go:build detectors
+// +build detectors
+
+package pagerdutyapikey
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestPagerDutyApiKey_FromChunk(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	defer cancel()
+	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+	if err != nil {
+		t.Fatalf("could not get test secrets from GCP: %s", err)
+	}
+	secret := testSecrets.MustGetField("PAGERDUTYAPIKEY_TOKEN")
+	invalidSecret := testSecrets.MustGetField("PAGERDUTYAPIKEY_INACTIVE")
+
+	type args struct {
+		ctx    context.Context
+		data   []byte
+		verify bool
+	}
+	tests := []struct {
+		name                string
+		s                   Scanner
+		args                args
+		want                []detectors.Result
+		wantErr             bool
+		wantVerificationErr bool
+	}{
+		{
+			name: "found, verified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a pagerdutyapikey secret %s within", secret)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_PagerDutyApiKey,
+					Verified:     true,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "found, would be verified if not for timeout",
+			s:    Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a pagerdutyapikey secret %s within", secret)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_PagerDutyApiKey,
+					Verified:     false,
+				},
+			},
+			wantErr:             false,
+			wantVerificationErr: true,
+		},
+		{
+			name: "found, verified but unexpected api surface",
+			s:    Scanner{client: common.ConstantResponseHttpClient(404, "")},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a pagerdutyapikey secret %s within", secret)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_PagerDutyApiKey,
+					Verified:     false,
+				},
+			},
+			wantErr:             false,
+			wantVerificationErr: true,
+		},
+		{
+			name: "found, unverified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a pagerdutyapikey secret %s within but not valid", invalidSecret)), // the secret would satisfy the regex but not pass validation
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_PagerDutyApiKey,
+					Verified:     false,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "not found",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte("You cannot find the secret within"),
+				verify: true,
+			},
+			want:    nil,
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("PagerDutyApiKey.FromData() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			for i := range got {
+				if len(got[i].Raw) == 0 {
+					t.Fatalf("no raw secret present: \n %+v", got[i])
+				}
+				if (got[i].VerificationError() != nil) != tt.wantVerificationErr {
+					t.Errorf("PagerDutyApiKey.FromData() verificationError = %v, wantVerificationErr %v", got[i].VerificationError(), tt.wantVerificationErr)
+				}
+			}
+			ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")
+			if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
+				t.Errorf("PagerDutyApiKey.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+			}
+		})
+	}
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+	ctx := context.Background()
+	s := Scanner{}
+	for name, data := range detectors.MustGetBenchmarkData() {
+		benchmark.Run(name, func(b *testing.B) {
+			b.ResetTimer()
+			for n := 0; n < b.N; n++ {
+				_, err := s.FromData(ctx, false, data)
+				if err != nil {
+					b.Fatal(err)
+				}
+			}
+		})
+	}
+}

pkg/detectors/pagerdutyapikey/pagerdutyapikey_test.go

@@ -1,156 +1,90 @@
-//go:build detectors
-// +build detectors
-
 package pagerdutyapikey
 
 import (
 	"context"
 	"fmt"
 	"testing"
-	"time"
 
 	"github.com/google/go-cmp/cmp"
-	"github.com/google/go-cmp/cmp/cmpopts"
+
-	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
 )
 
-func TestPagerDutyApiKey_FromChunk(t *testing.T) {
+var (
-	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	validPattern   = "u+C4lGv0tct7TlvSUDoc"
-	defer cancel()
+	invalidPattern = "u+C4lGv0?ct7TlvSUDoc"
-	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+	keyword        = "pagerdutyapikey"
-	if err != nil {
+)
-		t.Fatalf("could not get test secrets from GCP: %s", err)
-	}
-	secret := testSecrets.MustGetField("PAGERDUTYAPIKEY_TOKEN")
-	invalidSecret := testSecrets.MustGetField("PAGERDUTYAPIKEY_INACTIVE")
 
-	type args struct {
+func TestPagerDutyApiKey_Pattern(t *testing.T) {
-		ctx    context.Context
+	d := Scanner{}
-		data   []byte
+	ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
-		verify bool
-	}
 	tests := []struct {
-		name                string
+		name  string
-		s                   Scanner
+		input string
-		args                args
+		want  []string
-		want                []detectors.Result
-		wantErr             bool
-		wantVerificationErr bool
 	}{
 		{
-			name: "found, verified",
+			name:  "valid pattern - with keyword pagerdutyapikey",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
-			args: args{
+			want:  []string{validPattern},
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a pagerdutyapikey secret %s within", secret)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_PagerDutyApiKey,
-					Verified:     true,
-				},
-			},
-			wantErr: false,
 		},
 		{
-			name: "found, would be verified if not for timeout",
+			name:  "valid pattern - ignore duplicate",
-			s:    Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
+			input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
-			args: args{
+			want:  []string{validPattern},
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a pagerdutyapikey secret %s within", secret)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_PagerDutyApiKey,
-					Verified:     false,
-				},
-			},
-			wantErr:             false,
-			wantVerificationErr: true,
 		},
 		{
-			name: "found, verified but unexpected api surface",
+			name:  "valid pattern - key out of prefix range",
-			s:    Scanner{client: common.ConstantResponseHttpClient(404, "")},
+			input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
-			args: args{
+			want:  []string{},
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a pagerdutyapikey secret %s within", secret)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_PagerDutyApiKey,
-					Verified:     false,
-				},
-			},
-			wantErr:             false,
-			wantVerificationErr: true,
 		},
 		{
-			name: "found, unverified",
+			name:  "invalid pattern",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
-			args: args{
+			want:  []string{},
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a pagerdutyapikey secret %s within but not valid", invalidSecret)), // the secret would satisfy the regex but not pass validation
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_PagerDutyApiKey,
-					Verified:     false,
-				},
-			},
-			wantErr: false,
-		},
-		{
-			name: "not found",
-			s:    Scanner{},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte("You cannot find the secret within"),
-				verify: true,
-			},
-			want:    nil,
-			wantErr: false,
 		},
 	}
-	for _, tt := range tests {
+
-		t.Run(tt.name, func(t *testing.T) {
+	for _, test := range tests {
-			got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+		t.Run(test.name, func(t *testing.T) {
-			if (err != nil) != tt.wantErr {
+			matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
-				t.Errorf("PagerDutyApiKey.FromData() error = %v, wantErr %v", err, tt.wantErr)
+			if len(matchedDetectors) == 0 {
+				t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
 				return
 			}
-			for i := range got {
-				if len(got[i].Raw) == 0 {
-					t.Fatalf("no raw secret present: \n %+v", got[i])
-				}
-				if (got[i].VerificationError() != nil) != tt.wantVerificationErr {
-					t.Errorf("PagerDutyApiKey.FromData() verificationError = %v, wantVerificationErr %v", got[i].VerificationError(), tt.wantVerificationErr)
-				}
-			}
-			ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")
-			if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
-				t.Errorf("PagerDutyApiKey.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
-			}
-		})
-	}
-}
 
-func BenchmarkFromData(benchmark *testing.B) {
+			results, err := d.FromData(context.Background(), false, []byte(test.input))
-	ctx := context.Background()
+			if err != nil {
-	s := Scanner{}
+				t.Errorf("error = %v", err)
-	for name, data := range detectors.MustGetBenchmarkData() {
+				return
-		benchmark.Run(name, func(b *testing.B) {
+			}
-			b.ResetTimer()
+
-			for n := 0; n < b.N; n++ {
+			if len(results) != len(test.want) {
-				_, err := s.FromData(ctx, false, data)
+				if len(results) == 0 {
-				if err != nil {
+					t.Errorf("did not receive result")
-					b.Fatal(err)
+				} else {
+					t.Errorf("expected %d results, only received %d", len(test.want), len(results))
 				}
+				return
+			}
+
+			actual := make(map[string]struct{}, len(results))
+			for _, r := range results {
+				if len(r.RawV2) > 0 {
+					actual[string(r.RawV2)] = struct{}{}
+				} else {
+					actual[string(r.Raw)] = struct{}{}
+				}
+			}
+			expected := make(map[string]struct{}, len(test.want))
+			for _, v := range test.want {
+				expected[v] = struct{}{}
+			}
+
+			if diff := cmp.Diff(expected, actual); diff != "" {
+				t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
 			}
 		})
 	}

pkg/detectors/pandadoc/pandadoc_integration_test.go

@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package pandadoc
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/kylelemons/godebug/pretty"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestPandadoc_FromChunk(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	defer cancel()
+	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
+	if err != nil {
+		t.Fatalf("could not get test secrets from GCP: %s", err)
+	}
+	secret := testSecrets.MustGetField("PANDADOC")
+	inactiveSecret := testSecrets.MustGetField("PANDADOC_INACTIVE")
+
+	type args struct {
+		ctx    context.Context
+		data   []byte
+		verify bool
+	}
+	tests := []struct {
+		name    string
+		s       Scanner
+		args    args
+		want    []detectors.Result
+		wantErr bool
+	}{
+		{
+			name: "found, verified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a pandadoc secret %s within", secret)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Pandadoc,
+					Verified:     true,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "found, unverified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a pandadoc secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Pandadoc,
+					Verified:     false,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "not found",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte("You cannot find the secret within"),
+				verify: true,
+			},
+			want:    nil,
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			s := Scanner{}
+			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("Pandadoc.FromData() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			for i := range got {
+				if len(got[i].Raw) == 0 {
+					t.Fatalf("no raw secret present: \n %+v", got[i])
+				}
+				got[i].Raw = nil
+			}
+			if diff := pretty.Compare(got, tt.want); diff != "" {
+				t.Errorf("Pandadoc.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+			}
+		})
+	}
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+	ctx := context.Background()
+	s := Scanner{}
+	for name, data := range detectors.MustGetBenchmarkData() {
+		benchmark.Run(name, func(b *testing.B) {
+			b.ResetTimer()
+			for n := 0; n < b.N; n++ {
+				_, err := s.FromData(ctx, false, data)
+				if err != nil {
+					b.Fatal(err)
+				}
+			}
+		})
+	}
+}

pkg/detectors/pandadoc/pandadoc_test.go

@@ -1,119 +1,90 @@
-//go:build detectors
-// +build detectors
-
 package pandadoc
 
 import (
 	"context"
 	"fmt"
 	"testing"
-	"time"
 
-	"github.com/kylelemons/godebug/pretty"
+	"github.com/google/go-cmp/cmp"
+
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
-
+	"github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 )
 
-func TestPandadoc_FromChunk(t *testing.T) {
+var (
-	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	validPattern   = "gL1LKFp8p0q7ORUybiyovqIuGUsGjx4adxMRcHPh"
-	defer cancel()
+	invalidPattern = "gL1LKFp8p0q7ORUybiyo?qIuGUsGjx4adxMRcHPh"
-	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
+	keyword        = "pandadoc"
-	if err != nil {
+)
-		t.Fatalf("could not get test secrets from GCP: %s", err)
-	}
-	secret := testSecrets.MustGetField("PANDADOC")
-	inactiveSecret := testSecrets.MustGetField("PANDADOC_INACTIVE")
 
-	type args struct {
+func TestPandadoc_Pattern(t *testing.T) {
-		ctx    context.Context
+	d := Scanner{}
-		data   []byte
+	ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
-		verify bool
-	}
 	tests := []struct {
-		name    string
+		name  string
-		s       Scanner
+		input string
-		args    args
+		want  []string
-		want    []detectors.Result
-		wantErr bool
 	}{
 		{
-			name: "found, verified",
+			name:  "valid pattern - with keyword pandadoc",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
-			args: args{
+			want:  []string{validPattern},
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a pandadoc secret %s within", secret)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_Pandadoc,
-					Verified:     true,
-				},
-			},
-			wantErr: false,
 		},
 		{
-			name: "found, unverified",
+			name:  "valid pattern - ignore duplicate",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
-			args: args{
+			want:  []string{validPattern},
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a pandadoc secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_Pandadoc,
-					Verified:     false,
-				},
-			},
-			wantErr: false,
 		},
 		{
-			name: "not found",
+			name:  "valid pattern - key out of prefix range",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
-			args: args{
+			want:  []string{},
-				ctx:    context.Background(),
+		},
-				data:   []byte("You cannot find the secret within"),
+		{
-				verify: true,
+			name:  "invalid pattern",
-			},
+			input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
-			want:    nil,
+			want:  []string{},
-			wantErr: false,
 		},
 	}
-	for _, tt := range tests {
+
-		t.Run(tt.name, func(t *testing.T) {
+	for _, test := range tests {
-			s := Scanner{}
+		t.Run(test.name, func(t *testing.T) {
-			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
-			if (err != nil) != tt.wantErr {
+			if len(matchedDetectors) == 0 {
-				t.Errorf("Pandadoc.FromData() error = %v, wantErr %v", err, tt.wantErr)
+				t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
 				return
 			}
-			for i := range got {
-				if len(got[i].Raw) == 0 {
-					t.Fatalf("no raw secret present: \n %+v", got[i])
-				}
-				got[i].Raw = nil
-			}
-			if diff := pretty.Compare(got, tt.want); diff != "" {
-				t.Errorf("Pandadoc.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
-			}
-		})
-	}
-}
 
-func BenchmarkFromData(benchmark *testing.B) {
+			results, err := d.FromData(context.Background(), false, []byte(test.input))
-	ctx := context.Background()
+			if err != nil {
-	s := Scanner{}
+				t.Errorf("error = %v", err)
-	for name, data := range detectors.MustGetBenchmarkData() {
+				return
-		benchmark.Run(name, func(b *testing.B) {
+			}
-			b.ResetTimer()
+
-			for n := 0; n < b.N; n++ {
+			if len(results) != len(test.want) {
-				_, err := s.FromData(ctx, false, data)
+				if len(results) == 0 {
-				if err != nil {
+					t.Errorf("did not receive result")
-					b.Fatal(err)
+				} else {
+					t.Errorf("expected %d results, only received %d", len(test.want), len(results))
 				}
+				return
+			}
+
+			actual := make(map[string]struct{}, len(results))
+			for _, r := range results {
+				if len(r.RawV2) > 0 {
+					actual[string(r.RawV2)] = struct{}{}
+				} else {
+					actual[string(r.Raw)] = struct{}{}
+				}
+			}
+			expected := make(map[string]struct{}, len(test.want))
+			for _, v := range test.want {
+				expected[v] = struct{}{}
+			}
+
+			if diff := cmp.Diff(expected, actual); diff != "" {
+				t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
 			}
 		})
 	}

pkg/detectors/pandascore/pandascore_integration_test.go

@@ -0,0 +1,117 @@
+//go:build detectors
+// +build detectors
+
+package pandascore
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/kylelemons/godebug/pretty"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestPandaScore_FromChunk(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*10)
+	defer cancel()
+	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
+	if err != nil {
+		t.Fatalf("could not get test secrets from GCP: %s", err)
+	}
+	secret := testSecrets.MustGetField("PANDASCORE")
+	inactiveSecret := testSecrets.MustGetField("PANDASCORE_INACTIVE")
+
+	type args struct {
+		ctx    context.Context
+		data   []byte
+		verify bool
+	}
+	tests := []struct {
+		name    string
+		s       Scanner
+		args    args
+		want    []detectors.Result
+		wantErr bool
+	}{
+		{
+			name: "found, verified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a pandascore secret %s within", secret)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_PandaScore,
+					Verified:     true,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "found, unverified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a pandascore secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_PandaScore,
+					Verified:     false,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "not found",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte("You cannot find the secret within"),
+				verify: true,
+			},
+			want:    nil,
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			s := Scanner{}
+			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("PandaScore.FromData() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			for i := range got {
+				if len(got[i].Raw) == 0 {
+					t.Fatalf("no raw secret present: \n %+v", got[i])
+				}
+				got[i].Raw = nil
+			}
+			if diff := pretty.Compare(got, tt.want); diff != "" {
+				t.Errorf("PandaScore.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+			}
+		})
+	}
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+	ctx := context.Background()
+	s := Scanner{}
+	for name, data := range detectors.MustGetBenchmarkData() {
+		benchmark.Run(name, func(b *testing.B) {
+			b.ResetTimer()
+			for n := 0; n < b.N; n++ {
+				s.FromData(ctx, false, data)
+			}
+		})
+	}
+}

pkg/detectors/pandascore/pandascore_test.go

@@ -1,116 +1,90 @@
-//go:build detectors
-// +build detectors
-
 package pandascore
 
 import (
 	"context"
 	"fmt"
 	"testing"
-	"time"
 
-	"github.com/kylelemons/godebug/pretty"
+	"github.com/google/go-cmp/cmp"
 
-	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
 )
 
-func TestPandaScore_FromChunk(t *testing.T) {
+var (
-	ctx, cancel := context.WithTimeout(context.Background(), time.Second*10)
+	validPattern   = "iahNbqgnxq7tzca9hNBWiVexHeePoQ2nnXwN06wmaFSY5BMGyfq"
-	defer cancel()
+	invalidPattern = "iahNbqgnxq7tzca9hNBWiV?xHeePoQ2nnXwN06wmaFSY5BMGyfq"
-	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
+	keyword        = "pandascore"
-	if err != nil {
+)
-		t.Fatalf("could not get test secrets from GCP: %s", err)
-	}
-	secret := testSecrets.MustGetField("PANDASCORE")
-	inactiveSecret := testSecrets.MustGetField("PANDASCORE_INACTIVE")
 
-	type args struct {
+func TestPandaScore_Pattern(t *testing.T) {
-		ctx    context.Context
+	d := Scanner{}
-		data   []byte
+	ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
-		verify bool
-	}
 	tests := []struct {
-		name    string
+		name  string
-		s       Scanner
+		input string
-		args    args
+		want  []string
-		want    []detectors.Result
-		wantErr bool
 	}{
 		{
-			name: "found, verified",
+			name:  "valid pattern - with keyword pandascore",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s token = ' %s '", keyword, validPattern),
-			args: args{
+			want:  []string{validPattern},
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a pandascore secret %s within", secret)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_PandaScore,
-					Verified:     true,
-				},
-			},
-			wantErr: false,
 		},
 		{
-			name: "found, unverified",
+			name:  "valid pattern - ignore duplicate",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s token = ' %s ' | ' %s '", keyword, validPattern, validPattern),
-			args: args{
+			want:  []string{validPattern},
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a pandascore secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_PandaScore,
-					Verified:     false,
-				},
-			},
-			wantErr: false,
 		},
 		{
-			name: "not found",
+			name:  "valid pattern - key out of prefix range",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = ' %s '", keyword, validPattern),
-			args: args{
+			want:  []string{},
-				ctx:    context.Background(),
+		},
-				data:   []byte("You cannot find the secret within"),
+		{
-				verify: true,
+			name:  "invalid pattern",
-			},
+			input: fmt.Sprintf("%s = ' %s '", keyword, invalidPattern),
-			want:    nil,
+			want:  []string{},
-			wantErr: false,
 		},
 	}
-	for _, tt := range tests {
+
-		t.Run(tt.name, func(t *testing.T) {
+	for _, test := range tests {
-			s := Scanner{}
+		t.Run(test.name, func(t *testing.T) {
-			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
-			if (err != nil) != tt.wantErr {
+			if len(matchedDetectors) == 0 {
-				t.Errorf("PandaScore.FromData() error = %v, wantErr %v", err, tt.wantErr)
+				t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
 				return
 			}
-			for i := range got {
-				if len(got[i].Raw) == 0 {
-					t.Fatalf("no raw secret present: \n %+v", got[i])
-				}
-				got[i].Raw = nil
-			}
-			if diff := pretty.Compare(got, tt.want); diff != "" {
-				t.Errorf("PandaScore.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
-			}
-		})
-	}
-}
 
-func BenchmarkFromData(benchmark *testing.B) {
+			results, err := d.FromData(context.Background(), false, []byte(test.input))
-	ctx := context.Background()
+			if err != nil {
-	s := Scanner{}
+				t.Errorf("error = %v", err)
-	for name, data := range detectors.MustGetBenchmarkData() {
+				return
-		benchmark.Run(name, func(b *testing.B) {
+			}
-			b.ResetTimer()
+
-			for n := 0; n < b.N; n++ {
+			if len(results) != len(test.want) {
-				s.FromData(ctx, false, data)
+				if len(results) == 0 {
+					t.Errorf("did not receive result")
+				} else {
+					t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+				}
+				return
+			}
+
+			actual := make(map[string]struct{}, len(results))
+			for _, r := range results {
+				if len(r.RawV2) > 0 {
+					actual[string(r.RawV2)] = struct{}{}
+				} else {
+					actual[string(r.Raw)] = struct{}{}
+				}
+			}
+			expected := make(map[string]struct{}, len(test.want))
+			for _, v := range test.want {
+				expected[v] = struct{}{}
+			}
+
+			if diff := cmp.Diff(expected, actual); diff != "" {
+				t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
 			}
 		})
 	}

pkg/detectors/paperform/paperform_integration_test.go

@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package paperform
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/kylelemons/godebug/pretty"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestPaperform_FromChunk(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	defer cancel()
+	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
+	if err != nil {
+		t.Fatalf("could not get test secrets from GCP: %s", err)
+	}
+	secret := testSecrets.MustGetField("PAPERFORM")
+	inactiveSecret := testSecrets.MustGetField("PAPERFORM_INACTIVE")
+
+	type args struct {
+		ctx    context.Context
+		data   []byte
+		verify bool
+	}
+	tests := []struct {
+		name    string
+		s       Scanner
+		args    args
+		want    []detectors.Result
+		wantErr bool
+	}{
+		{
+			name: "found, verified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a paperform secret %s within", secret)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Paperform,
+					Verified:     true,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "found, unverified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a paperform secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Paperform,
+					Verified:     false,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "not found",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte("You cannot find the secret within"),
+				verify: true,
+			},
+			want:    nil,
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			s := Scanner{}
+			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("Paperform.FromData() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			for i := range got {
+				if len(got[i].Raw) == 0 {
+					t.Fatalf("no raw secret present: \n %+v", got[i])
+				}
+				got[i].Raw = nil
+			}
+			if diff := pretty.Compare(got, tt.want); diff != "" {
+				t.Errorf("Paperform.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+			}
+		})
+	}
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+	ctx := context.Background()
+	s := Scanner{}
+	for name, data := range detectors.MustGetBenchmarkData() {
+		benchmark.Run(name, func(b *testing.B) {
+			b.ResetTimer()
+			for n := 0; n < b.N; n++ {
+				_, err := s.FromData(ctx, false, data)
+				if err != nil {
+					b.Fatal(err)
+				}
+			}
+		})
+	}
+}

pkg/detectors/paperform/paperform_test.go

@@ -1,119 +1,90 @@
-//go:build detectors
-// +build detectors
-
 package paperform
 
 import (
 	"context"
 	"fmt"
 	"testing"
-	"time"
 
-	"github.com/kylelemons/godebug/pretty"
+	"github.com/google/go-cmp/cmp"
+
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
-
+	"github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 )
 
-func TestPaperform_FromChunk(t *testing.T) {
+var (
-	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	validPattern   = "NvBbE5vYVPhidc-tiZ7E6DvP.z4bAQj29KEZOCv_XEl1pIn.Y1q8JoeGNoSelGj.db1iQpLM0fsype86LD.Vk.p6yweF5A2MlfxXDrEd7nz2SBExnTpp4QRN94pBAeulPLLtzqTz--y_UuF1g6cjE_.kuW_u5He0QkBdLMkyAgSx94N3Csj9LY37XOmUp9IIi9LXpTvZGa8oywp6JuMfhzwg5OCEvdp9mx.UyfQcnnJtYzzP5dItmwfEC-KJIvjvS8LF2NU2w2japQHtnAJqBAn3_EP-FN78wHnDEWANANT0cfor6kDqyKraO0Y-26PdB6xBjm3_VpU.8hnKIyoKdLQ6S.HZwr5rx0Bx76zXTCBv4uEzhtDFcDqVPN8ZG_kE90P..ldReG0jU4w3YA2jbaOgi6i-8llYWGoCiFFBm3Od-zLOEDYL2BlGsUFRUkiEMjytCVDqcIOdfPT7GQcd3wdmort6FFv8SbCu95f2gBCcM.5.ZmMxIOybubMGmiRunYM8-pSaVvXfBQSkM2Eygh15tkKCDHf8X3InAkPh7HQn13mP5y1gFRLsVAUWb-91PeHASP6hluUEdsX3uLQ9OJFenKrk.0zS9Goy08bfttd4h4Jtb2JV8vbJ8-3Wb4AJWqf0eUALMxOChB3sSBKW37s4vDb1NKOnoqOeoYQUBijqRGu9YLKIAimwo7Uvl0CuD7bWNrERweBqNVWjfGhlE8Yvvklm5YhCk5XY02pOa3IjMf_TDKhbTr8bh_20SXevnDk80XKg_3mWbhuieL23kx835AokAg9JpEkgydBBqo8nzQg23R5xJzKRT64kgb5GBlzuM9Oxh7pXsmzlYf"
-	defer cancel()
+	invalidPattern = "NvBbE5vYVPhidc?tiZ7E6DvP.z4bAQj29KEZOCv_XEl1pIn.Y1q8JoeGNoSelGj.db1iQpLM0fsype86LD.Vk.p6yweF5A2MlfxXDrEd7nz2SBExnTpp4QRN94pBAeulPLLtzqTz--y_UuF1g6cjE_.kuW_u5He0QkBdLMkyAgSx94N3Csj9LY37XOmUp9IIi9LXpTvZGa8oywp6JuMfhzwg5OCEvdp9mx.UyfQcnnJtYzzP5dItmwfEC-KJIvjvS8LF2NU2w2japQHtnAJqBAn3_EP-FN78wHnDEWANANT0cfor6kDqyKraO0Y-26PdB6xBjm3_VpU.8hnKIyoKdLQ6S.HZwr5rx0Bx76zXTCBv4uEzhtDFcDqVPN8ZG_kE90P..ldReG0jU4w3YA2jbaOgi6i-8llYWGoCiFFBm3Od-zLOEDYL2BlGsUFRUkiEMjytCVDqcIOdfPT7GQcd3wdmort6FFv8SbCu95f2gBCcM.5.ZmMxIOybubMGmiRunYM8-pSaVvXfBQSkM2Eygh15tkKCDHf8X3InAkPh7HQn13mP5y1gFRLsVAUWb-91PeHASP6hluUEdsX3uLQ9OJFenKrk.0zS9Goy08bfttd4h4Jtb2JV8vbJ8-3Wb4AJWqf0eUALMxOChB3sSBKW37s4vDb1NKOnoqOeoYQUBijqRGu9YLKIAimwo7Uvl0CuD7bWNrERweBqNVWjfGhlE8Yvvklm5YhCk5XY02pOa3IjMf_TDKhbTr8bh_20SXevnDk80XKg_3mWbhuieL23kx835AokAg9JpEkgydBBqo8nzQg23R5xJzKRT64kgb5GBlzuM9Oxh7pXsmzlYf"
-	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
+	keyword        = "paperform"
-	if err != nil {
+)
-		t.Fatalf("could not get test secrets from GCP: %s", err)
-	}
-	secret := testSecrets.MustGetField("PAPERFORM")
-	inactiveSecret := testSecrets.MustGetField("PAPERFORM_INACTIVE")
 
-	type args struct {
+func TestPaperform_Pattern(t *testing.T) {
-		ctx    context.Context
+	d := Scanner{}
-		data   []byte
+	ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
-		verify bool
-	}
 	tests := []struct {
-		name    string
+		name  string
-		s       Scanner
+		input string
-		args    args
+		want  []string
-		want    []detectors.Result
-		wantErr bool
 	}{
 		{
-			name: "found, verified",
+			name:  "valid pattern - with keyword paperform",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
-			args: args{
+			want:  []string{validPattern},
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a paperform secret %s within", secret)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_Paperform,
-					Verified:     true,
-				},
-			},
-			wantErr: false,
 		},
 		{
-			name: "found, unverified",
+			name:  "valid pattern - ignore duplicate",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
-			args: args{
+			want:  []string{validPattern},
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a paperform secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_Paperform,
-					Verified:     false,
-				},
-			},
-			wantErr: false,
 		},
 		{
-			name: "not found",
+			name:  "valid pattern - key out of prefix range",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
-			args: args{
+			want:  []string{},
-				ctx:    context.Background(),
+		},
-				data:   []byte("You cannot find the secret within"),
+		{
-				verify: true,
+			name:  "invalid pattern",
-			},
+			input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
-			want:    nil,
+			want:  []string{},
-			wantErr: false,
 		},
 	}
-	for _, tt := range tests {
+
-		t.Run(tt.name, func(t *testing.T) {
+	for _, test := range tests {
-			s := Scanner{}
+		t.Run(test.name, func(t *testing.T) {
-			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
-			if (err != nil) != tt.wantErr {
+			if len(matchedDetectors) == 0 {
-				t.Errorf("Paperform.FromData() error = %v, wantErr %v", err, tt.wantErr)
+				t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
 				return
 			}
-			for i := range got {
-				if len(got[i].Raw) == 0 {
-					t.Fatalf("no raw secret present: \n %+v", got[i])
-				}
-				got[i].Raw = nil
-			}
-			if diff := pretty.Compare(got, tt.want); diff != "" {
-				t.Errorf("Paperform.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
-			}
-		})
-	}
-}
 
-func BenchmarkFromData(benchmark *testing.B) {
+			results, err := d.FromData(context.Background(), false, []byte(test.input))
-	ctx := context.Background()
+			if err != nil {
-	s := Scanner{}
+				t.Errorf("error = %v", err)
-	for name, data := range detectors.MustGetBenchmarkData() {
+				return
-		benchmark.Run(name, func(b *testing.B) {
+			}
-			b.ResetTimer()
+
-			for n := 0; n < b.N; n++ {
+			if len(results) != len(test.want) {
-				_, err := s.FromData(ctx, false, data)
+				if len(results) == 0 {
-				if err != nil {
+					t.Errorf("did not receive result")
-					b.Fatal(err)
+				} else {
+					t.Errorf("expected %d results, only received %d", len(test.want), len(results))
 				}
+				return
+			}
+
+			actual := make(map[string]struct{}, len(results))
+			for _, r := range results {
+				if len(r.RawV2) > 0 {
+					actual[string(r.RawV2)] = struct{}{}
+				} else {
+					actual[string(r.Raw)] = struct{}{}
+				}
+			}
+			expected := make(map[string]struct{}, len(test.want))
+			for _, v := range test.want {
+				expected[v] = struct{}{}
+			}
+
+			if diff := cmp.Diff(expected, actual); diff != "" {
+				t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
 			}
 		})
 	}

pkg/detectors/paralleldots/paralleldots_integration_test.go

@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package paralleldots
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/kylelemons/godebug/pretty"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestParalleldots_FromChunk(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	defer cancel()
+	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
+	if err != nil {
+		t.Fatalf("could not get test secrets from GCP: %s", err)
+	}
+	secret := testSecrets.MustGetField("PARALLELDOTS")
+	inactiveSecret := testSecrets.MustGetField("PARALLELDOTS_INACTIVE")
+
+	type args struct {
+		ctx    context.Context
+		data   []byte
+		verify bool
+	}
+	tests := []struct {
+		name    string
+		s       Scanner
+		args    args
+		want    []detectors.Result
+		wantErr bool
+	}{
+		{
+			name: "found, verified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a paralleldots secret %s within", secret)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_ParallelDots,
+					Verified:     true,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "found, unverified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a paralleldots secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_ParallelDots,
+					Verified:     false,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "not found",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte("You cannot find the secret within"),
+				verify: true,
+			},
+			want:    nil,
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			s := Scanner{}
+			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("Paralleldots.FromData() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			for i := range got {
+				if len(got[i].Raw) == 0 {
+					t.Fatalf("no raw secret present: \n %+v", got[i])
+				}
+				got[i].Raw = nil
+			}
+			if diff := pretty.Compare(got, tt.want); diff != "" {
+				t.Errorf("Paralleldots.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+			}
+		})
+	}
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+	ctx := context.Background()
+	s := Scanner{}
+	for name, data := range detectors.MustGetBenchmarkData() {
+		benchmark.Run(name, func(b *testing.B) {
+			b.ResetTimer()
+			for n := 0; n < b.N; n++ {
+				_, err := s.FromData(ctx, false, data)
+				if err != nil {
+					b.Fatal(err)
+				}
+			}
+		})
+	}
+}

pkg/detectors/paralleldots/paralleldots_test.go

@@ -1,119 +1,90 @@
-//go:build detectors
-// +build detectors
-
 package paralleldots
 
 import (
 	"context"
 	"fmt"
 	"testing"
-	"time"
 
-	"github.com/kylelemons/godebug/pretty"
+	"github.com/google/go-cmp/cmp"
+
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
-
+	"github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 )
 
-func TestParalleldots_FromChunk(t *testing.T) {
+var (
-	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	validPattern   = "Ki82DlwEBI9dqr4k3cZPH6Z5fP39XgEPktFfgZTM4mW"
-	defer cancel()
+	invalidPattern = "Ki82DlwEBI9dqr4k3cZPH?Z5fP39XgEPktFfgZTM4mW"
-	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
+	keyword        = "paralleldots"
-	if err != nil {
+)
-		t.Fatalf("could not get test secrets from GCP: %s", err)
-	}
-	secret := testSecrets.MustGetField("PARALLELDOTS")
-	inactiveSecret := testSecrets.MustGetField("PARALLELDOTS_INACTIVE")
 
-	type args struct {
+func TestParalleldots_Pattern(t *testing.T) {
-		ctx    context.Context
+	d := Scanner{}
-		data   []byte
+	ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
-		verify bool
-	}
 	tests := []struct {
-		name    string
+		name  string
-		s       Scanner
+		input string
-		args    args
+		want  []string
-		want    []detectors.Result
-		wantErr bool
 	}{
 		{
-			name: "found, verified",
+			name:  "valid pattern - with keyword paralleldots",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
-			args: args{
+			want:  []string{validPattern},
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a paralleldots secret %s within", secret)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_ParallelDots,
-					Verified:     true,
-				},
-			},
-			wantErr: false,
 		},
 		{
-			name: "found, unverified",
+			name:  "valid pattern - ignore duplicate",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
-			args: args{
+			want:  []string{validPattern},
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a paralleldots secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_ParallelDots,
-					Verified:     false,
-				},
-			},
-			wantErr: false,
 		},
 		{
-			name: "not found",
+			name:  "valid pattern - key out of prefix range",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
-			args: args{
+			want:  []string{},
-				ctx:    context.Background(),
+		},
-				data:   []byte("You cannot find the secret within"),
+		{
-				verify: true,
+			name:  "invalid pattern",
-			},
+			input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
-			want:    nil,
+			want:  []string{},
-			wantErr: false,
 		},
 	}
-	for _, tt := range tests {
+
-		t.Run(tt.name, func(t *testing.T) {
+	for _, test := range tests {
-			s := Scanner{}
+		t.Run(test.name, func(t *testing.T) {
-			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
-			if (err != nil) != tt.wantErr {
+			if len(matchedDetectors) == 0 {
-				t.Errorf("Paralleldots.FromData() error = %v, wantErr %v", err, tt.wantErr)
+				t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
 				return
 			}
-			for i := range got {
-				if len(got[i].Raw) == 0 {
-					t.Fatalf("no raw secret present: \n %+v", got[i])
-				}
-				got[i].Raw = nil
-			}
-			if diff := pretty.Compare(got, tt.want); diff != "" {
-				t.Errorf("Paralleldots.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
-			}
-		})
-	}
-}
 
-func BenchmarkFromData(benchmark *testing.B) {
+			results, err := d.FromData(context.Background(), false, []byte(test.input))
-	ctx := context.Background()
+			if err != nil {
-	s := Scanner{}
+				t.Errorf("error = %v", err)
-	for name, data := range detectors.MustGetBenchmarkData() {
+				return
-		benchmark.Run(name, func(b *testing.B) {
+			}
-			b.ResetTimer()
+
-			for n := 0; n < b.N; n++ {
+			if len(results) != len(test.want) {
-				_, err := s.FromData(ctx, false, data)
+				if len(results) == 0 {
-				if err != nil {
+					t.Errorf("did not receive result")
-					b.Fatal(err)
+				} else {
+					t.Errorf("expected %d results, only received %d", len(test.want), len(results))
 				}
+				return
+			}
+
+			actual := make(map[string]struct{}, len(results))
+			for _, r := range results {
+				if len(r.RawV2) > 0 {
+					actual[string(r.RawV2)] = struct{}{}
+				} else {
+					actual[string(r.Raw)] = struct{}{}
+				}
+			}
+			expected := make(map[string]struct{}, len(test.want))
+			for _, v := range test.want {
+				expected[v] = struct{}{}
+			}
+
+			if diff := cmp.Diff(expected, actual); diff != "" {
+				t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
 			}
 		})
 	}

pkg/detectors/parsehub/parsehub_integration_test.go

@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package parsehub
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/kylelemons/godebug/pretty"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestParsehub_FromChunk(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	defer cancel()
+	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+	if err != nil {
+		t.Fatalf("could not get test secrets from GCP: %s", err)
+	}
+	secret := testSecrets.MustGetField("PARSEHUB")
+	inactiveSecret := testSecrets.MustGetField("PARSEHUB_INACTIVE")
+
+	type args struct {
+		ctx    context.Context
+		data   []byte
+		verify bool
+	}
+	tests := []struct {
+		name    string
+		s       Scanner
+		args    args
+		want    []detectors.Result
+		wantErr bool
+	}{
+		{
+			name: "found, verified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a parsehub secret %s within", secret)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Parsehub,
+					Verified:     true,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "found, unverified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a parsehub secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Parsehub,
+					Verified:     false,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "not found",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte("You cannot find the secret within"),
+				verify: true,
+			},
+			want:    nil,
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			s := Scanner{}
+			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("Parsehub.FromData() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			for i := range got {
+				if len(got[i].Raw) == 0 {
+					t.Fatalf("no raw secret present: \n %+v", got[i])
+				}
+				got[i].Raw = nil
+			}
+			if diff := pretty.Compare(got, tt.want); diff != "" {
+				t.Errorf("Parsehub.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+			}
+		})
+	}
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+	ctx := context.Background()
+	s := Scanner{}
+	for name, data := range detectors.MustGetBenchmarkData() {
+		benchmark.Run(name, func(b *testing.B) {
+			b.ResetTimer()
+			for n := 0; n < b.N; n++ {
+				_, err := s.FromData(ctx, false, data)
+				if err != nil {
+					b.Fatal(err)
+				}
+			}
+		})
+	}
+}

pkg/detectors/parsehub/parsehub_test.go

@@ -1,119 +1,90 @@
-//go:build detectors
-// +build detectors
-
 package parsehub
 
 import (
 	"context"
 	"fmt"
 	"testing"
-	"time"
 
-	"github.com/kylelemons/godebug/pretty"
+	"github.com/google/go-cmp/cmp"
+
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
-
+	"github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 )
 
-func TestParsehub_FromChunk(t *testing.T) {
+var (
-	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	validPattern   = "vZrEclFwOqeA"
-	defer cancel()
+	invalidPattern = "vZrEcl?wOqeA"
-	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+	keyword        = "parsehub"
-	if err != nil {
+)
-		t.Fatalf("could not get test secrets from GCP: %s", err)
-	}
-	secret := testSecrets.MustGetField("PARSEHUB")
-	inactiveSecret := testSecrets.MustGetField("PARSEHUB_INACTIVE")
 
-	type args struct {
+func TestParsehub_Pattern(t *testing.T) {
-		ctx    context.Context
+	d := Scanner{}
-		data   []byte
+	ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
-		verify bool
-	}
 	tests := []struct {
-		name    string
+		name  string
-		s       Scanner
+		input string
-		args    args
+		want  []string
-		want    []detectors.Result
-		wantErr bool
 	}{
 		{
-			name: "found, verified",
+			name:  "valid pattern - with keyword parsehub",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
-			args: args{
+			want:  []string{validPattern},
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a parsehub secret %s within", secret)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_Parsehub,
-					Verified:     true,
-				},
-			},
-			wantErr: false,
 		},
 		{
-			name: "found, unverified",
+			name:  "valid pattern - ignore duplicate",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
-			args: args{
+			want:  []string{validPattern},
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a parsehub secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_Parsehub,
-					Verified:     false,
-				},
-			},
-			wantErr: false,
 		},
 		{
-			name: "not found",
+			name:  "valid pattern - key out of prefix range",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
-			args: args{
+			want:  []string{},
-				ctx:    context.Background(),
+		},
-				data:   []byte("You cannot find the secret within"),
+		{
-				verify: true,
+			name:  "invalid pattern",
-			},
+			input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
-			want:    nil,
+			want:  []string{},
-			wantErr: false,
 		},
 	}
-	for _, tt := range tests {
+
-		t.Run(tt.name, func(t *testing.T) {
+	for _, test := range tests {
-			s := Scanner{}
+		t.Run(test.name, func(t *testing.T) {
-			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
-			if (err != nil) != tt.wantErr {
+			if len(matchedDetectors) == 0 {
-				t.Errorf("Parsehub.FromData() error = %v, wantErr %v", err, tt.wantErr)
+				t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
 				return
 			}
-			for i := range got {
-				if len(got[i].Raw) == 0 {
-					t.Fatalf("no raw secret present: \n %+v", got[i])
-				}
-				got[i].Raw = nil
-			}
-			if diff := pretty.Compare(got, tt.want); diff != "" {
-				t.Errorf("Parsehub.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
-			}
-		})
-	}
-}
 
-func BenchmarkFromData(benchmark *testing.B) {
+			results, err := d.FromData(context.Background(), false, []byte(test.input))
-	ctx := context.Background()
+			if err != nil {
-	s := Scanner{}
+				t.Errorf("error = %v", err)
-	for name, data := range detectors.MustGetBenchmarkData() {
+				return
-		benchmark.Run(name, func(b *testing.B) {
+			}
-			b.ResetTimer()
+
-			for n := 0; n < b.N; n++ {
+			if len(results) != len(test.want) {
-				_, err := s.FromData(ctx, false, data)
+				if len(results) == 0 {
-				if err != nil {
+					t.Errorf("did not receive result")
-					b.Fatal(err)
+				} else {
+					t.Errorf("expected %d results, only received %d", len(test.want), len(results))
 				}
+				return
+			}
+
+			actual := make(map[string]struct{}, len(results))
+			for _, r := range results {
+				if len(r.RawV2) > 0 {
+					actual[string(r.RawV2)] = struct{}{}
+				} else {
+					actual[string(r.Raw)] = struct{}{}
+				}
+			}
+			expected := make(map[string]struct{}, len(test.want))
+			for _, v := range test.want {
+				expected[v] = struct{}{}
+			}
+
+			if diff := cmp.Diff(expected, actual); diff != "" {
+				t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
 			}
 		})
 	}

pkg/detectors/parsers/parsers_integration_test.go

@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package parsers
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/kylelemons/godebug/pretty"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestParsers_FromChunk(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	defer cancel()
+	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+	if err != nil {
+		t.Fatalf("could not get test secrets from GCP: %s", err)
+	}
+	secret := testSecrets.MustGetField("PARSERS")
+	inactiveSecret := testSecrets.MustGetField("PARSERS_INACTIVE")
+
+	type args struct {
+		ctx    context.Context
+		data   []byte
+		verify bool
+	}
+	tests := []struct {
+		name    string
+		s       Scanner
+		args    args
+		want    []detectors.Result
+		wantErr bool
+	}{
+		{
+			name: "found, verified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a parsers secret %s within", secret)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Parsers,
+					Verified:     true,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "found, unverified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a parsers secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Parsers,
+					Verified:     false,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "not found",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte("You cannot find the secret within"),
+				verify: true,
+			},
+			want:    nil,
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			s := Scanner{}
+			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("Parsers.FromData() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			for i := range got {
+				if len(got[i].Raw) == 0 {
+					t.Fatalf("no raw secret present: \n %+v", got[i])
+				}
+				got[i].Raw = nil
+			}
+			if diff := pretty.Compare(got, tt.want); diff != "" {
+				t.Errorf("Parsers.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+			}
+		})
+	}
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+	ctx := context.Background()
+	s := Scanner{}
+	for name, data := range detectors.MustGetBenchmarkData() {
+		benchmark.Run(name, func(b *testing.B) {
+			b.ResetTimer()
+			for n := 0; n < b.N; n++ {
+				_, err := s.FromData(ctx, false, data)
+				if err != nil {
+					b.Fatal(err)
+				}
+			}
+		})
+	}
+}

pkg/detectors/parsers/parsers_test.go

@@ -1,119 +1,90 @@
-//go:build detectors
-// +build detectors
-
 package parsers
 
 import (
 	"context"
 	"fmt"
 	"testing"
-	"time"
 
-	"github.com/kylelemons/godebug/pretty"
+	"github.com/google/go-cmp/cmp"
+
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
-
+	"github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 )
 
-func TestParsers_FromChunk(t *testing.T) {
+var (
-	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	validPattern   = "3ui8k045w9b9n6c2fuqbr44p1fg64cjsvn2dv2uvkyxfmjler9ddsls6uqtizt7q"
-	defer cancel()
+	invalidPattern = "3ui8k045w9b9n6c2fuqbr44p1fg64cjs?n2dv2uvkyxfmjler9ddsls6uqtizt7q"
-	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+	keyword        = "parsers"
-	if err != nil {
+)
-		t.Fatalf("could not get test secrets from GCP: %s", err)
-	}
-	secret := testSecrets.MustGetField("PARSERS")
-	inactiveSecret := testSecrets.MustGetField("PARSERS_INACTIVE")
 
-	type args struct {
+func TestParsers_Pattern(t *testing.T) {
-		ctx    context.Context
+	d := Scanner{}
-		data   []byte
+	ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
-		verify bool
-	}
 	tests := []struct {
-		name    string
+		name  string
-		s       Scanner
+		input string
-		args    args
+		want  []string
-		want    []detectors.Result
-		wantErr bool
 	}{
 		{
-			name: "found, verified",
+			name:  "valid pattern - with keyword parsers",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
-			args: args{
+			want:  []string{validPattern},
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a parsers secret %s within", secret)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_Parsers,
-					Verified:     true,
-				},
-			},
-			wantErr: false,
 		},
 		{
-			name: "found, unverified",
+			name:  "valid pattern - ignore duplicate",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
-			args: args{
+			want:  []string{validPattern},
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a parsers secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_Parsers,
-					Verified:     false,
-				},
-			},
-			wantErr: false,
 		},
 		{
-			name: "not found",
+			name:  "valid pattern - key out of prefix range",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
-			args: args{
+			want:  []string{},
-				ctx:    context.Background(),
+		},
-				data:   []byte("You cannot find the secret within"),
+		{
-				verify: true,
+			name:  "invalid pattern",
-			},
+			input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
-			want:    nil,
+			want:  []string{},
-			wantErr: false,
 		},
 	}
-	for _, tt := range tests {
+
-		t.Run(tt.name, func(t *testing.T) {
+	for _, test := range tests {
-			s := Scanner{}
+		t.Run(test.name, func(t *testing.T) {
-			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
-			if (err != nil) != tt.wantErr {
+			if len(matchedDetectors) == 0 {
-				t.Errorf("Parsers.FromData() error = %v, wantErr %v", err, tt.wantErr)
+				t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
 				return
 			}
-			for i := range got {
-				if len(got[i].Raw) == 0 {
-					t.Fatalf("no raw secret present: \n %+v", got[i])
-				}
-				got[i].Raw = nil
-			}
-			if diff := pretty.Compare(got, tt.want); diff != "" {
-				t.Errorf("Parsers.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
-			}
-		})
-	}
-}
 
-func BenchmarkFromData(benchmark *testing.B) {
+			results, err := d.FromData(context.Background(), false, []byte(test.input))
-	ctx := context.Background()
+			if err != nil {
-	s := Scanner{}
+				t.Errorf("error = %v", err)
-	for name, data := range detectors.MustGetBenchmarkData() {
+				return
-		benchmark.Run(name, func(b *testing.B) {
+			}
-			b.ResetTimer()
+
-			for n := 0; n < b.N; n++ {
+			if len(results) != len(test.want) {
-				_, err := s.FromData(ctx, false, data)
+				if len(results) == 0 {
-				if err != nil {
+					t.Errorf("did not receive result")
-					b.Fatal(err)
+				} else {
+					t.Errorf("expected %d results, only received %d", len(test.want), len(results))
 				}
+				return
+			}
+
+			actual := make(map[string]struct{}, len(results))
+			for _, r := range results {
+				if len(r.RawV2) > 0 {
+					actual[string(r.RawV2)] = struct{}{}
+				} else {
+					actual[string(r.Raw)] = struct{}{}
+				}
+			}
+			expected := make(map[string]struct{}, len(test.want))
+			for _, v := range test.want {
+				expected[v] = struct{}{}
+			}
+
+			if diff := cmp.Diff(expected, actual); diff != "" {
+				t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
 			}
 		})
 	}

pkg/detectors/partnerstack/partnerstack_integration_test.go

@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package partnerstack
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/kylelemons/godebug/pretty"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestPartnerstack_FromChunk(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	defer cancel()
+	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
+	if err != nil {
+		t.Fatalf("could not get test secrets from GCP: %s", err)
+	}
+	secret := testSecrets.MustGetField("PARTNERSTACK")
+	inactiveSecret := testSecrets.MustGetField("PARTNERSTACK_INACTIVE")
+
+	type args struct {
+		ctx    context.Context
+		data   []byte
+		verify bool
+	}
+	tests := []struct {
+		name    string
+		s       Scanner
+		args    args
+		want    []detectors.Result
+		wantErr bool
+	}{
+		{
+			name: "found, verified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a partnerstack secret %s within", secret)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Partnerstack,
+					Verified:     true,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "found, unverified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a partnerstack secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Partnerstack,
+					Verified:     false,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "not found",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte("You cannot find the secret within"),
+				verify: true,
+			},
+			want:    nil,
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			s := Scanner{}
+			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("Partnerstack.FromData() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			for i := range got {
+				if len(got[i].Raw) == 0 {
+					t.Fatalf("no raw secret present: \n %+v", got[i])
+				}
+				got[i].Raw = nil
+			}
+			if diff := pretty.Compare(got, tt.want); diff != "" {
+				t.Errorf("Partnerstack.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+			}
+		})
+	}
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+	ctx := context.Background()
+	s := Scanner{}
+	for name, data := range detectors.MustGetBenchmarkData() {
+		benchmark.Run(name, func(b *testing.B) {
+			b.ResetTimer()
+			for n := 0; n < b.N; n++ {
+				_, err := s.FromData(ctx, false, data)
+				if err != nil {
+					b.Fatal(err)
+				}
+			}
+		})
+	}
+}

pkg/detectors/partnerstack/partnerstack_test.go

@@ -1,119 +1,90 @@
-//go:build detectors
-// +build detectors
-
 package partnerstack
 
 import (
 	"context"
 	"fmt"
 	"testing"
-	"time"
 
-	"github.com/kylelemons/godebug/pretty"
+	"github.com/google/go-cmp/cmp"
+
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
-
+	"github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 )
 
-func TestPartnerstack_FromChunk(t *testing.T) {
+var (
-	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	validPattern   = "mDrW4HFb6fVh2ii74V41bh6EH52sWirlQuKr3svxfgGueyj32HW7OhIhNLZSgicW"
-	defer cancel()
+	invalidPattern = "mDrW4HFb6fVh2ii74V41bh6EH52sWirl?uKr3svxfgGueyj32HW7OhIhNLZSgicW"
-	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
+	keyword        = "partnerstack"
-	if err != nil {
+)
-		t.Fatalf("could not get test secrets from GCP: %s", err)
-	}
-	secret := testSecrets.MustGetField("PARTNERSTACK")
-	inactiveSecret := testSecrets.MustGetField("PARTNERSTACK_INACTIVE")
 
-	type args struct {
+func TestPartnerstack_Pattern(t *testing.T) {
-		ctx    context.Context
+	d := Scanner{}
-		data   []byte
+	ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
-		verify bool
-	}
 	tests := []struct {
-		name    string
+		name  string
-		s       Scanner
+		input string
-		args    args
+		want  []string
-		want    []detectors.Result
-		wantErr bool
 	}{
 		{
-			name: "found, verified",
+			name:  "valid pattern - with keyword partnerstack",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
-			args: args{
+			want:  []string{validPattern},
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a partnerstack secret %s within", secret)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_Partnerstack,
-					Verified:     true,
-				},
-			},
-			wantErr: false,
 		},
 		{
-			name: "found, unverified",
+			name:  "valid pattern - ignore duplicate",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
-			args: args{
+			want:  []string{validPattern},
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a partnerstack secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_Partnerstack,
-					Verified:     false,
-				},
-			},
-			wantErr: false,
 		},
 		{
-			name: "not found",
+			name:  "valid pattern - key out of prefix range",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
-			args: args{
+			want:  []string{},
-				ctx:    context.Background(),
+		},
-				data:   []byte("You cannot find the secret within"),
+		{
-				verify: true,
+			name:  "invalid pattern",
-			},
+			input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
-			want:    nil,
+			want:  []string{},
-			wantErr: false,
 		},
 	}
-	for _, tt := range tests {
+
-		t.Run(tt.name, func(t *testing.T) {
+	for _, test := range tests {
-			s := Scanner{}
+		t.Run(test.name, func(t *testing.T) {
-			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
-			if (err != nil) != tt.wantErr {
+			if len(matchedDetectors) == 0 {
-				t.Errorf("Partnerstack.FromData() error = %v, wantErr %v", err, tt.wantErr)
+				t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
 				return
 			}
-			for i := range got {
-				if len(got[i].Raw) == 0 {
-					t.Fatalf("no raw secret present: \n %+v", got[i])
-				}
-				got[i].Raw = nil
-			}
-			if diff := pretty.Compare(got, tt.want); diff != "" {
-				t.Errorf("Partnerstack.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
-			}
-		})
-	}
-}
 
-func BenchmarkFromData(benchmark *testing.B) {
+			results, err := d.FromData(context.Background(), false, []byte(test.input))
-	ctx := context.Background()
+			if err != nil {
-	s := Scanner{}
+				t.Errorf("error = %v", err)
-	for name, data := range detectors.MustGetBenchmarkData() {
+				return
-		benchmark.Run(name, func(b *testing.B) {
+			}
-			b.ResetTimer()
+
-			for n := 0; n < b.N; n++ {
+			if len(results) != len(test.want) {
-				_, err := s.FromData(ctx, false, data)
+				if len(results) == 0 {
-				if err != nil {
+					t.Errorf("did not receive result")
-					b.Fatal(err)
+				} else {
+					t.Errorf("expected %d results, only received %d", len(test.want), len(results))
 				}
+				return
+			}
+
+			actual := make(map[string]struct{}, len(results))
+			for _, r := range results {
+				if len(r.RawV2) > 0 {
+					actual[string(r.RawV2)] = struct{}{}
+				} else {
+					actual[string(r.Raw)] = struct{}{}
+				}
+			}
+			expected := make(map[string]struct{}, len(test.want))
+			for _, v := range test.want {
+				expected[v] = struct{}{}
+			}
+
+			if diff := cmp.Diff(expected, actual); diff != "" {
+				t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
 			}
 		})
 	}

pkg/detectors/pastebin/pastebin_integration_test.go

@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package pastebin
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/kylelemons/godebug/pretty"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestPastebin_FromChunk(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	defer cancel()
+	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
+	if err != nil {
+		t.Fatalf("could not get test secrets from GCP: %s", err)
+	}
+	secret := testSecrets.MustGetField("PASTEBIN")
+	inactiveSecret := testSecrets.MustGetField("PASTEBIN_INACTIVE")
+
+	type args struct {
+		ctx    context.Context
+		data   []byte
+		verify bool
+	}
+	tests := []struct {
+		name    string
+		s       Scanner
+		args    args
+		want    []detectors.Result
+		wantErr bool
+	}{
+		{
+			name: "found, verified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a pastebin secret %s within", secret)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Pastebin,
+					Verified:     true,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "found, unverified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a pastebin secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Pastebin,
+					Verified:     false,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "not found",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte("You cannot find the secret within"),
+				verify: true,
+			},
+			want:    nil,
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			s := Scanner{}
+			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("Pastebin.FromData() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			for i := range got {
+				if len(got[i].Raw) == 0 {
+					t.Fatalf("no raw secret present: \n %+v", got[i])
+				}
+				got[i].Raw = nil
+			}
+			if diff := pretty.Compare(got, tt.want); diff != "" {
+				t.Errorf("Pastebin.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+			}
+		})
+	}
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+	ctx := context.Background()
+	s := Scanner{}
+	for name, data := range detectors.MustGetBenchmarkData() {
+		benchmark.Run(name, func(b *testing.B) {
+			b.ResetTimer()
+			for n := 0; n < b.N; n++ {
+				_, err := s.FromData(ctx, false, data)
+				if err != nil {
+					b.Fatal(err)
+				}
+			}
+		})
+	}
+}

pkg/detectors/pastebin/pastebin_test.go

@@ -1,119 +1,90 @@
-//go:build detectors
-// +build detectors
-
 package pastebin
 
 import (
 	"context"
 	"fmt"
 	"testing"
-	"time"
 
-	"github.com/kylelemons/godebug/pretty"
+	"github.com/google/go-cmp/cmp"
+
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
-
+	"github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 )
 
-func TestPastebin_FromChunk(t *testing.T) {
+var (
-	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	validPattern   = "92q9vNtJQRsjQDdbiYRRHBUNBYHSymKL"
-	defer cancel()
+	invalidPattern = "92q9vNtJQRsjQDdb?YRRHBUNBYHSymKL"
-	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
+	keyword        = "pastebin"
-	if err != nil {
+)
-		t.Fatalf("could not get test secrets from GCP: %s", err)
-	}
-	secret := testSecrets.MustGetField("PASTEBIN")
-	inactiveSecret := testSecrets.MustGetField("PASTEBIN_INACTIVE")
 
-	type args struct {
+func TestPastebin_Pattern(t *testing.T) {
-		ctx    context.Context
+	d := Scanner{}
-		data   []byte
+	ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
-		verify bool
-	}
 	tests := []struct {
-		name    string
+		name  string
-		s       Scanner
+		input string
-		args    args
+		want  []string
-		want    []detectors.Result
-		wantErr bool
 	}{
 		{
-			name: "found, verified",
+			name:  "valid pattern - with keyword pastebin",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
-			args: args{
+			want:  []string{validPattern},
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a pastebin secret %s within", secret)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_Pastebin,
-					Verified:     true,
-				},
-			},
-			wantErr: false,
 		},
 		{
-			name: "found, unverified",
+			name:  "valid pattern - ignore duplicate",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
-			args: args{
+			want:  []string{validPattern},
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a pastebin secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_Pastebin,
-					Verified:     false,
-				},
-			},
-			wantErr: false,
 		},
 		{
-			name: "not found",
+			name:  "valid pattern - key out of prefix range",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
-			args: args{
+			want:  []string{},
-				ctx:    context.Background(),
+		},
-				data:   []byte("You cannot find the secret within"),
+		{
-				verify: true,
+			name:  "invalid pattern",
-			},
+			input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
-			want:    nil,
+			want:  []string{},
-			wantErr: false,
 		},
 	}
-	for _, tt := range tests {
+
-		t.Run(tt.name, func(t *testing.T) {
+	for _, test := range tests {
-			s := Scanner{}
+		t.Run(test.name, func(t *testing.T) {
-			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
-			if (err != nil) != tt.wantErr {
+			if len(matchedDetectors) == 0 {
-				t.Errorf("Pastebin.FromData() error = %v, wantErr %v", err, tt.wantErr)
+				t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
 				return
 			}
-			for i := range got {
-				if len(got[i].Raw) == 0 {
-					t.Fatalf("no raw secret present: \n %+v", got[i])
-				}
-				got[i].Raw = nil
-			}
-			if diff := pretty.Compare(got, tt.want); diff != "" {
-				t.Errorf("Pastebin.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
-			}
-		})
-	}
-}
 
-func BenchmarkFromData(benchmark *testing.B) {
+			results, err := d.FromData(context.Background(), false, []byte(test.input))
-	ctx := context.Background()
+			if err != nil {
-	s := Scanner{}
+				t.Errorf("error = %v", err)
-	for name, data := range detectors.MustGetBenchmarkData() {
+				return
-		benchmark.Run(name, func(b *testing.B) {
+			}
-			b.ResetTimer()
+
-			for n := 0; n < b.N; n++ {
+			if len(results) != len(test.want) {
-				_, err := s.FromData(ctx, false, data)
+				if len(results) == 0 {
-				if err != nil {
+					t.Errorf("did not receive result")
-					b.Fatal(err)
+				} else {
+					t.Errorf("expected %d results, only received %d", len(test.want), len(results))
 				}
+				return
+			}
+
+			actual := make(map[string]struct{}, len(results))
+			for _, r := range results {
+				if len(r.RawV2) > 0 {
+					actual[string(r.RawV2)] = struct{}{}
+				} else {
+					actual[string(r.Raw)] = struct{}{}
+				}
+			}
+			expected := make(map[string]struct{}, len(test.want))
+			for _, v := range test.want {
+				expected[v] = struct{}{}
+			}
+
+			if diff := cmp.Diff(expected, actual); diff != "" {
+				t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
 			}
 		})
 	}

pkg/detectors/paydirtapp/paydirtapp_integration_test.go

@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package paydirtapp
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/kylelemons/godebug/pretty"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestPaydirtyapp_FromChunk(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	defer cancel()
+	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+	if err != nil {
+		t.Fatalf("could not get test secrets from GCP: %s", err)
+	}
+	secret := testSecrets.MustGetField("PAYDIRTAPP")
+	inactiveSecret := testSecrets.MustGetField("PAYDIRTAPP_INACTIVE")
+
+	type args struct {
+		ctx    context.Context
+		data   []byte
+		verify bool
+	}
+	tests := []struct {
+		name    string
+		s       Scanner
+		args    args
+		want    []detectors.Result
+		wantErr bool
+	}{
+		{
+			name: "found, verified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a paydirtapp secret %s within", secret)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Paydirtapp,
+					Verified:     true,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "found, unverified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a paydirtapp secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Paydirtapp,
+					Verified:     false,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "not found",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte("You cannot find the secret within"),
+				verify: true,
+			},
+			want:    nil,
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			s := Scanner{}
+			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("Paydirtyapp.FromData() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			for i := range got {
+				if len(got[i].Raw) == 0 {
+					t.Fatalf("no raw secret present: \n %+v", got[i])
+				}
+				got[i].Raw = nil
+			}
+			if diff := pretty.Compare(got, tt.want); diff != "" {
+				t.Errorf("Paydirtyapp.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+			}
+		})
+	}
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+	ctx := context.Background()
+	s := Scanner{}
+	for name, data := range detectors.MustGetBenchmarkData() {
+		benchmark.Run(name, func(b *testing.B) {
+			b.ResetTimer()
+			for n := 0; n < b.N; n++ {
+				_, err := s.FromData(ctx, false, data)
+				if err != nil {
+					b.Fatal(err)
+				}
+			}
+		})
+	}
+}

pkg/detectors/paydirtapp/paydirtapp_test.go

@@ -1,119 +1,90 @@
-//go:build detectors
-// +build detectors
-
 package paydirtapp
 
 import (
 	"context"
 	"fmt"
 	"testing"
-	"time"
 
-	"github.com/kylelemons/godebug/pretty"
+	"github.com/google/go-cmp/cmp"
+
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
-
+	"github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 )
 
-func TestPaydirtyapp_FromChunk(t *testing.T) {
+var (
-	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	validPattern   = "teboslrqf1lirq5gx0ca6544y3tkzsq0"
-	defer cancel()
+	invalidPattern = "teboslrqf1lirq5g?0ca6544y3tkzsq0"
-	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+	keyword        = "paydirtapp"
-	if err != nil {
+)
-		t.Fatalf("could not get test secrets from GCP: %s", err)
-	}
-	secret := testSecrets.MustGetField("PAYDIRTAPP")
-	inactiveSecret := testSecrets.MustGetField("PAYDIRTAPP_INACTIVE")
 
-	type args struct {
+func TestPaydirtyapp_Pattern(t *testing.T) {
-		ctx    context.Context
+	d := Scanner{}
-		data   []byte
+	ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
-		verify bool
-	}
 	tests := []struct {
-		name    string
+		name  string
-		s       Scanner
+		input string
-		args    args
+		want  []string
-		want    []detectors.Result
-		wantErr bool
 	}{
 		{
-			name: "found, verified",
+			name:  "valid pattern - with keyword paydirtapp",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
-			args: args{
+			want:  []string{validPattern},
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a paydirtapp secret %s within", secret)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_Paydirtapp,
-					Verified:     true,
-				},
-			},
-			wantErr: false,
 		},
 		{
-			name: "found, unverified",
+			name:  "valid pattern - ignore duplicate",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
-			args: args{
+			want:  []string{validPattern},
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a paydirtapp secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_Paydirtapp,
-					Verified:     false,
-				},
-			},
-			wantErr: false,
 		},
 		{
-			name: "not found",
+			name:  "valid pattern - key out of prefix range",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
-			args: args{
+			want:  []string{},
-				ctx:    context.Background(),
+		},
-				data:   []byte("You cannot find the secret within"),
+		{
-				verify: true,
+			name:  "invalid pattern",
-			},
+			input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
-			want:    nil,
+			want:  []string{},
-			wantErr: false,
 		},
 	}
-	for _, tt := range tests {
+
-		t.Run(tt.name, func(t *testing.T) {
+	for _, test := range tests {
-			s := Scanner{}
+		t.Run(test.name, func(t *testing.T) {
-			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
-			if (err != nil) != tt.wantErr {
+			if len(matchedDetectors) == 0 {
-				t.Errorf("Paydirtyapp.FromData() error = %v, wantErr %v", err, tt.wantErr)
+				t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
 				return
 			}
-			for i := range got {
-				if len(got[i].Raw) == 0 {
-					t.Fatalf("no raw secret present: \n %+v", got[i])
-				}
-				got[i].Raw = nil
-			}
-			if diff := pretty.Compare(got, tt.want); diff != "" {
-				t.Errorf("Paydirtyapp.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
-			}
-		})
-	}
-}
 
-func BenchmarkFromData(benchmark *testing.B) {
+			results, err := d.FromData(context.Background(), false, []byte(test.input))
-	ctx := context.Background()
+			if err != nil {
-	s := Scanner{}
+				t.Errorf("error = %v", err)
-	for name, data := range detectors.MustGetBenchmarkData() {
+				return
-		benchmark.Run(name, func(b *testing.B) {
+			}
-			b.ResetTimer()
+
-			for n := 0; n < b.N; n++ {
+			if len(results) != len(test.want) {
-				_, err := s.FromData(ctx, false, data)
+				if len(results) == 0 {
-				if err != nil {
+					t.Errorf("did not receive result")
-					b.Fatal(err)
+				} else {
+					t.Errorf("expected %d results, only received %d", len(test.want), len(results))
 				}
+				return
+			}
+
+			actual := make(map[string]struct{}, len(results))
+			for _, r := range results {
+				if len(r.RawV2) > 0 {
+					actual[string(r.RawV2)] = struct{}{}
+				} else {
+					actual[string(r.Raw)] = struct{}{}
+				}
+			}
+			expected := make(map[string]struct{}, len(test.want))
+			for _, v := range test.want {
+				expected[v] = struct{}{}
+			}
+
+			if diff := cmp.Diff(expected, actual); diff != "" {
+				t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
 			}
 		})
 	}

pkg/detectors/paymoapp/paymoapp_integration_test.go

@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package paymoapp
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/kylelemons/godebug/pretty"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestPaymoapp_FromChunk(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	defer cancel()
+	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
+	if err != nil {
+		t.Fatalf("could not get test secrets from GCP: %s", err)
+	}
+	secret := testSecrets.MustGetField("PAYMOAPP")
+	inactiveSecret := testSecrets.MustGetField("PAYMOAPP_INACTIVE")
+
+	type args struct {
+		ctx    context.Context
+		data   []byte
+		verify bool
+	}
+	tests := []struct {
+		name    string
+		s       Scanner
+		args    args
+		want    []detectors.Result
+		wantErr bool
+	}{
+		{
+			name: "found, verified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a paymoapp secret %s within", secret)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Paymoapp,
+					Verified:     true,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "found, unverified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a paymoapp secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Paymoapp,
+					Verified:     false,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "not found",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte("You cannot find the secret within"),
+				verify: true,
+			},
+			want:    nil,
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			s := Scanner{}
+			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("Paymoapp.FromData() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			for i := range got {
+				if len(got[i].Raw) == 0 {
+					t.Fatalf("no raw secret present: \n %+v", got[i])
+				}
+				got[i].Raw = nil
+			}
+			if diff := pretty.Compare(got, tt.want); diff != "" {
+				t.Errorf("Paymoapp.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+			}
+		})
+	}
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+	ctx := context.Background()
+	s := Scanner{}
+	for name, data := range detectors.MustGetBenchmarkData() {
+		benchmark.Run(name, func(b *testing.B) {
+			b.ResetTimer()
+			for n := 0; n < b.N; n++ {
+				_, err := s.FromData(ctx, false, data)
+				if err != nil {
+					b.Fatal(err)
+				}
+			}
+		})
+	}
+}

pkg/detectors/paymoapp/paymoapp_test.go

@@ -1,119 +1,90 @@
-//go:build detectors
-// +build detectors
-
 package paymoapp
 
 import (
 	"context"
 	"fmt"
 	"testing"
-	"time"
 
-	"github.com/kylelemons/godebug/pretty"
+	"github.com/google/go-cmp/cmp"
+
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
-
+	"github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 )
 
-func TestPaymoapp_FromChunk(t *testing.T) {
+var (
-	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	validPattern   = "EpKoocVnygMTgD6XB0c4QQ2mYIqXlt1efBuIvuihzKjU"
-	defer cancel()
+	invalidPattern = "EpKoocVnygMTgD6XB0c4QQ?mYIqXlt1efBuIvuihzKjU"
-	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
+	keyword        = "paymoapp"
-	if err != nil {
+)
-		t.Fatalf("could not get test secrets from GCP: %s", err)
-	}
-	secret := testSecrets.MustGetField("PAYMOAPP")
-	inactiveSecret := testSecrets.MustGetField("PAYMOAPP_INACTIVE")
 
-	type args struct {
+func TestPaymoapp_Pattern(t *testing.T) {
-		ctx    context.Context
+	d := Scanner{}
-		data   []byte
+	ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
-		verify bool
-	}
 	tests := []struct {
-		name    string
+		name  string
-		s       Scanner
+		input string
-		args    args
+		want  []string
-		want    []detectors.Result
-		wantErr bool
 	}{
 		{
-			name: "found, verified",
+			name:  "valid pattern - with keyword paymoapp",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
-			args: args{
+			want:  []string{validPattern},
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a paymoapp secret %s within", secret)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_Paymoapp,
-					Verified:     true,
-				},
-			},
-			wantErr: false,
 		},
 		{
-			name: "found, unverified",
+			name:  "valid pattern - ignore duplicate",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
-			args: args{
+			want:  []string{validPattern},
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a paymoapp secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_Paymoapp,
-					Verified:     false,
-				},
-			},
-			wantErr: false,
 		},
 		{
-			name: "not found",
+			name:  "valid pattern - key out of prefix range",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
-			args: args{
+			want:  []string{},
-				ctx:    context.Background(),
+		},
-				data:   []byte("You cannot find the secret within"),
+		{
-				verify: true,
+			name:  "invalid pattern",
-			},
+			input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
-			want:    nil,
+			want:  []string{},
-			wantErr: false,
 		},
 	}
-	for _, tt := range tests {
+
-		t.Run(tt.name, func(t *testing.T) {
+	for _, test := range tests {
-			s := Scanner{}
+		t.Run(test.name, func(t *testing.T) {
-			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
-			if (err != nil) != tt.wantErr {
+			if len(matchedDetectors) == 0 {
-				t.Errorf("Paymoapp.FromData() error = %v, wantErr %v", err, tt.wantErr)
+				t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
 				return
 			}
-			for i := range got {
-				if len(got[i].Raw) == 0 {
-					t.Fatalf("no raw secret present: \n %+v", got[i])
-				}
-				got[i].Raw = nil
-			}
-			if diff := pretty.Compare(got, tt.want); diff != "" {
-				t.Errorf("Paymoapp.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
-			}
-		})
-	}
-}
 
-func BenchmarkFromData(benchmark *testing.B) {
+			results, err := d.FromData(context.Background(), false, []byte(test.input))
-	ctx := context.Background()
+			if err != nil {
-	s := Scanner{}
+				t.Errorf("error = %v", err)
-	for name, data := range detectors.MustGetBenchmarkData() {
+				return
-		benchmark.Run(name, func(b *testing.B) {
+			}
-			b.ResetTimer()
+
-			for n := 0; n < b.N; n++ {
+			if len(results) != len(test.want) {
-				_, err := s.FromData(ctx, false, data)
+				if len(results) == 0 {
-				if err != nil {
+					t.Errorf("did not receive result")
-					b.Fatal(err)
+				} else {
+					t.Errorf("expected %d results, only received %d", len(test.want), len(results))
 				}
+				return
+			}
+
+			actual := make(map[string]struct{}, len(results))
+			for _, r := range results {
+				if len(r.RawV2) > 0 {
+					actual[string(r.RawV2)] = struct{}{}
+				} else {
+					actual[string(r.Raw)] = struct{}{}
+				}
+			}
+			expected := make(map[string]struct{}, len(test.want))
+			for _, v := range test.want {
+				expected[v] = struct{}{}
+			}
+
+			if diff := cmp.Diff(expected, actual); diff != "" {
+				t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
 			}
 		})
 	}

pkg/detectors/paymongo/paymongo_integration_test.go

@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package paymongo
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/kylelemons/godebug/pretty"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestPaymongo_FromChunk(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	defer cancel()
+	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
+	if err != nil {
+		t.Fatalf("could not get test secrets from GCP: %s", err)
+	}
+	secret := testSecrets.MustGetField("PAYMONGO")
+	inactiveSecret := testSecrets.MustGetField("PAYMONGO_INACTIVE")
+
+	type args struct {
+		ctx    context.Context
+		data   []byte
+		verify bool
+	}
+	tests := []struct {
+		name    string
+		s       Scanner
+		args    args
+		want    []detectors.Result
+		wantErr bool
+	}{
+		{
+			name: "found, verified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a paymongo secret %s within", secret)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Paymongo,
+					Verified:     true,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "found, unverified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a paymongo secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Paymongo,
+					Verified:     false,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "not found",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte("You cannot find the secret within"),
+				verify: true,
+			},
+			want:    nil,
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			s := Scanner{}
+			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("Paymongo.FromData() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			for i := range got {
+				if len(got[i].Raw) == 0 {
+					t.Fatalf("no raw secret present: \n %+v", got[i])
+				}
+				got[i].Raw = nil
+			}
+			if diff := pretty.Compare(got, tt.want); diff != "" {
+				t.Errorf("Paymongo.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+			}
+		})
+	}
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+	ctx := context.Background()
+	s := Scanner{}
+	for name, data := range detectors.MustGetBenchmarkData() {
+		benchmark.Run(name, func(b *testing.B) {
+			b.ResetTimer()
+			for n := 0; n < b.N; n++ {
+				_, err := s.FromData(ctx, false, data)
+				if err != nil {
+					b.Fatal(err)
+				}
+			}
+		})
+	}
+}

pkg/detectors/paymongo/paymongo_test.go

@@ -1,119 +1,90 @@
-//go:build detectors
-// +build detectors
-
 package paymongo
 
 import (
 	"context"
 	"fmt"
 	"testing"
-	"time"
 
-	"github.com/kylelemons/godebug/pretty"
+	"github.com/google/go-cmp/cmp"
+
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
-
+	"github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 )
 
-func TestPaymongo_FromChunk(t *testing.T) {
+var (
-	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	validPattern   = "okrrdIMzwSl350mcZkrtMkzEJ_xuLaJc"
-	defer cancel()
+	invalidPattern = "okrrdIMzwSl350mc?krtMkzEJ_xuLaJc"
-	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
+	keyword        = "paymongo"
-	if err != nil {
+)
-		t.Fatalf("could not get test secrets from GCP: %s", err)
-	}
-	secret := testSecrets.MustGetField("PAYMONGO")
-	inactiveSecret := testSecrets.MustGetField("PAYMONGO_INACTIVE")
 
-	type args struct {
+func TestPaymongo_Pattern(t *testing.T) {
-		ctx    context.Context
+	d := Scanner{}
-		data   []byte
+	ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
-		verify bool
-	}
 	tests := []struct {
-		name    string
+		name  string
-		s       Scanner
+		input string
-		args    args
+		want  []string
-		want    []detectors.Result
-		wantErr bool
 	}{
 		{
-			name: "found, verified",
+			name:  "valid pattern - with keyword paymongo",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
-			args: args{
+			want:  []string{validPattern},
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a paymongo secret %s within", secret)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_Paymongo,
-					Verified:     true,
-				},
-			},
-			wantErr: false,
 		},
 		{
-			name: "found, unverified",
+			name:  "valid pattern - ignore duplicate",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
-			args: args{
+			want:  []string{validPattern},
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a paymongo secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_Paymongo,
-					Verified:     false,
-				},
-			},
-			wantErr: false,
 		},
 		{
-			name: "not found",
+			name:  "valid pattern - key out of prefix range",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
-			args: args{
+			want:  []string{},
-				ctx:    context.Background(),
+		},
-				data:   []byte("You cannot find the secret within"),
+		{
-				verify: true,
+			name:  "invalid pattern",
-			},
+			input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
-			want:    nil,
+			want:  []string{},
-			wantErr: false,
 		},
 	}
-	for _, tt := range tests {
+
-		t.Run(tt.name, func(t *testing.T) {
+	for _, test := range tests {
-			s := Scanner{}
+		t.Run(test.name, func(t *testing.T) {
-			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
-			if (err != nil) != tt.wantErr {
+			if len(matchedDetectors) == 0 {
-				t.Errorf("Paymongo.FromData() error = %v, wantErr %v", err, tt.wantErr)
+				t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
 				return
 			}
-			for i := range got {
-				if len(got[i].Raw) == 0 {
-					t.Fatalf("no raw secret present: \n %+v", got[i])
-				}
-				got[i].Raw = nil
-			}
-			if diff := pretty.Compare(got, tt.want); diff != "" {
-				t.Errorf("Paymongo.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
-			}
-		})
-	}
-}
 
-func BenchmarkFromData(benchmark *testing.B) {
+			results, err := d.FromData(context.Background(), false, []byte(test.input))
-	ctx := context.Background()
+			if err != nil {
-	s := Scanner{}
+				t.Errorf("error = %v", err)
-	for name, data := range detectors.MustGetBenchmarkData() {
+				return
-		benchmark.Run(name, func(b *testing.B) {
+			}
-			b.ResetTimer()
+
-			for n := 0; n < b.N; n++ {
+			if len(results) != len(test.want) {
-				_, err := s.FromData(ctx, false, data)
+				if len(results) == 0 {
-				if err != nil {
+					t.Errorf("did not receive result")
-					b.Fatal(err)
+				} else {
+					t.Errorf("expected %d results, only received %d", len(test.want), len(results))
 				}
+				return
+			}
+
+			actual := make(map[string]struct{}, len(results))
+			for _, r := range results {
+				if len(r.RawV2) > 0 {
+					actual[string(r.RawV2)] = struct{}{}
+				} else {
+					actual[string(r.Raw)] = struct{}{}
+				}
+			}
+			expected := make(map[string]struct{}, len(test.want))
+			for _, v := range test.want {
+				expected[v] = struct{}{}
+			}
+
+			if diff := cmp.Diff(expected, actual); diff != "" {
+				t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
 			}
 		})
 	}

pkg/detectors/paypaloauth/paypaloauth_integration_test.go

@@ -0,0 +1,152 @@
+//go:build detectors
+// +build detectors
+
+package paypaloauth
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/kylelemons/godebug/pretty"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestPaypalOauth_FromChunk(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	defer cancel()
+	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
+	if err != nil {
+		t.Fatalf("could not get test secrets from GCP: %s", err)
+	}
+	secret := testSecrets.MustGetField("PAYPALOAUTH_SECRET")
+	inactiveSecret := testSecrets.MustGetField("PAYPALOAUTH_SECRET_INACTIVE")
+	id := testSecrets.MustGetField("PAYPALOAUTH_CLIENTID")
+
+	newId := testSecrets.MustGetField("PAYPALOAUTH_NEW_INACTIVE_CLIENTID")
+	newSecret := testSecrets.MustGetField("PAYPALOAUTH_NEW_INACTIVE_SECRET")
+
+	type args struct {
+		ctx    context.Context
+		data   []byte
+		verify bool
+	}
+	tests := []struct {
+		name    string
+		s       Scanner
+		args    args
+		want    []detectors.Result
+		wantErr bool
+	}{
+		{
+			name: "found, verified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a paypaloauth secret %s within %s", secret, id)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_PaypalOauth,
+					Verified:     true,
+				},
+				{
+					DetectorType: detectorspb.DetectorType_PaypalOauth,
+					Verified:     false,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "found, unverified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a paypaloauth secret %s within %s but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_PaypalOauth,
+					Verified:     false,
+				},
+				{
+					DetectorType: detectorspb.DetectorType_PaypalOauth,
+					Verified:     false,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "new format, unverified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a paypaloauth secret %s within %s but not valid", newSecret, newId)), // the secret would satisfy the regex but not pass validation
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_PaypalOauth,
+					Verified:     false,
+				},
+				{
+					DetectorType: detectorspb.DetectorType_PaypalOauth,
+					Verified:     false,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "not found",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte("You cannot find the secret within"),
+				verify: true,
+			},
+			want:    nil,
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			s := Scanner{}
+			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("PaypalOauth.FromData() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			for i := range got {
+				if len(got[i].Raw) == 0 {
+					t.Fatalf("no raw secret present: \n %+v", got[i])
+				}
+				got[i].Raw = nil
+			}
+			if diff := pretty.Compare(got, tt.want); diff != "" {
+				t.Errorf("PaypalOauth.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+			}
+		})
+	}
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+	ctx := context.Background()
+	s := Scanner{}
+	for name, data := range detectors.MustGetBenchmarkData() {
+		benchmark.Run(name, func(b *testing.B) {
+			b.ResetTimer()
+			for n := 0; n < b.N; n++ {
+				_, err := s.FromData(ctx, false, data)
+				if err != nil {
+					b.Fatal(err)
+				}
+			}
+		})
+	}
+}

pkg/detectors/paypaloauth/paypaloauth_test.go

@@ -1,151 +1,82 @@
-//go:build detectors
-// +build detectors
-
 package paypaloauth
 
 import (
 	"context"
 	"fmt"
 	"testing"
-	"time"
 
-	"github.com/kylelemons/godebug/pretty"
+	"github.com/google/go-cmp/cmp"
+
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
-
+	"github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 )
 
-func TestPaypalOauth_FromChunk(t *testing.T) {
+var (
-	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	validId    = "Dddpt-5MySAKlcPX07LKjhzbHTbf9m2Xv9OFSw0bTdrZ"
-	defer cancel()
+	invalidId  = "Dddpt-5MySAKlcPX07LKjh?bHTbf9m2Xv9OFSw0bTdrZ"
-	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
+	validKey   = "PDMDWZ3BAwfXHoU.P.pJVUPIYlkJ2jVP-ns2icvSlWeb-_0Qa5"
-	if err != nil {
+	invalidKey = "PDMDWZ3BAwfXHoU.P.pJVUPIY?kJ2jVP-ns2icvSlWeb-_0Qa5"
-		t.Fatalf("could not get test secrets from GCP: %s", err)
+	keyword    = "paypaloauth"
-	}
+)
-	secret := testSecrets.MustGetField("PAYPALOAUTH_SECRET")
-	inactiveSecret := testSecrets.MustGetField("PAYPALOAUTH_SECRET_INACTIVE")
-	id := testSecrets.MustGetField("PAYPALOAUTH_CLIENTID")
 
-	newId := testSecrets.MustGetField("PAYPALOAUTH_NEW_INACTIVE_CLIENTID")
+func TestPaypalOauth_Pattern(t *testing.T) {
-	newSecret := testSecrets.MustGetField("PAYPALOAUTH_NEW_INACTIVE_SECRET")
+	d := Scanner{}
-
+	ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
-	type args struct {
-		ctx    context.Context
-		data   []byte
-		verify bool
-	}
 	tests := []struct {
-		name    string
+		name  string
-		s       Scanner
+		input string
-		args    args
+		want  []string
-		want    []detectors.Result
-		wantErr bool
 	}{
 		{
-			name: "found, verified",
+			name:  "valid pattern - with keyword paypaloauth",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, validId, keyword, validKey),
-			args: args{
+			want:  []string{validId, validKey},
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a paypaloauth secret %s within %s", secret, id)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_PaypalOauth,
-					Verified:     true,
-				},
-				{
-					DetectorType: detectorspb.DetectorType_PaypalOauth,
-					Verified:     false,
-				},
-			},
-			wantErr: false,
 		},
 		{
-			name: "found, unverified",
+			name:  "invalid pattern",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, invalidId, keyword, invalidKey),
-			args: args{
+			want:  []string{},
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a paypaloauth secret %s within %s but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_PaypalOauth,
-					Verified:     false,
-				},
-				{
-					DetectorType: detectorspb.DetectorType_PaypalOauth,
-					Verified:     false,
-				},
-			},
-			wantErr: false,
-		},
-		{
-			name: "new format, unverified",
-			s:    Scanner{},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a paypaloauth secret %s within %s but not valid", newSecret, newId)), // the secret would satisfy the regex but not pass validation
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_PaypalOauth,
-					Verified:     false,
-				},
-				{
-					DetectorType: detectorspb.DetectorType_PaypalOauth,
-					Verified:     false,
-				},
-			},
-			wantErr: false,
-		},
-		{
-			name: "not found",
-			s:    Scanner{},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte("You cannot find the secret within"),
-				verify: true,
-			},
-			want:    nil,
-			wantErr: false,
 		},
 	}
-	for _, tt := range tests {
+
-		t.Run(tt.name, func(t *testing.T) {
+	for _, test := range tests {
-			s := Scanner{}
+		t.Run(test.name, func(t *testing.T) {
-			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
-			if (err != nil) != tt.wantErr {
+			if len(matchedDetectors) == 0 {
-				t.Errorf("PaypalOauth.FromData() error = %v, wantErr %v", err, tt.wantErr)
+				t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
 				return
 			}
-			for i := range got {
-				if len(got[i].Raw) == 0 {
-					t.Fatalf("no raw secret present: \n %+v", got[i])
-				}
-				got[i].Raw = nil
-			}
-			if diff := pretty.Compare(got, tt.want); diff != "" {
-				t.Errorf("PaypalOauth.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
-			}
-		})
-	}
-}
 
-func BenchmarkFromData(benchmark *testing.B) {
+			results, err := d.FromData(context.Background(), false, []byte(test.input))
-	ctx := context.Background()
+			if err != nil {
-	s := Scanner{}
+				t.Errorf("error = %v", err)
-	for name, data := range detectors.MustGetBenchmarkData() {
+				return
-		benchmark.Run(name, func(b *testing.B) {
+			}
-			b.ResetTimer()
+
-			for n := 0; n < b.N; n++ {
+			if len(results) != len(test.want) {
-				_, err := s.FromData(ctx, false, data)
+				if len(results) == 0 {
-				if err != nil {
+					t.Errorf("did not receive result")
-					b.Fatal(err)
+				} else {
+					t.Errorf("expected %d results, only received %d", len(test.want), len(results))
 				}
+				return
+			}
+
+			actual := make(map[string]struct{}, len(results))
+			for _, r := range results {
+				if len(r.RawV2) > 0 {
+					actual[string(r.RawV2)] = struct{}{}
+				} else {
+					actual[string(r.Raw)] = struct{}{}
+				}
+			}
+			expected := make(map[string]struct{}, len(test.want))
+			for _, v := range test.want {
+				expected[v] = struct{}{}
+			}
+
+			if diff := cmp.Diff(expected, actual); diff != "" {
+				t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
 			}
 		})
 	}

pkg/detectors/paystack/paystack_integration_test.go

@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package paystack
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/kylelemons/godebug/pretty"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestPaystack_FromChunk(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	defer cancel()
+	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+	if err != nil {
+		t.Fatalf("could not get test secrets from GCP: %s", err)
+	}
+	secret := testSecrets.MustGetField("PAYSTACK_TOKEN")
+	inactiveSecret := testSecrets.MustGetField("PAYSTACK_INACTIVE")
+
+	type args struct {
+		ctx    context.Context
+		data   []byte
+		verify bool
+	}
+	tests := []struct {
+		name    string
+		s       Scanner
+		args    args
+		want    []detectors.Result
+		wantErr bool
+	}{
+		{
+			name: "found, verified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a paystack secret %s within", secret)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Paystack,
+					Verified:     true,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "found, unverified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a paystack secret %s within but unverified", inactiveSecret)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Paystack,
+					Verified:     false,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "not found",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte("You cannot find the secret within"),
+				verify: true,
+			},
+			want:    nil,
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			s := Scanner{}
+			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("Paystack.FromData() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			for i := range got {
+				if len(got[i].Raw) == 0 {
+					t.Fatalf("no raw secret present: \n %+v", got[i])
+				}
+				got[i].Raw = nil
+			}
+			if diff := pretty.Compare(got, tt.want); diff != "" {
+				t.Errorf("Paystack.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+			}
+		})
+	}
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+	ctx := context.Background()
+	s := Scanner{}
+	for name, data := range detectors.MustGetBenchmarkData() {
+		benchmark.Run(name, func(b *testing.B) {
+			b.ResetTimer()
+			for n := 0; n < b.N; n++ {
+				_, err := s.FromData(ctx, false, data)
+				if err != nil {
+					b.Fatal(err)
+				}
+			}
+		})
+	}
+}

pkg/detectors/paystack/paystack_test.go

@@ -1,119 +1,80 @@
-//go:build detectors
-// +build detectors
-
 package paystack
 
 import (
 	"context"
 	"fmt"
 	"testing"
-	"time"
 
-	"github.com/kylelemons/godebug/pretty"
+	"github.com/google/go-cmp/cmp"
+
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
-
+	"github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 )
 
-func TestPaystack_FromChunk(t *testing.T) {
+var (
-	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	validPattern   = "sk_xigrvarm_cJHGpWQwCTHajG2A2o8eC8TQaQGZdoMVhgXUA9Lm"
-	defer cancel()
+	invalidPattern = "sk_xigrvarm_cJHGpWQwCTHajG?A2o8eC8TQaQGZdoMVhgXUA9Lm"
-	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+	keyword        = "paystack"
-	if err != nil {
+)
-		t.Fatalf("could not get test secrets from GCP: %s", err)
-	}
-	secret := testSecrets.MustGetField("PAYSTACK_TOKEN")
-	inactiveSecret := testSecrets.MustGetField("PAYSTACK_INACTIVE")
 
-	type args struct {
+func TestPaystack_Pattern(t *testing.T) {
-		ctx    context.Context
+	d := Scanner{}
-		data   []byte
+	ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
-		verify bool
-	}
 	tests := []struct {
-		name    string
+		name  string
-		s       Scanner
+		input string
-		args    args
+		want  []string
-		want    []detectors.Result
-		wantErr bool
 	}{
 		{
-			name: "found, verified",
+			name:  "valid pattern - with keyword paystack",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
-			args: args{
+			want:  []string{validPattern},
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a paystack secret %s within", secret)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_Paystack,
-					Verified:     true,
-				},
-			},
-			wantErr: false,
 		},
 		{
-			name: "found, unverified",
+			name:  "invalid pattern",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
-			args: args{
+			want:  []string{},
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a paystack secret %s within but unverified", inactiveSecret)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_Paystack,
-					Verified:     false,
-				},
-			},
-			wantErr: false,
-		},
-		{
-			name: "not found",
-			s:    Scanner{},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte("You cannot find the secret within"),
-				verify: true,
-			},
-			want:    nil,
-			wantErr: false,
 		},
 	}
-	for _, tt := range tests {
+
-		t.Run(tt.name, func(t *testing.T) {
+	for _, test := range tests {
-			s := Scanner{}
+		t.Run(test.name, func(t *testing.T) {
-			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
-			if (err != nil) != tt.wantErr {
+			if len(matchedDetectors) == 0 {
-				t.Errorf("Paystack.FromData() error = %v, wantErr %v", err, tt.wantErr)
+				t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
 				return
 			}
-			for i := range got {
-				if len(got[i].Raw) == 0 {
-					t.Fatalf("no raw secret present: \n %+v", got[i])
-				}
-				got[i].Raw = nil
-			}
-			if diff := pretty.Compare(got, tt.want); diff != "" {
-				t.Errorf("Paystack.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
-			}
-		})
-	}
-}
 
-func BenchmarkFromData(benchmark *testing.B) {
+			results, err := d.FromData(context.Background(), false, []byte(test.input))
-	ctx := context.Background()
+			if err != nil {
-	s := Scanner{}
+				t.Errorf("error = %v", err)
-	for name, data := range detectors.MustGetBenchmarkData() {
+				return
-		benchmark.Run(name, func(b *testing.B) {
+			}
-			b.ResetTimer()
+
-			for n := 0; n < b.N; n++ {
+			if len(results) != len(test.want) {
-				_, err := s.FromData(ctx, false, data)
+				if len(results) == 0 {
-				if err != nil {
+					t.Errorf("did not receive result")
-					b.Fatal(err)
+				} else {
+					t.Errorf("expected %d results, only received %d", len(test.want), len(results))
 				}
+				return
+			}
+
+			actual := make(map[string]struct{}, len(results))
+			for _, r := range results {
+				if len(r.RawV2) > 0 {
+					actual[string(r.RawV2)] = struct{}{}
+				} else {
+					actual[string(r.Raw)] = struct{}{}
+				}
+			}
+			expected := make(map[string]struct{}, len(test.want))
+			for _, v := range test.want {
+				expected[v] = struct{}{}
+			}
+
+			if diff := cmp.Diff(expected, actual); diff != "" {
+				t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
 			}
 		})
 	}

pkg/detectors/pdflayer/pdflayer_integration_test.go

@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package pdflayer
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/kylelemons/godebug/pretty"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestPdfLayer_FromChunk(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	defer cancel()
+	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
+	if err != nil {
+		t.Fatalf("could not get test secrets from GCP: %s", err)
+	}
+	secret := testSecrets.MustGetField("PDFLAYER")
+	inactiveSecret := testSecrets.MustGetField("PDFLAYER_INACTIVE")
+
+	type args struct {
+		ctx    context.Context
+		data   []byte
+		verify bool
+	}
+	tests := []struct {
+		name    string
+		s       Scanner
+		args    args
+		want    []detectors.Result
+		wantErr bool
+	}{
+		{
+			name: "found, verified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a pdflayer secret %s within", secret)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_PdfLayer,
+					Verified:     true,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "found, unverified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a pdflayer secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_PdfLayer,
+					Verified:     false,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "not found",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte("You cannot find the secret within"),
+				verify: true,
+			},
+			want:    nil,
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			s := Scanner{}
+			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("PdfLayer.FromData() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			for i := range got {
+				if len(got[i].Raw) == 0 {
+					t.Fatalf("no raw secret present: \n %+v", got[i])
+				}
+				got[i].Raw = nil
+			}
+			if diff := pretty.Compare(got, tt.want); diff != "" {
+				t.Errorf("PdfLayer.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+			}
+		})
+	}
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+	ctx := context.Background()
+	s := Scanner{}
+	for name, data := range detectors.MustGetBenchmarkData() {
+		benchmark.Run(name, func(b *testing.B) {
+			b.ResetTimer()
+			for n := 0; n < b.N; n++ {
+				_, err := s.FromData(ctx, false, data)
+				if err != nil {
+					b.Fatal(err)
+				}
+			}
+		})
+	}
+}

pkg/detectors/pdflayer/pdflayer_test.go

@@ -1,119 +1,90 @@
-//go:build detectors
-// +build detectors
-
 package pdflayer
 
 import (
 	"context"
 	"fmt"
 	"testing"
-	"time"
 
-	"github.com/kylelemons/godebug/pretty"
+	"github.com/google/go-cmp/cmp"
+
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
-
+	"github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 )
 
-func TestPdfLayer_FromChunk(t *testing.T) {
+var (
-	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	validPattern   = "wv8d7n0j02zxzrsbcs8cyk9yvqch9gvr"
-	defer cancel()
+	invalidPattern = "wv8d7n0j02zxzrsb?s8cyk9yvqch9gvr"
-	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
+	keyword        = "pdflayer"
-	if err != nil {
+)
-		t.Fatalf("could not get test secrets from GCP: %s", err)
-	}
-	secret := testSecrets.MustGetField("PDFLAYER")
-	inactiveSecret := testSecrets.MustGetField("PDFLAYER_INACTIVE")
 
-	type args struct {
+func TestPdfLayer_Pattern(t *testing.T) {
-		ctx    context.Context
+	d := Scanner{}
-		data   []byte
+	ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
-		verify bool
-	}
 	tests := []struct {
-		name    string
+		name  string
-		s       Scanner
+		input string
-		args    args
+		want  []string
-		want    []detectors.Result
-		wantErr bool
 	}{
 		{
-			name: "found, verified",
+			name:  "valid pattern - with keyword pdflayer",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
-			args: args{
+			want:  []string{validPattern},
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a pdflayer secret %s within", secret)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_PdfLayer,
-					Verified:     true,
-				},
-			},
-			wantErr: false,
 		},
 		{
-			name: "found, unverified",
+			name:  "valid pattern - ignore duplicate",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
-			args: args{
+			want:  []string{validPattern},
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a pdflayer secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_PdfLayer,
-					Verified:     false,
-				},
-			},
-			wantErr: false,
 		},
 		{
-			name: "not found",
+			name:  "valid pattern - key out of prefix range",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
-			args: args{
+			want:  []string{},
-				ctx:    context.Background(),
+		},
-				data:   []byte("You cannot find the secret within"),
+		{
-				verify: true,
+			name:  "invalid pattern",
-			},
+			input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
-			want:    nil,
+			want:  []string{},
-			wantErr: false,
 		},
 	}
-	for _, tt := range tests {
+
-		t.Run(tt.name, func(t *testing.T) {
+	for _, test := range tests {
-			s := Scanner{}
+		t.Run(test.name, func(t *testing.T) {
-			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
-			if (err != nil) != tt.wantErr {
+			if len(matchedDetectors) == 0 {
-				t.Errorf("PdfLayer.FromData() error = %v, wantErr %v", err, tt.wantErr)
+				t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
 				return
 			}
-			for i := range got {
-				if len(got[i].Raw) == 0 {
-					t.Fatalf("no raw secret present: \n %+v", got[i])
-				}
-				got[i].Raw = nil
-			}
-			if diff := pretty.Compare(got, tt.want); diff != "" {
-				t.Errorf("PdfLayer.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
-			}
-		})
-	}
-}
 
-func BenchmarkFromData(benchmark *testing.B) {
+			results, err := d.FromData(context.Background(), false, []byte(test.input))
-	ctx := context.Background()
+			if err != nil {
-	s := Scanner{}
+				t.Errorf("error = %v", err)
-	for name, data := range detectors.MustGetBenchmarkData() {
+				return
-		benchmark.Run(name, func(b *testing.B) {
+			}
-			b.ResetTimer()
+
-			for n := 0; n < b.N; n++ {
+			if len(results) != len(test.want) {
-				_, err := s.FromData(ctx, false, data)
+				if len(results) == 0 {
-				if err != nil {
+					t.Errorf("did not receive result")
-					b.Fatal(err)
+				} else {
+					t.Errorf("expected %d results, only received %d", len(test.want), len(results))
 				}
+				return
+			}
+
+			actual := make(map[string]struct{}, len(results))
+			for _, r := range results {
+				if len(r.RawV2) > 0 {
+					actual[string(r.RawV2)] = struct{}{}
+				} else {
+					actual[string(r.Raw)] = struct{}{}
+				}
+			}
+			expected := make(map[string]struct{}, len(test.want))
+			for _, v := range test.want {
+				expected[v] = struct{}{}
+			}
+
+			if diff := cmp.Diff(expected, actual); diff != "" {
+				t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
 			}
 		})
 	}

pkg/detectors/pdfshift/pdfshift_integration_test.go

@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package pdfshift
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/kylelemons/godebug/pretty"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestPdfShift_FromChunk(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	defer cancel()
+	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
+	if err != nil {
+		t.Fatalf("could not get test secrets from GCP: %s", err)
+	}
+	secret := testSecrets.MustGetField("PDFSHIFT")
+	inactiveSecret := testSecrets.MustGetField("PDFSHIFT_INACTIVE")
+
+	type args struct {
+		ctx    context.Context
+		data   []byte
+		verify bool
+	}
+	tests := []struct {
+		name    string
+		s       Scanner
+		args    args
+		want    []detectors.Result
+		wantErr bool
+	}{
+		{
+			name: "found, verified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a pdfshift secret %s within", secret)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_PdfShift,
+					Verified:     true,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "found, unverified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a pdfshift secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_PdfShift,
+					Verified:     false,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "not found",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte("You cannot find the secret within"),
+				verify: true,
+			},
+			want:    nil,
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			s := Scanner{}
+			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("PdfShift.FromData() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			for i := range got {
+				if len(got[i].Raw) == 0 {
+					t.Fatalf("no raw secret present: \n %+v", got[i])
+				}
+				got[i].Raw = nil
+			}
+			if diff := pretty.Compare(got, tt.want); diff != "" {
+				t.Errorf("PdfShift.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+			}
+		})
+	}
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+	ctx := context.Background()
+	s := Scanner{}
+	for name, data := range detectors.MustGetBenchmarkData() {
+		benchmark.Run(name, func(b *testing.B) {
+			b.ResetTimer()
+			for n := 0; n < b.N; n++ {
+				_, err := s.FromData(ctx, false, data)
+				if err != nil {
+					b.Fatal(err)
+				}
+			}
+		})
+	}
+}

pkg/detectors/pdfshift/pdfshift_test.go

@@ -1,119 +1,90 @@
-//go:build detectors
-// +build detectors
-
 package pdfshift
 
 import (
 	"context"
 	"fmt"
 	"testing"
-	"time"
 
-	"github.com/kylelemons/godebug/pretty"
+	"github.com/google/go-cmp/cmp"
+
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
-
+	"github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 )
 
-func TestPdfShift_FromChunk(t *testing.T) {
+var (
-	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	validPattern   = "b03ed226557f474fda3f5a8fdd498f7c"
-	defer cancel()
+	invalidPattern = "b03e?226557f474fda3f5a8fdd498f7c"
-	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
+	keyword        = "pdfshift"
-	if err != nil {
+)
-		t.Fatalf("could not get test secrets from GCP: %s", err)
-	}
-	secret := testSecrets.MustGetField("PDFSHIFT")
-	inactiveSecret := testSecrets.MustGetField("PDFSHIFT_INACTIVE")
 
-	type args struct {
+func TestPdfShift_Pattern(t *testing.T) {
-		ctx    context.Context
+	d := Scanner{}
-		data   []byte
+	ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
-		verify bool
-	}
 	tests := []struct {
-		name    string
+		name  string
-		s       Scanner
+		input string
-		args    args
+		want  []string
-		want    []detectors.Result
-		wantErr bool
 	}{
 		{
-			name: "found, verified",
+			name:  "valid pattern - with keyword pdfshift",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
-			args: args{
+			want:  []string{validPattern},
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a pdfshift secret %s within", secret)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_PdfShift,
-					Verified:     true,
-				},
-			},
-			wantErr: false,
 		},
 		{
-			name: "found, unverified",
+			name:  "valid pattern - ignore duplicate",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
-			args: args{
+			want:  []string{validPattern},
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a pdfshift secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_PdfShift,
-					Verified:     false,
-				},
-			},
-			wantErr: false,
 		},
 		{
-			name: "not found",
+			name:  "valid pattern - key out of prefix range",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
-			args: args{
+			want:  []string{},
-				ctx:    context.Background(),
+		},
-				data:   []byte("You cannot find the secret within"),
+		{
-				verify: true,
+			name:  "invalid pattern",
-			},
+			input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
-			want:    nil,
+			want:  []string{},
-			wantErr: false,
 		},
 	}
-	for _, tt := range tests {
+
-		t.Run(tt.name, func(t *testing.T) {
+	for _, test := range tests {
-			s := Scanner{}
+		t.Run(test.name, func(t *testing.T) {
-			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
-			if (err != nil) != tt.wantErr {
+			if len(matchedDetectors) == 0 {
-				t.Errorf("PdfShift.FromData() error = %v, wantErr %v", err, tt.wantErr)
+				t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
 				return
 			}
-			for i := range got {
-				if len(got[i].Raw) == 0 {
-					t.Fatalf("no raw secret present: \n %+v", got[i])
-				}
-				got[i].Raw = nil
-			}
-			if diff := pretty.Compare(got, tt.want); diff != "" {
-				t.Errorf("PdfShift.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
-			}
-		})
-	}
-}
 
-func BenchmarkFromData(benchmark *testing.B) {
+			results, err := d.FromData(context.Background(), false, []byte(test.input))
-	ctx := context.Background()
+			if err != nil {
-	s := Scanner{}
+				t.Errorf("error = %v", err)
-	for name, data := range detectors.MustGetBenchmarkData() {
+				return
-		benchmark.Run(name, func(b *testing.B) {
+			}
-			b.ResetTimer()
+
-			for n := 0; n < b.N; n++ {
+			if len(results) != len(test.want) {
-				_, err := s.FromData(ctx, false, data)
+				if len(results) == 0 {
-				if err != nil {
+					t.Errorf("did not receive result")
-					b.Fatal(err)
+				} else {
+					t.Errorf("expected %d results, only received %d", len(test.want), len(results))
 				}
+				return
+			}
+
+			actual := make(map[string]struct{}, len(results))
+			for _, r := range results {
+				if len(r.RawV2) > 0 {
+					actual[string(r.RawV2)] = struct{}{}
+				} else {
+					actual[string(r.Raw)] = struct{}{}
+				}
+			}
+			expected := make(map[string]struct{}, len(test.want))
+			for _, v := range test.want {
+				expected[v] = struct{}{}
+			}
+
+			if diff := cmp.Diff(expected, actual); diff != "" {
+				t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
 			}
 		})
 	}

pkg/detectors/peopledatalabs/peopledatalabs_integration_test.go

@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package peopledatalabs
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/kylelemons/godebug/pretty"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestPeopleDataLabs_FromChunk(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	defer cancel()
+	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
+	if err != nil {
+		t.Fatalf("could not get test secrets from GCP: %s", err)
+	}
+	secret := testSecrets.MustGetField("PEOPLEDATALABS")
+	inactiveSecret := testSecrets.MustGetField("PEOPLEDATALABS_INACTIVE")
+
+	type args struct {
+		ctx    context.Context
+		data   []byte
+		verify bool
+	}
+	tests := []struct {
+		name    string
+		s       Scanner
+		args    args
+		want    []detectors.Result
+		wantErr bool
+	}{
+		{
+			name: "found, verified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a peopledatalabs secret %s within", secret)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_PeopleDataLabs,
+					Verified:     true,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "found, unverified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a peopledatalabs secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_PeopleDataLabs,
+					Verified:     false,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "not found",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte("You cannot find the secret within"),
+				verify: true,
+			},
+			want:    nil,
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			s := Scanner{}
+			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("PeopleDataLabs.FromData() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			for i := range got {
+				if len(got[i].Raw) == 0 {
+					t.Fatalf("no raw secret present: \n %+v", got[i])
+				}
+				got[i].Raw = nil
+			}
+			if diff := pretty.Compare(got, tt.want); diff != "" {
+				t.Errorf("PeopleDataLabs.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+			}
+		})
+	}
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+	ctx := context.Background()
+	s := Scanner{}
+	for name, data := range detectors.MustGetBenchmarkData() {
+		benchmark.Run(name, func(b *testing.B) {
+			b.ResetTimer()
+			for n := 0; n < b.N; n++ {
+				_, err := s.FromData(ctx, false, data)
+				if err != nil {
+					b.Fatal(err)
+				}
+			}
+		})
+	}
+}

pkg/detectors/peopledatalabs/peopledatalabs_test.go

@@ -1,119 +1,90 @@
-//go:build detectors
-// +build detectors
-
 package peopledatalabs
 
 import (
 	"context"
 	"fmt"
 	"testing"
-	"time"
 
-	"github.com/kylelemons/godebug/pretty"
+	"github.com/google/go-cmp/cmp"
+
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
-
+	"github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 )
 
-func TestPeopleDataLabs_FromChunk(t *testing.T) {
+var (
-	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	validPattern   = "tp8k005gi460y93zclyrifwjnij6492iiwignkeuccqjkrs1rqcw0lsuvjm39ij9"
-	defer cancel()
+	invalidPattern = "tp8k005g?460y93zclyrifwjnij6492iiwignkeuccqjkrs1rqcw0lsuvjm39ij9"
-	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
+	keyword        = "peopledatalabs"
-	if err != nil {
+)
-		t.Fatalf("could not get test secrets from GCP: %s", err)
-	}
-	secret := testSecrets.MustGetField("PEOPLEDATALABS")
-	inactiveSecret := testSecrets.MustGetField("PEOPLEDATALABS_INACTIVE")
 
-	type args struct {
+func TestPeopleDataLabs_Pattern(t *testing.T) {
-		ctx    context.Context
+	d := Scanner{}
-		data   []byte
+	ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
-		verify bool
-	}
 	tests := []struct {
-		name    string
+		name  string
-		s       Scanner
+		input string
-		args    args
+		want  []string
-		want    []detectors.Result
-		wantErr bool
 	}{
 		{
-			name: "found, verified",
+			name:  "valid pattern - with keyword peopledatalabs",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
-			args: args{
+			want:  []string{validPattern},
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a peopledatalabs secret %s within", secret)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_PeopleDataLabs,
-					Verified:     true,
-				},
-			},
-			wantErr: false,
 		},
 		{
-			name: "found, unverified",
+			name:  "valid pattern - ignore duplicate",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
-			args: args{
+			want:  []string{validPattern},
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a peopledatalabs secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_PeopleDataLabs,
-					Verified:     false,
-				},
-			},
-			wantErr: false,
 		},
 		{
-			name: "not found",
+			name:  "valid pattern - key out of prefix range",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
-			args: args{
+			want:  []string{},
-				ctx:    context.Background(),
+		},
-				data:   []byte("You cannot find the secret within"),
+		{
-				verify: true,
+			name:  "invalid pattern",
-			},
+			input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
-			want:    nil,
+			want:  []string{},
-			wantErr: false,
 		},
 	}
-	for _, tt := range tests {
+
-		t.Run(tt.name, func(t *testing.T) {
+	for _, test := range tests {
-			s := Scanner{}
+		t.Run(test.name, func(t *testing.T) {
-			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
-			if (err != nil) != tt.wantErr {
+			if len(matchedDetectors) == 0 {
-				t.Errorf("PeopleDataLabs.FromData() error = %v, wantErr %v", err, tt.wantErr)
+				t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
 				return
 			}
-			for i := range got {
-				if len(got[i].Raw) == 0 {
-					t.Fatalf("no raw secret present: \n %+v", got[i])
-				}
-				got[i].Raw = nil
-			}
-			if diff := pretty.Compare(got, tt.want); diff != "" {
-				t.Errorf("PeopleDataLabs.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
-			}
-		})
-	}
-}
 
-func BenchmarkFromData(benchmark *testing.B) {
+			results, err := d.FromData(context.Background(), false, []byte(test.input))
-	ctx := context.Background()
+			if err != nil {
-	s := Scanner{}
+				t.Errorf("error = %v", err)
-	for name, data := range detectors.MustGetBenchmarkData() {
+				return
-		benchmark.Run(name, func(b *testing.B) {
+			}
-			b.ResetTimer()
+
-			for n := 0; n < b.N; n++ {
+			if len(results) != len(test.want) {
-				_, err := s.FromData(ctx, false, data)
+				if len(results) == 0 {
-				if err != nil {
+					t.Errorf("did not receive result")
-					b.Fatal(err)
+				} else {
+					t.Errorf("expected %d results, only received %d", len(test.want), len(results))
 				}
+				return
+			}
+
+			actual := make(map[string]struct{}, len(results))
+			for _, r := range results {
+				if len(r.RawV2) > 0 {
+					actual[string(r.RawV2)] = struct{}{}
+				} else {
+					actual[string(r.Raw)] = struct{}{}
+				}
+			}
+			expected := make(map[string]struct{}, len(test.want))
+			for _, v := range test.want {
+				expected[v] = struct{}{}
+			}
+
+			if diff := cmp.Diff(expected, actual); diff != "" {
+				t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
 			}
 		})
 	}

pkg/detectors/pepipost/pepipost_integration_test.go

@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package pepipost
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/kylelemons/godebug/pretty"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestPepipost_FromChunk(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	defer cancel()
+	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+	if err != nil {
+		t.Fatalf("could not get test secrets from GCP: %s", err)
+	}
+	secret := testSecrets.MustGetField("PEPIPOST_TOKEN")
+	inactiveSecret := testSecrets.MustGetField("PEPIPOST_INACTIVE")
+
+	type args struct {
+		ctx    context.Context
+		data   []byte
+		verify bool
+	}
+	tests := []struct {
+		name    string
+		s       Scanner
+		args    args
+		want    []detectors.Result
+		wantErr bool
+	}{
+		{
+			name: "found, verified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a pepipost secret %s within", secret)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Pepipost,
+					Verified:     true,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "found, unverified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a pepipost secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Pepipost,
+					Verified:     false,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "not found",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte("You cannot find the secret within"),
+				verify: true,
+			},
+			want:    nil,
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			s := Scanner{}
+			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("Pepipost.FromData() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			for i := range got {
+				if len(got[i].Raw) == 0 {
+					t.Fatalf("no raw secret present: \n %+v", got[i])
+				}
+				got[i].Raw = nil
+			}
+			if diff := pretty.Compare(got, tt.want); diff != "" {
+				t.Errorf("Pepipost.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+			}
+		})
+	}
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+	ctx := context.Background()
+	s := Scanner{}
+	for name, data := range detectors.MustGetBenchmarkData() {
+		benchmark.Run(name, func(b *testing.B) {
+			b.ResetTimer()
+			for n := 0; n < b.N; n++ {
+				_, err := s.FromData(ctx, false, data)
+				if err != nil {
+					b.Fatal(err)
+				}
+			}
+		})
+	}
+}

pkg/detectors/pepipost/pepipost_test.go

@@ -1,119 +1,90 @@
-//go:build detectors
-// +build detectors
-
 package pepipost
 
 import (
 	"context"
 	"fmt"
 	"testing"
-	"time"
 
-	"github.com/kylelemons/godebug/pretty"
+	"github.com/google/go-cmp/cmp"
+
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
-
+	"github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 )
 
-func TestPepipost_FromChunk(t *testing.T) {
+var (
-	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	validPattern   = "zajmEjyCrhFdm1gQXoZD6r8WKLubC-B2"
-	defer cancel()
+	invalidPattern = "zajmEjyCrhFdm1gQ?oZD6r8WKLubC-B2"
-	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+	keyword        = "pepipost"
-	if err != nil {
+)
-		t.Fatalf("could not get test secrets from GCP: %s", err)
-	}
-	secret := testSecrets.MustGetField("PEPIPOST_TOKEN")
-	inactiveSecret := testSecrets.MustGetField("PEPIPOST_INACTIVE")
 
-	type args struct {
+func TestPepipost_Pattern(t *testing.T) {
-		ctx    context.Context
+	d := Scanner{}
-		data   []byte
+	ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
-		verify bool
-	}
 	tests := []struct {
-		name    string
+		name  string
-		s       Scanner
+		input string
-		args    args
+		want  []string
-		want    []detectors.Result
-		wantErr bool
 	}{
 		{
-			name: "found, verified",
+			name:  "valid pattern - with keyword pepipost",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
-			args: args{
+			want:  []string{validPattern},
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a pepipost secret %s within", secret)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_Pepipost,
-					Verified:     true,
-				},
-			},
-			wantErr: false,
 		},
 		{
-			name: "found, unverified",
+			name:  "valid pattern - ignore duplicate",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
-			args: args{
+			want:  []string{validPattern},
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a pepipost secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_Pepipost,
-					Verified:     false,
-				},
-			},
-			wantErr: false,
 		},
 		{
-			name: "not found",
+			name:  "valid pattern - key out of prefix range",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
-			args: args{
+			want:  []string{},
-				ctx:    context.Background(),
+		},
-				data:   []byte("You cannot find the secret within"),
+		{
-				verify: true,
+			name:  "invalid pattern",
-			},
+			input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
-			want:    nil,
+			want:  []string{},
-			wantErr: false,
 		},
 	}
-	for _, tt := range tests {
+
-		t.Run(tt.name, func(t *testing.T) {
+	for _, test := range tests {
-			s := Scanner{}
+		t.Run(test.name, func(t *testing.T) {
-			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
-			if (err != nil) != tt.wantErr {
+			if len(matchedDetectors) == 0 {
-				t.Errorf("Pepipost.FromData() error = %v, wantErr %v", err, tt.wantErr)
+				t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
 				return
 			}
-			for i := range got {
-				if len(got[i].Raw) == 0 {
-					t.Fatalf("no raw secret present: \n %+v", got[i])
-				}
-				got[i].Raw = nil
-			}
-			if diff := pretty.Compare(got, tt.want); diff != "" {
-				t.Errorf("Pepipost.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
-			}
-		})
-	}
-}
 
-func BenchmarkFromData(benchmark *testing.B) {
+			results, err := d.FromData(context.Background(), false, []byte(test.input))
-	ctx := context.Background()
+			if err != nil {
-	s := Scanner{}
+				t.Errorf("error = %v", err)
-	for name, data := range detectors.MustGetBenchmarkData() {
+				return
-		benchmark.Run(name, func(b *testing.B) {
+			}
-			b.ResetTimer()
+
-			for n := 0; n < b.N; n++ {
+			if len(results) != len(test.want) {
-				_, err := s.FromData(ctx, false, data)
+				if len(results) == 0 {
-				if err != nil {
+					t.Errorf("did not receive result")
-					b.Fatal(err)
+				} else {
+					t.Errorf("expected %d results, only received %d", len(test.want), len(results))
 				}
+				return
+			}
+
+			actual := make(map[string]struct{}, len(results))
+			for _, r := range results {
+				if len(r.RawV2) > 0 {
+					actual[string(r.RawV2)] = struct{}{}
+				} else {
+					actual[string(r.Raw)] = struct{}{}
+				}
+			}
+			expected := make(map[string]struct{}, len(test.want))
+			for _, v := range test.want {
+				expected[v] = struct{}{}
+			}
+
+			if diff := cmp.Diff(expected, actual); diff != "" {
+				t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
 			}
 		})
 	}

pkg/detectors/percy/percy_integration_test.go

@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package percy
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/kylelemons/godebug/pretty"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestPercy_FromChunk(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	defer cancel()
+	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
+	if err != nil {
+		t.Fatalf("could not get test secrets from GCP: %s", err)
+	}
+	secret := testSecrets.MustGetField("PERCY_VERIFIED")
+	inactiveSecret := testSecrets.MustGetField("PERCY_INACTIVE")
+
+	type args struct {
+		ctx    context.Context
+		data   []byte
+		verify bool
+	}
+	tests := []struct {
+		name    string
+		s       Scanner
+		args    args
+		want    []detectors.Result
+		wantErr bool
+	}{
+		{
+			name: "found, verified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a percy secret %s within", secret)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Percy,
+					Verified:     true,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "found, unverified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a percy secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Percy,
+					Verified:     false,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "not found",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte("You cannot find the secret within"),
+				verify: true,
+			},
+			want:    nil,
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			s := Scanner{}
+			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("Percy.FromData() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			for i := range got {
+				if len(got[i].Raw) == 0 {
+					t.Fatalf("no raw secret present: \n %+v", got[i])
+				}
+				got[i].Raw = nil
+			}
+			if diff := pretty.Compare(got, tt.want); diff != "" {
+				t.Errorf("Percy.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+			}
+		})
+	}
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+	ctx := context.Background()
+	s := Scanner{}
+	for name, data := range detectors.MustGetBenchmarkData() {
+		benchmark.Run(name, func(b *testing.B) {
+			b.ResetTimer()
+			for n := 0; n < b.N; n++ {
+				_, err := s.FromData(ctx, false, data)
+				if err != nil {
+					b.Fatal(err)
+				}
+			}
+		})
+	}
+}

pkg/detectors/percy/percy_test.go

@@ -1,119 +1,90 @@
-//go:build detectors
-// +build detectors
-
 package percy
 
 import (
 	"context"
 	"fmt"
 	"testing"
-	"time"
 
-	"github.com/kylelemons/godebug/pretty"
+	"github.com/google/go-cmp/cmp"
+
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
-
+	"github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 )
 
-func TestPercy_FromChunk(t *testing.T) {
+var (
-	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	validPattern   = "84f2cfA002913e5afbe0a43d71e49ac9389Ab4f4f827bceAed69ec34f844ed22"
-	defer cancel()
+	invalidPattern = "84f2cfA002913?5afbe0a43d71e49ac9389Ab4f4f827bceAed69ec34f844ed22"
-	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
+	keyword        = "percy"
-	if err != nil {
+)
-		t.Fatalf("could not get test secrets from GCP: %s", err)
-	}
-	secret := testSecrets.MustGetField("PERCY_VERIFIED")
-	inactiveSecret := testSecrets.MustGetField("PERCY_INACTIVE")
 
-	type args struct {
+func TestPercy_Pattern(t *testing.T) {
-		ctx    context.Context
+	d := Scanner{}
-		data   []byte
+	ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
-		verify bool
-	}
 	tests := []struct {
-		name    string
+		name  string
-		s       Scanner
+		input string
-		args    args
+		want  []string
-		want    []detectors.Result
-		wantErr bool
 	}{
 		{
-			name: "found, verified",
+			name:  "valid pattern - with keyword percy",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s token = 'PERCY_TOKEN=%s'", keyword, validPattern),
-			args: args{
+			want:  []string{validPattern},
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a percy secret %s within", secret)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_Percy,
-					Verified:     true,
-				},
-			},
-			wantErr: false,
 		},
 		{
-			name: "found, unverified",
+			name:  "valid pattern - ignore duplicate",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s token = 'PERCY_TOKEN=%s' | 'PERCY_TOKEN=%s'", keyword, validPattern, validPattern),
-			args: args{
+			want:  []string{validPattern},
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a percy secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_Percy,
-					Verified:     false,
-				},
-			},
-			wantErr: false,
 		},
 		{
-			name: "not found",
+			name:  "valid pattern - key out of prefix range",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = 'PERCY_TOKEN=%s'", keyword, validPattern),
-			args: args{
+			want:  []string{},
-				ctx:    context.Background(),
+		},
-				data:   []byte("You cannot find the secret within"),
+		{
-				verify: true,
+			name:  "invalid pattern",
-			},
+			input: fmt.Sprintf("%s = 'PERCY_TOKEN=%s'", keyword, invalidPattern),
-			want:    nil,
+			want:  []string{},
-			wantErr: false,
 		},
 	}
-	for _, tt := range tests {
+
-		t.Run(tt.name, func(t *testing.T) {
+	for _, test := range tests {
-			s := Scanner{}
+		t.Run(test.name, func(t *testing.T) {
-			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
-			if (err != nil) != tt.wantErr {
+			if len(matchedDetectors) == 0 {
-				t.Errorf("Percy.FromData() error = %v, wantErr %v", err, tt.wantErr)
+				t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
 				return
 			}
-			for i := range got {
-				if len(got[i].Raw) == 0 {
-					t.Fatalf("no raw secret present: \n %+v", got[i])
-				}
-				got[i].Raw = nil
-			}
-			if diff := pretty.Compare(got, tt.want); diff != "" {
-				t.Errorf("Percy.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
-			}
-		})
-	}
-}
 
-func BenchmarkFromData(benchmark *testing.B) {
+			results, err := d.FromData(context.Background(), false, []byte(test.input))
-	ctx := context.Background()
+			if err != nil {
-	s := Scanner{}
+				t.Errorf("error = %v", err)
-	for name, data := range detectors.MustGetBenchmarkData() {
+				return
-		benchmark.Run(name, func(b *testing.B) {
+			}
-			b.ResetTimer()
+
-			for n := 0; n < b.N; n++ {
+			if len(results) != len(test.want) {
-				_, err := s.FromData(ctx, false, data)
+				if len(results) == 0 {
-				if err != nil {
+					t.Errorf("did not receive result")
-					b.Fatal(err)
+				} else {
+					t.Errorf("expected %d results, only received %d", len(test.want), len(results))
 				}
+				return
+			}
+
+			actual := make(map[string]struct{}, len(results))
+			for _, r := range results {
+				if len(r.RawV2) > 0 {
+					actual[string(r.RawV2)] = struct{}{}
+				} else {
+					actual[string(r.Raw)] = struct{}{}
+				}
+			}
+			expected := make(map[string]struct{}, len(test.want))
+			for _, v := range test.want {
+				expected[v] = struct{}{}
+			}
+
+			if diff := cmp.Diff(expected, actual); diff != "" {
+				t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
 			}
 		})
 	}

pkg/detectors/pinata/pinata_integration_test.go

@@ -0,0 +1,121 @@
+//go:build detectors
+// +build detectors
+
+package pinata
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/kylelemons/godebug/pretty"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestPinata_FromChunk(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	defer cancel()
+	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+	if err != nil {
+		t.Fatalf("could not get test secrets from GCP: %s", err)
+	}
+	secret := testSecrets.MustGetField("PINATA")
+	key := testSecrets.MustGetField("PINATA_KEY")
+	inactiveSecret := testSecrets.MustGetField("PINATA_INACTIVE")
+
+	type args struct {
+		ctx    context.Context
+		data   []byte
+		verify bool
+	}
+	tests := []struct {
+		name    string
+		s       Scanner
+		args    args
+		want    []detectors.Result
+		wantErr bool
+	}{
+		{
+			name: "found, verified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a pinata secret %s within pinata %s", secret, key)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Pinata,
+					Verified:     true,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "found, unverified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a pinata secret %s within pinata %s but not valid", inactiveSecret, key)), // the secret would satisfy the regex but not pass validation
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Pinata,
+					Verified:     false,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "not found",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte("You cannot find the secret within"),
+				verify: true,
+			},
+			want:    nil,
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			s := Scanner{}
+			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("Pinata.FromData() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			for i := range got {
+				if len(got[i].Raw) == 0 {
+					t.Fatalf("no raw secret present: \n %+v", got[i])
+				}
+				got[i].Raw = nil
+			}
+			if diff := pretty.Compare(got, tt.want); diff != "" {
+				t.Errorf("Pinata.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+			}
+		})
+	}
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+	ctx := context.Background()
+	s := Scanner{}
+	for name, data := range detectors.MustGetBenchmarkData() {
+		benchmark.Run(name, func(b *testing.B) {
+			b.ResetTimer()
+			for n := 0; n < b.N; n++ {
+				_, err := s.FromData(ctx, false, data)
+				if err != nil {
+					b.Fatal(err)
+				}
+			}
+		})
+	}
+}

pkg/detectors/pinata/pinata_test.go

@@ -1,120 +1,82 @@
-//go:build detectors
-// +build detectors
-
 package pinata
 
 import (
 	"context"
 	"fmt"
 	"testing"
-	"time"
 
-	"github.com/kylelemons/godebug/pretty"
+	"github.com/google/go-cmp/cmp"
+
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
-
+	"github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 )
 
-func TestPinata_FromChunk(t *testing.T) {
+var (
-	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+    validKey   = "xw4xxzwjzerq6rf3zvd8zwnlh0yq62g4f7l97xxlg4u1043zrx4ndtptkoqdn49e"
-	defer cancel()
+    invalidKey = "xw4xxzwjzerq6r?3zvd8zwnlh0yq62g4f7l97xxlg4u1043zrx4ndtptkoqdn49e"
-	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+    validId    = "lnurl7e15mgdgv1hyalo"
-	if err != nil {
+    invalidId  = "lnurl7e15m?dgv1hyalo"
-		t.Fatalf("could not get test secrets from GCP: %s", err)
+    keyword    = "pinata"
-	}
+)
-	secret := testSecrets.MustGetField("PINATA")
-	key := testSecrets.MustGetField("PINATA_KEY")
-	inactiveSecret := testSecrets.MustGetField("PINATA_INACTIVE")
 
-	type args struct {
+func TestPinata_Pattern(t *testing.T) {
-		ctx    context.Context
+	d := Scanner{}
-		data   []byte
+	ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
-		verify bool
-	}
 	tests := []struct {
-		name    string
+		name  string
-		s       Scanner
+		input string
-		args    args
+		want  []string
-		want    []detectors.Result
-		wantErr bool
 	}{
 		{
-			name: "found, verified",
+			name:  "valid pattern - with keyword pinata",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, validKey, keyword, validId),
-			args: args{
+			want:  []string{validKey},
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a pinata secret %s within pinata %s", secret, key)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_Pinata,
-					Verified:     true,
-				},
-			},
-			wantErr: false,
 		},
 		{
-			name: "found, unverified",
+			name:  "invalid pattern",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, invalidKey, keyword, invalidId),
-			args: args{
+			want:  []string{},
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a pinata secret %s within pinata %s but not valid", inactiveSecret, key)), // the secret would satisfy the regex but not pass validation
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_Pinata,
-					Verified:     false,
-				},
-			},
-			wantErr: false,
-		},
-		{
-			name: "not found",
-			s:    Scanner{},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte("You cannot find the secret within"),
-				verify: true,
-			},
-			want:    nil,
-			wantErr: false,
 		},
 	}
-	for _, tt := range tests {
+
-		t.Run(tt.name, func(t *testing.T) {
+	for _, test := range tests {
-			s := Scanner{}
+		t.Run(test.name, func(t *testing.T) {
-			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
-			if (err != nil) != tt.wantErr {
+			if len(matchedDetectors) == 0 {
-				t.Errorf("Pinata.FromData() error = %v, wantErr %v", err, tt.wantErr)
+				t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
 				return
 			}
-			for i := range got {
-				if len(got[i].Raw) == 0 {
-					t.Fatalf("no raw secret present: \n %+v", got[i])
-				}
-				got[i].Raw = nil
-			}
-			if diff := pretty.Compare(got, tt.want); diff != "" {
-				t.Errorf("Pinata.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
-			}
-		})
-	}
-}
 
-func BenchmarkFromData(benchmark *testing.B) {
+			results, err := d.FromData(context.Background(), false, []byte(test.input))
-	ctx := context.Background()
+			if err != nil {
-	s := Scanner{}
+				t.Errorf("error = %v", err)
-	for name, data := range detectors.MustGetBenchmarkData() {
+				return
-		benchmark.Run(name, func(b *testing.B) {
+			}
-			b.ResetTimer()
+
-			for n := 0; n < b.N; n++ {
+			if len(results) != len(test.want) {
-				_, err := s.FromData(ctx, false, data)
+				if len(results) == 0 {
-				if err != nil {
+					t.Errorf("did not receive result")
-					b.Fatal(err)
+				} else {
+					t.Errorf("expected %d results, only received %d", len(test.want), len(results))
 				}
+				return
+			}
+
+			actual := make(map[string]struct{}, len(results))
+			for _, r := range results {
+				if len(r.RawV2) > 0 {
+					actual[string(r.RawV2)] = struct{}{}
+				} else {
+					actual[string(r.Raw)] = struct{}{}
+				}
+			}
+			expected := make(map[string]struct{}, len(test.want))
+			for _, v := range test.want {
+				expected[v] = struct{}{}
+			}
+
+			if diff := cmp.Diff(expected, actual); diff != "" {
+				t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
 			}
 		})
 	}

pkg/detectors/pipedream/pipedream_integration_test.go

@@ -0,0 +1,117 @@
+//go:build detectors
+// +build detectors
+
+package pipedream
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/kylelemons/godebug/pretty"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestPipedream_FromChunk(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	defer cancel()
+	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
+	if err != nil {
+		t.Fatalf("could not get test secrets from GCP: %s", err)
+	}
+	secret := testSecrets.MustGetField("PIPEDREAM")
+	inactiveSecret := testSecrets.MustGetField("PIPEDREAM_INACTIVE")
+
+	type args struct {
+		ctx    context.Context
+		data   []byte
+		verify bool
+	}
+	tests := []struct {
+		name    string
+		s       Scanner
+		args    args
+		want    []detectors.Result
+		wantErr bool
+	}{
+		{
+			name: "found, verified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a pipedream secret %s within", secret)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Pipedream,
+					Verified:     true,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "found, unverified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a pipedream secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Pipedream,
+					Verified:     false,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "not found",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte("You cannot find the secret within"),
+				verify: true,
+			},
+			want:    nil,
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			s := Scanner{}
+			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("Pipedream.FromData() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			for i := range got {
+				if len(got[i].Raw) == 0 {
+					t.Fatalf("no raw secret present: \n %+v", got[i])
+				}
+				got[i].Raw = nil
+			}
+			if diff := pretty.Compare(got, tt.want); diff != "" {
+				t.Errorf("Pipedream.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+			}
+		})
+	}
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+	ctx := context.Background()
+	s := Scanner{}
+	for name, data := range detectors.MustGetBenchmarkData() {
+		benchmark.Run(name, func(b *testing.B) {
+			b.ResetTimer()
+			for n := 0; n < b.N; n++ {
+				s.FromData(ctx, false, data)
+			}
+		})
+	}
+}

pkg/detectors/pipedream/pipedream_test.go

@@ -1,116 +1,90 @@
-//go:build detectors
-// +build detectors
-
 package pipedream
 
 import (
 	"context"
 	"fmt"
 	"testing"
-	"time"
 
-	"github.com/kylelemons/godebug/pretty"
+	"github.com/google/go-cmp/cmp"
 
-	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
 )
 
-func TestPipedream_FromChunk(t *testing.T) {
+var (
-	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	validPattern   = "dh6zkbdzrjlsz5gy4gj2zg9stqttbd65"
-	defer cancel()
+	invalidPattern = "dh6zkbdzrjlsz5gy?gj2zg9stqttbd65"
-	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
+	keyword        = "pipedream"
-	if err != nil {
+)
-		t.Fatalf("could not get test secrets from GCP: %s", err)
-	}
-	secret := testSecrets.MustGetField("PIPEDREAM")
-	inactiveSecret := testSecrets.MustGetField("PIPEDREAM_INACTIVE")
 
-	type args struct {
+func TestPipedream_Pattern(t *testing.T) {
-		ctx    context.Context
+	d := Scanner{}
-		data   []byte
+	ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
-		verify bool
-	}
 	tests := []struct {
-		name    string
+		name  string
-		s       Scanner
+		input string
-		args    args
+		want  []string
-		want    []detectors.Result
-		wantErr bool
 	}{
 		{
-			name: "found, verified",
+			name:  "valid pattern - with keyword pipedream",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
-			args: args{
+			want:  []string{validPattern},
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a pipedream secret %s within", secret)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_Pipedream,
-					Verified:     true,
-				},
-			},
-			wantErr: false,
 		},
 		{
-			name: "found, unverified",
+			name:  "valid pattern - ignore duplicate",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
-			args: args{
+			want:  []string{validPattern},
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a pipedream secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_Pipedream,
-					Verified:     false,
-				},
-			},
-			wantErr: false,
 		},
 		{
-			name: "not found",
+			name:  "valid pattern - key out of prefix range",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
-			args: args{
+			want:  []string{},
-				ctx:    context.Background(),
+		},
-				data:   []byte("You cannot find the secret within"),
+		{
-				verify: true,
+			name:  "invalid pattern",
-			},
+			input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
-			want:    nil,
+			want:  []string{},
-			wantErr: false,
 		},
 	}
-	for _, tt := range tests {
+
-		t.Run(tt.name, func(t *testing.T) {
+	for _, test := range tests {
-			s := Scanner{}
+		t.Run(test.name, func(t *testing.T) {
-			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
-			if (err != nil) != tt.wantErr {
+			if len(matchedDetectors) == 0 {
-				t.Errorf("Pipedream.FromData() error = %v, wantErr %v", err, tt.wantErr)
+				t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
 				return
 			}
-			for i := range got {
-				if len(got[i].Raw) == 0 {
-					t.Fatalf("no raw secret present: \n %+v", got[i])
-				}
-				got[i].Raw = nil
-			}
-			if diff := pretty.Compare(got, tt.want); diff != "" {
-				t.Errorf("Pipedream.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
-			}
-		})
-	}
-}
 
-func BenchmarkFromData(benchmark *testing.B) {
+			results, err := d.FromData(context.Background(), false, []byte(test.input))
-	ctx := context.Background()
+			if err != nil {
-	s := Scanner{}
+				t.Errorf("error = %v", err)
-	for name, data := range detectors.MustGetBenchmarkData() {
+				return
-		benchmark.Run(name, func(b *testing.B) {
+			}
-			b.ResetTimer()
+
-			for n := 0; n < b.N; n++ {
+			if len(results) != len(test.want) {
-				s.FromData(ctx, false, data)
+				if len(results) == 0 {
+					t.Errorf("did not receive result")
+				} else {
+					t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+				}
+				return
+			}
+
+			actual := make(map[string]struct{}, len(results))
+			for _, r := range results {
+				if len(r.RawV2) > 0 {
+					actual[string(r.RawV2)] = struct{}{}
+				} else {
+					actual[string(r.Raw)] = struct{}{}
+				}
+			}
+			expected := make(map[string]struct{}, len(test.want))
+			for _, v := range test.want {
+				expected[v] = struct{}{}
+			}
+
+			if diff := cmp.Diff(expected, actual); diff != "" {
+				t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
 			}
 		})
 	}

pkg/detectors/pipedrive/pipedrive_integration_test.go

@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package pipedrive
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/kylelemons/godebug/pretty"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestPipedrive_FromChunk(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	defer cancel()
+	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+	if err != nil {
+		t.Fatalf("could not get test secrets from GCP: %s", err)
+	}
+	secret := testSecrets.MustGetField("PIPEDRIVE_TOKEN")
+	inactiveSecret := testSecrets.MustGetField("PIPEDRIVE_INACTIVE")
+
+	type args struct {
+		ctx    context.Context
+		data   []byte
+		verify bool
+	}
+	tests := []struct {
+		name    string
+		s       Scanner
+		args    args
+		want    []detectors.Result
+		wantErr bool
+	}{
+		{
+			name: "found, verified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a pipedrive secret %s within", secret)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Pipedrive,
+					Verified:     true,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "found, unverified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a pipedrive secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Pipedrive,
+					Verified:     false,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "not found",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte("You cannot find the secret within"),
+				verify: true,
+			},
+			want:    nil,
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			s := Scanner{}
+			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("Pipedrive.FromData() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			for i := range got {
+				if len(got[i].Raw) == 0 {
+					t.Fatalf("no raw secret present: \n %+v", got[i])
+				}
+				got[i].Raw = nil
+			}
+			if diff := pretty.Compare(got, tt.want); diff != "" {
+				t.Errorf("Pipedrive.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+			}
+		})
+	}
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+	ctx := context.Background()
+	s := Scanner{}
+	for name, data := range detectors.MustGetBenchmarkData() {
+		benchmark.Run(name, func(b *testing.B) {
+			b.ResetTimer()
+			for n := 0; n < b.N; n++ {
+				_, err := s.FromData(ctx, false, data)
+				if err != nil {
+					b.Fatal(err)
+				}
+			}
+		})
+	}
+}

pkg/detectors/pipedrive/pipedrive_test.go

@@ -1,119 +1,90 @@
-//go:build detectors
-// +build detectors
-
 package pipedrive
 
 import (
 	"context"
 	"fmt"
 	"testing"
-	"time"
 
-	"github.com/kylelemons/godebug/pretty"
+	"github.com/google/go-cmp/cmp"
+
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
-
+	"github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 )
 
-func TestPipedrive_FromChunk(t *testing.T) {
+var (
-	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	validPattern   = "ZVt2kcilMoT5gvZDppcXkF5JPzyinJrkSwroZ9dB"
-	defer cancel()
+	invalidPattern = "ZVt2?cilMoT5gvZDppcXkF5JPzyinJrkSwroZ9dB"
-	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+	keyword        = "pipedrive"
-	if err != nil {
+)
-		t.Fatalf("could not get test secrets from GCP: %s", err)
-	}
-	secret := testSecrets.MustGetField("PIPEDRIVE_TOKEN")
-	inactiveSecret := testSecrets.MustGetField("PIPEDRIVE_INACTIVE")
 
-	type args struct {
+func TestPipedrive_Pattern(t *testing.T) {
-		ctx    context.Context
+	d := Scanner{}
-		data   []byte
+	ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
-		verify bool
-	}
 	tests := []struct {
-		name    string
+		name  string
-		s       Scanner
+		input string
-		args    args
+		want  []string
-		want    []detectors.Result
-		wantErr bool
 	}{
 		{
-			name: "found, verified",
+			name:  "valid pattern - with keyword pipedrive",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
-			args: args{
+			want:  []string{validPattern},
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a pipedrive secret %s within", secret)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_Pipedrive,
-					Verified:     true,
-				},
-			},
-			wantErr: false,
 		},
 		{
-			name: "found, unverified",
+			name:  "valid pattern - ignore duplicate",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
-			args: args{
+			want:  []string{validPattern},
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a pipedrive secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_Pipedrive,
-					Verified:     false,
-				},
-			},
-			wantErr: false,
 		},
 		{
-			name: "not found",
+			name:  "valid pattern - key out of prefix range",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
-			args: args{
+			want:  []string{},
-				ctx:    context.Background(),
+		},
-				data:   []byte("You cannot find the secret within"),
+		{
-				verify: true,
+			name:  "invalid pattern",
-			},
+			input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
-			want:    nil,
+			want:  []string{},
-			wantErr: false,
 		},
 	}
-	for _, tt := range tests {
+
-		t.Run(tt.name, func(t *testing.T) {
+	for _, test := range tests {
-			s := Scanner{}
+		t.Run(test.name, func(t *testing.T) {
-			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
-			if (err != nil) != tt.wantErr {
+			if len(matchedDetectors) == 0 {
-				t.Errorf("Pipedrive.FromData() error = %v, wantErr %v", err, tt.wantErr)
+				t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
 				return
 			}
-			for i := range got {
-				if len(got[i].Raw) == 0 {
-					t.Fatalf("no raw secret present: \n %+v", got[i])
-				}
-				got[i].Raw = nil
-			}
-			if diff := pretty.Compare(got, tt.want); diff != "" {
-				t.Errorf("Pipedrive.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
-			}
-		})
-	}
-}
 
-func BenchmarkFromData(benchmark *testing.B) {
+			results, err := d.FromData(context.Background(), false, []byte(test.input))
-	ctx := context.Background()
+			if err != nil {
-	s := Scanner{}
+				t.Errorf("error = %v", err)
-	for name, data := range detectors.MustGetBenchmarkData() {
+				return
-		benchmark.Run(name, func(b *testing.B) {
+			}
-			b.ResetTimer()
+
-			for n := 0; n < b.N; n++ {
+			if len(results) != len(test.want) {
-				_, err := s.FromData(ctx, false, data)
+				if len(results) == 0 {
-				if err != nil {
+					t.Errorf("did not receive result")
-					b.Fatal(err)
+				} else {
+					t.Errorf("expected %d results, only received %d", len(test.want), len(results))
 				}
+				return
+			}
+
+			actual := make(map[string]struct{}, len(results))
+			for _, r := range results {
+				if len(r.RawV2) > 0 {
+					actual[string(r.RawV2)] = struct{}{}
+				} else {
+					actual[string(r.Raw)] = struct{}{}
+				}
+			}
+			expected := make(map[string]struct{}, len(test.want))
+			for _, v := range test.want {
+				expected[v] = struct{}{}
+			}
+
+			if diff := cmp.Diff(expected, actual); diff != "" {
+				t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
 			}
 		})
 	}

pkg/detectors/pivotaltracker/pivotaltracker_integration_test.go

@@ -0,0 +1,119 @@
+//go:build detectors
+// +build detectors
+
+package pivotaltracker
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/kylelemons/godebug/pretty"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestPivotalTracker_FromChunk(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	defer cancel()
+	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
+	if err != nil {
+		t.Fatalf("could not get test secrets from GCP: %s", err)
+	}
+	secret := testSecrets.MustGetField("PIVOTALTRACKER")
+	secretInactive := testSecrets.MustGetField("PIVOTALTRACKER_INACTIVE")
+
+	type args struct {
+		ctx    context.Context
+		data   []byte
+		verify bool
+	}
+	tests := []struct {
+		name    string
+		s       Scanner
+		args    args
+		want    []detectors.Result
+		wantErr bool
+	}{
+		{
+			name: "found, verified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a pivotal secret %s within", secret)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_PivotalTracker,
+					Verified:     true,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "found, verified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a pivotal secret %s within", secretInactive)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_PivotalTracker,
+					Verified:     false,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "not found",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte("You cannot find the secret within"),
+				verify: true,
+			},
+			want:    nil,
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			s := Scanner{}
+			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("PivotalTracker.FromData() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			for i := range got {
+				if len(got[i].Raw) == 0 {
+					t.Fatal("no raw secret present")
+				}
+				got[i].Raw = nil
+			}
+			if diff := pretty.Compare(got, tt.want); diff != "" {
+				t.Errorf("PivotalTracker.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+			}
+		})
+	}
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+	ctx := context.Background()
+	s := Scanner{}
+	for name, data := range detectors.MustGetBenchmarkData() {
+		benchmark.Run(name, func(b *testing.B) {
+			b.ResetTimer()
+			for n := 0; n < b.N; n++ {
+				_, err := s.FromData(ctx, false, data)
+				if err != nil {
+					b.Fatal(err)
+				}
+			}
+		})
+	}
+}

pkg/detectors/pivotaltracker/pivotaltracker_test.go

@@ -1,118 +1,90 @@
-//go:build detectors
-// +build detectors
-
 package pivotaltracker
 
 import (
 	"context"
 	"fmt"
 	"testing"
-	"time"
 
-	"github.com/kylelemons/godebug/pretty"
+	"github.com/google/go-cmp/cmp"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
 )
 
-func TestPivotalTracker_FromChunk(t *testing.T) {
+var (
-	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	validPattern   = "hzwlc7d249nw76xwx372n09spjhjgidv"
-	defer cancel()
+	invalidPattern = "hzwlc7d249nw76?wx372n09spjhjgidv"
-	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
+	keyword        = "pivotaltracker"
-	if err != nil {
+)
-		t.Fatalf("could not get test secrets from GCP: %s", err)
-	}
-	secret := testSecrets.MustGetField("PIVOTALTRACKER")
-	secretInactive := testSecrets.MustGetField("PIVOTALTRACKER_INACTIVE")
 
-	type args struct {
+func TestPivotalTracker_Pattern(t *testing.T) {
-		ctx    context.Context
+	d := Scanner{}
-		data   []byte
+	ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
-		verify bool
-	}
 	tests := []struct {
-		name    string
+		name  string
-		s       Scanner
+		input string
-		args    args
+		want  []string
-		want    []detectors.Result
-		wantErr bool
 	}{
 		{
-			name: "found, verified",
+			name:  "valid pattern - with keyword pivotaltracker",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
-			args: args{
+			want:  []string{validPattern},
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a pivotal secret %s within", secret)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_PivotalTracker,
-					Verified:     true,
-				},
-			},
-			wantErr: false,
 		},
 		{
-			name: "found, verified",
+			name:  "valid pattern - ignore duplicate",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
-			args: args{
+			want:  []string{validPattern},
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a pivotal secret %s within", secretInactive)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_PivotalTracker,
-					Verified:     false,
-				},
-			},
-			wantErr: false,
 		},
 		{
-			name: "not found",
+			name:  "valid pattern - key out of prefix range",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
-			args: args{
+			want:  []string{},
-				ctx:    context.Background(),
+		},
-				data:   []byte("You cannot find the secret within"),
+		{
-				verify: true,
+			name:  "invalid pattern",
-			},
+			input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
-			want:    nil,
+			want:  []string{},
-			wantErr: false,
 		},
 	}
-	for _, tt := range tests {
+
-		t.Run(tt.name, func(t *testing.T) {
+	for _, test := range tests {
-			s := Scanner{}
+		t.Run(test.name, func(t *testing.T) {
-			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
-			if (err != nil) != tt.wantErr {
+			if len(matchedDetectors) == 0 {
-				t.Errorf("PivotalTracker.FromData() error = %v, wantErr %v", err, tt.wantErr)
+				t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
 				return
 			}
-			for i := range got {
-				if len(got[i].Raw) == 0 {
-					t.Fatal("no raw secret present")
-				}
-				got[i].Raw = nil
-			}
-			if diff := pretty.Compare(got, tt.want); diff != "" {
-				t.Errorf("PivotalTracker.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
-			}
-		})
-	}
-}
 
-func BenchmarkFromData(benchmark *testing.B) {
+			results, err := d.FromData(context.Background(), false, []byte(test.input))
-	ctx := context.Background()
+			if err != nil {
-	s := Scanner{}
+				t.Errorf("error = %v", err)
-	for name, data := range detectors.MustGetBenchmarkData() {
+				return
-		benchmark.Run(name, func(b *testing.B) {
+			}
-			b.ResetTimer()
+
-			for n := 0; n < b.N; n++ {
+			if len(results) != len(test.want) {
-				_, err := s.FromData(ctx, false, data)
+				if len(results) == 0 {
-				if err != nil {
+					t.Errorf("did not receive result")
-					b.Fatal(err)
+				} else {
+					t.Errorf("expected %d results, only received %d", len(test.want), len(results))
 				}
+				return
+			}
+
+			actual := make(map[string]struct{}, len(results))
+			for _, r := range results {
+				if len(r.RawV2) > 0 {
+					actual[string(r.RawV2)] = struct{}{}
+				} else {
+					actual[string(r.Raw)] = struct{}{}
+				}
+			}
+			expected := make(map[string]struct{}, len(test.want))
+			for _, v := range test.want {
+				expected[v] = struct{}{}
+			}
+
+			if diff := cmp.Diff(expected, actual); diff != "" {
+				t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
 			}
 		})
 	}

pkg/detectors/pixabay/pixabay_integration_test.go

@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package pixabay
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/kylelemons/godebug/pretty"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestPixabay_FromChunk(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	defer cancel()
+	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
+	if err != nil {
+		t.Fatalf("could not get test secrets from GCP: %s", err)
+	}
+	secret := testSecrets.MustGetField("PIXABAY")
+	inactiveSecret := testSecrets.MustGetField("PIXABAY_INACTIVE")
+
+	type args struct {
+		ctx    context.Context
+		data   []byte
+		verify bool
+	}
+	tests := []struct {
+		name    string
+		s       Scanner
+		args    args
+		want    []detectors.Result
+		wantErr bool
+	}{
+		{
+			name: "found, verified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a pixabay secret %s within", secret)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Pixabay,
+					Verified:     true,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "found, unverified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a pixabay secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Pixabay,
+					Verified:     false,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "not found",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte("You cannot find the secret within"),
+				verify: true,
+			},
+			want:    nil,
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			s := Scanner{}
+			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("Pixabay.FromData() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			for i := range got {
+				if len(got[i].Raw) == 0 {
+					t.Fatalf("no raw secret present: \n %+v", got[i])
+				}
+				got[i].Raw = nil
+			}
+			if diff := pretty.Compare(got, tt.want); diff != "" {
+				t.Errorf("Pixabay.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+			}
+		})
+	}
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+	ctx := context.Background()
+	s := Scanner{}
+	for name, data := range detectors.MustGetBenchmarkData() {
+		benchmark.Run(name, func(b *testing.B) {
+			b.ResetTimer()
+			for n := 0; n < b.N; n++ {
+				_, err := s.FromData(ctx, false, data)
+				if err != nil {
+					b.Fatal(err)
+				}
+			}
+		})
+	}
+}

pkg/detectors/pixabay/pixabay_test.go

@@ -1,119 +1,90 @@
-//go:build detectors
-// +build detectors
-
 package pixabay
 
 import (
 	"context"
 	"fmt"
 	"testing"
-	"time"
 
-	"github.com/kylelemons/godebug/pretty"
+	"github.com/google/go-cmp/cmp"
 
-	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
 )
 
-func TestPixabay_FromChunk(t *testing.T) {
+var (
-	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	validPattern   = "yz4gjwgew94zn73y9ds7hlwob6ytl70827"
-	defer cancel()
+	invalidPattern = "yz4gjwgew94zn73y9?s7hlwob6ytl70827"
-	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
+	keyword        = "pixabay"
-	if err != nil {
+)
-		t.Fatalf("could not get test secrets from GCP: %s", err)
-	}
-	secret := testSecrets.MustGetField("PIXABAY")
-	inactiveSecret := testSecrets.MustGetField("PIXABAY_INACTIVE")
 
-	type args struct {
+func TestPixabay_Pattern(t *testing.T) {
-		ctx    context.Context
+	d := Scanner{}
-		data   []byte
+	ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
-		verify bool
-	}
 	tests := []struct {
-		name    string
+		name  string
-		s       Scanner
+		input string
-		args    args
+		want  []string
-		want    []detectors.Result
-		wantErr bool
 	}{
 		{
-			name: "found, verified",
+			name:  "valid pattern - with keyword pixabay",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
-			args: args{
+			want:  []string{validPattern},
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a pixabay secret %s within", secret)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_Pixabay,
-					Verified:     true,
-				},
-			},
-			wantErr: false,
 		},
 		{
-			name: "found, unverified",
+			name:  "valid pattern - ignore duplicate",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
-			args: args{
+			want:  []string{validPattern},
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a pixabay secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_Pixabay,
-					Verified:     false,
-				},
-			},
-			wantErr: false,
 		},
 		{
-			name: "not found",
+			name:  "valid pattern - key out of prefix range",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
-			args: args{
+			want:  []string{},
-				ctx:    context.Background(),
+		},
-				data:   []byte("You cannot find the secret within"),
+		{
-				verify: true,
+			name:  "invalid pattern",
-			},
+			input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
-			want:    nil,
+			want:  []string{},
-			wantErr: false,
 		},
 	}
-	for _, tt := range tests {
+
-		t.Run(tt.name, func(t *testing.T) {
+	for _, test := range tests {
-			s := Scanner{}
+		t.Run(test.name, func(t *testing.T) {
-			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
-			if (err != nil) != tt.wantErr {
+			if len(matchedDetectors) == 0 {
-				t.Errorf("Pixabay.FromData() error = %v, wantErr %v", err, tt.wantErr)
+				t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
 				return
 			}
-			for i := range got {
-				if len(got[i].Raw) == 0 {
-					t.Fatalf("no raw secret present: \n %+v", got[i])
-				}
-				got[i].Raw = nil
-			}
-			if diff := pretty.Compare(got, tt.want); diff != "" {
-				t.Errorf("Pixabay.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
-			}
-		})
-	}
-}
 
-func BenchmarkFromData(benchmark *testing.B) {
+			results, err := d.FromData(context.Background(), false, []byte(test.input))
-	ctx := context.Background()
+			if err != nil {
-	s := Scanner{}
+				t.Errorf("error = %v", err)
-	for name, data := range detectors.MustGetBenchmarkData() {
+				return
-		benchmark.Run(name, func(b *testing.B) {
+			}
-			b.ResetTimer()
+
-			for n := 0; n < b.N; n++ {
+			if len(results) != len(test.want) {
-				_, err := s.FromData(ctx, false, data)
+				if len(results) == 0 {
-				if err != nil {
+					t.Errorf("did not receive result")
-					b.Fatal(err)
+				} else {
+					t.Errorf("expected %d results, only received %d", len(test.want), len(results))
 				}
+				return
+			}
+
+			actual := make(map[string]struct{}, len(results))
+			for _, r := range results {
+				if len(r.RawV2) > 0 {
+					actual[string(r.RawV2)] = struct{}{}
+				} else {
+					actual[string(r.Raw)] = struct{}{}
+				}
+			}
+			expected := make(map[string]struct{}, len(test.want))
+			for _, v := range test.want {
+				expected[v] = struct{}{}
+			}
+
+			if diff := cmp.Diff(expected, actual); diff != "" {
+				t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
 			}
 		})
 	}

pkg/detectors/plaidkey/plaidkey_integration_test.go

@@ -0,0 +1,126 @@
+//go:build detectors
+// +build detectors
+
+package plaidkey
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/kylelemons/godebug/pretty"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestPlaidKey_FromChunk(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	defer cancel()
+	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+	if err != nil {
+		t.Fatalf("could not get test secrets from GCP: %s", err)
+	}
+	secret := testSecrets.MustGetField("PLAIDKEY_SECRET")
+	inactiveSecret := testSecrets.MustGetField("PLAIDKEY_SECRET_INACTIVE")
+	id := testSecrets.MustGetField("PLAIDKEY_CLIENTID")
+	// env := testSecrets.MustGetField("PLAIDKEY_ENVIRONMENT") // development or production
+	env := "development"
+
+	type args struct {
+		ctx    context.Context
+		data   []byte
+		verify bool
+	}
+	tests := []struct {
+		name    string
+		s       Scanner
+		args    args
+		want    []detectors.Result
+		wantErr bool
+	}{
+		{
+			name: "found, verified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a plaidkey secret %s within plaidkey %s", secret, id)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_PlaidKey,
+					Verified:     true,
+					ExtraData: map[string]string{
+						"environment": fmt.Sprintf("https://%s.plaid.com", env),
+					},
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "found, unverified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a plaidkey secret %s within but plaidkey %s not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_PlaidKey,
+					Verified:     false,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "not found",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte("You cannot find the secret within"),
+				verify: true,
+			},
+			want:    nil,
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			s := Scanner{}
+			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("PlaidKey.FromData() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			for i := range got {
+				if len(got[i].Raw) == 0 {
+					t.Fatalf("no raw secret present: \n %+v", got[i])
+				}
+				got[i].Raw = nil
+			}
+			if diff := pretty.Compare(got, tt.want); diff != "" {
+				t.Errorf("PlaidKey.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+			}
+		})
+	}
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+	ctx := context.Background()
+	s := Scanner{}
+	for name, data := range detectors.MustGetBenchmarkData() {
+		benchmark.Run(name, func(b *testing.B) {
+			b.ResetTimer()
+			for n := 0; n < b.N; n++ {
+				_, err := s.FromData(ctx, false, data)
+				if err != nil {
+					b.Fatal(err)
+				}
+			}
+		})
+	}
+}

pkg/detectors/plaidkey/plaidkey_test.go

@@ -1,125 +1,82 @@
-//go:build detectors
-// +build detectors
-
 package plaidkey
 
 import (
 	"context"
 	"fmt"
 	"testing"
-	"time"
 
-	"github.com/kylelemons/godebug/pretty"
+	"github.com/google/go-cmp/cmp"
+
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
-
+	"github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 )
 
-func TestPlaidKey_FromChunk(t *testing.T) {
+var (
-	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	validKey   = "3vl81ihtozf9im7kqz7ldp6kxbsd8y"
-	defer cancel()
+	invalidKey = "3vl81ihtozf9im7?qz7ldp6kxbsd8y"
-	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+	validId    = "ic1mh5b49ycvmz2vgvlgxtb0"
-	if err != nil {
+	invalidId  = "ic1?h5b49ycvmz2vgvlgxtb0"
-		t.Fatalf("could not get test secrets from GCP: %s", err)
+	keyword    = "plaid"
-	}
+)
-	secret := testSecrets.MustGetField("PLAIDKEY_SECRET")
-	inactiveSecret := testSecrets.MustGetField("PLAIDKEY_SECRET_INACTIVE")
-	id := testSecrets.MustGetField("PLAIDKEY_CLIENTID")
-	// env := testSecrets.MustGetField("PLAIDKEY_ENVIRONMENT") // development or production
-	env := "development"
 
-	type args struct {
+func TestPlaidKey_Pattern(t *testing.T) {
-		ctx    context.Context
+	d := Scanner{}
-		data   []byte
+	ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
-		verify bool
-	}
 	tests := []struct {
-		name    string
+		name  string
-		s       Scanner
+		input string
-		args    args
+		want  []string
-		want    []detectors.Result
-		wantErr bool
 	}{
 		{
-			name: "found, verified",
+			name:  "valid pattern - with keyword plaid",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, validKey, keyword, validId),
-			args: args{
+			want:  []string{validKey},
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a plaidkey secret %s within plaidkey %s", secret, id)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_PlaidKey,
-					Verified:     true,
-					ExtraData: map[string]string{
-						"environment": fmt.Sprintf("https://%s.plaid.com", env),
-					},
-				},
-			},
-			wantErr: false,
 		},
 		{
-			name: "found, unverified",
+			name:  "invalid pattern",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, invalidKey, keyword, invalidId),
-			args: args{
+			want:  []string{},
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a plaidkey secret %s within but plaidkey %s not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_PlaidKey,
-					Verified:     false,
-				},
-			},
-			wantErr: false,
-		},
-		{
-			name: "not found",
-			s:    Scanner{},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte("You cannot find the secret within"),
-				verify: true,
-			},
-			want:    nil,
-			wantErr: false,
 		},
 	}
-	for _, tt := range tests {
+
-		t.Run(tt.name, func(t *testing.T) {
+	for _, test := range tests {
-			s := Scanner{}
+		t.Run(test.name, func(t *testing.T) {
-			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
-			if (err != nil) != tt.wantErr {
+			if len(matchedDetectors) == 0 {
-				t.Errorf("PlaidKey.FromData() error = %v, wantErr %v", err, tt.wantErr)
+				t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
 				return
 			}
-			for i := range got {
-				if len(got[i].Raw) == 0 {
-					t.Fatalf("no raw secret present: \n %+v", got[i])
-				}
-				got[i].Raw = nil
-			}
-			if diff := pretty.Compare(got, tt.want); diff != "" {
-				t.Errorf("PlaidKey.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
-			}
-		})
-	}
-}
 
-func BenchmarkFromData(benchmark *testing.B) {
+			results, err := d.FromData(context.Background(), false, []byte(test.input))
-	ctx := context.Background()
+			if err != nil {
-	s := Scanner{}
+				t.Errorf("error = %v", err)
-	for name, data := range detectors.MustGetBenchmarkData() {
+				return
-		benchmark.Run(name, func(b *testing.B) {
+			}
-			b.ResetTimer()
+
-			for n := 0; n < b.N; n++ {
+			if len(results) != len(test.want) {
-				_, err := s.FromData(ctx, false, data)
+				if len(results) == 0 {
-				if err != nil {
+					t.Errorf("did not receive result")
-					b.Fatal(err)
+				} else {
+					t.Errorf("expected %d results, only received %d", len(test.want), len(results))
 				}
+				return
+			}
+
+			actual := make(map[string]struct{}, len(results))
+			for _, r := range results {
+				if len(r.RawV2) > 0 {
+					actual[string(r.RawV2)] = struct{}{}
+				} else {
+					actual[string(r.Raw)] = struct{}{}
+				}
+			}
+			expected := make(map[string]struct{}, len(test.want))
+			for _, v := range test.want {
+				expected[v] = struct{}{}
+			}
+
+			if diff := cmp.Diff(expected, actual); diff != "" {
+				t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
 			}
 		})
 	}

pkg/detectors/planetscale/planetscale_integration_test.go

@@ -0,0 +1,164 @@
+//go:build detectors
+// +build detectors
+
+package planetscale
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestPlanetscale_FromChunk(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	defer cancel()
+	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
+	if err != nil {
+		t.Fatalf("could not get test secrets from GCP: %s", err)
+	}
+	secret := testSecrets.MustGetField("PLANET_SCALE_TOKEN")
+	secretID := testSecrets.MustGetField("PLANET_SCALE_ID")
+	inactiveSecret := testSecrets.MustGetField("PLANET_SCALE_TOKEN_INACTIVE")
+	inactiveSecretID := testSecrets.MustGetField("PLANET_SCALE_ID_INACTIVE")
+
+	type args struct {
+		ctx    context.Context
+		data   []byte
+		verify bool
+	}
+	tests := []struct {
+		name                string
+		s                   Scanner
+		args                args
+		want                []detectors.Result
+		wantErr             bool
+		wantVerificationErr bool
+	}{
+		{
+			name: "found, verified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a planetscale secret %s within with id %s", secret, secretID)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_PlanetScale,
+					Verified:     true,
+				},
+			},
+			wantErr:             false,
+			wantVerificationErr: false,
+		},
+		{
+			name: "found, unverified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a planetscale secret %s within with id %s but not valid", inactiveSecret, inactiveSecretID)), // the secret would satisfy the regex but not pass validation
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_PlanetScale,
+					Verified:     false,
+				},
+			},
+			wantErr:             false,
+			wantVerificationErr: false,
+		},
+		{
+			name: "not found",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte("You cannot find the secret within"),
+				verify: true,
+			},
+			want:                nil,
+			wantErr:             false,
+			wantVerificationErr: false,
+		},
+		{
+			name: "found, would be verified if not for timeout",
+			s:    Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a planetscale secret %s within with id %s", secret, secretID)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_PlanetScale,
+					Verified:     false,
+				},
+			},
+			wantErr:             false,
+			wantVerificationErr: true,
+		},
+		{
+			name: "found, verified but unexpected api surface",
+			s:    Scanner{client: common.ConstantResponseHttpClient(404, "")},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a planetscale secret %s within with id %s", secret, secretID)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_PlanetScale,
+					Verified:     false,
+				},
+			},
+			wantErr:             false,
+			wantVerificationErr: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("Planetscale.FromData() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			for i := range got {
+				if len(got[i].Raw) == 0 {
+					t.Fatalf("no raw secret present: \n %+v", got[i])
+				}
+				if (got[i].VerificationError() != nil) != tt.wantVerificationErr {
+					t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())
+				}
+			}
+			ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")
+			if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
+				t.Errorf("Planetscale.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+			}
+		})
+	}
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+	ctx := context.Background()
+	s := Scanner{}
+	for name, data := range detectors.MustGetBenchmarkData() {
+		benchmark.Run(name, func(b *testing.B) {
+			b.ResetTimer()
+			for n := 0; n < b.N; n++ {
+				_, err := s.FromData(ctx, false, data)
+				if err != nil {
+					b.Fatal(err)
+				}
+			}
+		})
+	}
+}

pkg/detectors/planetscale/planetscale_test.go

@@ -1,163 +1,82 @@
-//go:build detectors
-// +build detectors
-
 package planetscale
 
 import (
 	"context"
 	"fmt"
 	"testing"
-	"time"
 
 	"github.com/google/go-cmp/cmp"
-	"github.com/google/go-cmp/cmp/cmpopts"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
-
+	"github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 )
 
-func TestPlanetscale_FromChunk(t *testing.T) {
+var (
-	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	validUsername   = "fo57eya1lvvh"
-	defer cancel()
+	invalidUsername = "fo57ey?1lvvh"
-	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
+	validPassword   = "pscale_tkn_toLyJg1LfgY8vDQr9vhfF2HcT410U3q1alxt7012kcV"
-	if err != nil {
+	invalidPassword = "pscale_tkn_toLyJg1LfgY8vDQr?vhfF2HcT410U3q1alxt7012kcV"
-		t.Fatalf("could not get test secrets from GCP: %s", err)
+	keyword         = "planetscale"
-	}
+)
-	secret := testSecrets.MustGetField("PLANET_SCALE_TOKEN")
-	secretID := testSecrets.MustGetField("PLANET_SCALE_ID")
-	inactiveSecret := testSecrets.MustGetField("PLANET_SCALE_TOKEN_INACTIVE")
-	inactiveSecretID := testSecrets.MustGetField("PLANET_SCALE_ID_INACTIVE")
 
-	type args struct {
+func TestPlanetscale_Pattern(t *testing.T) {
-		ctx    context.Context
+	d := Scanner{}
-		data   []byte
+	ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
-		verify bool
-	}
 	tests := []struct {
-		name                string
+		name  string
-		s                   Scanner
+		input string
-		args                args
+		want  []string
-		want                []detectors.Result
-		wantErr             bool
-		wantVerificationErr bool
 	}{
 		{
-			name: "found, verified",
+			name:  "valid pattern - with keyword planetscale",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, validUsername, keyword, validPassword),
-			args: args{
+			want:  []string{validUsername + ":" + validPassword},
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a planetscale secret %s within with id %s", secret, secretID)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_PlanetScale,
-					Verified:     true,
-				},
-			},
-			wantErr:             false,
-			wantVerificationErr: false,
 		},
 		{
-			name: "found, unverified",
+			name:  "invalid pattern",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, invalidUsername, keyword, invalidPassword),
-			args: args{
+			want:  []string{},
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a planetscale secret %s within with id %s but not valid", inactiveSecret, inactiveSecretID)), // the secret would satisfy the regex but not pass validation
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_PlanetScale,
-					Verified:     false,
-				},
-			},
-			wantErr:             false,
-			wantVerificationErr: false,
-		},
-		{
-			name: "not found",
-			s:    Scanner{},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte("You cannot find the secret within"),
-				verify: true,
-			},
-			want:                nil,
-			wantErr:             false,
-			wantVerificationErr: false,
-		},
-		{
-			name: "found, would be verified if not for timeout",
-			s:    Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a planetscale secret %s within with id %s", secret, secretID)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_PlanetScale,
-					Verified:     false,
-				},
-			},
-			wantErr:             false,
-			wantVerificationErr: true,
-		},
-		{
-			name: "found, verified but unexpected api surface",
-			s:    Scanner{client: common.ConstantResponseHttpClient(404, "")},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a planetscale secret %s within with id %s", secret, secretID)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_PlanetScale,
-					Verified:     false,
-				},
-			},
-			wantErr:             false,
-			wantVerificationErr: true,
 		},
 	}
-	for _, tt := range tests {
+
-		t.Run(tt.name, func(t *testing.T) {
+	for _, test := range tests {
-			got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+		t.Run(test.name, func(t *testing.T) {
-			if (err != nil) != tt.wantErr {
+			matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
-				t.Errorf("Planetscale.FromData() error = %v, wantErr %v", err, tt.wantErr)
+			if len(matchedDetectors) == 0 {
+				t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
 				return
 			}
-			for i := range got {
-				if len(got[i].Raw) == 0 {
-					t.Fatalf("no raw secret present: \n %+v", got[i])
-				}
-				if (got[i].VerificationError() != nil) != tt.wantVerificationErr {
-					t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())
-				}
-			}
-			ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")
-			if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
-				t.Errorf("Planetscale.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
-			}
-		})
-	}
-}
 
-func BenchmarkFromData(benchmark *testing.B) {
+			results, err := d.FromData(context.Background(), false, []byte(test.input))
-	ctx := context.Background()
+			if err != nil {
-	s := Scanner{}
+				t.Errorf("error = %v", err)
-	for name, data := range detectors.MustGetBenchmarkData() {
+				return
-		benchmark.Run(name, func(b *testing.B) {
+			}
-			b.ResetTimer()
+
-			for n := 0; n < b.N; n++ {
+			if len(results) != len(test.want) {
-				_, err := s.FromData(ctx, false, data)
+				if len(results) == 0 {
-				if err != nil {
+					t.Errorf("did not receive result")
-					b.Fatal(err)
+				} else {
+					t.Errorf("expected %d results, only received %d", len(test.want), len(results))
 				}
+				return
+			}
+
+			actual := make(map[string]struct{}, len(results))
+			for _, r := range results {
+				if len(r.RawV2) > 0 {
+					actual[string(r.RawV2)] = struct{}{}
+				} else {
+					actual[string(r.Raw)] = struct{}{}
+				}
+			}
+			expected := make(map[string]struct{}, len(test.want))
+			for _, v := range test.want {
+				expected[v] = struct{}{}
+			}
+
+			if diff := cmp.Diff(expected, actual); diff != "" {
+				t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
 			}
 		})
 	}

pkg/detectors/planetscaledb/planetscaledb_integration_test.go

@@ -0,0 +1,129 @@
+//go:build detectors
+// +build detectors
+
+package planetscaledb
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestPlanetscaledb_FromChunk(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	defer cancel()
+	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
+	if err != nil {
+		t.Fatalf("could not get test secrets from GCP: %s", err)
+	}
+	username := testSecrets.MustGetField("PLANET_SCALEDB_USERNAME")
+	host := testSecrets.MustGetField("PLANET_SCALEDB_HOST")
+	password := testSecrets.MustGetField("PLANET_SCALEDB_PASSWORD")
+	inactivePassword := testSecrets.MustGetField("PLANET_SCALEDB_PASSWORD_INACTIVE")
+
+	type args struct {
+		ctx    context.Context
+		data   []byte
+		verify bool
+	}
+	tests := []struct {
+		name                string
+		s                   Scanner
+		args                args
+		want                []detectors.Result
+		wantErr             bool
+		wantVerificationErr bool
+	}{
+		{
+			name: "found, verified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a planetscaledb secret %s %s %s", username, password, host)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_PlanetScaleDb,
+					Verified:     true,
+				},
+			},
+			wantErr:             false,
+			wantVerificationErr: false,
+		},
+		{
+			name: "found, unverified",
+			s:    Scanner{},
+			args: args{
+				ctx:  context.Background(),
+				data: []byte(fmt.Sprintf("You can find a planetscaledb secret %s %s %s", username, inactivePassword, host)),
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_PlanetScaleDb,
+					Verified:     false,
+				},
+			},
+			wantErr:             false,
+			wantVerificationErr: false,
+		},
+		{
+			name: "not found",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte("You cannot find the secret within"),
+				verify: true,
+			},
+			want:                nil,
+			wantErr:             false,
+			wantVerificationErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("Planetscaledb.FromData() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			for i := range got {
+				if len(got[i].Raw) == 0 {
+					t.Fatalf("no raw secret present: \n %+v", got[i])
+				}
+				if (got[i].VerificationError() != nil) != tt.wantVerificationErr {
+					t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())
+				}
+			}
+			ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")
+			if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
+				t.Errorf("Planetscaledb.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+			}
+		})
+	}
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+	ctx := context.Background()
+	s := Scanner{}
+	for name, data := range detectors.MustGetBenchmarkData() {
+		benchmark.Run(name, func(b *testing.B) {
+			b.ResetTimer()
+			for n := 0; n < b.N; n++ {
+				_, err := s.FromData(ctx, false, data)
+				if err != nil {
+					b.Fatal(err)
+				}
+			}
+		})
+	}
+}

pkg/detectors/planetscaledb/planetscaledb_test.go

@@ -1,128 +1,84 @@
-//go:build detectors
-// +build detectors
-
 package planetscaledb
 
 import (
 	"context"
 	"fmt"
 	"testing"
-	"time"
 
 	"github.com/google/go-cmp/cmp"
-	"github.com/google/go-cmp/cmp/cmpopts"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
-
+	"github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 )
 
-func TestPlanetscaledb_FromChunk(t *testing.T) {
+var (
-	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	validUsername   = "82mygyuh2y23aw1k8lzv"
-	defer cancel()
+	invalidUsername = "8?mygyuh2y23aw1k8lzv"
-	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
+	validPassword   = "pscale_pw_iAhKQKjU8nUrHagaygAubhM6x0LTaBz6kOyqx9AIS6V"
-	if err != nil {
+	invalidPassword = "pscale_pw_iAhKQKjU8nUrHaga?gAubhM6x0LTaBz6kOyqx9AIS6V"
-		t.Fatalf("could not get test secrets from GCP: %s", err)
+	validHost       = "gcp.connect.psdb.cloud"
-	}
+	invalidHost     = "gcp?connect.psdb.cloud"
-	username := testSecrets.MustGetField("PLANET_SCALEDB_USERNAME")
+	keyword         = "planetscaledb"
-	host := testSecrets.MustGetField("PLANET_SCALEDB_HOST")
+)
-	password := testSecrets.MustGetField("PLANET_SCALEDB_PASSWORD")
-	inactivePassword := testSecrets.MustGetField("PLANET_SCALEDB_PASSWORD_INACTIVE")
 
-	type args struct {
+func TestPlanetscaledb_Pattern(t *testing.T) {
-		ctx    context.Context
+	d := Scanner{}
-		data   []byte
+	ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
-		verify bool
-	}
 	tests := []struct {
-		name                string
+		name  string
-		s                   Scanner
+		input string
-		args                args
+		want  []string
-		want                []detectors.Result
-		wantErr             bool
-		wantVerificationErr bool
 	}{
 		{
-			name: "found, verified",
+			name:  "valid pattern - with keyword planetscaledb",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n%s token - '%s'\n", keyword, validUsername, keyword, validPassword, keyword, validHost),
-			args: args{
+			want:  []string{validHost + "\t" + validUsername + "\t" + validPassword},
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a planetscaledb secret %s %s %s", username, password, host)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_PlanetScaleDb,
-					Verified:     true,
-				},
-			},
-			wantErr:             false,
-			wantVerificationErr: false,
 		},
 		{
-			name: "found, unverified",
+			name:  "invalid pattern",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n%s token - '%s'\n", keyword, invalidUsername, keyword, invalidPassword, keyword, invalidHost),
-			args: args{
+			want:  []string{},
-				ctx:  context.Background(),
-				data: []byte(fmt.Sprintf("You can find a planetscaledb secret %s %s %s", username, inactivePassword, host)),
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_PlanetScaleDb,
-					Verified:     false,
-				},
-			},
-			wantErr:             false,
-			wantVerificationErr: false,
-		},
-		{
-			name: "not found",
-			s:    Scanner{},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte("You cannot find the secret within"),
-				verify: true,
-			},
-			want:                nil,
-			wantErr:             false,
-			wantVerificationErr: false,
 		},
 	}
-	for _, tt := range tests {
+
-		t.Run(tt.name, func(t *testing.T) {
+	for _, test := range tests {
-			got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+		t.Run(test.name, func(t *testing.T) {
-			if (err != nil) != tt.wantErr {
+			matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
-				t.Errorf("Planetscaledb.FromData() error = %v, wantErr %v", err, tt.wantErr)
+			if len(matchedDetectors) == 0 {
+				t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
 				return
 			}
-			for i := range got {
-				if len(got[i].Raw) == 0 {
-					t.Fatalf("no raw secret present: \n %+v", got[i])
-				}
-				if (got[i].VerificationError() != nil) != tt.wantVerificationErr {
-					t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())
-				}
-			}
-			ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")
-			if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
-				t.Errorf("Planetscaledb.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
-			}
-		})
-	}
-}
 
-func BenchmarkFromData(benchmark *testing.B) {
+			results, err := d.FromData(context.Background(), false, []byte(test.input))
-	ctx := context.Background()
+			if err != nil {
-	s := Scanner{}
+				t.Errorf("error = %v", err)
-	for name, data := range detectors.MustGetBenchmarkData() {
+				return
-		benchmark.Run(name, func(b *testing.B) {
+			}
-			b.ResetTimer()
+
-			for n := 0; n < b.N; n++ {
+			if len(results) != len(test.want) {
-				_, err := s.FromData(ctx, false, data)
+				if len(results) == 0 {
-				if err != nil {
+					t.Errorf("did not receive result")
-					b.Fatal(err)
+				} else {
+					t.Errorf("expected %d results, only received %d", len(test.want), len(results))
 				}
+				return
+			}
+
+			actual := make(map[string]struct{}, len(results))
+			for _, r := range results {
+				if len(r.RawV2) > 0 {
+					actual[string(r.RawV2)] = struct{}{}
+				} else {
+					actual[string(r.Raw)] = struct{}{}
+				}
+			}
+			expected := make(map[string]struct{}, len(test.want))
+			for _, v := range test.want {
+				expected[v] = struct{}{}
+			}
+
+			if diff := cmp.Diff(expected, actual); diff != "" {
+				t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
 			}
 		})
 	}

pkg/detectors/planviewleankit/planviewleankit_integration_test.go

@@ -0,0 +1,122 @@
+//go:build detectors
+// +build detectors
+
+package planviewleankit
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/kylelemons/godebug/pretty"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestPlanviewLeanKit_FromChunk(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	defer cancel()
+	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
+	if err != nil {
+		t.Fatalf("could not get test secrets from GCP: %s", err)
+	}
+	secret := testSecrets.MustGetField("PLANVIEWLEANKIT")
+	inactiveSecret := testSecrets.MustGetField("PLANVIEWLEANKIT_INACTIVE")
+	subdomain := testSecrets.MustGetField("PLANVIEWLEANKIT_SUBDOMAIN")
+
+	// log.Println(secret)
+	// log.Println(inactiveSecret)
+	// log.Println(subdomain)
+
+	type args struct {
+		ctx    context.Context
+		data   []byte
+		verify bool
+	}
+	tests := []struct {
+		name    string
+		s       Scanner
+		args    args
+		want    []detectors.Result
+		wantErr bool
+	}{
+		{
+			name: "found, verified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a planviewleankit subdomain %s with planviewleankit secret %s within", subdomain, secret)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_PlanviewLeanKit,
+					Verified:     true,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "found, unverified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a planviewleankit subdomain %s with planviewleankit secret %s within but not valid", subdomain, inactiveSecret)), // the secret would satisfy the regex but not pass validation
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_PlanviewLeanKit,
+					Verified:     false,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "not found",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte("You cannot find the secret within"),
+				verify: true,
+			},
+			want:    nil,
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			s := Scanner{}
+			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("PlanviewLeanKit.FromData() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			for i := range got {
+				if len(got[i].Raw) == 0 {
+					t.Fatalf("no raw secret present: \n %+v", got[i])
+				}
+				got[i].Raw = nil
+			}
+			if diff := pretty.Compare(got, tt.want); diff != "" {
+				t.Errorf("PlanviewLeanKit.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+			}
+		})
+	}
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+	ctx := context.Background()
+	s := Scanner{}
+	for name, data := range detectors.MustGetBenchmarkData() {
+		benchmark.Run(name, func(b *testing.B) {
+			b.ResetTimer()
+			for n := 0; n < b.N; n++ {
+				s.FromData(ctx, false, data)
+			}
+		})
+	}
+}

pkg/detectors/planviewleankit/planviewleankit_test.go

@@ -1,121 +1,82 @@
-//go:build detectors
-// +build detectors
-
 package planviewleankit
 
 import (
 	"context"
 	"fmt"
 	"testing"
-	"time"
 
-	"github.com/kylelemons/godebug/pretty"
+	"github.com/google/go-cmp/cmp"
 
-	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
 )
 
-func TestPlanviewLeanKit_FromChunk(t *testing.T) {
+var (
-	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	validKey         = "b1bb66781f857e6edf539042389e77043396b123c85197676719ef79a420277bd5dc2f6c299bb01895c604f7038c27af9035937823c7c0e8e25c2efe7f6ddd4f"
-	defer cancel()
+	invalidKey       = "B1bb66781f857e6edf539042389e77043396b123c85197676719ef79a420277bd5dc2f6c299bb01895c604f7038c27af9035937823c7c0e8e25c2efe7f6ddd4F"
-	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
+	validSubDomain   = "subdomain.pYwPPnHSc"
-	if err != nil {
+	invalidSubDomain = "?ubdomain.pYwPPnHS?"
-		t.Fatalf("could not get test secrets from GCP: %s", err)
+	keyword          = "planviewleankit"
-	}
+)
-	secret := testSecrets.MustGetField("PLANVIEWLEANKIT")
-	inactiveSecret := testSecrets.MustGetField("PLANVIEWLEANKIT_INACTIVE")
-	subdomain := testSecrets.MustGetField("PLANVIEWLEANKIT_SUBDOMAIN")
 
-	// log.Println(secret)
+func TestPlanviewLeanKit_Pattern(t *testing.T) {
-	// log.Println(inactiveSecret)
+	d := Scanner{}
-	// log.Println(subdomain)
+	ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
-
-	type args struct {
-		ctx    context.Context
-		data   []byte
-		verify bool
-	}
 	tests := []struct {
-		name    string
+		name  string
-		s       Scanner
+		input string
-		args    args
+		want  []string
-		want    []detectors.Result
-		wantErr bool
 	}{
 		{
-			name: "found, verified",
+			name:  "valid pattern - with keyword planviewleankit",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, validKey, keyword, validSubDomain),
-			args: args{
+			want:  []string{validKey},
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a planviewleankit subdomain %s with planviewleankit secret %s within", subdomain, secret)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_PlanviewLeanKit,
-					Verified:     true,
-				},
-			},
-			wantErr: false,
 		},
 		{
-			name: "found, unverified",
+			name:  "invalid pattern",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, invalidKey, keyword, invalidSubDomain),
-			args: args{
+			want:  []string{},
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a planviewleankit subdomain %s with planviewleankit secret %s within but not valid", subdomain, inactiveSecret)), // the secret would satisfy the regex but not pass validation
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_PlanviewLeanKit,
-					Verified:     false,
-				},
-			},
-			wantErr: false,
-		},
-		{
-			name: "not found",
-			s:    Scanner{},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte("You cannot find the secret within"),
-				verify: true,
-			},
-			want:    nil,
-			wantErr: false,
 		},
 	}
-	for _, tt := range tests {
+
-		t.Run(tt.name, func(t *testing.T) {
+	for _, test := range tests {
-			s := Scanner{}
+		t.Run(test.name, func(t *testing.T) {
-			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
-			if (err != nil) != tt.wantErr {
+			if len(matchedDetectors) == 0 {
-				t.Errorf("PlanviewLeanKit.FromData() error = %v, wantErr %v", err, tt.wantErr)
+				t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
 				return
 			}
-			for i := range got {
-				if len(got[i].Raw) == 0 {
-					t.Fatalf("no raw secret present: \n %+v", got[i])
-				}
-				got[i].Raw = nil
-			}
-			if diff := pretty.Compare(got, tt.want); diff != "" {
-				t.Errorf("PlanviewLeanKit.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
-			}
-		})
-	}
-}
 
-func BenchmarkFromData(benchmark *testing.B) {
+			results, err := d.FromData(context.Background(), false, []byte(test.input))
-	ctx := context.Background()
+			if err != nil {
-	s := Scanner{}
+				t.Errorf("error = %v", err)
-	for name, data := range detectors.MustGetBenchmarkData() {
+				return
-		benchmark.Run(name, func(b *testing.B) {
+			}
-			b.ResetTimer()
+
-			for n := 0; n < b.N; n++ {
+			if len(results) != len(test.want) {
-				s.FromData(ctx, false, data)
+				if len(results) == 0 {
+					t.Errorf("did not receive result")
+				} else {
+					t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+				}
+				return
+			}
+
+			actual := make(map[string]struct{}, len(results))
+			for _, r := range results {
+				if len(r.RawV2) > 0 {
+					actual[string(r.RawV2)] = struct{}{}
+				} else {
+					actual[string(r.Raw)] = struct{}{}
+				}
+			}
+			expected := make(map[string]struct{}, len(test.want))
+			for _, v := range test.want {
+				expected[v] = struct{}{}
+			}
+
+			if diff := cmp.Diff(expected, actual); diff != "" {
+				t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
 			}
 		})
 	}

pkg/detectors/planyo/planyo_integration_test.go

@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package planyo
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/kylelemons/godebug/pretty"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestPlanyo_FromChunk(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	defer cancel()
+	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
+	if err != nil {
+		t.Fatalf("could not get test secrets from GCP: %s", err)
+	}
+	secret := testSecrets.MustGetField("PLANYO")
+	inactiveSecret := testSecrets.MustGetField("PLANYO_INACTIVE")
+
+	type args struct {
+		ctx    context.Context
+		data   []byte
+		verify bool
+	}
+	tests := []struct {
+		name    string
+		s       Scanner
+		args    args
+		want    []detectors.Result
+		wantErr bool
+	}{
+		{
+			name: "found, verified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a planyo secret %s within", secret)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Planyo,
+					Verified:     true,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "found, unverified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a planyo secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Planyo,
+					Verified:     false,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "not found",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte("You cannot find the secret within"),
+				verify: true,
+			},
+			want:    nil,
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			s := Scanner{}
+			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("Planyo.FromData() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			for i := range got {
+				if len(got[i].Raw) == 0 {
+					t.Fatalf("no raw secret present: \n %+v", got[i])
+				}
+				got[i].Raw = nil
+			}
+			if diff := pretty.Compare(got, tt.want); diff != "" {
+				t.Errorf("Planyo.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+			}
+		})
+	}
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+	ctx := context.Background()
+	s := Scanner{}
+	for name, data := range detectors.MustGetBenchmarkData() {
+		benchmark.Run(name, func(b *testing.B) {
+			b.ResetTimer()
+			for n := 0; n < b.N; n++ {
+				_, err := s.FromData(ctx, false, data)
+				if err != nil {
+					b.Fatal(err)
+				}
+			}
+		})
+	}
+}

pkg/detectors/planyo/planyo_test.go

@@ -1,119 +1,90 @@
-//go:build detectors
-// +build detectors
-
 package planyo
 
 import (
 	"context"
 	"fmt"
 	"testing"
-	"time"
 
-	"github.com/kylelemons/godebug/pretty"
+	"github.com/google/go-cmp/cmp"
+
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
-
+	"github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 )
 
-func TestPlanyo_FromChunk(t *testing.T) {
+var (
-	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	validPattern   = "big57xmqvemmsopa2s7s3805oo70dzxk7subgcao8zerjg89ze8walz5si63x7"
-	defer cancel()
+	invalidPattern = "big57xmqvemmsopa2s7s3805oo70dzx?7subgcao8zerjg89ze8walz5si63x7"
-	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
+	keyword        = "planyo"
-	if err != nil {
+)
-		t.Fatalf("could not get test secrets from GCP: %s", err)
-	}
-	secret := testSecrets.MustGetField("PLANYO")
-	inactiveSecret := testSecrets.MustGetField("PLANYO_INACTIVE")
 
-	type args struct {
+func TestPlanyo_Pattern(t *testing.T) {
-		ctx    context.Context
+	d := Scanner{}
-		data   []byte
+	ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
-		verify bool
-	}
 	tests := []struct {
-		name    string
+		name  string
-		s       Scanner
+		input string
-		args    args
+		want  []string
-		want    []detectors.Result
-		wantErr bool
 	}{
 		{
-			name: "found, verified",
+			name:  "valid pattern - with keyword planyo",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
-			args: args{
+			want:  []string{validPattern},
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a planyo secret %s within", secret)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_Planyo,
-					Verified:     true,
-				},
-			},
-			wantErr: false,
 		},
 		{
-			name: "found, unverified",
+			name:  "valid pattern - ignore duplicate",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
-			args: args{
+			want:  []string{validPattern},
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a planyo secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_Planyo,
-					Verified:     false,
-				},
-			},
-			wantErr: false,
 		},
 		{
-			name: "not found",
+			name:  "valid pattern - key out of prefix range",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
-			args: args{
+			want:  []string{},
-				ctx:    context.Background(),
+		},
-				data:   []byte("You cannot find the secret within"),
+		{
-				verify: true,
+			name:  "invalid pattern",
-			},
+			input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
-			want:    nil,
+			want:  []string{},
-			wantErr: false,
 		},
 	}
-	for _, tt := range tests {
+
-		t.Run(tt.name, func(t *testing.T) {
+	for _, test := range tests {
-			s := Scanner{}
+		t.Run(test.name, func(t *testing.T) {
-			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
-			if (err != nil) != tt.wantErr {
+			if len(matchedDetectors) == 0 {
-				t.Errorf("Planyo.FromData() error = %v, wantErr %v", err, tt.wantErr)
+				t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
 				return
 			}
-			for i := range got {
-				if len(got[i].Raw) == 0 {
-					t.Fatalf("no raw secret present: \n %+v", got[i])
-				}
-				got[i].Raw = nil
-			}
-			if diff := pretty.Compare(got, tt.want); diff != "" {
-				t.Errorf("Planyo.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
-			}
-		})
-	}
-}
 
-func BenchmarkFromData(benchmark *testing.B) {
+			results, err := d.FromData(context.Background(), false, []byte(test.input))
-	ctx := context.Background()
+			if err != nil {
-	s := Scanner{}
+				t.Errorf("error = %v", err)
-	for name, data := range detectors.MustGetBenchmarkData() {
+				return
-		benchmark.Run(name, func(b *testing.B) {
+			}
-			b.ResetTimer()
+
-			for n := 0; n < b.N; n++ {
+			if len(results) != len(test.want) {
-				_, err := s.FromData(ctx, false, data)
+				if len(results) == 0 {
-				if err != nil {
+					t.Errorf("did not receive result")
-					b.Fatal(err)
+				} else {
+					t.Errorf("expected %d results, only received %d", len(test.want), len(results))
 				}
+				return
+			}
+
+			actual := make(map[string]struct{}, len(results))
+			for _, r := range results {
+				if len(r.RawV2) > 0 {
+					actual[string(r.RawV2)] = struct{}{}
+				} else {
+					actual[string(r.Raw)] = struct{}{}
+				}
+			}
+			expected := make(map[string]struct{}, len(test.want))
+			for _, v := range test.want {
+				expected[v] = struct{}{}
+			}
+
+			if diff := cmp.Diff(expected, actual); diff != "" {
+				t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
 			}
 		})
 	}

pkg/detectors/plivo/plivo_integration_test.go

@@ -0,0 +1,123 @@
+//go:build detectors
+// +build detectors
+
+package plivo
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/kylelemons/godebug/pretty"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestPlivo_FromChunk(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	defer cancel()
+	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+	if err != nil {
+		t.Fatalf("could not get test secrets from GCP: %s", err)
+	}
+	secret := testSecrets.MustGetField("PLIVO_TOKEN")
+	inactiveSecret := testSecrets.MustGetField("PLIVO_INACTIVE")
+	id := testSecrets.MustGetField("PLIVO_ID")
+
+	type args struct {
+		ctx    context.Context
+		data   []byte
+		verify bool
+	}
+	tests := []struct {
+		name    string
+		s       Scanner
+		args    args
+		want    []detectors.Result
+		wantErr bool
+	}{
+		{
+			name: "found, verified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a plivo secret %s within https://api.plivo.com/v1/Account/%s/Number/", secret, id)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Plivo,
+					Redacted:     id,
+					Verified:     true,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "found, unverified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a plivo secret %s within https://api.plivo.com/v1/Account/%s/Number/ but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Plivo,
+					Redacted:     id,
+					Verified:     false,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "not found",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte("You cannot find the secret within"),
+				verify: true,
+			},
+			want:    nil,
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			s := Scanner{}
+			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("Plivo.FromData() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			for i := range got {
+				if len(got[i].Raw) == 0 {
+					t.Fatalf("no raw secret present: \n %+v", got[i])
+				}
+				got[i].Raw = nil
+			}
+			if diff := pretty.Compare(got, tt.want); diff != "" {
+				t.Errorf("Plivo.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+			}
+		})
+	}
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+	ctx := context.Background()
+	s := Scanner{}
+	for name, data := range detectors.MustGetBenchmarkData() {
+		benchmark.Run(name, func(b *testing.B) {
+			b.ResetTimer()
+			for n := 0; n < b.N; n++ {
+				_, err := s.FromData(ctx, false, data)
+				if err != nil {
+					b.Fatal(err)
+				}
+			}
+		})
+	}
+}

pkg/detectors/plivo/plivo_test.go

@@ -1,122 +1,82 @@
-//go:build detectors
-// +build detectors
-
 package plivo
 
 import (
 	"context"
 	"fmt"
 	"testing"
-	"time"
 
-	"github.com/kylelemons/godebug/pretty"
+	"github.com/google/go-cmp/cmp"
+
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
-
+	"github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 )
 
-func TestPlivo_FromChunk(t *testing.T) {
+var (
-	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	validId    = "YGIQXPGZSVVGVGOREMAE"
-	defer cancel()
+	invalidId  = "YGIQXPGZS?VGVGOREMAE"
-	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+	validKey   = "qFj32Da8vf-g_-8qLu9P_k8XPyAHGrvKrzGTQIN4"
-	if err != nil {
+	invalidKey = "qFj32Da8vf-g?-8qLu9P_k8XPyAHGrvKrzGTQIN4"
-		t.Fatalf("could not get test secrets from GCP: %s", err)
+	keyword    = "plivo"
-	}
+)
-	secret := testSecrets.MustGetField("PLIVO_TOKEN")
-	inactiveSecret := testSecrets.MustGetField("PLIVO_INACTIVE")
-	id := testSecrets.MustGetField("PLIVO_ID")
 
-	type args struct {
+func TestPlivo_Pattern(t *testing.T) {
-		ctx    context.Context
+	d := Scanner{}
-		data   []byte
+	ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
-		verify bool
-	}
 	tests := []struct {
-		name    string
+		name  string
-		s       Scanner
+		input string
-		args    args
+		want  []string
-		want    []detectors.Result
-		wantErr bool
 	}{
 		{
-			name: "found, verified",
+			name:  "valid pattern - with keyword plivo",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, validId, keyword, validKey),
-			args: args{
+			want:  []string{validKey + validId},
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a plivo secret %s within https://api.plivo.com/v1/Account/%s/Number/", secret, id)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_Plivo,
-					Redacted:     id,
-					Verified:     true,
-				},
-			},
-			wantErr: false,
 		},
 		{
-			name: "found, unverified",
+			name:  "invalid pattern",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, invalidId, keyword, invalidKey),
-			args: args{
+			want:  []string{},
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a plivo secret %s within https://api.plivo.com/v1/Account/%s/Number/ but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_Plivo,
-					Redacted:     id,
-					Verified:     false,
-				},
-			},
-			wantErr: false,
-		},
-		{
-			name: "not found",
-			s:    Scanner{},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte("You cannot find the secret within"),
-				verify: true,
-			},
-			want:    nil,
-			wantErr: false,
 		},
 	}
-	for _, tt := range tests {
+
-		t.Run(tt.name, func(t *testing.T) {
+	for _, test := range tests {
-			s := Scanner{}
+		t.Run(test.name, func(t *testing.T) {
-			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
-			if (err != nil) != tt.wantErr {
+			if len(matchedDetectors) == 0 {
-				t.Errorf("Plivo.FromData() error = %v, wantErr %v", err, tt.wantErr)
+				t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
 				return
 			}
-			for i := range got {
-				if len(got[i].Raw) == 0 {
-					t.Fatalf("no raw secret present: \n %+v", got[i])
-				}
-				got[i].Raw = nil
-			}
-			if diff := pretty.Compare(got, tt.want); diff != "" {
-				t.Errorf("Plivo.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
-			}
-		})
-	}
-}
 
-func BenchmarkFromData(benchmark *testing.B) {
+			results, err := d.FromData(context.Background(), false, []byte(test.input))
-	ctx := context.Background()
+			if err != nil {
-	s := Scanner{}
+				t.Errorf("error = %v", err)
-	for name, data := range detectors.MustGetBenchmarkData() {
+				return
-		benchmark.Run(name, func(b *testing.B) {
+			}
-			b.ResetTimer()
+
-			for n := 0; n < b.N; n++ {
+			if len(results) != len(test.want) {
-				_, err := s.FromData(ctx, false, data)
+				if len(results) == 0 {
-				if err != nil {
+					t.Errorf("did not receive result")
-					b.Fatal(err)
+				} else {
+					t.Errorf("expected %d results, only received %d", len(test.want), len(results))
 				}
+				return
+			}
+
+			actual := make(map[string]struct{}, len(results))
+			for _, r := range results {
+				if len(r.RawV2) > 0 {
+					actual[string(r.RawV2)] = struct{}{}
+				} else {
+					actual[string(r.Raw)] = struct{}{}
+				}
+			}
+			expected := make(map[string]struct{}, len(test.want))
+			for _, v := range test.want {
+				expected[v] = struct{}{}
+			}
+
+			if diff := cmp.Diff(expected, actual); diff != "" {
+				t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
 			}
 		})
 	}

pkg/detectors/podio/podio_integration_test.go

@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package podio
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/kylelemons/godebug/pretty"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestPodio_FromChunk(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	defer cancel()
+	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+	if err != nil {
+		t.Fatalf("could not get test secrets from GCP: %s", err)
+	}
+	secret := testSecrets.MustGetField("PODIO")
+	inactiveSecret := testSecrets.MustGetField("PODIO_INACTIVE")
+
+	type args struct {
+		ctx    context.Context
+		data   []byte
+		verify bool
+	}
+	tests := []struct {
+		name    string
+		s       Scanner
+		args    args
+		want    []detectors.Result
+		wantErr bool
+	}{
+		{
+			name: "found, verified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a podio secret %s within", secret)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Podio,
+					Verified:     true,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "found, unverified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a podio secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Podio,
+					Verified:     false,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "not found",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte("You cannot find the secret within"),
+				verify: true,
+			},
+			want:    nil,
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			s := Scanner{}
+			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("Podio.FromData() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			for i := range got {
+				if len(got[i].Raw) == 0 {
+					t.Fatalf("no raw secret present: \n %+v", got[i])
+				}
+				got[i].Raw = nil
+			}
+			if diff := pretty.Compare(got, tt.want); diff != "" {
+				t.Errorf("Podio.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+			}
+		})
+	}
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+	ctx := context.Background()
+	s := Scanner{}
+	for name, data := range detectors.MustGetBenchmarkData() {
+		benchmark.Run(name, func(b *testing.B) {
+			b.ResetTimer()
+			for n := 0; n < b.N; n++ {
+				_, err := s.FromData(ctx, false, data)
+				if err != nil {
+					b.Fatal(err)
+				}
+			}
+		})
+	}
+}

pkg/detectors/podio/podio_test.go

@@ -1,119 +1,90 @@
-//go:build detectors
-// +build detectors
-
 package podio
 
 import (
 	"context"
 	"fmt"
 	"testing"
-	"time"
 
-	"github.com/kylelemons/godebug/pretty"
+	"github.com/google/go-cmp/cmp"
+
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
-
+	"github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 )
 
-func TestPodio_FromChunk(t *testing.T) {
+var (
-	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	validPattern   = "d3ij9dt7n9pu9enq0seldrpwcbl6p1ky"
-	defer cancel()
+	invalidPattern = "d3ij9dt7n9pu9enq?seldrpwcbl6p1ky"
-	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+	keyword        = "podio"
-	if err != nil {
+)
-		t.Fatalf("could not get test secrets from GCP: %s", err)
-	}
-	secret := testSecrets.MustGetField("PODIO")
-	inactiveSecret := testSecrets.MustGetField("PODIO_INACTIVE")
 
-	type args struct {
+func TestPodio_Pattern(t *testing.T) {
-		ctx    context.Context
+	d := Scanner{}
-		data   []byte
+	ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
-		verify bool
-	}
 	tests := []struct {
-		name    string
+		name  string
-		s       Scanner
+		input string
-		args    args
+		want  []string
-		want    []detectors.Result
-		wantErr bool
 	}{
 		{
-			name: "found, verified",
+			name:  "valid pattern - with keyword podio",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
-			args: args{
+			want:  []string{validPattern},
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a podio secret %s within", secret)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_Podio,
-					Verified:     true,
-				},
-			},
-			wantErr: false,
 		},
 		{
-			name: "found, unverified",
+			name:  "valid pattern - ignore duplicate",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
-			args: args{
+			want:  []string{validPattern},
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a podio secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_Podio,
-					Verified:     false,
-				},
-			},
-			wantErr: false,
 		},
 		{
-			name: "not found",
+			name:  "valid pattern - key out of prefix range",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
-			args: args{
+			want:  []string{},
-				ctx:    context.Background(),
+		},
-				data:   []byte("You cannot find the secret within"),
+		{
-				verify: true,
+			name:  "invalid pattern",
-			},
+			input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
-			want:    nil,
+			want:  []string{},
-			wantErr: false,
 		},
 	}
-	for _, tt := range tests {
+
-		t.Run(tt.name, func(t *testing.T) {
+	for _, test := range tests {
-			s := Scanner{}
+		t.Run(test.name, func(t *testing.T) {
-			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
-			if (err != nil) != tt.wantErr {
+			if len(matchedDetectors) == 0 {
-				t.Errorf("Podio.FromData() error = %v, wantErr %v", err, tt.wantErr)
+				t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
 				return
 			}
-			for i := range got {
-				if len(got[i].Raw) == 0 {
-					t.Fatalf("no raw secret present: \n %+v", got[i])
-				}
-				got[i].Raw = nil
-			}
-			if diff := pretty.Compare(got, tt.want); diff != "" {
-				t.Errorf("Podio.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
-			}
-		})
-	}
-}
 
-func BenchmarkFromData(benchmark *testing.B) {
+			results, err := d.FromData(context.Background(), false, []byte(test.input))
-	ctx := context.Background()
+			if err != nil {
-	s := Scanner{}
+				t.Errorf("error = %v", err)
-	for name, data := range detectors.MustGetBenchmarkData() {
+				return
-		benchmark.Run(name, func(b *testing.B) {
+			}
-			b.ResetTimer()
+
-			for n := 0; n < b.N; n++ {
+			if len(results) != len(test.want) {
-				_, err := s.FromData(ctx, false, data)
+				if len(results) == 0 {
-				if err != nil {
+					t.Errorf("did not receive result")
-					b.Fatal(err)
+				} else {
+					t.Errorf("expected %d results, only received %d", len(test.want), len(results))
 				}
+				return
+			}
+
+			actual := make(map[string]struct{}, len(results))
+			for _, r := range results {
+				if len(r.RawV2) > 0 {
+					actual[string(r.RawV2)] = struct{}{}
+				} else {
+					actual[string(r.Raw)] = struct{}{}
+				}
+			}
+			expected := make(map[string]struct{}, len(test.want))
+			for _, v := range test.want {
+				expected[v] = struct{}{}
+			}
+
+			if diff := cmp.Diff(expected, actual); diff != "" {
+				t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
 			}
 		})
 	}

pkg/detectors/pollsapi/pollsapi_integration_test.go

@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package pollsapi
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/kylelemons/godebug/pretty"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestPollsAPI_FromChunk(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	defer cancel()
+	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+	if err != nil {
+		t.Fatalf("could not get test secrets from GCP: %s", err)
+	}
+	secret := testSecrets.MustGetField("POLLSAPI")
+	inactiveSecret := testSecrets.MustGetField("POLLSAPI_INACTIVE")
+
+	type args struct {
+		ctx    context.Context
+		data   []byte
+		verify bool
+	}
+	tests := []struct {
+		name    string
+		s       Scanner
+		args    args
+		want    []detectors.Result
+		wantErr bool
+	}{
+		{
+			name: "found, verified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a pollsapi secret %s within", secret)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_PollsAPI,
+					Verified:     true,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "found, unverified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a pollsapi secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_PollsAPI,
+					Verified:     false,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "not found",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte("You cannot find the secret within"),
+				verify: true,
+			},
+			want:    nil,
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			s := Scanner{}
+			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("PollsAPI.FromData() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			for i := range got {
+				if len(got[i].Raw) == 0 {
+					t.Fatalf("no raw secret present: \n %+v", got[i])
+				}
+				got[i].Raw = nil
+			}
+			if diff := pretty.Compare(got, tt.want); diff != "" {
+				t.Errorf("PollsAPI.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+			}
+		})
+	}
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+	ctx := context.Background()
+	s := Scanner{}
+	for name, data := range detectors.MustGetBenchmarkData() {
+		benchmark.Run(name, func(b *testing.B) {
+			b.ResetTimer()
+			for n := 0; n < b.N; n++ {
+				_, err := s.FromData(ctx, false, data)
+				if err != nil {
+					b.Fatal(err)
+				}
+			}
+		})
+	}
+}

pkg/detectors/pollsapi/pollsapi_test.go

@@ -1,119 +1,90 @@
-//go:build detectors
-// +build detectors
-
 package pollsapi
 
 import (
 	"context"
 	"fmt"
 	"testing"
-	"time"
 
-	"github.com/kylelemons/godebug/pretty"
+	"github.com/google/go-cmp/cmp"
+
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
-
+	"github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 )
 
-func TestPollsAPI_FromChunk(t *testing.T) {
+var (
-	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	validPattern   = "WHWPPKLOD23IRAUKKY24V9WWV6IC"
-	defer cancel()
+	invalidPattern = "WHWPPKLOD23IRA?KKY24V9WWV6IC"
-	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+	keyword        = "pollsapi"
-	if err != nil {
+)
-		t.Fatalf("could not get test secrets from GCP: %s", err)
-	}
-	secret := testSecrets.MustGetField("POLLSAPI")
-	inactiveSecret := testSecrets.MustGetField("POLLSAPI_INACTIVE")
 
-	type args struct {
+func TestPollsAPI_Pattern(t *testing.T) {
-		ctx    context.Context
+	d := Scanner{}
-		data   []byte
+	ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
-		verify bool
-	}
 	tests := []struct {
-		name    string
+		name  string
-		s       Scanner
+		input string
-		args    args
+		want  []string
-		want    []detectors.Result
-		wantErr bool
 	}{
 		{
-			name: "found, verified",
+			name:  "valid pattern - with keyword pollsapi",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
-			args: args{
+			want:  []string{validPattern},
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a pollsapi secret %s within", secret)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_PollsAPI,
-					Verified:     true,
-				},
-			},
-			wantErr: false,
 		},
 		{
-			name: "found, unverified",
+			name:  "valid pattern - ignore duplicate",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
-			args: args{
+			want:  []string{validPattern},
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a pollsapi secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_PollsAPI,
-					Verified:     false,
-				},
-			},
-			wantErr: false,
 		},
 		{
-			name: "not found",
+			name:  "valid pattern - key out of prefix range",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
-			args: args{
+			want:  []string{},
-				ctx:    context.Background(),
+		},
-				data:   []byte("You cannot find the secret within"),
+		{
-				verify: true,
+			name:  "invalid pattern",
-			},
+			input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
-			want:    nil,
+			want:  []string{},
-			wantErr: false,
 		},
 	}
-	for _, tt := range tests {
+
-		t.Run(tt.name, func(t *testing.T) {
+	for _, test := range tests {
-			s := Scanner{}
+		t.Run(test.name, func(t *testing.T) {
-			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
-			if (err != nil) != tt.wantErr {
+			if len(matchedDetectors) == 0 {
-				t.Errorf("PollsAPI.FromData() error = %v, wantErr %v", err, tt.wantErr)
+				t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
 				return
 			}
-			for i := range got {
-				if len(got[i].Raw) == 0 {
-					t.Fatalf("no raw secret present: \n %+v", got[i])
-				}
-				got[i].Raw = nil
-			}
-			if diff := pretty.Compare(got, tt.want); diff != "" {
-				t.Errorf("PollsAPI.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
-			}
-		})
-	}
-}
 
-func BenchmarkFromData(benchmark *testing.B) {
+			results, err := d.FromData(context.Background(), false, []byte(test.input))
-	ctx := context.Background()
+			if err != nil {
-	s := Scanner{}
+				t.Errorf("error = %v", err)
-	for name, data := range detectors.MustGetBenchmarkData() {
+				return
-		benchmark.Run(name, func(b *testing.B) {
+			}
-			b.ResetTimer()
+
-			for n := 0; n < b.N; n++ {
+			if len(results) != len(test.want) {
-				_, err := s.FromData(ctx, false, data)
+				if len(results) == 0 {
-				if err != nil {
+					t.Errorf("did not receive result")
-					b.Fatal(err)
+				} else {
+					t.Errorf("expected %d results, only received %d", len(test.want), len(results))
 				}
+				return
+			}
+
+			actual := make(map[string]struct{}, len(results))
+			for _, r := range results {
+				if len(r.RawV2) > 0 {
+					actual[string(r.RawV2)] = struct{}{}
+				} else {
+					actual[string(r.Raw)] = struct{}{}
+				}
+			}
+			expected := make(map[string]struct{}, len(test.want))
+			for _, v := range test.want {
+				expected[v] = struct{}{}
+			}
+
+			if diff := cmp.Diff(expected, actual); diff != "" {
+				t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
 			}
 		})
 	}

pkg/detectors/poloniex/poloniex_integration_test.go

@@ -0,0 +1,122 @@
+//go:build detectors
+// +build detectors
+
+package poloniex
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/kylelemons/godebug/pretty"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestPoloniex_FromChunk(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	defer cancel()
+	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
+	if err != nil {
+		t.Fatalf("could not get test secrets from GCP: %s", err)
+	}
+	key := testSecrets.MustGetField("POLONIEX_KEY")
+	inactiveKey := testSecrets.MustGetField("POLONIEX_KEY_INACTIVE")
+	secret := testSecrets.MustGetField("POLONIEX")
+	inactiveSecret := testSecrets.MustGetField("POLONIEX_INACTIVE")
+
+	type args struct {
+		ctx    context.Context
+		data   []byte
+		verify bool
+	}
+	tests := []struct {
+		name    string
+		s       Scanner
+		args    args
+		want    []detectors.Result
+		wantErr bool
+	}{
+		{
+			name: "found, verified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a poloniex key %s with poloniex secret %s within", key, secret)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Poloniex,
+					Verified:     true,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "found, unverified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a poloniex key %s with poloniex secret %s within but not valid", inactiveKey, inactiveSecret)), // the secret would satisfy the regex but not pass validation
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Poloniex,
+					Verified:     false,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "not found",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte("You cannot find the secret within"),
+				verify: true,
+			},
+			want:    nil,
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			s := Scanner{}
+			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("Poloniex.FromData() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			for i := range got {
+				if len(got[i].Raw) == 0 {
+					t.Fatalf("no raw secret present: \n %+v", got[i])
+				}
+				got[i].Raw = nil
+			}
+			if diff := pretty.Compare(got, tt.want); diff != "" {
+				t.Errorf("Poloniex.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+			}
+		})
+	}
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+	ctx := context.Background()
+	s := Scanner{}
+	for name, data := range detectors.MustGetBenchmarkData() {
+		benchmark.Run(name, func(b *testing.B) {
+			b.ResetTimer()
+			for n := 0; n < b.N; n++ {
+				_, err := s.FromData(ctx, false, data)
+				if err != nil {
+					b.Fatal(err)
+				}
+			}
+		})
+	}
+}

pkg/detectors/poloniex/poloniex_test.go

@@ -1,121 +1,82 @@
-//go:build detectors
-// +build detectors
-
 package poloniex
 
 import (
 	"context"
 	"fmt"
 	"testing"
-	"time"
 
-	"github.com/kylelemons/godebug/pretty"
+	"github.com/google/go-cmp/cmp"
+
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
-
+	"github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 )
 
-func TestPoloniex_FromChunk(t *testing.T) {
+var (
-	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	validKey      = "2ELRAUEF-NB06CX0R-ZGXL88HF-BX8B77S6"
-	defer cancel()
+	invalidKey    = "2ELRAUEF?NB06CX0R-ZGXL88HF-BX8B77S6"
-	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
+	validSecret   = "d5dc2f6c299bb01895c604f7038c27af9035937823c7c0e8e25c2efe7f6ddd4fb1bb66781f857e6edf539042389e77043396b123c85197676719ef79a420277b"
-	if err != nil {
+	invalidSecret = "D5dc2f6c299bb01895c604f7038c27af9035937823c7c0e8e25c2efe7f6ddd4fb1bb66781f857e6edf539042389e77043396b123c85197676719ef79a420277B"
-		t.Fatalf("could not get test secrets from GCP: %s", err)
+	keyword       = "poloniex"
-	}
+)
-	key := testSecrets.MustGetField("POLONIEX_KEY")
-	inactiveKey := testSecrets.MustGetField("POLONIEX_KEY_INACTIVE")
-	secret := testSecrets.MustGetField("POLONIEX")
-	inactiveSecret := testSecrets.MustGetField("POLONIEX_INACTIVE")
 
-	type args struct {
+func TestPoloniex_Pattern(t *testing.T) {
-		ctx    context.Context
+	d := Scanner{}
-		data   []byte
+	ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
-		verify bool
-	}
 	tests := []struct {
-		name    string
+		name  string
-		s       Scanner
+		input string
-		args    args
+		want  []string
-		want    []detectors.Result
-		wantErr bool
 	}{
 		{
-			name: "found, verified",
+			name:  "valid pattern - with keyword poloniex",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, validKey, keyword, validSecret),
-			args: args{
+			want:  []string{validKey + validSecret},
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a poloniex key %s with poloniex secret %s within", key, secret)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_Poloniex,
-					Verified:     true,
-				},
-			},
-			wantErr: false,
 		},
 		{
-			name: "found, unverified",
+			name:  "invalid pattern",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, invalidKey, keyword, invalidSecret),
-			args: args{
+			want:  []string{},
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a poloniex key %s with poloniex secret %s within but not valid", inactiveKey, inactiveSecret)), // the secret would satisfy the regex but not pass validation
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_Poloniex,
-					Verified:     false,
-				},
-			},
-			wantErr: false,
-		},
-		{
-			name: "not found",
-			s:    Scanner{},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte("You cannot find the secret within"),
-				verify: true,
-			},
-			want:    nil,
-			wantErr: false,
 		},
 	}
-	for _, tt := range tests {
+
-		t.Run(tt.name, func(t *testing.T) {
+	for _, test := range tests {
-			s := Scanner{}
+		t.Run(test.name, func(t *testing.T) {
-			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
-			if (err != nil) != tt.wantErr {
+			if len(matchedDetectors) == 0 {
-				t.Errorf("Poloniex.FromData() error = %v, wantErr %v", err, tt.wantErr)
+				t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
 				return
 			}
-			for i := range got {
-				if len(got[i].Raw) == 0 {
-					t.Fatalf("no raw secret present: \n %+v", got[i])
-				}
-				got[i].Raw = nil
-			}
-			if diff := pretty.Compare(got, tt.want); diff != "" {
-				t.Errorf("Poloniex.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
-			}
-		})
-	}
-}
 
-func BenchmarkFromData(benchmark *testing.B) {
+			results, err := d.FromData(context.Background(), false, []byte(test.input))
-	ctx := context.Background()
+			if err != nil {
-	s := Scanner{}
+				t.Errorf("error = %v", err)
-	for name, data := range detectors.MustGetBenchmarkData() {
+				return
-		benchmark.Run(name, func(b *testing.B) {
+			}
-			b.ResetTimer()
+
-			for n := 0; n < b.N; n++ {
+			if len(results) != len(test.want) {
-				_, err := s.FromData(ctx, false, data)
+				if len(results) == 0 {
-				if err != nil {
+					t.Errorf("did not receive result")
-					b.Fatal(err)
+				} else {
+					t.Errorf("expected %d results, only received %d", len(test.want), len(results))
 				}
+				return
+			}
+
+			actual := make(map[string]struct{}, len(results))
+			for _, r := range results {
+				if len(r.RawV2) > 0 {
+					actual[string(r.RawV2)] = struct{}{}
+				} else {
+					actual[string(r.Raw)] = struct{}{}
+				}
+			}
+			expected := make(map[string]struct{}, len(test.want))
+			for _, v := range test.want {
+				expected[v] = struct{}{}
+			}
+
+			if diff := cmp.Diff(expected, actual); diff != "" {
+				t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
 			}
 		})
 	}

pkg/detectors/polygon/polygon_integration_test.go

@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package polygon
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/kylelemons/godebug/pretty"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestPolygon_FromChunk(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	defer cancel()
+	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+	if err != nil {
+		t.Fatalf("could not get test secrets from GCP: %s", err)
+	}
+	secret := testSecrets.MustGetField("POLYGON_TOKEN")
+	inactiveSecret := testSecrets.MustGetField("POLYGON_INACTIVE")
+
+	type args struct {
+		ctx    context.Context
+		data   []byte
+		verify bool
+	}
+	tests := []struct {
+		name    string
+		s       Scanner
+		args    args
+		want    []detectors.Result
+		wantErr bool
+	}{
+		{
+			name: "found, verified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a polygon secret %s within", secret)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Polygon,
+					Verified:     true,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "found, unverified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a polygon secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Polygon,
+					Verified:     false,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "not found",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte("You cannot find the secret within"),
+				verify: true,
+			},
+			want:    nil,
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			s := Scanner{}
+			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("Polygon.FromData() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			for i := range got {
+				if len(got[i].Raw) == 0 {
+					t.Fatalf("no raw secret present: \n %+v", got[i])
+				}
+				got[i].Raw = nil
+			}
+			if diff := pretty.Compare(got, tt.want); diff != "" {
+				t.Errorf("Polygon.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+			}
+		})
+	}
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+	ctx := context.Background()
+	s := Scanner{}
+	for name, data := range detectors.MustGetBenchmarkData() {
+		benchmark.Run(name, func(b *testing.B) {
+			b.ResetTimer()
+			for n := 0; n < b.N; n++ {
+				_, err := s.FromData(ctx, false, data)
+				if err != nil {
+					b.Fatal(err)
+				}
+			}
+		})
+	}
+}

pkg/detectors/polygon/polygon_test.go

@@ -1,119 +1,90 @@
-//go:build detectors
-// +build detectors
-
 package polygon
 
 import (
 	"context"
 	"fmt"
 	"testing"
-	"time"
 
-	"github.com/kylelemons/godebug/pretty"
+	"github.com/google/go-cmp/cmp"
+
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
-
+	"github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 )
 
-func TestPolygon_FromChunk(t *testing.T) {
+var (
-	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	validPattern   = "vfINal1N4C0YH6fME3dxmRGaeuO1WjnP"
-	defer cancel()
+	invalidPattern = "vfINal1N4C0YH6fM?3dxmRGaeuO1WjnP"
-	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+	keyword        = "polygon"
-	if err != nil {
+)
-		t.Fatalf("could not get test secrets from GCP: %s", err)
-	}
-	secret := testSecrets.MustGetField("POLYGON_TOKEN")
-	inactiveSecret := testSecrets.MustGetField("POLYGON_INACTIVE")
 
-	type args struct {
+func TestPolygon_Pattern(t *testing.T) {
-		ctx    context.Context
+	d := Scanner{}
-		data   []byte
+	ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
-		verify bool
-	}
 	tests := []struct {
-		name    string
+		name  string
-		s       Scanner
+		input string
-		args    args
+		want  []string
-		want    []detectors.Result
-		wantErr bool
 	}{
 		{
-			name: "found, verified",
+			name:  "valid pattern - with keyword polygon",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
-			args: args{
+			want:  []string{validPattern},
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a polygon secret %s within", secret)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_Polygon,
-					Verified:     true,
-				},
-			},
-			wantErr: false,
 		},
 		{
-			name: "found, unverified",
+			name:  "valid pattern - ignore duplicate",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
-			args: args{
+			want:  []string{validPattern},
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a polygon secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_Polygon,
-					Verified:     false,
-				},
-			},
-			wantErr: false,
 		},
 		{
-			name: "not found",
+			name:  "valid pattern - key out of prefix range",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
-			args: args{
+			want:  []string{},
-				ctx:    context.Background(),
+		},
-				data:   []byte("You cannot find the secret within"),
+		{
-				verify: true,
+			name:  "invalid pattern",
-			},
+			input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
-			want:    nil,
+			want:  []string{},
-			wantErr: false,
 		},
 	}
-	for _, tt := range tests {
+
-		t.Run(tt.name, func(t *testing.T) {
+	for _, test := range tests {
-			s := Scanner{}
+		t.Run(test.name, func(t *testing.T) {
-			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
-			if (err != nil) != tt.wantErr {
+			if len(matchedDetectors) == 0 {
-				t.Errorf("Polygon.FromData() error = %v, wantErr %v", err, tt.wantErr)
+				t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
 				return
 			}
-			for i := range got {
-				if len(got[i].Raw) == 0 {
-					t.Fatalf("no raw secret present: \n %+v", got[i])
-				}
-				got[i].Raw = nil
-			}
-			if diff := pretty.Compare(got, tt.want); diff != "" {
-				t.Errorf("Polygon.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
-			}
-		})
-	}
-}
 
-func BenchmarkFromData(benchmark *testing.B) {
+			results, err := d.FromData(context.Background(), false, []byte(test.input))
-	ctx := context.Background()
+			if err != nil {
-	s := Scanner{}
+				t.Errorf("error = %v", err)
-	for name, data := range detectors.MustGetBenchmarkData() {
+				return
-		benchmark.Run(name, func(b *testing.B) {
+			}
-			b.ResetTimer()
+
-			for n := 0; n < b.N; n++ {
+			if len(results) != len(test.want) {
-				_, err := s.FromData(ctx, false, data)
+				if len(results) == 0 {
-				if err != nil {
+					t.Errorf("did not receive result")
-					b.Fatal(err)
+				} else {
+					t.Errorf("expected %d results, only received %d", len(test.want), len(results))
 				}
+				return
+			}
+
+			actual := make(map[string]struct{}, len(results))
+			for _, r := range results {
+				if len(r.RawV2) > 0 {
+					actual[string(r.RawV2)] = struct{}{}
+				} else {
+					actual[string(r.Raw)] = struct{}{}
+				}
+			}
+			expected := make(map[string]struct{}, len(test.want))
+			for _, v := range test.want {
+				expected[v] = struct{}{}
+			}
+
+			if diff := cmp.Diff(expected, actual); diff != "" {
+				t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
 			}
 		})
 	}

pkg/detectors/portainer/portainer_integration_test.go

@@ -0,0 +1,131 @@
+//go:build detectors
+// +build detectors
+
+package portainer
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestPortainer_FromChunk(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	defer cancel()
+	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
+	if err != nil {
+		t.Fatalf("could not get test secrets from GCP: %s", err)
+	}
+	secret := testSecrets.MustGetField("PORTAINER")
+	endpoint := testSecrets.MustGetField("PORTAINER_ENDPOINT")
+	inactiveSecret := testSecrets.MustGetField("PORTAINER_INACTIVE")
+
+	type args struct {
+		ctx    context.Context
+		data   []byte
+		verify bool
+	}
+	tests := []struct {
+		name                string
+		s                   Scanner
+		args                args
+		want                []detectors.Result
+		wantErr             bool
+		wantVerificationErr bool
+	}{
+		{
+			name: "found, verified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a portainer secret %s for portainer url %s", secret, endpoint)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Portainer,
+					Verified:     true,
+					RawV2:        []byte(secret + endpoint),
+				},
+			},
+			wantErr:             false,
+			wantVerificationErr: false,
+		},
+		{
+			name: "found, unverified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a portainer secret %s for portainer %s within but not valid", inactiveSecret, endpoint)), // the secret would satisfy the regex but not pass validation
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Portainer,
+					Verified:     false,
+					RawV2:        []byte(inactiveSecret + endpoint),
+				},
+			},
+			wantErr:             false,
+			wantVerificationErr: false,
+		},
+		{
+			name: "not found",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte("You cannot find the secret within"),
+				verify: true,
+			},
+			want:                nil,
+			wantErr:             false,
+			wantVerificationErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("Portainer.FromData() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			for i := range got {
+				if len(got[i].Raw) == 0 {
+					t.Fatalf("no raw secret present: \n %+v", got[i])
+				}
+				if (got[i].VerificationError() != nil) != tt.wantVerificationErr {
+					t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())
+				}
+			}
+			ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")
+			if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
+				t.Errorf("Portainer.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+			}
+		})
+	}
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+	ctx := context.Background()
+	s := Scanner{}
+	for name, data := range detectors.MustGetBenchmarkData() {
+		benchmark.Run(name, func(b *testing.B) {
+			b.ResetTimer()
+			for n := 0; n < b.N; n++ {
+				_, err := s.FromData(ctx, false, data)
+				if err != nil {
+					b.Fatal(err)
+				}
+			}
+		})
+	}
+}

pkg/detectors/portainer/portainer_test.go

@@ -1,130 +1,82 @@
-//go:build detectors
-// +build detectors
-
 package portainer
 
 import (
 	"context"
 	"fmt"
 	"testing"
-	"time"
 
 	"github.com/google/go-cmp/cmp"
-	"github.com/google/go-cmp/cmp/cmpopts"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
-
+	"github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 )
 
-func TestPortainer_FromChunk(t *testing.T) {
+var (
-	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	validEndpoint   = "http://>xC'w//b7U@CtF|>|Fqw'2Z"
-	defer cancel()
+	invalidEndpoint = "?ttp://>xC'w//b7U@CtF|>|Fqw'2?"
-	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
+	validToken      = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.EHHJE6Aht7Exhje8rUMLScr2bxcoTvWBl9bjMYZhCMYMLPD3EpUZL9SNd839DcI95lYtMfclPffpFrrJ0BbgryxnrfUSeeSKHu.W9Ur5_DLIBpXO3mfh404_7Kt9o8XZRnLTLyam2fdhB_"
-	if err != nil {
+	invalidToken    = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.EHHJE6Aht7Exhje8rUM?Scr2bxcoTvWBl9bjMYZhCMYMLPD3EpUZL9SNd839DcI95lYtMfclPffpFrrJ0BbgryxnrfUSeeSKHu.W9Ur5_DLIBpXO3mfh404_7Kt9o8XZRnLTLyam2fdhB_"
-		t.Fatalf("could not get test secrets from GCP: %s", err)
+	keyword         = "portainer"
-	}
+)
-	secret := testSecrets.MustGetField("PORTAINER")
-	endpoint := testSecrets.MustGetField("PORTAINER_ENDPOINT")
-	inactiveSecret := testSecrets.MustGetField("PORTAINER_INACTIVE")
 
-	type args struct {
+func TestPortainer_Pattern(t *testing.T) {
-		ctx    context.Context
+	d := Scanner{}
-		data   []byte
+	ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
-		verify bool
-	}
 	tests := []struct {
-		name                string
+		name  string
-		s                   Scanner
+		input string
-		args                args
+		want  []string
-		want                []detectors.Result
-		wantErr             bool
-		wantVerificationErr bool
 	}{
 		{
-			name: "found, verified",
+			name:  "valid pattern - with keyword portainer",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s token - '%s;'\n%s token - '%s'\n", keyword, validEndpoint, keyword, validToken),
-			args: args{
+			want:  []string{validToken + validEndpoint},
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a portainer secret %s for portainer url %s", secret, endpoint)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_Portainer,
-					Verified:     true,
-					RawV2:        []byte(secret + endpoint),
-				},
-			},
-			wantErr:             false,
-			wantVerificationErr: false,
 		},
 		{
-			name: "found, unverified",
+			name:  "invalid pattern",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s token - '%s;'\n%s token - '%s'\n", keyword, invalidEndpoint, keyword, invalidToken),
-			args: args{
+			want:  []string{},
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a portainer secret %s for portainer %s within but not valid", inactiveSecret, endpoint)), // the secret would satisfy the regex but not pass validation
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_Portainer,
-					Verified:     false,
-					RawV2:        []byte(inactiveSecret + endpoint),
-				},
-			},
-			wantErr:             false,
-			wantVerificationErr: false,
-		},
-		{
-			name: "not found",
-			s:    Scanner{},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte("You cannot find the secret within"),
-				verify: true,
-			},
-			want:                nil,
-			wantErr:             false,
-			wantVerificationErr: false,
 		},
 	}
-	for _, tt := range tests {
+
-		t.Run(tt.name, func(t *testing.T) {
+	for _, test := range tests {
-			got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+		t.Run(test.name, func(t *testing.T) {
-			if (err != nil) != tt.wantErr {
+			matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
-				t.Errorf("Portainer.FromData() error = %v, wantErr %v", err, tt.wantErr)
+			if len(matchedDetectors) == 0 {
+				t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
 				return
 			}
-			for i := range got {
-				if len(got[i].Raw) == 0 {
-					t.Fatalf("no raw secret present: \n %+v", got[i])
-				}
-				if (got[i].VerificationError() != nil) != tt.wantVerificationErr {
-					t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())
-				}
-			}
-			ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")
-			if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
-				t.Errorf("Portainer.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
-			}
-		})
-	}
-}
 
-func BenchmarkFromData(benchmark *testing.B) {
+			results, err := d.FromData(context.Background(), false, []byte(test.input))
-	ctx := context.Background()
+			if err != nil {
-	s := Scanner{}
+				t.Errorf("error = %v", err)
-	for name, data := range detectors.MustGetBenchmarkData() {
+				return
-		benchmark.Run(name, func(b *testing.B) {
+			}
-			b.ResetTimer()
+
-			for n := 0; n < b.N; n++ {
+			if len(results) != len(test.want) {
-				_, err := s.FromData(ctx, false, data)
+				if len(results) == 0 {
-				if err != nil {
+					t.Errorf("did not receive result")
-					b.Fatal(err)
+				} else {
+					t.Errorf("expected %d results, only received %d", len(test.want), len(results))
 				}
+				return
+			}
+
+			actual := make(map[string]struct{}, len(results))
+			for _, r := range results {
+				if len(r.RawV2) > 0 {
+					actual[string(r.RawV2)] = struct{}{}
+				} else {
+					actual[string(r.Raw)] = struct{}{}
+				}
+			}
+			expected := make(map[string]struct{}, len(test.want))
+			for _, v := range test.want {
+				expected[v] = struct{}{}
+			}
+
+			if diff := cmp.Diff(expected, actual); diff != "" {
+				t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
 			}
 		})
 	}

pkg/detectors/portainertoken/portainertoken.go

@@ -51,6 +51,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 
 			u, err := detectors.ParseURLAndStripPathAndParams(resEndpointMatch)
 			if err != nil {
+				fmt.Printf("\nINVALID URL\n")
 				// if the URL is invalid just move onto the next one
 				continue
 			}

pkg/detectors/portainertoken/portainertoken_integration_test.go

@@ -0,0 +1,131 @@
+//go:build detectors
+// +build detectors
+
+package portainertoken
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestPortainertoken_FromChunk(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	defer cancel()
+	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
+	if err != nil {
+		t.Fatalf("could not get test secrets from GCP: %s", err)
+	}
+	secret := testSecrets.MustGetField("PORTAINERTOKEN")
+	inactiveSecret := testSecrets.MustGetField("PORTAINERTOKEN_INACTIVE")
+	endpoint := testSecrets.MustGetField("PORTAINER_ENDPOINT")
+
+	type args struct {
+		ctx    context.Context
+		data   []byte
+		verify bool
+	}
+	tests := []struct {
+		name                string
+		s                   Scanner
+		args                args
+		want                []detectors.Result
+		wantErr             bool
+		wantVerificationErr bool
+	}{
+		{
+			name: "found, verified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a portainertoken secret %s within for portainer url %s", secret, endpoint)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_PortainerToken,
+					Verified:     true,
+					RawV2:        []byte(secret + endpoint),
+				},
+			},
+			wantErr:             false,
+			wantVerificationErr: false,
+		},
+		{
+			name: "found, unverified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a portainertoken secret %s within but not valid for portainer url %s", inactiveSecret, endpoint)), // the secret would satisfy the regex but not pass validation
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_PortainerToken,
+					Verified:     false,
+					RawV2:        []byte(inactiveSecret + endpoint),
+				},
+			},
+			wantErr:             false,
+			wantVerificationErr: false,
+		},
+		{
+			name: "not found",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte("You cannot find the secret within"),
+				verify: true,
+			},
+			want:                nil,
+			wantErr:             false,
+			wantVerificationErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("Portainertoken.FromData() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			for i := range got {
+				if len(got[i].Raw) == 0 {
+					t.Fatalf("no raw secret present: \n %+v", got[i])
+				}
+				if (got[i].VerificationError() != nil) != tt.wantVerificationErr {
+					t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())
+				}
+			}
+			ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")
+			if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
+				t.Errorf("Portainertoken.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+			}
+		})
+	}
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+	ctx := context.Background()
+	s := Scanner{}
+	for name, data := range detectors.MustGetBenchmarkData() {
+		benchmark.Run(name, func(b *testing.B) {
+			b.ResetTimer()
+			for n := 0; n < b.N; n++ {
+				_, err := s.FromData(ctx, false, data)
+				if err != nil {
+					b.Fatal(err)
+				}
+			}
+		})
+	}
+}

pkg/detectors/portainertoken/portainertoken_test.go

@@ -1,130 +1,82 @@
-//go:build detectors
-// +build detectors
-
 package portainertoken
 
 import (
 	"context"
 	"fmt"
 	"testing"
-	"time"
 
 	"github.com/google/go-cmp/cmp"
-	"github.com/google/go-cmp/cmp/cmpopts"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
-
+	"github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 )
 
-func TestPortainertoken_FromChunk(t *testing.T) {
+var (
-	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	validKey        = "ptr_9zmQMKyeMqEB_pei887xQBZGhGWm1jXCIT0gI"
-	defer cancel()
+	invalidKey      = "ptr_9zmQMKyeMqEB_pei?87xQBZGhGWm1jXCIT0gI"
-	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
+	validEndpoint   = "http://api.SAaiuc8123.com:12345"
-	if err != nil {
+	invalidEndpoint = "?ttp://api.SAaiuc8123.com:12345"
-		t.Fatalf("could not get test secrets from GCP: %s", err)
+	keyword         = "portainertoken"
-	}
+)
-	secret := testSecrets.MustGetField("PORTAINERTOKEN")
-	inactiveSecret := testSecrets.MustGetField("PORTAINERTOKEN_INACTIVE")
-	endpoint := testSecrets.MustGetField("PORTAINER_ENDPOINT")
 
-	type args struct {
+func TestPortainertoken_Pattern(t *testing.T) {
-		ctx    context.Context
+	d := Scanner{}
-		data   []byte
+	ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
-		verify bool
-	}
 	tests := []struct {
-		name                string
+		name  string
-		s                   Scanner
+		input string
-		args                args
+		want  []string
-		want                []detectors.Result
-		wantErr             bool
-		wantVerificationErr bool
 	}{
 		{
-			name: "found, verified",
+			name:  "valid pattern - with keyword portainertoken",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s token - '%s'\nportainer token - '%s'\n", keyword, validKey, validEndpoint),
-			args: args{
+			want:  []string{validKey + validEndpoint},
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a portainertoken secret %s within for portainer url %s", secret, endpoint)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_PortainerToken,
-					Verified:     true,
-					RawV2:        []byte(secret + endpoint),
-				},
-			},
-			wantErr:             false,
-			wantVerificationErr: false,
 		},
 		{
-			name: "found, unverified",
+			name:  "invalid pattern",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s token - '%s'\nportainer token - '%s'\n", keyword, invalidKey, invalidEndpoint),
-			args: args{
+			want:  []string{},
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a portainertoken secret %s within but not valid for portainer url %s", inactiveSecret, endpoint)), // the secret would satisfy the regex but not pass validation
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_PortainerToken,
-					Verified:     false,
-					RawV2:        []byte(inactiveSecret + endpoint),
-				},
-			},
-			wantErr:             false,
-			wantVerificationErr: false,
-		},
-		{
-			name: "not found",
-			s:    Scanner{},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte("You cannot find the secret within"),
-				verify: true,
-			},
-			want:                nil,
-			wantErr:             false,
-			wantVerificationErr: false,
 		},
 	}
-	for _, tt := range tests {
+
-		t.Run(tt.name, func(t *testing.T) {
+	for _, test := range tests {
-			got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+		t.Run(test.name, func(t *testing.T) {
-			if (err != nil) != tt.wantErr {
+			matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
-				t.Errorf("Portainertoken.FromData() error = %v, wantErr %v", err, tt.wantErr)
+			if len(matchedDetectors) == 0 {
+				t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
 				return
 			}
-			for i := range got {
-				if len(got[i].Raw) == 0 {
-					t.Fatalf("no raw secret present: \n %+v", got[i])
-				}
-				if (got[i].VerificationError() != nil) != tt.wantVerificationErr {
-					t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())
-				}
-			}
-			ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")
-			if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
-				t.Errorf("Portainertoken.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
-			}
-		})
-	}
-}
 
-func BenchmarkFromData(benchmark *testing.B) {
+			results, err := d.FromData(context.Background(), false, []byte(test.input))
-	ctx := context.Background()
+			if err != nil {
-	s := Scanner{}
+				t.Errorf("error = %v", err)
-	for name, data := range detectors.MustGetBenchmarkData() {
+				return
-		benchmark.Run(name, func(b *testing.B) {
+			}
-			b.ResetTimer()
+
-			for n := 0; n < b.N; n++ {
+			if len(results) != len(test.want) {
-				_, err := s.FromData(ctx, false, data)
+				if len(results) == 0 {
-				if err != nil {
+					t.Errorf("did not receive result")
-					b.Fatal(err)
+				} else {
+					t.Errorf("expected %d results, only received %d", len(test.want), len(results))
 				}
+				return
+			}
+
+			actual := make(map[string]struct{}, len(results))
+			for _, r := range results {
+				if len(r.RawV2) > 0 {
+					actual[string(r.RawV2)] = struct{}{}
+				} else {
+					actual[string(r.Raw)] = struct{}{}
+				}
+			}
+			expected := make(map[string]struct{}, len(test.want))
+			for _, v := range test.want {
+				expected[v] = struct{}{}
+			}
+
+			if diff := cmp.Diff(expected, actual); diff != "" {
+				t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
 			}
 		})
 	}

pkg/detectors/positionstack/positionstack_integration_test.go

@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package positionstack
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/kylelemons/godebug/pretty"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestPositionStack_FromChunk(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	defer cancel()
+	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
+	if err != nil {
+		t.Fatalf("could not get test secrets from GCP: %s", err)
+	}
+	secret := testSecrets.MustGetField("POSITIONSTACK")
+	inactiveSecret := testSecrets.MustGetField("POSITIONSTACK_INACTIVE")
+
+	type args struct {
+		ctx    context.Context
+		data   []byte
+		verify bool
+	}
+	tests := []struct {
+		name    string
+		s       Scanner
+		args    args
+		want    []detectors.Result
+		wantErr bool
+	}{
+		{
+			name: "found, verified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a positionstack secret %s within", secret)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_PositionStack,
+					Verified:     true,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "found, unverified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a positionstack secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_PositionStack,
+					Verified:     false,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "not found",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte("You cannot find the secret within"),
+				verify: true,
+			},
+			want:    nil,
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			s := Scanner{}
+			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("PositionStack.FromData() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			for i := range got {
+				if len(got[i].Raw) == 0 {
+					t.Fatalf("no raw secret present: \n %+v", got[i])
+				}
+				got[i].Raw = nil
+			}
+			if diff := pretty.Compare(got, tt.want); diff != "" {
+				t.Errorf("PositionStack.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+			}
+		})
+	}
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+	ctx := context.Background()
+	s := Scanner{}
+	for name, data := range detectors.MustGetBenchmarkData() {
+		benchmark.Run(name, func(b *testing.B) {
+			b.ResetTimer()
+			for n := 0; n < b.N; n++ {
+				_, err := s.FromData(ctx, false, data)
+				if err != nil {
+					b.Fatal(err)
+				}
+			}
+		})
+	}
+}

pkg/detectors/positionstack/positionstack_test.go

@@ -1,119 +1,90 @@
-//go:build detectors
-// +build detectors
-
 package positionstack
 
 import (
 	"context"
 	"fmt"
 	"testing"
-	"time"
 
-	"github.com/kylelemons/godebug/pretty"
+	"github.com/google/go-cmp/cmp"
+
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
-
+	"github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 )
 
-func TestPositionStack_FromChunk(t *testing.T) {
+var (
-	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	validPattern   = "iUgwG0nYZt0TY1x5bfyWMJj02PhW7EGX"
-	defer cancel()
+	invalidPattern = "iUgwG0nYZt0TY1x5?fyWMJj02PhW7EGX"
-	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
+	keyword        = "positionstack"
-	if err != nil {
+)
-		t.Fatalf("could not get test secrets from GCP: %s", err)
-	}
-	secret := testSecrets.MustGetField("POSITIONSTACK")
-	inactiveSecret := testSecrets.MustGetField("POSITIONSTACK_INACTIVE")
 
-	type args struct {
+func TestPositionStack_Pattern(t *testing.T) {
-		ctx    context.Context
+	d := Scanner{}
-		data   []byte
+	ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
-		verify bool
-	}
 	tests := []struct {
-		name    string
+		name  string
-		s       Scanner
+		input string
-		args    args
+		want  []string
-		want    []detectors.Result
-		wantErr bool
 	}{
 		{
-			name: "found, verified",
+			name:  "valid pattern - with keyword positionstack",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
-			args: args{
+			want:  []string{validPattern},
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a positionstack secret %s within", secret)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_PositionStack,
-					Verified:     true,
-				},
-			},
-			wantErr: false,
 		},
 		{
-			name: "found, unverified",
+			name:  "valid pattern - ignore duplicate",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
-			args: args{
+			want:  []string{validPattern},
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a positionstack secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_PositionStack,
-					Verified:     false,
-				},
-			},
-			wantErr: false,
 		},
 		{
-			name: "not found",
+			name:  "valid pattern - key out of prefix range",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
-			args: args{
+			want:  []string{},
-				ctx:    context.Background(),
+		},
-				data:   []byte("You cannot find the secret within"),
+		{
-				verify: true,
+			name:  "invalid pattern",
-			},
+			input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
-			want:    nil,
+			want:  []string{},
-			wantErr: false,
 		},
 	}
-	for _, tt := range tests {
+
-		t.Run(tt.name, func(t *testing.T) {
+	for _, test := range tests {
-			s := Scanner{}
+		t.Run(test.name, func(t *testing.T) {
-			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
-			if (err != nil) != tt.wantErr {
+			if len(matchedDetectors) == 0 {
-				t.Errorf("PositionStack.FromData() error = %v, wantErr %v", err, tt.wantErr)
+				t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
 				return
 			}
-			for i := range got {
-				if len(got[i].Raw) == 0 {
-					t.Fatalf("no raw secret present: \n %+v", got[i])
-				}
-				got[i].Raw = nil
-			}
-			if diff := pretty.Compare(got, tt.want); diff != "" {
-				t.Errorf("PositionStack.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
-			}
-		})
-	}
-}
 
-func BenchmarkFromData(benchmark *testing.B) {
+			results, err := d.FromData(context.Background(), false, []byte(test.input))
-	ctx := context.Background()
+			if err != nil {
-	s := Scanner{}
+				t.Errorf("error = %v", err)
-	for name, data := range detectors.MustGetBenchmarkData() {
+				return
-		benchmark.Run(name, func(b *testing.B) {
+			}
-			b.ResetTimer()
+
-			for n := 0; n < b.N; n++ {
+			if len(results) != len(test.want) {
-				_, err := s.FromData(ctx, false, data)
+				if len(results) == 0 {
-				if err != nil {
+					t.Errorf("did not receive result")
-					b.Fatal(err)
+				} else {
+					t.Errorf("expected %d results, only received %d", len(test.want), len(results))
 				}
+				return
+			}
+
+			actual := make(map[string]struct{}, len(results))
+			for _, r := range results {
+				if len(r.RawV2) > 0 {
+					actual[string(r.RawV2)] = struct{}{}
+				} else {
+					actual[string(r.Raw)] = struct{}{}
+				}
+			}
+			expected := make(map[string]struct{}, len(test.want))
+			for _, v := range test.want {
+				expected[v] = struct{}{}
+			}
+
+			if diff := cmp.Diff(expected, actual); diff != "" {
+				t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
 			}
 		})
 	}

pkg/detectors/postageapp/postageapp_integration_test.go

@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package postageapp
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/kylelemons/godebug/pretty"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestPostageApp_FromChunk(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	defer cancel()
+	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
+	if err != nil {
+		t.Fatalf("could not get test secrets from GCP: %s", err)
+	}
+	secret := testSecrets.MustGetField("POSTAGEAPP")
+	inactiveSecret := testSecrets.MustGetField("POSTAGEAPP_INACTIVE")
+
+	type args struct {
+		ctx    context.Context
+		data   []byte
+		verify bool
+	}
+	tests := []struct {
+		name    string
+		s       Scanner
+		args    args
+		want    []detectors.Result
+		wantErr bool
+	}{
+		{
+			name: "found, verified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a postageapp secret %s within", secret)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_PostageApp,
+					Verified:     true,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "found, unverified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a postageapp secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_PostageApp,
+					Verified:     false,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "not found",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte("You cannot find the secret within"),
+				verify: true,
+			},
+			want:    nil,
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			s := Scanner{}
+			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("PostageApp.FromData() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			for i := range got {
+				if len(got[i].Raw) == 0 {
+					t.Fatalf("no raw secret present: \n %+v", got[i])
+				}
+				got[i].Raw = nil
+			}
+			if diff := pretty.Compare(got, tt.want); diff != "" {
+				t.Errorf("PostageApp.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+			}
+		})
+	}
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+	ctx := context.Background()
+	s := Scanner{}
+	for name, data := range detectors.MustGetBenchmarkData() {
+		benchmark.Run(name, func(b *testing.B) {
+			b.ResetTimer()
+			for n := 0; n < b.N; n++ {
+				_, err := s.FromData(ctx, false, data)
+				if err != nil {
+					b.Fatal(err)
+				}
+			}
+		})
+	}
+}

pkg/detectors/postageapp/postageapp_test.go

@@ -1,119 +1,90 @@
-//go:build detectors
-// +build detectors
-
 package postageapp
 
 import (
 	"context"
 	"fmt"
 	"testing"
-	"time"
 
-	"github.com/kylelemons/godebug/pretty"
+	"github.com/google/go-cmp/cmp"
+
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
-
+	"github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 )
 
-func TestPostageApp_FromChunk(t *testing.T) {
+var (
-	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	validPattern   = "VJhmldVkyWB2bmLumsZzOtJKfxliCAXP"
-	defer cancel()
+	invalidPattern = "VJh?ldVkyWB2bmLumsZzOtJKfxliCAXP"
-	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
+	keyword        = "postageapp"
-	if err != nil {
+)
-		t.Fatalf("could not get test secrets from GCP: %s", err)
-	}
-	secret := testSecrets.MustGetField("POSTAGEAPP")
-	inactiveSecret := testSecrets.MustGetField("POSTAGEAPP_INACTIVE")
 
-	type args struct {
+func TestPostageApp_Pattern(t *testing.T) {
-		ctx    context.Context
+	d := Scanner{}
-		data   []byte
+	ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
-		verify bool
-	}
 	tests := []struct {
-		name    string
+		name  string
-		s       Scanner
+		input string
-		args    args
+		want  []string
-		want    []detectors.Result
-		wantErr bool
 	}{
 		{
-			name: "found, verified",
+			name:  "valid pattern - with keyword postageapp",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
-			args: args{
+			want:  []string{validPattern},
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a postageapp secret %s within", secret)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_PostageApp,
-					Verified:     true,
-				},
-			},
-			wantErr: false,
 		},
 		{
-			name: "found, unverified",
+			name:  "valid pattern - ignore duplicate",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
-			args: args{
+			want:  []string{validPattern},
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a postageapp secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_PostageApp,
-					Verified:     false,
-				},
-			},
-			wantErr: false,
 		},
 		{
-			name: "not found",
+			name:  "valid pattern - key out of prefix range",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
-			args: args{
+			want:  []string{},
-				ctx:    context.Background(),
+		},
-				data:   []byte("You cannot find the secret within"),
+		{
-				verify: true,
+			name:  "invalid pattern",
-			},
+			input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
-			want:    nil,
+			want:  []string{},
-			wantErr: false,
 		},
 	}
-	for _, tt := range tests {
+
-		t.Run(tt.name, func(t *testing.T) {
+	for _, test := range tests {
-			s := Scanner{}
+		t.Run(test.name, func(t *testing.T) {
-			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
-			if (err != nil) != tt.wantErr {
+			if len(matchedDetectors) == 0 {
-				t.Errorf("PostageApp.FromData() error = %v, wantErr %v", err, tt.wantErr)
+				t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
 				return
 			}
-			for i := range got {
-				if len(got[i].Raw) == 0 {
-					t.Fatalf("no raw secret present: \n %+v", got[i])
-				}
-				got[i].Raw = nil
-			}
-			if diff := pretty.Compare(got, tt.want); diff != "" {
-				t.Errorf("PostageApp.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
-			}
-		})
-	}
-}
 
-func BenchmarkFromData(benchmark *testing.B) {
+			results, err := d.FromData(context.Background(), false, []byte(test.input))
-	ctx := context.Background()
+			if err != nil {
-	s := Scanner{}
+				t.Errorf("error = %v", err)
-	for name, data := range detectors.MustGetBenchmarkData() {
+				return
-		benchmark.Run(name, func(b *testing.B) {
+			}
-			b.ResetTimer()
+
-			for n := 0; n < b.N; n++ {
+			if len(results) != len(test.want) {
-				_, err := s.FromData(ctx, false, data)
+				if len(results) == 0 {
-				if err != nil {
+					t.Errorf("did not receive result")
-					b.Fatal(err)
+				} else {
+					t.Errorf("expected %d results, only received %d", len(test.want), len(results))
 				}
+				return
+			}
+
+			actual := make(map[string]struct{}, len(results))
+			for _, r := range results {
+				if len(r.RawV2) > 0 {
+					actual[string(r.RawV2)] = struct{}{}
+				} else {
+					actual[string(r.Raw)] = struct{}{}
+				}
+			}
+			expected := make(map[string]struct{}, len(test.want))
+			for _, v := range test.want {
+				expected[v] = struct{}{}
+			}
+
+			if diff := cmp.Diff(expected, actual); diff != "" {
+				t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
 			}
 		})
 	}

pkg/detectors/postbacks/postbacks_integration_test.go

@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package postbacks
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/kylelemons/godebug/pretty"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestPostbacks_FromChunk(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	defer cancel()
+	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+	if err != nil {
+		t.Fatalf("could not get test secrets from GCP: %s", err)
+	}
+	secret := testSecrets.MustGetField("POSTBACKS")
+	inactiveSecret := testSecrets.MustGetField("POSTBACKS_INACTIVE")
+
+	type args struct {
+		ctx    context.Context
+		data   []byte
+		verify bool
+	}
+	tests := []struct {
+		name    string
+		s       Scanner
+		args    args
+		want    []detectors.Result
+		wantErr bool
+	}{
+		{
+			name: "found, verified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a postbacks secret %s within", secret)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Postbacks,
+					Verified:     true,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "found, unverified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a postbacks secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Postbacks,
+					Verified:     false,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "not found",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte("You cannot find the secret within"),
+				verify: true,
+			},
+			want:    nil,
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			s := Scanner{}
+			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("Postbacks.FromData() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			for i := range got {
+				if len(got[i].Raw) == 0 {
+					t.Fatalf("no raw secret present: \n %+v", got[i])
+				}
+				got[i].Raw = nil
+			}
+			if diff := pretty.Compare(got, tt.want); diff != "" {
+				t.Errorf("Postbacks.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+			}
+		})
+	}
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+	ctx := context.Background()
+	s := Scanner{}
+	for name, data := range detectors.MustGetBenchmarkData() {
+		benchmark.Run(name, func(b *testing.B) {
+			b.ResetTimer()
+			for n := 0; n < b.N; n++ {
+				_, err := s.FromData(ctx, false, data)
+				if err != nil {
+					b.Fatal(err)
+				}
+			}
+		})
+	}
+}

pkg/detectors/postbacks/postbacks_test.go

@@ -1,119 +1,90 @@
-//go:build detectors
-// +build detectors
-
 package postbacks
 
 import (
 	"context"
 	"fmt"
 	"testing"
-	"time"
 
-	"github.com/kylelemons/godebug/pretty"
+	"github.com/google/go-cmp/cmp"
+
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
-
+	"github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 )
 
-func TestPostbacks_FromChunk(t *testing.T) {
+var (
-	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	validPattern   = "64176f2e-e1da-3e29-2a08-19fa8bcb1838"
-	defer cancel()
+	invalidPattern = "64176f2e?e1da-3e29-2a08-19fa8bcb1838"
-	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+	keyword        = "postbacks"
-	if err != nil {
+)
-		t.Fatalf("could not get test secrets from GCP: %s", err)
-	}
-	secret := testSecrets.MustGetField("POSTBACKS")
-	inactiveSecret := testSecrets.MustGetField("POSTBACKS_INACTIVE")
 
-	type args struct {
+func TestPostbacks_Pattern(t *testing.T) {
-		ctx    context.Context
+	d := Scanner{}
-		data   []byte
+	ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
-		verify bool
-	}
 	tests := []struct {
-		name    string
+		name  string
-		s       Scanner
+		input string
-		args    args
+		want  []string
-		want    []detectors.Result
-		wantErr bool
 	}{
 		{
-			name: "found, verified",
+			name:  "valid pattern - with keyword postbacks",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
-			args: args{
+			want:  []string{validPattern},
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a postbacks secret %s within", secret)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_Postbacks,
-					Verified:     true,
-				},
-			},
-			wantErr: false,
 		},
 		{
-			name: "found, unverified",
+			name:  "valid pattern - ignore duplicate",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
-			args: args{
+			want:  []string{validPattern},
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a postbacks secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_Postbacks,
-					Verified:     false,
-				},
-			},
-			wantErr: false,
 		},
 		{
-			name: "not found",
+			name:  "valid pattern - key out of prefix range",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
-			args: args{
+			want:  []string{},
-				ctx:    context.Background(),
+		},
-				data:   []byte("You cannot find the secret within"),
+		{
-				verify: true,
+			name:  "invalid pattern",
-			},
+			input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
-			want:    nil,
+			want:  []string{},
-			wantErr: false,
 		},
 	}
-	for _, tt := range tests {
+
-		t.Run(tt.name, func(t *testing.T) {
+	for _, test := range tests {
-			s := Scanner{}
+		t.Run(test.name, func(t *testing.T) {
-			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
-			if (err != nil) != tt.wantErr {
+			if len(matchedDetectors) == 0 {
-				t.Errorf("Postbacks.FromData() error = %v, wantErr %v", err, tt.wantErr)
+				t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
 				return
 			}
-			for i := range got {
-				if len(got[i].Raw) == 0 {
-					t.Fatalf("no raw secret present: \n %+v", got[i])
-				}
-				got[i].Raw = nil
-			}
-			if diff := pretty.Compare(got, tt.want); diff != "" {
-				t.Errorf("Postbacks.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
-			}
-		})
-	}
-}
 
-func BenchmarkFromData(benchmark *testing.B) {
+			results, err := d.FromData(context.Background(), false, []byte(test.input))
-	ctx := context.Background()
+			if err != nil {
-	s := Scanner{}
+				t.Errorf("error = %v", err)
-	for name, data := range detectors.MustGetBenchmarkData() {
+				return
-		benchmark.Run(name, func(b *testing.B) {
+			}
-			b.ResetTimer()
+
-			for n := 0; n < b.N; n++ {
+			if len(results) != len(test.want) {
-				_, err := s.FromData(ctx, false, data)
+				if len(results) == 0 {
-				if err != nil {
+					t.Errorf("did not receive result")
-					b.Fatal(err)
+				} else {
+					t.Errorf("expected %d results, only received %d", len(test.want), len(results))
 				}
+				return
+			}
+
+			actual := make(map[string]struct{}, len(results))
+			for _, r := range results {
+				if len(r.RawV2) > 0 {
+					actual[string(r.RawV2)] = struct{}{}
+				} else {
+					actual[string(r.Raw)] = struct{}{}
+				}
+			}
+			expected := make(map[string]struct{}, len(test.want))
+			for _, v := range test.want {
+				expected[v] = struct{}{}
+			}
+
+			if diff := cmp.Diff(expected, actual); diff != "" {
+				t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
 			}
 		})
 	}

pkg/detectors/postgres/postgres_integration_test.go

@@ -0,0 +1,406 @@
+//go:build detectors
+// +build detectors
+
+package postgres
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"os/exec"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
+	"github.com/lib/pq"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+var postgresDockerHash string
+
+const (
+	postgresUser = "postgres"
+	postgresPass = "23201da=b56ca236f3dc6736c0f9afad"
+	postgresHost = "localhost"
+	postgresPort = "5434" // Do not use 5433, as local dev environments can use it for other things
+
+	inactivePass = "inactive"
+	inactiveHost = "192.0.2.0"
+)
+
+func TestPostgres_FromChunk(t *testing.T) {
+	if err := startPostgres(); err != nil {
+		if exitErr, ok := err.(*exec.ExitError); ok {
+			t.Fatalf("could not start local postgres: %v w/stderr:\n%s", err, string(exitErr.Stderr))
+		} else {
+			t.Fatalf("could not start local postgres: %v", err)
+		}
+	}
+	defer stopPostgres()
+
+	// The detector is written to connect to the database 'postgres' if no explicit database is found in the candidate
+	// secret (because pq uses 'postgres' as a default if no database is specified). If the target cluster doesn't
+	// actually have a database with this name, but our credentials are good, then Postgres will give us a "missing
+	// database" error message instead of an authentication failure.
+	//
+	// Unfortunately, directly validating this in the automated tests is awkward because the docker image's POSTGRES_DB
+	// environment variable doesn't appear to work: The database created is always named 'postgres', no matter what
+	// POSTGRES_DB is set to. This means that we can't replicate a cluster that has no database named 'postgres', so we
+	// can't directly test what happens if we see one. To work around this, all the automated tests try to connect to
+	// the nonexistent database 'postgres2'. In this way, we test the logic of attempting to connect to a non-existent
+	// database, even though the test cases are the inverse of what we'd see in the wild.
+	type args struct {
+		ctx    context.Context
+		data   []byte
+		verify bool
+	}
+	tests := []struct {
+		name    string
+		s       Scanner
+		args    args
+		want    []detectors.Result
+		wantErr bool
+	}{
+		{
+			name: "not found",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte("You cannot find the secret within"),
+				verify: true,
+			},
+			want:    nil,
+			wantErr: false,
+		},
+		{
+			name: "found connection URI with ssl mode unset, verified",
+			s:    Scanner{detectLoopback: true},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf(`postgresql://%s:%s@%s:%s/postgres2`, postgresUser, postgresPass, postgresHost, postgresPort)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Postgres,
+					Verified:     true,
+					Raw:          []byte("postgresql://postgres:23201da=b56ca236f3dc6736c0f9afad@localhost:5434"),
+					RawV2:        []byte("postgresql://postgres:23201da=b56ca236f3dc6736c0f9afad@localhost:5434"),
+					ExtraData:    map[string]string{"sslmode": "<unset>"},
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "found connection URI with ssl mode 'prefer', verified",
+			s:    Scanner{detectLoopback: true},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf(`postgresql://%s:%s@%s:%s/postgres2?sslmode=prefer`, postgresUser, postgresPass, postgresHost, postgresPort)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Postgres,
+					Verified:     true,
+					Raw:          []byte("postgresql://postgres:23201da=b56ca236f3dc6736c0f9afad@localhost:5434"),
+					RawV2:        []byte("postgresql://postgres:23201da=b56ca236f3dc6736c0f9afad@localhost:5434"),
+					ExtraData:    map[string]string{"sslmode": "prefer"},
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "found connection URI with ssl mode 'allow', verified",
+			s:    Scanner{detectLoopback: true},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf(`postgresql://%s:%s@%s:%s/postgres2?sslmode=allow`, postgresUser, postgresPass, postgresHost, postgresPort)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Postgres,
+					Verified:     true,
+					Raw:          []byte("postgresql://postgres:23201da=b56ca236f3dc6736c0f9afad@localhost:5434"),
+					RawV2:        []byte("postgresql://postgres:23201da=b56ca236f3dc6736c0f9afad@localhost:5434"),
+					ExtraData:    map[string]string{"sslmode": "allow"},
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "found connection URI with requiressl=0, verified",
+			s:    Scanner{detectLoopback: true},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf(`postgresql://%s:%s@%s:%s/postgres2?requiressl=0`, postgresUser, postgresPass, postgresHost, postgresPort)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Postgres,
+					Verified:     true,
+					Raw:          []byte("postgresql://postgres:23201da=b56ca236f3dc6736c0f9afad@localhost:5434"),
+					RawV2:        []byte("postgresql://postgres:23201da=b56ca236f3dc6736c0f9afad@localhost:5434"),
+					ExtraData:    map[string]string{"sslmode": "prefer"},
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "found connection URI without database, verified",
+			s:    Scanner{detectLoopback: true},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf(`postgresql://%s:%s@%s:%s/`, postgresUser, postgresPass, postgresHost, postgresPort)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Postgres,
+					Verified:     true,
+					Raw:          []byte("postgresql://postgres:23201da=b56ca236f3dc6736c0f9afad@localhost:5434"),
+					RawV2:        []byte("postgresql://postgres:23201da=b56ca236f3dc6736c0f9afad@localhost:5434"),
+					ExtraData:    map[string]string{"sslmode": "<unset>"},
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "found connection URI, unverified",
+			s:    Scanner{detectLoopback: true},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf(`postgresql://%s:%s@%s:%s/postgres2`, postgresUser, inactivePass, postgresHost, postgresPort)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Postgres,
+					Verified:     false,
+					Raw:          []byte("postgresql://postgres:inactive@localhost:5434"),
+					RawV2:        []byte("postgresql://postgres:inactive@localhost:5434"),
+					ExtraData:    map[string]string{"sslmode": "<unset>"},
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "ignored localhost",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf(`postgresql://%s:%s@%s:%s/postgres2`, postgresUser, postgresPass, "localhost", postgresPort)),
+				verify: true,
+			},
+			want:    nil,
+			wantErr: false,
+		},
+		{
+			name: "ignored 127.0.0.1",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf(`postgresql://%s:%s@%s:%s/postgres2`, postgresUser, postgresPass, "127.0.0.1", postgresPort)),
+				verify: true,
+			},
+			want:    nil,
+			wantErr: false,
+		},
+		{
+			name: "found connection URI, unverified due to error - inactive host",
+			s:    Scanner{},
+			args: func() args {
+				ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
+				defer cancel()
+				return args{
+					ctx:    ctx,
+					data:   []byte(fmt.Sprintf(`postgresql://%s:%s@%s:%s/postgres2`, postgresUser, postgresPass, inactiveHost, postgresPort)),
+					verify: true,
+				}
+			}(),
+			want: func() []detectors.Result {
+				r := detectors.Result{
+					DetectorType: detectorspb.DetectorType_Postgres,
+					Verified:     false,
+					Raw:          []byte("postgresql://postgres:[email protected]:5434"),
+					RawV2:        []byte("postgresql://postgres:[email protected]:5434"),
+					ExtraData:    map[string]string{"sslmode": "<unset>"},
+				}
+				r.SetVerificationError(errors.New("i/o timeout"))
+				return []detectors.Result{r}
+			}(),
+			wantErr: false,
+		},
+		{
+			name: "found connection URI, unverified due to error - wrong port",
+			s:    Scanner{detectLoopback: true},
+			args: func() args {
+				ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
+				defer cancel()
+				return args{
+					ctx:    ctx,
+					data:   []byte(fmt.Sprintf(`postgresql://%s:%s@%s/postgres2`, postgresUser, postgresPass, postgresHost)),
+					verify: true,
+				}
+			}(),
+			want: func() []detectors.Result {
+				r := detectors.Result{
+					DetectorType: detectorspb.DetectorType_Postgres,
+					Verified:     false,
+					Raw:          []byte("postgresql://postgres:23201da=b56ca236f3dc6736c0f9afad@localhost:5432"),
+					RawV2:        []byte("postgresql://postgres:23201da=b56ca236f3dc6736c0f9afad@localhost:5432"),
+					ExtraData:    map[string]string{"sslmode": "<unset>"},
+				}
+				r.SetVerificationError(errors.New("connection refused"))
+				return []detectors.Result{r}
+			}(),
+			wantErr: false,
+		},
+		{
+			name: "found connection URI, unverified due to error - ssl not supported (using sslmode)",
+			s:    Scanner{detectLoopback: true},
+			args: func() args {
+				ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
+				defer cancel()
+				return args{
+					ctx:    ctx,
+					data:   []byte(fmt.Sprintf(`postgresql://%s:%s@%s:%s/postgres2?sslmode=require`, postgresUser, postgresPass, postgresHost, postgresPort)),
+					verify: true,
+				}
+			}(),
+			want: func() []detectors.Result {
+				r := detectors.Result{
+					DetectorType: detectorspb.DetectorType_Postgres,
+					Verified:     false,
+					Raw:          []byte("postgresql://postgres:23201da=b56ca236f3dc6736c0f9afad@localhost:5434"),
+					RawV2:        []byte("postgresql://postgres:23201da=b56ca236f3dc6736c0f9afad@localhost:5434"),
+					ExtraData:    map[string]string{"sslmode": "require"},
+				}
+				r.SetVerificationError(pq.ErrSSLNotSupported)
+				return []detectors.Result{r}
+			}(),
+			wantErr: false,
+		},
+		{
+			name: "found connection URI, unverified due to error - ssl not supported (using requiressl)",
+			s:    Scanner{detectLoopback: true},
+			args: func() args {
+				ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
+				defer cancel()
+				return args{
+					ctx:    ctx,
+					data:   []byte(fmt.Sprintf(`postgresql://%s:%s@%s:%s/postgres2?requiressl=1`, postgresUser, postgresPass, postgresHost, postgresPort)),
+					verify: true,
+				}
+			}(),
+			want: func() []detectors.Result {
+				r := detectors.Result{
+					DetectorType: detectorspb.DetectorType_Postgres,
+					Verified:     false,
+					Raw:          []byte("postgresql://postgres:23201da=b56ca236f3dc6736c0f9afad@localhost:5434"),
+					RawV2:        []byte("postgresql://postgres:23201da=b56ca236f3dc6736c0f9afad@localhost:5434"),
+					ExtraData:    map[string]string{"sslmode": "require"},
+				}
+				r.SetVerificationError(pq.ErrSSLNotSupported)
+				return []detectors.Result{r}
+			}(),
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("postgres.FromData() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			for i := range got {
+				if len(got[i].Raw) == 0 {
+					t.Fatalf("no raw secret present: \n %+v", got[i])
+				}
+				gotErr := ""
+				if got[i].VerificationError() != nil {
+					gotErr = got[i].VerificationError().Error()
+				}
+				wantErr := ""
+				if tt.want[i].VerificationError() != nil {
+					wantErr = tt.want[i].VerificationError().Error()
+				}
+				if gotErr != wantErr {
+					t.Fatalf("wantVerificationError = %v, verification error = %v", tt.want[i].VerificationError(), got[i].VerificationError())
+				}
+			}
+			ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "verificationError", "AnalysisInfo")
+			if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
+				t.Errorf("Postgres.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+			}
+		})
+	}
+}
+
+func dockerLogLine(hash string, needle string) chan struct{} {
+	ch := make(chan struct{}, 1)
+	go func() {
+		for {
+			out, err := exec.Command("docker", "logs", hash).CombinedOutput()
+			if err != nil {
+				panic(err)
+			}
+			if strings.Contains(string(out), needle) {
+				ch <- struct{}{}
+				return
+			}
+			time.Sleep(1 * time.Second)
+		}
+	}()
+	return ch
+}
+
+func startPostgres() error {
+	cmd := exec.Command(
+		"docker", "run", "--rm", "-p", postgresPort+":"+defaultPort,
+		"-e", "POSTGRES_PASSWORD="+postgresPass,
+		"-e", "POSTGRES_USER="+postgresUser,
+		"-d", "postgres",
+	)
+	fmt.Println(cmd.String())
+	out, err := cmd.Output()
+	if err != nil {
+		return err
+	}
+	postgresDockerHash = string(bytes.TrimSpace(out))
+	select {
+	case <-dockerLogLine(postgresDockerHash, "PostgreSQL init process complete; ready for start up."):
+		return nil
+	case <-time.After(30 * time.Second):
+		stopPostgres()
+		return errors.New("timeout waiting for postgres database to be ready")
+	}
+}
+
+func stopPostgres() {
+	exec.Command("docker", "kill", postgresDockerHash).Run()
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+	ctx := context.Background()
+	s := Scanner{}
+	for name, data := range detectors.MustGetBenchmarkData() {
+		benchmark.Run(name, func(b *testing.B) {
+			b.ResetTimer()
+			for n := 0; n < b.N; n++ {
+				_, err := s.FromData(ctx, false, data)
+				if err != nil {
+					b.Fatal(err)
+				}
+			}
+		})
+	}
+}

pkg/detectors/postgres/postgres_test.go

@@ -1,405 +1,82 @@
-//go:build detectors
-// +build detectors
-
 package postgres
 
 import (
-	"bytes"
 	"context"
-	"errors"
 	"fmt"
-	"os/exec"
-	"strings"
 	"testing"
-	"time"
 
 	"github.com/google/go-cmp/cmp"
-	"github.com/google/go-cmp/cmp/cmpopts"
+
-	"github.com/lib/pq"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
-
+	"github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 )
 
-var postgresDockerHash string
+var (
-
+	validUriPattern           = "postgres://sN19x:[email protected]:5432"
-const (
+	invalidUriPattern         = "?ostgres://sN19x:[email protected]:5432"
-	postgresUser = "postgres"
+	validConnStrPartPattern   = "gVmMTdkwLwmZljcIOXhEmuZ='.jD#=-;|9tD!r^6('"
-	postgresPass = "23201da=b56ca236f3dc6736c0f9afad"
+	invalidConnStrPartPattern = "gVmMTdkwLwmZljcIOXhEmu?='.jD#=-;|9tD!r^6('"
-	postgresHost = "localhost"
+	keyword                   = "postgres"
-	postgresPort = "5434" // Do not use 5433, as local dev environments can use it for other things
-
-	inactivePass = "inactive"
-	inactiveHost = "192.0.2.0"
 )
 
-func TestPostgres_FromChunk(t *testing.T) {
+func TestPostgres_Pattern(t *testing.T) {
-	if err := startPostgres(); err != nil {
+	d := Scanner{}
-		if exitErr, ok := err.(*exec.ExitError); ok {
+	ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
-			t.Fatalf("could not start local postgres: %v w/stderr:\n%s", err, string(exitErr.Stderr))
-		} else {
-			t.Fatalf("could not start local postgres: %v", err)
-		}
-	}
-	defer stopPostgres()
-
-	// The detector is written to connect to the database 'postgres' if no explicit database is found in the candidate
-	// secret (because pq uses 'postgres' as a default if no database is specified). If the target cluster doesn't
-	// actually have a database with this name, but our credentials are good, then Postgres will give us a "missing
-	// database" error message instead of an authentication failure.
-	//
-	// Unfortunately, directly validating this in the automated tests is awkward because the docker image's POSTGRES_DB
-	// environment variable doesn't appear to work: The database created is always named 'postgres', no matter what
-	// POSTGRES_DB is set to. This means that we can't replicate a cluster that has no database named 'postgres', so we
-	// can't directly test what happens if we see one. To work around this, all the automated tests try to connect to
-	// the nonexistent database 'postgres2'. In this way, we test the logic of attempting to connect to a non-existent
-	// database, even though the test cases are the inverse of what we'd see in the wild.
-	type args struct {
-		ctx    context.Context
-		data   []byte
-		verify bool
-	}
 	tests := []struct {
-		name    string
+		name  string
-		s       Scanner
+		input string
-		args    args
+		want  []string
-		want    []detectors.Result
-		wantErr bool
 	}{
 		{
-			name: "not found",
+			name:  "valid pattern - with keyword postgres",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, validUriPattern, keyword, validConnStrPartPattern),
-			args: args{
+			want:  []string{validUriPattern},
-				ctx:    context.Background(),
-				data:   []byte("You cannot find the secret within"),
-				verify: true,
-			},
-			want:    nil,
-			wantErr: false,
 		},
 		{
-			name: "found connection URI with ssl mode unset, verified",
+			name:  "invalid pattern",
-			s:    Scanner{detectLoopback: true},
+			input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, invalidUriPattern, keyword, invalidConnStrPartPattern),
-			args: args{
+			want:  []string{},
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf(`postgresql://%s:%s@%s:%s/postgres2`, postgresUser, postgresPass, postgresHost, postgresPort)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_Postgres,
-					Verified:     true,
-					Raw:          []byte("postgresql://postgres:23201da=b56ca236f3dc6736c0f9afad@localhost:5434"),
-					RawV2:        []byte("postgresql://postgres:23201da=b56ca236f3dc6736c0f9afad@localhost:5434"),
-					ExtraData:    map[string]string{"sslmode": "<unset>"},
-				},
-			},
-			wantErr: false,
-		},
-		{
-			name: "found connection URI with ssl mode 'prefer', verified",
-			s:    Scanner{detectLoopback: true},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf(`postgresql://%s:%s@%s:%s/postgres2?sslmode=prefer`, postgresUser, postgresPass, postgresHost, postgresPort)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_Postgres,
-					Verified:     true,
-					Raw:          []byte("postgresql://postgres:23201da=b56ca236f3dc6736c0f9afad@localhost:5434"),
-					RawV2:        []byte("postgresql://postgres:23201da=b56ca236f3dc6736c0f9afad@localhost:5434"),
-					ExtraData:    map[string]string{"sslmode": "prefer"},
-				},
-			},
-			wantErr: false,
-		},
-		{
-			name: "found connection URI with ssl mode 'allow', verified",
-			s:    Scanner{detectLoopback: true},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf(`postgresql://%s:%s@%s:%s/postgres2?sslmode=allow`, postgresUser, postgresPass, postgresHost, postgresPort)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_Postgres,
-					Verified:     true,
-					Raw:          []byte("postgresql://postgres:23201da=b56ca236f3dc6736c0f9afad@localhost:5434"),
-					RawV2:        []byte("postgresql://postgres:23201da=b56ca236f3dc6736c0f9afad@localhost:5434"),
-					ExtraData:    map[string]string{"sslmode": "allow"},
-				},
-			},
-			wantErr: false,
-		},
-		{
-			name: "found connection URI with requiressl=0, verified",
-			s:    Scanner{detectLoopback: true},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf(`postgresql://%s:%s@%s:%s/postgres2?requiressl=0`, postgresUser, postgresPass, postgresHost, postgresPort)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_Postgres,
-					Verified:     true,
-					Raw:          []byte("postgresql://postgres:23201da=b56ca236f3dc6736c0f9afad@localhost:5434"),
-					RawV2:        []byte("postgresql://postgres:23201da=b56ca236f3dc6736c0f9afad@localhost:5434"),
-					ExtraData:    map[string]string{"sslmode": "prefer"},
-				},
-			},
-			wantErr: false,
-		},
-		{
-			name: "found connection URI without database, verified",
-			s:    Scanner{detectLoopback: true},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf(`postgresql://%s:%s@%s:%s/`, postgresUser, postgresPass, postgresHost, postgresPort)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_Postgres,
-					Verified:     true,
-					Raw:          []byte("postgresql://postgres:23201da=b56ca236f3dc6736c0f9afad@localhost:5434"),
-					RawV2:        []byte("postgresql://postgres:23201da=b56ca236f3dc6736c0f9afad@localhost:5434"),
-					ExtraData:    map[string]string{"sslmode": "<unset>"},
-				},
-			},
-			wantErr: false,
-		},
-		{
-			name: "found connection URI, unverified",
-			s:    Scanner{detectLoopback: true},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf(`postgresql://%s:%s@%s:%s/postgres2`, postgresUser, inactivePass, postgresHost, postgresPort)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_Postgres,
-					Verified:     false,
-					Raw:          []byte("postgresql://postgres:inactive@localhost:5434"),
-					RawV2:        []byte("postgresql://postgres:inactive@localhost:5434"),
-					ExtraData:    map[string]string{"sslmode": "<unset>"},
-				},
-			},
-			wantErr: false,
-		},
-		{
-			name: "ignored localhost",
-			s:    Scanner{},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf(`postgresql://%s:%s@%s:%s/postgres2`, postgresUser, postgresPass, "localhost", postgresPort)),
-				verify: true,
-			},
-			want:    nil,
-			wantErr: false,
-		},
-		{
-			name: "ignored 127.0.0.1",
-			s:    Scanner{},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf(`postgresql://%s:%s@%s:%s/postgres2`, postgresUser, postgresPass, "127.0.0.1", postgresPort)),
-				verify: true,
-			},
-			want:    nil,
-			wantErr: false,
-		},
-		{
-			name: "found connection URI, unverified due to error - inactive host",
-			s:    Scanner{},
-			args: func() args {
-				ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
-				defer cancel()
-				return args{
-					ctx:    ctx,
-					data:   []byte(fmt.Sprintf(`postgresql://%s:%s@%s:%s/postgres2`, postgresUser, postgresPass, inactiveHost, postgresPort)),
-					verify: true,
-				}
-			}(),
-			want: func() []detectors.Result {
-				r := detectors.Result{
-					DetectorType: detectorspb.DetectorType_Postgres,
-					Verified:     false,
-					Raw:          []byte("postgresql://postgres:[email protected]:5434"),
-					RawV2:        []byte("postgresql://postgres:[email protected]:5434"),
-					ExtraData:    map[string]string{"sslmode": "<unset>"},
-				}
-				r.SetVerificationError(errors.New("i/o timeout"))
-				return []detectors.Result{r}
-			}(),
-			wantErr: false,
-		},
-		{
-			name: "found connection URI, unverified due to error - wrong port",
-			s:    Scanner{detectLoopback: true},
-			args: func() args {
-				ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
-				defer cancel()
-				return args{
-					ctx:    ctx,
-					data:   []byte(fmt.Sprintf(`postgresql://%s:%s@%s/postgres2`, postgresUser, postgresPass, postgresHost)),
-					verify: true,
-				}
-			}(),
-			want: func() []detectors.Result {
-				r := detectors.Result{
-					DetectorType: detectorspb.DetectorType_Postgres,
-					Verified:     false,
-					Raw:          []byte("postgresql://postgres:23201da=b56ca236f3dc6736c0f9afad@localhost:5432"),
-					RawV2:        []byte("postgresql://postgres:23201da=b56ca236f3dc6736c0f9afad@localhost:5432"),
-					ExtraData:    map[string]string{"sslmode": "<unset>"},
-				}
-				r.SetVerificationError(errors.New("connection refused"))
-				return []detectors.Result{r}
-			}(),
-			wantErr: false,
-		},
-		{
-			name: "found connection URI, unverified due to error - ssl not supported (using sslmode)",
-			s:    Scanner{detectLoopback: true},
-			args: func() args {
-				ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
-				defer cancel()
-				return args{
-					ctx:    ctx,
-					data:   []byte(fmt.Sprintf(`postgresql://%s:%s@%s:%s/postgres2?sslmode=require`, postgresUser, postgresPass, postgresHost, postgresPort)),
-					verify: true,
-				}
-			}(),
-			want: func() []detectors.Result {
-				r := detectors.Result{
-					DetectorType: detectorspb.DetectorType_Postgres,
-					Verified:     false,
-					Raw:          []byte("postgresql://postgres:23201da=b56ca236f3dc6736c0f9afad@localhost:5434"),
-					RawV2:        []byte("postgresql://postgres:23201da=b56ca236f3dc6736c0f9afad@localhost:5434"),
-					ExtraData:    map[string]string{"sslmode": "require"},
-				}
-				r.SetVerificationError(pq.ErrSSLNotSupported)
-				return []detectors.Result{r}
-			}(),
-			wantErr: false,
-		},
-		{
-			name: "found connection URI, unverified due to error - ssl not supported (using requiressl)",
-			s:    Scanner{detectLoopback: true},
-			args: func() args {
-				ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
-				defer cancel()
-				return args{
-					ctx:    ctx,
-					data:   []byte(fmt.Sprintf(`postgresql://%s:%s@%s:%s/postgres2?requiressl=1`, postgresUser, postgresPass, postgresHost, postgresPort)),
-					verify: true,
-				}
-			}(),
-			want: func() []detectors.Result {
-				r := detectors.Result{
-					DetectorType: detectorspb.DetectorType_Postgres,
-					Verified:     false,
-					Raw:          []byte("postgresql://postgres:23201da=b56ca236f3dc6736c0f9afad@localhost:5434"),
-					RawV2:        []byte("postgresql://postgres:23201da=b56ca236f3dc6736c0f9afad@localhost:5434"),
-					ExtraData:    map[string]string{"sslmode": "require"},
-				}
-				r.SetVerificationError(pq.ErrSSLNotSupported)
-				return []detectors.Result{r}
-			}(),
-			wantErr: false,
 		},
 	}
-	for _, tt := range tests {
+
-		t.Run(tt.name, func(t *testing.T) {
+	for _, test := range tests {
-			got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+		t.Run(test.name, func(t *testing.T) {
-			if (err != nil) != tt.wantErr {
+			matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
-				t.Errorf("postgres.FromData() error = %v, wantErr %v", err, tt.wantErr)
+			if len(matchedDetectors) == 0 {
+				t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
 				return
 			}
-			for i := range got {
-				if len(got[i].Raw) == 0 {
-					t.Fatalf("no raw secret present: \n %+v", got[i])
-				}
-				gotErr := ""
-				if got[i].VerificationError() != nil {
-					gotErr = got[i].VerificationError().Error()
-				}
-				wantErr := ""
-				if tt.want[i].VerificationError() != nil {
-					wantErr = tt.want[i].VerificationError().Error()
-				}
-				if gotErr != wantErr {
-					t.Fatalf("wantVerificationError = %v, verification error = %v", tt.want[i].VerificationError(), got[i].VerificationError())
-				}
-			}
-			ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "verificationError", "AnalysisInfo")
-			if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
-				t.Errorf("Postgres.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
-			}
-		})
-	}
-}
 
-func dockerLogLine(hash string, needle string) chan struct{} {
+			results, err := d.FromData(context.Background(), false, []byte(test.input))
-	ch := make(chan struct{}, 1)
-	go func() {
-		for {
-			out, err := exec.Command("docker", "logs", hash).CombinedOutput()
 			if err != nil {
-				panic(err)
+				t.Errorf("error = %v", err)
-			}
-			if strings.Contains(string(out), needle) {
-				ch <- struct{}{}
 				return
 			}
-			time.Sleep(1 * time.Second)
-		}
-	}()
-	return ch
-}
 
-func startPostgres() error {
+			if len(results) != len(test.want) {
-	cmd := exec.Command(
+				if len(results) == 0 {
-		"docker", "run", "--rm", "-p", postgresPort+":"+defaultPort,
+					t.Errorf("did not receive result")
-		"-e", "POSTGRES_PASSWORD="+postgresPass,
+				} else {
-		"-e", "POSTGRES_USER="+postgresUser,
+					t.Errorf("expected %d results, only received %d", len(test.want), len(results))
-		"-d", "postgres",
-	)
-	fmt.Println(cmd.String())
-	out, err := cmd.Output()
-	if err != nil {
-		return err
-	}
-	postgresDockerHash = string(bytes.TrimSpace(out))
-	select {
-	case <-dockerLogLine(postgresDockerHash, "PostgreSQL init process complete; ready for start up."):
-		return nil
-	case <-time.After(30 * time.Second):
-		stopPostgres()
-		return errors.New("timeout waiting for postgres database to be ready")
-	}
-}
-
-func stopPostgres() {
-	exec.Command("docker", "kill", postgresDockerHash).Run()
-}
-
-func BenchmarkFromData(benchmark *testing.B) {
-	ctx := context.Background()
-	s := Scanner{}
-	for name, data := range detectors.MustGetBenchmarkData() {
-		benchmark.Run(name, func(b *testing.B) {
-			b.ResetTimer()
-			for n := 0; n < b.N; n++ {
-				_, err := s.FromData(ctx, false, data)
-				if err != nil {
-					b.Fatal(err)
 				}
+				return
+			}
+
+			actual := make(map[string]struct{}, len(results))
+			for _, r := range results {
+				if len(r.RawV2) > 0 {
+					actual[string(r.RawV2)] = struct{}{}
+				} else {
+					actual[string(r.Raw)] = struct{}{}
+				}
+			}
+			expected := make(map[string]struct{}, len(test.want))
+			for _, v := range test.want {
+				expected[v] = struct{}{}
+			}
+
+			if diff := cmp.Diff(expected, actual); diff != "" {
+				t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
 			}
 		})
 	}

pkg/detectors/posthog/posthog_integration_test.go

@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package posthog
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/kylelemons/godebug/pretty"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestAppPosthog_FromChunk(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	defer cancel()
+	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+	if err != nil {
+		t.Fatalf("could not get test secrets from GCP: %s", err)
+	}
+	secret := testSecrets.MustGetField("APPPOSTHOG_TOKEN")
+	inactiveSecret := testSecrets.MustGetField("APPPOSTHOG_INACTIVE")
+
+	type args struct {
+		ctx    context.Context
+		data   []byte
+		verify bool
+	}
+	tests := []struct {
+		name    string
+		s       Scanner
+		args    args
+		want    []detectors.Result
+		wantErr bool
+	}{
+		{
+			name: "found, verified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a appposthog secret %s within", secret)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_PosthogApp,
+					Verified:     true,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "found, unverified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a appposthog secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_PosthogApp,
+					Verified:     false,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "not found",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte("You cannot find the secret within"),
+				verify: true,
+			},
+			want:    nil,
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			s := Scanner{}
+			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("AppPosthog.FromData() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			for i := range got {
+				if len(got[i].Raw) == 0 {
+					t.Fatalf("no raw secret present: \n %+v", got[i])
+				}
+				got[i].Raw = nil
+			}
+			if diff := pretty.Compare(got, tt.want); diff != "" {
+				t.Errorf("AppPosthog.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+			}
+		})
+	}
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+	ctx := context.Background()
+	s := Scanner{}
+	for name, data := range detectors.MustGetBenchmarkData() {
+		benchmark.Run(name, func(b *testing.B) {
+			b.ResetTimer()
+			for n := 0; n < b.N; n++ {
+				_, err := s.FromData(ctx, false, data)
+				if err != nil {
+					b.Fatal(err)
+				}
+			}
+		})
+	}
+}

pkg/detectors/posthog/posthog_test.go

@@ -1,119 +1,80 @@
-//go:build detectors
-// +build detectors
-
 package posthog
 
 import (
 	"context"
 	"fmt"
 	"testing"
-	"time"
 
-	"github.com/kylelemons/godebug/pretty"
+	"github.com/google/go-cmp/cmp"
+
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
-
+	"github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 )
 
-func TestAppPosthog_FromChunk(t *testing.T) {
+var (
-	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	validPattern   = "phx_C1rP9fnAtEFJvb0IYCFdeQhar2WdwUFBYHJym1F_Zqr"
-	defer cancel()
+	invalidPattern = "phx_C1rP9fnAtEFJvb0IYCF?eQhar2WdwUFBYHJym1F_Zqr"
-	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+	keyword        = "posthog"
-	if err != nil {
+)
-		t.Fatalf("could not get test secrets from GCP: %s", err)
-	}
-	secret := testSecrets.MustGetField("APPPOSTHOG_TOKEN")
-	inactiveSecret := testSecrets.MustGetField("APPPOSTHOG_INACTIVE")
 
-	type args struct {
+func TestAppPosthog_Pattern(t *testing.T) {
-		ctx    context.Context
+	d := Scanner{}
-		data   []byte
+	ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
-		verify bool
-	}
 	tests := []struct {
-		name    string
+		name  string
-		s       Scanner
+		input string
-		args    args
+		want  []string
-		want    []detectors.Result
-		wantErr bool
 	}{
 		{
-			name: "found, verified",
+			name:  "valid pattern - with keyword posthog",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
-			args: args{
+			want:  []string{validPattern},
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a appposthog secret %s within", secret)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_PosthogApp,
-					Verified:     true,
-				},
-			},
-			wantErr: false,
 		},
 		{
-			name: "found, unverified",
+			name:  "invalid pattern",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
-			args: args{
+			want:  []string{},
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a appposthog secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_PosthogApp,
-					Verified:     false,
-				},
-			},
-			wantErr: false,
-		},
-		{
-			name: "not found",
-			s:    Scanner{},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte("You cannot find the secret within"),
-				verify: true,
-			},
-			want:    nil,
-			wantErr: false,
 		},
 	}
-	for _, tt := range tests {
+
-		t.Run(tt.name, func(t *testing.T) {
+	for _, test := range tests {
-			s := Scanner{}
+		t.Run(test.name, func(t *testing.T) {
-			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
-			if (err != nil) != tt.wantErr {
+			if len(matchedDetectors) == 0 {
-				t.Errorf("AppPosthog.FromData() error = %v, wantErr %v", err, tt.wantErr)
+				t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
 				return
 			}
-			for i := range got {
-				if len(got[i].Raw) == 0 {
-					t.Fatalf("no raw secret present: \n %+v", got[i])
-				}
-				got[i].Raw = nil
-			}
-			if diff := pretty.Compare(got, tt.want); diff != "" {
-				t.Errorf("AppPosthog.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
-			}
-		})
-	}
-}
 
-func BenchmarkFromData(benchmark *testing.B) {
+			results, err := d.FromData(context.Background(), false, []byte(test.input))
-	ctx := context.Background()
+			if err != nil {
-	s := Scanner{}
+				t.Errorf("error = %v", err)
-	for name, data := range detectors.MustGetBenchmarkData() {
+				return
-		benchmark.Run(name, func(b *testing.B) {
+			}
-			b.ResetTimer()
+
-			for n := 0; n < b.N; n++ {
+			if len(results) != len(test.want) {
-				_, err := s.FromData(ctx, false, data)
+				if len(results) == 0 {
-				if err != nil {
+					t.Errorf("did not receive result")
-					b.Fatal(err)
+				} else {
+					t.Errorf("expected %d results, only received %d", len(test.want), len(results))
 				}
+				return
+			}
+
+			actual := make(map[string]struct{}, len(results))
+			for _, r := range results {
+				if len(r.RawV2) > 0 {
+					actual[string(r.RawV2)] = struct{}{}
+				} else {
+					actual[string(r.Raw)] = struct{}{}
+				}
+			}
+			expected := make(map[string]struct{}, len(test.want))
+			for _, v := range test.want {
+				expected[v] = struct{}{}
+			}
+
+			if diff := cmp.Diff(expected, actual); diff != "" {
+				t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
 			}
 		})
 	}

pkg/detectors/postman/postman_integration_test.go

@@ -0,0 +1,158 @@
+//go:build detectors
+// +build detectors
+
+package postman
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestPostman_FromChunk(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	defer cancel()
+	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+	if err != nil {
+		t.Fatalf("could not get test secrets from GCP: %s", err)
+	}
+	secret := testSecrets.MustGetField("POSTMAN_TOKEN")
+	inactiveSecret := testSecrets.MustGetField("POSTMAN_INACTIVE")
+
+	type args struct {
+		ctx    context.Context
+		data   []byte
+		verify bool
+	}
+	tests := []struct {
+		name                string
+		s                   Scanner
+		args                args
+		want                []detectors.Result
+		wantErr             bool
+		wantVerificationErr bool
+	}{
+		{
+			name: "found, verified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a postman secret %s within", secret)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Postman,
+					Verified:     true,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "found, would be verified if not for timeout",
+			s:    Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a postman secret %s within", secret)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Postman,
+					Verified:     false,
+				},
+			},
+			wantErr:             false,
+			wantVerificationErr: true,
+		},
+		{
+			name: "found, verified but unexpected api surface",
+			s:    Scanner{client: common.ConstantResponseHttpClient(404, "")},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a postman secret %s within", secret)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Postman,
+					Verified:     false,
+				},
+			},
+			wantErr:             false,
+			wantVerificationErr: true,
+		},
+		{
+			name: "found, unverified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a postman secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Postman,
+					Verified:     false,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "not found",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte("You cannot find the secret within"),
+				verify: true,
+			},
+			want:    nil,
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("Postman.FromData() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			for i := range got {
+				if len(got[i].Raw) == 0 {
+					t.Fatalf("no raw secret present: \n %+v", got[i])
+				}
+				if (got[i].VerificationError() != nil) != tt.wantVerificationErr {
+					t.Errorf("Postman.FromData() error = %v, wantErr %v", got[i].VerificationError(), tt.wantVerificationErr)
+				}
+			}
+			ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError", "AnalysisInfo")
+			if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
+				t.Errorf("Postman.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+			}
+		})
+	}
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+	ctx := context.Background()
+	s := Scanner{}
+	for name, data := range detectors.MustGetBenchmarkData() {
+		benchmark.Run(name, func(b *testing.B) {
+			b.ResetTimer()
+			for n := 0; n < b.N; n++ {
+				_, err := s.FromData(ctx, false, data)
+				if err != nil {
+					b.Fatal(err)
+				}
+			}
+		})
+	}
+}

pkg/detectors/postman/postman_test.go

@@ -1,157 +1,80 @@
-//go:build detectors
-// +build detectors
-
 package postman
 
 import (
 	"context"
 	"fmt"
 	"testing"
-	"time"
 
 	"github.com/google/go-cmp/cmp"
-	"github.com/google/go-cmp/cmp/cmpopts"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 
-	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
 )
 
-func TestPostman_FromChunk(t *testing.T) {
+var (
-	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	validPattern   = "PMAK-aMUVdak2ohpOeeJQVlb1G9EWB09YVQfbvH6YH8HpPRkXHPBg5qYvp9YGOhI"
-	defer cancel()
+	invalidPattern = "PMAK-aMUVdak2ohpOeeJQVlb1G9EWB09?VQfbvH6YH8HpPRkXHPBg5qYvp9YGOhI"
-	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+	keyword        = "postman"
-	if err != nil {
+)
-		t.Fatalf("could not get test secrets from GCP: %s", err)
-	}
-	secret := testSecrets.MustGetField("POSTMAN_TOKEN")
-	inactiveSecret := testSecrets.MustGetField("POSTMAN_INACTIVE")
 
-	type args struct {
+func TestPostman_Pattern(t *testing.T) {
-		ctx    context.Context
+	d := Scanner{}
-		data   []byte
+	ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
-		verify bool
-	}
 	tests := []struct {
-		name                string
+		name  string
-		s                   Scanner
+		input string
-		args                args
+		want  []string
-		want                []detectors.Result
-		wantErr             bool
-		wantVerificationErr bool
 	}{
 		{
-			name: "found, verified",
+			name:  "valid pattern - with keyword postman",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
-			args: args{
+			want:  []string{validPattern},
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a postman secret %s within", secret)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_Postman,
-					Verified:     true,
-				},
-			},
-			wantErr: false,
 		},
 		{
-			name: "found, would be verified if not for timeout",
+			name:  "invalid pattern",
-			s:    Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
+			input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
-			args: args{
+			want:  []string{},
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a postman secret %s within", secret)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_Postman,
-					Verified:     false,
-				},
-			},
-			wantErr:             false,
-			wantVerificationErr: true,
-		},
-		{
-			name: "found, verified but unexpected api surface",
-			s:    Scanner{client: common.ConstantResponseHttpClient(404, "")},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a postman secret %s within", secret)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_Postman,
-					Verified:     false,
-				},
-			},
-			wantErr:             false,
-			wantVerificationErr: true,
-		},
-		{
-			name: "found, unverified",
-			s:    Scanner{},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a postman secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_Postman,
-					Verified:     false,
-				},
-			},
-			wantErr: false,
-		},
-		{
-			name: "not found",
-			s:    Scanner{},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte("You cannot find the secret within"),
-				verify: true,
-			},
-			want:    nil,
-			wantErr: false,
 		},
 	}
-	for _, tt := range tests {
+
-		t.Run(tt.name, func(t *testing.T) {
+	for _, test := range tests {
-			got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+		t.Run(test.name, func(t *testing.T) {
-			if (err != nil) != tt.wantErr {
+			matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
-				t.Errorf("Postman.FromData() error = %v, wantErr %v", err, tt.wantErr)
+			if len(matchedDetectors) == 0 {
+				t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
 				return
 			}
-			for i := range got {
-				if len(got[i].Raw) == 0 {
-					t.Fatalf("no raw secret present: \n %+v", got[i])
-				}
-				if (got[i].VerificationError() != nil) != tt.wantVerificationErr {
-					t.Errorf("Postman.FromData() error = %v, wantErr %v", got[i].VerificationError(), tt.wantVerificationErr)
-				}
-			}
-			ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError", "AnalysisInfo")
-			if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
-				t.Errorf("Postman.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
-			}
-		})
-	}
-}
 
-func BenchmarkFromData(benchmark *testing.B) {
+			results, err := d.FromData(context.Background(), false, []byte(test.input))
-	ctx := context.Background()
+			if err != nil {
-	s := Scanner{}
+				t.Errorf("error = %v", err)
-	for name, data := range detectors.MustGetBenchmarkData() {
+				return
-		benchmark.Run(name, func(b *testing.B) {
+			}
-			b.ResetTimer()
+
-			for n := 0; n < b.N; n++ {
+			if len(results) != len(test.want) {
-				_, err := s.FromData(ctx, false, data)
+				if len(results) == 0 {
-				if err != nil {
+					t.Errorf("did not receive result")
-					b.Fatal(err)
+				} else {
+					t.Errorf("expected %d results, only received %d", len(test.want), len(results))
 				}
+				return
+			}
+
+			actual := make(map[string]struct{}, len(results))
+			for _, r := range results {
+				if len(r.RawV2) > 0 {
+					actual[string(r.RawV2)] = struct{}{}
+				} else {
+					actual[string(r.Raw)] = struct{}{}
+				}
+			}
+			expected := make(map[string]struct{}, len(test.want))
+			for _, v := range test.want {
+				expected[v] = struct{}{}
+			}
+
+			if diff := cmp.Diff(expected, actual); diff != "" {
+				t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
 			}
 		})
 	}

pkg/detectors/postmark/postmark_integration_test.go

@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package postmark
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/kylelemons/godebug/pretty"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestPostmark_FromChunk(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	defer cancel()
+	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+	if err != nil {
+		t.Fatalf("could not get test secrets from GCP: %s", err)
+	}
+	secret := testSecrets.MustGetField("POSTMARK_TOKEN")
+	inactiveSecret := testSecrets.MustGetField("POSTMARK_INACTIVE")
+
+	type args struct {
+		ctx    context.Context
+		data   []byte
+		verify bool
+	}
+	tests := []struct {
+		name    string
+		s       Scanner
+		args    args
+		want    []detectors.Result
+		wantErr bool
+	}{
+		{
+			name: "found, verified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a postmark secret %s within", secret)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Postmark,
+					Verified:     true,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "found, unverified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a postmark secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Postmark,
+					Verified:     false,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "not found",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte("You cannot find the secret within"),
+				verify: true,
+			},
+			want:    nil,
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			s := Scanner{}
+			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("Postmark.FromData() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			for i := range got {
+				if len(got[i].Raw) == 0 {
+					t.Fatalf("no raw secret present: \n %+v", got[i])
+				}
+				got[i].Raw = nil
+			}
+			if diff := pretty.Compare(got, tt.want); diff != "" {
+				t.Errorf("Postmark.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+			}
+		})
+	}
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+	ctx := context.Background()
+	s := Scanner{}
+	for name, data := range detectors.MustGetBenchmarkData() {
+		benchmark.Run(name, func(b *testing.B) {
+			b.ResetTimer()
+			for n := 0; n < b.N; n++ {
+				_, err := s.FromData(ctx, false, data)
+				if err != nil {
+					b.Fatal(err)
+				}
+			}
+		})
+	}
+}

pkg/detectors/postmark/postmark_test.go

@@ -1,119 +1,90 @@
-//go:build detectors
-// +build detectors
-
 package postmark
 
 import (
 	"context"
 	"fmt"
 	"testing"
-	"time"
 
-	"github.com/kylelemons/godebug/pretty"
+	"github.com/google/go-cmp/cmp"
 
-	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
 )
 
-func TestPostmark_FromChunk(t *testing.T) {
+var (
-	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	validPattern   = "uigi9zkt-7q3u-v1v9-5x2s-91p78wlmsuxv"
-	defer cancel()
+	invalidPattern = "uigi9zkt?7q3u-v1v9-5x2s-91p78wlmsuxv"
-	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+	keyword        = "postmark"
-	if err != nil {
+)
-		t.Fatalf("could not get test secrets from GCP: %s", err)
-	}
-	secret := testSecrets.MustGetField("POSTMARK_TOKEN")
-	inactiveSecret := testSecrets.MustGetField("POSTMARK_INACTIVE")
 
-	type args struct {
+func TestPostmark_Pattern(t *testing.T) {
-		ctx    context.Context
+	d := Scanner{}
-		data   []byte
+	ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
-		verify bool
-	}
 	tests := []struct {
-		name    string
+		name  string
-		s       Scanner
+		input string
-		args    args
+		want  []string
-		want    []detectors.Result
-		wantErr bool
 	}{
 		{
-			name: "found, verified",
+			name:  "valid pattern - with keyword postmark",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
-			args: args{
+			want:  []string{validPattern},
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a postmark secret %s within", secret)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_Postmark,
-					Verified:     true,
-				},
-			},
-			wantErr: false,
 		},
 		{
-			name: "found, unverified",
+			name:  "valid pattern - ignore duplicate",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
-			args: args{
+			want:  []string{validPattern},
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a postmark secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_Postmark,
-					Verified:     false,
-				},
-			},
-			wantErr: false,
 		},
 		{
-			name: "not found",
+			name:  "valid pattern - key out of prefix range",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
-			args: args{
+			want:  []string{},
-				ctx:    context.Background(),
+		},
-				data:   []byte("You cannot find the secret within"),
+		{
-				verify: true,
+			name:  "invalid pattern",
-			},
+			input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
-			want:    nil,
+			want:  []string{},
-			wantErr: false,
 		},
 	}
-	for _, tt := range tests {
+
-		t.Run(tt.name, func(t *testing.T) {
+	for _, test := range tests {
-			s := Scanner{}
+		t.Run(test.name, func(t *testing.T) {
-			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
-			if (err != nil) != tt.wantErr {
+			if len(matchedDetectors) == 0 {
-				t.Errorf("Postmark.FromData() error = %v, wantErr %v", err, tt.wantErr)
+				t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
 				return
 			}
-			for i := range got {
-				if len(got[i].Raw) == 0 {
-					t.Fatalf("no raw secret present: \n %+v", got[i])
-				}
-				got[i].Raw = nil
-			}
-			if diff := pretty.Compare(got, tt.want); diff != "" {
-				t.Errorf("Postmark.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
-			}
-		})
-	}
-}
 
-func BenchmarkFromData(benchmark *testing.B) {
+			results, err := d.FromData(context.Background(), false, []byte(test.input))
-	ctx := context.Background()
+			if err != nil {
-	s := Scanner{}
+				t.Errorf("error = %v", err)
-	for name, data := range detectors.MustGetBenchmarkData() {
+				return
-		benchmark.Run(name, func(b *testing.B) {
+			}
-			b.ResetTimer()
+
-			for n := 0; n < b.N; n++ {
+			if len(results) != len(test.want) {
-				_, err := s.FromData(ctx, false, data)
+				if len(results) == 0 {
-				if err != nil {
+					t.Errorf("did not receive result")
-					b.Fatal(err)
+				} else {
+					t.Errorf("expected %d results, only received %d", len(test.want), len(results))
 				}
+				return
+			}
+
+			actual := make(map[string]struct{}, len(results))
+			for _, r := range results {
+				if len(r.RawV2) > 0 {
+					actual[string(r.RawV2)] = struct{}{}
+				} else {
+					actual[string(r.Raw)] = struct{}{}
+				}
+			}
+			expected := make(map[string]struct{}, len(test.want))
+			for _, v := range test.want {
+				expected[v] = struct{}{}
+			}
+
+			if diff := cmp.Diff(expected, actual); diff != "" {
+				t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
 			}
 		})
 	}

pkg/detectors/powrbot/powrbot_integration_test.go

@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package powrbot
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/kylelemons/godebug/pretty"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestPowrbot_FromChunk(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	defer cancel()
+	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+	if err != nil {
+		t.Fatalf("could not get test secrets from GCP: %s", err)
+	}
+	secret := testSecrets.MustGetField("POWRBOT_TOKEN")
+	inactiveSecret := testSecrets.MustGetField("POWRBOT_INACTIVE")
+
+	type args struct {
+		ctx    context.Context
+		data   []byte
+		verify bool
+	}
+	tests := []struct {
+		name    string
+		s       Scanner
+		args    args
+		want    []detectors.Result
+		wantErr bool
+	}{
+		{
+			name: "found, verified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a powrbot secret %s within", secret)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Powrbot,
+					Verified:     true,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "found, unverified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a powrbot secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Powrbot,
+					Verified:     false,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "not found",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte("You cannot find the secret within"),
+				verify: true,
+			},
+			want:    nil,
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			s := Scanner{}
+			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("Powrbot.FromData() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			for i := range got {
+				if len(got[i].Raw) == 0 {
+					t.Fatalf("no raw secret present: \n %+v", got[i])
+				}
+				got[i].Raw = nil
+			}
+			if diff := pretty.Compare(got, tt.want); diff != "" {
+				t.Errorf("Powrbot.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+			}
+		})
+	}
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+	ctx := context.Background()
+	s := Scanner{}
+	for name, data := range detectors.MustGetBenchmarkData() {
+		benchmark.Run(name, func(b *testing.B) {
+			b.ResetTimer()
+			for n := 0; n < b.N; n++ {
+				_, err := s.FromData(ctx, false, data)
+				if err != nil {
+					b.Fatal(err)
+				}
+			}
+		})
+	}
+}

pkg/detectors/powrbot/powrbot_test.go

@@ -1,119 +1,90 @@
-//go:build detectors
-// +build detectors
-
 package powrbot
 
 import (
 	"context"
 	"fmt"
 	"testing"
-	"time"
 
-	"github.com/kylelemons/godebug/pretty"
+	"github.com/google/go-cmp/cmp"
+
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
-
+	"github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 )
 
-func TestPowrbot_FromChunk(t *testing.T) {
+var (
-	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	validPattern   = "BuvR1k59n67R96aInATbIUbchmLcpLVN10oyqKuB"
-	defer cancel()
+	invalidPattern = "BuvR1k59n67R96a?nATbIUbchmLcpLVN10oyqKuB"
-	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+	keyword        = "powrbot"
-	if err != nil {
+)
-		t.Fatalf("could not get test secrets from GCP: %s", err)
-	}
-	secret := testSecrets.MustGetField("POWRBOT_TOKEN")
-	inactiveSecret := testSecrets.MustGetField("POWRBOT_INACTIVE")
 
-	type args struct {
+func TestPowrbot_Pattern(t *testing.T) {
-		ctx    context.Context
+	d := Scanner{}
-		data   []byte
+	ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
-		verify bool
-	}
 	tests := []struct {
-		name    string
+		name  string
-		s       Scanner
+		input string
-		args    args
+		want  []string
-		want    []detectors.Result
-		wantErr bool
 	}{
 		{
-			name: "found, verified",
+			name:  "valid pattern - with keyword powrbot",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
-			args: args{
+			want:  []string{validPattern},
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a powrbot secret %s within", secret)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_Powrbot,
-					Verified:     true,
-				},
-			},
-			wantErr: false,
 		},
 		{
-			name: "found, unverified",
+			name:  "valid pattern - ignore duplicate",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
-			args: args{
+			want:  []string{validPattern},
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a powrbot secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_Powrbot,
-					Verified:     false,
-				},
-			},
-			wantErr: false,
 		},
 		{
-			name: "not found",
+			name:  "valid pattern - key out of prefix range",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
-			args: args{
+			want:  []string{},
-				ctx:    context.Background(),
+		},
-				data:   []byte("You cannot find the secret within"),
+		{
-				verify: true,
+			name:  "invalid pattern",
-			},
+			input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
-			want:    nil,
+			want:  []string{},
-			wantErr: false,
 		},
 	}
-	for _, tt := range tests {
+
-		t.Run(tt.name, func(t *testing.T) {
+	for _, test := range tests {
-			s := Scanner{}
+		t.Run(test.name, func(t *testing.T) {
-			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
-			if (err != nil) != tt.wantErr {
+			if len(matchedDetectors) == 0 {
-				t.Errorf("Powrbot.FromData() error = %v, wantErr %v", err, tt.wantErr)
+				t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
 				return
 			}
-			for i := range got {
-				if len(got[i].Raw) == 0 {
-					t.Fatalf("no raw secret present: \n %+v", got[i])
-				}
-				got[i].Raw = nil
-			}
-			if diff := pretty.Compare(got, tt.want); diff != "" {
-				t.Errorf("Powrbot.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
-			}
-		})
-	}
-}
 
-func BenchmarkFromData(benchmark *testing.B) {
+			results, err := d.FromData(context.Background(), false, []byte(test.input))
-	ctx := context.Background()
+			if err != nil {
-	s := Scanner{}
+				t.Errorf("error = %v", err)
-	for name, data := range detectors.MustGetBenchmarkData() {
+				return
-		benchmark.Run(name, func(b *testing.B) {
+			}
-			b.ResetTimer()
+
-			for n := 0; n < b.N; n++ {
+			if len(results) != len(test.want) {
-				_, err := s.FromData(ctx, false, data)
+				if len(results) == 0 {
-				if err != nil {
+					t.Errorf("did not receive result")
-					b.Fatal(err)
+				} else {
+					t.Errorf("expected %d results, only received %d", len(test.want), len(results))
 				}
+				return
+			}
+
+			actual := make(map[string]struct{}, len(results))
+			for _, r := range results {
+				if len(r.RawV2) > 0 {
+					actual[string(r.RawV2)] = struct{}{}
+				} else {
+					actual[string(r.Raw)] = struct{}{}
+				}
+			}
+			expected := make(map[string]struct{}, len(test.want))
+			for _, v := range test.want {
+				expected[v] = struct{}{}
+			}
+
+			if diff := cmp.Diff(expected, actual); diff != "" {
+				t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
 			}
 		})
 	}

pkg/detectors/prefect/prefect_integration_test.go

@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package prefect
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/kylelemons/godebug/pretty"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestPrefect_FromChunk(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	defer cancel()
+	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
+	if err != nil {
+		t.Fatalf("could not get test secrets from GCP: %s", err)
+	}
+	secret := testSecrets.MustGetField("PREFECT")
+	inactiveSecret := testSecrets.MustGetField("PREFECT_INACTIVE")
+
+	type args struct {
+		ctx    context.Context
+		data   []byte
+		verify bool
+	}
+	tests := []struct {
+		name    string
+		s       Scanner
+		args    args
+		want    []detectors.Result
+		wantErr bool
+	}{
+		{
+			name: "found, verified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a prefect secret %s within", secret)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Prefect,
+					Verified:     true,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "found, unverified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a prefect secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Prefect,
+					Verified:     false,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "not found",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte("You cannot find the secret within"),
+				verify: true,
+			},
+			want:    nil,
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			s := Scanner{}
+			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("Prefect.FromData() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			for i := range got {
+				if len(got[i].Raw) == 0 {
+					t.Fatalf("no raw secret present: \n %+v", got[i])
+				}
+				got[i].Raw = nil
+			}
+			if diff := pretty.Compare(got, tt.want); diff != "" {
+				t.Errorf("Prefect.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+			}
+		})
+	}
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+	ctx := context.Background()
+	s := Scanner{}
+	for name, data := range detectors.MustGetBenchmarkData() {
+		benchmark.Run(name, func(b *testing.B) {
+			b.ResetTimer()
+			for n := 0; n < b.N; n++ {
+				_, err := s.FromData(ctx, false, data)
+				if err != nil {
+					b.Fatal(err)
+				}
+			}
+		})
+	}
+}

pkg/detectors/prefect/prefect_test.go

@@ -1,119 +1,80 @@
-//go:build detectors
-// +build detectors
-
 package prefect
 
 import (
 	"context"
 	"fmt"
 	"testing"
-	"time"
 
-	"github.com/kylelemons/godebug/pretty"
+	"github.com/google/go-cmp/cmp"
+
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
-
+	"github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 )
 
-func TestPrefect_FromChunk(t *testing.T) {
+var (
-	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	validPattern   = "pnu_v7mBGSR1R1WT0He5KS2w83RZgInZO8Qwalvr"
-	defer cancel()
+	invalidPattern = "pnu_v7mBGSR1R1WT0He5?S2w83RZgInZO8Qwalvr"
-	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
+	keyword        = "prefect"
-	if err != nil {
+)
-		t.Fatalf("could not get test secrets from GCP: %s", err)
-	}
-	secret := testSecrets.MustGetField("PREFECT")
-	inactiveSecret := testSecrets.MustGetField("PREFECT_INACTIVE")
 
-	type args struct {
+func TestPrefect_Pattern(t *testing.T) {
-		ctx    context.Context
+	d := Scanner{}
-		data   []byte
+	ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
-		verify bool
-	}
 	tests := []struct {
-		name    string
+		name  string
-		s       Scanner
+		input string
-		args    args
+		want  []string
-		want    []detectors.Result
-		wantErr bool
 	}{
 		{
-			name: "found, verified",
+			name:  "valid pattern - with keyword prefect",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
-			args: args{
+			want:  []string{validPattern},
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a prefect secret %s within", secret)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_Prefect,
-					Verified:     true,
-				},
-			},
-			wantErr: false,
 		},
 		{
-			name: "found, unverified",
+			name:  "invalid pattern",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
-			args: args{
+			want:  []string{},
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a prefect secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_Prefect,
-					Verified:     false,
-				},
-			},
-			wantErr: false,
-		},
-		{
-			name: "not found",
-			s:    Scanner{},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte("You cannot find the secret within"),
-				verify: true,
-			},
-			want:    nil,
-			wantErr: false,
 		},
 	}
-	for _, tt := range tests {
+
-		t.Run(tt.name, func(t *testing.T) {
+	for _, test := range tests {
-			s := Scanner{}
+		t.Run(test.name, func(t *testing.T) {
-			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
-			if (err != nil) != tt.wantErr {
+			if len(matchedDetectors) == 0 {
-				t.Errorf("Prefect.FromData() error = %v, wantErr %v", err, tt.wantErr)
+				t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
 				return
 			}
-			for i := range got {
-				if len(got[i].Raw) == 0 {
-					t.Fatalf("no raw secret present: \n %+v", got[i])
-				}
-				got[i].Raw = nil
-			}
-			if diff := pretty.Compare(got, tt.want); diff != "" {
-				t.Errorf("Prefect.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
-			}
-		})
-	}
-}
 
-func BenchmarkFromData(benchmark *testing.B) {
+			results, err := d.FromData(context.Background(), false, []byte(test.input))
-	ctx := context.Background()
+			if err != nil {
-	s := Scanner{}
+				t.Errorf("error = %v", err)
-	for name, data := range detectors.MustGetBenchmarkData() {
+				return
-		benchmark.Run(name, func(b *testing.B) {
+			}
-			b.ResetTimer()
+
-			for n := 0; n < b.N; n++ {
+			if len(results) != len(test.want) {
-				_, err := s.FromData(ctx, false, data)
+				if len(results) == 0 {
-				if err != nil {
+					t.Errorf("did not receive result")
-					b.Fatal(err)
+				} else {
+					t.Errorf("expected %d results, only received %d", len(test.want), len(results))
 				}
+				return
+			}
+
+			actual := make(map[string]struct{}, len(results))
+			for _, r := range results {
+				if len(r.RawV2) > 0 {
+					actual[string(r.RawV2)] = struct{}{}
+				} else {
+					actual[string(r.Raw)] = struct{}{}
+				}
+			}
+			expected := make(map[string]struct{}, len(test.want))
+			for _, v := range test.want {
+				expected[v] = struct{}{}
+			}
+
+			if diff := cmp.Diff(expected, actual); diff != "" {
+				t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
 			}
 		})
 	}

pkg/detectors/privacy/privacy_integration_test.go

@@ -0,0 +1,162 @@
+//go:build detectors
+// +build detectors
+
+package privacy
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestPrivacy_FromChunk(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	defer cancel()
+	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
+	if err != nil {
+		t.Fatalf("could not get test secrets from GCP: %s", err)
+	}
+	secret := testSecrets.MustGetField("PRIVACY")
+	inactiveSecret := testSecrets.MustGetField("PRIVACY_INACTIVE")
+
+	type args struct {
+		ctx    context.Context
+		data   []byte
+		verify bool
+	}
+	tests := []struct {
+		name                string
+		s                   Scanner
+		args                args
+		want                []detectors.Result
+		wantErr             bool
+		wantVerificationErr bool
+	}{
+		{
+			name: "found, verified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a privacy secret %s within", secret)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Privacy,
+					Verified:     true,
+				},
+			},
+			wantErr:             false,
+			wantVerificationErr: false,
+		},
+		{
+			name: "found, unverified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a privacy secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Privacy,
+					Verified:     false,
+				},
+			},
+			wantErr:             false,
+			wantVerificationErr: false,
+		},
+		{
+			name: "not found",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte("You cannot find the secret within"),
+				verify: true,
+			},
+			want:                nil,
+			wantErr:             false,
+			wantVerificationErr: false,
+		},
+		{
+			name: "found, would be verified if not for timeout",
+			s:    Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a privacy secret %s within", secret)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Privacy,
+					Verified:     false,
+				},
+			},
+			wantErr:             false,
+			wantVerificationErr: true,
+		},
+		{
+			name: "found, verified but unexpected api surface",
+			s:    Scanner{client: common.ConstantResponseHttpClient(404, "")},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a privacy secret %s within", secret)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Privacy,
+					Verified:     false,
+				},
+			},
+			wantErr:             false,
+			wantVerificationErr: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("Privacy.FromData() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			for i := range got {
+				if len(got[i].Raw) == 0 {
+					t.Fatalf("no raw secret present: \n %+v", got[i])
+				}
+				if (got[i].VerificationError() != nil) != tt.wantVerificationErr {
+					t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())
+				}
+			}
+			ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")
+			if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
+				t.Errorf("Privacy.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+			}
+		})
+	}
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+	ctx := context.Background()
+	s := Scanner{}
+	for name, data := range detectors.MustGetBenchmarkData() {
+		benchmark.Run(name, func(b *testing.B) {
+			b.ResetTimer()
+			for n := 0; n < b.N; n++ {
+				_, err := s.FromData(ctx, false, data)
+				if err != nil {
+					b.Fatal(err)
+				}
+			}
+		})
+	}
+}

pkg/detectors/privacy/privacy_test.go

@@ -1,161 +1,90 @@
-//go:build detectors
-// +build detectors
-
 package privacy
 
 import (
 	"context"
 	"fmt"
 	"testing"
-	"time"
 
 	"github.com/google/go-cmp/cmp"
-	"github.com/google/go-cmp/cmp/cmpopts"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
-
+	"github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 )
 
-func TestPrivacy_FromChunk(t *testing.T) {
+var (
-	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	validPattern   = "sccqe60k-nsak-h1m1-vzq8-vr7u4gxihiwl"
-	defer cancel()
+	invalidPattern = "sccqe60k?nsak-h1m1-vzq8-vr7u4gxihiwl"
-	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
+	keyword        = "privacy"
-	if err != nil {
+)
-		t.Fatalf("could not get test secrets from GCP: %s", err)
-	}
-	secret := testSecrets.MustGetField("PRIVACY")
-	inactiveSecret := testSecrets.MustGetField("PRIVACY_INACTIVE")
 
-	type args struct {
+func TestPrivacy_Pattern(t *testing.T) {
-		ctx    context.Context
+	d := Scanner{}
-		data   []byte
+	ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
-		verify bool
-	}
 	tests := []struct {
-		name                string
+		name  string
-		s                   Scanner
+		input string
-		args                args
+		want  []string
-		want                []detectors.Result
-		wantErr             bool
-		wantVerificationErr bool
 	}{
 		{
-			name: "found, verified",
+			name:  "valid pattern - with keyword privacy",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
-			args: args{
+			want:  []string{validPattern},
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a privacy secret %s within", secret)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_Privacy,
-					Verified:     true,
-				},
-			},
-			wantErr:             false,
-			wantVerificationErr: false,
 		},
 		{
-			name: "found, unverified",
+			name:  "valid pattern - ignore duplicate",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
-			args: args{
+			want:  []string{validPattern},
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a privacy secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_Privacy,
-					Verified:     false,
-				},
-			},
-			wantErr:             false,
-			wantVerificationErr: false,
 		},
 		{
-			name: "not found",
+			name:  "valid pattern - key out of prefix range",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
-			args: args{
+			want:  []string{},
-				ctx:    context.Background(),
-				data:   []byte("You cannot find the secret within"),
-				verify: true,
-			},
-			want:                nil,
-			wantErr:             false,
-			wantVerificationErr: false,
 		},
 		{
-			name: "found, would be verified if not for timeout",
+			name:  "invalid pattern",
-			s:    Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
+			input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
-			args: args{
+			want:  []string{},
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a privacy secret %s within", secret)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_Privacy,
-					Verified:     false,
-				},
-			},
-			wantErr:             false,
-			wantVerificationErr: true,
-		},
-		{
-			name: "found, verified but unexpected api surface",
-			s:    Scanner{client: common.ConstantResponseHttpClient(404, "")},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a privacy secret %s within", secret)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_Privacy,
-					Verified:     false,
-				},
-			},
-			wantErr:             false,
-			wantVerificationErr: true,
 		},
 	}
-	for _, tt := range tests {
+
-		t.Run(tt.name, func(t *testing.T) {
+	for _, test := range tests {
-			got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+		t.Run(test.name, func(t *testing.T) {
-			if (err != nil) != tt.wantErr {
+			matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
-				t.Errorf("Privacy.FromData() error = %v, wantErr %v", err, tt.wantErr)
+			if len(matchedDetectors) == 0 {
+				t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
 				return
 			}
-			for i := range got {
-				if len(got[i].Raw) == 0 {
-					t.Fatalf("no raw secret present: \n %+v", got[i])
-				}
-				if (got[i].VerificationError() != nil) != tt.wantVerificationErr {
-					t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())
-				}
-			}
-			ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")
-			if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
-				t.Errorf("Privacy.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
-			}
-		})
-	}
-}
 
-func BenchmarkFromData(benchmark *testing.B) {
+			results, err := d.FromData(context.Background(), false, []byte(test.input))
-	ctx := context.Background()
+			if err != nil {
-	s := Scanner{}
+				t.Errorf("error = %v", err)
-	for name, data := range detectors.MustGetBenchmarkData() {
+				return
-		benchmark.Run(name, func(b *testing.B) {
+			}
-			b.ResetTimer()
+
-			for n := 0; n < b.N; n++ {
+			if len(results) != len(test.want) {
-				_, err := s.FromData(ctx, false, data)
+				if len(results) == 0 {
-				if err != nil {
+					t.Errorf("did not receive result")
-					b.Fatal(err)
+				} else {
+					t.Errorf("expected %d results, only received %d", len(test.want), len(results))
 				}
+				return
+			}
+
+			actual := make(map[string]struct{}, len(results))
+			for _, r := range results {
+				if len(r.RawV2) > 0 {
+					actual[string(r.RawV2)] = struct{}{}
+				} else {
+					actual[string(r.Raw)] = struct{}{}
+				}
+			}
+			expected := make(map[string]struct{}, len(test.want))
+			for _, v := range test.want {
+				expected[v] = struct{}{}
+			}
+
+			if diff := cmp.Diff(expected, actual); diff != "" {
+				t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
 			}
 		})
 	}

pkg/detectors/privatekey/privatekey_integration_test.go

@@ -0,0 +1,216 @@
+//go:build detectors
+// +build detectors
+
+package privatekey
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"reflect"
+	"testing"
+	"time"
+
+	"github.com/kylelemons/godebug/pretty"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestPrivatekey_FromChunk(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	defer cancel()
+	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4")
+	if err != nil {
+		t.Fatalf("could not get test secrets from GCP: %s", err)
+	}
+	secretTLS := testSecrets.MustGetField("PRIVATEKEY_TLS")
+	secretGitHub := testSecrets.MustGetField("PRIVATEKEY_GITHUB")
+	secretGitHubEncrypted := testSecrets.MustGetField("PRIVATEKEY_GITHUB_ENCRYPTED")
+	secretInactive := testSecrets.MustGetField("PRIVATEKEY_UNVERIFIED")
+
+	type args struct {
+		ctx    context.Context
+		data   []byte
+		verify bool
+	}
+
+	tests := []struct {
+		name    string
+		s       Scanner
+		args    args
+		want    []detectors.Result
+		wantErr bool
+	}{
+		{
+			name: "found, unverified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find privatekey secret %s within", secretInactive)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_PrivateKey,
+					Verified:     false,
+					Redacted:     "-----BEGIN PRIVATE KEY-----\nMIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYw",
+				},
+				{
+					DetectorType: detectorspb.DetectorType_PrivateKey,
+					Verified:     false,
+					Redacted:     "-----BEGIN PRIVATE KEY-----\nMIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgw",
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "found TLS private key, verified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(secretTLS),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_PrivateKey,
+					Verified:     true,
+					Redacted:     "-----BEGIN PRIVATE KEY-----\nMIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgw",
+					ExtraData: map[string]string{
+						"certificate_urls": "https://crt.sh/?q=1e20c40deb44a8539dd3ac3e8c53b72750cb19f9, https://crt.sh/?q=0e9de31fb2ee16465a4d5d93b227d54f870326d1",
+					},
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "found GitHub SSH private key, verified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(secretGitHub),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_PrivateKey,
+					Verified:     true,
+					Redacted:     "-----BEGIN OPENSSH PRIVATE KEY-----\nb3BlbnNzaC1rZXktdjEAAAAABG5v",
+					ExtraData: map[string]string{
+						"github_user": "sirdetectsalot",
+					},
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "found encrypted GitHub SSH private key, verified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(secretGitHubEncrypted),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_PrivateKey,
+					Verified:     true,
+					Redacted:     "-----BEGIN OPENSSH PRIVATE KEY-----\nb3BlbnNzaC1rZXktdjEAAAAACmFl",
+					ExtraData: map[string]string{
+						"github_user":                   "sirdetectsalot",
+						"encrypted":                     "true",
+						"cracked_encryption_passphrase": "true",
+					},
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "not found",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte("You cannot find the secret within"),
+				verify: true,
+			},
+			want:    nil,
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			s := Scanner{IncludeExpired: true}
+			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("PrivatekeyCI.FromData() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if os.Getenv("FORCE_PASS_DIFF") == "true" {
+				return
+			}
+			for i := range got {
+				if len(got[i].Raw) == 0 {
+					t.Fatal("no raw secret present")
+				}
+				got[i].Raw = nil
+			}
+			if diff := pretty.Compare(got, tt.want); diff != "" {
+				t.Errorf("PrivatekeyCI.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+			}
+		})
+	}
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+	ctx := context.Background()
+	s := Scanner{}
+	for name, data := range detectors.MustGetBenchmarkData() {
+		benchmark.Run(name, func(b *testing.B) {
+			b.ResetTimer()
+			for n := 0; n < b.N; n++ {
+				_, err := s.FromData(ctx, false, data)
+				if err != nil {
+					b.Fatal(err)
+				}
+			}
+		})
+	}
+}
+
+func Test_lookupFingerprint(t *testing.T) {
+	tests := []struct {
+		name                      string
+		publicKeyFingerprintInHex string
+		wantFingerprints          bool
+		wantErr                   bool
+		includeExpired            bool
+	}{
+		{
+			name:                      "got some",
+			publicKeyFingerprintInHex: "4c5da06caa1c81df9c8e1abe43bac385de1bda76",
+			wantFingerprints:          true,
+			wantErr:                   false,
+			includeExpired:            true,
+		},
+		{
+			name:                      "got some",
+			publicKeyFingerprintInHex: "none",
+			wantFingerprints:          false,
+			wantErr:                   false,
+			includeExpired:            true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			gotFingerprints, err := lookupFingerprint(context.TODO(), tt.publicKeyFingerprintInHex, tt.includeExpired)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("lookupFingerprint() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if !reflect.DeepEqual(gotFingerprints != nil && len(gotFingerprints.CertificateURLs) > 0, tt.wantFingerprints) {
+				t.Errorf("lookupFingerprint() = %v, want %v", gotFingerprints, tt.wantFingerprints)
+			}
+		})
+	}
+}

pkg/detectors/privatekey/privatekey_test.go

@@ -1,215 +1,97 @@
-//go:build detectors
-// +build detectors
-
 package privatekey
 
 import (
 	"context"
 	"fmt"
-	"os"
-	"reflect"
 	"testing"
-	"time"
 
-	"github.com/kylelemons/godebug/pretty"
+	"github.com/google/go-cmp/cmp"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
 )
 
-func TestPrivatekey_FromChunk(t *testing.T) {
+var (
-	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	validPattern = `-----BEGIN RSA PRIVATE KEY-----
-	defer cancel()
+MIIBOgIBAAJBAKj34GkxFhD90vcNLYLInFEX6Ppy1tPf9Cnzj4p4WGeKLs1Pt8Qu
-	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4")
+KUpRKfFLfRYC9AIKjbJTWit+CqvjWYzvQwECAwEAAQJAIJLixBy2qpFoS4DSmoEm
-	if err != nil {
+o3qGy0t6z09AIJtH+5OeRV1be+N4cDYJKffGzDa88vQENZiRm0GRq6a+HPGQMd2k
-		t.Fatalf("could not get test secrets from GCP: %s", err)
+TQIhAKMSvzIBnni7ot/OSie2TmJLY4SwTQAevXysE2RbFDYdAiEBCUEaRQnMnbp7
-	}
+9mxDXDf6AU0cN/RPBjb9qSHDcWZHGzUCIG2Es59z8ugGrDY+pxLQnwfotadxd+Uy
-	secretTLS := testSecrets.MustGetField("PRIVATEKEY_TLS")
+v/Ow5T0q5gIJAiEAyS4RaI9YG8EWx/2w0T67ZUVAw8eOMB6BIUg0Xcu+3okCIBOs
-	secretGitHub := testSecrets.MustGetField("PRIVATEKEY_GITHUB")
+/5OiPgoTdSy7bcF9IGpSE8ZgGKzgYQVZeN97YE00
-	secretGitHubEncrypted := testSecrets.MustGetField("PRIVATEKEY_GITHUB_ENCRYPTED")
+-----END RSA PRIVATE KEY-----
-	secretInactive := testSecrets.MustGetField("PRIVATEKEY_UNVERIFIED")
+`
-
+	invalidPattern = `-----BEGIN?RSA?PRIVATE?KEY-----
-	type args struct {
+MIIBOgIBAAJBAKj34GkxFhD90vcNLYLInFEX6Ppy1tPf9Cnzj4p4WGeKLs1Pt8Qu
-		ctx    context.Context
+KUpRKfFLfRYC9AIKjbJTWit+CqvjWYzvQwECAwEAAQJAIJLixBy2qpFoS4DSmoEm
-		data   []byte
+o3qGy0t6z09AIJtH+5OeRV1be+N4cDYJKffGzDa88vQENZiRm0GRq6a+HPGQMd2k
-		verify bool
+TQIhAKMSvzIBnni7ot/OSie2TmJLY4SwTQAevXysE2RbFDYdAiEBCUEaRQnMnbp7
-	}
+9mxDXDf6AU0cN/RPBjb9qSHDcWZHGzUCIG2Es59z8ugGrDY+pxLQnwfotadxd+Uy
+v/Ow5T0q5gIJAiEAyS4RaI9YG8EWx/2w0T67ZUVAw8eOMB6BIUg0Xcu+3okCIBOs
+/5OiPgoTdSy7bcF9IGpSE8ZgGKzgYQVZeN97YE00
+-----END RSA PRIVATE KEY-----`
+	keyword = "privatekey"
+)
 
+func TestPrivatekey_Pattern(t *testing.T) {
+	d := Scanner{}
+	ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
 	tests := []struct {
-		name    string
+		name  string
-		s       Scanner
+		input string
-		args    args
+		want  []string
-		want    []detectors.Result
-		wantErr bool
 	}{
 		{
-			name: "found, unverified",
+			name:  "valid pattern - with keyword privatekey",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
-			args: args{
+			want:  []string{validPattern},
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find privatekey secret %s within", secretInactive)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_PrivateKey,
-					Verified:     false,
-					Redacted:     "-----BEGIN PRIVATE KEY-----\nMIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYw",
-				},
-				{
-					DetectorType: detectorspb.DetectorType_PrivateKey,
-					Verified:     false,
-					Redacted:     "-----BEGIN PRIVATE KEY-----\nMIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgw",
-				},
-			},
-			wantErr: false,
 		},
 		{
-			name: "found TLS private key, verified",
+			name:  "invalid pattern",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
-			args: args{
+			want:  []string{},
-				ctx:    context.Background(),
-				data:   []byte(secretTLS),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_PrivateKey,
-					Verified:     true,
-					Redacted:     "-----BEGIN PRIVATE KEY-----\nMIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgw",
-					ExtraData: map[string]string{
-						"certificate_urls": "https://crt.sh/?q=1e20c40deb44a8539dd3ac3e8c53b72750cb19f9, https://crt.sh/?q=0e9de31fb2ee16465a4d5d93b227d54f870326d1",
-					},
-				},
-			},
-			wantErr: false,
-		},
-		{
-			name: "found GitHub SSH private key, verified",
-			s:    Scanner{},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte(secretGitHub),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_PrivateKey,
-					Verified:     true,
-					Redacted:     "-----BEGIN OPENSSH PRIVATE KEY-----\nb3BlbnNzaC1rZXktdjEAAAAABG5v",
-					ExtraData: map[string]string{
-						"github_user": "sirdetectsalot",
-					},
-				},
-			},
-			wantErr: false,
-		},
-		{
-			name: "found encrypted GitHub SSH private key, verified",
-			s:    Scanner{},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte(secretGitHubEncrypted),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_PrivateKey,
-					Verified:     true,
-					Redacted:     "-----BEGIN OPENSSH PRIVATE KEY-----\nb3BlbnNzaC1rZXktdjEAAAAACmFl",
-					ExtraData: map[string]string{
-						"github_user":                   "sirdetectsalot",
-						"encrypted":                     "true",
-						"cracked_encryption_passphrase": "true",
-					},
-				},
-			},
-			wantErr: false,
-		},
-		{
-			name: "not found",
-			s:    Scanner{},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte("You cannot find the secret within"),
-				verify: true,
-			},
-			want:    nil,
-			wantErr: false,
 		},
 	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			s := Scanner{IncludeExpired: true}
-			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
-			if (err != nil) != tt.wantErr {
-				t.Errorf("PrivatekeyCI.FromData() error = %v, wantErr %v", err, tt.wantErr)
-				return
-			}
-			if os.Getenv("FORCE_PASS_DIFF") == "true" {
-				return
-			}
-			for i := range got {
-				if len(got[i].Raw) == 0 {
-					t.Fatal("no raw secret present")
-				}
-				got[i].Raw = nil
-			}
-			if diff := pretty.Compare(got, tt.want); diff != "" {
-				t.Errorf("PrivatekeyCI.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
-			}
-		})
-	}
-}
 
-func BenchmarkFromData(benchmark *testing.B) {
+	for _, test := range tests {
-	ctx := context.Background()
+		t.Run(test.name, func(t *testing.T) {
-	s := Scanner{}
+			matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
-	for name, data := range detectors.MustGetBenchmarkData() {
+			if len(matchedDetectors) == 0 {
-		benchmark.Run(name, func(b *testing.B) {
+				t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
-			b.ResetTimer()
+				return
-			for n := 0; n < b.N; n++ {
+			}
-				_, err := s.FromData(ctx, false, data)
+
-				if err != nil {
+			results, err := d.FromData(context.Background(), false, []byte(test.input))
-					b.Fatal(err)
+			if err != nil {
+				t.Errorf("error = %v", err)
+				return
+			}
+
+			if len(results) != len(test.want) {
+				if len(results) == 0 {
+					t.Errorf("did not receive result")
+				} else {
+					t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+				}
+				return
+			}
+
+			actual := make(map[string]struct{}, len(results))
+			for _, r := range results {
+				if len(r.RawV2) > 0 {
+					actual[string(r.RawV2)] = struct{}{}
+				} else {
+					actual[string(r.Raw)] = struct{}{}
 				}
 			}
-		})
+			expected := make(map[string]struct{}, len(test.want))
-	}
+			for _, v := range test.want {
-}
+				expected[v] = struct{}{}
+			}
 
-func Test_lookupFingerprint(t *testing.T) {
+			if diff := cmp.Diff(expected, actual); diff != "" {
-	tests := []struct {
+				t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
-		name                      string
-		publicKeyFingerprintInHex string
-		wantFingerprints          bool
-		wantErr                   bool
-		includeExpired            bool
-	}{
-		{
-			name:                      "got some",
-			publicKeyFingerprintInHex: "4c5da06caa1c81df9c8e1abe43bac385de1bda76",
-			wantFingerprints:          true,
-			wantErr:                   false,
-			includeExpired:            true,
-		},
-		{
-			name:                      "got some",
-			publicKeyFingerprintInHex: "none",
-			wantFingerprints:          false,
-			wantErr:                   false,
-			includeExpired:            true,
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			gotFingerprints, err := lookupFingerprint(context.TODO(), tt.publicKeyFingerprintInHex, tt.includeExpired)
-			if (err != nil) != tt.wantErr {
-				t.Errorf("lookupFingerprint() error = %v, wantErr %v", err, tt.wantErr)
-				return
-			}
-			if !reflect.DeepEqual(gotFingerprints != nil && len(gotFingerprints.CertificateURLs) > 0, tt.wantFingerprints) {
-				t.Errorf("lookupFingerprint() = %v, want %v", gotFingerprints, tt.wantFingerprints)
 			}
 		})
 	}

pkg/detectors/prodpad/prodpad_integration_test.go

@@ -0,0 +1,117 @@
+package prodpad
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/kylelemons/godebug/pretty"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestProdpad_FromChunk(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	defer cancel()
+	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+	if err != nil {
+		t.Fatalf("could not get test secrets from GCP: %s", err)
+	}
+	secret := testSecrets.MustGetField("PRODPAD")
+	inactiveSecret := testSecrets.MustGetField("PRODPAD_INACTIVE")
+
+	type args struct {
+		ctx    context.Context
+		data   []byte
+		verify bool
+	}
+	tests := []struct {
+		name    string
+		s       Scanner
+		args    args
+		want    []detectors.Result
+		wantErr bool
+	}{
+		{
+			name: "found, verified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a prodpad secret %s within", secret)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Prodpad,
+					Verified:     true,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "found, unverified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a prodpad secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Prodpad,
+					Verified:     false,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "not found",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte("You cannot find the secret within"),
+				verify: true,
+			},
+			want:    nil,
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			s := Scanner{}
+			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("Prodpad.FromData() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			for i := range got {
+				if len(got[i].Raw) == 0 {
+					t.Fatalf("no raw secret present: \n %+v", got[i])
+				}
+				got[i].Raw = nil
+			}
+			if diff := pretty.Compare(got, tt.want); diff != "" {
+				t.Errorf("Prodpad.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+			}
+		})
+	}
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+	ctx := context.Background()
+	s := Scanner{}
+	for name, data := range detectors.MustGetBenchmarkData() {
+		benchmark.Run(name, func(b *testing.B) {
+			b.ResetTimer()
+			for n := 0; n < b.N; n++ {
+				_, err := s.FromData(ctx, false, data)
+				if err != nil {
+					b.Fatal(err)
+				}
+			}
+		})
+	}
+}

pkg/detectors/prodpad/prodpad_test.go

@@ -4,113 +4,87 @@ import (
 	"context"
 	"fmt"
 	"testing"
-	"time"
 
-	"github.com/kylelemons/godebug/pretty"
+	"github.com/google/go-cmp/cmp"
+
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
-
+	"github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 )
 
-func TestProdpad_FromChunk(t *testing.T) {
+var (
-	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	validPattern   = "96d3a2234e30b05b6c0ec74df1943b95ec708d09b0c45779874be404ce004335"
-	defer cancel()
+	invalidPattern = "96d3a2234?30b05b6c0ec74df1943b95ec708d09b0c45779874be404ce004335"
-	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+	keyword        = "prodpad"
-	if err != nil {
+)
-		t.Fatalf("could not get test secrets from GCP: %s", err)
-	}
-	secret := testSecrets.MustGetField("PRODPAD")
-	inactiveSecret := testSecrets.MustGetField("PRODPAD_INACTIVE")
 
-	type args struct {
+func TestProdpad_Pattern(t *testing.T) {
-		ctx    context.Context
+	d := Scanner{}
-		data   []byte
+	ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
-		verify bool
-	}
 	tests := []struct {
-		name    string
+		name  string
-		s       Scanner
+		input string
-		args    args
+		want  []string
-		want    []detectors.Result
-		wantErr bool
 	}{
 		{
-			name: "found, verified",
+			name:  "valid pattern - with keyword prodpad",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
-			args: args{
+			want:  []string{validPattern},
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a prodpad secret %s within", secret)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_Prodpad,
-					Verified:     true,
-				},
-			},
-			wantErr: false,
 		},
 		{
-			name: "found, unverified",
+			name:  "valid pattern - ignore duplicate",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
-			args: args{
+			want:  []string{validPattern},
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a prodpad secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_Prodpad,
-					Verified:     false,
-				},
-			},
-			wantErr: false,
 		},
 		{
-			name: "not found",
+			name:  "valid pattern - key out of prefix range",
-			s:    Scanner{},
+			input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
-			args: args{
+			want:  []string{},
-				ctx:    context.Background(),
+		},
-				data:   []byte("You cannot find the secret within"),
+		{
-				verify: true,
+			name:  "invalid pattern",
-			},
+			input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
-			want:    nil,
+			want:  []string{},
-			wantErr: false,
 		},
 	}
-	for _, tt := range tests {
+
-		t.Run(tt.name, func(t *testing.T) {
+	for _, test := range tests {
-			s := Scanner{}
+		t.Run(test.name, func(t *testing.T) {
-			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
-			if (err != nil) != tt.wantErr {
+			if len(matchedDetectors) == 0 {
-				t.Errorf("Prodpad.FromData() error = %v, wantErr %v", err, tt.wantErr)
+				t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
 				return
 			}
-			for i := range got {
-				if len(got[i].Raw) == 0 {
-					t.Fatalf("no raw secret present: \n %+v", got[i])
-				}
-				got[i].Raw = nil
-			}
-			if diff := pretty.Compare(got, tt.want); diff != "" {
-				t.Errorf("Prodpad.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
-			}
-		})
-	}
-}
 
-func BenchmarkFromData(benchmark *testing.B) {
+			results, err := d.FromData(context.Background(), false, []byte(test.input))
-	ctx := context.Background()
+			if err != nil {
-	s := Scanner{}
+				t.Errorf("error = %v", err)
-	for name, data := range detectors.MustGetBenchmarkData() {
+				return
-		benchmark.Run(name, func(b *testing.B) {
+			}
-			b.ResetTimer()
+
-			for n := 0; n < b.N; n++ {
+			if len(results) != len(test.want) {
-				_, err := s.FromData(ctx, false, data)
+				if len(results) == 0 {
-				if err != nil {
+					t.Errorf("did not receive result")
-					b.Fatal(err)
+				} else {
+					t.Errorf("expected %d results, only received %d", len(test.want), len(results))
 				}
+				return
+			}
+
+			actual := make(map[string]struct{}, len(results))
+			for _, r := range results {
+				if len(r.RawV2) > 0 {
+					actual[string(r.RawV2)] = struct{}{}
+				} else {
+					actual[string(r.Raw)] = struct{}{}
+				}
+			}
+			expected := make(map[string]struct{}, len(test.want))
+			for _, v := range test.want {
+				expected[v] = struct{}{}
+			}
+
+			if diff := cmp.Diff(expected, actual); diff != "" {
+				t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
 			}
 		})
 	}

pkg/detectors/prospectcrm/prospectcrm_integration_test.go

@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package prospectcrm
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/kylelemons/godebug/pretty"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestProspectCRM_FromChunk(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	defer cancel()
+	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
+	if err != nil {
+		t.Fatalf("could not get test secrets from GCP: %s", err)
+	}
+	secret := testSecrets.MustGetField("PROSPECTCRM")
+	inactiveSecret := testSecrets.MustGetField("PROSPECTCRM_INACTIVE")
+
+	type args struct {
+		ctx    context.Context
+		data   []byte
+		verify bool
+	}
+	tests := []struct {
+		name    string
+		s       Scanner
+		args    args
+		want    []detectors.Result
+		wantErr bool
+	}{
+		{
+			name: "found, verified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a prospectcrm secret %s within", secret)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_ProspectCRM,
+					Verified:     true,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "found, unverified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a prospectcrm secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_ProspectCRM,
+					Verified:     false,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "not found",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte("You cannot find the secret within"),
+				verify: true,
+			},
+			want:    nil,
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			s := Scanner{}
+			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("ProspectCRM.FromData() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			for i := range got {
+				if len(got[i].Raw) == 0 {
+					t.Fatalf("no raw secret present: \n %+v", got[i])
+				}
+				got[i].Raw = nil
+			}
+			if diff := pretty.Compare(got, tt.want); diff != "" {
+				t.Errorf("ProspectCRM.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+			}
+		})
+	}
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+	ctx := context.Background()
+	s := Scanner{}
+	for name, data := range detectors.MustGetBenchmarkData() {
+		benchmark.Run(name, func(b *testing.B) {
+			b.ResetTimer()
+			for n := 0; n < b.N; n++ {
+				_, err := s.FromData(ctx, false, data)
+				if err != nil {
+					b.Fatal(err)
+				}
+			}
+		})
+	}
+}