chore: run unit tests (#3773)

Makefile

@@ -28,19 +28,19 @@ test-failing:
 	CGO_ENABLED=0 go test -timeout=5m $(shell go list ./... | grep -v /vendor/) | grep FAIL
 
 test:
-	CGO_ENABLED=0 go test -timeout=5m $(shell go list ./... | grep -v /vendor/ | grep -v pkg/detectors)
+	CGO_ENABLED=0 go test -timeout=5m $(shell go list ./... | grep -v /vendor/)
 
 test-integration:
-	CGO_ENABLED=0 go test -timeout=5m -tags=integration $(shell go list ./... | grep -v /vendor/ | grep -v pkg/detectors)
+	CGO_ENABLED=0 go test -timeout=5m -tags=integration $(shell go list ./... | grep -v /vendor/)
 
 test-race:
-	CGO_ENABLED=1 go test -timeout=5m -race $(shell go list ./... | grep -v /vendor/ | grep -v pkg/detectors)
+	CGO_ENABLED=1 go test -timeout=5m -race $(shell go list ./... | grep -v /vendor/)
 
 test-detectors:
 	CGO_ENABLED=0 go test -tags=detectors -timeout=5m $(shell go list ./... | grep pkg/detectors)
 
 test-community:
-	CGO_ENABLED=0 go test -timeout=5m $(shell go list ./... | grep -v /vendor/ | grep -v pkg/detectors | grep -v pkg/sources | grep -v pkg/analyzer/analyzers)
+	CGO_ENABLED=0 go test -timeout=5m $(shell go list ./... | grep -v /vendor/ | grep -v pkg/sources | grep -v pkg/analyzer/analyzers)
 
 bench:
 	CGO_ENABLED=0 go test $(shell go list ./pkg/secrets/... | grep -v /vendor/) -benchmem -run=xxx -bench .

pkg/detectors/algoliaadminkey/algoliaadminkey_test.go

@@ -28,7 +28,7 @@ func TestAlgoliaAdminKey_Pattern(t *testing.T) {
 		{
 			name:  "valid pattern",
 			input: fmt.Sprintf("algolia: '%s'", validPattern),
-			want:  []string{"AlgoliaAdminKey010TESTINGPlmiOPCADMIN01KEY"},
+			want:  []string{"ADMIN01KEY:AlgoliaAdminKey010TESTINGPlmiOPC"},
 		},
 		{
 			name:  "valid pattern - key out of prefix range",

pkg/detectors/aws/access_keys/accesskey_test.go

@@ -37,7 +37,7 @@ func TestAWS_Pattern(t *testing.T) {
 		{
 			name:  "valid pattern",
 			input: validPattern,
-			want:  []string{"ABIAS9L8MS5IPHTZPPUQv2QPKHl7LcdVYsjaR4LgQiZ1zw3MAnMyiondXC63"},
+			want:  []string{"ABIAS9L8MS5IPHTZPPUQ:v2QPKHl7LcdVYsjaR4LgQiZ1zw3MAnMyiondXC63"},
 		},
 		{
 			name:  "invalid pattern",

pkg/detectors/aws/session_keys/sessionkeys_test.go

@@ -15,7 +15,7 @@ var (
 		aws credentials{
 			id: ASIABBKK02W42Q3IPSPG
 			secret: fkhIiUwQY32Zu9e4a86g9r3WpTzfE1aXljVcgn8O
-			session: >aSqfp/GTZbJP+=tXPNCZ9GoveoM0vgxtlYXdzPQ2uYNMPPgUkt0VT7SoTLasAo7iVqWWREOUC6DEenlcgDEKyzIEgQ=W5Ju/b9K/Z176uD2HJYCfq/lyowHtt5PvJi7LR=uf/urSorGbTcqNUvP=i42YP1Ps/4F6He9hQA1io3EAGBC3ICGHXWf2IlvFoTNUyPTqhjnPEKMWZ42jblqNA=dD7hLpzNXmmGhdLCjy99XK8+gjHdZHkOeD/FIjRPRZ7Jl0tdwdqFEwzRVCzL2uelMVMd3UaZ+d4I4Kf+J464piO//jxx48Fs/mG3zr5ba9m2S+6gvUZJq4j+0uJ+jf6cG/x2G9XSybqYQRwvxfNquKB4TcKiGVH5+ZbJT4ASkARadwoSPMGfvMPje+X2zAziSzXfsxYfIQKf6iJ9p7VavlDGi+Acr4kwFXW5IfQs4uGk6AVQFsoZK3o1hhLOkuOwWQEWhDQGNLXwJbFqXfELOnUQvM0Z5NUm46bjAAi4g+X9gLPNR/KjzXuuTTaWYrQEjXLb7PxS0sIttAb1w+sTXXtc1kDIsABC6KcsyGlEwji5sLkbkUa~
+			session: aSqfp/GTZbJP+tXPNCZ9GoveoM0vgxtlYXdzPQ2uYNMPPgUkt0VT7SoTLasAo7iVqWWREOUC6DEenlcgDEKyzIEgQW5Ju/b9K/Z176uD2HJYCfq/lyowHtt5PvJi7LRuf/urSorGbTcqNUvPi42YP1Ps/4F6He9hQA1io3EAGBC3ICGHXWf2IlvFoTNUyPTqhjnPEKMWZ42jblqNAdD7hLpzNXmmGhdLCjy99XK8+gjHdZHkOeD/FIjRPRZ7Jl0tdwdqFEwzRVCzL2uelMVMd3UaZ+d4I4Kf+J464piO//jxx48Fs/mG3zr5ba9m2S+6gvUZJq4j+0uJ+jf6cG/x2G9XSybqYQRwvxfNquKB4TcKiGVH5+ZbJT4ASkARadwoSPMGfvMPje+X2zAziSzXfsxYfIQKf6iJ9p7VavlDGi+Acr4kwFXW5IfQs4uGk6AVQFsoZK3o1hhLOkuOwWQEWhDQGNLXwJbFqXfELOnUQvM0Z5NUm46bjAAi4g+X9gLPNR/KjzXuuTTaWYrQEjXLb7PxS0sIttAb1w+sTXXtc1kDIsABC6KcsyGlEwji5sLkbkUa=
 		}
 	`
 	invalidPattern = `
@@ -27,7 +27,7 @@ var (
 )
 
 func TestAWSSessionKey_Pattern(t *testing.T) {
-	d := scanner{}
+	d := New()
 	ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
 
 	tests := []struct {
@@ -38,7 +38,7 @@ func TestAWSSessionKey_Pattern(t *testing.T) {
 		{
 			name:  "valid pattern",
 			input: validPattern,
-			want:  []string{"ASIABBKK02W42Q3IPSPGfkhIiUwQY32Zu9e4a86g9r3WpTzfE1aXljVcgn8OaSqfp/GTZbJP+=tXPNCZ9GoveoM0vgxtlYXdzPQ2uYNMPPgUkt0VT7SoTLasAo7iVqWWREOUC6DEenlcgDEKyzIEgQ=W5Ju/b9K/Z176uD2HJYCfq/lyowHtt5PvJi7LR=uf/urSorGbTcqNUvP=i42YP1Ps/4F6He9hQA1io3EAGBC3ICGHXWf2IlvFoTNUyPTqhjnPEKMWZ42jblqNA=dD7hLpzNXmmGhdLCjy99XK8+gjHdZHkOeD/FIjRPRZ7Jl0tdwdqFEwzRVCzL2uelMVMd3UaZ+d4I4Kf+J464piO//jxx48Fs/mG3zr5ba9m2S+6gvUZJq4j+0uJ+jf6cG/x2G9XSybqYQRwvxfNquKB4TcKiGVH5+ZbJT4ASkARadwoSPMGfvMPje+X2zAziSzXfsxYfIQKf6iJ9p7VavlDGi+Acr4kwFXW5IfQs4uGk6AVQFsoZK3o1hhLOkuOwWQEWhDQGNLXwJbFqXfELOnUQvM0Z5NUm46bjAAi4g+X9gLPNR/KjzXuuTTaWYrQEjXLb7PxS0sIttAb1w+sTXXtc1kDIsABC6KcsyGlEwji5sLkbkUa"},
+			want:  []string{"ASIABBKK02W42Q3IPSPG:fkhIiUwQY32Zu9e4a86g9r3WpTzfE1aXljVcgn8O:aSqfp/GTZbJP+tXPNCZ9GoveoM0vgxtlYXdzPQ2uYNMPPgUkt0VT7SoTLasAo7iVqWWREOUC6DEenlcgDEKyzIEgQW5Ju/b9K/Z176uD2HJYCfq/lyowHtt5PvJi7LRuf/urSorGbTcqNUvPi42YP1Ps/4F6He9hQA1io3EAGBC3ICGHXWf2IlvFoTNUyPTqhjnPEKMWZ42jblqNAdD7hLpzNXmmGhdLCjy99XK8+gjHdZHkOeD/FIjRPRZ7Jl0tdwdqFEwzRVCzL2uelMVMd3UaZ+d4I4Kf+J464piO//jxx48Fs/mG3zr5ba9m2S+6gvUZJq4j+0uJ+jf6cG/x2G9XSybqYQRwvxfNquKB4TcKiGVH5+ZbJT4ASkARadwoSPMGfvMPje+X2zAziSzXfsxYfIQKf6iJ9p7VavlDGi+Acr4kwFXW5IfQs4uGk6AVQFsoZK3o1hhLOkuOwWQEWhDQGNLXwJbFqXfELOnUQvM0Z5NUm46bjAAi4g+X9gLPNR/KjzXuuTTaWYrQEjXLb7PxS0sIttAb1w+sTXXtc1kDIsABC6KcsyGlEwji5sLkbkUa="},
 		},
 		{
 			name:  "invalid pattern",

pkg/detectors/azure_entra/serviceprincipal/v1/spv1_test.go

@@ -1,94 +1,11 @@
 package v1
 
 import (
-	"context"
 	"testing"
 
 	"github.com/google/go-cmp/cmp"
-
-	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
 )
 
-var (
-	validPattern = `
-	azure credentials:
-		azureClientID: clientid9304d5df4-aac1-6117-552c-7f70c89a40d9
-		azureTenant: tenant_idid9304d5df4-aac1-6117-552c-7f70c89a40d9
-		azureClientSecret: clientsecretY_0w|[cGpan41k6ng.ol414sp4ccw2v_rkfmbs537i
-	`
-	invalidPattern = `
-	azure credentials:
-		azureClientID: 9304d5df4-aac1-6117-552c-7f70c89a
-		azureTenant: id9304d5df4-aac1-6117-55-7f70c89a40d9
-		azureClientSecret: Y_0w|[cGpan41k6ng.
-	`
-)
-
-func TestAzure_Pattern(t *testing.T) {
-	d := Scanner{}
-	ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
-
-	tests := []struct {
-		name  string
-		input string
-		want  []string
-	}{
-		{
-			name:  "valid pattern",
-			input: validPattern,
-			want:  []string{"304d5df4-aac1-6117-552c-7f70c89a40d9cGpan41k6ng.ol414sp4ccw2v_rkfmbs53304d5df4-aac1-6117-552c-7f70c89a40d9"},
-		},
-		{
-			name:  "invalid pattern",
-			input: invalidPattern,
-			want:  nil,
-		},
-	}
-
-	for _, test := range tests {
-		t.Run(test.name, func(t *testing.T) {
-			matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
-			if len(matchedDetectors) == 0 {
-				t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
-				return
-			}
-
-			results, err := d.FromData(context.Background(), false, []byte(test.input))
-			if err != nil {
-				t.Errorf("error = %v", err)
-				return
-			}
-
-			if len(results) != len(test.want) {
-				if len(results) == 0 {
-					t.Errorf("did not receive result")
-				} else {
-					t.Errorf("expected %d results, only received %d", len(test.want), len(results))
-				}
-				return
-			}
-
-			actual := make(map[string]struct{}, len(results))
-			for _, r := range results {
-				if len(r.RawV2) > 0 {
-					actual[string(r.RawV2)] = struct{}{}
-				} else {
-					actual[string(r.Raw)] = struct{}{}
-				}
-			}
-			expected := make(map[string]struct{}, len(test.want))
-			for _, v := range test.want {
-				expected[v] = struct{}{}
-			}
-
-			if diff := cmp.Diff(expected, actual); diff != "" {
-				t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
-			}
-		})
-	}
-}
-
 type testCase struct {
 	Input    string
 	Expected map[string]struct{}
@@ -155,10 +72,10 @@ configs = {"fs.azure.account.auth.type": "OAuth"`,
 			Input:    ` "AZUREAD-AKS-APPID-SECRET": "8w__IGsaY.6g6jUxb1.pPGK262._pgX.q-",`,
 			Expected: map[string]struct{}{"8w__IGsaY.6g6jUxb1.pPGK262._pgX.q-": {}},
 		},
-		//"client_secret6": {
+		// "client_secret6": {
 		//	Input:    ``,
 		//	Expected: map[string]struct{}{"": {}},
-		//},
+		// },
 
 		"password": {
 			Input: `# Login using Service Principal
@@ -175,12 +92,12 @@ $Credential = New-Object -TypeName System.Management.Automation.PSCredential -Ar
 az login --service-principal --username {{http://azure-cli-service-principal}} --password {{secret}} --tenant {{someone.onmicrosoft.com}}`,
 			Expected: nil,
 		},
-		//"client_secret3": {
+		// "client_secret3": {
 		//	Input: ``,
 		//	Expected: map[string]struct{}{
 		//		"": {},
 		//	},
-		//},
+		// },
 	}
 
 	for name, test := range cases {

pkg/detectors/gcp/gcp_test.go

@@ -67,7 +67,7 @@ func TestGCP_Pattern(t *testing.T) {
 		{
 			name:  "escaped JSON",
 			input: `{"credentials":"{\n  \"type\": \"service_account\",\n  \"project_id\": \"unit-test\",\n  \"private_key_id\": \"10f922eb17fba903dc59f7baf753976233520012\",\n  \"private_key\": \"-----BEGIN PRIVATE KEY-----\\nMIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQCgyAZHbtJu1MRf\\ng9+Wg==\\n-----END PRIVATE KEY-----\\n\",\n  \"client_email\": \"[email protected]\",\n  \"client_id\": \"123456476766156356779\",\n  \"auth_uri\": \"https://accounts.google.com/o/oauth2/auth\",\n  \"token_uri\": \"https://oauth2.googleapis.com/token\",\n  \"auth_provider_x509_cert_url\": \"https://www.googleapis.com/oauth2/v1/certs\",\n  \"client_x509_cert_url\": \"https://www.googleapis.com/robot/v1/metadata/x509/fake-value%40unit-test.iam.gserviceaccount.com\"\n}\n"}`,
-			want:  []string{"{\"type\":\"service_account\",\"project_id\":\"unit-test\",\"private_key_id\":\"10f922eb17fba903dc59f7baf753976233520012\",\"private_key\":\"-----BEGIN PRIVATE KEY-----\\nMIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQCgyAZHbtJu1MRf\\ng9+Wg==\\n-----END PRIVATE KEY-----\\n\",\"client_email\":\"[email protected]\",\"client_id\":\"123456476766156356779\",\"auth_uri\":\"https://accounts.google.com/o/oauth2/auth\",\"token_uri\":\"https://oauth2.googleapis.com/token\",\"auth_provider_x509_cert_url\":\"https://www.googleapis.com/oauth2/v1/certs\",\"client_x509_cert_url\":\"https://www.googleapis.com/robot/v1/metadata/x509/fake-value%40unit-test.iam.gserviceaccount.com\"}g"},
+			want:  []string{"{\"type\":\"service_account\",\"project_id\":\"unit-test\",\"private_key_id\":\"10f922eb17fba903dc59f7baf753976233520012\",\"private_key\":\"-----BEGIN PRIVATE KEY-----\\nMIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQCgyAZHbtJu1MRf\\ng9+Wg==\\n-----END PRIVATE KEY-----\\n\",\"client_email\":\"[email protected]\",\"client_id\":\"123456476766156356779\",\"auth_uri\":\"https://accounts.google.com/o/oauth2/auth\",\"token_uri\":\"https://oauth2.googleapis.com/token\",\"auth_provider_x509_cert_url\":\"https://www.googleapis.com/oauth2/v1/certs\",\"client_x509_cert_url\":\"https://www.googleapis.com/robot/v1/metadata/x509/fake-value%40unit-test.iam.gserviceaccount.com\"}"},
 		},
 		{
 			name: "no private_key_id (1)",
@@ -103,21 +103,21 @@ GOOGLE_SERVICE_ACCOUNT_PRIVATE_KEY="-----BEGIN PRIVATE KEY-----\nMIIEvAIBADANBgk
 			skip: true,
 		},
 		// TODO: Create an example of these.
-		//{
+		// {
 		//	name:  "Slack mangled email",
 		//	input: ``,
 		//	want:  []string{""},
-		//},
+		// },
-		//{
+		// {
 		//	name:  "Empty client email",
 		//	input: ``,
 		//	want:  []string{""},
-		//},
+		// },
-		//{
+		// {
 		//	name:  "Carets",
 		//	input: ``,
 		//	want:  []string{""},
-		//},
+		// },
 	}
 
 	for _, test := range tests {

pkg/detectors/hubspot_apikey/v1/apikey_test.go

@@ -29,7 +29,7 @@ const hs = new HubSpotAPI({ hapikey: HAPI_KEY });`,
 		//	{
 		//		name: "hubapi",
 		//		input: `curl https://api.hubapi.com/contacts/v1/lists/all/contacts/all \
-		//--header "Authorization: Bearer b71aa2ed-9c76-417d-bd8e-c5f4980d21ef"`,
+		// --header "Authorization: Bearer b71aa2ed-9c76-417d-bd8e-c5f4980d21ef"`,
 		//		want: []string{"b71aa2ed-9c76-417d-bd8e-c5f4980d21ef"},
 		//	},
 		{
@@ -61,7 +61,7 @@ const blogPosts = hs.blog.getPosts({ name: 'Inbound' });`,
 		"method": "GET",
 		"deprecated": false
 	}]`,
-			want: []string{""},
+			want: []string{"hDNxPGyQ-AOMZ-w9Sp-aw5t-TwKLBQjQ85go"},
 		},
 	}
 

pkg/detectors/kanbantool/kanbantool_integration_test.go

@@ -1,3 +1,6 @@
+//go:build detectors
+// +build detectors
+
 package kanbantool
 
 import (

pkg/detectors/lemlist/lemlist_integration_test.go

@@ -1,3 +1,6 @@
+//go:build detectors
+// +build detectors
+
 package lemlist
 
 import (
@@ -7,9 +10,9 @@ import (
 	"time"
 
 	"github.com/kylelemons/godebug/pretty"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 )
 

pkg/detectors/meraki/meraki_integration_test.go

@@ -1,3 +1,6 @@
+//go:build detectors
+// +build detectors
+
 package meraki
 
 import (

pkg/detectors/monkeylearn/monkeylearn_integration_test.go

@@ -1,3 +1,6 @@
+//go:build detectors
+// +build detectors
+
 package monkeylearn
 
 import (
@@ -7,9 +10,9 @@ import (
 	"time"
 
 	"github.com/kylelemons/godebug/pretty"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 )
 

pkg/detectors/opsgenie/opsgenie_integration_test.go

@@ -1,3 +1,6 @@
+//go:build detectors
+// +build detectors
+
 package opsgenie
 
 import (
@@ -7,9 +10,9 @@ import (
 	"time"
 
 	"github.com/kylelemons/godebug/pretty"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 )
 

pkg/detectors/privatekey/ssh_integration_test.go

@@ -1,3 +1,6 @@
+//go:build detectors
+// +build detectors
+
 package privatekey
 
 import (

pkg/detectors/prodpad/prodpad_integration_test.go

@@ -1,3 +1,6 @@
+//go:build detectors
+// +build detectors
+
 package prodpad
 
 import (
@@ -7,9 +10,9 @@ import (
 	"time"
 
 	"github.com/kylelemons/godebug/pretty"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 )
 

pkg/detectors/salesmate/salesmate_integration_test.go

@@ -1,3 +1,6 @@
+//go:build detectors
+// +build detectors
+
 package salesmate
 
 import (

pkg/detectors/scalr/scalr_integration_test.go

@@ -1,3 +1,6 @@
+//go:build detectors
+// +build detectors
+
 package scalr
 
 import (

pkg/detectors/shopify/shopify_integration_test.go

@@ -1,3 +1,6 @@
+//go:build detectors
+// +build detectors
+
 package shopify
 
 import (

pkg/detectors/tokeet/tokeet_integration_test.go

@@ -1,3 +1,6 @@
+//go:build detectors
+// +build detectors
+
 package tokeet
 
 import (
@@ -7,9 +10,9 @@ import (
 	"time"
 
 	"github.com/kylelemons/godebug/pretty"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 )
 

pkg/detectors/transferwise/transferwise_integration_test.go

@@ -1,3 +1,6 @@
+//go:build detectors
+// +build detectors
+
 package transferwise
 
 import (
@@ -7,9 +10,9 @@ import (
 	"time"
 
 	"github.com/kylelemons/godebug/pretty"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 )
 

pkg/detectors/twist/twist_integration_test.go

@@ -1,3 +1,6 @@
+//go:build detectors
+// +build detectors
+
 package twist
 
 import (
@@ -7,9 +10,9 @@ import (
 	"time"
 
 	"github.com/kylelemons/godebug/pretty"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 )
 

pkg/detectors/websitepulse/websitepulse_integration_test.go

@@ -1,3 +1,6 @@
+//go:build detectors
+// +build detectors
+
 package websitepulse
 
 import (