fixed and improved squareapp detector (#3993)

pkg/detectors/squareapp/squareapp.go

@@ -5,7 +5,9 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"io"
 	"net/http"
+	"strings"
 
 	regexp "github.com/wasilibs/go-re2"
 
@@ -14,7 +16,7 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 )
 
-type Scanner struct{
+type Scanner struct {
 	detectors.DefaultMultiPartCredentialProvider
 }
 
@@ -22,10 +24,17 @@ type Scanner struct{
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
-	// possibly always `sq0csp` for secret
+	client = common.SaneHttpClient()
-	// and `sq0idb` for app
+	/*
-	keyPat = regexp.MustCompile(`[\w\-]*sq0i[a-z]{2}-[0-9A-Za-z\-_]{22,43}`)
+		The sandbox id and secret has word `sandbox-` as prefix
-	secPat = regexp.MustCompile(`[\w\-]*sq0c[a-z]{2}-[0-9A-Za-z\-_]{40,50}`)
+		possibly always `sq0csp` for secret and `sq0idb` for app
+	*/
+	keyPat = regexp.MustCompile(`(?:sandbox-)?sq0i[a-z]{2}-[0-9A-Za-z_-]{22,43}`)
+	secPat = regexp.MustCompile(`(?:sandbox-)?sq0c[a-z]{2}-[0-9A-Za-z_-]{40,50}`)
+
+	// api endpoints
+	sandboxEndpoint = "https://connect.squareupsandbox.com/oauth2/revoke"
+	prodEndpoint    = "https://connect.squareup.com/oauth2/revoke"
 )
 
 // Keywords are used for efficiently pre-filtering chunks.
@@ -34,57 +43,6 @@ func (s Scanner) Keywords() []string {
 	return []string{"sq0i"}
 }
 
-// FromData will find and optionally verify SquareApp secrets in a given set of bytes.
-func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	dataStr := string(data)
-
-	matches := keyPat.FindAllString(dataStr, -1)
-	secMatches := secPat.FindAllString(dataStr, -1)
-	for _, match := range matches {
-		for _, secMatch := range secMatches {
-
-			result := detectors.Result{
-				DetectorType: detectorspb.DetectorType_SquareApp,
-				Raw:          []byte(match),
-				Redacted:     match,
-			}
-
-			if verify {
-				baseURL := "https://connect.squareupsandbox.com/oauth2/revoke"
-
-				client := common.SaneHttpClient()
-				reqData, err := json.Marshal(map[string]string{
-					"client_id":    match,
-					"access_token": "fakeTruffleHogAccessTokenForVerification",
-				})
-				if err != nil {
-					return results, err
-				}
-
-				req, err := http.NewRequestWithContext(ctx, "POST", baseURL, bytes.NewReader(reqData))
-				if err != nil {
-					continue
-				}
-				req.Header.Add("Authorization", fmt.Sprintf("Client %s", secMatch))
-				req.Header.Add("Content-Type", "application/json")
-
-				res, err := client.Do(req)
-				if err == nil {
-					res.Body.Close() // The request body is unused.
-
-					// 404 = Correct credentials. The fake access token should not be found.
-					if res.StatusCode == http.StatusNotFound {
-						result.Verified = true
-					}
-				}
-			}
-
-			results = append(results, result)
-		}
-	}
-	return
-}
-
 func (s Scanner) Type() detectorspb.DetectorType {
 	return detectorspb.DetectorType_SquareApp
 }
@@ -92,3 +50,107 @@ func (s Scanner) Type() detectorspb.DetectorType {
 func (s Scanner) Description() string {
 	return "Square is a financial services and mobile payment company. Square credentials can be used to access and manage payment processing and other financial services."
 }
+
+// FromData will find and optionally verify SquareApp secrets in a given set of bytes.
+func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
+	dataStr := string(data)
+
+	var uniqueIDMatches, uniqueSecretMatches = make(map[string]struct{}), make(map[string]struct{})
+
+	for _, match := range keyPat.FindAllString(dataStr, -1) {
+		uniqueIDMatches[match] = struct{}{}
+	}
+
+	for _, match := range secPat.FindAllString(dataStr, -1) {
+		uniqueSecretMatches[match] = struct{}{}
+	}
+
+	for id := range uniqueIDMatches {
+		for secret := range uniqueSecretMatches {
+			// if both are not from same env, continue
+			if !hasSamePrefix(id, secret) {
+				continue
+			}
+
+			result := detectors.Result{
+				DetectorType: detectorspb.DetectorType_SquareApp,
+				Raw:          []byte(id),
+				Redacted:     id,
+				ExtraData:    map[string]string{},
+			}
+
+			var isVerified bool
+			var verificationErr error
+
+			// verify against sandbox endpoint
+			if verify && isSandbox(id) {
+				isVerified, verificationErr = verifySquareApp(ctx, client, sandboxEndpoint, id, secret)
+				result.ExtraData["Env"] = "Sandbox"
+			}
+
+			// verify against prod endpoint
+			if verify && !isSandbox(id) {
+				isVerified, verificationErr = verifySquareApp(ctx, client, prodEndpoint, id, secret)
+				result.ExtraData["Env"] = "Production"
+			}
+
+			result.Verified = isVerified
+			result.SetVerificationError(verificationErr)
+
+			results = append(results, result)
+
+			// once a secret is verified with id, remove it from the list
+			if isVerified {
+				delete(uniqueSecretMatches, secret)
+			}
+		}
+	}
+
+	return results, nil
+}
+
+func verifySquareApp(ctx context.Context, client *http.Client, endpoint, id, secret string) (bool, error) {
+	reqData, err := json.Marshal(map[string]string{
+		"client_id":    id,
+		"access_token": "fakeTruffleHogAccessTokenForVerification",
+	})
+	if err != nil {
+		return false, err
+	}
+
+	req, err := http.NewRequestWithContext(ctx, "POST", endpoint, bytes.NewReader(reqData))
+	if err != nil {
+		return false, err
+	}
+	req.Header.Add("Authorization", fmt.Sprintf("Client %s", secret))
+	req.Header.Add("Content-Type", "application/json")
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return false, err
+	}
+
+	defer func() {
+		_, _ = io.Copy(io.Discard, resp.Body)
+		_ = resp.Body.Close()
+	}()
+
+	switch resp.StatusCode {
+	case http.StatusNotFound:
+		return true, nil
+	default:
+		return false, fmt.Errorf("unexpected status code: %d", resp.StatusCode)
+	}
+}
+
+func hasSamePrefix(id, secret string) bool {
+	idHasPrefix := strings.HasPrefix(id, "sandbox-")
+	secretHasPrefix := strings.HasPrefix(secret, "sandbox-")
+
+	return idHasPrefix == secretHasPrefix
+}
+
+// isSandbox check if provided key(id or secret) is of sandbox env
+func isSandbox(key string) bool {
+	return strings.HasPrefix(key, "sandbox-")
+}

pkg/detectors/squareapp/squareapp_integration_test.go

@@ -19,13 +19,15 @@ import (
 func TestSquareApp_FromChunk(t *testing.T) {
 	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
 	defer cancel()
-	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
+	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
 	if err != nil {
 		t.Fatalf("could not get test secrets from GCP: %s", err)
 	}
+
+	id := testSecrets.MustGetField("SQUAREAPP_ID")
 	secret := testSecrets.MustGetField("SQUAREAPP_SECRET")
 	secretInactive := testSecrets.MustGetField("SQUAREAPP_INACTIVE")
-	id := testSecrets.MustGetField("SQUAREAPP_ID")
+
 	type args struct {
 		ctx    context.Context
 		data   []byte
@@ -48,28 +50,23 @@ func TestSquareApp_FromChunk(t *testing.T) {
 			},
 			want: []detectors.Result{
 				{
-					DetectorType: detectorspb.DetectorType_Square,
+					DetectorType: detectorspb.DetectorType_SquareApp,
 					Verified:     true,
 					Redacted:     id,
+					ExtraData:    map[string]string{"Env": "Sandbox"},
 				},
 			},
 			wantErr: false,
 		},
 		{
-			name: "found, unverified",
+			name: "found, unverified - detected but not added in result due to mismatch of env",
 			s:    Scanner{},
 			args: args{
 				ctx:    context.Background(),
 				data:   []byte(fmt.Sprintf("You can find a squareapp secret %s within awsId %s", secretInactive, id)),
 				verify: true,
 			},
-			want: []detectors.Result{
+			want:    []detectors.Result{},
-				{
-					DetectorType: detectorspb.DetectorType_Square,
-					Verified:     false,
-					Redacted:     id,
-				},
-			},
 			wantErr: false,
 		},
 		{

pkg/detectors/squareapp/squareapp_test.go

@@ -12,11 +12,15 @@ import (
 )
 
 var (
-    validKey   = "YJsq0ige-a9khwVJOSwlzBvX0wp4j8t90s2d"
+	validKey   = "sq0ige-a9khwVJOSwlzBvX0wp4j8t90s2d"
-    invalidKey = "YJsq0ige-a9kh?VJOSwlzBvX0wp4j8t90s2d"
+	invalidKey = "YJsq0ige-a9kh?VJOSwlzBvX0wp4j8t90s2d"
-    validSec   = "4sSPeeM_jk0VZiTFZJqEwzvXHjcCd6fsq0cvn-4pvyBIQ1OvY6dOv4X2AK5r6UJaFf0Xkp5NjV6lGhtbM"
+	validSec   = "4sSPeeM_jk0VZiTFZJqEwzvXHjcCd6fsq0cvn-4pvyBIQ1OvY6dOv4X2AK5r6UJaFf0Xkp5NjV6lGhtbM"
-    invalidSec = "4sSPeeM_jk0VZiTFZJqEwz?XHjcCd6fsq0cvn-4pvyBIQ1OvY6dOv4X2AK5r6UJaFf0Xkp5NjV6lGhtbM"
+	invalidSec = "4sSPeeM_jk0VZiTFZJqEwz?XHjcCd6fsq0cvn-4pvyBIQ1OvY6dOv4X2AK5r6UJaFf0Xkp5NjV6lGhtbM"
-    keyword    = "squareapp"
+
+	// sandbox
+	validSandboxKey    = "sandbox-sq0idb-hFAKEQrhLGgFAKELZEDgpo"
+	validSandboxSecret = "sandbox-sq0csb-o6cs8xFAKExEgIDGbzn2hFAKEZPbzhe713Q-FAKEfbY"
+	keyword            = "squareapp"
 )
 
 func TestSquareApp_Pattern(t *testing.T) {
@@ -32,6 +36,11 @@ func TestSquareApp_Pattern(t *testing.T) {
 			input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, validKey, keyword, validSec),
 			want:  []string{validKey},
 		},
+		{
+			name:  "valid sandbox pattern - with keyword squareapp",
+			input: fmt.Sprintf("token - '%s'\n secret - '%s'\n", validSandboxKey, validSandboxSecret),
+			want:  []string{validSandboxKey},
+		},
 		{
 			name:  "invalid pattern",
 			input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, invalidKey, keyword, invalidSec),