Refactor Netlify Detector (#4102)

* test: added netlify v2 tokens in GCP detectors5

* refactor: refactored netlify detector based on new detector implementations

* refactor: response status handling using switch case

* refactor: PR feedback incorporated.
Unchanged variables moved to const.
ExtraData removed from verifyMatch function.

* task: added AnalysisInfo map in netlify detectors for analyzer

* fix: analyzed required key for analyzer and modified

pkg/detectors/netlify/v1/netlify_v1.go

@@ -3,9 +3,9 @@ package netlify
 import (
 	"context"
 	"fmt"
+	"io"
 	"net/http"
 	"strconv"
-	"strings"
 
 	regexp "github.com/wasilibs/go-re2"
 
@@ -22,10 +22,14 @@ var _ detectors.Versioner = (*Scanner)(nil)
 
 var (
 	client = common.SaneHttpClient()
-
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"netlify"}) + `\b([A-Za-z0-9_-]{43,45})\b`)
 )
 
+const (
+	rotationGuideUrl = "https://howtorotate.com/docs/tutorials/netlify/"
+	verificationUrl  = "https://api.netlify.com/api/v1/sites"
+)
+
 func (Scanner) Version() int { return 1 }
 
 // Keywords are used for efficiently pre-filtering chunks.
@@ -38,32 +42,29 @@ func (s Scanner) Keywords() []string {
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
 	dataStr := string(data)
 
-	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+	uniqueMatches := make(map[string]struct{})
 
-	for _, match := range matches {
-		resMatch := strings.TrimSpace(match[1])
+	for _, match := range keyPat.FindAllStringSubmatch(dataStr, -1) {
+		uniqueMatches[match[1]] = struct{}{}
+	}
 
+	for match := range uniqueMatches {
 		s1 := detectors.Result{
 			DetectorType: detectorspb.DetectorType_Netlify,
-			Raw:          []byte(resMatch),
+			Raw:          []byte(match),
 		}
 		s1.ExtraData = map[string]string{
-			"rotation_guide": "https://howtorotate.com/docs/tutorials/netlify/",
+			"rotation_guide": rotationGuideUrl,
 			"version":        strconv.Itoa(s.Version()),
 		}
 
 		if verify {
-			req, err := http.NewRequestWithContext(ctx, "GET", "https://api.netlify.com/api/v1/sites", nil)
-			if err != nil {
-				continue
-			}
-			req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))
-			res, err := client.Do(req)
-			if err == nil {
-				defer res.Body.Close()
-				if res.StatusCode >= 200 && res.StatusCode < 300 {
-					s1.Verified = true
-				}
+			isVerified, verificationErr := verifyMatch(ctx, client, match)
+			s1.Verified = isVerified
+			s1.SetVerificationError(verificationErr, match)
+
+			if s1.Verified {
+				s1.AnalysisInfo = map[string]string{"key": match}
 			}
 		}
 
@@ -73,6 +74,33 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 	return results, nil
 }
 
+func verifyMatch(ctx context.Context, client *http.Client, token string) (bool, error) {
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, verificationUrl, nil)
+	if err != nil {
+		return false, nil
+	}
+	req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token))
+	res, err := client.Do(req)
+
+	if err != nil {
+		return false, err
+	}
+
+	defer func() {
+		_, _ = io.Copy(io.Discard, res.Body)
+		_ = res.Body.Close()
+	}()
+
+	switch res.StatusCode {
+	case http.StatusOK:
+		return true, nil
+	case http.StatusUnauthorized:
+		return false, nil
+	default:
+		return false, fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
+	}
+}
+
 func (s Scanner) Type() detectorspb.DetectorType {
 	return detectorspb.DetectorType_Netlify
 }

pkg/detectors/netlify/v1/netlify_v1_integration_test.go

@@ -103,6 +103,7 @@ func TestNetlify_FromChunk(t *testing.T) {
 					t.Fatalf("no raw secret present: \n %+v", got[i])
 				}
 				got[i].Raw = nil
+				got[i].AnalysisInfo = nil
 			}
 			if diff := pretty.Compare(got, tt.want); diff != "" {
 				t.Errorf("Netlify.FromData() %s diff: (-got +want)\n%s", tt.name, diff)

pkg/detectors/netlify/v2/netlify_v2.go

@@ -3,9 +3,9 @@ package netlify
 import (
 	"context"
 	"fmt"
+	"io"
 	"net/http"
 	"strconv"
-	"strings"
 
 	regexp "github.com/wasilibs/go-re2"
 
@@ -22,10 +22,14 @@ var _ detectors.Versioner = (*Scanner)(nil)
 
 var (
 	client = common.SaneHttpClient()
-
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"netlify"}) + `\b(nfp_[a-zA-Z0-9_]{36})\b`)
 )
 
+const (
+	rotationGuideUrl = "https://howtorotate.com/docs/tutorials/netlify/"
+	verificationUrl  = "https://api.netlify.com/api/v1/sites"
+)
+
 func (Scanner) Version() int { return 2 }
 
 // Keywords are used for efficiently pre-filtering chunks.
@@ -38,32 +42,29 @@ func (s Scanner) Keywords() []string {
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
 	dataStr := string(data)
 
-	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+	uniqueMatches := make(map[string]struct{})
 
-	for _, match := range matches {
-		resMatch := strings.TrimSpace(match[1])
+	for _, match := range keyPat.FindAllStringSubmatch(dataStr, -1) {
+		uniqueMatches[match[1]] = struct{}{}
+	}
 
+	for match := range uniqueMatches {
 		s1 := detectors.Result{
 			DetectorType: detectorspb.DetectorType_Netlify,
-			Raw:          []byte(resMatch),
+			Raw:          []byte(match),
 		}
 		s1.ExtraData = map[string]string{
-			"rotation_guide": "https://howtorotate.com/docs/tutorials/netlify/",
+			"rotation_guide": rotationGuideUrl,
 			"version":        strconv.Itoa(s.Version()),
 		}
 
 		if verify {
-			req, err := http.NewRequestWithContext(ctx, "GET", "https://api.netlify.com/api/v1/sites", nil)
-			if err != nil {
-				continue
-			}
-			req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))
-			res, err := client.Do(req)
-			if err == nil {
-				defer res.Body.Close()
-				if res.StatusCode >= 200 && res.StatusCode < 300 {
-					s1.Verified = true
-				}
+			isVerified, verificationErr := verifyMatch(ctx, client, match)
+			s1.Verified = isVerified
+			s1.SetVerificationError(verificationErr, match)
+
+			if s1.Verified {
+				s1.AnalysisInfo = map[string]string{"key": match}
 			}
 		}
 
@@ -73,6 +74,33 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 	return results, nil
 }
 
+func verifyMatch(ctx context.Context, client *http.Client, token string) (bool, error) {
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, verificationUrl, nil)
+	if err != nil {
+		return false, nil
+	}
+	req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token))
+	res, err := client.Do(req)
+
+	if err != nil {
+		return false, err
+	}
+
+	defer func() {
+		_, _ = io.Copy(io.Discard, res.Body)
+		_ = res.Body.Close()
+	}()
+
+	switch res.StatusCode {
+	case http.StatusOK:
+		return true, nil
+	case http.StatusUnauthorized:
+		return false, nil
+	default:
+		return false, fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
+	}
+}
+
 func (s Scanner) Type() detectorspb.DetectorType {
 	return detectorspb.DetectorType_Netlify
 }

pkg/detectors/netlify/v2/netlify_v2_integration_test.go

@@ -19,12 +19,12 @@ import (
 func TestNetlify_FromChunk(t *testing.T) {
 	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
 	defer cancel()
-	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
 	if err != nil {
 		t.Fatalf("could not get test secrets from GCP: %s", err)
 	}
-	secret := testSecrets.MustGetField("NETLIFY_TOKEN")
-	inactiveSecret := testSecrets.MustGetField("NETLIFY_INACTIVE")
+	secret := testSecrets.MustGetField("NETLIFY_V2_TOKEN")
+	inactiveSecret := testSecrets.MustGetField("NETLIFY_V2_INACTIVE")
 
 	type args struct {
 		ctx    context.Context
@@ -103,6 +103,7 @@ func TestNetlify_FromChunk(t *testing.T) {
 					t.Fatalf("no raw secret present: \n %+v", got[i])
 				}
 				got[i].Raw = nil
+				got[i].AnalysisInfo = nil
 			}
 			if diff := pretty.Compare(got, tt.want); diff != "" {
 				t.Errorf("Netlify.FromData() %s diff: (-got +want)\n%s", tt.name, diff)