trufflehog/pkg/detectors/uri/uri.go

package uri

import (
	"context"
	"net/http"
	"net/url"
	"strings"
	"time"

	regexp "github.com/wasilibs/go-re2"

	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)

type Scanner struct {
	allowKnownTestSites bool
	client              *http.Client
}

// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var _ detectors.CustomFalsePositiveChecker = (*Scanner)(nil)

var (
	keyPat = regexp.MustCompile(`\b(?:https?:)?\/\/[\S]{3,50}:([\S]{3,50})@[-.%\w\/:]+\b`)

	// TODO: make local addr opt-out
	defaultClient = detectors.DetectorHttpClientWithNoLocalAddresses
)

// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
	return []string{"http"}
}

// FromData will find and optionally verify URI secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
	dataStr := string(data)

	matches := keyPat.FindAllStringSubmatch(dataStr, -1)

	for _, match := range matches {

		if !s.allowKnownTestSites {
			if strings.Contains(match[0], "httpbin.org") {
				continue
			}
			if strings.Contains(match[0], "httpwatch.com") {
				continue
			}
		}

		urlMatch := match[0]
		password := match[1]

		// Skip findings where the password only has "*" characters, this is a redacted password
		// Also include the url encoded "*" characters: "%2A"
		if strings.Trim(password, "*") == "" || strings.Trim(password, "%2A") == "" {
			continue
		}

		parsedURL, err := url.Parse(urlMatch)
		if err != nil {
			continue
		}
		if _, ok := parsedURL.User.Password(); !ok {
			continue
		}

		rawURL, _ := url.Parse(urlMatch)
		rawURLStr := rawURL.String()
		// Removing the path causes possible deduplication issues if some paths have basic auth and some do not.
		rawURL.Path = ""

		s1 := detectors.Result{
			DetectorType: detectorspb.DetectorType_URI,
			Raw:          []byte(rawURL.String()),
			RawV2:        []byte(rawURLStr),
			Redacted:     detectors.RedactURL(*rawURL),
		}

		if verify {
			if s.client == nil {
				s.client = defaultClient
			}
			isVerified, verificationError := verifyURL(ctx, s.client, parsedURL)
			s1.Verified = isVerified
			s1.SetVerificationError(verificationError, password)
		}

		if !s1.Verified {
			// Skip unverified findings where the password starts with a `$` - it's almost certainly a variable.
			if strings.HasPrefix(password, "$") {
				continue
			}
		}

		results = append(results, s1)
	}

	return results, nil
}

func (s Scanner) IsFalsePositive(_ detectors.Result) (bool, string) {
	return false, ""
}

func verifyURL(ctx context.Context, client *http.Client, u *url.URL) (bool, error) {
	// defuse most SSRF payloads
	u.Path = strings.TrimSuffix(u.Path, "/")
	u.RawQuery = ""
	u.Fragment = ""

	credentialedURL := u.String()

	u.User = nil
	nonCredentialedURL := u.String()

	req, err := http.NewRequestWithContext(ctx, "GET", credentialedURL, nil)
	if err != nil {
		return false, err
	}
	req = req.WithContext(ctx)
	credentialedRes, err := client.Do(req)
	if err != nil {
		return false, err
	}
	credentialedRes.Body.Close()

	// If the credentialed URL returns a non 2XX code, we can assume it's a false positive.
	if credentialedRes.StatusCode < 200 || credentialedRes.StatusCode > 299 {
		return false, nil
	}

	time.Sleep(time.Millisecond * 10)

	req, err = http.NewRequestWithContext(ctx, "GET", nonCredentialedURL, nil)
	if err != nil {
		return false, err
	}
	req = req.WithContext(ctx)
	nonCredentialedRes, err := client.Do(req)
	if err != nil {
		return false, err
	}
	nonCredentialedRes.Body.Close()

	// If the non-credentialed URL returns a non 400-428 code and basic auth header, we can assume it's verified now.
	if nonCredentialedRes.StatusCode >= 400 && nonCredentialedRes.StatusCode < 429 {
		if nonCredentialedRes.Header.Get("WWW-Authenticate") != "" {
			return true, nil
		}
	}

	return false, nil
}

func (s Scanner) Type() detectorspb.DetectorType {
	return detectorspb.DetectorType_URI
}

func (s Scanner) Description() string {
	return "This detector identifies URLs with embedded credentials, which can be used to access web resources without explicit user interaction."
}