trufflehog/pkg/detectors/detectors.go

package detectors

import (
	"context"
	"crypto/rand"
	"errors"
	"fmt"
	"math/big"
	"net/http"
	"net/url"
	"strings"
	"unicode"

	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb"
	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
)

// Detector defines an interface for scanning for and verifying secrets.
type Detector interface {
	// FromData will scan bytes for results and optionally verify them.
	//
	// FromData can be called concurrently from multiple goroutines.
	// Any modification to the receiver or to global variables will need to use some kind of synchronization.
	FromData(ctx context.Context, verify bool, data []byte) ([]Result, error)

	// Keywords are used for efficiently pre-filtering chunks using substring operations.
	// Use unique identifiers that are part of the secret if you can, or the provider name.
	//
	// When multiple keywords are provided, they are is treated as a *union* of filtering terms.
	// That is, if any of the keywords are found in a chunk, the chunk will be run through the detector.
	Keywords() []string

	// Type returns the DetectorType number from detector_type.proto for the given detector.
	Type() detector_typepb.DetectorType

	// Description returns a description for the result being detected
	Description() string
}

// CustomResultsCleaner is an optional interface that a detector can implement to customize how its generated results
// are "cleaned," which is defined as removing superfluous results from those found in a given chunk. The default
// implementation of this logic removes all unverified results if there are any verified results, and all unverified
// results except for one otherwise, but this interface allows a detector to specify different logic. (This logic must
// be implemented outside results generation because there are circumstances under which the engine should not execute
// it.)
type CustomResultsCleaner interface {
	// CleanResults removes "superfluous" results from a result set (where the definition of "superfluous" is detector-
	// specific).
	CleanResults(results []Result, verificationEnabled bool) []Result
	// ShouldCleanResultsIrrespectiveOfConfiguration allows a custom cleaner to instruct the engine to ignore
	// user-provided configuration that controls whether results are cleaned. (User-provided configuration is not the
	// only factor that determines whether the engine runs cleaning logic.)
	ShouldCleanResultsIrrespectiveOfConfiguration() bool
}

// Versioner is an optional interface that a detector can implement to
// differentiate instances of the same detector type.
type Versioner interface {
	Version() int
}

// MaxSecretSizeProvider is an optional interface that a detector can implement to
// provide a custom max size for the secret it finds.
type MaxSecretSizeProvider interface {
	MaxSecretSize() int64
}

// StartOffsetProvider is an optional interface that a detector can implement to
// provide a custom start offset for the secret it finds.
type StartOffsetProvider interface {
	StartOffset() int64
}

// MultiPartCredentialProvider is an optional interface that a detector can implement
// to indicate its compatibility with multi-part credentials and provide the maximum
// secret size for the credential it finds.
type MultiPartCredentialProvider interface {
	// MaxCredentialSpan returns the maximum span or range of characters that the
	// detector should consider when searching for a multi-part credential.
	MaxCredentialSpan() int64
}

// EndpointCustomizer is an optional interface that a detector can implement to
// support verifying against user-supplied endpoints.
type EndpointCustomizer interface {
	SetConfiguredEndpoints(...string) error
	SetCloudEndpoint(string)
	UseCloudEndpoint(bool)
	UseFoundEndpoints(bool)
}

type CloudProvider interface {
	CloudEndpoint() string
}

type Result struct {
	// DetectorType is the type of Detector.
	DetectorType detector_typepb.DetectorType
	// DetectorName is the name of the Detector. Used for custom detectors.
	DetectorName string
	// Verified indicates whether the result was verified or not.
	Verified bool
	// VerificationFromCache indicates whether this result's verification result came from the verification cache rather
	// than an actual remote request.
	VerificationFromCache bool
	// Raw contains the raw secret identifier data. Prefer IDs over secrets since it is used for deduping after hashing.
	Raw []byte
	// RawV2 contains the raw secret identifier that is a combination of both the ID and the secret.
	// This is used for secrets that are multi part and could have the same ID. Ex: AWS credentials
	RawV2 []byte
	// Redacted contains the redacted version of the raw secret identification data for display purposes.
	// A secret ID should be used if available.
	Redacted       string
	ExtraData      map[string]string
	StructuredData *detectorspb.StructuredData

	// verificationError should be populated if the verification process itself failed in a way that provides no
	// information about the verification status of the candidate secret, such as if the verification request timed out.
	verificationError error

	// SecretParts holds the individual components of a (potentially multi-part)
	// credential, keyed by a component name. It is used by analyzers where
	// the keys are analyzer specific and should match what the
	// corresponding analyzer expects.
	SecretParts map[string]string

	// primarySecret is used when a detector has multiple secret patterns.
	// This secret is designated to determine the line number.
	// If set, the line number will correspond to this secret.
	primarySecret struct {
		Value string
		Line  int64
	}

	// chunkOffset stores the byte position of this result's secret within chunk data.
	// Used to disambiguate line numbers when the same secret appears multiple times.
	chunkOffset    int64
	chunkOffsetSet bool
}

// CopyVerificationInfo clones verification info (status and error) from another Result struct. This is used when
// loading verification info from a verification cache. (A method is necessary because verification errors are not
// exported, to prevent the accidental storage of sensitive information in them.)
func (r *Result) CopyVerificationInfo(from *Result) {
	r.Verified = from.Verified
	r.verificationError = from.verificationError
}

// SetVerificationError is the only way to set a new verification error. Any sensitive values should be passed-in as secrets to be redacted.
func (r *Result) SetVerificationError(err error, secrets ...string) {
	if err != nil {
		r.verificationError = redactSecrets(err, secrets...)
	}
}

// Public accessors for the fields could also be provided if needed.
func (r *Result) VerificationError() error {
	return r.verificationError
}

// SetPrimarySecretValue set the value passed as primary secret in the result
func (r *Result) SetPrimarySecretValue(value string) {
	if value != "" {
		r.primarySecret.Value = value
	}
}

// SetPrimarySecretLine set the passed line number as primary secret line number
func (r *Result) SetPrimarySecretLine(line int64) {
	// line number is only set if value is set for primary secret
	if r.primarySecret.Value != "" {
		r.primarySecret.Line = line
	}
}

// GetPrimarySecretValue return primary secret match value
func (r *Result) GetPrimarySecretValue() string {
	return r.primarySecret.Value
}

// SetChunkOffset records the byte position of this result's secret within the chunk data.
func (r *Result) SetChunkOffset(offset int64) {
	r.chunkOffset = offset
	r.chunkOffsetSet = true
}

// ChunkOffset returns the byte position of this result's secret within the chunk data.
func (r *Result) ChunkOffset() int64 {
	return r.chunkOffset
}

// HasChunkOffset reports whether a chunk offset has been explicitly set on this result.
func (r *Result) HasChunkOffset() bool {
	return r.chunkOffsetSet
}

// redactSecrets replaces all instances of the given secrets with [REDACTED] in the error message.
func redactSecrets(err error, secrets ...string) error {
	lastErr := unwrapToLast(err)
	errStr := lastErr.Error()
	for _, secret := range secrets {
		errStr = strings.ReplaceAll(errStr, secret, "[REDACTED]")
	}
	return errors.New(errStr)
}

// unwrapToLast returns the last error in the chain of errors.
// This is added to exclude non-essential details (like URLs) for brevity and security.
// Also helps us optimize performance in redaction and enhance log clarity.
func unwrapToLast(err error) error {
	for {
		unwrapped := errors.Unwrap(err)
		if unwrapped == nil {
			// We've reached the last error in the chain
			return err
		}
		err = unwrapped
	}
}

type ResultWithMetadata struct {
	// IsWordlistFalsePositive indicates whether this secret was flagged as a false positive based on a wordlist check
	IsWordlistFalsePositive bool
	// SourceMetadata contains source-specific contextual information.
	SourceMetadata *source_metadatapb.MetaData
	// SourceID is the ID of the source that the API uses to map secrets to specific sources.
	SourceID sources.SourceID
	// JobID is the ID of the job that the API uses to map secrets to specific jobs.
	JobID sources.JobID
	// SecretID is the ID of the secret, if it exists.
	// Only secrets that are being reverified will have a SecretID.
	SecretID int64
	// SourceType is the type of Source.
	SourceType sourcespb.SourceType
	// SourceName is the name of the Source.
	SourceName string
	Result
	// DetectorDescription is the description of the Detector.
	DetectorDescription string
	// DecoderType is the type of decoder that was used to generate this result's data.
	DecoderType detectorspb.DecoderType
	// ChunkData holds the original pre-decode source chunk data, preserved
	// for secret storage encryption in the dispatcher.
	ChunkData []byte
}

// CopyMetadata returns a detector result with included metadata from the source chunk.
func CopyMetadata(chunk *sources.Chunk, result Result) ResultWithMetadata {
	// OriginalData may be nil when CopyMetadata is called outside the engine
	// pipeline (e.g., in tests or external consumers that construct chunks directly).
	chunkData := chunk.OriginalData
	if chunkData == nil {
		chunkData = chunk.Data
	}
	return ResultWithMetadata{
		SourceMetadata: chunk.SourceMetadata,
		SourceID:       chunk.SourceID,
		JobID:          chunk.JobID,
		SecretID:       chunk.SecretID,
		SourceType:     chunk.SourceType,
		SourceName:     chunk.SourceName,
		Result:         result,
		ChunkData:      chunkData,
	}
}

// CleanResults returns all verified secrets, and if there are no verified secrets,
// just one unverified secret if there are any.
func CleanResults(results []Result, _ bool) []Result {
	if len(results) == 0 {
		return results
	}

	var cleaned = make(map[string]Result, 0)

	for _, s := range results {
		if s.Verified {
			cleaned[s.Redacted] = s
		}
	}

	if len(cleaned) == 0 {
		return results[:1]
	}

	results = results[:0]
	for _, r := range cleaned {
		results = append(results, r)
	}

	return results
}

// PrefixRegex ensures that at least one of the given keywords is within
// 40 characters of the capturing group that follows.
// This can help prevent false positives.
func PrefixRegex(keywords []string) string {
	pre := `(?i:`
	middle := strings.Join(keywords, "|")
	post := `)(?:.|[\n\r]){0,40}?`
	return pre + middle + post
}

// KeyIsRandom is a Low cost check to make sure that 'keys' include a number to reduce FPs.
// Golang doesn't support regex lookaheads, so must be done in separate calls.
// TODO improve checks. Shannon entropy did not work well.
func KeyIsRandom(key string) bool {
	for _, ch := range key {
		if unicode.IsDigit(ch) {
			return true
		}
	}

	return false
}

func MustGetBenchmarkData() map[string][]byte {
	sizes := map[string]int{
		"xsmall":  10,          // 10 bytes
		"small":   100,         // 100 bytes
		"medium":  1024,        // 1KB
		"large":   10 * 1024,   // 10KB
		"xlarge":  100 * 1024,  // 100KB
		"xxlarge": 1024 * 1024, // 1MB
	}
	data := make(map[string][]byte)

	for key, size := range sizes {
		// Generating a byte slice of a specific size with random data.
		content := make([]byte, size)
		for i := range size {
			randomByte, err := rand.Int(rand.Reader, big.NewInt(256))
			if err != nil {
				panic(err)
			}
			content[i] = byte(randomByte.Int64())
		}
		data[key] = content
	}

	return data
}

func RedactURL(u url.URL) string {
	u.User = url.UserPassword(u.User.Username(), "********")
	return strings.TrimSpace(strings.ReplaceAll(u.String(), "%2A", "*"))
}

func ParseURLAndStripPathAndParams(u string) (*url.URL, error) {
	parsedURL, err := url.Parse(u)
	if err != nil {
		return nil, err
	}
	parsedURL.Path = ""
	parsedURL.RawQuery = ""
	return parsedURL, nil
}

type dedupKeyContextKey struct{}

func withDedupKey(ctx context.Context, detType detector_typepb.DetectorType, credential string) context.Context {
	key := fmt.Sprintf("%d:%s", int32(detType), credential)
	return context.WithValue(ctx, dedupKeyContextKey{}, key)
}

// DoWithDedup executes req through client, coalescing concurrent requests that share
// the same detector type and credential into a single network call via singleflight.
// The response body is fully buffered and replayed to every waiting caller.
//
// Use this instead of client.Do for all verification requests on a client created
// with NewClientWithDedup or WithDedup — it is the only way to activate deduplication.
func DoWithDedup(client *http.Client, detType detector_typepb.DetectorType, credential string, req *http.Request) (*http.Response, error) {
	return client.Do(req.WithContext(withDedupKey(req.Context(), detType, credential)))
}