trufflehog/main.go

package main

import (
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	_ "net/http/pprof"
	"os"
	"os/exec"
	"os/signal"
	"runtime"
	"strconv"
	"strings"
	"sync"
	"syscall"

	"github.com/alecthomas/kingpin/v2"
	"github.com/fatih/color"
	"github.com/felixge/fgprof"
	"github.com/go-logr/logr"
	"github.com/jpillora/overseer"
	"github.com/mattn/go-isatty"
	"go.uber.org/automaxprocs/maxprocs"

	"github.com/trufflesecurity/trufflehog/v3/pkg/analyzer"
	"github.com/trufflesecurity/trufflehog/v3/pkg/cache/simple"
	"github.com/trufflesecurity/trufflehog/v3/pkg/cleantemp"
	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
	"github.com/trufflesecurity/trufflehog/v3/pkg/config"
	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
	"github.com/trufflesecurity/trufflehog/v3/pkg/engine"
	"github.com/trufflesecurity/trufflehog/v3/pkg/engine/defaults"
	"github.com/trufflesecurity/trufflehog/v3/pkg/feature"
	"github.com/trufflesecurity/trufflehog/v3/pkg/handlers"
	"github.com/trufflesecurity/trufflehog/v3/pkg/log"
	"github.com/trufflesecurity/trufflehog/v3/pkg/output"
	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
	"github.com/trufflesecurity/trufflehog/v3/pkg/tui"
	"github.com/trufflesecurity/trufflehog/v3/pkg/updater"
	"github.com/trufflesecurity/trufflehog/v3/pkg/verificationcache"
	"github.com/trufflesecurity/trufflehog/v3/pkg/version"
)

var (
	cli = kingpin.New("TruffleHog", "TruffleHog is a tool for finding credentials.")
	cmd string
	// https://github.com/trufflesecurity/trufflehog/blob/main/CONTRIBUTING.md#logging-in-trufflehog
	logLevel            = cli.Flag("log-level", `Logging verbosity on a scale of 0 (info) to 5 (trace). Can be disabled with "-1".`).Default("0").Int()
	debug               = cli.Flag("debug", "Run in debug mode.").Hidden().Bool()
	trace               = cli.Flag("trace", "Run in trace mode.").Hidden().Bool()
	profile             = cli.Flag("profile", "Enables profiling and sets a pprof and fgprof server on :18066.").Bool()
	localDev            = cli.Flag("local-dev", "Hidden feature to disable overseer for local dev.").Hidden().Bool()
	jsonOut             = cli.Flag("json", "Output in JSON format.").Short('j').Bool()
	jsonLegacy          = cli.Flag("json-legacy", "Use the pre-v3.0 JSON format. Only works with git, gitlab, and github sources.").Bool()
	gitHubActionsFormat = cli.Flag("github-actions", "Output in GitHub Actions format.").Bool()
	concurrency         = cli.Flag("concurrency", "Number of concurrent workers.").Default(strconv.Itoa(runtime.NumCPU())).Int()
	noVerification      = cli.Flag("no-verification", "Don't verify the results.").Bool()
	onlyVerified        = cli.Flag("only-verified", "Only output verified results.").Hidden().Bool()
	results             = cli.Flag("results", "Specifies which type(s) of results to output: verified (confirmed valid by API), unknown (verification failed due to error), unverified (detected but not verified), filtered_unverified (unverified but would have been filtered out). Defaults to verified,unverified,unknown.").String()
	noColor             = cli.Flag("no-color", "Disable colorized output").Bool()
	noColour            = cli.Flag("no-colour", "Alias for --no-color").Hidden().Bool()

	allowVerificationOverlap   = cli.Flag("allow-verification-overlap", "Allow verification of similar credentials across detectors").Bool()
	filterUnverified           = cli.Flag("filter-unverified", "Only output first unverified result per chunk per detector if there are more than one results.").Bool()
	filterEntropy              = cli.Flag("filter-entropy", "Filter unverified results with Shannon entropy. Start with 3.0.").Float64()
	scanEntireChunk            = cli.Flag("scan-entire-chunk", "Scan the entire chunk for secrets.").Hidden().Default("false").Bool()
	compareDetectionStrategies = cli.Flag("compare-detection-strategies", "Compare different detection strategies for matching spans").Hidden().Default("false").Bool()
	configFilename             = cli.Flag("config", "Path to configuration file.").ExistingFile()
	// rules = cli.Flag("rules", "Path to file with custom rules.").String()
	printAvgDetectorTime = cli.Flag("print-avg-detector-time", "Print the average time spent on each detector.").Bool()
	noUpdate             = cli.Flag("no-update", "Don't check for updates.").Bool()
	fail                 = cli.Flag("fail", "Exit with code 183 if results are found.").Bool()
	failOnScanErrors     = cli.Flag("fail-on-scan-errors", "Exit with non-zero error code if an error occurs during the scan.").Bool()
	verifiers            = cli.Flag("verifier", "Set custom verification endpoints.").StringMap()
	customVerifiersOnly  = cli.Flag("custom-verifiers-only", "Only use custom verification endpoints.").Bool()
	detectorTimeout      = cli.Flag("detector-timeout", "Maximum time to spend scanning chunks per detector (e.g., 30s).").Duration()
	archiveMaxSize       = cli.Flag("archive-max-size", "Maximum size of archive to scan. (Byte units eg. 512B, 2KB, 4MB)").Bytes()
	archiveMaxDepth      = cli.Flag("archive-max-depth", "Maximum depth of archive to scan.").Int()
	archiveTimeout       = cli.Flag("archive-timeout", "Maximum time to spend extracting an archive.").Duration()
	includeDetectors     = cli.Flag("include-detectors", "Comma separated list of detector types to include. Protobuf name or IDs may be used, as well as ranges.").Default("all").String()
	excludeDetectors     = cli.Flag("exclude-detectors", "Comma separated list of detector types to exclude. Protobuf name or IDs may be used, as well as ranges. IDs defined here take precedence over the include list.").String()
	jobReportFile        = cli.Flag("output-report", "Write a scan report to the provided path.").Hidden().OpenFile(os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666)

	noVerificationCache = cli.Flag("no-verification-cache", "Disable verification caching").Bool()

	// Add feature flags
	forceSkipBinaries  = cli.Flag("force-skip-binaries", "Force skipping binaries.").Bool()
	forceSkipArchives  = cli.Flag("force-skip-archives", "Force skipping archives.").Bool()
	gitCloneTimeout    = cli.Flag("git-clone-timeout", "Maximum time to spend cloning a repository, as a duration.").Hidden().Duration()
	skipAdditionalRefs = cli.Flag("skip-additional-refs", "Skip additional references.").Bool()
	userAgentSuffix    = cli.Flag("user-agent-suffix", "Suffix to add to User-Agent.").String()

	gitScan                = cli.Command("git", "Find credentials in git repositories.")
	gitScanURI             = gitScan.Arg("uri", "Git repository URL. https://, file://, or ssh:// schema expected.").Required().String()
	gitScanIncludePaths    = gitScan.Flag("include-paths", "Path to file with newline separated regexes for files to include in scan.").Short('i').String()
	gitScanExcludePaths    = gitScan.Flag("exclude-paths", "Path to file with newline separated regexes for files to exclude in scan.").Short('x').String()
	gitScanExcludeGlobs    = gitScan.Flag("exclude-globs", "Comma separated list of globs to exclude in scan. This option filters at the `git log` level, resulting in faster scans.").String()
	gitScanSinceCommit     = gitScan.Flag("since-commit", "Commit to start scan from.").String()
	gitScanBranch          = gitScan.Flag("branch", "Branch to scan.").String()
	gitScanMaxDepth        = gitScan.Flag("max-depth", "Maximum depth of commits to scan.").Int()
	gitScanBare            = gitScan.Flag("bare", "Scan bare repository (e.g. useful while using in pre-receive hooks)").Bool()
	gitClonePath           = gitScan.Flag("clone-path", "Custom path where the repository should be cloned (default: temp dir).").String()
	gitNoCleanup           = gitScan.Flag("no-cleanup", "Do not delete cloned repositories after scanning (can only be used with --clone-path).").Bool()
	gitTrustLocalGitConfig = gitScan.Flag("trust-local-git-config", "Trust local git config.").Bool()
	_                      = gitScan.Flag("allow", "No-op flag for backwards compat.").Bool()
	_                      = gitScan.Flag("entropy", "No-op flag for backwards compat.").Bool()
	_                      = gitScan.Flag("regex", "No-op flag for backwards compat.").Bool()

	githubScan                  = cli.Command("github", "Find credentials in GitHub repositories.")
	githubScanEndpoint          = githubScan.Flag("endpoint", "GitHub endpoint.").Default("https://api.github.com").String()
	githubScanRepos             = githubScan.Flag("repo", `GitHub repository to scan. You can repeat this flag. Example: "https://github.com/dustin-decker/secretsandstuff"`).Strings()
	githubScanOrgs              = githubScan.Flag("org", `GitHub organization to scan. You can repeat this flag. Example: "trufflesecurity"`).Strings()
	githubScanToken             = githubScan.Flag("token", "GitHub token. Can be provided with environment variable GITHUB_TOKEN.").Envar("GITHUB_TOKEN").String()
	githubIncludeForks          = githubScan.Flag("include-forks", "Include forks in scan.").Bool()
	githubIncludeMembers        = githubScan.Flag("include-members", "Include organization member repositories in scan.").Bool()
	githubIncludeRepos          = githubScan.Flag("include-repos", `Repositories to include in an org scan. This can also be a glob pattern. You can repeat this flag. Must use Github repo full name. Example: "trufflesecurity/trufflehog", "trufflesecurity/t*"`).Strings()
	githubIncludeWikis          = githubScan.Flag("include-wikis", "Include repository wikisin scan.").Bool()
	githubExcludeRepos          = githubScan.Flag("exclude-repos", `Repositories to exclude in an org scan. This can also be a glob pattern. You can repeat this flag. Must use Github repo full name. Example: "trufflesecurity/driftwood", "trufflesecurity/d*"`).Strings()
	githubScanIncludePaths      = githubScan.Flag("include-paths", "Path to file with newline separated regexes for files to include in scan.").Short('i').String()
	githubScanExcludePaths      = githubScan.Flag("exclude-paths", "Path to file with newline separated regexes for files to exclude in scan.").Short('x').String()
	githubScanIssueComments     = githubScan.Flag("issue-comments", "Include issue descriptions and comments in scan.").Bool()
	githubScanPRComments        = githubScan.Flag("pr-comments", "Include pull request descriptions and comments in scan.").Bool()
	githubScanGistComments      = githubScan.Flag("gist-comments", "Include gist comments in scan.").Bool()
	githubCommentsTimeframeDays = githubScan.Flag("comments-timeframe", "Number of days in the past to review when scanning issue, PR, and gist comments.").Uint32()
	githubAuthInUrl             = githubScan.Flag("auth-in-url", "Embed authentication credentials in repository URLs instead of using secure HTTP headers").Bool()
	githubClonePath             = githubScan.Flag("clone-path", "Custom path where the repository should be cloned (default: temp dir).").String()
	githubNoCleanup             = githubScan.Flag("no-cleanup", "Do not delete cloned repositories after scanning (can only be used with --clone-path).").Bool()
	githubIgnoreGists           = githubScan.Flag("ignore-gists", "Ignore all gists in scan.").Bool()

	// GitHub Cross Fork Object Reference Experimental Feature
	githubExperimentalScan = cli.Command("github-experimental", "Run an experimental GitHub scan. Must specify at least one experimental sub-module to run: object-discovery.")
	// GitHub Experimental SubModules
	githubExperimentalObjectDiscovery = githubExperimentalScan.Flag("object-discovery", "Discover hidden data objects in GitHub repositories.").Bool()
	// GitHub Experimental Options
	githubExperimentalToken              = githubExperimentalScan.Flag("token", "GitHub token. Can be provided with environment variable GITHUB_TOKEN.").Envar("GITHUB_TOKEN").String()
	githubExperimentalRepo               = githubExperimentalScan.Flag("repo", "GitHub repository to scan. Example: https://github.com/<user>/<repo>.git").Required().String()
	githubExperimentalCollisionThreshold = githubExperimentalScan.Flag("collision-threshold", "Threshold for short-sha collisions in object-discovery submodule. Default is 1.").Default("1").Int()
	githubExperimentalDeleteCache        = githubExperimentalScan.Flag("delete-cached-data", "Delete cached data after object-discovery secret scanning.").Bool()

	gitlabScan = cli.Command("gitlab", "Find credentials in GitLab repositories.")
	// TODO: Add more GitLab options
	gitlabScanEndpoint     = gitlabScan.Flag("endpoint", "GitLab endpoint.").Default("https://gitlab.com").String()
	gitlabScanRepos        = gitlabScan.Flag("repo", "GitLab repo url. You can repeat this flag. Leave empty to scan all repos accessible with provided credential. Example: https://gitlab.com/org/repo.git").Strings()
	gitlabScanToken        = gitlabScan.Flag("token", "GitLab token. Can be provided with environment variable GITLAB_TOKEN.").Envar("GITLAB_TOKEN").Required().String()
	gitlabScanGroupIds     = gitlabScan.Flag("group-id", "GitLab group ID. If provided, it will scan the group and its subgroups. You can repeat this flag.").Strings()
	gitlabScanIncludePaths = gitlabScan.Flag("include-paths", "Path to file with newline separated regexes for files to include in scan.").Short('i').String()
	gitlabScanExcludePaths = gitlabScan.Flag("exclude-paths", "Path to file with newline separated regexes for files to exclude in scan.").Short('x').String()
	gitlabScanIncludeRepos = gitlabScan.Flag("include-repos", `Repositories to include in an org scan. This can also be a glob pattern. You can repeat this flag. Must use Gitlab repo full name. Example: "trufflesecurity/trufflehog", "trufflesecurity/t*"`).Strings()
	gitlabScanExcludeRepos = gitlabScan.Flag("exclude-repos", `Repositories to exclude in an org scan. This can also be a glob pattern. You can repeat this flag. Must use Gitlab repo full name. Example: "trufflesecurity/driftwood", "trufflesecurity/d*"`).Strings()
	gitlabAuthInUrl        = gitlabScan.Flag("auth-in-url", "Embed authentication credentials in repository URLs instead of using secure HTTP headers").Bool()
	gitlabClonePath        = gitlabScan.Flag("clone-path", "Custom path where the repository should be cloned (default: temp dir)").String()
	gitlabNoCleanup        = gitlabScan.Flag("no-cleanup", "Do not delete cloned repositories after scanning (can only be used with --clone-path).").Bool()

	filesystemScan  = cli.Command("filesystem", "Find credentials in a filesystem.")
	filesystemPaths = filesystemScan.Arg("path", "Path to file or directory to scan.").Strings()
	// DEPRECATED: --directory is deprecated in favor of arguments.
	filesystemDirectories = filesystemScan.Flag("directory", "Path to directory to scan. You can repeat this flag.").Strings()
	// TODO: Add more filesystem scan options. Currently only supports scanning a list of directories.
	// filesystemScanRecursive = filesystemScan.Flag("recursive", "Scan recursively.").Short('r').Bool()
	filesystemScanIncludePaths = filesystemScan.Flag("include-paths", "Path to file with newline separated regexes for files to include in scan.").Short('i').String()
	filesystemScanExcludePaths = filesystemScan.Flag("exclude-paths", "Path to file with newline separated regexes for files to exclude in scan.").Short('x').String()

	s3Scan              = cli.Command("s3", "Find credentials in S3 buckets.")
	s3ScanKey           = s3Scan.Flag("key", "S3 key used to authenticate. Can be provided with environment variable AWS_ACCESS_KEY_ID.").Envar("AWS_ACCESS_KEY_ID").String()
	s3ScanRoleArns      = s3Scan.Flag("role-arn", "Specify the ARN of an IAM role to assume for scanning. You can repeat this flag.").Strings()
	s3ScanSecret        = s3Scan.Flag("secret", "S3 secret used to authenticate. Can be provided with environment variable AWS_SECRET_ACCESS_KEY.").Envar("AWS_SECRET_ACCESS_KEY").String()
	s3ScanSessionToken  = s3Scan.Flag("session-token", "S3 session token used to authenticate temporary credentials. Can be provided with environment variable AWS_SESSION_TOKEN.").Envar("AWS_SESSION_TOKEN").String()
	s3ScanCloudEnv      = s3Scan.Flag("cloud-environment", "Use IAM credentials in cloud environment.").Bool()
	s3ScanBuckets       = s3Scan.Flag("bucket", "Name of S3 bucket to scan. You can repeat this flag. Incompatible with --ignore-bucket.").Strings()
	s3ScanIgnoreBuckets = s3Scan.Flag("ignore-bucket", "Name of S3 bucket to ignore. You can repeat this flag. Incompatible with --bucket.").Strings()
	s3ScanMaxObjectSize = s3Scan.Flag("max-object-size", "Maximum size of objects to scan. Objects larger than this will be skipped. (Byte units eg. 512B, 2KB, 4MB)").Default("250MB").Bytes()

	gcsScan           = cli.Command("gcs", "Find credentials in GCS buckets.")
	gcsProjectID      = gcsScan.Flag("project-id", "GCS project ID used to authenticate. Can NOT be used with unauth scan. Can be provided with environment variable GOOGLE_CLOUD_PROJECT.").Envar("GOOGLE_CLOUD_PROJECT").String()
	gcsCloudEnv       = gcsScan.Flag("cloud-environment", "Use Application Default Credentials, IAM credentials to authenticate.").Bool()
	gcsServiceAccount = gcsScan.Flag("service-account", "Path to GCS service account JSON file.").ExistingFile()
	gcsWithoutAuth    = gcsScan.Flag("without-auth", "Scan GCS buckets without authentication. This will only work for public buckets").Bool()
	gcsAPIKey         = gcsScan.Flag("api-key", "GCS API key used to authenticate. Can be provided with environment variable GOOGLE_API_KEY.").Envar("GOOGLE_API_KEY").String()
	gcsIncludeBuckets = gcsScan.Flag("include-buckets", "Buckets to scan. Comma separated list of buckets. You can repeat this flag. Globs are supported").Short('I').Strings()
	gcsExcludeBuckets = gcsScan.Flag("exclude-buckets", "Buckets to exclude from scan. Comma separated list of buckets. Globs are supported").Short('X').Strings()
	gcsIncludeObjects = gcsScan.Flag("include-objects", "Objects to scan. Comma separated list of objects. you can repeat this flag. Globs are supported").Short('i').Strings()
	gcsExcludeObjects = gcsScan.Flag("exclude-objects", "Objects to exclude from scan. Comma separated list of objects. You can repeat this flag. Globs are supported").Short('x').Strings()
	gcsMaxObjectSize  = gcsScan.Flag("max-object-size", "Maximum size of objects to scan. Objects larger than this will be skipped. (Byte units eg. 512B, 2KB, 4MB)").Default("10MB").Bytes()

	syslogScan     = cli.Command("syslog", "Scan syslog")
	syslogAddress  = syslogScan.Flag("address", "Address and port to listen on for syslog. Example: 127.0.0.1:514").String()
	syslogProtocol = syslogScan.Flag("protocol", "Protocol to listen on. udp or tcp").String()
	syslogTLSCert  = syslogScan.Flag("cert", "Path to TLS cert.").String()
	syslogTLSKey   = syslogScan.Flag("key", "Path to TLS key.").String()
	syslogFormat   = syslogScan.Flag("format", "Log format. Can be rfc3164 or rfc5424").Required().String()

	circleCiScan      = cli.Command("circleci", "Scan CircleCI")
	circleCiScanToken = circleCiScan.Flag("token", "CircleCI token. Can also be provided with environment variable").Envar("CIRCLECI_TOKEN").Required().String()

	dockerScan              = cli.Command("docker", "Scan Docker Image")
	dockerScanImages        = dockerScan.Flag("image", "Docker image to scan. Use the file:// prefix to point to a local tarball, the docker:// prefix to point to the docker daemon, otherwise an image registry is assumed.").Strings()
	dockerScanToken         = dockerScan.Flag("token", "Docker bearer token. Can also be provided with environment variable").Envar("DOCKER_TOKEN").String()
	dockerExcludePaths      = dockerScan.Flag("exclude-paths", "Comma separated list of paths to exclude from scan").String()
	dockerScanNamespace     = dockerScan.Flag("namespace", "Docker namespace (organization or user). For non-Docker Hub registries, include the registry address as well (e.g., ghcr.io/namespace or quay.io/namespace).").String()
	dockerScanRegistryToken = dockerScan.Flag("registry-token", "Optional Docker registry access token. Provide this if you want to include private images within the specified namespace.").String()

	travisCiScan      = cli.Command("travisci", "Scan TravisCI")
	travisCiScanToken = travisCiScan.Flag("token", "TravisCI token. Can also be provided with environment variable").Envar("TRAVISCI_TOKEN").Required().String()

	// Postman is hidden for now until we get more feedback from the community.
	postmanScan  = cli.Command("postman", "Scan Postman")
	postmanToken = postmanScan.Flag("token", "Postman token. Can also be provided with environment variable").Envar("POSTMAN_TOKEN").String()

	postmanWorkspaces   = postmanScan.Flag("workspace", "Postman workspace to scan. You can repeat this flag. Deprecated flag.").Hidden().Strings()
	postmanWorkspaceIDs = postmanScan.Flag("workspace-id", "Postman workspace ID to scan. You can repeat this flag.").Strings()

	postmanCollections   = postmanScan.Flag("collection", "Postman collection to scan. You can repeat this flag. Deprecated flag.").Hidden().Strings()
	postmanCollectionIDs = postmanScan.Flag("collection-id", "Postman collection ID to scan. You can repeat this flag.").Strings()

	postmanEnvironments = postmanScan.Flag("environment", "Postman environment to scan. You can repeat this flag.").Strings()

	postmanIncludeCollections   = postmanScan.Flag("include-collections", "Collections to include in scan. You can repeat this flag. Deprecated flag.").Hidden().Strings()
	postmanIncludeCollectionIDs = postmanScan.Flag("include-collection-id", "Collection ID to include in scan. You can repeat this flag.").Strings()

	postmanIncludeEnvironments = postmanScan.Flag("include-environments", "Environments to include in scan. You can repeat this flag.").Strings()

	postmanExcludeCollections   = postmanScan.Flag("exclude-collections", "Collections to exclude from scan. You can repeat this flag. Deprecated flag.").Hidden().Strings()
	postmanExcludeCollectionIDs = postmanScan.Flag("exclude-collection-id", "Collection ID to exclude from scan. You can repeat this flag.").Strings()

	postmanExcludeEnvironments = postmanScan.Flag("exclude-environments", "Environments to exclude from scan. You can repeat this flag.").Strings()
	postmanWorkspacePaths      = postmanScan.Flag("workspace-paths", "Path to Postman workspaces.").Strings()
	postmanCollectionPaths     = postmanScan.Flag("collection-paths", "Path to Postman collections.").Strings()
	postmanEnvironmentPaths    = postmanScan.Flag("environment-paths", "Path to Postman environments.").Strings()

	elasticsearchScan           = cli.Command("elasticsearch", "Scan Elasticsearch")
	elasticsearchNodes          = elasticsearchScan.Flag("nodes", "Elasticsearch nodes").Envar("ELASTICSEARCH_NODES").Strings()
	elasticsearchUsername       = elasticsearchScan.Flag("username", "Elasticsearch username").Envar("ELASTICSEARCH_USERNAME").String()
	elasticsearchPassword       = elasticsearchScan.Flag("password", "Elasticsearch password").Envar("ELASTICSEARCH_PASSWORD").String()
	elasticsearchServiceToken   = elasticsearchScan.Flag("service-token", "Elasticsearch service token").Envar("ELASTICSEARCH_SERVICE_TOKEN").String()
	elasticsearchCloudId        = elasticsearchScan.Flag("cloud-id", "Elasticsearch cloud ID. Can also be provided with environment variable").Envar("ELASTICSEARCH_CLOUD_ID").String()
	elasticsearchAPIKey         = elasticsearchScan.Flag("api-key", "Elasticsearch API key. Can also be provided with environment variable").Envar("ELASTICSEARCH_API_KEY").String()
	elasticsearchIndexPattern   = elasticsearchScan.Flag("index-pattern", "Filters the indices to search").Default("*").Envar("ELASTICSEARCH_INDEX_PATTERN").String()
	elasticsearchQueryJSON      = elasticsearchScan.Flag("query-json", "Filters the documents to search").Envar("ELASTICSEARCH_QUERY_JSON").String()
	elasticsearchSinceTimestamp = elasticsearchScan.Flag("since-timestamp", "Filters the documents to search to those created since this timestamp; overrides any timestamp from --query-json").Envar("ELASTICSEARCH_SINCE_TIMESTAMP").String()
	elasticsearchBestEffortScan = elasticsearchScan.Flag("best-effort-scan", "Attempts to continuously scan a cluster").Envar("ELASTICSEARCH_BEST_EFFORT_SCAN").Bool()

	jenkinsScan                  = cli.Command("jenkins", "Scan Jenkins")
	jenkinsURL                   = jenkinsScan.Flag("url", "Jenkins URL").Envar("JENKINS_URL").Required().String()
	jenkinsUsername              = jenkinsScan.Flag("username", "Jenkins username").Envar("JENKINS_USERNAME").String()
	jenkinsPassword              = jenkinsScan.Flag("password", "Jenkins password").Envar("JENKINS_PASSWORD").String()
	jenkinsInsecureSkipVerifyTLS = jenkinsScan.Flag("insecure-skip-verify-tls", "Skip TLS verification").Envar("JENKINS_INSECURE_SKIP_VERIFY_TLS").Bool()

	huggingfaceScan     = cli.Command("huggingface", "Find credentials in HuggingFace datasets, models and spaces.")
	huggingfaceEndpoint = huggingfaceScan.Flag("endpoint", "HuggingFace endpoint.").Default("https://huggingface.co").String()
	huggingfaceModels   = huggingfaceScan.Flag("model", "HuggingFace model to scan. You can repeat this flag. Example: 'username/model'").Strings()
	huggingfaceSpaces   = huggingfaceScan.Flag("space", "HuggingFace space to scan. You can repeat this flag. Example: 'username/space'").Strings()
	huggingfaceDatasets = huggingfaceScan.Flag("dataset", "HuggingFace dataset to scan. You can repeat this flag. Example: 'username/dataset'").Strings()
	huggingfaceOrgs     = huggingfaceScan.Flag("org", `HuggingFace organization to scan. You can repeat this flag. Example: "trufflesecurity"`).Strings()
	huggingfaceUsers    = huggingfaceScan.Flag("user", `HuggingFace user to scan. You can repeat this flag. Example: "trufflesecurity"`).Strings()
	huggingfaceToken    = huggingfaceScan.Flag("token", "HuggingFace token. Can be provided with environment variable HUGGINGFACE_TOKEN.").Envar("HUGGINGFACE_TOKEN").String()

	huggingfaceIncludeModels      = huggingfaceScan.Flag("include-models", "Models to include in scan. You can repeat this flag. Must use HuggingFace model full name. Example: 'username/model' (Only used with --user or --org)").Strings()
	huggingfaceIncludeSpaces      = huggingfaceScan.Flag("include-spaces", "Spaces to include in scan. You can repeat this flag. Must use HuggingFace space full name. Example: 'username/space' (Only used with --user or --org)").Strings()
	huggingfaceIncludeDatasets    = huggingfaceScan.Flag("include-datasets", "Datasets to include in scan. You can repeat this flag. Must use HuggingFace dataset full name. Example: 'username/dataset' (Only used with --user or --org)").Strings()
	huggingfaceIgnoreModels       = huggingfaceScan.Flag("ignore-models", "Models to ignore in scan. You can repeat this flag. Must use HuggingFace model full name. Example: 'username/model' (Only used with --user or --org)").Strings()
	huggingfaceIgnoreSpaces       = huggingfaceScan.Flag("ignore-spaces", "Spaces to ignore in scan. You can repeat this flag. Must use HuggingFace space full name. Example: 'username/space' (Only used with --user or --org)").Strings()
	huggingfaceIgnoreDatasets     = huggingfaceScan.Flag("ignore-datasets", "Datasets to ignore in scan. You can repeat this flag. Must use HuggingFace dataset full name. Example: 'username/dataset' (Only used with --user or --org)").Strings()
	huggingfaceSkipAllModels      = huggingfaceScan.Flag("skip-all-models", "Skip all model scans. (Only used with --user or --org)").Bool()
	huggingfaceSkipAllSpaces      = huggingfaceScan.Flag("skip-all-spaces", "Skip all space scans. (Only used with --user or --org)").Bool()
	huggingfaceSkipAllDatasets    = huggingfaceScan.Flag("skip-all-datasets", "Skip all dataset scans. (Only used with --user or --org)").Bool()
	huggingfaceIncludeDiscussions = huggingfaceScan.Flag("include-discussions", "Include discussions in scan.").Bool()
	huggingfaceIncludePrs         = huggingfaceScan.Flag("include-prs", "Include pull requests in scan.").Bool()

	stdinInputScan = cli.Command("stdin", "Find credentials from stdin.")
	multiScanScan  = cli.Command("multi-scan", "Find credentials in multiple sources defined in configuration.")

	analyzeCmd = analyzer.Command(cli)
	usingTUI   = false
)

func init() {
	_, _ = maxprocs.Set()

	for i, arg := range os.Args {
		if strings.HasPrefix(arg, "--") {
			split := strings.SplitN(arg, "=", 2)
			split[0] = strings.ReplaceAll(split[0], "_", "-")
			os.Args[i] = strings.Join(split, "=")
		}
	}

	cli.Version("trufflehog " + version.BuildVersion)

	// Support -h for help and write it to stdout.
	cli.HelpFlag.Short('h')
	cli.UsageWriter(os.Stdout)

	// Check if the TUI environment variable is set.
	if ok, err := strconv.ParseBool(os.Getenv("TUI_PARENT")); err == nil {
		usingTUI = ok
	}

	if isatty.IsTerminal(os.Stdout.Fd()) && (len(os.Args) <= 1 || os.Args[1] == analyzeCmd.FullCommand()) {
		args := tui.Run(os.Args[1:])
		if len(args) == 0 {
			os.Exit(0)
		}

		binary, err := exec.LookPath("sh")
		if err == nil {
			// On success, this call will never return. On failure, fallthrough
			// to overwriting os.Args.
			cmd := strings.Join(append(os.Args[:1], args...), " ")
			_ = syscall.Exec(binary, []string{"sh", "-c", cmd}, append(os.Environ(), "TUI_PARENT=true"))
		}

		// Overwrite the Args slice so overseer works properly.
		os.Args = os.Args[:1]
		os.Args = append(os.Args, args...)

		usingTUI = true
	}

	cmd = kingpin.MustParse(cli.Parse(os.Args[1:]))

	// Configure logging.
	switch {
	case *trace:
		log.SetLevel(5)
	case *debug:
		log.SetLevel(2)
	default:
		l := int8(*logLevel)
		if l < -1 || l > 5 {
			fmt.Fprintf(os.Stderr, "invalid log level: %d\n", *logLevel)
			os.Exit(1)
		}

		if l == -1 {
			// Zap uses "5" as the value for fatal.
			// We need to pass in "-5" because `SetLevel` passes the negation.
			log.SetLevel(-5)
		} else {
			log.SetLevel(l)
		}
	}

	if *noColor || *noColour {
		color.NoColor = true // disables colorized output
	}
}

func main() {
	// setup logger
	logFormat := log.WithConsoleSink
	if *jsonOut {
		logFormat = log.WithJSONSink
	}
	logger, sync := log.New("trufflehog", logFormat(os.Stderr, log.WithGlobalRedaction()))
	// make it the default logger for contexts
	context.SetDefaultLogger(logger)

	if *localDev {
		run(overseer.State{})
		os.Exit(0)
	}

	defer func() { _ = sync() }()
	logFatal := logFatalFunc(logger)

	updateCfg := overseer.Config{
		Program:       run,
		Debug:         *debug,
		RestartSignal: syscall.SIGTERM,
		// TODO: Eventually add a PreUpgrade func for signature check w/ x509 PKCS1v15
		// PreUpgrade: checkUpdateSignature(binaryPath string),
	}

	if !*noUpdate {
		topLevelCmd, _, _ := strings.Cut(cmd, " ")
		updateCfg.Fetcher = updater.Fetcher(topLevelCmd, usingTUI)
	}
	if version.BuildVersion == "dev" {
		updateCfg.Fetcher = nil
	}

	err := overseer.RunErr(updateCfg)
	if err != nil {
		logFatal(err, "error occurred with trufflehog updater 🐷")
	}
}

func run(state overseer.State) {

	ctx, cancel := context.WithCancelCause(context.Background())
	defer cancel(nil)

	go func() {
		if err := cleantemp.CleanTempArtifacts(ctx); err != nil {
			ctx.Logger().Error(err, "error cleaning temporary artifacts")
		}
	}()

	logger := ctx.Logger()
	logFatal := logFatalFunc(logger)

	killSignal := make(chan os.Signal, 1)
	signal.Notify(killSignal, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT)
	go func() {
		<-killSignal
		logger.Info("Received signal, shutting down.")
		cancel(fmt.Errorf("canceling context due to signal"))

		if err := cleantemp.CleanTempArtifacts(ctx); err != nil {
			logger.Error(err, "error cleaning temporary artifacts")
		} else {
			logger.Info("cleaned temporary artifacts")
		}
		os.Exit(0)
	}()

	logger.V(2).Info(fmt.Sprintf("trufflehog %s", version.BuildVersion))

	if *githubScanToken != "" {
		// NOTE: this kludge is here to do an authenticated shallow commit
		// TODO: refactor to better pass credentials
		os.Setenv("GITHUB_TOKEN", *githubScanToken)
	}

	// When setting a base commit, chunks must be scanned in order.
	if *gitScanSinceCommit != "" {
		*concurrency = 1
	}

	if *profile {
		runtime.SetBlockProfileRate(1)
		runtime.SetMutexProfileFraction(-1)
		go func() {
			router := http.NewServeMux()
			router.Handle("/debug/pprof/", http.DefaultServeMux)
			router.Handle("/debug/fgprof", fgprof.Handler())
			logger.Info("starting pprof and fgprof server on :18066 /debug/pprof and /debug/fgprof")
			if err := http.ListenAndServe(":18066", router); err != nil {
				logger.Error(err, "error serving pprof and fgprof")
			}
		}()
	}

	// Set feature configurations from CLI flags
	if *forceSkipBinaries {
		feature.ForceSkipBinaries.Store(true)
	}

	if *forceSkipArchives {
		feature.ForceSkipArchives.Store(true)
	}

	if gitCloneTimeout != nil {
		feature.GitCloneTimeoutDuration.Store(int64(*gitCloneTimeout))
	}

	if *skipAdditionalRefs {
		feature.SkipAdditionalRefs.Store(true)
	}

	if *userAgentSuffix != "" {
		feature.UserAgentSuffix.Store(*userAgentSuffix)
	}

	// OSS Default APK handling on
	feature.EnableAPKHandler.Store(true)

	// OSS Default Use Git Mirror on
	feature.UseGitMirror.Store(true)

	// OSS Default simplified gitlab enumeration
	feature.UseSimplifiedGitlabEnumeration.Store(true)
	feature.GitlabProjectsPerPage.Store(100)

	// OSS Default using github graphql api for issues, pr's and comments
	feature.UseGithubGraphQLAPI.Store(false)

	conf := &config.Config{}
	if *configFilename != "" {
		var err error
		conf, err = config.Read(*configFilename)
		if err != nil {
			logFatal(err, "error parsing the provided configuration file")
		}
	}

	if *detectorTimeout != 0 {
		logger.Info("Setting detector timeout", "timeout", detectorTimeout.String())
		engine.SetDetectorTimeout(*detectorTimeout)
		detectors.OverrideDetectorTimeout(*detectorTimeout)
	}
	if *archiveMaxSize != 0 {
		handlers.SetArchiveMaxSize(int(*archiveMaxSize))
	}
	if *archiveMaxDepth != 0 {
		handlers.SetArchiveMaxDepth(*archiveMaxDepth)
	}
	if *archiveTimeout != 0 {
		handlers.SetArchiveMaxTimeout(*archiveTimeout)
	}

	// Set how the engine will print its results.
	var printer engine.Printer
	switch {
	case *jsonLegacy:
		printer = new(output.LegacyJSONPrinter)
	case *jsonOut:
		printer = new(output.JSONPrinter)
	case *gitHubActionsFormat:
		printer = new(output.GitHubActionsPrinter)
	default:
		printer = new(output.PlainPrinter)
	}

	if !*jsonLegacy && !*jsonOut {
		fmt.Fprintf(os.Stderr, "🐷🔑🐷  TruffleHog. Unearth your secrets. 🐷🔑🐷\n\n")
	}

	// Parse --results flag.
	if *onlyVerified {
		r := "verified"
		results = &r
	}
	parsedResults, err := parseResults(results)
	if err != nil {
		logFatal(err, "failed to configure results flag")
	}

	verificationCacheMetrics := verificationcache.InMemoryMetrics{}

	engConf := engine.Config{
		Concurrency:       *concurrency,
		ConfiguredSources: conf.Sources,
		// The engine must always be configured with the list of
		// default detectors, which can be further filtered by the
		// user. The filters are applied by the engine and are only
		// subtractive.
		Detectors:                append(defaults.DefaultDetectors(), conf.Detectors...),
		Verify:                   !*noVerification,
		IncludeDetectors:         *includeDetectors,
		ExcludeDetectors:         *excludeDetectors,
		CustomVerifiersOnly:      *customVerifiersOnly,
		VerifierEndpoints:        *verifiers,
		Dispatcher:               engine.NewPrinterDispatcher(printer),
		FilterUnverified:         *filterUnverified,
		FilterEntropy:            *filterEntropy,
		VerificationOverlap:      *allowVerificationOverlap,
		Results:                  parsedResults,
		PrintAvgDetectorTime:     *printAvgDetectorTime,
		ShouldScanEntireChunk:    *scanEntireChunk,
		VerificationCacheMetrics: &verificationCacheMetrics,
	}

	if !*noVerificationCache {
		engConf.VerificationResultCache = simple.NewCache[detectors.Result]()
	}

	// Check that there are no sources defined for non-scan subcommands. If
	// there are, return an error as it is ambiguous what the user is
	// trying to do.
	if cmd != multiScanScan.FullCommand() && len(conf.Sources) > 0 {
		logFatal(
			fmt.Errorf("ambiguous configuration"),
			"sources should only be defined in configuration for the 'multi-scan' command",
		)
	}

	if *compareDetectionStrategies {
		if err := compareScans(ctx, cmd, engConf); err != nil {
			logFatal(err, "error comparing detection strategies")
		}
		return
	}

	metrics, err := runSingleScan(ctx, cmd, engConf)
	if err != nil {
		logFatal(err, "error running scan")
	}

	verificationCacheMetricsSnapshot := struct {
		Hits                    int32
		Misses                  int32
		HitsWasted              int32
		AttemptsSaved           int32
		VerificationTimeSpentMS int64
	}{
		Hits:                    verificationCacheMetrics.ResultCacheHits.Load(),
		Misses:                  verificationCacheMetrics.ResultCacheMisses.Load(),
		HitsWasted:              verificationCacheMetrics.ResultCacheHitsWasted.Load(),
		AttemptsSaved:           verificationCacheMetrics.CredentialVerificationsSaved.Load(),
		VerificationTimeSpentMS: verificationCacheMetrics.FromDataVerifyTimeSpentMS.Load(),
	}

	// Print results.
	logger.Info("finished scanning",
		"chunks", metrics.ChunksScanned,
		"bytes", metrics.BytesScanned,
		"verified_secrets", metrics.VerifiedSecretsFound,
		"unverified_secrets", metrics.UnverifiedSecretsFound,
		"scan_duration", metrics.ScanDuration.String(),
		"trufflehog_version", version.BuildVersion,
		"verification_caching", verificationCacheMetricsSnapshot,
	)

	if metrics.hasFoundResults && *fail {
		logger.V(2).Info("exiting with code 183 because results were found")
		os.Exit(183)
	}
}

func compareScans(ctx context.Context, cmd string, cfg engine.Config) error {
	var (
		entireMetrics    metrics
		maxLengthMetrics metrics
		err              error
	)

	var wg sync.WaitGroup
	wg.Add(1)

	go func() {
		defer wg.Done()
		// Run scan with entire chunk span calculator.
		cfg.ShouldScanEntireChunk = true
		entireMetrics, err = runSingleScan(ctx, cmd, cfg)
		if err != nil {
			ctx.Logger().Error(err, "error running scan with entire chunk span calculator")
		}
	}()

	// Run scan with max-length span calculator.
	maxLengthMetrics, err = runSingleScan(ctx, cmd, cfg)
	if err != nil {
		return fmt.Errorf("error running scan with custom span calculator: %v", err)
	}

	wg.Wait()

	return compareMetrics(maxLengthMetrics.Metrics, entireMetrics.Metrics)
}

func compareMetrics(customMetrics, entireMetrics engine.Metrics) error {
	fmt.Printf("Comparison of scan results: \n")
	fmt.Printf("Custom span - Chunks: %d, Bytes: %d, Verified Secrets: %d, Unverified Secrets: %d, Duration: %s\n",
		customMetrics.ChunksScanned, customMetrics.BytesScanned, customMetrics.VerifiedSecretsFound, customMetrics.UnverifiedSecretsFound, customMetrics.ScanDuration.String())
	fmt.Printf("Entire chunk - Chunks: %d, Bytes: %d, Verified Secrets: %d, Unverified Secrets: %d, Duration: %s\n",
		entireMetrics.ChunksScanned, entireMetrics.BytesScanned, entireMetrics.VerifiedSecretsFound, entireMetrics.UnverifiedSecretsFound, entireMetrics.ScanDuration.String())

	// Check for differences in scan metrics.
	if customMetrics.ChunksScanned != entireMetrics.ChunksScanned ||
		customMetrics.BytesScanned != entireMetrics.BytesScanned ||
		customMetrics.VerifiedSecretsFound != entireMetrics.VerifiedSecretsFound {
		return fmt.Errorf("scan metrics do not match")
	}

	return nil
}

type metrics struct {
	engine.Metrics
	hasFoundResults bool
}

func runSingleScan(ctx context.Context, cmd string, cfg engine.Config) (metrics, error) {
	var scanMetrics metrics

	// Setup job report writer if provided
	var jobReportWriter io.WriteCloser
	if *jobReportFile != nil {
		jobReportWriter = *jobReportFile
	}

	handleFinishedMetrics := func(ctx context.Context, finishedMetrics <-chan sources.UnitMetrics, jobReportWriter io.WriteCloser) {
		go func() {
			defer func() {
				jobReportWriter.Close()
				if namer, ok := jobReportWriter.(interface{ Name() string }); ok {
					ctx.Logger().Info("report written", "path", namer.Name())
				} else {
					ctx.Logger().Info("report written")
				}
			}()

			for metrics := range finishedMetrics {
				metrics.Errors = common.ExportErrors(metrics.Errors...)
				details, err := json.Marshal(map[string]any{
					"version": 1,
					"data":    metrics,
				})
				if err != nil {
					ctx.Logger().Error(err, "error marshalling job details")
					continue
				}
				if _, err := jobReportWriter.Write(append(details, '\n')); err != nil {
					ctx.Logger().Error(err, "error writing to file")
				}
			}
		}()
	}

	const defaultOutputBufferSize = 64
	opts := []func(*sources.SourceManager){
		sources.WithConcurrentSources(cfg.Concurrency),
		sources.WithConcurrentUnits(cfg.Concurrency),
		sources.WithSourceUnits(),
		sources.WithBufferedOutput(defaultOutputBufferSize),
	}

	if jobReportWriter != nil {
		unitHook, finishedMetrics := sources.NewUnitHook(ctx)
		opts = append(opts, sources.WithReportHook(unitHook))
		handleFinishedMetrics(ctx, finishedMetrics, jobReportWriter)
	}

	cfg.SourceManager = sources.NewManager(opts...)

	eng, err := engine.NewEngine(ctx, &cfg)
	if err != nil {
		return scanMetrics, fmt.Errorf("error initializing engine: %v", err)
	}
	eng.Start(ctx)

	persistGitRepo := *gitNoCleanup || *githubNoCleanup || *gitlabNoCleanup
	gitCloneTempPath := ""

	defer func() {
		// Clean up temporary artifacts.
		if err := cleantemp.CleanTempArtifacts(ctx); err != nil {
			ctx.Logger().Error(err, "error cleaning temp artifacts")
		}

		if *jsonLegacy {
			// If JSON legacy is enabled, that means the cloned repos are not deleted yet
			// because they were needed for outputting legacy JSON.
			// We only clean them up here if the user did not request to persist them.
			if !persistGitRepo {
				if err := cleantemp.CleanTempDirsForLegacyJSON(gitCloneTempPath); err != nil {
					ctx.Logger().Error(err, "error cleaning temp artifacts for legacy JSON")
				}
			}
		}
	}()

	var refs []sources.JobProgressRef
	switch cmd {
	case gitScan.FullCommand():

		if err := validateClonePath(*gitClonePath, *gitNoCleanup); err != nil {
			return scanMetrics, err
		}

		gitCfg := sources.GitConfig{
			URI:                 *gitScanURI,
			IncludePathsFile:    *gitScanIncludePaths,
			ExcludePathsFile:    *gitScanExcludePaths,
			HeadRef:             *gitScanBranch,
			BaseRef:             *gitScanSinceCommit,
			MaxDepth:            *gitScanMaxDepth,
			Bare:                *gitScanBare,
			ExcludeGlobs:        *gitScanExcludeGlobs,
			ClonePath:           *gitClonePath,
			NoCleanup:           *gitNoCleanup,
			PrintLegacyJSON:     *jsonLegacy,
			TrustLocalGitConfig: *gitTrustLocalGitConfig,
		}
		if ref, err := eng.ScanGit(ctx, gitCfg); err != nil {
			return scanMetrics, fmt.Errorf("failed to scan Git: %v", err)
		} else {
			refs = []sources.JobProgressRef{ref}
		}
	case githubScan.FullCommand():
		gitCloneTempPath = *githubClonePath
		filter, err := common.FilterFromFiles(*githubScanIncludePaths, *githubScanExcludePaths)
		if err != nil {
			return scanMetrics, fmt.Errorf("could not create filter: %v", err)
		}
		if len(*githubScanOrgs) == 0 && len(*githubScanRepos) == 0 {
			return scanMetrics, fmt.Errorf("invalid config: you must specify at least one organization or repository")
		}
		if len(*githubScanOrgs) > 0 && len(*githubScanRepos) > 0 {
			return scanMetrics, fmt.Errorf("invalid config: you cannot specify both organizations and repositories at the same time")
		}

		if err := validateClonePath(*githubClonePath, *githubNoCleanup); err != nil {
			return scanMetrics, err
		}

		cfg := sources.GithubConfig{
			Endpoint:                   *githubScanEndpoint,
			Token:                      *githubScanToken,
			IncludeForks:               *githubIncludeForks,
			IncludeMembers:             *githubIncludeMembers,
			IncludeWikis:               *githubIncludeWikis,
			Concurrency:                *concurrency,
			ExcludeRepos:               *githubExcludeRepos,
			IncludeRepos:               *githubIncludeRepos,
			Repos:                      *githubScanRepos,
			Orgs:                       *githubScanOrgs,
			IncludeIssueComments:       *githubScanIssueComments,
			IncludePullRequestComments: *githubScanPRComments,
			IncludeGistComments:        *githubScanGistComments,
			CommentsTimeframeDays:      *githubCommentsTimeframeDays,
			Filter:                     filter,
			AuthInUrl:                  *githubAuthInUrl,
			ClonePath:                  *githubClonePath,
			NoCleanup:                  *githubNoCleanup,
			IgnoreGists:                *githubIgnoreGists,
			PrintLegacyJSON:            *jsonLegacy,
		}

		if ref, err := eng.ScanGitHub(ctx, cfg); err != nil {
			return scanMetrics, fmt.Errorf("failed to scan Github: %v", err)
		} else {
			refs = []sources.JobProgressRef{ref}
		}
	case githubExperimentalScan.FullCommand():
		cfg := sources.GitHubExperimentalConfig{
			Token:              *githubExperimentalToken,
			Repository:         *githubExperimentalRepo,
			ObjectDiscovery:    *githubExperimentalObjectDiscovery,
			CollisionThreshold: *githubExperimentalCollisionThreshold,
			DeleteCachedData:   *githubExperimentalDeleteCache,
		}
		if ref, err := eng.ScanGitHubExperimental(ctx, cfg); err != nil {
			return scanMetrics, fmt.Errorf("failed to scan using Github Experimental: %v", err)
		} else {
			refs = []sources.JobProgressRef{ref}
		}
	case gitlabScan.FullCommand():
		gitCloneTempPath = *gitlabClonePath
		filter, err := common.FilterFromFiles(*gitlabScanIncludePaths, *gitlabScanExcludePaths)
		if err != nil {
			return scanMetrics, fmt.Errorf("could not create filter: %v", err)
		}

		if len(*gitlabScanRepos) > 0 && len(*gitlabScanGroupIds) > 0 {
			return scanMetrics, fmt.Errorf("invalid config: you cannot specify both repositories and groups at the same time")
		}

		if err := validateClonePath(*gitlabClonePath, *gitlabNoCleanup); err != nil {
			return scanMetrics, err
		}

		cfg := sources.GitlabConfig{
			Endpoint:        *gitlabScanEndpoint,
			Token:           *gitlabScanToken,
			Repos:           *gitlabScanRepos,
			GroupIds:        *gitlabScanGroupIds,
			IncludeRepos:    *gitlabScanIncludeRepos,
			ExcludeRepos:    *gitlabScanExcludeRepos,
			Filter:          filter,
			AuthInUrl:       *gitlabAuthInUrl,
			ClonePath:       *gitlabClonePath,
			NoCleanup:       *gitlabNoCleanup,
			PrintLegacyJSON: *jsonLegacy,
		}

		if ref, err := eng.ScanGitLab(ctx, cfg); err != nil {
			return scanMetrics, fmt.Errorf("failed to scan GitLab: %v", err)
		} else {
			refs = []sources.JobProgressRef{ref}
		}
	case filesystemScan.FullCommand():
		if len(*filesystemDirectories) > 0 {
			ctx.Logger().Info("--directory flag is deprecated, please pass directories as arguments")
		}
		paths := make([]string, 0, len(*filesystemPaths)+len(*filesystemDirectories))
		paths = append(paths, *filesystemPaths...)
		paths = append(paths, *filesystemDirectories...)
		cfg := sources.FilesystemConfig{
			Paths:            paths,
			IncludePathsFile: *filesystemScanIncludePaths,
			ExcludePathsFile: *filesystemScanExcludePaths,
		}
		if ref, err := eng.ScanFileSystem(ctx, cfg); err != nil {
			return scanMetrics, fmt.Errorf("failed to scan filesystem: %v", err)
		} else {
			refs = []sources.JobProgressRef{ref}
		}
	case s3Scan.FullCommand():
		cfg := sources.S3Config{
			Key:           *s3ScanKey,
			Secret:        *s3ScanSecret,
			SessionToken:  *s3ScanSessionToken,
			Buckets:       *s3ScanBuckets,
			IgnoreBuckets: *s3ScanIgnoreBuckets,
			Roles:         *s3ScanRoleArns,
			CloudCred:     *s3ScanCloudEnv,
			MaxObjectSize: int64(*s3ScanMaxObjectSize),
		}
		if ref, err := eng.ScanS3(ctx, cfg); err != nil {
			return scanMetrics, fmt.Errorf("failed to scan S3: %v", err)
		} else {
			refs = []sources.JobProgressRef{ref}
		}
	case syslogScan.FullCommand():
		cfg := sources.SyslogConfig{
			Address:     *syslogAddress,
			Format:      *syslogFormat,
			Protocol:    *syslogProtocol,
			CertPath:    *syslogTLSCert,
			KeyPath:     *syslogTLSKey,
			Concurrency: *concurrency,
		}
		if ref, err := eng.ScanSyslog(ctx, cfg); err != nil {
			return scanMetrics, fmt.Errorf("failed to scan syslog: %v", err)
		} else {
			refs = []sources.JobProgressRef{ref}
		}
	case circleCiScan.FullCommand():
		if ref, err := eng.ScanCircleCI(ctx, *circleCiScanToken); err != nil {
			return scanMetrics, fmt.Errorf("failed to scan CircleCI: %v", err)
		} else {
			refs = []sources.JobProgressRef{ref}
		}
	case travisCiScan.FullCommand():
		if ref, err := eng.ScanTravisCI(ctx, *travisCiScanToken); err != nil {
			return scanMetrics, fmt.Errorf("failed to scan TravisCI: %v", err)
		} else {
			refs = []sources.JobProgressRef{ref}
		}
	case gcsScan.FullCommand():
		cfg := sources.GCSConfig{
			ProjectID:      *gcsProjectID,
			CloudCred:      *gcsCloudEnv,
			ServiceAccount: *gcsServiceAccount,
			WithoutAuth:    *gcsWithoutAuth,
			ApiKey:         *gcsAPIKey,
			IncludeBuckets: commaSeparatedToSlice(*gcsIncludeBuckets),
			ExcludeBuckets: commaSeparatedToSlice(*gcsExcludeBuckets),
			IncludeObjects: commaSeparatedToSlice(*gcsIncludeObjects),
			ExcludeObjects: commaSeparatedToSlice(*gcsExcludeObjects),
			Concurrency:    *concurrency,
			MaxObjectSize:  int64(*gcsMaxObjectSize),
		}
		if ref, err := eng.ScanGCS(ctx, cfg); err != nil {
			return scanMetrics, fmt.Errorf("failed to scan GCS: %v", err)
		} else {
			refs = []sources.JobProgressRef{ref}
		}
	case dockerScan.FullCommand():
		if *dockerScanImages != nil && *dockerScanNamespace != "" {
			return scanMetrics, fmt.Errorf("invalid config: you cannot specify both images and namespace at the same time")
		}

		if *dockerScanImages == nil && *dockerScanNamespace == "" {
			return scanMetrics, fmt.Errorf("invalid config: both images and namespace cannot be empty; one is required")
		}

		if *dockerScanRegistryToken != "" && *dockerScanNamespace == "" {
			return scanMetrics, fmt.Errorf("invalid config: registry token can only be used with registry namespace")
		}

		cfg := sources.DockerConfig{
			BearerToken:       *dockerScanToken,
			Images:            *dockerScanImages,
			UseDockerKeychain: *dockerScanToken == "",
			ExcludePaths:      strings.Split(*dockerExcludePaths, ","),
			Namespace:         *dockerScanNamespace,
			RegistryToken:     *dockerScanRegistryToken,
		}
		if ref, err := eng.ScanDocker(ctx, cfg); err != nil {
			return scanMetrics, fmt.Errorf("failed to scan Docker: %v", err)
		} else {
			refs = []sources.JobProgressRef{ref}
		}
	case postmanScan.FullCommand():
		// handle deprecated flag
		workspaceIDs := make([]string, 0, len(*postmanWorkspaceIDs)+len(*postmanWorkspaces))
		workspaceIDs = append(workspaceIDs, *postmanWorkspaceIDs...)
		workspaceIDs = append(workspaceIDs, *postmanWorkspaces...)

		// handle deprecated flag
		collectionIDs := make([]string, 0, len(*postmanCollectionIDs)+len(*postmanCollections))
		collectionIDs = append(collectionIDs, *postmanCollectionIDs...)
		collectionIDs = append(collectionIDs, *postmanCollections...)

		// handle deprecated flag
		includeCollectionIDs := make([]string, 0, len(*postmanIncludeCollectionIDs)+len(*postmanIncludeCollections))
		includeCollectionIDs = append(includeCollectionIDs, *postmanIncludeCollectionIDs...)
		includeCollectionIDs = append(includeCollectionIDs, *postmanIncludeCollections...)

		// handle deprecated flag
		excludeCollectionIDs := make([]string, 0, len(*postmanExcludeCollectionIDs)+len(*postmanExcludeCollections))
		excludeCollectionIDs = append(excludeCollectionIDs, *postmanExcludeCollectionIDs...)
		excludeCollectionIDs = append(excludeCollectionIDs, *postmanExcludeCollections...)

		cfg := sources.PostmanConfig{
			Token:               *postmanToken,
			Workspaces:          workspaceIDs,
			Collections:         collectionIDs,
			Environments:        *postmanEnvironments,
			IncludeCollections:  includeCollectionIDs,
			IncludeEnvironments: *postmanIncludeEnvironments,
			ExcludeCollections:  excludeCollectionIDs,
			ExcludeEnvironments: *postmanExcludeEnvironments,
			CollectionPaths:     *postmanCollectionPaths,
			WorkspacePaths:      *postmanWorkspacePaths,
			EnvironmentPaths:    *postmanEnvironmentPaths,
		}
		if ref, err := eng.ScanPostman(ctx, cfg); err != nil {
			return scanMetrics, fmt.Errorf("failed to scan Postman: %v", err)
		} else {
			refs = []sources.JobProgressRef{ref}
		}
	case elasticsearchScan.FullCommand():
		cfg := sources.ElasticsearchConfig{
			Nodes:          *elasticsearchNodes,
			Username:       *elasticsearchUsername,
			Password:       *elasticsearchPassword,
			CloudID:        *elasticsearchCloudId,
			APIKey:         *elasticsearchAPIKey,
			ServiceToken:   *elasticsearchServiceToken,
			IndexPattern:   *elasticsearchIndexPattern,
			QueryJSON:      *elasticsearchQueryJSON,
			SinceTimestamp: *elasticsearchSinceTimestamp,
			BestEffortScan: *elasticsearchBestEffortScan,
		}
		if ref, err := eng.ScanElasticsearch(ctx, cfg); err != nil {
			return scanMetrics, fmt.Errorf("failed to scan Elasticsearch: %v", err)
		} else {
			refs = []sources.JobProgressRef{ref}
		}
	case jenkinsScan.FullCommand():
		cfg := engine.JenkinsConfig{
			Endpoint:              *jenkinsURL,
			InsecureSkipVerifyTLS: *jenkinsInsecureSkipVerifyTLS,
			Username:              *jenkinsUsername,
			Password:              *jenkinsPassword,
		}
		if ref, err := eng.ScanJenkins(ctx, cfg); err != nil {
			return scanMetrics, fmt.Errorf("failed to scan Jenkins: %v", err)
		} else {
			refs = []sources.JobProgressRef{ref}
		}
	case huggingfaceScan.FullCommand():
		if *huggingfaceEndpoint != "" {
			*huggingfaceEndpoint = strings.TrimRight(*huggingfaceEndpoint, "/")
		}

		if len(*huggingfaceModels) == 0 && len(*huggingfaceSpaces) == 0 && len(*huggingfaceDatasets) == 0 && len(*huggingfaceOrgs) == 0 && len(*huggingfaceUsers) == 0 {
			return scanMetrics, fmt.Errorf("invalid config: you must specify at least one organization, user, model, space or dataset")
		}

		cfg := engine.HuggingfaceConfig{
			Endpoint:           *huggingfaceEndpoint,
			Models:             *huggingfaceModels,
			Spaces:             *huggingfaceSpaces,
			Datasets:           *huggingfaceDatasets,
			Organizations:      *huggingfaceOrgs,
			Users:              *huggingfaceUsers,
			Token:              *huggingfaceToken,
			IncludeModels:      *huggingfaceIncludeModels,
			IncludeSpaces:      *huggingfaceIncludeSpaces,
			IncludeDatasets:    *huggingfaceIncludeDatasets,
			IgnoreModels:       *huggingfaceIgnoreModels,
			IgnoreSpaces:       *huggingfaceIgnoreSpaces,
			IgnoreDatasets:     *huggingfaceIgnoreDatasets,
			SkipAllModels:      *huggingfaceSkipAllModels,
			SkipAllSpaces:      *huggingfaceSkipAllSpaces,
			SkipAllDatasets:    *huggingfaceSkipAllDatasets,
			IncludeDiscussions: *huggingfaceIncludeDiscussions,
			IncludePrs:         *huggingfaceIncludePrs,
			Concurrency:        *concurrency,
		}
		if ref, err := eng.ScanHuggingface(ctx, cfg); err != nil {
			return scanMetrics, fmt.Errorf("failed to scan HuggingFace: %v", err)
		} else {
			refs = []sources.JobProgressRef{ref}
		}
	case multiScanScan.FullCommand():
		if *configFilename == "" {
			return scanMetrics, fmt.Errorf("missing required flag: --config")
		}
		if rs, err := eng.ScanConfig(ctx, cfg.ConfiguredSources...); err != nil {
			return scanMetrics, fmt.Errorf("failed to scan via config: %w", err)
		} else {
			refs = rs
		}
	case stdinInputScan.FullCommand():
		cfg := sources.StdinConfig{}
		if ref, err := eng.ScanStdinInput(ctx, cfg); err != nil {
			return scanMetrics, fmt.Errorf("failed to scan stdin input: %v", err)
		} else {
			refs = []sources.JobProgressRef{ref}
		}
	default:
		return scanMetrics, fmt.Errorf("invalid command: %s", cmd)
	}

	// Wait for all workers to finish.
	if err = eng.Finish(ctx); err != nil {
		return scanMetrics, fmt.Errorf("engine failed to finish execution: %v", err)
	}

	// Print any non-fatal errors reported during the scan.
	var retErr error
	for _, ref := range refs {
		if errs := ref.Snapshot().Errors; len(errs) > 0 {
			if *failOnScanErrors {
				retErr = fmt.Errorf("encountered errors during scan")
			}
			errMsgs := make([]string, len(errs))
			for i := 0; i < len(errs); i++ {
				errMsgs[i] = errs[i].Error()
			}
			ctx.Logger().Error(nil, "encountered errors during scan",
				"job", ref.JobID,
				"source_name", ref.SourceName,
				"errors", errMsgs,
			)
		}
	}

	if *printAvgDetectorTime {
		printAverageDetectorTime(eng)
	}

	return metrics{Metrics: eng.GetMetrics(), hasFoundResults: eng.HasFoundResults()}, retErr
}

// parseResults ensures that users provide valid CSV input to `--results`.
//
// This is a work-around to kingpin not supporting CSVs.
// See: https://github.com/trufflesecurity/trufflehog/pull/2372#issuecomment-1983868917
func parseResults(input *string) (map[string]struct{}, error) {
	if *input == "" {
		return nil, nil
	}

	var (
		values  = strings.Split(strings.ToLower(*input), ",")
		results = make(map[string]struct{}, 3)
	)
	for _, value := range values {
		switch value {
		case "verified", "unknown", "unverified", "filtered_unverified":
			results[value] = struct{}{}
		default:
			return nil, fmt.Errorf("invalid value '%s', valid values are 'verified,unknown,unverified,filtered_unverified'", value)
		}
	}
	return results, nil
}

// logFatalFunc returns a log.Fatal style function. Calling the returned
// function will terminate the program without cleanup.
func logFatalFunc(logger logr.Logger) func(error, string, ...any) {
	return func(err error, message string, keyAndVals ...any) {
		logger.Error(err, message, keyAndVals...)
		if err != nil {
			os.Exit(1)
			return
		}
		os.Exit(0)
	}
}

func commaSeparatedToSlice(s []string) []string {
	var result []string
	for _, items := range s {
		for _, item := range strings.Split(items, ",") {
			item = strings.TrimSpace(item)
			if item == "" {
				continue
			}
			result = append(result, item)
		}
	}
	return result
}

func printAverageDetectorTime(e *engine.Engine) {
	fmt.Fprintln(
		os.Stderr,
		"Average detector time is the measurement of average time spent on each detector when results are returned.",
	)
	for detectorName, duration := range e.GetDetectorsMetrics() {
		fmt.Fprintf(os.Stderr, "%s: %s\n", detectorName, duration)
	}
}

// validateClonePath ensures that --clone-path, if provided, exists and is a directory.
// It also verifies that --no-cleanup is only allowed when --clone-path is set.
// Note: without a custom clone path, repositories are cloned into temporary directories, which should never be retained.
func validateClonePath(clonePath string, noCleanup bool) error {
	if noCleanup && clonePath == "" {
		return fmt.Errorf("invalid configuration: --no-cleanup can only be used together with --clone-path")
	}

	if clonePath == "" {
		return nil
	}

	info, err := os.Stat(clonePath)
	if err != nil {
		if os.IsNotExist(err) {
			return fmt.Errorf("path provided to --clone-path: %q does not exist", clonePath)
		}

		return fmt.Errorf("failed to access --clone-path %q: %w", clonePath, err)
	}

	if !info.IsDir() {
		return fmt.Errorf("path provided to --clone-path: %q is not a directory", clonePath)
	}

	return nil
}