Files
trufflehog/main.go
meredith 4b7d1d3a68 Low-memory Git Scanning (#5257)
* Refactor gitparse for reduced peak memory use

Instead of one single `git log` that grows huge, get a commit list with
"log" then request `git show` groups of those commits.

This is currently slightly slower than main, and parallelizing this part
further doesn't improve things because of the diffChan consumer in the
caller.  It does pipeline just enough that one git show can run while
scanning consumes the prior set of diffs.

* Group commits while reading instead of slurp and split

* Make the low-memory scan mode a non-default option
2026-09-10 12:18:33 -05:00

1419 lines
68 KiB
Go

package main
import (
"encoding/json"
"fmt"
"io"
"net/http"
_ "net/http/pprof"
"os"
"os/exec"
"os/signal"
"runtime"
"strconv"
"strings"
"sync"
"syscall"
"github.com/alecthomas/kingpin/v2"
"github.com/fatih/color"
"github.com/felixge/fgprof"
"github.com/go-logr/logr"
"github.com/jpillora/overseer"
"github.com/mattn/go-isatty"
"go.uber.org/automaxprocs/maxprocs"
"github.com/trufflesecurity/trufflehog/v3/pkg/analyzer"
"github.com/trufflesecurity/trufflehog/v3/pkg/cache/simple"
"github.com/trufflesecurity/trufflehog/v3/pkg/cleantemp"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/config"
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/engine"
"github.com/trufflesecurity/trufflehog/v3/pkg/engine/defaults"
"github.com/trufflesecurity/trufflehog/v3/pkg/feature"
"github.com/trufflesecurity/trufflehog/v3/pkg/handlers"
"github.com/trufflesecurity/trufflehog/v3/pkg/log"
"github.com/trufflesecurity/trufflehog/v3/pkg/output"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
"github.com/trufflesecurity/trufflehog/v3/pkg/tui"
"github.com/trufflesecurity/trufflehog/v3/pkg/updater"
"github.com/trufflesecurity/trufflehog/v3/pkg/verificationcache"
"github.com/trufflesecurity/trufflehog/v3/pkg/version"
)
var (
cli = kingpin.New("trufflehog", "TruffleHog is a tool for finding credentials. Run without a command for interactive mode.")
cmd string
// https://github.com/trufflesecurity/trufflehog/blob/main/CONTRIBUTING.md#logging-in-trufflehog
logLevel = cli.Flag("log-level", `Logging verbosity on a scale of 0 (info) to 5 (trace). Can be disabled with "-1".`).Default("0").Int()
debug = cli.Flag("debug", "Run in debug mode.").Hidden().Bool()
trace = cli.Flag("trace", "Run in trace mode.").Hidden().Bool()
profile = cli.Flag("profile", "Enables profiling and sets a pprof and fgprof server on :18066.").Bool()
localDev = cli.Flag("local-dev", "Hidden feature to disable overseer for local dev.").Hidden().Bool()
jsonOut = cli.Flag("json", "Output in JSON format.").Short('j').Bool()
jsonLegacy = cli.Flag("json-legacy", "Use the pre-v3.0 JSON format. Only works with git, gitlab, and github sources.").Bool()
gitHubActionsFormat = cli.Flag("github-actions", "Output in GitHub Actions format.").Bool()
sarifOut = cli.Flag("sarif", "Output in SARIF format for upload to GitHub code scanning (e.g. via github/codeql-action/upload-sarif).").Bool()
concurrency = cli.Flag("concurrency", "Number of concurrent workers.").PlaceHolder("N").Int()
noVerification = cli.Flag("no-verification", "Don't verify the results.").Bool()
onlyVerified = cli.Flag("only-verified", "Only output verified results.").Hidden().Bool()
results = cli.Flag("results", "Specifies which type(s) of results to output: verified (confirmed valid by API), unknown (verification failed due to error), unverified (detected but not verified), filtered_unverified (unverified but would have been filtered out). Defaults to verified,unverified,unknown.").String()
noColor = cli.Flag("no-color", "Disable colorized output").Bool()
noColour = cli.Flag("no-colour", "Alias for --no-color").Hidden().Bool()
allowVerificationOverlap = cli.Flag("allow-verification-overlap", "Allow verification of similar credentials across detectors").Bool()
filterUnverified = cli.Flag("filter-unverified", "Only output first unverified result per chunk per detector if there are more than one results.").Bool()
filterEntropy = cli.Flag("filter-entropy", "Filter unverified results with Shannon entropy. Start with 3.0.").Float64()
scanEntireChunk = cli.Flag("scan-entire-chunk", "Scan the entire chunk for secrets.").Hidden().Default("false").Bool()
maxDecodeDepth = cli.Flag("max-decode-depth", "Maximum depth of iterative decoding. Each decoder's output is fed back through all decoders, up to this limit. 1 = single pass, 2+ = chained decoding (e.g., base64 inside utf16).").Default("5").Int()
compareDetectionStrategies = cli.Flag("compare-detection-strategies", "Compare different detection strategies for matching spans").Hidden().Default("false").Bool()
configFilename = cli.Flag("config", "Path to configuration file.").ExistingFile()
// rules = cli.Flag("rules", "Path to file with custom rules.").String()
printAvgDetectorTime = cli.Flag("print-avg-detector-time", "Print the average time spent on each detector.").Bool()
noUpdate = cli.Flag("no-update", "Don't check for updates.").Bool()
fail = cli.Flag("fail", "Exit with code 183 if results are found.").Bool()
failOnScanErrors = cli.Flag("fail-on-scan-errors", "Exit with non-zero error code if an error occurs during the scan.").Bool()
verifiers = cli.Flag("verifier", "Set custom verification endpoints.").StringMap()
customVerifiersOnly = cli.Flag("custom-verifiers-only", "Only use custom verification endpoints.").Bool()
detectorTimeout = cli.Flag("detector-timeout", "Maximum time to spend scanning chunks per detector (e.g., 30s).").Duration()
archiveMaxSize = cli.Flag("archive-max-size", "Maximum size of archive to scan. (Byte units eg. 512B, 2KB, 4MB)").Bytes()
archiveMaxDepth = cli.Flag("archive-max-depth", "Maximum depth of archive to scan.").Int()
archiveTimeout = cli.Flag("archive-timeout", "Maximum time to spend extracting an archive.").Duration()
includeDetectors = cli.Flag("include-detectors", "Comma separated list of detector types to include. Protobuf name or IDs may be used, as well as ranges.").Default("all").String()
excludeDetectors = cli.Flag("exclude-detectors", "Comma separated list of detector types to exclude. Protobuf name or IDs may be used, as well as ranges. IDs defined here take precedence over the include list.").String()
jobReportFile = cli.Flag("output-report", "Write a scan report to the provided path.").Hidden().OpenFile(os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666)
noVerificationCache = cli.Flag("no-verification-cache", "Disable verification caching").Bool()
// Add feature flags
forceSkipBinaries = cli.Flag("force-skip-binaries", "Force skipping binaries.").Bool()
forceSkipArchives = cli.Flag("force-skip-archives", "Force skipping archives.").Bool()
gitCloneTimeout = cli.Flag("git-clone-timeout", "Maximum time to spend cloning a repository, as a duration.").Hidden().Duration()
gitLowMemoryScan = cli.Flag("git-low-memory-scan", "Reduce memory use for git scanning.").Hidden().Bool()
skipAdditionalRefs = cli.Flag("skip-additional-refs", "Skip additional references.").Bool()
userAgentSuffix = cli.Flag("user-agent-suffix", "Suffix to add to User-Agent.").String()
dropUnverifiedJWTResults = cli.Flag("drop-unverified-jwt-results", "Drop unverified results without any verification errors from the JWT detector.").Bool()
gitScan = cli.Command("git", "Find credentials in git repositories.")
gitScanURI = gitScan.Arg("uri", "Git repository URL. https://, file://, or ssh:// schema expected.").Required().String()
gitScanIncludePaths = gitScan.Flag("include-paths", "Path to file with newline separated regexes for files to include in scan.").Short('i').String()
gitScanExcludePaths = gitScan.Flag("exclude-paths", "Path to file with newline separated regexes for files to exclude in scan.").Short('x').String()
gitScanExcludeGlobs = gitScan.Flag("exclude-globs", "Comma separated list of globs to exclude in scan. This option filters at the `git log` level, resulting in faster scans.").String()
gitScanSinceCommit = gitScan.Flag("since-commit", "Commit to start scan from.").String()
gitScanBranch = gitScan.Flag("branch", "Branch to scan.").String()
gitScanMaxDepth = gitScan.Flag("max-depth", "Maximum depth of commits to scan.").Int()
gitScanBare = gitScan.Flag("bare", "Scan bare repository (e.g. useful while using in pre-receive hooks)").Bool()
gitClonePath = gitScan.Flag("clone-path", "Custom path where the repository should be cloned (default: temp dir).").String()
gitNoCleanup = gitScan.Flag("no-cleanup", "Do not delete cloned repositories after scanning (can only be used with --clone-path). Each clone is kept in its own directory, so ensure sufficient disk space: usage grows with every repository scanned and every run.").Bool()
gitTrustLocalGitConfig = gitScan.Flag("trust-local-git-config", "Trust local git config.").Bool()
_ = gitScan.Flag("allow", "No-op flag for backwards compat.").Bool()
_ = gitScan.Flag("entropy", "No-op flag for backwards compat.").Bool()
_ = gitScan.Flag("regex", "No-op flag for backwards compat.").Bool()
githubScan = cli.Command("github", "Find credentials in GitHub repositories.")
githubScanEndpoint = githubScan.Flag("endpoint", "GitHub endpoint.").Default("https://api.github.com").String()
githubScanRepos = githubScan.Flag("repo", `GitHub repository to scan. You can repeat this flag. Example: "https://github.com/dustin-decker/secretsandstuff"`).Strings()
githubScanOrgs = githubScan.Flag("org", `GitHub organization to scan. You can repeat this flag. Example: "trufflesecurity"`).Strings()
githubScanToken = githubScan.Flag("token", "GitHub token. Can be provided with environment variable GITHUB_TOKEN.").Envar("GITHUB_TOKEN").String()
githubIncludeForks = githubScan.Flag("include-forks", "Include forks in scan.").Bool()
githubIncludeMembers = githubScan.Flag("include-members", "Include organization member repositories in scan.").Bool()
githubIncludeRepos = githubScan.Flag("include-repos", `Repositories to include in an org scan. This can also be a glob pattern. You can repeat this flag. Must use Github repo full name. Example: "trufflesecurity/trufflehog", "trufflesecurity/t*"`).Strings()
githubIncludeWikis = githubScan.Flag("include-wikis", "Include repository wikisin scan.").Bool()
githubExcludeRepos = githubScan.Flag("exclude-repos", `Repositories to exclude in an org scan. This can also be a glob pattern. You can repeat this flag. Must use Github repo full name. Example: "trufflesecurity/driftwood", "trufflesecurity/d*"`).Strings()
githubScanIncludePaths = githubScan.Flag("include-paths", "Path to file with newline separated regexes for files to include in scan.").Short('i').String()
githubScanExcludePaths = githubScan.Flag("exclude-paths", "Path to file with newline separated regexes for files to exclude in scan.").Short('x').String()
githubScanIssueComments = githubScan.Flag("issue-comments", "Include issue descriptions and comments in scan.").Bool()
githubScanPRComments = githubScan.Flag("pr-comments", "Include pull request descriptions and comments in scan.").Bool()
githubScanGistComments = githubScan.Flag("gist-comments", "Include gist comments in scan.").Bool()
githubCommentsTimeframeDays = githubScan.Flag("comments-timeframe", "Number of days in the past to review when scanning issue, PR, and gist comments.").Uint32()
githubAuthInUrl = githubScan.Flag("auth-in-url", "Embed authentication credentials in repository URLs instead of using secure HTTP headers").Bool()
githubClonePath = githubScan.Flag("clone-path", "Custom path where the repository should be cloned (default: temp dir).").String()
githubNoCleanup = githubScan.Flag("no-cleanup", "Do not delete cloned repositories after scanning (can only be used with --clone-path). Each clone is kept in its own directory, so ensure sufficient disk space: usage grows with every repository scanned and every run.").Bool()
githubIgnoreGists = githubScan.Flag("ignore-gists", "Ignore all gists in scan.").Bool()
githubExcludeArchived = githubScan.Flag("exclude-archived", "Exclude archived repositories from scan.").Bool()
// GitHub Cross Fork Object Reference Experimental Feature
githubExperimentalScan = cli.Command("github-experimental", "Run an experimental GitHub scan. Must specify at least one experimental sub-module to run: object-discovery.")
// GitHub Experimental SubModules
githubExperimentalObjectDiscovery = githubExperimentalScan.Flag("object-discovery", "Discover hidden data objects in GitHub repositories.").Bool()
// GitHub Experimental Options
githubExperimentalToken = githubExperimentalScan.Flag("token", "GitHub token. Can be provided with environment variable GITHUB_TOKEN.").Envar("GITHUB_TOKEN").String()
githubExperimentalRepo = githubExperimentalScan.Flag("repo", "GitHub repository to scan. Example: https://github.com/<user>/<repo>.git").Required().String()
githubExperimentalCollisionThreshold = githubExperimentalScan.Flag("collision-threshold", "Threshold for short-sha collisions in object-discovery submodule. Default is 1.").Default("1").Int()
githubExperimentalDeleteCache = githubExperimentalScan.Flag("delete-cached-data", "Delete cached data after object-discovery secret scanning.").Bool()
gitlabScan = cli.Command("gitlab", "Find credentials in GitLab repositories.")
// TODO: Add more GitLab options
gitlabScanEndpoint = gitlabScan.Flag("endpoint", "GitLab endpoint.").Default("https://gitlab.com").String()
gitlabScanRepos = gitlabScan.Flag("repo", "GitLab repo url. You can repeat this flag. Leave empty to scan all repos accessible with provided credential. Example: https://gitlab.com/org/repo.git").Strings()
gitlabScanToken = gitlabScan.Flag("token", "GitLab token. Can be provided with environment variable GITLAB_TOKEN.").Envar("GITLAB_TOKEN").Required().String()
gitlabScanGroupIds = gitlabScan.Flag("group-id", "GitLab group ID. If provided, it will scan the group and its subgroups. You can repeat this flag.").Strings()
gitlabScanIncludePaths = gitlabScan.Flag("include-paths", "Path to file with newline separated regexes for files to include in scan.").Short('i').String()
gitlabScanExcludePaths = gitlabScan.Flag("exclude-paths", "Path to file with newline separated regexes for files to exclude in scan.").Short('x').String()
gitlabScanIncludeRepos = gitlabScan.Flag("include-repos", `Repositories to include in an org scan. This can also be a glob pattern. You can repeat this flag. Must use Gitlab repo full name. Example: "trufflesecurity/trufflehog", "trufflesecurity/t*"`).Strings()
gitlabScanExcludeRepos = gitlabScan.Flag("exclude-repos", `Repositories to exclude in an org scan. This can also be a glob pattern. You can repeat this flag. Must use Gitlab repo full name. Example: "trufflesecurity/driftwood", "trufflesecurity/d*"`).Strings()
gitlabAuthInUrl = gitlabScan.Flag("auth-in-url", "Embed authentication credentials in repository URLs instead of using secure HTTP headers").Bool()
gitlabClonePath = gitlabScan.Flag("clone-path", "Custom path where the repository should be cloned (default: temp dir).").String()
gitlabNoCleanup = gitlabScan.Flag("no-cleanup", "Do not delete cloned repositories after scanning (can only be used with --clone-path). Each clone is kept in its own directory, so ensure sufficient disk space: usage grows with every repository scanned and every run.").Bool()
filesystemScan = cli.Command("filesystem", "Find credentials in a filesystem.")
filesystemPaths = filesystemScan.Arg("path", "Path to file or directory to scan.").Strings()
// DEPRECATED: --directory is deprecated in favor of arguments.
filesystemDirectories = filesystemScan.Flag("directory", "Path to directory to scan. You can repeat this flag.").Strings()
// TODO: Add more filesystem scan options. Currently only supports scanning a list of directories.
// filesystemScanRecursive = filesystemScan.Flag("recursive", "Scan recursively.").Short('r').Bool()
filesystemScanIncludePaths = filesystemScan.Flag("include-paths", "Path to file with newline separated regexes for files to include in scan.").Short('i').String()
filesystemScanExcludePaths = filesystemScan.Flag("exclude-paths", "Path to file with newline separated regexes for files to exclude in scan.").Short('x').String()
filesystemScanMaxSymlinkDepth = filesystemScan.Flag("max-symlink-depth", "Maximum depth to follow symlinks during filesystem scan.").Short('s').Int32()
s3Scan = cli.Command("s3", "Find credentials in S3 buckets.")
s3ScanKey = s3Scan.Flag("key", "S3 key used to authenticate. Can be provided with environment variable AWS_ACCESS_KEY_ID.").Envar("AWS_ACCESS_KEY_ID").String()
s3ScanRoleArns = s3Scan.Flag("role-arn", "Specify the ARN of an IAM role to assume for scanning. You can repeat this flag.").Strings()
s3ScanSecret = s3Scan.Flag("secret", "S3 secret used to authenticate. Can be provided with environment variable AWS_SECRET_ACCESS_KEY.").Envar("AWS_SECRET_ACCESS_KEY").String()
s3ScanSessionToken = s3Scan.Flag("session-token", "S3 session token used to authenticate temporary credentials. Can be provided with environment variable AWS_SESSION_TOKEN.").Envar("AWS_SESSION_TOKEN").String()
s3ScanCloudEnv = s3Scan.Flag("cloud-environment", "Use IAM credentials in cloud environment.").Bool()
s3ScanBuckets = s3Scan.Flag("bucket", "Name of S3 bucket to scan. You can repeat this flag. Incompatible with --ignore-bucket.").Strings()
s3ScanIgnoreBuckets = s3Scan.Flag("ignore-bucket", "Name of S3 bucket to ignore. You can repeat this flag. Incompatible with --bucket.").Strings()
s3ScanMaxObjectSize = s3Scan.Flag("max-object-size", "Maximum size of objects to scan. Objects larger than this will be skipped. (Byte units eg. 512B, 2KB, 4MB)").Default("250MB").Bytes()
s3ScanEndpoint = s3Scan.Flag("endpoint", "Endpoint of an S3-compatible service to scan instead of AWS S3. (eg. https://s3.internal.example.com)").String()
s3ScanRegion = s3Scan.Flag("region", "Region used to sign requests. Defaults to us-east-1.").String()
s3ScanIncludePrefixes = s3Scan.Flag("include-prefix", "Only scan objects whose key starts with this prefix. You can repeat this flag.").Strings()
s3ScanExcludePrefixes = s3Scan.Flag("exclude-prefix", "Skip objects whose key starts with this prefix. You can repeat this flag. Takes precedence over --include-prefix.").Strings()
s3ScanIncludeExtensions = s3Scan.Flag("include-extension", "Only scan objects with this file extension, written without a leading dot (eg. tf). You can repeat this flag. Incompatible with --exclude-extension.").Strings()
s3ScanExcludeExtensions = s3Scan.Flag("exclude-extension", "Skip objects with this file extension, written without a leading dot (eg. zip). You can repeat this flag. Incompatible with --include-extension.").Strings()
gcsScan = cli.Command("gcs", "Find credentials in GCS buckets.")
gcsProjectID = gcsScan.Flag("project-id", "GCS project ID used to authenticate. Can NOT be used with unauth scan. Can be provided with environment variable GOOGLE_CLOUD_PROJECT.").Envar("GOOGLE_CLOUD_PROJECT").String()
gcsCloudEnv = gcsScan.Flag("cloud-environment", "Use Application Default Credentials, IAM credentials to authenticate.").Bool()
gcsServiceAccount = gcsScan.Flag("service-account", "Path to GCS service account JSON file.").ExistingFile()
gcsWithoutAuth = gcsScan.Flag("without-auth", "Scan GCS buckets without authentication. This will only work for public buckets").Bool()
gcsAPIKey = gcsScan.Flag("api-key", "GCS API key used to authenticate. Can be provided with environment variable GOOGLE_API_KEY.").Envar("GOOGLE_API_KEY").String()
gcsIncludeBuckets = gcsScan.Flag("include-buckets", "Buckets to scan. Comma separated list of buckets. You can repeat this flag. Globs are supported").Short('I').Strings()
gcsExcludeBuckets = gcsScan.Flag("exclude-buckets", "Buckets to exclude from scan. Comma separated list of buckets. Globs are supported").Short('X').Strings()
gcsIncludeObjects = gcsScan.Flag("include-objects", "Objects to scan. Comma separated list of objects. you can repeat this flag. Globs are supported").Short('i').Strings()
gcsExcludeObjects = gcsScan.Flag("exclude-objects", "Objects to exclude from scan. Comma separated list of objects. You can repeat this flag. Globs are supported").Short('x').Strings()
gcsMaxObjectSize = gcsScan.Flag("max-object-size", "Maximum size of objects to scan. Objects larger than this will be skipped. (Byte units eg. 512B, 2KB, 4MB)").Default("10MB").Bytes()
syslogScan = cli.Command("syslog", "Scan syslog")
syslogAddress = syslogScan.Flag("address", "Address and port to listen on for syslog. Example: 127.0.0.1:514").String()
syslogProtocol = syslogScan.Flag("protocol", "Protocol to listen on. udp or tcp").String()
syslogTLSCert = syslogScan.Flag("cert", "Path to TLS cert.").String()
syslogTLSKey = syslogScan.Flag("key", "Path to TLS key.").String()
syslogFormat = syslogScan.Flag("format", "Log format. Can be rfc3164 or rfc5424").Required().String()
circleCiScan = cli.Command("circleci", "Scan CircleCI")
circleCiScanToken = circleCiScan.Flag("token", "CircleCI token. Can also be provided with environment variable").Envar("CIRCLECI_TOKEN").Required().String()
dockerScan = cli.Command("docker", "Scan Docker Image")
dockerScanImages = dockerScan.Flag("image", "Docker image to scan. Use the file:// prefix to point to a local tarball, the docker:// prefix to point to the docker daemon, otherwise an image registry is assumed.").Strings()
dockerScanToken = dockerScan.Flag("token", "Docker bearer token. Can also be provided with environment variable").Envar("DOCKER_TOKEN").String()
dockerExcludePaths = dockerScan.Flag("exclude-paths", "Comma separated list of paths to exclude from scan").String()
dockerScanNamespace = dockerScan.Flag("namespace", "Docker namespace (organization or user). For non-Docker Hub registries, include the registry address as well (e.g., ghcr.io/namespace or quay.io/namespace).").String()
dockerScanRegistryToken = dockerScan.Flag("registry-token", "Optional Docker registry access token. Provide this if you want to include private images within the specified namespace.").String()
travisCiScan = cli.Command("travisci", "Scan TravisCI")
travisCiScanToken = travisCiScan.Flag("token", "TravisCI token. Can also be provided with environment variable").Envar("TRAVISCI_TOKEN").Required().String()
// Postman is hidden for now until we get more feedback from the community.
postmanScan = cli.Command("postman", "Scan Postman")
postmanToken = postmanScan.Flag("token", "Postman token. Can also be provided with environment variable").Envar("POSTMAN_TOKEN").String()
postmanWorkspaces = postmanScan.Flag("workspace", "Postman workspace to scan. You can repeat this flag. Deprecated flag.").Hidden().Strings()
postmanWorkspaceIDs = postmanScan.Flag("workspace-id", "Postman workspace ID to scan. You can repeat this flag.").Strings()
postmanCollections = postmanScan.Flag("collection", "Postman collection to scan. You can repeat this flag. Deprecated flag.").Hidden().Strings()
postmanCollectionIDs = postmanScan.Flag("collection-id", "Postman collection ID to scan. You can repeat this flag.").Strings()
postmanEnvironments = postmanScan.Flag("environment", "Postman environment to scan. You can repeat this flag.").Strings()
postmanIncludeCollections = postmanScan.Flag("include-collections", "Collections to include in scan. You can repeat this flag. Deprecated flag.").Hidden().Strings()
postmanIncludeCollectionIDs = postmanScan.Flag("include-collection-id", "Collection ID to include in scan. You can repeat this flag.").Strings()
postmanIncludeEnvironments = postmanScan.Flag("include-environments", "Environments to include in scan. You can repeat this flag.").Strings()
postmanExcludeCollections = postmanScan.Flag("exclude-collections", "Collections to exclude from scan. You can repeat this flag. Deprecated flag.").Hidden().Strings()
postmanExcludeCollectionIDs = postmanScan.Flag("exclude-collection-id", "Collection ID to exclude from scan. You can repeat this flag.").Strings()
postmanExcludeEnvironments = postmanScan.Flag("exclude-environments", "Environments to exclude from scan. You can repeat this flag.").Strings()
postmanWorkspacePaths = postmanScan.Flag("workspace-paths", "Path to Postman workspaces.").Strings()
postmanCollectionPaths = postmanScan.Flag("collection-paths", "Path to Postman collections.").Strings()
postmanEnvironmentPaths = postmanScan.Flag("environment-paths", "Path to Postman environments.").Strings()
elasticsearchScan = cli.Command("elasticsearch", "Scan Elasticsearch")
elasticsearchNodes = elasticsearchScan.Flag("nodes", "Elasticsearch nodes").Envar("ELASTICSEARCH_NODES").Strings()
elasticsearchUsername = elasticsearchScan.Flag("username", "Elasticsearch username").Envar("ELASTICSEARCH_USERNAME").String()
elasticsearchPassword = elasticsearchScan.Flag("password", "Elasticsearch password").Envar("ELASTICSEARCH_PASSWORD").String()
elasticsearchServiceToken = elasticsearchScan.Flag("service-token", "Elasticsearch service token").Envar("ELASTICSEARCH_SERVICE_TOKEN").String()
elasticsearchCloudId = elasticsearchScan.Flag("cloud-id", "Elasticsearch cloud ID. Can also be provided with environment variable").Envar("ELASTICSEARCH_CLOUD_ID").String()
elasticsearchAPIKey = elasticsearchScan.Flag("api-key", "Elasticsearch API key. Can also be provided with environment variable").Envar("ELASTICSEARCH_API_KEY").String()
elasticsearchIndexPattern = elasticsearchScan.Flag("index-pattern", "Filters the indices to search").Default("*").Envar("ELASTICSEARCH_INDEX_PATTERN").String()
elasticsearchQueryJSON = elasticsearchScan.Flag("query-json", "Filters the documents to search").Envar("ELASTICSEARCH_QUERY_JSON").String()
elasticsearchSinceTimestamp = elasticsearchScan.Flag("since-timestamp", "Filters the documents to search to those created since this timestamp; overrides any timestamp from --query-json").Envar("ELASTICSEARCH_SINCE_TIMESTAMP").String()
elasticsearchBestEffortScan = elasticsearchScan.Flag("best-effort-scan", "Attempts to continuously scan a cluster").Envar("ELASTICSEARCH_BEST_EFFORT_SCAN").Bool()
jenkinsScan = cli.Command("jenkins", "Scan Jenkins")
jenkinsURL = jenkinsScan.Flag("url", "Jenkins URL").Envar("JENKINS_URL").Required().String()
jenkinsUsername = jenkinsScan.Flag("username", "Jenkins username").Envar("JENKINS_USERNAME").String()
jenkinsPassword = jenkinsScan.Flag("password", "Jenkins password").Envar("JENKINS_PASSWORD").String()
jenkinsInsecureSkipVerifyTLS = jenkinsScan.Flag("insecure-skip-verify-tls", "Skip TLS verification").Envar("JENKINS_INSECURE_SKIP_VERIFY_TLS").Bool()
huggingfaceScan = cli.Command("huggingface", "Find credentials in HuggingFace datasets, models, spaces and buckets.")
huggingfaceEndpoint = huggingfaceScan.Flag("endpoint", "HuggingFace endpoint.").Default("https://huggingface.co").String()
huggingfaceModels = huggingfaceScan.Flag("model", "HuggingFace model to scan. You can repeat this flag. Example: 'username/model'").Strings()
huggingfaceSpaces = huggingfaceScan.Flag("space", "HuggingFace space to scan. You can repeat this flag. Example: 'username/space'").Strings()
huggingfaceDatasets = huggingfaceScan.Flag("dataset", "HuggingFace dataset to scan. You can repeat this flag. Example: 'username/dataset'").Strings()
huggingfaceBuckets = huggingfaceScan.Flag("bucket", "HuggingFace bucket to scan. You can repeat this flag. Example: 'username/bucket'").Strings()
huggingfaceOrgs = huggingfaceScan.Flag("org", `HuggingFace organization to scan. You can repeat this flag. Example: "trufflesecurity"`).Strings()
huggingfaceUsers = huggingfaceScan.Flag("user", `HuggingFace user to scan. You can repeat this flag. Example: "trufflesecurity"`).Strings()
huggingfaceToken = huggingfaceScan.Flag("token", "HuggingFace token. Can be provided with environment variable HUGGINGFACE_TOKEN.").Envar("HUGGINGFACE_TOKEN").String()
huggingfaceIncludeModels = huggingfaceScan.Flag("include-models", "Models to include in scan. You can repeat this flag. Must use HuggingFace model full name. Example: 'username/model' (Only used with --user or --org)").Strings()
huggingfaceIncludeSpaces = huggingfaceScan.Flag("include-spaces", "Spaces to include in scan. You can repeat this flag. Must use HuggingFace space full name. Example: 'username/space' (Only used with --user or --org)").Strings()
huggingfaceIncludeDatasets = huggingfaceScan.Flag("include-datasets", "Datasets to include in scan. You can repeat this flag. Must use HuggingFace dataset full name. Example: 'username/dataset' (Only used with --user or --org)").Strings()
huggingfaceIgnoreModels = huggingfaceScan.Flag("ignore-models", "Models to ignore in scan. You can repeat this flag. Must use HuggingFace model full name. Example: 'username/model' (Only used with --user or --org)").Strings()
huggingfaceIgnoreSpaces = huggingfaceScan.Flag("ignore-spaces", "Spaces to ignore in scan. You can repeat this flag. Must use HuggingFace space full name. Example: 'username/space' (Only used with --user or --org)").Strings()
huggingfaceIgnoreDatasets = huggingfaceScan.Flag("ignore-datasets", "Datasets to ignore in scan. You can repeat this flag. Must use HuggingFace dataset full name. Example: 'username/dataset' (Only used with --user or --org)").Strings()
huggingfaceIncludeBuckets = huggingfaceScan.Flag("include-buckets", "Buckets to include in scan. You can repeat this flag. Must use HuggingFace bucket full name. Example: 'username/bucket' (Only used with --user or --org)").Strings()
huggingfaceIgnoreBuckets = huggingfaceScan.Flag("ignore-buckets", "Buckets to ignore in scan. You can repeat this flag. Must use HuggingFace bucket full name. Example: 'username/bucket' (Only used with --user or --org)").Strings()
huggingfaceSkipAllModels = huggingfaceScan.Flag("skip-all-models", "Skip all model scans. (Only used with --user or --org)").Bool()
huggingfaceSkipAllSpaces = huggingfaceScan.Flag("skip-all-spaces", "Skip all space scans. (Only used with --user or --org)").Bool()
huggingfaceSkipAllDatasets = huggingfaceScan.Flag("skip-all-datasets", "Skip all dataset scans. (Only used with --user or --org)").Bool()
huggingfaceSkipAllBuckets = huggingfaceScan.Flag("skip-all-buckets", "Skip all bucket scans. (Only used with --user or --org)").Bool()
huggingfaceIncludeDiscussions = huggingfaceScan.Flag("include-discussions", "Include discussions in scan.").Bool()
huggingfaceIncludePrs = huggingfaceScan.Flag("include-prs", "Include pull requests in scan.").Bool()
stdinInputScan = cli.Command("stdin", "Find credentials from stdin.")
multiScanScan = cli.Command("multi-scan", "Find credentials in multiple sources defined in configuration.")
jsonEnumeratorScan = cli.Command("json-enumerator", "Find credentials from a JSON enumerator input.")
jsonEnumeratorPaths = jsonEnumeratorScan.Arg("path", "Path to JSON enumerator file to scan.").Strings()
analyzeCmd = analyzer.Command(cli)
usingTUI = false
)
// expandTilde replaces a leading ~ in each argument with the user's home
// directory. This compensates for bypassing the shell (which would normally
// perform this expansion) while still avoiding shell metacharacter injection.
func expandTilde(args []string) []string {
home, err := os.UserHomeDir()
if err != nil || home == "" {
return args
}
out := make([]string, len(args))
for i, arg := range args {
switch {
case arg == "~":
out[i] = home
case strings.HasPrefix(arg, "~/"):
out[i] = home + arg[1:]
default:
out[i] = arg
}
}
return out
}
func init() {
_, _ = maxprocs.Set()
for i, arg := range os.Args {
if strings.HasPrefix(arg, "--") {
split := strings.SplitN(arg, "=", 2)
split[0] = strings.ReplaceAll(split[0], "_", "-")
os.Args[i] = strings.Join(split, "=")
}
}
cli.Version("trufflehog " + version.BuildVersion)
// Support -h for help and write it to stdout.
cli.HelpFlag.Short('h')
cli.UsageWriter(os.Stdout)
registerManPageFlag(cli)
// Check if the TUI environment variable is set.
if ok, err := strconv.ParseBool(os.Getenv("TUI_PARENT")); err == nil {
usingTUI = ok
}
if isatty.IsTerminal(os.Stdout.Fd()) && (len(os.Args) <= 1 || os.Args[1] == analyzeCmd.FullCommand()) {
args := tui.Run(os.Args[1:])
if len(args) == 0 {
os.Exit(0)
}
// The TUI passes user input as literal strings. Since we no longer
// invoke a shell, we must expand ~ ourselves so paths like ~/foo
// resolve correctly.
args = expandTilde(args)
binary, err := exec.LookPath(os.Args[0])
if err == nil {
// On success, this call will never return. On failure, fallthrough
// to overwriting os.Args.
execArgs := append([]string{binary}, args...)
_ = syscall.Exec(binary, execArgs, append(os.Environ(), "TUI_PARENT=true"))
}
// Overwrite the Args slice so overseer works properly.
os.Args = os.Args[:1]
os.Args = append(os.Args, args...)
usingTUI = true
}
cmd = kingpin.MustParse(cli.Parse(os.Args[1:]))
// Configure logging.
switch {
case *trace:
log.SetLevel(5)
case *debug:
log.SetLevel(2)
default:
l := int8(*logLevel)
if l < -1 || l > 5 {
fmt.Fprintf(os.Stderr, "invalid log level: %d\n", *logLevel)
os.Exit(1)
}
if l == -1 {
// Zap uses "5" as the value for fatal.
// We need to pass in "-5" because `SetLevel` passes the negation.
log.SetLevel(-5)
} else {
log.SetLevel(l)
}
}
if *noColor || *noColour {
color.NoColor = true // disables colorized output
}
}
// syncLogs flushes logs when the program exits.
func syncLogs(syncFn func() error) {
if syncFn != nil {
_ = syncFn()
}
}
func main() {
// setup logger
logFormat := log.WithConsoleSink
if *jsonOut {
logFormat = log.WithJSONSink
}
logger, sync := log.New("trufflehog", logFormat(os.Stderr, log.WithGlobalRedaction()))
// make it the default logger for contexts
context.SetDefaultLogger(logger)
if *localDev {
run(overseer.State{}, sync)
os.Exit(0)
}
logFatal := logFatalFunc(logger, sync)
updateCfg := overseer.Config{
Program: func(s overseer.State) {
run(s, sync)
},
Debug: *debug,
RestartSignal: syscall.SIGTERM,
// TODO: Eventually add a PreUpgrade func for signature check w/ x509 PKCS1v15
// PreUpgrade: checkUpdateSignature(binaryPath string),
}
if !*noUpdate {
topLevelCmd, _, _ := strings.Cut(cmd, " ")
updateCfg.Fetcher = updater.Fetcher(topLevelCmd, usingTUI)
}
if version.BuildVersion == "dev" {
updateCfg.Fetcher = nil
}
err := overseer.RunErr(updateCfg)
if err != nil {
logFatal(err, "error occurred with trufflehog updater 🐷")
}
}
func run(state overseer.State, logSync func() error) {
ctx, cancel := context.WithCancelCause(context.Background())
defer cancel(nil)
defer syncLogs(logSync)
go func() {
if err := cleantemp.CleanTempArtifacts(ctx); err != nil {
ctx.Logger().Error(err, "error cleaning temporary artifacts")
}
}()
logger := ctx.Logger()
logFatal := logFatalFunc(logger, logSync)
killSignal := make(chan os.Signal, 1)
signal.Notify(killSignal, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT)
go func() {
<-killSignal
logger.Info("Received signal, shutting down.")
cancel(fmt.Errorf("canceling context due to signal"))
if err := cleantemp.CleanTempArtifacts(ctx); err != nil {
logger.Error(err, "error cleaning temporary artifacts")
} else {
logger.Info("cleaned temporary artifacts")
}
syncLogs(logSync)
os.Exit(0)
}()
logger.V(2).Info(fmt.Sprintf("trufflehog %s", version.BuildVersion))
if *githubScanToken != "" {
// NOTE: this kludge is here to do an authenticated shallow commit
// TODO: refactor to better pass credentials
_ = os.Setenv("GITHUB_TOKEN", *githubScanToken)
}
if *concurrency <= 0 {
*concurrency = runtime.NumCPU()
}
// When setting a base commit, chunks must be scanned in order.
if *gitScanSinceCommit != "" {
*concurrency = 1
}
if *profile {
runtime.SetBlockProfileRate(1)
runtime.SetMutexProfileFraction(-1)
go func() {
router := http.NewServeMux()
router.Handle("/debug/pprof/", http.DefaultServeMux)
router.Handle("/debug/fgprof", fgprof.Handler())
logger.Info("starting pprof and fgprof server on :18066 /debug/pprof and /debug/fgprof")
if err := http.ListenAndServe(":18066", router); err != nil {
logger.Error(err, "error serving pprof and fgprof")
}
}()
}
// Set feature configurations from CLI flags
if *forceSkipBinaries {
feature.ForceSkipBinaries.Store(true)
}
if *forceSkipArchives {
feature.ForceSkipArchives.Store(true)
}
if gitCloneTimeout != nil {
feature.GitCloneTimeoutDuration.Store(int64(*gitCloneTimeout))
}
if *gitLowMemoryScan {
feature.UseGitLowMemoryScan.Store(true)
}
if *skipAdditionalRefs {
feature.SkipAdditionalRefs.Store(true)
}
if *userAgentSuffix != "" {
feature.UserAgentSuffix.Store(*userAgentSuffix)
}
if *dropUnverifiedJWTResults {
feature.DropUnverifiedJWTResults.Store(true)
}
// OSS Default APK handling on
feature.EnableAPKHandler.Store(true)
// OSS Default Use Git Mirror on
feature.UseGitMirror.Store(true)
// OSS Default simplified gitlab enumeration
feature.UseSimplifiedGitlabEnumeration.Store(true)
feature.GitlabProjectsPerPage.Store(100)
// OSS Default using github graphql api for issues, pr's and comments
feature.UseGithubGraphQLAPI.Store(false)
// OSS Default Use HTML Decoder on
feature.HTMLDecoderEnabled.Store(true)
// New detector flags
feature.PineconeDetectorEnabled.Store(true)
feature.CloudinaryDetectorEnabled.Store(true)
feature.GitLabOAuthDetectorEnabled.Store(true)
feature.FigmaV3DetectorEnabled.Store(true)
feature.SonarCloudV2DetectorEnabled.Store(true)
feature.EnigmaDetectorEnabled.Store(true)
feature.DatadogApiKeyDetectorEnabled.Store(true)
feature.TlyDetectorEnabled.Store(true)
feature.WitDetectorEnabled.Store(true)
feature.RevDetectorEnabled.Store(true)
feature.UserDetectorEnabled.Store(true)
feature.BraintrustDetectorEnabled.Store(true)
feature.PgAnalyzeReadKeyDetectorEnabled.Store(true)
feature.RedHatPyxisDetectorEnabled.Store(true)
feature.OctopusDeployDetectorEnabled.Store(true)
feature.OpenRouterDetectorEnabled.Store(true)
feature.NewRelicInsightsInsertKeyDetectorEnabled.Store(true)
feature.DuffelTokenDetectorEnabled.Store(true)
feature.ShippoDetectorEnabled.Store(true)
feature.IPInfoDetectorEnabled.Store(true)
feature.LobDetectorEnabled.Store(true)
feature.HashiCorpVaultBatchTokenDetectorEnabled.Store(true)
feature.HashiCorpVaultTokenDetectorEnabled.Store(true)
feature.CloudflareApiTokenV2DetectorEnabled.Store(true)
feature.CloudflareGlobalApiKeyV2DetectorEnabled.Store(true)
feature.DuoDetectorEnabled.Store(true)
feature.NewRelicLicenseKeyDetectorEnabled.Store(true)
feature.NewRelicBrowserKeyDetectorEnabled.Store(true)
feature.NewRelicUserKeyDetectorEnabled.Store(true)
feature.NewRelicInsightsQueryKeyDetectorEnabled.Store(true)
feature.NewRelicMobileAppTokenDetectorEnabled.Store(true)
feature.MSTeamsWebhookV2DetectorEnabled.Store(true)
feature.SolarwindsDetectorEnabled.Store(true)
feature.ResendDetectorEnabled.Store(true)
feature.WeightsAndBiasesV2DetectorEnabled.Store(true)
feature.HumioAPITokenDetectorEnabled.Store(true)
conf := &config.Config{}
if *configFilename != "" {
var err error
conf, err = config.Read(*configFilename)
if err != nil {
logFatal(err, "error parsing the provided configuration file")
}
}
if *detectorTimeout != 0 {
logger.Info("Setting detector timeout", "timeout", detectorTimeout.String())
engine.SetDetectorTimeout(*detectorTimeout)
detectors.OverrideDetectorTimeout(*detectorTimeout)
}
if *archiveMaxSize != 0 {
handlers.SetArchiveMaxSize(int(*archiveMaxSize))
}
if *archiveMaxDepth != 0 {
handlers.SetArchiveMaxDepth(*archiveMaxDepth)
}
if *archiveTimeout != 0 {
handlers.SetArchiveMaxTimeout(*archiveTimeout)
}
// Set how the engine will print its results.
var printer engine.Printer
switch {
case *jsonLegacy:
printer = new(output.LegacyJSONPrinter)
case *jsonOut:
printer = new(output.JSONPrinter)
case *gitHubActionsFormat:
printer = new(output.GitHubActionsPrinter)
case *sarifOut:
printer = new(output.SarifPrinter)
default:
printer = new(output.PlainPrinter)
}
if !*jsonLegacy && !*jsonOut && !*sarifOut {
fmt.Fprintf(os.Stderr, "🐷🔑🐷 TruffleHog. Unearth your secrets. 🐷🔑🐷\n\n")
}
// Parse --results flag.
if *onlyVerified {
r := "verified"
results = &r
}
parsedResults, err := parseResults(results)
if err != nil {
logFatal(err, "failed to configure results flag")
}
verificationCacheMetrics := verificationcache.InMemoryMetrics{}
engConf := engine.Config{
Concurrency: *concurrency,
ConfiguredSources: conf.Sources,
// The engine must always be configured with the list of
// default detectors, which can be further filtered by the
// user. The filters are applied by the engine and are only
// subtractive.
Detectors: append(defaults.DefaultDetectors(), conf.Detectors...),
Verify: !*noVerification,
IncludeDetectors: *includeDetectors,
ExcludeDetectors: *excludeDetectors,
CustomVerifiersOnly: *customVerifiersOnly,
VerifierEndpoints: *verifiers,
Dispatcher: engine.NewPrinterDispatcher(printer),
FilterUnverified: *filterUnverified,
FilterEntropy: *filterEntropy,
VerificationOverlap: *allowVerificationOverlap,
Results: parsedResults,
PrintAvgDetectorTime: *printAvgDetectorTime,
ShouldScanEntireChunk: *scanEntireChunk,
MaxDecodeDepth: *maxDecodeDepth,
VerificationCacheMetrics: &verificationCacheMetrics,
}
if !*noVerificationCache {
engConf.VerificationResultCache = simple.NewCache[detectors.Result]()
}
// Check that there are no sources defined for non-scan subcommands. If
// there are, return an error as it is ambiguous what the user is
// trying to do.
if cmd != multiScanScan.FullCommand() && len(conf.Sources) > 0 {
logFatal(
fmt.Errorf("ambiguous configuration"),
"sources should only be defined in configuration for the 'multi-scan' command",
)
}
if *compareDetectionStrategies {
if err := compareScans(ctx, cmd, engConf); err != nil {
logFatal(err, "error comparing detection strategies")
}
return
}
metrics, err := runSingleScan(ctx, cmd, engConf)
if err != nil {
logFatal(err, "error running scan")
}
// SARIF can't be streamed like the other output formats: it's a single JSON document
// wrapping every result, so it's buffered by the printer and written out here, once
// scanning has fully finished.
if sarifPrinter, ok := printer.(*output.SarifPrinter); ok {
if err := sarifPrinter.Flush(os.Stdout); err != nil {
logFatal(err, "error writing SARIF output")
}
}
verificationCacheMetricsSnapshot := struct {
Hits int32
Misses int32
HitsWasted int32
AttemptsSaved int32
VerificationTimeSpentMS int64
}{
Hits: verificationCacheMetrics.ResultCacheHits.Load(),
Misses: verificationCacheMetrics.ResultCacheMisses.Load(),
HitsWasted: verificationCacheMetrics.ResultCacheHitsWasted.Load(),
AttemptsSaved: verificationCacheMetrics.CredentialVerificationsSaved.Load(),
VerificationTimeSpentMS: verificationCacheMetrics.FromDataVerifyTimeSpentMS.Load(),
}
// Print results.
logger.Info("finished scanning",
"chunks", metrics.ChunksScanned,
"bytes", metrics.BytesScanned,
"verified_secrets", metrics.VerifiedSecretsFound,
"unverified_secrets", metrics.UnverifiedSecretsFound,
"scan_duration", metrics.ScanDuration.String(),
"trufflehog_version", version.BuildVersion,
"verification_caching", verificationCacheMetricsSnapshot,
)
if metrics.hasFoundResults && *fail {
logger.V(2).Info("exiting with code 183 because results were found")
syncLogs(logSync)
os.Exit(183)
}
}
func compareScans(ctx context.Context, cmd string, cfg engine.Config) error {
var (
entireMetrics metrics
maxLengthMetrics metrics
err error
)
var wg sync.WaitGroup
wg.Add(1)
go func() {
defer wg.Done()
// Run scan with entire chunk span calculator.
cfg.ShouldScanEntireChunk = true
entireMetrics, err = runSingleScan(ctx, cmd, cfg)
if err != nil {
ctx.Logger().Error(err, "error running scan with entire chunk span calculator")
}
}()
// Run scan with max-length span calculator.
maxLengthMetrics, err = runSingleScan(ctx, cmd, cfg)
if err != nil {
return fmt.Errorf("error running scan with custom span calculator: %v", err)
}
wg.Wait()
return compareMetrics(maxLengthMetrics.Metrics, entireMetrics.Metrics)
}
func compareMetrics(customMetrics, entireMetrics engine.Metrics) error {
fmt.Printf("Comparison of scan results: \n")
fmt.Printf("Custom span - Chunks: %d, Bytes: %d, Verified Secrets: %d, Unverified Secrets: %d, Duration: %s\n",
customMetrics.ChunksScanned, customMetrics.BytesScanned, customMetrics.VerifiedSecretsFound, customMetrics.UnverifiedSecretsFound, customMetrics.ScanDuration.String())
fmt.Printf("Entire chunk - Chunks: %d, Bytes: %d, Verified Secrets: %d, Unverified Secrets: %d, Duration: %s\n",
entireMetrics.ChunksScanned, entireMetrics.BytesScanned, entireMetrics.VerifiedSecretsFound, entireMetrics.UnverifiedSecretsFound, entireMetrics.ScanDuration.String())
// Check for differences in scan metrics.
if customMetrics.ChunksScanned != entireMetrics.ChunksScanned ||
customMetrics.BytesScanned != entireMetrics.BytesScanned ||
customMetrics.VerifiedSecretsFound != entireMetrics.VerifiedSecretsFound {
return fmt.Errorf("scan metrics do not match")
}
return nil
}
type metrics struct {
engine.Metrics
hasFoundResults bool
}
func runSingleScan(ctx context.Context, cmd string, cfg engine.Config) (metrics, error) {
var scanMetrics metrics
// Setup job report writer if provided
var jobReportWriter io.WriteCloser
if *jobReportFile != nil {
jobReportWriter = *jobReportFile
}
handleFinishedMetrics := func(ctx context.Context, finishedMetrics <-chan sources.UnitMetrics, jobReportWriter io.WriteCloser) {
go func() {
defer func() {
_ = jobReportWriter.Close()
if namer, ok := jobReportWriter.(interface{ Name() string }); ok {
ctx.Logger().Info("report written", "path", namer.Name())
} else {
ctx.Logger().Info("report written")
}
}()
for metrics := range finishedMetrics {
metrics.Errors = common.ExportErrors(metrics.Errors...)
details, err := json.Marshal(map[string]any{
"version": 1,
"data": metrics,
})
if err != nil {
ctx.Logger().Error(err, "error marshalling job details")
continue
}
if _, err := jobReportWriter.Write(append(details, '\n')); err != nil {
ctx.Logger().Error(err, "error writing to file")
}
}
}()
}
const defaultOutputBufferSize = 64
opts := []func(*sources.SourceManager){
sources.WithConcurrentSources(cfg.Concurrency),
sources.WithConcurrentUnits(cfg.Concurrency),
sources.WithSourceUnits(),
sources.WithBufferedOutput(defaultOutputBufferSize),
}
if jobReportWriter != nil {
unitHook, finishedMetrics := sources.NewUnitHook(ctx)
opts = append(opts, sources.WithReportHook(unitHook))
handleFinishedMetrics(ctx, finishedMetrics, jobReportWriter)
}
cfg.SourceManager = sources.NewManager(opts...)
eng, err := engine.NewEngine(ctx, &cfg)
if err != nil {
return scanMetrics, fmt.Errorf("error initializing engine: %v", err)
}
eng.Start(ctx)
persistGitRepo := *gitNoCleanup || *githubNoCleanup || *gitlabNoCleanup
gitCloneTempPath := ""
defer func() {
// Clean up temporary artifacts.
if err := cleantemp.CleanTempArtifacts(ctx); err != nil {
ctx.Logger().Error(err, "error cleaning temp artifacts")
}
if *jsonLegacy {
// If JSON legacy is enabled, that means the cloned repos are not deleted yet
// because they were needed for outputting legacy JSON.
// We only clean them up here if the user did not request to persist them.
if !persistGitRepo {
if err := cleantemp.CleanTempDirsForLegacyJSON(gitCloneTempPath); err != nil {
ctx.Logger().Error(err, "error cleaning temp artifacts for legacy JSON")
}
}
}
}()
var refs []sources.JobProgressRef
switch cmd {
case gitScan.FullCommand():
if err := validateClonePath(*gitClonePath, *gitNoCleanup); err != nil {
return scanMetrics, err
}
gitCfg := sources.GitConfig{
URI: *gitScanURI,
IncludePathsFile: *gitScanIncludePaths,
ExcludePathsFile: *gitScanExcludePaths,
HeadRef: *gitScanBranch,
BaseRef: *gitScanSinceCommit,
MaxDepth: *gitScanMaxDepth,
Bare: *gitScanBare,
ExcludeGlobs: *gitScanExcludeGlobs,
ClonePath: *gitClonePath,
NoCleanup: *gitNoCleanup,
PrintLegacyJSON: *jsonLegacy,
TrustLocalGitConfig: *gitTrustLocalGitConfig,
}
// detect if trufflehog is running git source as a pre-commit hook
if isPreCommitHook() {
ctx.Logger().Info("Running as a pre-commit hook, overriding default flags for hook context")
// Override git configuration for pre-commit hook context
gitCfg.TrustLocalGitConfig = true
gitCfg.BaseRef = "HEAD" // Only scan staged changes
// Override result filters for pre-commit hook context
// In hook mode, we only want to show verified secrets and unknown findings
*results = "verified,unknown"
// Override failure behavior for pre-commit hook context
// In hook mode, we want to fail the commit if any secrets are found
*fail = true
}
if ref, err := eng.ScanGit(ctx, gitCfg); err != nil {
return scanMetrics, fmt.Errorf("failed to scan Git: %v", err)
} else {
refs = []sources.JobProgressRef{ref}
}
case githubScan.FullCommand():
gitCloneTempPath = *githubClonePath
filter, err := common.FilterFromFiles(*githubScanIncludePaths, *githubScanExcludePaths)
if err != nil {
return scanMetrics, fmt.Errorf("could not create filter: %v", err)
}
if len(*githubScanOrgs) == 0 && len(*githubScanRepos) == 0 {
return scanMetrics, fmt.Errorf("invalid config: you must specify at least one organization or repository")
}
if len(*githubScanOrgs) > 0 && len(*githubScanRepos) > 0 {
return scanMetrics, fmt.Errorf("invalid config: you cannot specify both organizations and repositories at the same time")
}
// --include-repos/--exclude-repos only filter repos enumerated from an org/user scan;
// explicit --repo entries bypass that filtering step entirely (see enumerate() in
// pkg/sources/github/github.go), so combining them with --repo would silently no-op
// the filters instead of erroring.
if len(*githubScanRepos) > 0 && (len(*githubIncludeRepos) > 0 || len(*githubExcludeRepos) > 0) {
return scanMetrics, fmt.Errorf("invalid config: --include-repos and --exclude-repos only apply to organization or user scans and cannot be used with --repo")
}
if err := validateClonePath(*githubClonePath, *githubNoCleanup); err != nil {
return scanMetrics, err
}
cfg := sources.GithubConfig{
Endpoint: *githubScanEndpoint,
Token: *githubScanToken,
IncludeForks: *githubIncludeForks,
IncludeMembers: *githubIncludeMembers,
IncludeWikis: *githubIncludeWikis,
Concurrency: *concurrency,
ExcludeRepos: *githubExcludeRepos,
IncludeRepos: *githubIncludeRepos,
Repos: *githubScanRepos,
Orgs: *githubScanOrgs,
IncludeIssueComments: *githubScanIssueComments,
IncludePullRequestComments: *githubScanPRComments,
IncludeGistComments: *githubScanGistComments,
CommentsTimeframeDays: *githubCommentsTimeframeDays,
Filter: filter,
AuthInUrl: *githubAuthInUrl,
ClonePath: *githubClonePath,
NoCleanup: *githubNoCleanup,
IgnoreGists: *githubIgnoreGists,
ExcludeArchived: *githubExcludeArchived,
PrintLegacyJSON: *jsonLegacy,
}
if ref, err := eng.ScanGitHub(ctx, cfg); err != nil {
return scanMetrics, fmt.Errorf("failed to scan Github: %v", err)
} else {
refs = []sources.JobProgressRef{ref}
}
case githubExperimentalScan.FullCommand():
cfg := sources.GitHubExperimentalConfig{
Token: *githubExperimentalToken,
Repository: *githubExperimentalRepo,
ObjectDiscovery: *githubExperimentalObjectDiscovery,
CollisionThreshold: *githubExperimentalCollisionThreshold,
DeleteCachedData: *githubExperimentalDeleteCache,
}
if ref, err := eng.ScanGitHubExperimental(ctx, cfg); err != nil {
return scanMetrics, fmt.Errorf("failed to scan using Github Experimental: %v", err)
} else {
refs = []sources.JobProgressRef{ref}
}
case gitlabScan.FullCommand():
gitCloneTempPath = *gitlabClonePath
filter, err := common.FilterFromFiles(*gitlabScanIncludePaths, *gitlabScanExcludePaths)
if err != nil {
return scanMetrics, fmt.Errorf("could not create filter: %v", err)
}
if len(*gitlabScanRepos) > 0 && len(*gitlabScanGroupIds) > 0 {
return scanMetrics, fmt.Errorf("invalid config: you cannot specify both repositories and groups at the same time")
}
if err := validateClonePath(*gitlabClonePath, *gitlabNoCleanup); err != nil {
return scanMetrics, err
}
cfg := sources.GitlabConfig{
Endpoint: *gitlabScanEndpoint,
Token: *gitlabScanToken,
Repos: *gitlabScanRepos,
GroupIds: *gitlabScanGroupIds,
IncludeRepos: *gitlabScanIncludeRepos,
ExcludeRepos: *gitlabScanExcludeRepos,
Filter: filter,
AuthInUrl: *gitlabAuthInUrl,
ClonePath: *gitlabClonePath,
NoCleanup: *gitlabNoCleanup,
PrintLegacyJSON: *jsonLegacy,
}
if ref, err := eng.ScanGitLab(ctx, cfg); err != nil {
return scanMetrics, fmt.Errorf("failed to scan GitLab: %v", err)
} else {
refs = []sources.JobProgressRef{ref}
}
case filesystemScan.FullCommand():
if len(*filesystemDirectories) > 0 {
ctx.Logger().Info("--directory flag is deprecated, please pass directories as arguments")
}
paths := make([]string, 0, len(*filesystemPaths)+len(*filesystemDirectories))
paths = append(paths, *filesystemPaths...)
paths = append(paths, *filesystemDirectories...)
cfg := sources.FilesystemConfig{
Paths: paths,
IncludePathsFile: *filesystemScanIncludePaths,
ExcludePathsFile: *filesystemScanExcludePaths,
MaxSymlinkDepth: *filesystemScanMaxSymlinkDepth,
}
if ref, err := eng.ScanFileSystem(ctx, cfg); err != nil {
return scanMetrics, fmt.Errorf("failed to scan filesystem: %v", err)
} else {
refs = []sources.JobProgressRef{ref}
}
case s3Scan.FullCommand():
cfg := sources.S3Config{
Key: *s3ScanKey,
Secret: *s3ScanSecret,
SessionToken: *s3ScanSessionToken,
Buckets: *s3ScanBuckets,
IgnoreBuckets: *s3ScanIgnoreBuckets,
Roles: *s3ScanRoleArns,
CloudCred: *s3ScanCloudEnv,
MaxObjectSize: int64(*s3ScanMaxObjectSize),
Endpoint: *s3ScanEndpoint,
Region: *s3ScanRegion,
IncludePrefixes: *s3ScanIncludePrefixes,
ExcludePrefixes: *s3ScanExcludePrefixes,
IncludeExtensions: *s3ScanIncludeExtensions,
ExcludeExtensions: *s3ScanExcludeExtensions,
}
if ref, err := eng.ScanS3(ctx, cfg); err != nil {
return scanMetrics, fmt.Errorf("failed to scan S3: %v", err)
} else {
refs = []sources.JobProgressRef{ref}
}
case syslogScan.FullCommand():
cfg := sources.SyslogConfig{
Address: *syslogAddress,
Format: *syslogFormat,
Protocol: *syslogProtocol,
CertPath: *syslogTLSCert,
KeyPath: *syslogTLSKey,
Concurrency: *concurrency,
}
if ref, err := eng.ScanSyslog(ctx, cfg); err != nil {
return scanMetrics, fmt.Errorf("failed to scan syslog: %v", err)
} else {
refs = []sources.JobProgressRef{ref}
}
case circleCiScan.FullCommand():
if ref, err := eng.ScanCircleCI(ctx, *circleCiScanToken); err != nil {
return scanMetrics, fmt.Errorf("failed to scan CircleCI: %v", err)
} else {
refs = []sources.JobProgressRef{ref}
}
case travisCiScan.FullCommand():
if ref, err := eng.ScanTravisCI(ctx, *travisCiScanToken); err != nil {
return scanMetrics, fmt.Errorf("failed to scan TravisCI: %v", err)
} else {
refs = []sources.JobProgressRef{ref}
}
case gcsScan.FullCommand():
cfg := sources.GCSConfig{
ProjectID: *gcsProjectID,
CloudCred: *gcsCloudEnv,
ServiceAccount: *gcsServiceAccount,
WithoutAuth: *gcsWithoutAuth,
ApiKey: *gcsAPIKey,
IncludeBuckets: commaSeparatedToSlice(*gcsIncludeBuckets),
ExcludeBuckets: commaSeparatedToSlice(*gcsExcludeBuckets),
IncludeObjects: commaSeparatedToSlice(*gcsIncludeObjects),
ExcludeObjects: commaSeparatedToSlice(*gcsExcludeObjects),
Concurrency: *concurrency,
MaxObjectSize: int64(*gcsMaxObjectSize),
}
if ref, err := eng.ScanGCS(ctx, cfg); err != nil {
return scanMetrics, fmt.Errorf("failed to scan GCS: %v", err)
} else {
refs = []sources.JobProgressRef{ref}
}
case dockerScan.FullCommand():
if *dockerScanImages != nil && *dockerScanNamespace != "" {
return scanMetrics, fmt.Errorf("invalid config: you cannot specify both images and namespace at the same time")
}
if *dockerScanImages == nil && *dockerScanNamespace == "" {
return scanMetrics, fmt.Errorf("invalid config: both images and namespace cannot be empty; one is required")
}
if *dockerScanRegistryToken != "" && *dockerScanNamespace == "" {
return scanMetrics, fmt.Errorf("invalid config: registry token can only be used with registry namespace")
}
cfg := sources.DockerConfig{
BearerToken: *dockerScanToken,
Images: *dockerScanImages,
UseDockerKeychain: *dockerScanToken == "",
ExcludePaths: strings.Split(*dockerExcludePaths, ","),
Namespace: *dockerScanNamespace,
RegistryToken: *dockerScanRegistryToken,
}
if ref, err := eng.ScanDocker(ctx, cfg); err != nil {
return scanMetrics, fmt.Errorf("failed to scan Docker: %v", err)
} else {
refs = []sources.JobProgressRef{ref}
}
case postmanScan.FullCommand():
// handle deprecated flag
workspaceIDs := make([]string, 0, len(*postmanWorkspaceIDs)+len(*postmanWorkspaces))
workspaceIDs = append(workspaceIDs, *postmanWorkspaceIDs...)
workspaceIDs = append(workspaceIDs, *postmanWorkspaces...)
// handle deprecated flag
collectionIDs := make([]string, 0, len(*postmanCollectionIDs)+len(*postmanCollections))
collectionIDs = append(collectionIDs, *postmanCollectionIDs...)
collectionIDs = append(collectionIDs, *postmanCollections...)
// handle deprecated flag
includeCollectionIDs := make([]string, 0, len(*postmanIncludeCollectionIDs)+len(*postmanIncludeCollections))
includeCollectionIDs = append(includeCollectionIDs, *postmanIncludeCollectionIDs...)
includeCollectionIDs = append(includeCollectionIDs, *postmanIncludeCollections...)
// handle deprecated flag
excludeCollectionIDs := make([]string, 0, len(*postmanExcludeCollectionIDs)+len(*postmanExcludeCollections))
excludeCollectionIDs = append(excludeCollectionIDs, *postmanExcludeCollectionIDs...)
excludeCollectionIDs = append(excludeCollectionIDs, *postmanExcludeCollections...)
cfg := sources.PostmanConfig{
Token: *postmanToken,
Workspaces: workspaceIDs,
Collections: collectionIDs,
Environments: *postmanEnvironments,
IncludeCollections: includeCollectionIDs,
IncludeEnvironments: *postmanIncludeEnvironments,
ExcludeCollections: excludeCollectionIDs,
ExcludeEnvironments: *postmanExcludeEnvironments,
CollectionPaths: *postmanCollectionPaths,
WorkspacePaths: *postmanWorkspacePaths,
EnvironmentPaths: *postmanEnvironmentPaths,
}
if ref, err := eng.ScanPostman(ctx, cfg); err != nil {
return scanMetrics, fmt.Errorf("failed to scan Postman: %v", err)
} else {
refs = []sources.JobProgressRef{ref}
}
case elasticsearchScan.FullCommand():
cfg := sources.ElasticsearchConfig{
Nodes: *elasticsearchNodes,
Username: *elasticsearchUsername,
Password: *elasticsearchPassword,
CloudID: *elasticsearchCloudId,
APIKey: *elasticsearchAPIKey,
ServiceToken: *elasticsearchServiceToken,
IndexPattern: *elasticsearchIndexPattern,
QueryJSON: *elasticsearchQueryJSON,
SinceTimestamp: *elasticsearchSinceTimestamp,
BestEffortScan: *elasticsearchBestEffortScan,
}
if ref, err := eng.ScanElasticsearch(ctx, cfg); err != nil {
return scanMetrics, fmt.Errorf("failed to scan Elasticsearch: %v", err)
} else {
refs = []sources.JobProgressRef{ref}
}
case jenkinsScan.FullCommand():
cfg := engine.JenkinsConfig{
Endpoint: *jenkinsURL,
InsecureSkipVerifyTLS: *jenkinsInsecureSkipVerifyTLS,
Username: *jenkinsUsername,
Password: *jenkinsPassword,
}
if ref, err := eng.ScanJenkins(ctx, cfg); err != nil {
return scanMetrics, fmt.Errorf("failed to scan Jenkins: %v", err)
} else {
refs = []sources.JobProgressRef{ref}
}
case huggingfaceScan.FullCommand():
if *huggingfaceEndpoint != "" {
*huggingfaceEndpoint = strings.TrimRight(*huggingfaceEndpoint, "/")
}
if len(*huggingfaceModels) == 0 && len(*huggingfaceSpaces) == 0 && len(*huggingfaceDatasets) == 0 && len(*huggingfaceBuckets) == 0 && len(*huggingfaceOrgs) == 0 && len(*huggingfaceUsers) == 0 {
return scanMetrics, fmt.Errorf("invalid config: you must specify at least one organization, user, model, space, dataset or bucket")
}
cfg := engine.HuggingfaceConfig{
Endpoint: *huggingfaceEndpoint,
Models: *huggingfaceModels,
Spaces: *huggingfaceSpaces,
Datasets: *huggingfaceDatasets,
Buckets: *huggingfaceBuckets,
Organizations: *huggingfaceOrgs,
Users: *huggingfaceUsers,
Token: *huggingfaceToken,
IncludeModels: *huggingfaceIncludeModels,
IncludeSpaces: *huggingfaceIncludeSpaces,
IncludeDatasets: *huggingfaceIncludeDatasets,
IncludeBuckets: *huggingfaceIncludeBuckets,
IgnoreModels: *huggingfaceIgnoreModels,
IgnoreSpaces: *huggingfaceIgnoreSpaces,
IgnoreDatasets: *huggingfaceIgnoreDatasets,
IgnoreBuckets: *huggingfaceIgnoreBuckets,
SkipAllModels: *huggingfaceSkipAllModels,
SkipAllSpaces: *huggingfaceSkipAllSpaces,
SkipAllDatasets: *huggingfaceSkipAllDatasets,
SkipAllBuckets: *huggingfaceSkipAllBuckets,
IncludeDiscussions: *huggingfaceIncludeDiscussions,
IncludePrs: *huggingfaceIncludePrs,
Concurrency: *concurrency,
}
if ref, err := eng.ScanHuggingface(ctx, cfg); err != nil {
return scanMetrics, fmt.Errorf("failed to scan HuggingFace: %v", err)
} else {
refs = []sources.JobProgressRef{ref}
}
case multiScanScan.FullCommand():
if *configFilename == "" {
return scanMetrics, fmt.Errorf("missing required flag: --config")
}
if rs, err := eng.ScanConfig(ctx, cfg.ConfiguredSources...); err != nil {
return scanMetrics, fmt.Errorf("failed to scan via config: %w", err)
} else {
refs = rs
}
case stdinInputScan.FullCommand():
cfg := sources.StdinConfig{}
if ref, err := eng.ScanStdinInput(ctx, cfg); err != nil {
return scanMetrics, fmt.Errorf("failed to scan stdin input: %v", err)
} else {
refs = []sources.JobProgressRef{ref}
}
case jsonEnumeratorScan.FullCommand():
cfg := sources.JSONEnumeratorConfig{Paths: *jsonEnumeratorPaths}
if ref, err := eng.ScanJSONEnumeratorInput(ctx, cfg); err != nil {
return scanMetrics, fmt.Errorf("failed to scan JSON enumerator input: %v", err)
} else {
refs = []sources.JobProgressRef{ref}
}
default:
return scanMetrics, fmt.Errorf("invalid command: %s", cmd)
}
// Wait for all workers to finish.
if err = eng.Finish(ctx); err != nil {
return scanMetrics, fmt.Errorf("engine failed to finish execution: %v", err)
}
// Print any non-fatal errors reported during the scan.
var retErr error
for _, ref := range refs {
if errs := ref.Snapshot().Errors; len(errs) > 0 {
if *failOnScanErrors {
retErr = fmt.Errorf("encountered errors during scan")
}
errMsgs := make([]string, len(errs))
for i := 0; i < len(errs); i++ {
errMsgs[i] = errs[i].Error()
}
ctx.Logger().Error(nil, "encountered errors during scan",
"job", ref.JobID,
"source_name", ref.SourceName,
"errors", errMsgs,
)
}
}
if *printAvgDetectorTime {
printAverageDetectorTime(eng)
}
return metrics{Metrics: eng.GetMetrics(), hasFoundResults: eng.HasFoundResults()}, retErr
}
// parseResults ensures that users provide valid CSV input to `--results`.
//
// This is a work-around to kingpin not supporting CSVs.
// See: https://github.com/trufflesecurity/trufflehog/pull/2372#issuecomment-1983868917
func parseResults(input *string) (map[string]struct{}, error) {
if *input == "" {
return nil, nil
}
var (
values = strings.Split(strings.ToLower(*input), ",")
results = make(map[string]struct{}, 3)
)
for _, value := range values {
switch value {
case "verified", "unknown", "unverified", "filtered_unverified":
results[value] = struct{}{}
default:
return nil, fmt.Errorf("invalid value '%s', valid values are 'verified,unknown,unverified,filtered_unverified'", value)
}
}
return results, nil
}
// logFatalFunc returns a log.Fatal style function. Calling the returned
// function will terminate the program without cleanup.
func logFatalFunc(logger logr.Logger, logSync func() error) func(error, string, ...any) {
return func(err error, message string, keyAndVals ...any) {
logger.Error(err, message, keyAndVals...)
syncLogs(logSync)
if err != nil {
os.Exit(1)
return
}
os.Exit(0)
}
}
func commaSeparatedToSlice(s []string) []string {
var result []string
for _, items := range s {
for _, item := range strings.Split(items, ",") {
item = strings.TrimSpace(item)
if item == "" {
continue
}
result = append(result, item)
}
}
return result
}
func printAverageDetectorTime(e *engine.Engine) {
fmt.Fprintln(
os.Stderr,
"Average detector time is the measurement of average time spent on each detector when results are returned.",
)
for detectorName, duration := range e.GetDetectorsMetrics() {
fmt.Fprintf(os.Stderr, "%s: %s\n", detectorName, duration)
}
}
// validateClonePath ensures that --clone-path, if provided, exists and is a directory.
// It also verifies that --no-cleanup is only allowed when --clone-path is set.
// Note: without a custom clone path, repositories are cloned into temporary directories, which should never be retained.
func validateClonePath(clonePath string, noCleanup bool) error {
if noCleanup && clonePath == "" {
return fmt.Errorf("invalid configuration: --no-cleanup can only be used together with --clone-path")
}
if clonePath == "" {
return nil
}
info, err := os.Stat(clonePath)
if err != nil {
if os.IsNotExist(err) {
return fmt.Errorf("path provided to --clone-path: %q does not exist", clonePath)
}
return fmt.Errorf("failed to access --clone-path %q: %w", clonePath, err)
}
if !info.IsDir() {
return fmt.Errorf("path provided to --clone-path: %q is not a directory", clonePath)
}
return nil
}
// isPreCommitHook detects if trufflehog is running as a pre-commit hook
func isPreCommitHook() bool {
// Pre-commit.com framework detection
// Docs: https://pre-commit.com/#pre-commit
// Sets PRE_COMMIT=1 environment variable when running hooks
if os.Getenv("PRE_COMMIT") == "1" {
return true
}
// Husky framework detection (modern versions)
// Docs: https://typicode.github.io/husky/get-started.html#disabling-hooks
// Sets HUSKY=1 environment variable for all hooks
if os.Getenv("HUSKY") == "1" {
return true
}
// Husky legacy detection (versions < 4.0)
// Sets HUSKY_GIT_PARAMS for git hooks, containing commit parameters
// Reference: https://github.com/typicode/husky/tree/v0.14.3
if os.Getenv("HUSKY_GIT_PARAMS") != "" {
return true
}
// Local Git hook detection (non-framework)
// Native Git hooks don't set specific environment variables by default.
// To detect local hooks without frameworks, we must explicitly set
// an environment variable in the hook script:
// Example in .git/hooks/pre-commit:
// export TRUFFLEHOG_PRE_COMMIT=1
// Than we can detect it
if os.Getenv("TRUFFLEHOG_PRE_COMMIT") == "1" {
return true
}
return false
}