package main import ( "encoding/json" "fmt" "io" "net/http" _ "net/http/pprof" "os" "os/exec" "os/signal" "runtime" "strconv" "strings" "sync" "syscall" "github.com/alecthomas/kingpin/v2" "github.com/fatih/color" "github.com/felixge/fgprof" "github.com/go-logr/logr" "github.com/jpillora/overseer" "github.com/mattn/go-isatty" "go.uber.org/automaxprocs/maxprocs" "github.com/trufflesecurity/trufflehog/v3/pkg/analyzer" "github.com/trufflesecurity/trufflehog/v3/pkg/cache/simple" "github.com/trufflesecurity/trufflehog/v3/pkg/cleantemp" "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/config" "github.com/trufflesecurity/trufflehog/v3/pkg/context" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/engine" "github.com/trufflesecurity/trufflehog/v3/pkg/engine/defaults" "github.com/trufflesecurity/trufflehog/v3/pkg/feature" "github.com/trufflesecurity/trufflehog/v3/pkg/handlers" "github.com/trufflesecurity/trufflehog/v3/pkg/log" "github.com/trufflesecurity/trufflehog/v3/pkg/output" "github.com/trufflesecurity/trufflehog/v3/pkg/sources" "github.com/trufflesecurity/trufflehog/v3/pkg/tui" "github.com/trufflesecurity/trufflehog/v3/pkg/updater" "github.com/trufflesecurity/trufflehog/v3/pkg/verificationcache" "github.com/trufflesecurity/trufflehog/v3/pkg/version" ) var ( cli = kingpin.New("trufflehog", "TruffleHog is a tool for finding credentials. Run without a command for interactive mode.") cmd string // https://github.com/trufflesecurity/trufflehog/blob/main/CONTRIBUTING.md#logging-in-trufflehog logLevel = cli.Flag("log-level", `Logging verbosity on a scale of 0 (info) to 5 (trace). Can be disabled with "-1".`).Default("0").Int() debug = cli.Flag("debug", "Run in debug mode.").Hidden().Bool() trace = cli.Flag("trace", "Run in trace mode.").Hidden().Bool() profile = cli.Flag("profile", "Enables profiling and sets a pprof and fgprof server on :18066.").Bool() localDev = cli.Flag("local-dev", "Hidden feature to disable overseer for local dev.").Hidden().Bool() jsonOut = cli.Flag("json", "Output in JSON format.").Short('j').Bool() jsonLegacy = cli.Flag("json-legacy", "Use the pre-v3.0 JSON format. Only works with git, gitlab, and github sources.").Bool() gitHubActionsFormat = cli.Flag("github-actions", "Output in GitHub Actions format.").Bool() sarifOut = cli.Flag("sarif", "Output in SARIF format for upload to GitHub code scanning (e.g. via github/codeql-action/upload-sarif).").Bool() concurrency = cli.Flag("concurrency", "Number of concurrent workers.").PlaceHolder("N").Int() noVerification = cli.Flag("no-verification", "Don't verify the results.").Bool() onlyVerified = cli.Flag("only-verified", "Only output verified results.").Hidden().Bool() results = cli.Flag("results", "Specifies which type(s) of results to output: verified (confirmed valid by API), unknown (verification failed due to error), unverified (detected but not verified), filtered_unverified (unverified but would have been filtered out). Defaults to verified,unverified,unknown.").String() noColor = cli.Flag("no-color", "Disable colorized output").Bool() noColour = cli.Flag("no-colour", "Alias for --no-color").Hidden().Bool() allowVerificationOverlap = cli.Flag("allow-verification-overlap", "Allow verification of similar credentials across detectors").Bool() filterUnverified = cli.Flag("filter-unverified", "Only output first unverified result per chunk per detector if there are more than one results.").Bool() filterEntropy = cli.Flag("filter-entropy", "Filter unverified results with Shannon entropy. Start with 3.0.").Float64() noIgnoreTag = cli.Flag("no-ignore-tag", "Report results even if the line has a 'trufflehog:ignore' comment.").Bool() scanEntireChunk = cli.Flag("scan-entire-chunk", "Scan the entire chunk for secrets.").Hidden().Default("false").Bool() maxDecodeDepth = cli.Flag("max-decode-depth", "Maximum depth of iterative decoding. Each decoder's output is fed back through all decoders, up to this limit. 1 = single pass, 2+ = chained decoding (e.g., base64 inside utf16).").Default("5").Int() compareDetectionStrategies = cli.Flag("compare-detection-strategies", "Compare different detection strategies for matching spans").Hidden().Default("false").Bool() configFilename = cli.Flag("config", "Path to configuration file.").ExistingFile() // rules = cli.Flag("rules", "Path to file with custom rules.").String() printAvgDetectorTime = cli.Flag("print-avg-detector-time", "Print the average time spent on each detector.").Bool() noUpdate = cli.Flag("no-update", "Don't check for updates.").Bool() fail = cli.Flag("fail", "Exit with code 183 if results are found.").Bool() failOnScanErrors = cli.Flag("fail-on-scan-errors", "Exit with non-zero error code if an error occurs during the scan.").Bool() verifiers = cli.Flag("verifier", "Set custom verification endpoints.").StringMap() customVerifiersOnly = cli.Flag("custom-verifiers-only", "Only use custom verification endpoints.").Bool() detectorTimeout = cli.Flag("detector-timeout", "Maximum time to spend scanning chunks per detector (e.g., 30s).").Duration() archiveMaxSize = cli.Flag("archive-max-size", "Maximum size of archive to scan. (Byte units eg. 512B, 2KB, 4MB)").Bytes() archiveMaxDepth = cli.Flag("archive-max-depth", "Maximum depth of archive to scan.").Int() archiveTimeout = cli.Flag("archive-timeout", "Maximum time to spend extracting an archive.").Duration() includeDetectors = cli.Flag("include-detectors", "Comma separated list of detector types to include. Protobuf name or IDs may be used, as well as ranges.").Default("all").String() excludeDetectors = cli.Flag("exclude-detectors", "Comma separated list of detector types to exclude. Protobuf name or IDs may be used, as well as ranges. IDs defined here take precedence over the include list.").String() jobReportFile = cli.Flag("output-report", "Write a scan report to the provided path.").Hidden().OpenFile(os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666) noVerificationCache = cli.Flag("no-verification-cache", "Disable verification caching").Bool() // Add feature flags forceSkipBinaries = cli.Flag("force-skip-binaries", "Force skipping binaries.").Bool() forceSkipArchives = cli.Flag("force-skip-archives", "Force skipping archives.").Bool() gitCloneTimeout = cli.Flag("git-clone-timeout", "Maximum time to spend cloning a repository, as a duration.").Hidden().Duration() gitLowMemoryScan = cli.Flag("git-low-memory-scan", "Reduce memory use for git scanning.").Hidden().Bool() skipAdditionalRefs = cli.Flag("skip-additional-refs", "Skip additional references.").Bool() userAgentSuffix = cli.Flag("user-agent-suffix", "Suffix to add to User-Agent.").String() dropUnverifiedJWTResults = cli.Flag("drop-unverified-jwt-results", "Drop unverified results without any verification errors from the JWT detector.").Bool() gitScan = cli.Command("git", "Find credentials in git repositories.") gitScanURI = gitScan.Arg("uri", "Git repository URL. https://, file://, or ssh:// schema expected.").Required().String() gitScanIncludePaths = gitScan.Flag("include-paths", "Path to file with newline separated regexes for files to include in scan.").Short('i').String() gitScanExcludePaths = gitScan.Flag("exclude-paths", "Path to file with newline separated regexes for files to exclude in scan.").Short('x').String() gitScanExcludeGlobs = gitScan.Flag("exclude-globs", "Comma separated list of globs to exclude in scan. This option filters at the `git log` level, resulting in faster scans.").String() gitScanSinceCommit = gitScan.Flag("since-commit", "Commit to start scan from.").String() gitScanBranch = gitScan.Flag("branch", "Branch to scan.").String() gitScanMaxDepth = gitScan.Flag("max-depth", "Maximum depth of commits to scan.").Int() gitScanBare = gitScan.Flag("bare", "Scan bare repository (e.g. useful while using in pre-receive hooks)").Bool() gitClonePath = gitScan.Flag("clone-path", "Custom path where the repository should be cloned (default: temp dir).").String() gitNoCleanup = gitScan.Flag("no-cleanup", "Do not delete cloned repositories after scanning (can only be used with --clone-path). Each clone is kept in its own directory, so ensure sufficient disk space: usage grows with every repository scanned and every run.").Bool() gitTrustLocalGitConfig = gitScan.Flag("trust-local-git-config", "Trust local git config.").Bool() _ = gitScan.Flag("allow", "No-op flag for backwards compat.").Bool() _ = gitScan.Flag("entropy", "No-op flag for backwards compat.").Bool() _ = gitScan.Flag("regex", "No-op flag for backwards compat.").Bool() githubScan = cli.Command("github", "Find credentials in GitHub repositories.") githubScanEndpoint = githubScan.Flag("endpoint", "GitHub endpoint.").Default("https://api.github.com").String() githubScanRepos = githubScan.Flag("repo", `GitHub repository to scan. You can repeat this flag. Example: "https://github.com/dustin-decker/secretsandstuff"`).Strings() githubScanOrgs = githubScan.Flag("org", `GitHub organization to scan. You can repeat this flag. Example: "trufflesecurity"`).Strings() githubScanToken = githubScan.Flag("token", "GitHub token. Can be provided with environment variable GITHUB_TOKEN.").Envar("GITHUB_TOKEN").String() githubIncludeForks = githubScan.Flag("include-forks", "Include forks in scan.").Bool() githubIncludeMembers = githubScan.Flag("include-members", "Include organization member repositories in scan.").Bool() githubIncludeRepos = githubScan.Flag("include-repos", `Repositories to include in an org scan. This can also be a glob pattern. You can repeat this flag. Must use Github repo full name. Example: "trufflesecurity/trufflehog", "trufflesecurity/t*"`).Strings() githubIncludeWikis = githubScan.Flag("include-wikis", "Include repository wikisin scan.").Bool() githubExcludeRepos = githubScan.Flag("exclude-repos", `Repositories to exclude in an org scan. This can also be a glob pattern. You can repeat this flag. Must use Github repo full name. Example: "trufflesecurity/driftwood", "trufflesecurity/d*"`).Strings() githubScanIncludePaths = githubScan.Flag("include-paths", "Path to file with newline separated regexes for files to include in scan.").Short('i').String() githubScanExcludePaths = githubScan.Flag("exclude-paths", "Path to file with newline separated regexes for files to exclude in scan.").Short('x').String() githubScanIssueComments = githubScan.Flag("issue-comments", "Include issue descriptions and comments in scan.").Bool() githubScanPRComments = githubScan.Flag("pr-comments", "Include pull request descriptions and comments in scan.").Bool() githubScanGistComments = githubScan.Flag("gist-comments", "Include gist comments in scan.").Bool() githubCommentsTimeframeDays = githubScan.Flag("comments-timeframe", "Number of days in the past to review when scanning issue, PR, and gist comments.").Uint32() githubAuthInUrl = githubScan.Flag("auth-in-url", "Embed authentication credentials in repository URLs instead of using secure HTTP headers").Bool() githubClonePath = githubScan.Flag("clone-path", "Custom path where the repository should be cloned (default: temp dir).").String() githubNoCleanup = githubScan.Flag("no-cleanup", "Do not delete cloned repositories after scanning (can only be used with --clone-path). Each clone is kept in its own directory, so ensure sufficient disk space: usage grows with every repository scanned and every run.").Bool() githubIgnoreGists = githubScan.Flag("ignore-gists", "Ignore all gists in scan.").Bool() githubExcludeArchived = githubScan.Flag("exclude-archived", "Exclude archived repositories from scan.").Bool() // GitHub Cross Fork Object Reference Experimental Feature githubExperimentalScan = cli.Command("github-experimental", "Run an experimental GitHub scan. Must specify at least one experimental sub-module to run: object-discovery.") // GitHub Experimental SubModules githubExperimentalObjectDiscovery = githubExperimentalScan.Flag("object-discovery", "Discover hidden data objects in GitHub repositories.").Bool() // GitHub Experimental Options githubExperimentalToken = githubExperimentalScan.Flag("token", "GitHub token. Can be provided with environment variable GITHUB_TOKEN.").Envar("GITHUB_TOKEN").String() githubExperimentalRepo = githubExperimentalScan.Flag("repo", "GitHub repository to scan. Example: https://github.com//.git").Required().String() githubExperimentalCollisionThreshold = githubExperimentalScan.Flag("collision-threshold", "Threshold for short-sha collisions in object-discovery submodule. Default is 1.").Default("1").Int() githubExperimentalDeleteCache = githubExperimentalScan.Flag("delete-cached-data", "Delete cached data after object-discovery secret scanning.").Bool() gitlabScan = cli.Command("gitlab", "Find credentials in GitLab repositories.") // TODO: Add more GitLab options gitlabScanEndpoint = gitlabScan.Flag("endpoint", "GitLab endpoint.").Default("https://gitlab.com").String() gitlabScanRepos = gitlabScan.Flag("repo", "GitLab repo url. You can repeat this flag. Leave empty to scan all repos accessible with provided credential. Example: https://gitlab.com/org/repo.git").Strings() gitlabScanToken = gitlabScan.Flag("token", "GitLab token. Can be provided with environment variable GITLAB_TOKEN.").Envar("GITLAB_TOKEN").Required().String() gitlabScanGroupIds = gitlabScan.Flag("group-id", "GitLab group ID. If provided, it will scan the group and its subgroups. You can repeat this flag.").Strings() gitlabScanIncludePaths = gitlabScan.Flag("include-paths", "Path to file with newline separated regexes for files to include in scan.").Short('i').String() gitlabScanExcludePaths = gitlabScan.Flag("exclude-paths", "Path to file with newline separated regexes for files to exclude in scan.").Short('x').String() gitlabScanIncludeRepos = gitlabScan.Flag("include-repos", `Repositories to include in an org scan. This can also be a glob pattern. You can repeat this flag. Must use Gitlab repo full name. Example: "trufflesecurity/trufflehog", "trufflesecurity/t*"`).Strings() gitlabScanExcludeRepos = gitlabScan.Flag("exclude-repos", `Repositories to exclude in an org scan. This can also be a glob pattern. You can repeat this flag. Must use Gitlab repo full name. Example: "trufflesecurity/driftwood", "trufflesecurity/d*"`).Strings() gitlabAuthInUrl = gitlabScan.Flag("auth-in-url", "Embed authentication credentials in repository URLs instead of using secure HTTP headers").Bool() gitlabClonePath = gitlabScan.Flag("clone-path", "Custom path where the repository should be cloned (default: temp dir).").String() gitlabNoCleanup = gitlabScan.Flag("no-cleanup", "Do not delete cloned repositories after scanning (can only be used with --clone-path). Each clone is kept in its own directory, so ensure sufficient disk space: usage grows with every repository scanned and every run.").Bool() filesystemScan = cli.Command("filesystem", "Find credentials in a filesystem.") filesystemPaths = filesystemScan.Arg("path", "Path to file or directory to scan.").Strings() // DEPRECATED: --directory is deprecated in favor of arguments. filesystemDirectories = filesystemScan.Flag("directory", "Path to directory to scan. You can repeat this flag.").Strings() // TODO: Add more filesystem scan options. Currently only supports scanning a list of directories. // filesystemScanRecursive = filesystemScan.Flag("recursive", "Scan recursively.").Short('r').Bool() filesystemScanIncludePaths = filesystemScan.Flag("include-paths", "Path to file with newline separated regexes for files to include in scan.").Short('i').String() filesystemScanExcludePaths = filesystemScan.Flag("exclude-paths", "Path to file with newline separated regexes for files to exclude in scan.").Short('x').String() filesystemScanMaxSymlinkDepth = filesystemScan.Flag("max-symlink-depth", "Maximum depth to follow symlinks during filesystem scan.").Short('s').Int32() s3Scan = cli.Command("s3", "Find credentials in S3 buckets.") s3ScanKey = s3Scan.Flag("key", "S3 key used to authenticate. Can be provided with environment variable AWS_ACCESS_KEY_ID.").Envar("AWS_ACCESS_KEY_ID").String() s3ScanRoleArns = s3Scan.Flag("role-arn", "Specify the ARN of an IAM role to assume for scanning. You can repeat this flag.").Strings() s3ScanSecret = s3Scan.Flag("secret", "S3 secret used to authenticate. Can be provided with environment variable AWS_SECRET_ACCESS_KEY.").Envar("AWS_SECRET_ACCESS_KEY").String() s3ScanSessionToken = s3Scan.Flag("session-token", "S3 session token used to authenticate temporary credentials. Can be provided with environment variable AWS_SESSION_TOKEN.").Envar("AWS_SESSION_TOKEN").String() s3ScanCloudEnv = s3Scan.Flag("cloud-environment", "Use IAM credentials in cloud environment.").Bool() s3ScanBuckets = s3Scan.Flag("bucket", "Name of S3 bucket to scan. You can repeat this flag. Incompatible with --ignore-bucket.").Strings() s3ScanIgnoreBuckets = s3Scan.Flag("ignore-bucket", "Name of S3 bucket to ignore. You can repeat this flag. Incompatible with --bucket.").Strings() s3ScanMaxObjectSize = s3Scan.Flag("max-object-size", "Maximum size of objects to scan. Objects larger than this will be skipped. (Byte units eg. 512B, 2KB, 4MB)").Default("250MB").Bytes() s3ScanEndpoint = s3Scan.Flag("endpoint", "Endpoint of an S3-compatible service to scan instead of AWS S3. (eg. https://s3.internal.example.com)").String() s3ScanRegion = s3Scan.Flag("region", "Region used to sign requests. Defaults to us-east-1.").String() s3ScanIncludePrefixes = s3Scan.Flag("include-prefix", "Only scan objects whose key starts with this prefix. You can repeat this flag.").Strings() s3ScanExcludePrefixes = s3Scan.Flag("exclude-prefix", "Skip objects whose key starts with this prefix. You can repeat this flag. Takes precedence over --include-prefix.").Strings() s3ScanIncludeExtensions = s3Scan.Flag("include-extension", "Only scan objects with this file extension, written without a leading dot (eg. tf). You can repeat this flag. Incompatible with --exclude-extension.").Strings() s3ScanExcludeExtensions = s3Scan.Flag("exclude-extension", "Skip objects with this file extension, written without a leading dot (eg. zip). You can repeat this flag. Incompatible with --include-extension.").Strings() gcsScan = cli.Command("gcs", "Find credentials in GCS buckets.") gcsProjectID = gcsScan.Flag("project-id", "GCS project ID used to authenticate. Can NOT be used with unauth scan. Can be provided with environment variable GOOGLE_CLOUD_PROJECT.").Envar("GOOGLE_CLOUD_PROJECT").String() gcsCloudEnv = gcsScan.Flag("cloud-environment", "Use Application Default Credentials, IAM credentials to authenticate.").Bool() gcsServiceAccount = gcsScan.Flag("service-account", "Path to GCS service account JSON file.").ExistingFile() gcsWithoutAuth = gcsScan.Flag("without-auth", "Scan GCS buckets without authentication. This will only work for public buckets").Bool() gcsAPIKey = gcsScan.Flag("api-key", "GCS API key used to authenticate. Can be provided with environment variable GOOGLE_API_KEY.").Envar("GOOGLE_API_KEY").String() gcsIncludeBuckets = gcsScan.Flag("include-buckets", "Buckets to scan. Comma separated list of buckets. You can repeat this flag. Globs are supported").Short('I').Strings() gcsExcludeBuckets = gcsScan.Flag("exclude-buckets", "Buckets to exclude from scan. Comma separated list of buckets. Globs are supported").Short('X').Strings() gcsIncludeObjects = gcsScan.Flag("include-objects", "Objects to scan. Comma separated list of objects. you can repeat this flag. Globs are supported").Short('i').Strings() gcsExcludeObjects = gcsScan.Flag("exclude-objects", "Objects to exclude from scan. Comma separated list of objects. You can repeat this flag. Globs are supported").Short('x').Strings() gcsMaxObjectSize = gcsScan.Flag("max-object-size", "Maximum size of objects to scan. Objects larger than this will be skipped. (Byte units eg. 512B, 2KB, 4MB)").Default("10MB").Bytes() syslogScan = cli.Command("syslog", "Scan syslog") syslogAddress = syslogScan.Flag("address", "Address and port to listen on for syslog. Example: 127.0.0.1:514").String() syslogProtocol = syslogScan.Flag("protocol", "Protocol to listen on. udp or tcp").String() syslogTLSCert = syslogScan.Flag("cert", "Path to TLS cert.").String() syslogTLSKey = syslogScan.Flag("key", "Path to TLS key.").String() syslogFormat = syslogScan.Flag("format", "Log format. Can be rfc3164 or rfc5424").Required().String() circleCiScan = cli.Command("circleci", "Scan CircleCI") circleCiScanToken = circleCiScan.Flag("token", "CircleCI token. Can also be provided with environment variable").Envar("CIRCLECI_TOKEN").Required().String() dockerScan = cli.Command("docker", "Scan Docker Image") dockerScanImages = dockerScan.Flag("image", "Docker image to scan. Use the file:// prefix to point to a local tarball, the docker:// prefix to point to the docker daemon, otherwise an image registry is assumed.").Strings() dockerScanToken = dockerScan.Flag("token", "Docker bearer token. Can also be provided with environment variable").Envar("DOCKER_TOKEN").String() dockerExcludePaths = dockerScan.Flag("exclude-paths", "Comma separated list of paths to exclude from scan").String() dockerScanNamespace = dockerScan.Flag("namespace", "Docker namespace (organization or user). For non-Docker Hub registries, include the registry address as well (e.g., ghcr.io/namespace or quay.io/namespace).").String() dockerScanRegistryToken = dockerScan.Flag("registry-token", "Optional Docker registry access token. Provide this if you want to include private images within the specified namespace.").String() travisCiScan = cli.Command("travisci", "Scan TravisCI") travisCiScanToken = travisCiScan.Flag("token", "TravisCI token. Can also be provided with environment variable").Envar("TRAVISCI_TOKEN").Required().String() // Postman is hidden for now until we get more feedback from the community. postmanScan = cli.Command("postman", "Scan Postman") postmanToken = postmanScan.Flag("token", "Postman token. Can also be provided with environment variable").Envar("POSTMAN_TOKEN").String() postmanWorkspaces = postmanScan.Flag("workspace", "Postman workspace to scan. You can repeat this flag. Deprecated flag.").Hidden().Strings() postmanWorkspaceIDs = postmanScan.Flag("workspace-id", "Postman workspace ID to scan. You can repeat this flag.").Strings() postmanCollections = postmanScan.Flag("collection", "Postman collection to scan. You can repeat this flag. Deprecated flag.").Hidden().Strings() postmanCollectionIDs = postmanScan.Flag("collection-id", "Postman collection ID to scan. You can repeat this flag.").Strings() postmanEnvironments = postmanScan.Flag("environment", "Postman environment to scan. You can repeat this flag.").Strings() postmanIncludeCollections = postmanScan.Flag("include-collections", "Collections to include in scan. You can repeat this flag. Deprecated flag.").Hidden().Strings() postmanIncludeCollectionIDs = postmanScan.Flag("include-collection-id", "Collection ID to include in scan. You can repeat this flag.").Strings() postmanIncludeEnvironments = postmanScan.Flag("include-environments", "Environments to include in scan. You can repeat this flag.").Strings() postmanExcludeCollections = postmanScan.Flag("exclude-collections", "Collections to exclude from scan. You can repeat this flag. Deprecated flag.").Hidden().Strings() postmanExcludeCollectionIDs = postmanScan.Flag("exclude-collection-id", "Collection ID to exclude from scan. You can repeat this flag.").Strings() postmanExcludeEnvironments = postmanScan.Flag("exclude-environments", "Environments to exclude from scan. You can repeat this flag.").Strings() postmanWorkspacePaths = postmanScan.Flag("workspace-paths", "Path to Postman workspaces.").Strings() postmanCollectionPaths = postmanScan.Flag("collection-paths", "Path to Postman collections.").Strings() postmanEnvironmentPaths = postmanScan.Flag("environment-paths", "Path to Postman environments.").Strings() elasticsearchScan = cli.Command("elasticsearch", "Scan Elasticsearch") elasticsearchNodes = elasticsearchScan.Flag("nodes", "Elasticsearch nodes").Envar("ELASTICSEARCH_NODES").Strings() elasticsearchUsername = elasticsearchScan.Flag("username", "Elasticsearch username").Envar("ELASTICSEARCH_USERNAME").String() elasticsearchPassword = elasticsearchScan.Flag("password", "Elasticsearch password").Envar("ELASTICSEARCH_PASSWORD").String() elasticsearchServiceToken = elasticsearchScan.Flag("service-token", "Elasticsearch service token").Envar("ELASTICSEARCH_SERVICE_TOKEN").String() elasticsearchCloudId = elasticsearchScan.Flag("cloud-id", "Elasticsearch cloud ID. Can also be provided with environment variable").Envar("ELASTICSEARCH_CLOUD_ID").String() elasticsearchAPIKey = elasticsearchScan.Flag("api-key", "Elasticsearch API key. Can also be provided with environment variable").Envar("ELASTICSEARCH_API_KEY").String() elasticsearchIndexPattern = elasticsearchScan.Flag("index-pattern", "Filters the indices to search").Default("*").Envar("ELASTICSEARCH_INDEX_PATTERN").String() elasticsearchQueryJSON = elasticsearchScan.Flag("query-json", "Filters the documents to search").Envar("ELASTICSEARCH_QUERY_JSON").String() elasticsearchSinceTimestamp = elasticsearchScan.Flag("since-timestamp", "Filters the documents to search to those created since this timestamp; overrides any timestamp from --query-json").Envar("ELASTICSEARCH_SINCE_TIMESTAMP").String() elasticsearchBestEffortScan = elasticsearchScan.Flag("best-effort-scan", "Attempts to continuously scan a cluster").Envar("ELASTICSEARCH_BEST_EFFORT_SCAN").Bool() jenkinsScan = cli.Command("jenkins", "Scan Jenkins") jenkinsURL = jenkinsScan.Flag("url", "Jenkins URL").Envar("JENKINS_URL").Required().String() jenkinsUsername = jenkinsScan.Flag("username", "Jenkins username").Envar("JENKINS_USERNAME").String() jenkinsPassword = jenkinsScan.Flag("password", "Jenkins password").Envar("JENKINS_PASSWORD").String() jenkinsInsecureSkipVerifyTLS = jenkinsScan.Flag("insecure-skip-verify-tls", "Skip TLS verification").Envar("JENKINS_INSECURE_SKIP_VERIFY_TLS").Bool() huggingfaceScan = cli.Command("huggingface", "Find credentials in HuggingFace datasets, models, spaces and buckets.") huggingfaceEndpoint = huggingfaceScan.Flag("endpoint", "HuggingFace endpoint.").Default("https://huggingface.co").String() huggingfaceModels = huggingfaceScan.Flag("model", "HuggingFace model to scan. You can repeat this flag. Example: 'username/model'").Strings() huggingfaceSpaces = huggingfaceScan.Flag("space", "HuggingFace space to scan. You can repeat this flag. Example: 'username/space'").Strings() huggingfaceDatasets = huggingfaceScan.Flag("dataset", "HuggingFace dataset to scan. You can repeat this flag. Example: 'username/dataset'").Strings() huggingfaceBuckets = huggingfaceScan.Flag("bucket", "HuggingFace bucket to scan. You can repeat this flag. Example: 'username/bucket'").Strings() huggingfaceOrgs = huggingfaceScan.Flag("org", `HuggingFace organization to scan. You can repeat this flag. Example: "trufflesecurity"`).Strings() huggingfaceUsers = huggingfaceScan.Flag("user", `HuggingFace user to scan. You can repeat this flag. Example: "trufflesecurity"`).Strings() huggingfaceToken = huggingfaceScan.Flag("token", "HuggingFace token. Can be provided with environment variable HUGGINGFACE_TOKEN.").Envar("HUGGINGFACE_TOKEN").String() huggingfaceIncludeModels = huggingfaceScan.Flag("include-models", "Models to include in scan. You can repeat this flag. Must use HuggingFace model full name. Example: 'username/model' (Only used with --user or --org)").Strings() huggingfaceIncludeSpaces = huggingfaceScan.Flag("include-spaces", "Spaces to include in scan. You can repeat this flag. Must use HuggingFace space full name. Example: 'username/space' (Only used with --user or --org)").Strings() huggingfaceIncludeDatasets = huggingfaceScan.Flag("include-datasets", "Datasets to include in scan. You can repeat this flag. Must use HuggingFace dataset full name. Example: 'username/dataset' (Only used with --user or --org)").Strings() huggingfaceIgnoreModels = huggingfaceScan.Flag("ignore-models", "Models to ignore in scan. You can repeat this flag. Must use HuggingFace model full name. Example: 'username/model' (Only used with --user or --org)").Strings() huggingfaceIgnoreSpaces = huggingfaceScan.Flag("ignore-spaces", "Spaces to ignore in scan. You can repeat this flag. Must use HuggingFace space full name. Example: 'username/space' (Only used with --user or --org)").Strings() huggingfaceIgnoreDatasets = huggingfaceScan.Flag("ignore-datasets", "Datasets to ignore in scan. You can repeat this flag. Must use HuggingFace dataset full name. Example: 'username/dataset' (Only used with --user or --org)").Strings() huggingfaceIncludeBuckets = huggingfaceScan.Flag("include-buckets", "Buckets to include in scan. You can repeat this flag. Must use HuggingFace bucket full name. Example: 'username/bucket' (Only used with --user or --org)").Strings() huggingfaceIgnoreBuckets = huggingfaceScan.Flag("ignore-buckets", "Buckets to ignore in scan. You can repeat this flag. Must use HuggingFace bucket full name. Example: 'username/bucket' (Only used with --user or --org)").Strings() huggingfaceSkipAllModels = huggingfaceScan.Flag("skip-all-models", "Skip all model scans. (Only used with --user or --org)").Bool() huggingfaceSkipAllSpaces = huggingfaceScan.Flag("skip-all-spaces", "Skip all space scans. (Only used with --user or --org)").Bool() huggingfaceSkipAllDatasets = huggingfaceScan.Flag("skip-all-datasets", "Skip all dataset scans. (Only used with --user or --org)").Bool() huggingfaceSkipAllBuckets = huggingfaceScan.Flag("skip-all-buckets", "Skip all bucket scans. (Only used with --user or --org)").Bool() huggingfaceIncludeDiscussions = huggingfaceScan.Flag("include-discussions", "Include discussions in scan.").Bool() huggingfaceIncludePrs = huggingfaceScan.Flag("include-prs", "Include pull requests in scan.").Bool() stdinInputScan = cli.Command("stdin", "Find credentials from stdin.") multiScanScan = cli.Command("multi-scan", "Find credentials in multiple sources defined in configuration.") jsonEnumeratorScan = cli.Command("json-enumerator", "Find credentials from a JSON enumerator input.") jsonEnumeratorPaths = jsonEnumeratorScan.Arg("path", "Path to JSON enumerator file to scan.").Strings() analyzeCmd = analyzer.Command(cli) usingTUI = false ) // expandTilde replaces a leading ~ in each argument with the user's home // directory. This compensates for bypassing the shell (which would normally // perform this expansion) while still avoiding shell metacharacter injection. func expandTilde(args []string) []string { home, err := os.UserHomeDir() if err != nil || home == "" { return args } out := make([]string, len(args)) for i, arg := range args { switch { case arg == "~": out[i] = home case strings.HasPrefix(arg, "~/"): out[i] = home + arg[1:] default: out[i] = arg } } return out } func init() { _, _ = maxprocs.Set() for i, arg := range os.Args { if strings.HasPrefix(arg, "--") { split := strings.SplitN(arg, "=", 2) split[0] = strings.ReplaceAll(split[0], "_", "-") os.Args[i] = strings.Join(split, "=") } } cli.Version("trufflehog " + version.BuildVersion) // Support -h for help and write it to stdout. cli.HelpFlag.Short('h') cli.UsageWriter(os.Stdout) registerManPageFlag(cli) // Check if the TUI environment variable is set. if ok, err := strconv.ParseBool(os.Getenv("TUI_PARENT")); err == nil { usingTUI = ok } if isatty.IsTerminal(os.Stdout.Fd()) && (len(os.Args) <= 1 || os.Args[1] == analyzeCmd.FullCommand()) { args := tui.Run(os.Args[1:]) if len(args) == 0 { os.Exit(0) } // The TUI passes user input as literal strings. Since we no longer // invoke a shell, we must expand ~ ourselves so paths like ~/foo // resolve correctly. args = expandTilde(args) binary, err := exec.LookPath(os.Args[0]) if err == nil { // On success, this call will never return. On failure, fallthrough // to overwriting os.Args. execArgs := append([]string{binary}, args...) _ = syscall.Exec(binary, execArgs, append(os.Environ(), "TUI_PARENT=true")) } // Overwrite the Args slice so overseer works properly. os.Args = os.Args[:1] os.Args = append(os.Args, args...) usingTUI = true } cmd = kingpin.MustParse(cli.Parse(os.Args[1:])) // Configure logging. switch { case *trace: log.SetLevel(5) case *debug: log.SetLevel(2) default: l := int8(*logLevel) if l < -1 || l > 5 { fmt.Fprintf(os.Stderr, "invalid log level: %d\n", *logLevel) os.Exit(1) } if l == -1 { // Zap uses "5" as the value for fatal. // We need to pass in "-5" because `SetLevel` passes the negation. log.SetLevel(-5) } else { log.SetLevel(l) } } if *noColor || *noColour { color.NoColor = true // disables colorized output } } // syncLogs flushes logs when the program exits. func syncLogs(syncFn func() error) { if syncFn != nil { _ = syncFn() } } func main() { // setup logger logFormat := log.WithConsoleSink if *jsonOut { logFormat = log.WithJSONSink } logger, sync := log.New("trufflehog", logFormat(os.Stderr, log.WithGlobalRedaction())) // make it the default logger for contexts context.SetDefaultLogger(logger) if *localDev { run(overseer.State{}, sync) os.Exit(0) } logFatal := logFatalFunc(logger, sync) updateCfg := overseer.Config{ Program: func(s overseer.State) { run(s, sync) }, Debug: *debug, RestartSignal: syscall.SIGTERM, // TODO: Eventually add a PreUpgrade func for signature check w/ x509 PKCS1v15 // PreUpgrade: checkUpdateSignature(binaryPath string), } if !*noUpdate { topLevelCmd, _, _ := strings.Cut(cmd, " ") updateCfg.Fetcher = updater.Fetcher(topLevelCmd, usingTUI) } if version.BuildVersion == "dev" { updateCfg.Fetcher = nil } err := overseer.RunErr(updateCfg) if err != nil { logFatal(err, "error occurred with trufflehog updater 🐷") } } func run(state overseer.State, logSync func() error) { ctx, cancel := context.WithCancelCause(context.Background()) defer cancel(nil) defer syncLogs(logSync) go func() { if err := cleantemp.CleanTempArtifacts(ctx); err != nil { ctx.Logger().Error(err, "error cleaning temporary artifacts") } }() logger := ctx.Logger() logFatal := logFatalFunc(logger, logSync) killSignal := make(chan os.Signal, 1) signal.Notify(killSignal, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT) go func() { <-killSignal logger.Info("Received signal, shutting down.") cancel(fmt.Errorf("canceling context due to signal")) if err := cleantemp.CleanTempArtifacts(ctx); err != nil { logger.Error(err, "error cleaning temporary artifacts") } else { logger.Info("cleaned temporary artifacts") } syncLogs(logSync) os.Exit(0) }() logger.V(2).Info(fmt.Sprintf("trufflehog %s", version.BuildVersion)) if *githubScanToken != "" { // NOTE: this kludge is here to do an authenticated shallow commit // TODO: refactor to better pass credentials _ = os.Setenv("GITHUB_TOKEN", *githubScanToken) } if *concurrency <= 0 { *concurrency = runtime.NumCPU() } // When setting a base commit, chunks must be scanned in order. if *gitScanSinceCommit != "" { *concurrency = 1 } if *profile { runtime.SetBlockProfileRate(1) runtime.SetMutexProfileFraction(-1) go func() { router := http.NewServeMux() router.Handle("/debug/pprof/", http.DefaultServeMux) router.Handle("/debug/fgprof", fgprof.Handler()) logger.Info("starting pprof and fgprof server on :18066 /debug/pprof and /debug/fgprof") if err := http.ListenAndServe(":18066", router); err != nil { logger.Error(err, "error serving pprof and fgprof") } }() } // Set feature configurations from CLI flags if *forceSkipBinaries { feature.ForceSkipBinaries.Store(true) } if *forceSkipArchives { feature.ForceSkipArchives.Store(true) } if gitCloneTimeout != nil { feature.GitCloneTimeoutDuration.Store(int64(*gitCloneTimeout)) } if *gitLowMemoryScan { feature.UseGitLowMemoryScan.Store(true) } if *skipAdditionalRefs { feature.SkipAdditionalRefs.Store(true) } if *userAgentSuffix != "" { feature.UserAgentSuffix.Store(*userAgentSuffix) } if *dropUnverifiedJWTResults { feature.DropUnverifiedJWTResults.Store(true) } // OSS Default APK handling on feature.EnableAPKHandler.Store(true) // OSS Default Use Git Mirror on feature.UseGitMirror.Store(true) // OSS Default simplified gitlab enumeration feature.UseSimplifiedGitlabEnumeration.Store(true) feature.GitlabProjectsPerPage.Store(100) // OSS Default using github graphql api for issues, pr's and comments feature.UseGithubGraphQLAPI.Store(false) // OSS Default Use HTML Decoder on feature.HTMLDecoderEnabled.Store(true) // New detector flags feature.PineconeDetectorEnabled.Store(true) feature.CloudinaryDetectorEnabled.Store(true) feature.GitLabOAuthDetectorEnabled.Store(true) feature.FigmaV3DetectorEnabled.Store(true) feature.SonarCloudV2DetectorEnabled.Store(true) feature.EnigmaDetectorEnabled.Store(true) feature.DatadogApiKeyDetectorEnabled.Store(true) feature.TlyDetectorEnabled.Store(true) feature.WitDetectorEnabled.Store(true) feature.RevDetectorEnabled.Store(true) feature.UserDetectorEnabled.Store(true) feature.BraintrustDetectorEnabled.Store(true) feature.PgAnalyzeReadKeyDetectorEnabled.Store(true) feature.RedHatPyxisDetectorEnabled.Store(true) feature.OctopusDeployDetectorEnabled.Store(true) feature.OpenRouterDetectorEnabled.Store(true) feature.NewRelicInsightsInsertKeyDetectorEnabled.Store(true) feature.DuffelTokenDetectorEnabled.Store(true) feature.ShippoDetectorEnabled.Store(true) feature.IPInfoDetectorEnabled.Store(true) feature.LobDetectorEnabled.Store(true) feature.HashiCorpVaultBatchTokenDetectorEnabled.Store(true) feature.HashiCorpVaultTokenDetectorEnabled.Store(true) feature.CloudflareApiTokenV2DetectorEnabled.Store(true) feature.CloudflareGlobalApiKeyV2DetectorEnabled.Store(true) feature.DuoDetectorEnabled.Store(true) feature.NewRelicLicenseKeyDetectorEnabled.Store(true) feature.NewRelicBrowserKeyDetectorEnabled.Store(true) feature.NewRelicUserKeyDetectorEnabled.Store(true) feature.NewRelicInsightsQueryKeyDetectorEnabled.Store(true) feature.NewRelicMobileAppTokenDetectorEnabled.Store(true) feature.MSTeamsWebhookV2DetectorEnabled.Store(true) feature.SolarwindsDetectorEnabled.Store(true) feature.ResendDetectorEnabled.Store(true) feature.WeightsAndBiasesV2DetectorEnabled.Store(true) feature.HumioAPITokenDetectorEnabled.Store(true) conf := &config.Config{} if *configFilename != "" { var err error conf, err = config.Read(*configFilename) if err != nil { logFatal(err, "error parsing the provided configuration file") } } if *detectorTimeout != 0 { logger.Info("Setting detector timeout", "timeout", detectorTimeout.String()) engine.SetDetectorTimeout(*detectorTimeout) detectors.OverrideDetectorTimeout(*detectorTimeout) } if *archiveMaxSize != 0 { handlers.SetArchiveMaxSize(int(*archiveMaxSize)) } if *archiveMaxDepth != 0 { handlers.SetArchiveMaxDepth(*archiveMaxDepth) } if *archiveTimeout != 0 { handlers.SetArchiveMaxTimeout(*archiveTimeout) } // Set how the engine will print its results. var printer engine.Printer switch { case *jsonLegacy: printer = new(output.LegacyJSONPrinter) case *jsonOut: printer = new(output.JSONPrinter) case *gitHubActionsFormat: printer = new(output.GitHubActionsPrinter) case *sarifOut: printer = new(output.SarifPrinter) default: printer = new(output.PlainPrinter) } if !*jsonLegacy && !*jsonOut && !*sarifOut { fmt.Fprintf(os.Stderr, "🐷🔑🐷 TruffleHog. Unearth your secrets. 🐷🔑🐷\n\n") } // Parse --results flag. if *onlyVerified { r := "verified" results = &r } parsedResults, err := parseResults(results) if err != nil { logFatal(err, "failed to configure results flag") } verificationCacheMetrics := verificationcache.InMemoryMetrics{} engConf := engine.Config{ Concurrency: *concurrency, ConfiguredSources: conf.Sources, // The engine must always be configured with the list of // default detectors, which can be further filtered by the // user. The filters are applied by the engine and are only // subtractive. Detectors: append(defaults.DefaultDetectors(), conf.Detectors...), Verify: !*noVerification, IncludeDetectors: *includeDetectors, ExcludeDetectors: *excludeDetectors, CustomVerifiersOnly: *customVerifiersOnly, VerifierEndpoints: *verifiers, Dispatcher: engine.NewPrinterDispatcher(printer), FilterUnverified: *filterUnverified, FilterEntropy: *filterEntropy, NoIgnoreTag: *noIgnoreTag, VerificationOverlap: *allowVerificationOverlap, Results: parsedResults, PrintAvgDetectorTime: *printAvgDetectorTime, ShouldScanEntireChunk: *scanEntireChunk, MaxDecodeDepth: *maxDecodeDepth, VerificationCacheMetrics: &verificationCacheMetrics, } if !*noVerificationCache { engConf.VerificationResultCache = simple.NewCache[detectors.Result]() } // Check that there are no sources defined for non-scan subcommands. If // there are, return an error as it is ambiguous what the user is // trying to do. if cmd != multiScanScan.FullCommand() && len(conf.Sources) > 0 { logFatal( fmt.Errorf("ambiguous configuration"), "sources should only be defined in configuration for the 'multi-scan' command", ) } if *compareDetectionStrategies { if err := compareScans(ctx, cmd, engConf); err != nil { logFatal(err, "error comparing detection strategies") } return } metrics, err := runSingleScan(ctx, cmd, engConf) if err != nil { logFatal(err, "error running scan") } // SARIF can't be streamed like the other output formats: it's a single JSON document // wrapping every result, so it's buffered by the printer and written out here, once // scanning has fully finished. if sarifPrinter, ok := printer.(*output.SarifPrinter); ok { if err := sarifPrinter.Flush(os.Stdout); err != nil { logFatal(err, "error writing SARIF output") } } verificationCacheMetricsSnapshot := struct { Hits int32 Misses int32 HitsWasted int32 AttemptsSaved int32 VerificationTimeSpentMS int64 }{ Hits: verificationCacheMetrics.ResultCacheHits.Load(), Misses: verificationCacheMetrics.ResultCacheMisses.Load(), HitsWasted: verificationCacheMetrics.ResultCacheHitsWasted.Load(), AttemptsSaved: verificationCacheMetrics.CredentialVerificationsSaved.Load(), VerificationTimeSpentMS: verificationCacheMetrics.FromDataVerifyTimeSpentMS.Load(), } // Print results. logger.Info("finished scanning", "chunks", metrics.ChunksScanned, "bytes", metrics.BytesScanned, "verified_secrets", metrics.VerifiedSecretsFound, "unverified_secrets", metrics.UnverifiedSecretsFound, "scan_duration", metrics.ScanDuration.String(), "trufflehog_version", version.BuildVersion, "verification_caching", verificationCacheMetricsSnapshot, ) if metrics.hasFoundResults && *fail { logger.V(2).Info("exiting with code 183 because results were found") syncLogs(logSync) os.Exit(183) } } func compareScans(ctx context.Context, cmd string, cfg engine.Config) error { var ( entireMetrics metrics maxLengthMetrics metrics err error ) var wg sync.WaitGroup wg.Add(1) go func() { defer wg.Done() // Run scan with entire chunk span calculator. cfg.ShouldScanEntireChunk = true entireMetrics, err = runSingleScan(ctx, cmd, cfg) if err != nil { ctx.Logger().Error(err, "error running scan with entire chunk span calculator") } }() // Run scan with max-length span calculator. maxLengthMetrics, err = runSingleScan(ctx, cmd, cfg) if err != nil { return fmt.Errorf("error running scan with custom span calculator: %v", err) } wg.Wait() return compareMetrics(maxLengthMetrics.Metrics, entireMetrics.Metrics) } func compareMetrics(customMetrics, entireMetrics engine.Metrics) error { fmt.Printf("Comparison of scan results: \n") fmt.Printf("Custom span - Chunks: %d, Bytes: %d, Verified Secrets: %d, Unverified Secrets: %d, Duration: %s\n", customMetrics.ChunksScanned, customMetrics.BytesScanned, customMetrics.VerifiedSecretsFound, customMetrics.UnverifiedSecretsFound, customMetrics.ScanDuration.String()) fmt.Printf("Entire chunk - Chunks: %d, Bytes: %d, Verified Secrets: %d, Unverified Secrets: %d, Duration: %s\n", entireMetrics.ChunksScanned, entireMetrics.BytesScanned, entireMetrics.VerifiedSecretsFound, entireMetrics.UnverifiedSecretsFound, entireMetrics.ScanDuration.String()) // Check for differences in scan metrics. if customMetrics.ChunksScanned != entireMetrics.ChunksScanned || customMetrics.BytesScanned != entireMetrics.BytesScanned || customMetrics.VerifiedSecretsFound != entireMetrics.VerifiedSecretsFound { return fmt.Errorf("scan metrics do not match") } return nil } type metrics struct { engine.Metrics hasFoundResults bool } func runSingleScan(ctx context.Context, cmd string, cfg engine.Config) (metrics, error) { var scanMetrics metrics // Setup job report writer if provided var jobReportWriter io.WriteCloser if *jobReportFile != nil { jobReportWriter = *jobReportFile } handleFinishedMetrics := func(ctx context.Context, finishedMetrics <-chan sources.UnitMetrics, jobReportWriter io.WriteCloser) { go func() { defer func() { _ = jobReportWriter.Close() if namer, ok := jobReportWriter.(interface{ Name() string }); ok { ctx.Logger().Info("report written", "path", namer.Name()) } else { ctx.Logger().Info("report written") } }() for metrics := range finishedMetrics { metrics.Errors = common.ExportErrors(metrics.Errors...) details, err := json.Marshal(map[string]any{ "version": 1, "data": metrics, }) if err != nil { ctx.Logger().Error(err, "error marshalling job details") continue } if _, err := jobReportWriter.Write(append(details, '\n')); err != nil { ctx.Logger().Error(err, "error writing to file") } } }() } const defaultOutputBufferSize = 64 opts := []func(*sources.SourceManager){ sources.WithConcurrentSources(cfg.Concurrency), sources.WithConcurrentUnits(cfg.Concurrency), sources.WithSourceUnits(), sources.WithBufferedOutput(defaultOutputBufferSize), } if jobReportWriter != nil { unitHook, finishedMetrics := sources.NewUnitHook(ctx) opts = append(opts, sources.WithReportHook(unitHook)) handleFinishedMetrics(ctx, finishedMetrics, jobReportWriter) } cfg.SourceManager = sources.NewManager(opts...) eng, err := engine.NewEngine(ctx, &cfg) if err != nil { return scanMetrics, fmt.Errorf("error initializing engine: %v", err) } eng.Start(ctx) persistGitRepo := *gitNoCleanup || *githubNoCleanup || *gitlabNoCleanup gitCloneTempPath := "" defer func() { // Clean up temporary artifacts. if err := cleantemp.CleanTempArtifacts(ctx); err != nil { ctx.Logger().Error(err, "error cleaning temp artifacts") } if *jsonLegacy { // If JSON legacy is enabled, that means the cloned repos are not deleted yet // because they were needed for outputting legacy JSON. // We only clean them up here if the user did not request to persist them. if !persistGitRepo { if err := cleantemp.CleanTempDirsForLegacyJSON(gitCloneTempPath); err != nil { ctx.Logger().Error(err, "error cleaning temp artifacts for legacy JSON") } } } }() var refs []sources.JobProgressRef switch cmd { case gitScan.FullCommand(): if err := validateClonePath(*gitClonePath, *gitNoCleanup); err != nil { return scanMetrics, err } gitCfg := sources.GitConfig{ URI: *gitScanURI, IncludePathsFile: *gitScanIncludePaths, ExcludePathsFile: *gitScanExcludePaths, HeadRef: *gitScanBranch, BaseRef: *gitScanSinceCommit, MaxDepth: *gitScanMaxDepth, Bare: *gitScanBare, ExcludeGlobs: *gitScanExcludeGlobs, ClonePath: *gitClonePath, NoCleanup: *gitNoCleanup, PrintLegacyJSON: *jsonLegacy, TrustLocalGitConfig: *gitTrustLocalGitConfig, } // detect if trufflehog is running git source as a pre-commit hook if isPreCommitHook() { ctx.Logger().Info("Running as a pre-commit hook, overriding default flags for hook context") // Override git configuration for pre-commit hook context gitCfg.TrustLocalGitConfig = true gitCfg.BaseRef = "HEAD" // Only scan staged changes // Override result filters for pre-commit hook context // In hook mode, we only want to show verified secrets and unknown findings *results = "verified,unknown" // Override failure behavior for pre-commit hook context // In hook mode, we want to fail the commit if any secrets are found *fail = true } if ref, err := eng.ScanGit(ctx, gitCfg); err != nil { return scanMetrics, fmt.Errorf("failed to scan Git: %v", err) } else { refs = []sources.JobProgressRef{ref} } case githubScan.FullCommand(): gitCloneTempPath = *githubClonePath filter, err := common.FilterFromFiles(*githubScanIncludePaths, *githubScanExcludePaths) if err != nil { return scanMetrics, fmt.Errorf("could not create filter: %v", err) } if len(*githubScanOrgs) == 0 && len(*githubScanRepos) == 0 { return scanMetrics, fmt.Errorf("invalid config: you must specify at least one organization or repository") } if len(*githubScanOrgs) > 0 && len(*githubScanRepos) > 0 { return scanMetrics, fmt.Errorf("invalid config: you cannot specify both organizations and repositories at the same time") } // --include-repos/--exclude-repos only filter repos enumerated from an org/user scan; // explicit --repo entries bypass that filtering step entirely (see enumerate() in // pkg/sources/github/github.go), so combining them with --repo would silently no-op // the filters instead of erroring. if len(*githubScanRepos) > 0 && (len(*githubIncludeRepos) > 0 || len(*githubExcludeRepos) > 0) { return scanMetrics, fmt.Errorf("invalid config: --include-repos and --exclude-repos only apply to organization or user scans and cannot be used with --repo") } if err := validateClonePath(*githubClonePath, *githubNoCleanup); err != nil { return scanMetrics, err } cfg := sources.GithubConfig{ Endpoint: *githubScanEndpoint, Token: *githubScanToken, IncludeForks: *githubIncludeForks, IncludeMembers: *githubIncludeMembers, IncludeWikis: *githubIncludeWikis, Concurrency: *concurrency, ExcludeRepos: *githubExcludeRepos, IncludeRepos: *githubIncludeRepos, Repos: *githubScanRepos, Orgs: *githubScanOrgs, IncludeIssueComments: *githubScanIssueComments, IncludePullRequestComments: *githubScanPRComments, IncludeGistComments: *githubScanGistComments, CommentsTimeframeDays: *githubCommentsTimeframeDays, Filter: filter, AuthInUrl: *githubAuthInUrl, ClonePath: *githubClonePath, NoCleanup: *githubNoCleanup, IgnoreGists: *githubIgnoreGists, ExcludeArchived: *githubExcludeArchived, PrintLegacyJSON: *jsonLegacy, } if ref, err := eng.ScanGitHub(ctx, cfg); err != nil { return scanMetrics, fmt.Errorf("failed to scan Github: %v", err) } else { refs = []sources.JobProgressRef{ref} } case githubExperimentalScan.FullCommand(): cfg := sources.GitHubExperimentalConfig{ Token: *githubExperimentalToken, Repository: *githubExperimentalRepo, ObjectDiscovery: *githubExperimentalObjectDiscovery, CollisionThreshold: *githubExperimentalCollisionThreshold, DeleteCachedData: *githubExperimentalDeleteCache, } if ref, err := eng.ScanGitHubExperimental(ctx, cfg); err != nil { return scanMetrics, fmt.Errorf("failed to scan using Github Experimental: %v", err) } else { refs = []sources.JobProgressRef{ref} } case gitlabScan.FullCommand(): gitCloneTempPath = *gitlabClonePath filter, err := common.FilterFromFiles(*gitlabScanIncludePaths, *gitlabScanExcludePaths) if err != nil { return scanMetrics, fmt.Errorf("could not create filter: %v", err) } if len(*gitlabScanRepos) > 0 && len(*gitlabScanGroupIds) > 0 { return scanMetrics, fmt.Errorf("invalid config: you cannot specify both repositories and groups at the same time") } if err := validateClonePath(*gitlabClonePath, *gitlabNoCleanup); err != nil { return scanMetrics, err } cfg := sources.GitlabConfig{ Endpoint: *gitlabScanEndpoint, Token: *gitlabScanToken, Repos: *gitlabScanRepos, GroupIds: *gitlabScanGroupIds, IncludeRepos: *gitlabScanIncludeRepos, ExcludeRepos: *gitlabScanExcludeRepos, Filter: filter, AuthInUrl: *gitlabAuthInUrl, ClonePath: *gitlabClonePath, NoCleanup: *gitlabNoCleanup, PrintLegacyJSON: *jsonLegacy, } if ref, err := eng.ScanGitLab(ctx, cfg); err != nil { return scanMetrics, fmt.Errorf("failed to scan GitLab: %v", err) } else { refs = []sources.JobProgressRef{ref} } case filesystemScan.FullCommand(): if len(*filesystemDirectories) > 0 { ctx.Logger().Info("--directory flag is deprecated, please pass directories as arguments") } paths := make([]string, 0, len(*filesystemPaths)+len(*filesystemDirectories)) paths = append(paths, *filesystemPaths...) paths = append(paths, *filesystemDirectories...) cfg := sources.FilesystemConfig{ Paths: paths, IncludePathsFile: *filesystemScanIncludePaths, ExcludePathsFile: *filesystemScanExcludePaths, MaxSymlinkDepth: *filesystemScanMaxSymlinkDepth, } if ref, err := eng.ScanFileSystem(ctx, cfg); err != nil { return scanMetrics, fmt.Errorf("failed to scan filesystem: %v", err) } else { refs = []sources.JobProgressRef{ref} } case s3Scan.FullCommand(): cfg := sources.S3Config{ Key: *s3ScanKey, Secret: *s3ScanSecret, SessionToken: *s3ScanSessionToken, Buckets: *s3ScanBuckets, IgnoreBuckets: *s3ScanIgnoreBuckets, Roles: *s3ScanRoleArns, CloudCred: *s3ScanCloudEnv, MaxObjectSize: int64(*s3ScanMaxObjectSize), Endpoint: *s3ScanEndpoint, Region: *s3ScanRegion, IncludePrefixes: *s3ScanIncludePrefixes, ExcludePrefixes: *s3ScanExcludePrefixes, IncludeExtensions: *s3ScanIncludeExtensions, ExcludeExtensions: *s3ScanExcludeExtensions, } if ref, err := eng.ScanS3(ctx, cfg); err != nil { return scanMetrics, fmt.Errorf("failed to scan S3: %v", err) } else { refs = []sources.JobProgressRef{ref} } case syslogScan.FullCommand(): cfg := sources.SyslogConfig{ Address: *syslogAddress, Format: *syslogFormat, Protocol: *syslogProtocol, CertPath: *syslogTLSCert, KeyPath: *syslogTLSKey, Concurrency: *concurrency, } if ref, err := eng.ScanSyslog(ctx, cfg); err != nil { return scanMetrics, fmt.Errorf("failed to scan syslog: %v", err) } else { refs = []sources.JobProgressRef{ref} } case circleCiScan.FullCommand(): if ref, err := eng.ScanCircleCI(ctx, *circleCiScanToken); err != nil { return scanMetrics, fmt.Errorf("failed to scan CircleCI: %v", err) } else { refs = []sources.JobProgressRef{ref} } case travisCiScan.FullCommand(): if ref, err := eng.ScanTravisCI(ctx, *travisCiScanToken); err != nil { return scanMetrics, fmt.Errorf("failed to scan TravisCI: %v", err) } else { refs = []sources.JobProgressRef{ref} } case gcsScan.FullCommand(): cfg := sources.GCSConfig{ ProjectID: *gcsProjectID, CloudCred: *gcsCloudEnv, ServiceAccount: *gcsServiceAccount, WithoutAuth: *gcsWithoutAuth, ApiKey: *gcsAPIKey, IncludeBuckets: commaSeparatedToSlice(*gcsIncludeBuckets), ExcludeBuckets: commaSeparatedToSlice(*gcsExcludeBuckets), IncludeObjects: commaSeparatedToSlice(*gcsIncludeObjects), ExcludeObjects: commaSeparatedToSlice(*gcsExcludeObjects), Concurrency: *concurrency, MaxObjectSize: int64(*gcsMaxObjectSize), } if ref, err := eng.ScanGCS(ctx, cfg); err != nil { return scanMetrics, fmt.Errorf("failed to scan GCS: %v", err) } else { refs = []sources.JobProgressRef{ref} } case dockerScan.FullCommand(): if *dockerScanImages != nil && *dockerScanNamespace != "" { return scanMetrics, fmt.Errorf("invalid config: you cannot specify both images and namespace at the same time") } if *dockerScanImages == nil && *dockerScanNamespace == "" { return scanMetrics, fmt.Errorf("invalid config: both images and namespace cannot be empty; one is required") } if *dockerScanRegistryToken != "" && *dockerScanNamespace == "" { return scanMetrics, fmt.Errorf("invalid config: registry token can only be used with registry namespace") } cfg := sources.DockerConfig{ BearerToken: *dockerScanToken, Images: *dockerScanImages, UseDockerKeychain: *dockerScanToken == "", ExcludePaths: strings.Split(*dockerExcludePaths, ","), Namespace: *dockerScanNamespace, RegistryToken: *dockerScanRegistryToken, } if ref, err := eng.ScanDocker(ctx, cfg); err != nil { return scanMetrics, fmt.Errorf("failed to scan Docker: %v", err) } else { refs = []sources.JobProgressRef{ref} } case postmanScan.FullCommand(): // handle deprecated flag workspaceIDs := make([]string, 0, len(*postmanWorkspaceIDs)+len(*postmanWorkspaces)) workspaceIDs = append(workspaceIDs, *postmanWorkspaceIDs...) workspaceIDs = append(workspaceIDs, *postmanWorkspaces...) // handle deprecated flag collectionIDs := make([]string, 0, len(*postmanCollectionIDs)+len(*postmanCollections)) collectionIDs = append(collectionIDs, *postmanCollectionIDs...) collectionIDs = append(collectionIDs, *postmanCollections...) // handle deprecated flag includeCollectionIDs := make([]string, 0, len(*postmanIncludeCollectionIDs)+len(*postmanIncludeCollections)) includeCollectionIDs = append(includeCollectionIDs, *postmanIncludeCollectionIDs...) includeCollectionIDs = append(includeCollectionIDs, *postmanIncludeCollections...) // handle deprecated flag excludeCollectionIDs := make([]string, 0, len(*postmanExcludeCollectionIDs)+len(*postmanExcludeCollections)) excludeCollectionIDs = append(excludeCollectionIDs, *postmanExcludeCollectionIDs...) excludeCollectionIDs = append(excludeCollectionIDs, *postmanExcludeCollections...) cfg := sources.PostmanConfig{ Token: *postmanToken, Workspaces: workspaceIDs, Collections: collectionIDs, Environments: *postmanEnvironments, IncludeCollections: includeCollectionIDs, IncludeEnvironments: *postmanIncludeEnvironments, ExcludeCollections: excludeCollectionIDs, ExcludeEnvironments: *postmanExcludeEnvironments, CollectionPaths: *postmanCollectionPaths, WorkspacePaths: *postmanWorkspacePaths, EnvironmentPaths: *postmanEnvironmentPaths, } if ref, err := eng.ScanPostman(ctx, cfg); err != nil { return scanMetrics, fmt.Errorf("failed to scan Postman: %v", err) } else { refs = []sources.JobProgressRef{ref} } case elasticsearchScan.FullCommand(): cfg := sources.ElasticsearchConfig{ Nodes: *elasticsearchNodes, Username: *elasticsearchUsername, Password: *elasticsearchPassword, CloudID: *elasticsearchCloudId, APIKey: *elasticsearchAPIKey, ServiceToken: *elasticsearchServiceToken, IndexPattern: *elasticsearchIndexPattern, QueryJSON: *elasticsearchQueryJSON, SinceTimestamp: *elasticsearchSinceTimestamp, BestEffortScan: *elasticsearchBestEffortScan, } if ref, err := eng.ScanElasticsearch(ctx, cfg); err != nil { return scanMetrics, fmt.Errorf("failed to scan Elasticsearch: %v", err) } else { refs = []sources.JobProgressRef{ref} } case jenkinsScan.FullCommand(): cfg := engine.JenkinsConfig{ Endpoint: *jenkinsURL, InsecureSkipVerifyTLS: *jenkinsInsecureSkipVerifyTLS, Username: *jenkinsUsername, Password: *jenkinsPassword, } if ref, err := eng.ScanJenkins(ctx, cfg); err != nil { return scanMetrics, fmt.Errorf("failed to scan Jenkins: %v", err) } else { refs = []sources.JobProgressRef{ref} } case huggingfaceScan.FullCommand(): if *huggingfaceEndpoint != "" { *huggingfaceEndpoint = strings.TrimRight(*huggingfaceEndpoint, "/") } if len(*huggingfaceModels) == 0 && len(*huggingfaceSpaces) == 0 && len(*huggingfaceDatasets) == 0 && len(*huggingfaceBuckets) == 0 && len(*huggingfaceOrgs) == 0 && len(*huggingfaceUsers) == 0 { return scanMetrics, fmt.Errorf("invalid config: you must specify at least one organization, user, model, space, dataset or bucket") } cfg := engine.HuggingfaceConfig{ Endpoint: *huggingfaceEndpoint, Models: *huggingfaceModels, Spaces: *huggingfaceSpaces, Datasets: *huggingfaceDatasets, Buckets: *huggingfaceBuckets, Organizations: *huggingfaceOrgs, Users: *huggingfaceUsers, Token: *huggingfaceToken, IncludeModels: *huggingfaceIncludeModels, IncludeSpaces: *huggingfaceIncludeSpaces, IncludeDatasets: *huggingfaceIncludeDatasets, IncludeBuckets: *huggingfaceIncludeBuckets, IgnoreModels: *huggingfaceIgnoreModels, IgnoreSpaces: *huggingfaceIgnoreSpaces, IgnoreDatasets: *huggingfaceIgnoreDatasets, IgnoreBuckets: *huggingfaceIgnoreBuckets, SkipAllModels: *huggingfaceSkipAllModels, SkipAllSpaces: *huggingfaceSkipAllSpaces, SkipAllDatasets: *huggingfaceSkipAllDatasets, SkipAllBuckets: *huggingfaceSkipAllBuckets, IncludeDiscussions: *huggingfaceIncludeDiscussions, IncludePrs: *huggingfaceIncludePrs, Concurrency: *concurrency, } if ref, err := eng.ScanHuggingface(ctx, cfg); err != nil { return scanMetrics, fmt.Errorf("failed to scan HuggingFace: %v", err) } else { refs = []sources.JobProgressRef{ref} } case multiScanScan.FullCommand(): if *configFilename == "" { return scanMetrics, fmt.Errorf("missing required flag: --config") } if rs, err := eng.ScanConfig(ctx, cfg.ConfiguredSources...); err != nil { return scanMetrics, fmt.Errorf("failed to scan via config: %w", err) } else { refs = rs } case stdinInputScan.FullCommand(): cfg := sources.StdinConfig{} if ref, err := eng.ScanStdinInput(ctx, cfg); err != nil { return scanMetrics, fmt.Errorf("failed to scan stdin input: %v", err) } else { refs = []sources.JobProgressRef{ref} } case jsonEnumeratorScan.FullCommand(): cfg := sources.JSONEnumeratorConfig{Paths: *jsonEnumeratorPaths} if ref, err := eng.ScanJSONEnumeratorInput(ctx, cfg); err != nil { return scanMetrics, fmt.Errorf("failed to scan JSON enumerator input: %v", err) } else { refs = []sources.JobProgressRef{ref} } default: return scanMetrics, fmt.Errorf("invalid command: %s", cmd) } // Wait for all workers to finish. if err = eng.Finish(ctx); err != nil { return scanMetrics, fmt.Errorf("engine failed to finish execution: %v", err) } // Print any non-fatal errors reported during the scan. var retErr error for _, ref := range refs { if errs := ref.Snapshot().Errors; len(errs) > 0 { if *failOnScanErrors { retErr = fmt.Errorf("encountered errors during scan") } errMsgs := make([]string, len(errs)) for i := 0; i < len(errs); i++ { errMsgs[i] = errs[i].Error() } ctx.Logger().Error(nil, "encountered errors during scan", "job", ref.JobID, "source_name", ref.SourceName, "errors", errMsgs, ) } } if *printAvgDetectorTime { printAverageDetectorTime(eng) } return metrics{Metrics: eng.GetMetrics(), hasFoundResults: eng.HasFoundResults()}, retErr } // parseResults ensures that users provide valid CSV input to `--results`. // // This is a work-around to kingpin not supporting CSVs. // See: https://github.com/trufflesecurity/trufflehog/pull/2372#issuecomment-1983868917 func parseResults(input *string) (map[string]struct{}, error) { if *input == "" { return nil, nil } var ( values = strings.Split(strings.ToLower(*input), ",") results = make(map[string]struct{}, 3) ) for _, value := range values { switch value { case "verified", "unknown", "unverified", "filtered_unverified": results[value] = struct{}{} default: return nil, fmt.Errorf("invalid value '%s', valid values are 'verified,unknown,unverified,filtered_unverified'", value) } } return results, nil } // logFatalFunc returns a log.Fatal style function. Calling the returned // function will terminate the program without cleanup. func logFatalFunc(logger logr.Logger, logSync func() error) func(error, string, ...any) { return func(err error, message string, keyAndVals ...any) { logger.Error(err, message, keyAndVals...) syncLogs(logSync) if err != nil { os.Exit(1) return } os.Exit(0) } } func commaSeparatedToSlice(s []string) []string { var result []string for _, items := range s { for _, item := range strings.Split(items, ",") { item = strings.TrimSpace(item) if item == "" { continue } result = append(result, item) } } return result } func printAverageDetectorTime(e *engine.Engine) { fmt.Fprintln( os.Stderr, "Average detector time is the measurement of average time spent on each detector when results are returned.", ) for detectorName, duration := range e.GetDetectorsMetrics() { fmt.Fprintf(os.Stderr, "%s: %s\n", detectorName, duration) } } // validateClonePath ensures that --clone-path, if provided, exists and is a directory. // It also verifies that --no-cleanup is only allowed when --clone-path is set. // Note: without a custom clone path, repositories are cloned into temporary directories, which should never be retained. func validateClonePath(clonePath string, noCleanup bool) error { if noCleanup && clonePath == "" { return fmt.Errorf("invalid configuration: --no-cleanup can only be used together with --clone-path") } if clonePath == "" { return nil } info, err := os.Stat(clonePath) if err != nil { if os.IsNotExist(err) { return fmt.Errorf("path provided to --clone-path: %q does not exist", clonePath) } return fmt.Errorf("failed to access --clone-path %q: %w", clonePath, err) } if !info.IsDir() { return fmt.Errorf("path provided to --clone-path: %q is not a directory", clonePath) } return nil } // isPreCommitHook detects if trufflehog is running as a pre-commit hook func isPreCommitHook() bool { // Pre-commit.com framework detection // Docs: https://pre-commit.com/#pre-commit // Sets PRE_COMMIT=1 environment variable when running hooks if os.Getenv("PRE_COMMIT") == "1" { return true } // Husky framework detection (modern versions) // Docs: https://typicode.github.io/husky/get-started.html#disabling-hooks // Sets HUSKY=1 environment variable for all hooks if os.Getenv("HUSKY") == "1" { return true } // Husky legacy detection (versions < 4.0) // Sets HUSKY_GIT_PARAMS for git hooks, containing commit parameters // Reference: https://github.com/typicode/husky/tree/v0.14.3 if os.Getenv("HUSKY_GIT_PARAMS") != "" { return true } // Local Git hook detection (non-framework) // Native Git hooks don't set specific environment variables by default. // To detect local hooks without frameworks, we must explicitly set // an environment variable in the hook script: // Example in .git/hooks/pre-commit: // export TRUFFLEHOG_PRE_COMMIT=1 // Than we can detect it if os.Getenv("TRUFFLEHOG_PRE_COMMIT") == "1" { return true } return false }