diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a203e7d09..ef7a58a29 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -38,7 +38,7 @@ jobs: with: go-version: '1.21' - name: Cosign install - uses: sigstore/cosign-installer@1fc5bd396d372bee37d608f955b336615edf79c8 # v3.2.0 + uses: sigstore/cosign-installer@9614fae9e5c5eddabb09f90a270fcb487c9f7149 # v3.3.0 - name: Run GoReleaser uses: goreleaser/goreleaser-action@v5 with: diff --git a/.github/workflows/secrets.yml b/.github/workflows/secrets.yml index 311c020cb..45a50b389 100644 --- a/.github/workflows/secrets.yml +++ b/.github/workflows/secrets.yml @@ -13,10 +13,6 @@ jobs: test: runs-on: ubuntu-latest steps: - - name: Install Go - uses: actions/setup-go@v4 - with: - go-version: '1.21' - name: Checkout code uses: actions/checkout@v4 with: @@ -26,7 +22,4 @@ jobs: uses: ./ id: dogfood with: - path: ./ - base: ${{ github.event.repository.default_branch }} - head: HEAD extra_args: --only-verified diff --git a/Dockerfile b/Dockerfile index 1fdfce29e..abfe664d7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,7 +8,7 @@ RUN --mount=type=cache,target=/go/pkg/mod \ --mount=type=cache,target=/root/.cache/go-build \ GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build -o trufflehog . -FROM alpine:3.18 +FROM alpine:3.19 RUN apk add --no-cache bash git openssh-client ca-certificates rpm2cpio binutils cpio \ && rm -rf /var/cache/apk/* && update-ca-certificates COPY --from=builder /build/trufflehog /usr/bin/trufflehog diff --git a/Dockerfile.goreleaser b/Dockerfile.goreleaser index bd1b1aad0..59bef10a9 100644 --- a/Dockerfile.goreleaser +++ b/Dockerfile.goreleaser @@ -1,4 +1,4 @@ -FROM alpine:3.18 +FROM alpine:3.19 RUN apk add --no-cache bash git openssh-client ca-certificates rpm2cpio binutils cpio \ && rm -rf /var/cache/apk/* && update-ca-certificates diff --git a/README.md b/README.md index 503a964af..8507980d9 100644 --- a/README.md +++ b/README.md @@ -65,6 +65,10 @@ cd trufflehog; go install # Using installation script curl -sSfL https://raw.githubusercontent.com/trufflesecurity/trufflehog/main/scripts/install.sh | sh -s -- -b /usr/local/bin + +# Using installation script, verify checksum signature (requires cosign to be installed) +curl -sSfL https://raw.githubusercontent.com/trufflesecurity/trufflehog/main/scripts/install.sh | sh -s -- -v -b /usr/local/bin + # Using installation script to install a specific version curl -sSfL https://raw.githubusercontent.com/trufflesecurity/trufflehog/main/scripts/install.sh | sh -s -- -b /usr/local/bin ``` @@ -103,6 +107,9 @@ Verification steps are as follow: Replace `{version}` with the downloaded files version +Alternatively, if you are using installation script, pass `-v` option to perform signature verification. +This required Cosign binary to be installed prior to running installation script. + # :rocket: Quick Start ## 1: Scan a repo for only verified secrets @@ -336,6 +343,62 @@ Exit Codes: ## :octocat: TruffleHog Github Action +### General Usage + +``` +on: + push: + branches: + - main + pull_request: + +jobs: + test: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Secret Scanning + uses: trufflesecurity/trufflehog@main + with: + extra_args: --only-verified +``` + +In the example config above, we're scanning for live secrets in all PRs and Pushes to `main`. Only code changes in the referenced commits are scanned. If you'd like to scan an entire branch, please see the "Advanced Usage" section below. + + +### Shallow Cloning + +If you're incorporating TruffleHog into a standalone workflow and aren't running any other CI/CD tooling alongside TruffleHog, then we recommend using [Shallow Cloning](https://git-scm.com/docs/git-clone#Documentation/git-clone.txt---depthltdepthgt) to speed up your workflow. Here's an example for how to do it: + +``` +... + - shell: bash + run: | + if [ "${{ github.event_name }}" == "push" ]; then + echo "depth=$(($(jq length <<< '${{ toJson(github.event.commits) }}') + 2))" >> $GITHUB_ENV + echo "branch=${{ github.ref_name }}" >> $GITHUB_ENV + fi + if [ "${{ github.event_name }}" == "pull_request" ]; then + echo "depth=$((${{ github.event.pull_request.commits }}+2))" >> $GITHUB_ENV + echo "branch=${{ github.event.pull_request.head.ref }}" >> $GITHUB_ENV + fi + - uses: actions/checkout@v3 + with: + ref: ${{env.branch}} + fetch-depth: ${{env.depth}} + - uses: trufflesecurity/trufflehog@main + with: + extra_args: --only-verified +... +``` + +Depending on the event type (push or PR), we calculate the number of commits present. Then we add 2, so that we can reference a base commit before our code changes. We pass that integer value to the `fetch-depth` flag in the checkout action in addition to the relevant branch. Now our checkout process should be much shorter. + +### Advanced Usage + ```yaml - name: TruffleHog uses: trufflesecurity/trufflehog@main @@ -350,34 +413,27 @@ Exit Codes: extra_args: --debug --only-verified ``` -The TruffleHog OSS Github Action can be used to scan a range of commits for leaked credentials. The action will fail if -any results are found. +If you'd like to specify specific `base` and `head` refs, you can use the `base` argument (`--since-commit` flag in TruffleHog CLI) and the `head` argument (`--branch` flag in the TruffleHog CLI). We only recommend using these arguments for very specific use cases, where the default behavior does not work. -For example, to scan the contents of pull requests you could use the following workflow: - -```yaml -name: TruffleHog Secrets Scan -on: [pull_request] -jobs: - TruffleHog: - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - name: TruffleHog OSS +#### Advanced Usage: Scan entire branch +``` +- name: scan-push uses: trufflesecurity/trufflehog@main with: - path: ./ - base: ${{ github.event.repository.default_branch }} - head: HEAD - extra_args: --debug --only-verified + base: "" + head: ${{ github.ref_name }} + extra_args: --only-verified ``` ## Pre-commit Hook Trufflehog can be used in a pre-commit hook to prevent credentials from leaking before they ever leave your computer. + +**Key Usage Note:** + +- **For optimal hook efficacy, execute `git add` followed by `git commit` separately.** This ensures Trufflehog analyzes all intended changes. +- **Avoid using `git commit -am`, as it might bypass pre-commit hook execution for unstaged modifications.** + An example `.pre-commit-config.yaml` is provided (see [pre-commit.com](https://pre-commit.com/) for installation). ```yaml diff --git a/action.yml b/action.yml index 3632ebbbe..0ed2548a4 100644 --- a/action.yml +++ b/action.yml @@ -1,11 +1,12 @@ name: 'TruffleHog OSS' -description: 'Scan Github Actions with TruffleHog' +description: 'Scan Github Actions with TruffleHog.' author: Truffle Security Co. inputs: path: description: Repository path - required: true + required: false + default: "./" base: description: Start scanning from here (usually main branch). required: false @@ -20,17 +21,73 @@ inputs: branding: icon: "shield" color: "green" + runs: - using: "docker" - image: "docker://ghcr.io/trufflesecurity/trufflehog:latest" - args: - - git - - file://${{ inputs.path }} - - --since-commit - - ${{ inputs.base }} - - --branch - - ${{ inputs.head }} - - --fail - - --no-update - - --github-actions - - ${{ inputs.extra_args }} + using: "composite" + steps: + - shell: bash + env: + REPO_PATH: ${{ inputs.path }} + BASE: ${{ inputs.base }} + HEAD: ${{ inputs.head }} + ARGS: ${{ inputs.extra_args }} + COMMITS: ${{ toJson(github.event.commits) }} + run: | + ########################################## + ## ADVANCED USAGE ## + ## Scan by BASE & HEAD user inputs ## + ## If BASE == HEAD, exit with error ## + ########################################## + if [ -n "$BASE" ] || [ -n "$HEAD" ]; then + if [ -n "$BASE" ]; then + base_commit=$(git rev-parse "$BASE" 2>/dev/null) || true + else + base_commit="" + fi + if [ -n "$HEAD" ]; then + head_commit=$(git rev-parse "$HEAD" 2>/dev/null) || true + else + head_commit="" + fi + if [ $base_commit == $head_commit ] ; then + echo "::error::BASE and HEAD commits are the same. TruffleHog won't scan anything. Please see documentation (https://github.com/trufflesecurity/trufflehog#octocat-trufflehog-github-action)." + exit 1 + fi + ########################################## + ## Scan commits based on event type ## + ########################################## + else + if [ "${{ github.event_name }}" == "push" ]; then + COMMIT_LENGTH=$(printenv COMMITS | jq length) + if [ $COMMIT_LENGTH == "0" ]; then + echo "No commits to scan" + exit 0 + fi + HEAD=${{ github.event.after }} + if [ ${{ github.event.before }} == "0000000000000000000000000000000000000000" ]; then + BASE=$(git rev-parse $HEAD~$COMMIT_LENGTH) + else + BASE=${{ github.event.before }} + fi + elif [ "${{ github.event_name }}" == "workflow_dispatch" ] || [ "${{ github.event_name }}" == "schedule" ]; then + BASE="" + HEAD="" + elif [ "${{ github.event_name }}" == "pull_request" ]; then + BASE=${{github.event.pull_request.base.sha}} + HEAD=${{github.event.pull_request.head.sha}} + fi + fi + ########################################## + ## Run TruffleHog ## + ########################################## + docker run --rm -v "$REPO_PATH":/tmp \ + ghcr.io/trufflesecurity/trufflehog:latest \ + git file:///tmp/ \ + --since-commit \ + ${BASE:-''} \ + --branch \ + ${HEAD:-''} \ + --fail \ + --no-update \ + --github-actions \ + ${ARGS:-''} diff --git a/go.mod b/go.mod index 16fbc266a..2305c8066 100644 --- a/go.mod +++ b/go.mod @@ -6,17 +6,17 @@ replace github.com/jpillora/overseer => github.com/trufflesecurity/overseer v1.2 require ( cloud.google.com/go/secretmanager v1.11.4 - cloud.google.com/go/storage v1.35.1 + cloud.google.com/go/storage v1.36.0 github.com/Azure/go-autorest/autorest/azure/auth v0.5.12 - github.com/AzureAD/microsoft-authentication-library-for-go v1.2.0 + github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 github.com/BobuSumisu/aho-corasick v1.0.3 github.com/TheZeroSlave/zapsentry v1.19.0 github.com/alecthomas/kingpin/v2 v2.4.0 - github.com/aws/aws-sdk-go v1.48.12 + github.com/aws/aws-sdk-go v1.49.19 github.com/aymanbagabas/go-osc52 v1.2.2 github.com/bill-rich/go-syslog v0.0.0-20220413021637-49edb52a574c github.com/bitfinexcom/bitfinex-api-go v0.0.0-20210608095005-9e0b26f200fb - github.com/bradleyfalzon/ghinstallation/v2 v2.8.0 + github.com/bradleyfalzon/ghinstallation/v2 v2.9.0 github.com/charmbracelet/bubbles v0.16.1 github.com/charmbracelet/bubbletea v0.24.2 github.com/charmbracelet/glamour v0.6.0 @@ -28,9 +28,10 @@ require ( github.com/envoyproxy/protoc-gen-validate v1.0.2 github.com/fatih/color v1.16.0 github.com/felixge/fgprof v0.9.3 + github.com/gabriel-vasile/mimetype v1.4.3 github.com/getsentry/sentry-go v0.25.0 github.com/go-errors/errors v1.5.1 - github.com/go-git/go-git/v5 v5.10.1 + github.com/go-git/go-git/v5 v5.11.0 github.com/go-ldap/ldap/v3 v3.4.6 github.com/go-logr/logr v1.3.0 github.com/go-logr/zapr v1.3.0 @@ -56,7 +57,7 @@ require ( github.com/lrstanley/bubblezone v0.0.0-20230911164824-e3824f1adde9 github.com/marusama/semaphore/v2 v2.5.0 github.com/mattn/go-isatty v0.0.20 - github.com/mattn/go-sqlite3 v1.14.18 + github.com/mattn/go-sqlite3 v1.14.19 github.com/mholt/archiver/v4 v4.0.0-alpha.8 github.com/mitchellh/go-ps v1.0.0 github.com/muesli/reflow v0.3.0 @@ -75,8 +76,8 @@ require ( go.mongodb.org/mongo-driver v1.12.1 go.uber.org/mock v0.3.0 go.uber.org/zap v1.26.0 - golang.org/x/crypto v0.16.0 - golang.org/x/exp v0.0.0-20231127185646-65229373498e + golang.org/x/crypto v0.17.0 + golang.org/x/exp v0.0.0-20240110193028-0dcbfd608b1e golang.org/x/net v0.19.0 golang.org/x/oauth2 v0.15.0 golang.org/x/sync v0.5.0 @@ -137,7 +138,7 @@ require ( github.com/bodgit/sevenzip v1.4.5 // indirect github.com/bodgit/windows v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect - github.com/cloudflare/circl v1.3.3 // indirect + github.com/cloudflare/circl v1.3.7 // indirect github.com/containerd/console v1.0.4-0.20230313162750-1ae8d489ac81 // indirect github.com/containerd/stargz-snapshotter/estargz v0.14.3 // indirect github.com/couchbase/gocbcore/v10 v10.3.0 // indirect @@ -154,10 +155,9 @@ require ( github.com/docker/docker v24.0.7+incompatible // indirect github.com/docker/docker-credential-helpers v0.7.0 // indirect github.com/dsnet/compress v0.0.1 // indirect - github.com/dvsekhvalnov/jose2go v1.5.0 // indirect + github.com/dvsekhvalnov/jose2go v1.6.0 // indirect github.com/emirpasic/gods v1.18.1 // indirect github.com/form3tech-oss/jwt-go v3.2.5+incompatible // indirect - github.com/gabriel-vasile/mimetype v1.4.2 // indirect github.com/go-asn1-ber/asn1-ber v1.5.5 // indirect github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect github.com/go-git/go-billy/v5 v5.5.0 // indirect @@ -172,6 +172,7 @@ require ( github.com/golang/snappy v0.0.4 // indirect github.com/google/flatbuffers v23.1.21+incompatible // indirect github.com/google/go-github/v56 v56.0.0 // indirect + github.com/google/go-github/v57 v57.0.0 // indirect github.com/google/go-querystring v1.1.0 // indirect github.com/google/pprof v0.0.0-20211214055906-6f57359322fd // indirect github.com/google/s2a-go v0.1.7 // indirect diff --git a/go.sum b/go.sum index ec93c09f1..401bd150c 100644 --- a/go.sum +++ b/go.sum @@ -28,6 +28,8 @@ cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiy cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= cloud.google.com/go/storage v1.35.1 h1:B59ahL//eDfx2IIKFBeT5Atm9wnNmj3+8xG/W4WB//w= cloud.google.com/go/storage v1.35.1/go.mod h1:M6M/3V/D3KpzMTJyPOR/HU6n2Si5QdaXYEsng2xgOs8= +cloud.google.com/go/storage v1.36.0 h1:P0mOkAcaJxhCTvAkMhxMfrTKiNcub4YmmPBtlhAyTr8= +cloud.google.com/go/storage v1.36.0/go.mod h1:M6M/3V/D3KpzMTJyPOR/HU6n2Si5QdaXYEsng2xgOs8= dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= @@ -68,6 +70,8 @@ github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 h1:mFRzDkZVAjdal+ github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU= github.com/AzureAD/microsoft-authentication-library-for-go v1.2.0 h1:hVeq+yCyUi+MsoO/CU95yqCIcdzra5ovzk8Q2BBpV2M= github.com/AzureAD/microsoft-authentication-library-for-go v1.2.0/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= +github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 h1:DzHpqpoJVaCgOUdVHxE8QB52S6NiVdDQvGlny1qvPqA= +github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= github.com/BobuSumisu/aho-corasick v1.0.3 h1:uuf+JHwU9CHP2Vx+wAy6jcksJThhJS9ehR8a+4nPE9g= github.com/BobuSumisu/aho-corasick v1.0.3/go.mod h1:hm4jLcvZKI2vRF2WDU1N4p/jpWtpOzp3nLmi9AzX/XE= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= @@ -104,6 +108,10 @@ github.com/atotto/clipboard v0.1.4 h1:EH0zSVneZPSuFR11BlR9YppQTVDbh5+16AmcJi4g1z github.com/atotto/clipboard v0.1.4/go.mod h1:ZY9tmq7sm5xIbd9bOK4onWV4S6X0u6GY7Vn0Yu86PYI= github.com/aws/aws-sdk-go v1.48.12 h1:n+eGzflzzvYubu2cOjqpVll7lF+Ci0ThyCpg5kzfzbo= github.com/aws/aws-sdk-go v1.48.12/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= +github.com/aws/aws-sdk-go v1.49.18 h1:g/iMXkfXeJQ7MvnLwroxWsTTNkHtdVJGxIgrAIEG62M= +github.com/aws/aws-sdk-go v1.49.18/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= +github.com/aws/aws-sdk-go v1.49.19 h1:oZryiqeQpeJsIcAmZlp86duMu/s/DJ43qyfwa51qmLg= +github.com/aws/aws-sdk-go v1.49.19/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= github.com/aws/aws-sdk-go-v2 v1.17.7 h1:CLSjnhJSTSogvqUGhIC6LqFKATMRexcxLZ0i/Nzk9Eg= github.com/aws/aws-sdk-go-v2 v1.17.7/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 h1:dK82zF6kkPeCo8J1e+tGx4JdvDIQzj7ygIoLg8WMuGs= @@ -154,19 +162,18 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bill-rich/go-syslog v0.0.0-20220413021637-49edb52a574c h1:tSME5FDS02qQll3JYodI6RZR/g4EKOHApGv1wMZT+Z0= github.com/bill-rich/go-syslog v0.0.0-20220413021637-49edb52a574c/go.mod h1:+sCc6hztur+oZCLOsNk6wCCy+GLrnSNHSRmTnnL+8iQ= -github.com/bill-rich/sevenzip v0.0.0-20231205215127-9521c543efac h1:7jDHN8fn8cNf1ibps8CrHivEM7GtXpsuaJeD5CWkw/Y= -github.com/bill-rich/sevenzip v0.0.0-20231205215127-9521c543efac/go.mod h1:aHY9hBGPQUQIimuGB0H+LJPqCI/68ZZFxRRab0gk/nU= github.com/bitfinexcom/bitfinex-api-go v0.0.0-20210608095005-9e0b26f200fb h1:9v7Bzlg+1EBYi2IYcUmOwHReBEfqBbYIj3ZCi9cIe1Q= github.com/bitfinexcom/bitfinex-api-go v0.0.0-20210608095005-9e0b26f200fb/go.mod h1:EkOqCuelvo7DY8vCOoZ09p7pHvAK9B1PHI9GeM4Rdxc= github.com/bodgit/plumbing v1.3.0 h1:pf9Itz1JOQgn7vEOE7v7nlEfBykYqvUYioC61TwWCFU= github.com/bodgit/plumbing v1.3.0/go.mod h1:JOTb4XiRu5xfnmdnDJo6GmSbSbtSyufrsyZFByMtKEs= -github.com/bodgit/sevenzip v1.4.3/go.mod h1:F8n3+0CwbdxqmNy3wFeOAtanza02Ur66AGfs/hbYblI= github.com/bodgit/sevenzip v1.4.5 h1:HFJQ+nbjppfyf2xbQEJBbmVo+o2kTg1FXV4i7YOx87s= github.com/bodgit/sevenzip v1.4.5/go.mod h1:LAcAg/UQzyjzCQSGBPZFYzoiHMfT6Gk+3tMSjUk3foY= github.com/bodgit/windows v1.0.1 h1:tF7K6KOluPYygXa3Z2594zxlkbKPAOvqr97etrGNIz4= github.com/bodgit/windows v1.0.1/go.mod h1:a6JLwrB4KrTR5hBpp8FI9/9W9jJfeQ2h4XDXU74ZCdM= github.com/bradleyfalzon/ghinstallation/v2 v2.8.0 h1:yUmoVv70H3J4UOqxqsee39+KlXxNEDfTbAp8c/qULKk= github.com/bradleyfalzon/ghinstallation/v2 v2.8.0/go.mod h1:fmPmvCiBWhJla3zDv9ZTQSZc8AbwyRnGW1yg5ep1Pcs= +github.com/bradleyfalzon/ghinstallation/v2 v2.9.0 h1:HmxIYqnxubRYcYGRc5v3wUekmo5Wv2uX3gukmWJ0AFk= +github.com/bradleyfalzon/ghinstallation/v2 v2.9.0/go.mod h1:wmkTDJf8CmVypxE8ijIStFnKoTa6solK5QfdmJrP9KI= github.com/bwesterb/go-ristretto v1.2.3/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= @@ -183,8 +190,9 @@ github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWR github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cloudflare/circl v1.3.3 h1:fE/Qz0QdIGqeWfnwq0RE0R7MI51s0M2E4Ga9kq5AEMs= github.com/cloudflare/circl v1.3.3/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA= +github.com/cloudflare/circl v1.3.7 h1:qlCDlTPz2n9fu58M0Nh1J/JzcFpfgkFHHX3O35r5vcU= +github.com/cloudflare/circl v1.3.7/go.mod h1:sRTcRWXGLrKw6yIGJ+l7amYJFfAXbZG0kBSc8r4zxgA= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/coinbase/waas-client-library-go v1.0.8 h1:AdbTmBQpsSUx947GZd5/68BhNBw1CSwTfE2PcnVy3Ao= github.com/coinbase/waas-client-library-go v1.0.8/go.mod h1:RVKozprfdfMiK92ATZUWHRs0EFGHQj4rbEJjzzZzI1I= @@ -234,8 +242,8 @@ github.com/docker/docker-credential-helpers v0.7.0/go.mod h1:rETQfLdHNT3foU5kuNk github.com/dsnet/compress v0.0.1 h1:PlZu0n3Tuv04TzpfPbrnI0HW/YwodEXDS+oPKahKF0Q= github.com/dsnet/compress v0.0.1/go.mod h1:Aw8dCMJ7RioblQeTqt88akK31OvO8Dhf5JflhBbQEHo= github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY= -github.com/dvsekhvalnov/jose2go v1.5.0 h1:3j8ya4Z4kMCwT5nXIKFSV84YS+HdqSSO0VsTQxaLAeM= -github.com/dvsekhvalnov/jose2go v1.5.0/go.mod h1:QsHjhyTlD/lAVqn/NSbVZmSCGeDehTB/mPZadG+mhXU= +github.com/dvsekhvalnov/jose2go v1.6.0 h1:Y9gnSnP4qEI0+/uQkHvFXeD2PLPJeXEL+ySMEA2EjTY= +github.com/dvsekhvalnov/jose2go v1.6.0/go.mod h1:QsHjhyTlD/lAVqn/NSbVZmSCGeDehTB/mPZadG+mhXU= github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a h1:mATvB/9r/3gvcejNsXKSkQ6lcIaNec2nyfOdlTBR2lU= github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM= github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= @@ -257,6 +265,8 @@ github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWo github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU= github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA= +github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0= +github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk= github.com/getsentry/sentry-go v0.25.0 h1:q6Eo+hS+yoJlTO3uu/azhQadsD8V+jQn2D8VvX1eOyI= github.com/getsentry/sentry-go v0.25.0/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w1gwTf25trprY= github.com/gliderlabs/ssh v0.3.5 h1:OcaySEmAQJgyYcArR+gGGTHCyE7nvhEMTlYY+Dp8CpY= @@ -271,8 +281,8 @@ github.com/go-git/go-billy/v5 v5.5.0 h1:yEY4yhzCDuMGSv83oGxiBotRzhwhNr8VZyphhiu+ github.com/go-git/go-billy/v5 v5.5.0/go.mod h1:hmexnoNsr2SJU1Ju67OaNz5ASJY3+sHgFRpCtpDCKow= github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399 h1:eMje31YglSBqCdIqdhKBW8lokaMrL3uTkpGYlE2OOT4= github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399/go.mod h1:1OCfN199q1Jm3HZlxleg+Dw/mwps2Wbk9frAWm+4FII= -github.com/go-git/go-git/v5 v5.10.1 h1:tu8/D8i+TWxgKpzQ3Vc43e+kkhXqtsZCKI/egajKnxk= -github.com/go-git/go-git/v5 v5.10.1/go.mod h1:uEuHjxkHap8kAl//V5F/nNWwqIYtP/402ddd05mp0wg= +github.com/go-git/go-git/v5 v5.11.0 h1:XIZc1p+8YzypNr34itUfSvYJcv+eYdTnTvOZ2vD3cA4= +github.com/go-git/go-git/v5 v5.11.0/go.mod h1:6GFcX2P3NM7FPBfpePbpLd21XxsgdAt+lKqXmCUiUCY= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= @@ -366,6 +376,8 @@ github.com/google/go-github/v42 v42.0.0 h1:YNT0FwjPrEysRkLIiKuEfSvBPCGKphW5aS5Px github.com/google/go-github/v42 v42.0.0/go.mod h1:jgg/jvyI0YlDOM1/ps6XYh04HNQ3vKf0CVko62/EhRg= github.com/google/go-github/v56 v56.0.0 h1:TysL7dMa/r7wsQi44BjqlwaHvwlFlqkK8CtBWCX3gb4= github.com/google/go-github/v56 v56.0.0/go.mod h1:D8cdcX98YWJvi7TLo7zM4/h8ZTx6u6fwGEkCdisopo0= +github.com/google/go-github/v57 v57.0.0 h1:L+Y3UPTY8ALM8x+TV0lg+IEBI+upibemtBD8Q9u7zHs= +github.com/google/go-github/v57 v57.0.0/go.mod h1:s0omdnye0hvK/ecLvpsGfJMiRt85PimQh4oygmLIxHw= github.com/google/go-querystring v0.0.0-20170111101155-53e6ce116135/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= @@ -513,6 +525,8 @@ github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZ github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mattn/go-sqlite3 v1.14.18 h1:JL0eqdCOq6DJVNPSvArO/bIV9/P7fbGrV00LZHc+5aI= github.com/mattn/go-sqlite3 v1.14.18/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= +github.com/mattn/go-sqlite3 v1.14.19 h1:fhGleo2h1p8tVChob4I9HpmVFIAkKGpiukdrgQbWfGI= +github.com/mattn/go-sqlite3 v1.14.19/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/mholt/archiver/v4 v4.0.0-alpha.8 h1:tRGQuDVPh66WCOelqe6LIGh0gwmfwxUrSSDunscGsRM= @@ -571,8 +585,6 @@ github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaR github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= github.com/paulbellamy/ratecounter v0.2.0 h1:2L/RhJq+HA8gBQImDXtLPrDXK5qAj6ozWVK/zFXVJGs= github.com/paulbellamy/ratecounter v0.2.0/go.mod h1:Hfx1hDpSGoqxkVVpBi/IlYD7kChlfo5C6hzIHwPqfFE= -github.com/pierrec/lz4/v4 v4.1.18 h1:xaKrnTkyoqfh1YItXl56+6KJNVYWlEEPuAQW9xsplYQ= -github.com/pierrec/lz4/v4 v4.1.18/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pierrec/lz4/v4 v4.1.19 h1:tYLzDnjDXh9qIxSTKHwXwOYmm9d887Y7Y1ZkyXYHAN4= github.com/pierrec/lz4/v4 v4.1.19/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= @@ -726,8 +738,8 @@ golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0 golang.org/x/crypto v0.3.1-0.20221117191849-2c476679df9a/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= -golang.org/x/crypto v0.16.0 h1:mMMrFzRSCF0GvB7Ne27XVtVAaXLrPmgPC7/v0tkwHaY= -golang.org/x/crypto v0.16.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= +golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= +golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -738,6 +750,10 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20231127185646-65229373498e h1:Gvh4YaCaXNs6dKTlfgismwWZKyjVZXwOPfIyUaqU3No= golang.org/x/exp v0.0.0-20231127185646-65229373498e/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI= +golang.org/x/exp v0.0.0-20240103183307-be819d1f06fc h1:ao2WRsKSzW6KuUY9IWPwWahcHCgR0s52IfwutMfEbdM= +golang.org/x/exp v0.0.0-20240103183307-be819d1f06fc/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI= +golang.org/x/exp v0.0.0-20240110193028-0dcbfd608b1e h1:723BNChdd0c2Wk6WOE320qGBiPtYx0F0Bbm1kriShfE= +golang.org/x/exp v0.0.0-20240110193028-0dcbfd608b1e/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= diff --git a/hack/snifftest/main.go b/hack/snifftest/main.go index 651f51541..219e7c57d 100644 --- a/hack/snifftest/main.go +++ b/hack/snifftest/main.go @@ -203,6 +203,7 @@ func main() { } }, true, + false, ) logger.Info("scanning repo", "repo", r) diff --git a/main.go b/main.go index 8b1859d32..ff1651c78 100644 --- a/main.go +++ b/main.go @@ -5,10 +5,12 @@ import ( "net/http" _ "net/http/pprof" "os" + "os/signal" "runtime" "strconv" "strings" "syscall" + "time" "github.com/alecthomas/kingpin/v2" "github.com/felixge/fgprof" @@ -55,6 +57,7 @@ var ( noUpdate = cli.Flag("no-update", "Don't check for updates.").Bool() fail = cli.Flag("fail", "Exit with code 183 if results are found.").Bool() verifiers = cli.Flag("verifier", "Set custom verification endpoints.").StringMap() + customVerifiersOnly = cli.Flag("custom-verifiers-only", "Only use custom verification endpoints.").Bool() archiveMaxSize = cli.Flag("archive-max-size", "Maximum size of archive to scan. (Byte units eg. 512B, 2KB, 4MB)").Bytes() archiveMaxDepth = cli.Flag("archive-max-depth", "Maximum depth of archive to scan.").Int() archiveTimeout = cli.Flag("archive-timeout", "Maximum time to spend extracting an archive.").Duration() @@ -216,17 +219,39 @@ func main() { if err != nil { logFatal(err, "error occurred with trufflehog updater 🐷") } - - ctx := context.Background() - - go cleantemp.RunCleanupLoop(ctx) } func run(state overseer.State) { - ctx := context.Background() + ctx, cancel := context.WithCancelCause(context.Background()) + defer cancel(nil) + + go func() { + if err := cleantemp.CleanTempArtifacts(ctx); err != nil { + ctx.Logger().Error(err, "error cleaning temporary artifacts") + } + }() + logger := ctx.Logger() logFatal := logFatalFunc(logger) + killSignal := make(chan os.Signal, 1) + signal.Notify(killSignal, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT) + go func() { + <-killSignal + logger.Info("Received signal, shutting down.") + cancel(fmt.Errorf("canceling context due to signal")) + + if err := cleantemp.CleanTempArtifacts(ctx); err != nil { + logger.Error(err, "error cleaning temporary artifacts") + } else { + logger.Info("cleaned temporary artifacts") + } + + time.Sleep(time.Second * 10) + logger.Info("10 seconds elapsed. Forcing shutdown.") + os.Exit(0) + }() + logger.V(2).Info(fmt.Sprintf("trufflehog %s", version.BuildVersion)) if *githubScanToken != "" { @@ -341,8 +366,9 @@ func run(state overseer.State) { "detector", id, ) } - // TODO: Add flag to ignore the default endpoint. - urls = append(urls, customizer.DefaultEndpoint()) + if !*customVerifiersOnly || len(urls) == 0 { + urls = append(urls, customizer.DefaultEndpoint()) + } if err := customizer.SetEndpoints(urls...); err != nil { logFatal(err, "failed configuring custom endpoint for detector", "detector", id) } @@ -446,10 +472,6 @@ func run(state overseer.State) { logFatal(err, "Failed to scan GitLab.") } case filesystemScan.FullCommand(): - filter, err := common.FilterFromFiles(*filesystemScanIncludePaths, *filesystemScanExcludePaths) - if err != nil { - logFatal(err, "could not create filter") - } if len(*filesystemDirectories) > 0 { ctx.Logger().Info("--directory flag is deprecated, please pass directories as arguments") } @@ -457,8 +479,9 @@ func run(state overseer.State) { paths = append(paths, *filesystemPaths...) paths = append(paths, *filesystemDirectories...) cfg := sources.FilesystemConfig{ - Paths: paths, - Filter: filter, + Paths: paths, + IncludePathsFile: *filesystemScanIncludePaths, + ExcludePathsFile: *filesystemScanExcludePaths, } if err = e.ScanFileSystem(ctx, cfg); err != nil { logFatal(err, "Failed to scan filesystem") diff --git a/pkg/cache/cache.go b/pkg/cache/cache.go index 65a0bf2a8..da7156ac7 100644 --- a/pkg/cache/cache.go +++ b/pkg/cache/cache.go @@ -4,9 +4,9 @@ package cache // Cache is used to store key/value pairs. type Cache interface { // Set stores the given key/value pair. - Set(string, string) + Set(string, any) // Get returns the value for the given key and a boolean indicating if the key was found. - Get(string) (string, bool) + Get(string) (any, bool) // Exists returns true if the given key exists in the cache. Exists(string) bool // Delete the given key from the cache. @@ -18,7 +18,7 @@ type Cache interface { // Keys returns all keys in the cache. Keys() []string // Values returns all values in the cache. - Values() []string + Values() []any // Contents returns all keys in the cache encoded as a string. Contents() string } diff --git a/pkg/cache/memory/memory.go b/pkg/cache/memory/memory.go index b4e9c87c3..ce5bd4365 100644 --- a/pkg/cache/memory/memory.go +++ b/pkg/cache/memory/memory.go @@ -5,52 +5,77 @@ import ( "time" "github.com/patrickmn/go-cache" - - "github.com/trufflesecurity/trufflehog/v3/pkg/context" ) const ( - expirationInterval = 12 * time.Hour - purgeInterval = 13 * time.Hour - defaultExpiration = cache.DefaultExpiration + defaultExpirationInterval = 12 * time.Hour + defaultPurgeInterval = 13 * time.Hour + defaultExpiration = cache.DefaultExpiration ) -// Cache is a wrapper around the go-cache library. +// Cache wraps the go-cache library to provide an in-memory key-value store. type Cache struct { - c *cache.Cache + c *cache.Cache + expiration time.Duration + purgeInterval time.Duration } -// New constructs a new in-memory cache. -func New() *Cache { - c := cache.New(expirationInterval, purgeInterval) - return &Cache{c: c} +// CacheOption defines a function type used for configuring a Cache. +type CacheOption func(*Cache) + +// WithExpirationInterval returns a CacheOption to set the expiration interval of cache items. +// The interval determines the duration a cached item remains in the cache before it is expired. +func WithExpirationInterval(interval time.Duration) CacheOption { + return func(c *Cache) { c.expiration = interval } +} + +// WithPurgeInterval returns a CacheOption to set the interval at which the cache purges expired items. +// Regular purging helps in freeing up memory by removing stale entries. +func WithPurgeInterval(interval time.Duration) CacheOption { + return func(c *Cache) { c.purgeInterval = interval } +} + +// New constructs a new in-memory cache instance with optional configurations. +// By default, it sets the expiration and purge intervals to 12 and 13 hours, respectively. +// These defaults can be overridden using the functional options: WithExpirationInterval and WithPurgeInterval. +func New(opts ...CacheOption) *Cache { + return NewWithData(nil, opts...) +} + +// CacheEntry represents a single entry in the cache, consisting of a key and its corresponding value. +type CacheEntry struct { + // Key is the unique identifier for the entry. + Key string + // Value is the data stored in the entry. + Value any } // NewWithData constructs a new in-memory cache with existing data. -func NewWithData(ctx context.Context, data []string) *Cache { - ctx.Logger().V(3).Info("Loading cache", "num-items", len(data)) - - items := make(map[string]cache.Item, len(data)) - for _, d := range data { - items[d] = cache.Item{Object: d, Expiration: int64(defaultExpiration)} +// It also accepts CacheOption parameters to override default configuration values. +func NewWithData(data []CacheEntry, opts ...CacheOption) *Cache { + instance := &Cache{expiration: defaultExpirationInterval, purgeInterval: defaultPurgeInterval} + for _, opt := range opts { + opt(instance) } - c := cache.NewFrom(expirationInterval, purgeInterval, items) - return &Cache{c: c} + // Convert data slice to map required by go-cache. + items := make(map[string]cache.Item, len(data)) + for _, d := range data { + items[d.Key] = cache.Item{Object: d.Value, Expiration: int64(defaultExpiration)} + } + + instance.c = cache.NewFrom(instance.expiration, instance.purgeInterval, items) + return instance } // Set adds a key-value pair to the cache. -func (c *Cache) Set(key, value string) { +func (c *Cache) Set(key string, value any) { c.c.Set(key, value, defaultExpiration) } // Get returns the value for the given key. -func (c *Cache) Get(key string) (string, bool) { - res, ok := c.c.Get(key) - if !ok { - return "", ok - } - return res.(string), ok +func (c *Cache) Get(key string) (any, bool) { + return c.c.Get(key) } // Exists returns true if the given key exists in the cache. @@ -85,11 +110,11 @@ func (c *Cache) Keys() []string { } // Values returns all values in the cache. -func (c *Cache) Values() []string { +func (c *Cache) Values() []any { items := c.c.Items() - res := make([]string, 0, len(items)) + res := make([]any, 0, len(items)) for _, v := range items { - res = append(res, v.Object.(string)) + res = append(res, v.Object) } return res } diff --git a/pkg/cache/memory/memory_test.go b/pkg/cache/memory/memory_test.go index e1df315c8..0ec9c926b 100644 --- a/pkg/cache/memory/memory_test.go +++ b/pkg/cache/memory/memory_test.go @@ -7,8 +7,6 @@ import ( "testing" "github.com/google/go-cmp/cmp" - - logContext "github.com/trufflesecurity/trufflehog/v3/pkg/context" ) func TestCache(t *testing.T) { @@ -34,7 +32,7 @@ func TestCache(t *testing.T) { // Test delete. c.Delete("key1") v, ok = c.Get("key1") - if ok || v != "" { + if ok || v != nil { t.Fatalf("Unexpected value for key1 after delete: %v, %v", v, ok) } @@ -42,7 +40,7 @@ func TestCache(t *testing.T) { c.Set("key10", "key10") c.Clear() v, ok = c.Get("key10") - if ok || v != "" { + if ok || v != nil { t.Fatalf("Unexpected value for key10 after clear: %v, %v", v, ok) } @@ -60,7 +58,10 @@ func TestCache(t *testing.T) { } // Test getting only the values. - vals := c.Values() + vals := make([]string, 0, c.Count()) + for _, v := range c.Values() { + vals = append(vals, v.(string)) + } sort.Strings(vals) sort.Strings(values) if !cmp.Equal(values, vals) { @@ -82,7 +83,8 @@ func TestCache(t *testing.T) { } func TestCache_NewWithData(t *testing.T) { - c := NewWithData(logContext.Background(), []string{"key1", "key2", "key3"}) + data := []CacheEntry{{"key1", "value1"}, {"key2", "value2"}, {"key3", "value3"}} + c := NewWithData(data) // Test the count. if c.Count() != 3 { diff --git a/pkg/cleantemp/cleantemp.go b/pkg/cleantemp/cleantemp.go index b638aeb70..2abc7e680 100644 --- a/pkg/cleantemp/cleantemp.go +++ b/pkg/cleantemp/cleantemp.go @@ -2,12 +2,12 @@ package cleantemp import ( "fmt" + "io" "os" "path/filepath" "regexp" "strconv" "strings" - "time" "github.com/mitchellh/go-ps" @@ -63,65 +63,57 @@ func CleanTempArtifacts(ctx logContext.Context) error { } } - tempDir := os.TempDir() - artifacts, err := os.ReadDir(tempDir) - if err != nil { - return fmt.Errorf("error reading temp dir: %w", err) + if len(pids) == 0 { + ctx.Logger().V(5).Info("No trufflehog processes were found") + return nil } - for _, artifact := range artifacts { - if trufflehogRE.MatchString(artifact.Name()) { + tempDir := os.TempDir() + dir, err := os.Open(tempDir) + if err != nil { + return fmt.Errorf("error opening temp dir: %w", err) + } + defer dir.Close() + + for { + entries, err := dir.ReadDir(1) // read only one entry + if err != nil { + if err == io.EOF { + break + } + continue + } + entry := entries[0] + + if trufflehogRE.MatchString(entry.Name()) { + // Mark these artifacts initially as ones that should be deleted. shouldDelete := true // Check if the name matches any live PIDs. + // Potential race condition here if a PID is started and creates tmp data after the initial check. for _, pidval := range pids { - if strings.Contains(artifact.Name(), fmt.Sprintf("-%s-", pidval)) { + if strings.Contains(entry.Name(), fmt.Sprintf("-%s-", pidval)) { shouldDelete = false break } } if shouldDelete { - artifactPath := filepath.Join(tempDir, artifact.Name()) - - var err error - if artifact.IsDir() { - err = os.RemoveAll(artifactPath) + path := filepath.Join(tempDir, entry.Name()) + isDir := entry.IsDir() + if isDir { + err = os.RemoveAll(path) } else { - err = os.Remove(artifactPath) + err = os.Remove(path) } if err != nil { - return fmt.Errorf("Error deleting temp artifact: %s", artifactPath) + return fmt.Errorf("error deleting temp artifact (dir: %v) %s: %w", isDir, path, err) } - ctx.Logger().Info("Deleted orphaned temp artifact", "artifact", artifactPath) + ctx.Logger().V(4).Info("Deleted orphaned temp artifact", "artifact", path) } } } return nil } - -// RunCleanupLoop runs a loop that cleans up orphaned directories every 15 seconds. -func RunCleanupLoop(ctx logContext.Context) { - err := CleanTempArtifacts(ctx) - if err != nil { - ctx.Logger().Error(err, "Error cleaning up orphaned directories ") - } - - const cleanupLoopInterval = 15 * time.Second - ticker := time.NewTicker(cleanupLoopInterval) - defer ticker.Stop() - - for { - select { - case <-ticker.C: - if err := CleanTempArtifacts(ctx); err != nil { - ctx.Logger().Error(err, "error cleaning up orphaned directories") - } - case <-ctx.Done(): - ctx.Logger().Info("Cleanup loop exiting due to context cancellation") - return - } - } -} diff --git a/pkg/common/vars.go b/pkg/common/vars.go index e21c3dd7e..c674bd9da 100644 --- a/pkg/common/vars.go +++ b/pkg/common/vars.go @@ -28,6 +28,7 @@ var ( "wav", "flac", "webp", + "pdf", // images "png", @@ -79,6 +80,9 @@ var ( "vxd": {}, // Virtual device driver in Windows "sfx": {}, // Self-extracting archive "bundle": {}, // Mac OS X application bundle + "pyo": {}, // Compiled Python file + "pyc": {}, // Compiled Python file + "sym": {}, // Symbolic link, Unix/Linux } ) diff --git a/pkg/detectors/github/github.go b/pkg/detectors/github/github.go index ad8346cd1..192f3d97a 100644 --- a/pkg/detectors/github/github.go +++ b/pkg/detectors/github/github.go @@ -43,6 +43,8 @@ type userRes struct { Name string `json:"name"` Company string `json:"company"` UserURL string `json:"html_url"` + // Included in GitHub Enterprise Server. + LdapDN string `json:"ldap_dn"` } // Keywords are used for efficiently pre-filtering chunks. @@ -77,12 +79,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result client := common.SaneHttpClient() // https://developer.github.com/v3/users/#get-the-authenticated-user for _, url := range s.Endpoints(s.DefaultEndpoint()) { - req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("%s/user", url), nil) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, fmt.Sprintf("%s/user", url), nil) if err != nil { continue } - req.Header.Add("Content-Type", "application/json; charset=utf-8") - req.Header.Add("Authorization", fmt.Sprintf("token %s", token)) + + req.Header.Set("Content-Type", "application/json; charset=utf-8") + req.Header.Set("Authorization", fmt.Sprintf("token %s", token)) res, err := client.Do(req) if err == nil { if res.StatusCode >= 200 && res.StatusCode < 300 { @@ -94,10 +97,28 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result s1.ExtraData["username"] = userResponse.Login s1.ExtraData["url"] = userResponse.UserURL s1.ExtraData["account_type"] = userResponse.Type - s1.ExtraData["site_admin"] = fmt.Sprintf("%t", userResponse.SiteAdmin) - s1.ExtraData["name"] = userResponse.Name - s1.ExtraData["company"] = userResponse.Company - s1.ExtraData["scopes"] = res.Header.Get("X-OAuth-Scopes") + if userResponse.SiteAdmin { + s1.ExtraData["site_admin"] = "true" + } + if userResponse.Name != "" { + s1.ExtraData["name"] = userResponse.Name + } + if userResponse.Company != "" { + s1.ExtraData["company"] = userResponse.Company + } + if userResponse.LdapDN != "" { + s1.ExtraData["ldap_dn"] = userResponse.LdapDN + } + + // GitHub does not seem to consistently return this header. + scopes := res.Header.Get("X-OAuth-Scopes") + if scopes != "" { + s1.ExtraData["scopes"] = scopes + } + expiry := res.Header.Get("github-authentication-token-expiration") + if expiry != "" { + s1.ExtraData["expires_at"] = expiry + } } } } else { diff --git a/pkg/detectors/github_old/github_old.go b/pkg/detectors/github_old/github_old.go index b5edc1596..45724309c 100644 --- a/pkg/detectors/github_old/github_old.go +++ b/pkg/detectors/github_old/github_old.go @@ -39,6 +39,8 @@ type userRes struct { Name string `json:"name"` Company string `json:"company"` UserURL string `json:"html_url"` + // Included in GitHub Enterprise Server. + LdapDN string `json:"ldap_dn"` } // Keywords are used for efficiently pre-filtering chunks. @@ -78,12 +80,12 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result client := common.SaneHttpClient() // https://developer.github.com/v3/users/#get-the-authenticated-user for _, url := range s.Endpoints(s.DefaultEndpoint()) { - req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("%s/user", url), nil) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, fmt.Sprintf("%s/user", url), nil) if err != nil { continue } - req.Header.Add("Content-Type", "application/json; charset=utf-8") - req.Header.Add("Authorization", fmt.Sprintf("token %s", token)) + req.Header.Set("Content-Type", "application/json; charset=utf-8") + req.Header.Set("Authorization", fmt.Sprintf("token %s", token)) res, err := client.Do(req) if err == nil { if res.StatusCode >= 200 && res.StatusCode < 300 { @@ -98,10 +100,28 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result s1.ExtraData["username"] = userResponse.Login s1.ExtraData["url"] = userResponse.UserURL s1.ExtraData["account_type"] = userResponse.Type - s1.ExtraData["site_admin"] = fmt.Sprintf("%t", userResponse.SiteAdmin) - s1.ExtraData["name"] = userResponse.Name - s1.ExtraData["company"] = userResponse.Company - s1.ExtraData["scopes"] = res.Header.Get("X-OAuth-Scopes") + if userResponse.SiteAdmin { + s1.ExtraData["site_admin"] = "true" + } + if userResponse.Name != "" { + s1.ExtraData["name"] = userResponse.Name + } + if userResponse.Company != "" { + s1.ExtraData["company"] = userResponse.Company + } + if userResponse.LdapDN != "" { + s1.ExtraData["ldap_dn"] = userResponse.LdapDN + } + + // GitHub does not seem to consistently return this header. + scopes := res.Header.Get("X-OAuth-Scopes") + if scopes != "" { + s1.ExtraData["scopes"] = scopes + } + expiry := res.Header.Get("github-authentication-token-expiration") + if expiry != "" { + s1.ExtraData["expires_at"] = expiry + } } } } diff --git a/pkg/detectors/huggingface/huggingface.go b/pkg/detectors/huggingface/huggingface.go index 725c3678c..4bdf14c9a 100644 --- a/pkg/detectors/huggingface/huggingface.go +++ b/pkg/detectors/huggingface/huggingface.go @@ -23,13 +23,13 @@ var _ detectors.Detector = (*Scanner)(nil) var ( defaultClient = common.SaneHttpClient() // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. - keyPat = regexp.MustCompile(`\bhf_[a-zA-Z0-9]{34}\b`) + keyPat = regexp.MustCompile(`\b(?:hf_|api_org_)[a-zA-Z0-9]{34}\b`) ) // Keywords are used for efficiently pre-filtering chunks. // Use identifiers in the secret preferably, or the provider name. func (s Scanner) Keywords() []string { - return []string{"huggingface", "hugging_face", "hf"} // Huggingface docs occasionally use "hf" instead of "huggingface" + return []string{"hf_", "api_org_"} // Huggingface docs occasionally use "hf" instead of "huggingface" } // FromData will find and optionally verify Huggingface secrets in a given set of bytes. @@ -92,15 +92,29 @@ func (s Scanner) verifyResult(ctx context.Context, apiKey string) (bool, map[str return true, nil, err } - t := whoamiRes.Auth.AccessToken + var tokenInfo string + switch { + case whoamiRes.Auth.AccessToken.DisplayName != "" || whoamiRes.Auth.AccessToken.Role != "": + // hf_xxxx token + t := whoamiRes.Auth.AccessToken + tokenInfo = fmt.Sprintf("%s (%s)", t.DisplayName, t.Role) + + case whoamiRes.Auth.Type != "": + // api_org_xxxx token + tokenInfo = whoamiRes.Auth.Type + + default: + tokenInfo = "Unknown Token Type" + } + extraData := map[string]string{ "Username": whoamiRes.Name, "Email": whoamiRes.Email, - "Token": fmt.Sprintf("%s (%s)", t.DisplayName, t.Role), + "Token": tokenInfo, } // Condense a list of organizations + roles. - var orgs []string + orgs := make([]string, 0, len(whoamiRes.Organizations)) for _, org := range whoamiRes.Organizations { orgs = append(orgs, fmt.Sprintf("%s:%s", org.Name, org.Role)) } @@ -136,7 +150,8 @@ type organization struct { type auth struct { AccessToken struct { - DisplayName string `json:"displayName"` - Role string `json:"role"` - } `json:"accessToken"` + DisplayName string `json:"displayName,omitempty"` + Role string `json:"role,omitempty"` + } `json:"accessToken,omitempty"` + Type string `json:"type,omitempty"` } diff --git a/pkg/detectors/parseur/parseur.go b/pkg/detectors/parseur/parseur.go index 4cad1da9d..5f389edae 100644 --- a/pkg/detectors/parseur/parseur.go +++ b/pkg/detectors/parseur/parseur.go @@ -12,16 +12,18 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct { + client *http.Client +} // Ensure the Scanner satisfies the interface at compile time var _ detectors.Detector = (*Scanner)(nil) var ( - client = common.SaneHttpClient() + defaultClient = common.SaneHttpClient() // Make sure that your group is surrounded in boundary characters such as below to reduce false positives - keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"parseur"}) + `\b([a-f0-9]{40})\b`) + keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"parseur[^il]"}) + `\b([a-f0-9]{40})\b`) ) // Keywords are used for efficiently pre-filtering chunks. @@ -35,35 +37,30 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result dataStr := string(data) matches := keyPat.FindAllStringSubmatch(dataStr, -1) - for _, match := range matches { if len(match) != 2 { continue } - resMatch := strings.TrimSpace(match[1]) + resMatch := strings.TrimSpace(match[1]) s1 := detectors.Result{ DetectorType: detectorspb.DetectorType_Parseur, Raw: []byte(resMatch), } if verify { - req, err := http.NewRequestWithContext(ctx, "GET", "https://api.parseur.com/", nil) - if err != nil { - continue + if s.client == nil { + s.client = defaultClient } - req.Header.Add("Authorization", fmt.Sprintf("Token %s", resMatch)) - res, err := client.Do(req) - if err == nil { - defer res.Body.Close() - if res.StatusCode >= 200 && res.StatusCode < 300 { - s1.Verified = true - } else { - // This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key - if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) { - continue - } - } + isVerified, verificationErr := verifyResult(ctx, s.client, resMatch) + s1.Verified = isVerified + s1.SetVerificationError(verificationErr, resMatch) + } + + if !s1.Verified { + // This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key + if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) { + continue } } @@ -73,6 +70,25 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result return results, nil } +func verifyResult(ctx context.Context, client *http.Client, token string) (bool, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://api.parseur.com/", nil) + if err != nil { + return false, err + } + + req.Header.Add("Authorization", fmt.Sprintf("Token %s", token)) + res, err := client.Do(req) + if err != nil { + return false, err + } + + defer res.Body.Close() + if res.StatusCode >= 200 && res.StatusCode < 300 { + return true, nil + } + return false, nil +} + func (s Scanner) Type() detectorspb.DetectorType { return detectorspb.DetectorType_Parseur } diff --git a/pkg/detectors/parseur/parseur_test.go b/pkg/detectors/parseur/parseur_test.go index fcc9d0c36..9c30cf98d 100644 --- a/pkg/detectors/parseur/parseur_test.go +++ b/pkg/detectors/parseur/parseur_test.go @@ -7,12 +7,117 @@ import ( "time" "github.com/kylelemons/godebug/pretty" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) +func TestParseur_Pattern(t *testing.T) { + tests := []struct { + name string + data string + shouldMatch bool + match string + }{ + // True positives + { + name: "valid", + data: `const parseurToken = "6813a07afc6b4ed35518635c6fb70abf4e721962";`, + shouldMatch: true, + match: "6813a07afc6b4ed35518635c6fb70abf4e721962", + }, + // This technically isn't valid but shouldn't be excluded based on the current pattern. + { + name: "valid", + data: `commit 6813a07afc6b4ed35518635c6fb70abf4e721962 +Author: Stéphane Borel +Date: Thu Dec 30 13:59:59 1999 +0000 + + * Modifications de quelques erreurs sur le parseur + +commit 2c65bd981d308d264aa0c07083b2bc914905deb3`, + shouldMatch: true, + match: "2c65bd981d308d264aa0c07083b2bc914905deb3", + }, + + // False positives + { + name: `invalid_parseuri_package.json`, + data: `{ + "dist": { + "shasum": "80204a50d4dbb779bfdc6ebe2778d90e4bce320a", + "tarball": "https://registry.npmjs.org/parseuri/-/parseuri-0.0.5.tgz" + }, + "gitHead": "792c9a63162a4484eb6b4f95fc611ccf224a24b6",`, + shouldMatch: false, + }, + // https://github.com/airalab/airapkgs/blob/cb3f8021303f79345f65b5328b75117044bde852/pkgs/servers/mesh/meshviewer/yarn.nix#L6066 + { + name: `invalid_parseuri_nix`, + data: ` + { + name = "parseuri-0.0.5.tgz"; + path = fetchurl { + name = "parseuri-0.0.5.tgz"; + url = "https://registry.yarnpkg.com/parseuri/-/parseuri-0.0.5.tgz"; + sha1 = "80204a50d4dbb779bfdc6ebe2778d90e4bce320a"; + }; + }`, + shouldMatch: false, + }, + { + name: `invalid_parseurl_yarn`, + data: `parseurl@~1.3.1: + version "1.3.1" + resolved "https://registry.yarnpkg.com/parseurl/-/parseurl-1.3.1.tgz#c8ab8c9223ba34888aa64a297b28853bec18da56"`, + shouldMatch: false, + }, + // https://github.com/tolerious/django-wechat/blob/18f3f2d5d8377c7dde8700afc5977861c8488b68/django_weixin/Sample.py#L30 + { + name: `invalid_parseurl_func`, + data: `#sVerifyMsgSig=HttpUtils.ParseUrl("msg_signature") + sVerifyMsgSig="5c45ff5e21c57e6ad56bac8758b79b1d9ac89fd3"`, + shouldMatch: false, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + s := Scanner{} + + results, err := s.FromData(context.Background(), false, []byte(test.data)) + if err != nil { + t.Errorf("Parseur.FromData() error = %v", err) + return + } + + if test.shouldMatch { + if len(results) == 0 { + t.Errorf("%s: did not receive a match for '%v' when one was expected", test.name, test.data) + return + } + expected := test.data + if test.match != "" { + expected = test.match + } + result := results[0] + resultData := string(result.Raw) + if resultData != expected { + t.Errorf("%s: did not receive expected match.\n\texpected: '%s'\n\t actual: '%s'", test.name, expected, resultData) + return + } + } else { + if len(results) > 0 { + t.Errorf("%s: received a match for '%v' when one wasn't wanted", test.name, test.data) + return + } + } + }) + } +} + func TestParseur_FromChunk(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) defer cancel() diff --git a/pkg/detectors/postgres/postgres.go b/pkg/detectors/postgres/postgres.go new file mode 100644 index 000000000..cb22536cd --- /dev/null +++ b/pkg/detectors/postgres/postgres.go @@ -0,0 +1,254 @@ +package postgres + +import ( + "context" + "database/sql" + "fmt" + "net/url" + "regexp" + "strings" + "time" + + _ "github.com/lib/pq" // PostgreSQL driver + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +) + +const ( + defaultPort = "5432" + defaultHost = "localhost" +) + +var ( + _ detectors.Detector = (*Scanner)(nil) + uriPattern = regexp.MustCompile(`\b(?i)postgresql://[\S]+\b`) + hostnamePattern = regexp.MustCompile(`(?i)(?:host|server|address).{0,40}?(\b[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*\b)`) + portPattern = regexp.MustCompile(`(?i)(?:port|p).{0,40}?(\b[0-9]{1,5}\b)`) + usernamePattern = regexp.MustCompile(`(?im)(?:user|usr)\S{0,40}?[:=\s]{1,3}[ '"=]{0,1}([^:'"\s]{4,40})`) + passwordPattern = regexp.MustCompile(`(?im)(?:pass)\S{0,40}?[:=\s]{1,3}[ '"=]{0,1}([^:'"\s]{4,40})`) +) + +type Scanner struct{} + +func (s Scanner) Keywords() []string { + return []string{"postgres", "psql", "pghost"} +} + +func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) ([]detectors.Result, error) { + var results []detectors.Result + var pgURLs []url.URL + pgURLs = append(pgURLs, findUriMatches(string(data))) + pgURLs = append(pgURLs, findComponentMatches(verify, string(data))...) + + for _, pgURL := range pgURLs { + if pgURL.User == nil { + continue + } + username := pgURL.User.Username() + password, _ := pgURL.User.Password() + hostport := pgURL.Host + result := detectors.Result{ + DetectorType: detectorspb.DetectorType_Postgres, + Raw: []byte(hostport + username + password), + RawV2: []byte(hostport + username + password), + } + + if verify { + timeoutInSeconds := getDeadlineInSeconds(ctx) + isVerified, verificationErr := verifyPostgres(&pgURL, timeoutInSeconds) + result.Verified = isVerified + result.SetVerificationError(verificationErr, password) + } + + if !result.Verified && detectors.IsKnownFalsePositive(password, detectors.DefaultFalsePositives, true) { + continue + } + results = append(results, result) + } + + return results, nil +} + +func getDeadlineInSeconds(ctx context.Context) int { + deadline, ok := ctx.Deadline() + if !ok { + // Context does not have a deadline + return 0 + } + + duration := time.Until(deadline) + return int(duration.Seconds()) +} + +func findUriMatches(dataStr string) url.URL { + var pgURL url.URL + for _, uri := range uriPattern.FindAllString(dataStr, -1) { + pgURL, err := url.Parse(uri) + if err != nil { + continue + } + if pgURL.User != nil { + return *pgURL + } + } + return pgURL +} + +// check if postgres is running +func postgresRunning(hostname, port string) bool { + connStr := fmt.Sprintf("host=%s port=%s sslmode=disable", hostname, port) + db, err := sql.Open("postgres", connStr) + if err != nil { + return false + } + defer db.Close() + return true +} + +func findComponentMatches(verify bool, dataStr string) []url.URL { + usernameMatches := usernamePattern.FindAllStringSubmatch(dataStr, -1) + passwordMatches := passwordPattern.FindAllStringSubmatch(dataStr, -1) + hostnameMatches := hostnamePattern.FindAllStringSubmatch(dataStr, -1) + portMatches := portPattern.FindAllStringSubmatch(dataStr, -1) + + var pgURLs []url.URL + + hosts := findHosts(verify, hostnameMatches, portMatches) + + for _, username := range dedupMatches(usernameMatches) { + for _, password := range dedupMatches(passwordMatches) { + for _, host := range hosts { + hostname, port := strings.Split(host, ":")[0], strings.Split(host, ":")[1] + if combinedLength := len(username) + len(password) + len(hostname); combinedLength > 255 { + continue + } + postgresURL := url.URL{ + Scheme: "postgresql", + User: url.UserPassword(username, password), + Host: fmt.Sprintf("%s:%s", hostname, port), + } + pgURLs = append(pgURLs, postgresURL) + } + } + } + return pgURLs +} + +// if verification is turned on, and we can confirm that postgres is running on at least one host, +// return only hosts where it's running. otherwise return all hosts. +func findHosts(verify bool, hostnameMatches, portMatches [][]string) []string { + hostnames := dedupMatches(hostnameMatches) + ports := dedupMatches(portMatches) + var hosts []string + + if len(hostnames) < 1 { + hostnames = append(hostnames, defaultHost) + } + + if len(ports) < 1 { + ports = append(ports, defaultPort) + } + + for _, hostname := range hostnames { + for _, port := range ports { + hosts = append(hosts, fmt.Sprintf("%s:%s", hostname, port)) + } + } + + if verify { + var verifiedHosts []string + for _, host := range hosts { + parts := strings.Split(host, ":") + hostname, port := parts[0], parts[1] + if postgresRunning(hostname, port) { + verifiedHosts = append(verifiedHosts, host) + } + } + if len(verifiedHosts) > 0 { + return verifiedHosts + } + } + + return hosts +} + +// deduplicate matches in order to reduce the number of verification requests +func dedupMatches(matches [][]string) []string { + setOfMatches := make(map[string]struct{}) + for _, match := range matches { + if len(match) > 1 { + setOfMatches[match[1]] = struct{}{} + } + } + var results []string + for match := range setOfMatches { + results = append(results, match) + } + return results +} + +func verifyPostgres(pgURL *url.URL, timeoutInSeconds int) (bool, error) { + if pgURL.User == nil { + return false, nil + } + username := pgURL.User.Username() + password, _ := pgURL.User.Password() + + hostname, port := pgURL.Hostname(), pgURL.Port() + if hostname == "" { + hostname = defaultHost + } + if port == "" { + port = defaultPort + } + + sslmode := determineSSLMode(pgURL) + + connStr := fmt.Sprintf("user=%s password=%s host=%s port=%s sslmode=%s", username, password, hostname, port, sslmode) + if timeoutInSeconds > 0 { + connStr = fmt.Sprintf("%s connect_timeout=%d", connStr, timeoutInSeconds) + } + + db, err := sql.Open("postgres", connStr) + if err != nil { + if strings.Contains(err.Error(), "connection refused") { + // inactive host + return false, nil + } + return false, err + } + defer db.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) + defer cancel() + + err = db.PingContext(ctx) + if err == nil { + return true, nil + } else if strings.Contains(err.Error(), "password authentication failed") || // incorrect username or password + strings.Contains(err.Error(), "connection refused") { // inactive host + return false, nil + } + + // if ssl is not enabled, manually fall-back to sslmode=disable + if strings.Contains(err.Error(), "SSL is not enabled on the server") { + pgURL.RawQuery = fmt.Sprintf("sslmode=%s", "disable") + return verifyPostgres(pgURL, timeoutInSeconds) + } + return false, err +} + +func determineSSLMode(pgURL *url.URL) string { + // default ssl mode is "prefer" per https://www.postgresql.org/docs/current/libpq-ssl.html + // but is currently not implemented in the driver per https://github.com/lib/pq/issues/1006 + // default for the driver is "require". ideally we would use "allow" but that is also not supported by the driver. + sslmode := "require" + if sslQuery, ok := pgURL.Query()["sslmode"]; ok && len(sslQuery) > 0 { + sslmode = sslQuery[0] + } + return sslmode +} + +func (s Scanner) Type() detectorspb.DetectorType { + return detectorspb.DetectorType_Postgres +} diff --git a/pkg/detectors/postgres/postgres_test.go b/pkg/detectors/postgres/postgres_test.go new file mode 100644 index 000000000..08da6dd2f --- /dev/null +++ b/pkg/detectors/postgres/postgres_test.go @@ -0,0 +1,340 @@ +//go:build detectors +// +build detectors + +package postgres + +import ( + "bytes" + "context" + "errors" + "fmt" + "os/exec" + "strings" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +) + +var postgresDockerHash string + +const ( + postgresUser = "postgres" + postgresPass = "23201dabb56ca236f3dc6736c0f9afad" + postgresHost = "localhost" + postgresPort = "5433" + + inactiveUser = "inactive" + inactivePass = "inactive" + inactivePort = "61000" + inactiveHost = "192.0.2.0" +) + +func TestPostgres_FromChunk(t *testing.T) { + startPostgres() + defer stopPostgres() + + type args struct { + ctx context.Context + data []byte + verify bool + } + tests := []struct { + name string + s Scanner + args args + want []detectors.Result + wantErr bool + }{ + { + name: "not found", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte("You cannot find the secret within"), + verify: true, + }, + want: nil, + wantErr: false, + }, + { + name: "found with seperated credentials, verified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf(` + POSTGRES_USER=%s + POSTGRES_PASSWORD=%s + POSTGRES_ADDRESS=%s + POSTGRES_PORT=%s + `, postgresUser, postgresPass, postgresHost, postgresPort)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_Postgres, + Verified: true, + }, + }, + wantErr: false, + }, + { + name: "found with single line credentials, verified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf(`postgresql://%s:%s@%s:%s/postgres`, postgresUser, postgresPass, postgresHost, postgresPort)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_Postgres, + Verified: true, + }, + }, + wantErr: false, + }, + { + name: "found with json credentials, verified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf( + `DB_CONFIG={"user": "%s", "password": "%s", "host": "%s", "port": "%s", "database": "postgres"}`, postgresUser, postgresPass, postgresHost, postgresPort)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_Postgres, + Verified: true, + }, + }, + wantErr: false, + }, + { + name: "found with seperated credentials, unverified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf(` + POSTGRES_USER=%s + POSTGRES_PASSWORD=%s + POSTGRES_ADDRESS=%s + POSTGRES_PORT=%s + `, postgresUser, inactivePass, postgresHost, postgresPort)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_Postgres, + Verified: false, + }, + }, + wantErr: false, + }, + { + name: "found with seperated credentials - no port, unverified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf(` + POSTGRES_USER=%s + POSTGRES_PASSWORD=%s + POSTGRES_ADDRESS=%s + `, postgresUser, inactivePass, postgresHost)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_Postgres, + Verified: false, + }, + }, + wantErr: false, + }, + { + name: "found with single line credentials, unverified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf(`postgresql://%s:%s@%s:%s/postgres`, postgresUser, inactivePass, postgresHost, postgresPort)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_Postgres, + Verified: false, + }, + }, + wantErr: false, + }, + { + name: "found with json credentials, unverified - inactive password", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf( + `DB_CONFIG={"user": "%s", "password": "%s", "host": "%s", "port": "%s", "database": "postgres"}`, postgresUser, inactivePass, postgresHost, postgresPort)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_Postgres, + Verified: false, + }, + }, + wantErr: false, + }, + { + name: "found with json credentials, unverified - inactive user", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf( + `DB_CONFIG={"user": "%s", "password": "%s", "host": "%s", "port": "%s", "database": "postgres"}`, inactiveUser, postgresPass, postgresHost, postgresPort)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_Postgres, + Verified: false, + }, + }, + wantErr: false, + }, + { + name: "found, unverified due to error - inactive port", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf(`postgresql://%s:%s@%s:%s/postgres`, postgresUser, postgresPass, postgresHost, inactivePort)), + verify: true, + }, + want: func() []detectors.Result { + r := detectors.Result{ + DetectorType: detectorspb.DetectorType_Postgres, + Verified: false, + } + return []detectors.Result{r} + }(), + wantErr: false, + }, + // This test seems take a long time to run (70s+) even with the timeout set to 1s. It's not clear why. + { + name: "found, unverified due to error - inactive host", + s: Scanner{}, + args: func() args { + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + return args{ + ctx: ctx, + data: []byte(fmt.Sprintf(`postgresql://%s:%s@%s:%s/postgres`, postgresUser, postgresPass, inactiveHost, postgresPort)), + verify: true, + } + }(), + want: func() []detectors.Result { + r := detectors.Result{ + DetectorType: detectorspb.DetectorType_Postgres, + Verified: false, + } + r.SetVerificationError(errors.New("i/o timeout")) + return []detectors.Result{r} + }(), + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := Scanner{} + got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + if (err != nil) != tt.wantErr { + t.Errorf("postgres.FromData() error = %v, wantErr %v", err, tt.wantErr) + return + } + for i := range got { + if len(got[i].Raw) == 0 { + t.Fatalf("no raw secret present: \n %+v", got[i]) + } + gotErr := "" + if got[i].VerificationError() != nil { + gotErr = got[i].VerificationError().Error() + } + wantErr := "" + if tt.want[i].VerificationError() != nil { + wantErr = tt.want[i].VerificationError().Error() + } + if gotErr != wantErr { + t.Fatalf("wantVerificationError = %v, verification error = %v", tt.want[i].VerificationError(), got[i].VerificationError()) + } + } + ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError") + if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" { + t.Errorf("Postgres.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + } + }) + } +} + +func dockerLogLine(hash string, needle string) chan struct{} { + ch := make(chan struct{}, 1) + go func() { + for { + out, err := exec.Command("docker", "logs", hash).CombinedOutput() + if err != nil { + panic(err) + } + if strings.Contains(string(out), needle) { + ch <- struct{}{} + return + } + time.Sleep(1 * time.Second) + } + }() + return ch +} + +func startPostgres() error { + cmd := exec.Command( + "docker", "run", "--rm", "-p", postgresPort+":"+defaultPort, + "-e", "POSTGRES_PASSWORD="+postgresPass, + "-e", "POSTGRES_USER="+postgresUser, + "-d", "postgres", + ) + fmt.Println(cmd.String()) + out, err := cmd.Output() + if err != nil { + return err + } + postgresDockerHash = string(bytes.TrimSpace(out)) + select { + case <-dockerLogLine(postgresDockerHash, "PostgreSQL init process complete; ready for start up."): + return nil + case <-time.After(30 * time.Second): + stopPostgres() + return errors.New("timeout waiting for postgres database to be ready") + } +} + +func stopPostgres() { + exec.Command("docker", "kill", postgresDockerHash).Run() +} + +func BenchmarkFromData(benchmark *testing.B) { + ctx := context.Background() + s := Scanner{} + for name, data := range detectors.MustGetBenchmarkData() { + benchmark.Run(name, func(b *testing.B) { + b.ResetTimer() + for n := 0; n < b.N; n++ { + _, err := s.FromData(ctx, false, data) + if err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/pkg/detectors/signable/signable.go b/pkg/detectors/signable/signable.go index d3e9c539a..cb583c6a4 100644 --- a/pkg/detectors/signable/signable.go +++ b/pkg/detectors/signable/signable.go @@ -13,16 +13,19 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct { + client *http.Client +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) var ( - client = common.SaneHttpClient() + defaultClient = common.SaneHttpClient() // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. - keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"signable"}) + `\b([a-zA-Z-0-9]{32})\b`) + tokenPat = regexp.MustCompile(detectors.PrefixRegex([]string{".{0,2}signable"}) + `\b([a-zA-Z-0-9]{32})\b`) + keywordPat = regexp.MustCompile(`(?i)([a-z]{2})signable`) ) // Keywords are used for efficiently pre-filtering chunks. @@ -35,41 +38,34 @@ func (s Scanner) Keywords() []string { func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) { dataStr := string(data) - matches := keyPat.FindAllStringSubmatch(dataStr, -1) - + matches := tokenPat.FindAllStringSubmatch(dataStr, -1) for _, match := range matches { if len(match) != 2 { continue } - resMatch := strings.TrimSpace(match[1]) + if isCommonFalsePositive(match[0]) { + continue + } + + resMatch := strings.TrimSpace(match[1]) s1 := detectors.Result{ DetectorType: detectorspb.DetectorType_Signable, Raw: []byte(resMatch), } if verify { - data := fmt.Sprintf("%s:", resMatch) - sEnc := b64.StdEncoding.EncodeToString([]byte(data)) - - req, err := http.NewRequestWithContext(ctx, "GET", "https://api.signable.co.uk/v1/templates?offset=0&limit=5", nil) - if err != nil { - continue + if s.client == nil { + s.client = defaultClient } + isVerified, verificationErr := verifyResult(ctx, s.client, resMatch) + s1.Verified = isVerified + s1.SetVerificationError(verificationErr, resMatch) + } - req.Header.Add("Authorization", fmt.Sprintf("Basic %s", sEnc)) - res, err := client.Do(req) - if err == nil { - defer res.Body.Close() - if res.StatusCode >= 200 && res.StatusCode < 300 { - s1.Verified = true - } else { - // This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key. - if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) { - continue - } - } - } + // This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key. + if !s1.Verified && detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) { + continue } results = append(results, s1) @@ -78,6 +74,44 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result return results, nil } +// Eliminate the most common false positive. +func isCommonFalsePositive(line string) bool { + // TODO: Skip lock files altogether. (https://github.com/trufflesecurity/trufflehog/issues/1517) + if strings.Contains(line, "helper-explode-assignable-expression") { + return true + } + + // Eliminate false positives from `assignable` and `designable`. + for _, m := range keywordPat.FindAllStringSubmatch(line, -1) { + if strings.EqualFold(m[1], "as") || strings.EqualFold(m[1], "de") { + return true + } + } + return false +} + +func verifyResult(ctx context.Context, client *http.Client, token string) (bool, error) { + data := fmt.Sprintf("%s:", token) + sEnc := b64.StdEncoding.EncodeToString([]byte(data)) + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://api.signable.co.uk/v1/templates?offset=0&limit=5", nil) + if err != nil { + return false, err + } + + req.Header.Add("Authorization", fmt.Sprintf("Basic %s", sEnc)) + res, err := client.Do(req) + if err != nil { + return false, err + } + + defer res.Body.Close() + if res.StatusCode >= 200 && res.StatusCode < 300 { + return true, nil + } + return false, nil +} + func (s Scanner) Type() detectorspb.DetectorType { return detectorspb.DetectorType_Signable } diff --git a/pkg/detectors/signable/signable_test.go b/pkg/detectors/signable/signable_test.go index 9287a53c8..74328a79c 100644 --- a/pkg/detectors/signable/signable_test.go +++ b/pkg/detectors/signable/signable_test.go @@ -10,12 +10,162 @@ import ( "time" "github.com/kylelemons/godebug/pretty" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) +func TestSignable_Pattern(t *testing.T) { + tests := []struct { + name string + data string + shouldMatch bool + match string + }{ + // True positives + { + name: "valid", + data: `const signableToken = '40a1cd917bff1288f699a94a75b37a1a'`, + shouldMatch: true, + match: "40a1cd917bff1288f699a94a75b37a1a", + }, + + // False positives + { + name: `invalid_assignable_yarn`, + data: `" babel-helper-explode-assignable-expression@^6.24.1: + version "6.24.1" + resolved "https://registry.npmjs.org/babel-helper-explode-assignable-expression/-/babel-helper-explode-assignable-expression-6.24.1.tgz#f25b82cf7dc10433c55f70592d5746400ac22caa" + dependencies: + babel-runtime "^6.22.0" + babel-traverse "^6.24.1" + babel-types "^6.24.1"`, + shouldMatch: false, + }, + { + name: `invalid_assignable_yarn`, + data: `"@babel/helper-explode-assignable-expression@^7.16.7": + version "7.16.7" + resolved "https://registry.yarnpkg.com/@babel/helper-explode-assignable-expression/-/helper-explode-assignable-expression-7.16.7.tgz#12a6d8522fdd834f194e868af6354e8650242b7a" + integrity sha512-KyUenhWMC8VrxzkGP0Jizjo4/Zx+1nNZhgocs+gLzyZyB8SHidhoq9KK/8Ato4anhwsivfkBLftky7gvzbZMtQ== + dependencies: + "@babel/types" "^7.16.7"`, + shouldMatch: false, + }, + // https://github.com/tbenst/purescript-nix-example/blob/558c8d6cb605742218cfa14a3fa93c062324b885/yarn.nix + { + name: `invalid_assignable_nix`, + data: ` { + name = "_babel_helper_explode_assignable_expression___helper_explode_assignable_expression_7.8.3.tgz"; + path = fetchurl { + name = "_babel_helper_explode_assignable_expression___helper_explode_assignable_expression_7.8.3.tgz"; + url = "https://registry.yarnpkg.com/@babel/helper-explode-assignable-expression/-/helper-explode-assignable-expression-7.8.3.tgz"; + sha1 = "a728dc5b4e89e30fc2dfc7d04fa28a930653f982"; + }; + }`, + shouldMatch: false, + }, + { + name: `invalid_assignable`, + data: `

Public Member Functions

+ +void constraints () +`, + shouldMatch: false, + }, + { + name: `invalid_assignable`, + data: `File: enumIsAssignableToBuiltInEnum.kt - 6396cf8549625bfce8b8ca2511d7f347 + NL("\n") + packageHeader`, + shouldMatch: false, + }, + { + name: `invalid_assignable_php`, + data: `'./include/SugarObjects/forms/PersonFormBase.php' => '2c1846ef127d60a40ecbab2c0b312ff5', + './include/SugarObjects/implements/assignable/language/en_us.lang.php' => '90f14b03e22e1eed2a1b93e10b975ef5', + './include/SugarObjects/implements/assignable/vardefs.php' => '358e0c47f753c5577fbdc0de08553c02', + './include/SugarObjects/implements/security_groups/language/en_us.lang.php' => 'ac1fd4817cb4662e3bdf973836558bdb',`, + shouldMatch: false, + }, + // https://github.com/past-due/warzone2100/blob/3e5637a7ed3d67ab92e439b94bf93f89f7bbea51/ChangeLog#L318 + { + name: `invalid_designable`, + data: ` * Fix: Prevent map selection button list from going off the form (commit:aea66eb1aa557c73d97b8019e5e66fccbb79f66e, #1347) + * Fix: Fix odd EMP mortar pathway; Add EMP mortar to designable weapons (commit:bcf93b7fe640c09e8b1239fabfd901fde9760259, #1535)`, + shouldMatch: false, + }, + // https://github.com/exis-io/Exis/blob/5383174f7b52112a97aadd09e6b9ea837c2fa07b/CardsAgainstHumanityDemo/swiftCardsAgainst/Pods/Pods.xcodeproj/project.pbxproj + { + name: `invalid_designable`, + data: ` 570767CBD99941F484DED46232044DC3 /* DesignableView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 93111B182BD71051B9ED0B14A9EF6EB6 /* DesignableView.swift */; }; + 57140D31D50A2FDE1E26729DDE7CB762 /* M13ProgressHUD.h in Headers */ = {isa = PBXBuildFile; fileRef = ECF777CB8B4C090E8D271E62729F7DD3 /* M13ProgressHUD.h */; settings = {ATTRIBUTES = (Public, ); }; };`, + shouldMatch: false, + }, + { + name: `invalid_designable`, + data: `{"nick":"hamster88","message":"this is my gist > https://gist.github.com/thedesignable/a05f628c649a81aae757945c352a8392","date":"2016-06-19T11:05:13.776Z","type":"message"}`, + shouldMatch: false, + }, + { + name: `invalid_designable`, + data: `返回到故事板文件，选择视图(我将假设从现在起视图被选中)并打开 Identity Inspector。你会注意到一个*可设计的*状态指示器已经出现在自定义类部分。 + +![Designables status](img/84bc9afc942815899347a31a425af7c6.png)`, + shouldMatch: false, + }, + { + name: `invalid_designable`, + data: `<h3 id="ibdesignable-x-paintcode:0b699a3cd6d609650a3fca90a5cd32cc">IBDesignable x PaintCode</h3>`, + shouldMatch: false, + }, + { + name: `invalid_designable`, + data: ` + + + `, + shouldMatch: false, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + s := Scanner{} + + results, err := s.FromData(context.Background(), false, []byte(test.data)) + if err != nil { + t.Errorf("Signable.FromData() error = %v", err) + return + } + + if test.shouldMatch { + if len(results) == 0 { + t.Errorf("%s: did not receive a match for '%v' when one was expected", test.name, test.data) + return + } + expected := test.data + if test.match != "" { + expected = test.match + } + result := results[0] + resultData := string(result.Raw) + if resultData != expected { + t.Errorf("%s: did not receive expected match.\n\texpected: '%s'\n\t actual: '%s'", test.name, expected, resultData) + return + } + } else { + if len(results) > 0 { + t.Errorf("%s: received a match for '%v' when one wasn't wanted", test.name, test.data) + return + } + } + }) + } +} + func TestSignable_FromChunk(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) defer cancel() diff --git a/pkg/detectors/snowflake/snowflake.go b/pkg/detectors/snowflake/snowflake.go index 0b837f9e1..7ec16ef32 100644 --- a/pkg/detectors/snowflake/snowflake.go +++ b/pkg/detectors/snowflake/snowflake.go @@ -6,6 +6,7 @@ import ( "fmt" "regexp" "strings" + "time" "unicode" _ "github.com/snowflakedb/gosnowflake" @@ -116,6 +117,10 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result ctx = context.Background() } + // Disable pool + retries to prevent flooding the server with failed login attemps. + db.SetConnMaxLifetime(time.Second) + db.SetMaxOpenConns(1) + err = db.PingContext(ctx) if err != nil { if strings.Contains(err.Error(), "Incorrect username or password was specified") { diff --git a/pkg/detectors/sourcegraph/sourcegraph.go b/pkg/detectors/sourcegraph/sourcegraph.go index 7580eb35e..46d3767e9 100644 --- a/pkg/detectors/sourcegraph/sourcegraph.go +++ b/pkg/detectors/sourcegraph/sourcegraph.go @@ -22,7 +22,7 @@ var _ detectors.Detector = (*Scanner)(nil) var ( defaultClient = common.SaneHttpClient() // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. - keyPat = regexp.MustCompile(`\b(sgp_[a-f0-9]{40})\b`) + keyPat = regexp.MustCompile(`\b(sgp_(?:[a-fA-F0-9]{16}|local)_[a-fA-F0-9]{40}|sgp_[a-fA-F0-9]{40}|[a-fA-F0-9]{40})\b`) ) // Keywords are used for efficiently pre-filtering chunks. @@ -47,6 +47,9 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result DetectorType: detectorspb.DetectorType_Sourcegraph, Raw: []byte(resMatch), } + s1.ExtraData = map[string]string{ + "rotation_guide": "https://howtorotate.com/docs/tutorials/sourcegraph/", + } if verify { client := s.client diff --git a/pkg/detectors/sourcegraph/sourcegraph_test.go b/pkg/detectors/sourcegraph/sourcegraph_test.go index 83b0de6f7..a260a63ab 100644 --- a/pkg/detectors/sourcegraph/sourcegraph_test.go +++ b/pkg/detectors/sourcegraph/sourcegraph_test.go @@ -25,15 +25,24 @@ func TestSourcegraph_FromChunk(t *testing.T) { if err != nil { t.Fatalf("could not get test secrets from GCP: %s", err) } - secret := testSecrets.MustGetField("SOURCEGRAPH") - inactiveSecret := testSecrets.MustGetField("SOURCEGRAPH_INACTIVE") + + secretV1 := testSecrets.MustGetField("SOURCEGRAPH_V1") + secretV2 := testSecrets.MustGetField("SOURCEGRAPH_V2") + secretV3 := testSecrets.MustGetField("SOURCEGRAPH_V3") + + inactiveSecretV1 := testSecrets.MustGetField("SOURCEGRAPH_INACTIVE_V1") + inactiveSecretV2 := testSecrets.MustGetField("SOURCEGRAPH_INACTIVE_V2") + inactiveSecretV3 := testSecrets.MustGetField("SOURCEGRAPH_INACTIVE_V3") + + secrets := []string{secretV1, secretV2, secretV3, inactiveSecretV1, inactiveSecretV2, inactiveSecretV3} type args struct { ctx context.Context data []byte verify bool } - tests := []struct { + for _, secret := range secrets { + tests = append(tests, []struct { name string s Scanner args args @@ -122,6 +131,7 @@ func TestSourcegraph_FromChunk(t *testing.T) { wantVerificationErr: true, }, } +} for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) @@ -137,7 +147,7 @@ func TestSourcegraph_FromChunk(t *testing.T) { t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError()) } } - ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError") + ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "VerificationError", "ExtraData") if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" { t.Errorf("Sourcegraph.FromData() %s diff: (-got +want)\n%s", tt.name, diff) } diff --git a/pkg/detectors/stripe/stripe.go b/pkg/detectors/stripe/stripe.go index 98c3a7682..498904f85 100644 --- a/pkg/detectors/stripe/stripe.go +++ b/pkg/detectors/stripe/stripe.go @@ -18,7 +18,7 @@ var _ detectors.Detector = (*Scanner)(nil) var ( //doesn't include test keys with "sk_test" - secretKey = regexp.MustCompile(`[rs]k_live_[a-zA-Z0-9]{20,30}`) + secretKey = regexp.MustCompile(`[rs]k_live_[a-zA-Z0-9]{20,247}`) ) // Keywords are used for efficiently pre-filtering chunks. diff --git a/pkg/engine/defaults.go b/pkg/engine/defaults.go index aec2ef61d..0cd901ab9 100644 --- a/pkg/engine/defaults.go +++ b/pkg/engine/defaults.go @@ -1496,6 +1496,8 @@ func DefaultDetectors() []detectors.Detector { speechtextai.Scanner{}, databox.Scanner{}, postbacks.Scanner{}, + // Too noisy, needs attention + // postgres.Scanner{}, collect2.Scanner{}, uclassify.Scanner{}, holistic.Scanner{}, diff --git a/pkg/engine/defaults_test.go b/pkg/engine/defaults_test.go index 67e41bc86..6131b211f 100644 --- a/pkg/engine/defaults_test.go +++ b/pkg/engine/defaults_test.go @@ -48,3 +48,20 @@ func TestDefaultDetectorTypesImplementing(t *testing.T) { ) } } + +func TestDefaultVersionerDetectorsHaveNonZeroVersions(t *testing.T) { + // Loop through all our default detectors and find the ones that + // implement Versioner. Of those, check each version is not zero. + // This is required due to an implementation detail of filtering detectors. + // See: https://github.com/trufflesecurity/trufflehog/blob/v3.63.7/main.go#L624-L638 + for _, detector := range DefaultDetectors() { + v, ok := detector.(detectors.Versioner) + if !ok || v.Version() != 0 { + continue + } + t.Errorf( + "detector %q implements Versioner that returns a zero version", + detectorspb.DetectorType_name[int32(detector.Type())], + ) + } +} diff --git a/pkg/engine/engine.go b/pkg/engine/engine.go index a8113d637..38f0da443 100644 --- a/pkg/engine/engine.go +++ b/pkg/engine/engine.go @@ -11,6 +11,7 @@ import ( lru "github.com/hashicorp/golang-lru" "google.golang.org/protobuf/proto" + "github.com/trufflesecurity/trufflehog/v3/pkg/cleantemp" "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/config" "github.com/trufflesecurity/trufflehog/v3/pkg/context" @@ -425,6 +426,10 @@ func (e *Engine) Finish(ctx context.Context) error { close(e.results) // Detector workers are done, close the results channel and call it a day. e.WgNotifier.Wait() // Wait for the notifier workers to finish notifying results. + if err := cleantemp.CleanTempArtifacts(ctx); err != nil { + ctx.Logger().Error(err, "error cleaning temp artifacts") + } + e.metrics.ScanDuration = time.Since(e.metrics.scanStartTime) return err diff --git a/pkg/engine/filesystem.go b/pkg/engine/filesystem.go index 9b49ae853..169e52c31 100644 --- a/pkg/engine/filesystem.go +++ b/pkg/engine/filesystem.go @@ -15,7 +15,9 @@ import ( // ScanFileSystem scans a given file system. func (e *Engine) ScanFileSystem(ctx context.Context, c sources.FilesystemConfig) error { connection := &sourcespb.Filesystem{ - Paths: c.Paths, + Paths: c.Paths, + IncludePathsFile: c.IncludePathsFile, + ExcludePathsFile: c.ExcludePathsFile, } var conn anypb.Any err := anypb.MarshalFrom(&conn, connection, proto.MarshalOptions{}) @@ -28,7 +30,6 @@ func (e *Engine) ScanFileSystem(ctx context.Context, c sources.FilesystemConfig) sourceID, jobID, _ := e.sourceManager.GetIDs(ctx, sourceName, filesystem.SourceType) fileSystemSource := &filesystem.Source{} - fileSystemSource.WithFilter(c.Filter) if err := fileSystemSource.Init(ctx, sourceName, jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil { return err } diff --git a/pkg/gitparse/gitparse.go b/pkg/gitparse/gitparse.go index f7b912448..dcd544c85 100644 --- a/pkg/gitparse/gitparse.go +++ b/pkg/gitparse/gitparse.go @@ -4,6 +4,7 @@ import ( "bufio" "bytes" "fmt" + "github.com/go-logr/logr" "io" "os" "os/exec" @@ -92,6 +93,7 @@ func (state ParseState) String() string { "BinaryFileLine", "HunkLineNumberLine", "HunkContentLine", + "ParseFailure", }[state] } @@ -313,7 +315,7 @@ func (c *Parser) FromReader(ctx context.Context, stdOut io.Reader, commitChan ch case isMessageLine(isStaged, latestState, line): latestState = MessageLine - currentCommit.Message.Write(line[4:]) + currentCommit.Message.Write(line[4:]) // Messages are indented with 4 spaces. case isMessageEndLine(isStaged, latestState, line): latestState = MessageEndLine // NoOp @@ -425,13 +427,14 @@ func (c *Parser) FromReader(ctx context.Context, stdOut io.Reader, commitChan ch // Here be dragons... // Build an informative error message. - var err error + err := fmt.Errorf(`invalid line "%s" after state "%s"`, line, latestState) + var logger logr.Logger if currentCommit != nil && currentCommit.Hash != "" { - err = fmt.Errorf(`failed to parse line "%s" after state "%s" (commit=%s)`, line, latestState, currentCommit.Hash) + logger = ctx.Logger().WithValues("commit", currentCommit.Hash) } else { - err = fmt.Errorf(`failed to parse line "%s" after state "%s"`, line, latestState) + logger = ctx.Logger() } - ctx.Logger().V(2).Error(err, "Recovering at the latest commit or diff...\n") + logger.Error(err, "failed to parse Git input. Recovering at the latest commit or diff...") latestState = ParseFailure } @@ -612,8 +615,9 @@ func pathFromBinaryLine(line []byte) string { } // --- a/internal/addrs/move_endpoint_module.go +// --- /dev/null func isFromFileLine(isStaged bool, latestState ParseState, line []byte) bool { - if latestState != IndexLine { + if !(latestState == IndexLine || latestState == ModeLine) { return false } if len(line) >= 6 && bytes.Equal(line[:4], []byte("--- ")) { diff --git a/pkg/gitparse/gitparse_test.go b/pkg/gitparse/gitparse_test.go index 33d6e4483..f343aead0 100644 --- a/pkg/gitparse/gitparse_test.go +++ b/pkg/gitparse/gitparse_test.go @@ -323,6 +323,13 @@ func TestLineChecks(t *testing.T) { IndexLine, []byte("--- /dev/null"), }, + // New file (https://github.com/trufflesecurity/trufflehog/issues/2109) + // diff --git a/libs/Unfit-1.0 b/libs/Unfit-1.0 + // new file mode 160000 + { + ModeLine, + []byte("--- /dev/null"), + }, // Uncommon but valid prefixes. Will these ever show up? // https://stackoverflow.com/a/2530012 // https://git-scm.com/docs/git-config#Documentation/git-config.txt-diffmnemonicPrefix @@ -1148,7 +1155,45 @@ func TestMaxCommitSize(t *testing.T) { } -const commitLog = `commit 4727ffb7ad6dc5130bf4b4dd166e00705abdd018 (HEAD -> master) +const commitLog = `commit fd6e99e7a80199b76a694603be57c5ade1de18e7 +Author: Jaliborc +Date: Mon Apr 25 16:28:06 2011 +0100 + + Added Unusable coloring + +diff --git a/components/item.lua b/components/item.lua +index fc74534..f8d7d50 100755 +--- a/components/item.lua ++++ b/components/item.lua +@@ -9,6 +9,7 @@ ItemSlot:Hide() + Bagnon.ItemSlot = ItemSlot + + local ItemSearch = LibStub('LibItemSearch-1.0') ++local Unfit = LibStub('Unfit-1.0') + + local function hasBlizzQuestHighlight() + return GetContainerItemQuestInfo and true or false +diff --git a/embeds.xml b/embeds.xml +index d3f4e7c..0c2df69 100755 +--- a/embeds.xml ++++ b/embeds.xml +@@ -6,6 +6,7 @@ + + + ++