Merge branch 'main' of github.com:trufflesecurity/trufflehog into feature/config-verification
This commit is contained in:
Zachary Rice
@@ -38,7 +38,7 @@ jobs:
|
||||
with:
|
||||
go-version: '1.21'
|
||||
- name: Cosign install
|
||||
uses: sigstore/cosign-installer@1fc5bd396d372bee37d608f955b336615edf79c8 # v3.2.0
|
||||
uses: sigstore/cosign-installer@9614fae9e5c5eddabb09f90a270fcb487c9f7149 # v3.3.0
|
||||
- name: Run GoReleaser
|
||||
uses: goreleaser/goreleaser-action@v5
|
||||
with:
|
||||
|
||||
@@ -13,10 +13,6 @@ jobs:
|
||||
test:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Install Go
|
||||
uses: actions/setup-go@v4
|
||||
with:
|
||||
go-version: '1.21'
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
@@ -26,7 +22,4 @@ jobs:
|
||||
uses: ./
|
||||
id: dogfood
|
||||
with:
|
||||
path: ./
|
||||
base: ${{ github.event.repository.default_branch }}
|
||||
head: HEAD
|
||||
extra_args: --only-verified
|
||||
|
||||
+1
-1
@@ -8,7 +8,7 @@ RUN --mount=type=cache,target=/go/pkg/mod \
|
||||
--mount=type=cache,target=/root/.cache/go-build \
|
||||
GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build -o trufflehog .
|
||||
|
||||
FROM alpine:3.18
|
||||
FROM alpine:3.19
|
||||
RUN apk add --no-cache bash git openssh-client ca-certificates rpm2cpio binutils cpio \
|
||||
&& rm -rf /var/cache/apk/* && update-ca-certificates
|
||||
COPY --from=builder /build/trufflehog /usr/bin/trufflehog
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
FROM alpine:3.18
|
||||
FROM alpine:3.19
|
||||
|
||||
RUN apk add --no-cache bash git openssh-client ca-certificates rpm2cpio binutils cpio \
|
||||
&& rm -rf /var/cache/apk/* && update-ca-certificates
|
||||
|
||||
@@ -65,6 +65,10 @@ cd trufflehog; go install
|
||||
|
||||
# Using installation script
|
||||
curl -sSfL https://raw.githubusercontent.com/trufflesecurity/trufflehog/main/scripts/install.sh | sh -s -- -b /usr/local/bin
|
||||
|
||||
# Using installation script, verify checksum signature (requires cosign to be installed)
|
||||
curl -sSfL https://raw.githubusercontent.com/trufflesecurity/trufflehog/main/scripts/install.sh | sh -s -- -v -b /usr/local/bin
|
||||
|
||||
# Using installation script to install a specific version
|
||||
curl -sSfL https://raw.githubusercontent.com/trufflesecurity/trufflehog/main/scripts/install.sh | sh -s -- -b /usr/local/bin <ReleaseTag like v3.56.0>
|
||||
```
|
||||
@@ -103,6 +107,9 @@ Verification steps are as follow:
|
||||
|
||||
Replace `{version}` with the downloaded files version
|
||||
|
||||
Alternatively, if you are using installation script, pass `-v` option to perform signature verification.
|
||||
This required Cosign binary to be installed prior to running installation script.
|
||||
|
||||
# :rocket: Quick Start
|
||||
|
||||
## 1: Scan a repo for only verified secrets
|
||||
@@ -336,6 +343,62 @@ Exit Codes:
|
||||
|
||||
## :octocat: TruffleHog Github Action
|
||||
|
||||
### General Usage
|
||||
|
||||
```
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
pull_request:
|
||||
|
||||
jobs:
|
||||
test:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- name: Secret Scanning
|
||||
uses: trufflesecurity/trufflehog@main
|
||||
with:
|
||||
extra_args: --only-verified
|
||||
```
|
||||
|
||||
In the example config above, we're scanning for live secrets in all PRs and Pushes to `main`. Only code changes in the referenced commits are scanned. If you'd like to scan an entire branch, please see the "Advanced Usage" section below.
|
||||
|
||||
|
||||
### Shallow Cloning
|
||||
|
||||
If you're incorporating TruffleHog into a standalone workflow and aren't running any other CI/CD tooling alongside TruffleHog, then we recommend using [Shallow Cloning](https://git-scm.com/docs/git-clone#Documentation/git-clone.txt---depthltdepthgt) to speed up your workflow. Here's an example for how to do it:
|
||||
|
||||
```
|
||||
...
|
||||
- shell: bash
|
||||
run: |
|
||||
if [ "${{ github.event_name }}" == "push" ]; then
|
||||
echo "depth=$(($(jq length <<< '${{ toJson(github.event.commits) }}') + 2))" >> $GITHUB_ENV
|
||||
echo "branch=${{ github.ref_name }}" >> $GITHUB_ENV
|
||||
fi
|
||||
if [ "${{ github.event_name }}" == "pull_request" ]; then
|
||||
echo "depth=$((${{ github.event.pull_request.commits }}+2))" >> $GITHUB_ENV
|
||||
echo "branch=${{ github.event.pull_request.head.ref }}" >> $GITHUB_ENV
|
||||
fi
|
||||
- uses: actions/checkout@v3
|
||||
with:
|
||||
ref: ${{env.branch}}
|
||||
fetch-depth: ${{env.depth}}
|
||||
- uses: trufflesecurity/trufflehog@main
|
||||
with:
|
||||
extra_args: --only-verified
|
||||
...
|
||||
```
|
||||
|
||||
Depending on the event type (push or PR), we calculate the number of commits present. Then we add 2, so that we can reference a base commit before our code changes. We pass that integer value to the `fetch-depth` flag in the checkout action in addition to the relevant branch. Now our checkout process should be much shorter.
|
||||
|
||||
### Advanced Usage
|
||||
|
||||
```yaml
|
||||
- name: TruffleHog
|
||||
uses: trufflesecurity/trufflehog@main
|
||||
@@ -350,34 +413,27 @@ Exit Codes:
|
||||
extra_args: --debug --only-verified
|
||||
```
|
||||
|
||||
The TruffleHog OSS Github Action can be used to scan a range of commits for leaked credentials. The action will fail if
|
||||
any results are found.
|
||||
If you'd like to specify specific `base` and `head` refs, you can use the `base` argument (`--since-commit` flag in TruffleHog CLI) and the `head` argument (`--branch` flag in the TruffleHog CLI). We only recommend using these arguments for very specific use cases, where the default behavior does not work.
|
||||
|
||||
For example, to scan the contents of pull requests you could use the following workflow:
|
||||
|
||||
```yaml
|
||||
name: TruffleHog Secrets Scan
|
||||
on: [pull_request]
|
||||
jobs:
|
||||
TruffleHog:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- name: TruffleHog OSS
|
||||
#### Advanced Usage: Scan entire branch
|
||||
```
|
||||
- name: scan-push
|
||||
uses: trufflesecurity/trufflehog@main
|
||||
with:
|
||||
path: ./
|
||||
base: ${{ github.event.repository.default_branch }}
|
||||
head: HEAD
|
||||
extra_args: --debug --only-verified
|
||||
base: ""
|
||||
head: ${{ github.ref_name }}
|
||||
extra_args: --only-verified
|
||||
```
|
||||
|
||||
## Pre-commit Hook
|
||||
|
||||
Trufflehog can be used in a pre-commit hook to prevent credentials from leaking before they ever leave your computer.
|
||||
|
||||
**Key Usage Note:**
|
||||
|
||||
- **For optimal hook efficacy, execute `git add` followed by `git commit` separately.** This ensures Trufflehog analyzes all intended changes.
|
||||
- **Avoid using `git commit -am`, as it might bypass pre-commit hook execution for unstaged modifications.**
|
||||
|
||||
An example `.pre-commit-config.yaml` is provided (see [pre-commit.com](https://pre-commit.com/) for installation).
|
||||
|
||||
```yaml
|
||||
|
||||
+72
-15
@@ -1,11 +1,12 @@
|
||||
name: 'TruffleHog OSS'
|
||||
description: 'Scan Github Actions with TruffleHog'
|
||||
description: 'Scan Github Actions with TruffleHog.'
|
||||
author: Truffle Security Co. <[email protected]>
|
||||
|
||||
inputs:
|
||||
path:
|
||||
description: Repository path
|
||||
required: true
|
||||
required: false
|
||||
default: "./"
|
||||
base:
|
||||
description: Start scanning from here (usually main branch).
|
||||
required: false
|
||||
@@ -20,17 +21,73 @@ inputs:
|
||||
branding:
|
||||
icon: "shield"
|
||||
color: "green"
|
||||
|
||||
runs:
|
||||
using: "docker"
|
||||
image: "docker://ghcr.io/trufflesecurity/trufflehog:latest"
|
||||
args:
|
||||
- git
|
||||
- file://${{ inputs.path }}
|
||||
- --since-commit
|
||||
- ${{ inputs.base }}
|
||||
- --branch
|
||||
- ${{ inputs.head }}
|
||||
- --fail
|
||||
- --no-update
|
||||
- --github-actions
|
||||
- ${{ inputs.extra_args }}
|
||||
using: "composite"
|
||||
steps:
|
||||
- shell: bash
|
||||
env:
|
||||
REPO_PATH: ${{ inputs.path }}
|
||||
BASE: ${{ inputs.base }}
|
||||
HEAD: ${{ inputs.head }}
|
||||
ARGS: ${{ inputs.extra_args }}
|
||||
COMMITS: ${{ toJson(github.event.commits) }}
|
||||
run: |
|
||||
##########################################
|
||||
## ADVANCED USAGE ##
|
||||
## Scan by BASE & HEAD user inputs ##
|
||||
## If BASE == HEAD, exit with error ##
|
||||
##########################################
|
||||
if [ -n "$BASE" ] || [ -n "$HEAD" ]; then
|
||||
if [ -n "$BASE" ]; then
|
||||
base_commit=$(git rev-parse "$BASE" 2>/dev/null) || true
|
||||
else
|
||||
base_commit=""
|
||||
fi
|
||||
if [ -n "$HEAD" ]; then
|
||||
head_commit=$(git rev-parse "$HEAD" 2>/dev/null) || true
|
||||
else
|
||||
head_commit=""
|
||||
fi
|
||||
if [ $base_commit == $head_commit ] ; then
|
||||
echo "::error::BASE and HEAD commits are the same. TruffleHog won't scan anything. Please see documentation (https://github.com/trufflesecurity/trufflehog#octocat-trufflehog-github-action)."
|
||||
exit 1
|
||||
fi
|
||||
##########################################
|
||||
## Scan commits based on event type ##
|
||||
##########################################
|
||||
else
|
||||
if [ "${{ github.event_name }}" == "push" ]; then
|
||||
COMMIT_LENGTH=$(printenv COMMITS | jq length)
|
||||
if [ $COMMIT_LENGTH == "0" ]; then
|
||||
echo "No commits to scan"
|
||||
exit 0
|
||||
fi
|
||||
HEAD=${{ github.event.after }}
|
||||
if [ ${{ github.event.before }} == "0000000000000000000000000000000000000000" ]; then
|
||||
BASE=$(git rev-parse $HEAD~$COMMIT_LENGTH)
|
||||
else
|
||||
BASE=${{ github.event.before }}
|
||||
fi
|
||||
elif [ "${{ github.event_name }}" == "workflow_dispatch" ] || [ "${{ github.event_name }}" == "schedule" ]; then
|
||||
BASE=""
|
||||
HEAD=""
|
||||
elif [ "${{ github.event_name }}" == "pull_request" ]; then
|
||||
BASE=${{github.event.pull_request.base.sha}}
|
||||
HEAD=${{github.event.pull_request.head.sha}}
|
||||
fi
|
||||
fi
|
||||
##########################################
|
||||
## Run TruffleHog ##
|
||||
##########################################
|
||||
docker run --rm -v "$REPO_PATH":/tmp \
|
||||
ghcr.io/trufflesecurity/trufflehog:latest \
|
||||
git file:///tmp/ \
|
||||
--since-commit \
|
||||
${BASE:-''} \
|
||||
--branch \
|
||||
${HEAD:-''} \
|
||||
--fail \
|
||||
--no-update \
|
||||
--github-actions \
|
||||
${ARGS:-''}
|
||||
|
||||
@@ -6,17 +6,17 @@ replace github.com/jpillora/overseer => github.com/trufflesecurity/overseer v1.2
|
||||
|
||||
require (
|
||||
cloud.google.com/go/secretmanager v1.11.4
|
||||
cloud.google.com/go/storage v1.35.1
|
||||
cloud.google.com/go/storage v1.36.0
|
||||
github.com/Azure/go-autorest/autorest/azure/auth v0.5.12
|
||||
github.com/AzureAD/microsoft-authentication-library-for-go v1.2.0
|
||||
github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1
|
||||
github.com/BobuSumisu/aho-corasick v1.0.3
|
||||
github.com/TheZeroSlave/zapsentry v1.19.0
|
||||
github.com/alecthomas/kingpin/v2 v2.4.0
|
||||
github.com/aws/aws-sdk-go v1.48.12
|
||||
github.com/aws/aws-sdk-go v1.49.19
|
||||
github.com/aymanbagabas/go-osc52 v1.2.2
|
||||
github.com/bill-rich/go-syslog v0.0.0-20220413021637-49edb52a574c
|
||||
github.com/bitfinexcom/bitfinex-api-go v0.0.0-20210608095005-9e0b26f200fb
|
||||
github.com/bradleyfalzon/ghinstallation/v2 v2.8.0
|
||||
github.com/bradleyfalzon/ghinstallation/v2 v2.9.0
|
||||
github.com/charmbracelet/bubbles v0.16.1
|
||||
github.com/charmbracelet/bubbletea v0.24.2
|
||||
github.com/charmbracelet/glamour v0.6.0
|
||||
@@ -28,9 +28,10 @@ require (
|
||||
github.com/envoyproxy/protoc-gen-validate v1.0.2
|
||||
github.com/fatih/color v1.16.0
|
||||
github.com/felixge/fgprof v0.9.3
|
||||
github.com/gabriel-vasile/mimetype v1.4.3
|
||||
github.com/getsentry/sentry-go v0.25.0
|
||||
github.com/go-errors/errors v1.5.1
|
||||
github.com/go-git/go-git/v5 v5.10.1
|
||||
github.com/go-git/go-git/v5 v5.11.0
|
||||
github.com/go-ldap/ldap/v3 v3.4.6
|
||||
github.com/go-logr/logr v1.3.0
|
||||
github.com/go-logr/zapr v1.3.0
|
||||
@@ -56,7 +57,7 @@ require (
|
||||
github.com/lrstanley/bubblezone v0.0.0-20230911164824-e3824f1adde9
|
||||
github.com/marusama/semaphore/v2 v2.5.0
|
||||
github.com/mattn/go-isatty v0.0.20
|
||||
github.com/mattn/go-sqlite3 v1.14.18
|
||||
github.com/mattn/go-sqlite3 v1.14.19
|
||||
github.com/mholt/archiver/v4 v4.0.0-alpha.8
|
||||
github.com/mitchellh/go-ps v1.0.0
|
||||
github.com/muesli/reflow v0.3.0
|
||||
@@ -75,8 +76,8 @@ require (
|
||||
go.mongodb.org/mongo-driver v1.12.1
|
||||
go.uber.org/mock v0.3.0
|
||||
go.uber.org/zap v1.26.0
|
||||
golang.org/x/crypto v0.16.0
|
||||
golang.org/x/exp v0.0.0-20231127185646-65229373498e
|
||||
golang.org/x/crypto v0.17.0
|
||||
golang.org/x/exp v0.0.0-20240110193028-0dcbfd608b1e
|
||||
golang.org/x/net v0.19.0
|
||||
golang.org/x/oauth2 v0.15.0
|
||||
golang.org/x/sync v0.5.0
|
||||
@@ -137,7 +138,7 @@ require (
|
||||
github.com/bodgit/sevenzip v1.4.5 // indirect
|
||||
github.com/bodgit/windows v1.0.1 // indirect
|
||||
github.com/cespare/xxhash/v2 v2.2.0 // indirect
|
||||
github.com/cloudflare/circl v1.3.3 // indirect
|
||||
github.com/cloudflare/circl v1.3.7 // indirect
|
||||
github.com/containerd/console v1.0.4-0.20230313162750-1ae8d489ac81 // indirect
|
||||
github.com/containerd/stargz-snapshotter/estargz v0.14.3 // indirect
|
||||
github.com/couchbase/gocbcore/v10 v10.3.0 // indirect
|
||||
@@ -154,10 +155,9 @@ require (
|
||||
github.com/docker/docker v24.0.7+incompatible // indirect
|
||||
github.com/docker/docker-credential-helpers v0.7.0 // indirect
|
||||
github.com/dsnet/compress v0.0.1 // indirect
|
||||
github.com/dvsekhvalnov/jose2go v1.5.0 // indirect
|
||||
github.com/dvsekhvalnov/jose2go v1.6.0 // indirect
|
||||
github.com/emirpasic/gods v1.18.1 // indirect
|
||||
github.com/form3tech-oss/jwt-go v3.2.5+incompatible // indirect
|
||||
github.com/gabriel-vasile/mimetype v1.4.2 // indirect
|
||||
github.com/go-asn1-ber/asn1-ber v1.5.5 // indirect
|
||||
github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
|
||||
github.com/go-git/go-billy/v5 v5.5.0 // indirect
|
||||
@@ -172,6 +172,7 @@ require (
|
||||
github.com/golang/snappy v0.0.4 // indirect
|
||||
github.com/google/flatbuffers v23.1.21+incompatible // indirect
|
||||
github.com/google/go-github/v56 v56.0.0 // indirect
|
||||
github.com/google/go-github/v57 v57.0.0 // indirect
|
||||
github.com/google/go-querystring v1.1.0 // indirect
|
||||
github.com/google/pprof v0.0.0-20211214055906-6f57359322fd // indirect
|
||||
github.com/google/s2a-go v0.1.7 // indirect
|
||||
|
||||
@@ -28,6 +28,8 @@ cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiy
|
||||
cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos=
|
||||
cloud.google.com/go/storage v1.35.1 h1:B59ahL//eDfx2IIKFBeT5Atm9wnNmj3+8xG/W4WB//w=
|
||||
cloud.google.com/go/storage v1.35.1/go.mod h1:M6M/3V/D3KpzMTJyPOR/HU6n2Si5QdaXYEsng2xgOs8=
|
||||
cloud.google.com/go/storage v1.36.0 h1:P0mOkAcaJxhCTvAkMhxMfrTKiNcub4YmmPBtlhAyTr8=
|
||||
cloud.google.com/go/storage v1.36.0/go.mod h1:M6M/3V/D3KpzMTJyPOR/HU6n2Si5QdaXYEsng2xgOs8=
|
||||
dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk=
|
||||
dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk=
|
||||
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
|
||||
@@ -68,6 +70,8 @@ github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 h1:mFRzDkZVAjdal+
|
||||
github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU=
|
||||
github.com/AzureAD/microsoft-authentication-library-for-go v1.2.0 h1:hVeq+yCyUi+MsoO/CU95yqCIcdzra5ovzk8Q2BBpV2M=
|
||||
github.com/AzureAD/microsoft-authentication-library-for-go v1.2.0/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI=
|
||||
github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 h1:DzHpqpoJVaCgOUdVHxE8QB52S6NiVdDQvGlny1qvPqA=
|
||||
github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI=
|
||||
github.com/BobuSumisu/aho-corasick v1.0.3 h1:uuf+JHwU9CHP2Vx+wAy6jcksJThhJS9ehR8a+4nPE9g=
|
||||
github.com/BobuSumisu/aho-corasick v1.0.3/go.mod h1:hm4jLcvZKI2vRF2WDU1N4p/jpWtpOzp3nLmi9AzX/XE=
|
||||
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
|
||||
@@ -104,6 +108,10 @@ github.com/atotto/clipboard v0.1.4 h1:EH0zSVneZPSuFR11BlR9YppQTVDbh5+16AmcJi4g1z
|
||||
github.com/atotto/clipboard v0.1.4/go.mod h1:ZY9tmq7sm5xIbd9bOK4onWV4S6X0u6GY7Vn0Yu86PYI=
|
||||
github.com/aws/aws-sdk-go v1.48.12 h1:n+eGzflzzvYubu2cOjqpVll7lF+Ci0ThyCpg5kzfzbo=
|
||||
github.com/aws/aws-sdk-go v1.48.12/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk=
|
||||
github.com/aws/aws-sdk-go v1.49.18 h1:g/iMXkfXeJQ7MvnLwroxWsTTNkHtdVJGxIgrAIEG62M=
|
||||
github.com/aws/aws-sdk-go v1.49.18/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk=
|
||||
github.com/aws/aws-sdk-go v1.49.19 h1:oZryiqeQpeJsIcAmZlp86duMu/s/DJ43qyfwa51qmLg=
|
||||
github.com/aws/aws-sdk-go v1.49.19/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk=
|
||||
github.com/aws/aws-sdk-go-v2 v1.17.7 h1:CLSjnhJSTSogvqUGhIC6LqFKATMRexcxLZ0i/Nzk9Eg=
|
||||
github.com/aws/aws-sdk-go-v2 v1.17.7/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 h1:dK82zF6kkPeCo8J1e+tGx4JdvDIQzj7ygIoLg8WMuGs=
|
||||
@@ -154,19 +162,18 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
|
||||
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
|
||||
github.com/bill-rich/go-syslog v0.0.0-20220413021637-49edb52a574c h1:tSME5FDS02qQll3JYodI6RZR/g4EKOHApGv1wMZT+Z0=
|
||||
github.com/bill-rich/go-syslog v0.0.0-20220413021637-49edb52a574c/go.mod h1:+sCc6hztur+oZCLOsNk6wCCy+GLrnSNHSRmTnnL+8iQ=
|
||||
github.com/bill-rich/sevenzip v0.0.0-20231205215127-9521c543efac h1:7jDHN8fn8cNf1ibps8CrHivEM7GtXpsuaJeD5CWkw/Y=
|
||||
github.com/bill-rich/sevenzip v0.0.0-20231205215127-9521c543efac/go.mod h1:aHY9hBGPQUQIimuGB0H+LJPqCI/68ZZFxRRab0gk/nU=
|
||||
github.com/bitfinexcom/bitfinex-api-go v0.0.0-20210608095005-9e0b26f200fb h1:9v7Bzlg+1EBYi2IYcUmOwHReBEfqBbYIj3ZCi9cIe1Q=
|
||||
github.com/bitfinexcom/bitfinex-api-go v0.0.0-20210608095005-9e0b26f200fb/go.mod h1:EkOqCuelvo7DY8vCOoZ09p7pHvAK9B1PHI9GeM4Rdxc=
|
||||
github.com/bodgit/plumbing v1.3.0 h1:pf9Itz1JOQgn7vEOE7v7nlEfBykYqvUYioC61TwWCFU=
|
||||
github.com/bodgit/plumbing v1.3.0/go.mod h1:JOTb4XiRu5xfnmdnDJo6GmSbSbtSyufrsyZFByMtKEs=
|
||||
github.com/bodgit/sevenzip v1.4.3/go.mod h1:F8n3+0CwbdxqmNy3wFeOAtanza02Ur66AGfs/hbYblI=
|
||||
github.com/bodgit/sevenzip v1.4.5 h1:HFJQ+nbjppfyf2xbQEJBbmVo+o2kTg1FXV4i7YOx87s=
|
||||
github.com/bodgit/sevenzip v1.4.5/go.mod h1:LAcAg/UQzyjzCQSGBPZFYzoiHMfT6Gk+3tMSjUk3foY=
|
||||
github.com/bodgit/windows v1.0.1 h1:tF7K6KOluPYygXa3Z2594zxlkbKPAOvqr97etrGNIz4=
|
||||
github.com/bodgit/windows v1.0.1/go.mod h1:a6JLwrB4KrTR5hBpp8FI9/9W9jJfeQ2h4XDXU74ZCdM=
|
||||
github.com/bradleyfalzon/ghinstallation/v2 v2.8.0 h1:yUmoVv70H3J4UOqxqsee39+KlXxNEDfTbAp8c/qULKk=
|
||||
github.com/bradleyfalzon/ghinstallation/v2 v2.8.0/go.mod h1:fmPmvCiBWhJla3zDv9ZTQSZc8AbwyRnGW1yg5ep1Pcs=
|
||||
github.com/bradleyfalzon/ghinstallation/v2 v2.9.0 h1:HmxIYqnxubRYcYGRc5v3wUekmo5Wv2uX3gukmWJ0AFk=
|
||||
github.com/bradleyfalzon/ghinstallation/v2 v2.9.0/go.mod h1:wmkTDJf8CmVypxE8ijIStFnKoTa6solK5QfdmJrP9KI=
|
||||
github.com/bwesterb/go-ristretto v1.2.3/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0=
|
||||
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
|
||||
github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
|
||||
@@ -183,8 +190,9 @@ github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWR
|
||||
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
|
||||
github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
|
||||
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
|
||||
github.com/cloudflare/circl v1.3.3 h1:fE/Qz0QdIGqeWfnwq0RE0R7MI51s0M2E4Ga9kq5AEMs=
|
||||
github.com/cloudflare/circl v1.3.3/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA=
|
||||
github.com/cloudflare/circl v1.3.7 h1:qlCDlTPz2n9fu58M0Nh1J/JzcFpfgkFHHX3O35r5vcU=
|
||||
github.com/cloudflare/circl v1.3.7/go.mod h1:sRTcRWXGLrKw6yIGJ+l7amYJFfAXbZG0kBSc8r4zxgA=
|
||||
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
|
||||
github.com/coinbase/waas-client-library-go v1.0.8 h1:AdbTmBQpsSUx947GZd5/68BhNBw1CSwTfE2PcnVy3Ao=
|
||||
github.com/coinbase/waas-client-library-go v1.0.8/go.mod h1:RVKozprfdfMiK92ATZUWHRs0EFGHQj4rbEJjzzZzI1I=
|
||||
@@ -234,8 +242,8 @@ github.com/docker/docker-credential-helpers v0.7.0/go.mod h1:rETQfLdHNT3foU5kuNk
|
||||
github.com/dsnet/compress v0.0.1 h1:PlZu0n3Tuv04TzpfPbrnI0HW/YwodEXDS+oPKahKF0Q=
|
||||
github.com/dsnet/compress v0.0.1/go.mod h1:Aw8dCMJ7RioblQeTqt88akK31OvO8Dhf5JflhBbQEHo=
|
||||
github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY=
|
||||
github.com/dvsekhvalnov/jose2go v1.5.0 h1:3j8ya4Z4kMCwT5nXIKFSV84YS+HdqSSO0VsTQxaLAeM=
|
||||
github.com/dvsekhvalnov/jose2go v1.5.0/go.mod h1:QsHjhyTlD/lAVqn/NSbVZmSCGeDehTB/mPZadG+mhXU=
|
||||
github.com/dvsekhvalnov/jose2go v1.6.0 h1:Y9gnSnP4qEI0+/uQkHvFXeD2PLPJeXEL+ySMEA2EjTY=
|
||||
github.com/dvsekhvalnov/jose2go v1.6.0/go.mod h1:QsHjhyTlD/lAVqn/NSbVZmSCGeDehTB/mPZadG+mhXU=
|
||||
github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a h1:mATvB/9r/3gvcejNsXKSkQ6lcIaNec2nyfOdlTBR2lU=
|
||||
github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM=
|
||||
github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc=
|
||||
@@ -257,6 +265,8 @@ github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWo
|
||||
github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
|
||||
github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU=
|
||||
github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA=
|
||||
github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0=
|
||||
github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk=
|
||||
github.com/getsentry/sentry-go v0.25.0 h1:q6Eo+hS+yoJlTO3uu/azhQadsD8V+jQn2D8VvX1eOyI=
|
||||
github.com/getsentry/sentry-go v0.25.0/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w1gwTf25trprY=
|
||||
github.com/gliderlabs/ssh v0.3.5 h1:OcaySEmAQJgyYcArR+gGGTHCyE7nvhEMTlYY+Dp8CpY=
|
||||
@@ -271,8 +281,8 @@ github.com/go-git/go-billy/v5 v5.5.0 h1:yEY4yhzCDuMGSv83oGxiBotRzhwhNr8VZyphhiu+
|
||||
github.com/go-git/go-billy/v5 v5.5.0/go.mod h1:hmexnoNsr2SJU1Ju67OaNz5ASJY3+sHgFRpCtpDCKow=
|
||||
github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399 h1:eMje31YglSBqCdIqdhKBW8lokaMrL3uTkpGYlE2OOT4=
|
||||
github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399/go.mod h1:1OCfN199q1Jm3HZlxleg+Dw/mwps2Wbk9frAWm+4FII=
|
||||
github.com/go-git/go-git/v5 v5.10.1 h1:tu8/D8i+TWxgKpzQ3Vc43e+kkhXqtsZCKI/egajKnxk=
|
||||
github.com/go-git/go-git/v5 v5.10.1/go.mod h1:uEuHjxkHap8kAl//V5F/nNWwqIYtP/402ddd05mp0wg=
|
||||
github.com/go-git/go-git/v5 v5.11.0 h1:XIZc1p+8YzypNr34itUfSvYJcv+eYdTnTvOZ2vD3cA4=
|
||||
github.com/go-git/go-git/v5 v5.11.0/go.mod h1:6GFcX2P3NM7FPBfpePbpLd21XxsgdAt+lKqXmCUiUCY=
|
||||
github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
|
||||
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
|
||||
github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY=
|
||||
@@ -366,6 +376,8 @@ github.com/google/go-github/v42 v42.0.0 h1:YNT0FwjPrEysRkLIiKuEfSvBPCGKphW5aS5Px
|
||||
github.com/google/go-github/v42 v42.0.0/go.mod h1:jgg/jvyI0YlDOM1/ps6XYh04HNQ3vKf0CVko62/EhRg=
|
||||
github.com/google/go-github/v56 v56.0.0 h1:TysL7dMa/r7wsQi44BjqlwaHvwlFlqkK8CtBWCX3gb4=
|
||||
github.com/google/go-github/v56 v56.0.0/go.mod h1:D8cdcX98YWJvi7TLo7zM4/h8ZTx6u6fwGEkCdisopo0=
|
||||
github.com/google/go-github/v57 v57.0.0 h1:L+Y3UPTY8ALM8x+TV0lg+IEBI+upibemtBD8Q9u7zHs=
|
||||
github.com/google/go-github/v57 v57.0.0/go.mod h1:s0omdnye0hvK/ecLvpsGfJMiRt85PimQh4oygmLIxHw=
|
||||
github.com/google/go-querystring v0.0.0-20170111101155-53e6ce116135/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
|
||||
github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8=
|
||||
github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU=
|
||||
@@ -513,6 +525,8 @@ github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZ
|
||||
github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
|
||||
github.com/mattn/go-sqlite3 v1.14.18 h1:JL0eqdCOq6DJVNPSvArO/bIV9/P7fbGrV00LZHc+5aI=
|
||||
github.com/mattn/go-sqlite3 v1.14.18/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
|
||||
github.com/mattn/go-sqlite3 v1.14.19 h1:fhGleo2h1p8tVChob4I9HpmVFIAkKGpiukdrgQbWfGI=
|
||||
github.com/mattn/go-sqlite3 v1.14.19/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo=
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
|
||||
github.com/mholt/archiver/v4 v4.0.0-alpha.8 h1:tRGQuDVPh66WCOelqe6LIGh0gwmfwxUrSSDunscGsRM=
|
||||
@@ -571,8 +585,6 @@ github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaR
|
||||
github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
|
||||
github.com/paulbellamy/ratecounter v0.2.0 h1:2L/RhJq+HA8gBQImDXtLPrDXK5qAj6ozWVK/zFXVJGs=
|
||||
github.com/paulbellamy/ratecounter v0.2.0/go.mod h1:Hfx1hDpSGoqxkVVpBi/IlYD7kChlfo5C6hzIHwPqfFE=
|
||||
github.com/pierrec/lz4/v4 v4.1.18 h1:xaKrnTkyoqfh1YItXl56+6KJNVYWlEEPuAQW9xsplYQ=
|
||||
github.com/pierrec/lz4/v4 v4.1.18/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
|
||||
github.com/pierrec/lz4/v4 v4.1.19 h1:tYLzDnjDXh9qIxSTKHwXwOYmm9d887Y7Y1ZkyXYHAN4=
|
||||
github.com/pierrec/lz4/v4 v4.1.19/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
|
||||
github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4=
|
||||
@@ -726,8 +738,8 @@ golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0
|
||||
golang.org/x/crypto v0.3.1-0.20221117191849-2c476679df9a/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4=
|
||||
golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU=
|
||||
golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
|
||||
golang.org/x/crypto v0.16.0 h1:mMMrFzRSCF0GvB7Ne27XVtVAaXLrPmgPC7/v0tkwHaY=
|
||||
golang.org/x/crypto v0.16.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4=
|
||||
golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k=
|
||||
golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4=
|
||||
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
|
||||
@@ -738,6 +750,10 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0
|
||||
golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
|
||||
golang.org/x/exp v0.0.0-20231127185646-65229373498e h1:Gvh4YaCaXNs6dKTlfgismwWZKyjVZXwOPfIyUaqU3No=
|
||||
golang.org/x/exp v0.0.0-20231127185646-65229373498e/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI=
|
||||
golang.org/x/exp v0.0.0-20240103183307-be819d1f06fc h1:ao2WRsKSzW6KuUY9IWPwWahcHCgR0s52IfwutMfEbdM=
|
||||
golang.org/x/exp v0.0.0-20240103183307-be819d1f06fc/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI=
|
||||
golang.org/x/exp v0.0.0-20240110193028-0dcbfd608b1e h1:723BNChdd0c2Wk6WOE320qGBiPtYx0F0Bbm1kriShfE=
|
||||
golang.org/x/exp v0.0.0-20240110193028-0dcbfd608b1e/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI=
|
||||
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
|
||||
golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
|
||||
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
|
||||
@@ -203,6 +203,7 @@ func main() {
|
||||
}
|
||||
},
|
||||
true,
|
||||
false,
|
||||
)
|
||||
|
||||
logger.Info("scanning repo", "repo", r)
|
||||
|
||||
@@ -5,10 +5,12 @@ import (
|
||||
"net/http"
|
||||
_ "net/http/pprof"
|
||||
"os"
|
||||
"os/signal"
|
||||
"runtime"
|
||||
"strconv"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/alecthomas/kingpin/v2"
|
||||
"github.com/felixge/fgprof"
|
||||
@@ -55,6 +57,7 @@ var (
|
||||
noUpdate = cli.Flag("no-update", "Don't check for updates.").Bool()
|
||||
fail = cli.Flag("fail", "Exit with code 183 if results are found.").Bool()
|
||||
verifiers = cli.Flag("verifier", "Set custom verification endpoints.").StringMap()
|
||||
customVerifiersOnly = cli.Flag("custom-verifiers-only", "Only use custom verification endpoints.").Bool()
|
||||
archiveMaxSize = cli.Flag("archive-max-size", "Maximum size of archive to scan. (Byte units eg. 512B, 2KB, 4MB)").Bytes()
|
||||
archiveMaxDepth = cli.Flag("archive-max-depth", "Maximum depth of archive to scan.").Int()
|
||||
archiveTimeout = cli.Flag("archive-timeout", "Maximum time to spend extracting an archive.").Duration()
|
||||
@@ -216,17 +219,39 @@ func main() {
|
||||
if err != nil {
|
||||
logFatal(err, "error occurred with trufflehog updater 🐷")
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
go cleantemp.RunCleanupLoop(ctx)
|
||||
}
|
||||
|
||||
func run(state overseer.State) {
|
||||
ctx := context.Background()
|
||||
ctx, cancel := context.WithCancelCause(context.Background())
|
||||
defer cancel(nil)
|
||||
|
||||
go func() {
|
||||
if err := cleantemp.CleanTempArtifacts(ctx); err != nil {
|
||||
ctx.Logger().Error(err, "error cleaning temporary artifacts")
|
||||
}
|
||||
}()
|
||||
|
||||
logger := ctx.Logger()
|
||||
logFatal := logFatalFunc(logger)
|
||||
|
||||
killSignal := make(chan os.Signal, 1)
|
||||
signal.Notify(killSignal, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT)
|
||||
go func() {
|
||||
<-killSignal
|
||||
logger.Info("Received signal, shutting down.")
|
||||
cancel(fmt.Errorf("canceling context due to signal"))
|
||||
|
||||
if err := cleantemp.CleanTempArtifacts(ctx); err != nil {
|
||||
logger.Error(err, "error cleaning temporary artifacts")
|
||||
} else {
|
||||
logger.Info("cleaned temporary artifacts")
|
||||
}
|
||||
|
||||
time.Sleep(time.Second * 10)
|
||||
logger.Info("10 seconds elapsed. Forcing shutdown.")
|
||||
os.Exit(0)
|
||||
}()
|
||||
|
||||
logger.V(2).Info(fmt.Sprintf("trufflehog %s", version.BuildVersion))
|
||||
|
||||
if *githubScanToken != "" {
|
||||
@@ -341,8 +366,9 @@ func run(state overseer.State) {
|
||||
"detector", id,
|
||||
)
|
||||
}
|
||||
// TODO: Add flag to ignore the default endpoint.
|
||||
urls = append(urls, customizer.DefaultEndpoint())
|
||||
if !*customVerifiersOnly || len(urls) == 0 {
|
||||
urls = append(urls, customizer.DefaultEndpoint())
|
||||
}
|
||||
if err := customizer.SetEndpoints(urls...); err != nil {
|
||||
logFatal(err, "failed configuring custom endpoint for detector", "detector", id)
|
||||
}
|
||||
@@ -446,10 +472,6 @@ func run(state overseer.State) {
|
||||
logFatal(err, "Failed to scan GitLab.")
|
||||
}
|
||||
case filesystemScan.FullCommand():
|
||||
filter, err := common.FilterFromFiles(*filesystemScanIncludePaths, *filesystemScanExcludePaths)
|
||||
if err != nil {
|
||||
logFatal(err, "could not create filter")
|
||||
}
|
||||
if len(*filesystemDirectories) > 0 {
|
||||
ctx.Logger().Info("--directory flag is deprecated, please pass directories as arguments")
|
||||
}
|
||||
@@ -457,8 +479,9 @@ func run(state overseer.State) {
|
||||
paths = append(paths, *filesystemPaths...)
|
||||
paths = append(paths, *filesystemDirectories...)
|
||||
cfg := sources.FilesystemConfig{
|
||||
Paths: paths,
|
||||
Filter: filter,
|
||||
Paths: paths,
|
||||
IncludePathsFile: *filesystemScanIncludePaths,
|
||||
ExcludePathsFile: *filesystemScanExcludePaths,
|
||||
}
|
||||
if err = e.ScanFileSystem(ctx, cfg); err != nil {
|
||||
logFatal(err, "Failed to scan filesystem")
|
||||
|
||||
Vendored
+3
-3
@@ -4,9 +4,9 @@ package cache
|
||||
// Cache is used to store key/value pairs.
|
||||
type Cache interface {
|
||||
// Set stores the given key/value pair.
|
||||
Set(string, string)
|
||||
Set(string, any)
|
||||
// Get returns the value for the given key and a boolean indicating if the key was found.
|
||||
Get(string) (string, bool)
|
||||
Get(string) (any, bool)
|
||||
// Exists returns true if the given key exists in the cache.
|
||||
Exists(string) bool
|
||||
// Delete the given key from the cache.
|
||||
@@ -18,7 +18,7 @@ type Cache interface {
|
||||
// Keys returns all keys in the cache.
|
||||
Keys() []string
|
||||
// Values returns all values in the cache.
|
||||
Values() []string
|
||||
Values() []any
|
||||
// Contents returns all keys in the cache encoded as a string.
|
||||
Contents() string
|
||||
}
|
||||
|
||||
Vendored
+54
-29
@@ -5,52 +5,77 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/patrickmn/go-cache"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
)
|
||||
|
||||
const (
|
||||
expirationInterval = 12 * time.Hour
|
||||
purgeInterval = 13 * time.Hour
|
||||
defaultExpiration = cache.DefaultExpiration
|
||||
defaultExpirationInterval = 12 * time.Hour
|
||||
defaultPurgeInterval = 13 * time.Hour
|
||||
defaultExpiration = cache.DefaultExpiration
|
||||
)
|
||||
|
||||
// Cache is a wrapper around the go-cache library.
|
||||
// Cache wraps the go-cache library to provide an in-memory key-value store.
|
||||
type Cache struct {
|
||||
c *cache.Cache
|
||||
c *cache.Cache
|
||||
expiration time.Duration
|
||||
purgeInterval time.Duration
|
||||
}
|
||||
|
||||
// New constructs a new in-memory cache.
|
||||
func New() *Cache {
|
||||
c := cache.New(expirationInterval, purgeInterval)
|
||||
return &Cache{c: c}
|
||||
// CacheOption defines a function type used for configuring a Cache.
|
||||
type CacheOption func(*Cache)
|
||||
|
||||
// WithExpirationInterval returns a CacheOption to set the expiration interval of cache items.
|
||||
// The interval determines the duration a cached item remains in the cache before it is expired.
|
||||
func WithExpirationInterval(interval time.Duration) CacheOption {
|
||||
return func(c *Cache) { c.expiration = interval }
|
||||
}
|
||||
|
||||
// WithPurgeInterval returns a CacheOption to set the interval at which the cache purges expired items.
|
||||
// Regular purging helps in freeing up memory by removing stale entries.
|
||||
func WithPurgeInterval(interval time.Duration) CacheOption {
|
||||
return func(c *Cache) { c.purgeInterval = interval }
|
||||
}
|
||||
|
||||
// New constructs a new in-memory cache instance with optional configurations.
|
||||
// By default, it sets the expiration and purge intervals to 12 and 13 hours, respectively.
|
||||
// These defaults can be overridden using the functional options: WithExpirationInterval and WithPurgeInterval.
|
||||
func New(opts ...CacheOption) *Cache {
|
||||
return NewWithData(nil, opts...)
|
||||
}
|
||||
|
||||
// CacheEntry represents a single entry in the cache, consisting of a key and its corresponding value.
|
||||
type CacheEntry struct {
|
||||
// Key is the unique identifier for the entry.
|
||||
Key string
|
||||
// Value is the data stored in the entry.
|
||||
Value any
|
||||
}
|
||||
|
||||
// NewWithData constructs a new in-memory cache with existing data.
|
||||
func NewWithData(ctx context.Context, data []string) *Cache {
|
||||
ctx.Logger().V(3).Info("Loading cache", "num-items", len(data))
|
||||
|
||||
items := make(map[string]cache.Item, len(data))
|
||||
for _, d := range data {
|
||||
items[d] = cache.Item{Object: d, Expiration: int64(defaultExpiration)}
|
||||
// It also accepts CacheOption parameters to override default configuration values.
|
||||
func NewWithData(data []CacheEntry, opts ...CacheOption) *Cache {
|
||||
instance := &Cache{expiration: defaultExpirationInterval, purgeInterval: defaultPurgeInterval}
|
||||
for _, opt := range opts {
|
||||
opt(instance)
|
||||
}
|
||||
|
||||
c := cache.NewFrom(expirationInterval, purgeInterval, items)
|
||||
return &Cache{c: c}
|
||||
// Convert data slice to map required by go-cache.
|
||||
items := make(map[string]cache.Item, len(data))
|
||||
for _, d := range data {
|
||||
items[d.Key] = cache.Item{Object: d.Value, Expiration: int64(defaultExpiration)}
|
||||
}
|
||||
|
||||
instance.c = cache.NewFrom(instance.expiration, instance.purgeInterval, items)
|
||||
return instance
|
||||
}
|
||||
|
||||
// Set adds a key-value pair to the cache.
|
||||
func (c *Cache) Set(key, value string) {
|
||||
func (c *Cache) Set(key string, value any) {
|
||||
c.c.Set(key, value, defaultExpiration)
|
||||
}
|
||||
|
||||
// Get returns the value for the given key.
|
||||
func (c *Cache) Get(key string) (string, bool) {
|
||||
res, ok := c.c.Get(key)
|
||||
if !ok {
|
||||
return "", ok
|
||||
}
|
||||
return res.(string), ok
|
||||
func (c *Cache) Get(key string) (any, bool) {
|
||||
return c.c.Get(key)
|
||||
}
|
||||
|
||||
// Exists returns true if the given key exists in the cache.
|
||||
@@ -85,11 +110,11 @@ func (c *Cache) Keys() []string {
|
||||
}
|
||||
|
||||
// Values returns all values in the cache.
|
||||
func (c *Cache) Values() []string {
|
||||
func (c *Cache) Values() []any {
|
||||
items := c.c.Items()
|
||||
res := make([]string, 0, len(items))
|
||||
res := make([]any, 0, len(items))
|
||||
for _, v := range items {
|
||||
res = append(res, v.Object.(string))
|
||||
res = append(res, v.Object)
|
||||
}
|
||||
return res
|
||||
}
|
||||
|
||||
Vendored
+8
-6
@@ -7,8 +7,6 @@ import (
|
||||
"testing"
|
||||
|
||||
"github.com/google/go-cmp/cmp"
|
||||
|
||||
logContext "github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
)
|
||||
|
||||
func TestCache(t *testing.T) {
|
||||
@@ -34,7 +32,7 @@ func TestCache(t *testing.T) {
|
||||
// Test delete.
|
||||
c.Delete("key1")
|
||||
v, ok = c.Get("key1")
|
||||
if ok || v != "" {
|
||||
if ok || v != nil {
|
||||
t.Fatalf("Unexpected value for key1 after delete: %v, %v", v, ok)
|
||||
}
|
||||
|
||||
@@ -42,7 +40,7 @@ func TestCache(t *testing.T) {
|
||||
c.Set("key10", "key10")
|
||||
c.Clear()
|
||||
v, ok = c.Get("key10")
|
||||
if ok || v != "" {
|
||||
if ok || v != nil {
|
||||
t.Fatalf("Unexpected value for key10 after clear: %v, %v", v, ok)
|
||||
}
|
||||
|
||||
@@ -60,7 +58,10 @@ func TestCache(t *testing.T) {
|
||||
}
|
||||
|
||||
// Test getting only the values.
|
||||
vals := c.Values()
|
||||
vals := make([]string, 0, c.Count())
|
||||
for _, v := range c.Values() {
|
||||
vals = append(vals, v.(string))
|
||||
}
|
||||
sort.Strings(vals)
|
||||
sort.Strings(values)
|
||||
if !cmp.Equal(values, vals) {
|
||||
@@ -82,7 +83,8 @@ func TestCache(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestCache_NewWithData(t *testing.T) {
|
||||
c := NewWithData(logContext.Background(), []string{"key1", "key2", "key3"})
|
||||
data := []CacheEntry{{"key1", "value1"}, {"key2", "value2"}, {"key3", "value3"}}
|
||||
c := NewWithData(data)
|
||||
|
||||
// Test the count.
|
||||
if c.Count() != 3 {
|
||||
|
||||
+32
-40
@@ -2,12 +2,12 @@ package cleantemp
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/mitchellh/go-ps"
|
||||
|
||||
@@ -63,65 +63,57 @@ func CleanTempArtifacts(ctx logContext.Context) error {
|
||||
}
|
||||
}
|
||||
|
||||
tempDir := os.TempDir()
|
||||
artifacts, err := os.ReadDir(tempDir)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error reading temp dir: %w", err)
|
||||
if len(pids) == 0 {
|
||||
ctx.Logger().V(5).Info("No trufflehog processes were found")
|
||||
return nil
|
||||
}
|
||||
|
||||
for _, artifact := range artifacts {
|
||||
if trufflehogRE.MatchString(artifact.Name()) {
|
||||
tempDir := os.TempDir()
|
||||
dir, err := os.Open(tempDir)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error opening temp dir: %w", err)
|
||||
}
|
||||
defer dir.Close()
|
||||
|
||||
for {
|
||||
entries, err := dir.ReadDir(1) // read only one entry
|
||||
if err != nil {
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
continue
|
||||
}
|
||||
entry := entries[0]
|
||||
|
||||
if trufflehogRE.MatchString(entry.Name()) {
|
||||
|
||||
// Mark these artifacts initially as ones that should be deleted.
|
||||
shouldDelete := true
|
||||
// Check if the name matches any live PIDs.
|
||||
// Potential race condition here if a PID is started and creates tmp data after the initial check.
|
||||
for _, pidval := range pids {
|
||||
if strings.Contains(artifact.Name(), fmt.Sprintf("-%s-", pidval)) {
|
||||
if strings.Contains(entry.Name(), fmt.Sprintf("-%s-", pidval)) {
|
||||
shouldDelete = false
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if shouldDelete {
|
||||
artifactPath := filepath.Join(tempDir, artifact.Name())
|
||||
|
||||
var err error
|
||||
if artifact.IsDir() {
|
||||
err = os.RemoveAll(artifactPath)
|
||||
path := filepath.Join(tempDir, entry.Name())
|
||||
isDir := entry.IsDir()
|
||||
if isDir {
|
||||
err = os.RemoveAll(path)
|
||||
} else {
|
||||
err = os.Remove(artifactPath)
|
||||
err = os.Remove(path)
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("Error deleting temp artifact: %s", artifactPath)
|
||||
return fmt.Errorf("error deleting temp artifact (dir: %v) %s: %w", isDir, path, err)
|
||||
}
|
||||
|
||||
ctx.Logger().Info("Deleted orphaned temp artifact", "artifact", artifactPath)
|
||||
ctx.Logger().V(4).Info("Deleted orphaned temp artifact", "artifact", path)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// RunCleanupLoop runs a loop that cleans up orphaned directories every 15 seconds.
|
||||
func RunCleanupLoop(ctx logContext.Context) {
|
||||
err := CleanTempArtifacts(ctx)
|
||||
if err != nil {
|
||||
ctx.Logger().Error(err, "Error cleaning up orphaned directories ")
|
||||
}
|
||||
|
||||
const cleanupLoopInterval = 15 * time.Second
|
||||
ticker := time.NewTicker(cleanupLoopInterval)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ticker.C:
|
||||
if err := CleanTempArtifacts(ctx); err != nil {
|
||||
ctx.Logger().Error(err, "error cleaning up orphaned directories")
|
||||
}
|
||||
case <-ctx.Done():
|
||||
ctx.Logger().Info("Cleanup loop exiting due to context cancellation")
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -28,6 +28,7 @@ var (
|
||||
"wav",
|
||||
"flac",
|
||||
"webp",
|
||||
"pdf",
|
||||
|
||||
// images
|
||||
"png",
|
||||
@@ -79,6 +80,9 @@ var (
|
||||
"vxd": {}, // Virtual device driver in Windows
|
||||
"sfx": {}, // Self-extracting archive
|
||||
"bundle": {}, // Mac OS X application bundle
|
||||
"pyo": {}, // Compiled Python file
|
||||
"pyc": {}, // Compiled Python file
|
||||
"sym": {}, // Symbolic link, Unix/Linux
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@@ -43,6 +43,8 @@ type userRes struct {
|
||||
Name string `json:"name"`
|
||||
Company string `json:"company"`
|
||||
UserURL string `json:"html_url"`
|
||||
// Included in GitHub Enterprise Server.
|
||||
LdapDN string `json:"ldap_dn"`
|
||||
}
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
@@ -77,12 +79,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
client := common.SaneHttpClient()
|
||||
// https://developer.github.com/v3/users/#get-the-authenticated-user
|
||||
for _, url := range s.Endpoints(s.DefaultEndpoint()) {
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("%s/user", url), nil)
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, fmt.Sprintf("%s/user", url), nil)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
req.Header.Add("Content-Type", "application/json; charset=utf-8")
|
||||
req.Header.Add("Authorization", fmt.Sprintf("token %s", token))
|
||||
|
||||
req.Header.Set("Content-Type", "application/json; charset=utf-8")
|
||||
req.Header.Set("Authorization", fmt.Sprintf("token %s", token))
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
@@ -94,10 +97,28 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
s1.ExtraData["username"] = userResponse.Login
|
||||
s1.ExtraData["url"] = userResponse.UserURL
|
||||
s1.ExtraData["account_type"] = userResponse.Type
|
||||
s1.ExtraData["site_admin"] = fmt.Sprintf("%t", userResponse.SiteAdmin)
|
||||
s1.ExtraData["name"] = userResponse.Name
|
||||
s1.ExtraData["company"] = userResponse.Company
|
||||
s1.ExtraData["scopes"] = res.Header.Get("X-OAuth-Scopes")
|
||||
if userResponse.SiteAdmin {
|
||||
s1.ExtraData["site_admin"] = "true"
|
||||
}
|
||||
if userResponse.Name != "" {
|
||||
s1.ExtraData["name"] = userResponse.Name
|
||||
}
|
||||
if userResponse.Company != "" {
|
||||
s1.ExtraData["company"] = userResponse.Company
|
||||
}
|
||||
if userResponse.LdapDN != "" {
|
||||
s1.ExtraData["ldap_dn"] = userResponse.LdapDN
|
||||
}
|
||||
|
||||
// GitHub does not seem to consistently return this header.
|
||||
scopes := res.Header.Get("X-OAuth-Scopes")
|
||||
if scopes != "" {
|
||||
s1.ExtraData["scopes"] = scopes
|
||||
}
|
||||
expiry := res.Header.Get("github-authentication-token-expiration")
|
||||
if expiry != "" {
|
||||
s1.ExtraData["expires_at"] = expiry
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
|
||||
@@ -39,6 +39,8 @@ type userRes struct {
|
||||
Name string `json:"name"`
|
||||
Company string `json:"company"`
|
||||
UserURL string `json:"html_url"`
|
||||
// Included in GitHub Enterprise Server.
|
||||
LdapDN string `json:"ldap_dn"`
|
||||
}
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
@@ -78,12 +80,12 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
client := common.SaneHttpClient()
|
||||
// https://developer.github.com/v3/users/#get-the-authenticated-user
|
||||
for _, url := range s.Endpoints(s.DefaultEndpoint()) {
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("%s/user", url), nil)
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, fmt.Sprintf("%s/user", url), nil)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
req.Header.Add("Content-Type", "application/json; charset=utf-8")
|
||||
req.Header.Add("Authorization", fmt.Sprintf("token %s", token))
|
||||
req.Header.Set("Content-Type", "application/json; charset=utf-8")
|
||||
req.Header.Set("Authorization", fmt.Sprintf("token %s", token))
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
@@ -98,10 +100,28 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
s1.ExtraData["username"] = userResponse.Login
|
||||
s1.ExtraData["url"] = userResponse.UserURL
|
||||
s1.ExtraData["account_type"] = userResponse.Type
|
||||
s1.ExtraData["site_admin"] = fmt.Sprintf("%t", userResponse.SiteAdmin)
|
||||
s1.ExtraData["name"] = userResponse.Name
|
||||
s1.ExtraData["company"] = userResponse.Company
|
||||
s1.ExtraData["scopes"] = res.Header.Get("X-OAuth-Scopes")
|
||||
if userResponse.SiteAdmin {
|
||||
s1.ExtraData["site_admin"] = "true"
|
||||
}
|
||||
if userResponse.Name != "" {
|
||||
s1.ExtraData["name"] = userResponse.Name
|
||||
}
|
||||
if userResponse.Company != "" {
|
||||
s1.ExtraData["company"] = userResponse.Company
|
||||
}
|
||||
if userResponse.LdapDN != "" {
|
||||
s1.ExtraData["ldap_dn"] = userResponse.LdapDN
|
||||
}
|
||||
|
||||
// GitHub does not seem to consistently return this header.
|
||||
scopes := res.Header.Get("X-OAuth-Scopes")
|
||||
if scopes != "" {
|
||||
s1.ExtraData["scopes"] = scopes
|
||||
}
|
||||
expiry := res.Header.Get("github-authentication-token-expiration")
|
||||
if expiry != "" {
|
||||
s1.ExtraData["expires_at"] = expiry
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -23,13 +23,13 @@ var _ detectors.Detector = (*Scanner)(nil)
|
||||
var (
|
||||
defaultClient = common.SaneHttpClient()
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(`\bhf_[a-zA-Z0-9]{34}\b`)
|
||||
keyPat = regexp.MustCompile(`\b(?:hf_|api_org_)[a-zA-Z0-9]{34}\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"huggingface", "hugging_face", "hf"} // Huggingface docs occasionally use "hf" instead of "huggingface"
|
||||
return []string{"hf_", "api_org_"} // Huggingface docs occasionally use "hf" instead of "huggingface"
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify Huggingface secrets in a given set of bytes.
|
||||
@@ -92,15 +92,29 @@ func (s Scanner) verifyResult(ctx context.Context, apiKey string) (bool, map[str
|
||||
return true, nil, err
|
||||
}
|
||||
|
||||
t := whoamiRes.Auth.AccessToken
|
||||
var tokenInfo string
|
||||
switch {
|
||||
case whoamiRes.Auth.AccessToken.DisplayName != "" || whoamiRes.Auth.AccessToken.Role != "":
|
||||
// hf_xxxx token
|
||||
t := whoamiRes.Auth.AccessToken
|
||||
tokenInfo = fmt.Sprintf("%s (%s)", t.DisplayName, t.Role)
|
||||
|
||||
case whoamiRes.Auth.Type != "":
|
||||
// api_org_xxxx token
|
||||
tokenInfo = whoamiRes.Auth.Type
|
||||
|
||||
default:
|
||||
tokenInfo = "Unknown Token Type"
|
||||
}
|
||||
|
||||
extraData := map[string]string{
|
||||
"Username": whoamiRes.Name,
|
||||
"Email": whoamiRes.Email,
|
||||
"Token": fmt.Sprintf("%s (%s)", t.DisplayName, t.Role),
|
||||
"Token": tokenInfo,
|
||||
}
|
||||
|
||||
// Condense a list of organizations + roles.
|
||||
var orgs []string
|
||||
orgs := make([]string, 0, len(whoamiRes.Organizations))
|
||||
for _, org := range whoamiRes.Organizations {
|
||||
orgs = append(orgs, fmt.Sprintf("%s:%s", org.Name, org.Role))
|
||||
}
|
||||
@@ -136,7 +150,8 @@ type organization struct {
|
||||
|
||||
type auth struct {
|
||||
AccessToken struct {
|
||||
DisplayName string `json:"displayName"`
|
||||
Role string `json:"role"`
|
||||
} `json:"accessToken"`
|
||||
DisplayName string `json:"displayName,omitempty"`
|
||||
Role string `json:"role,omitempty"`
|
||||
} `json:"accessToken,omitempty"`
|
||||
Type string `json:"type,omitempty"`
|
||||
}
|
||||
|
||||
@@ -12,16 +12,18 @@ import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
type Scanner struct{}
|
||||
type Scanner struct {
|
||||
client *http.Client
|
||||
}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
client = common.SaneHttpClient()
|
||||
defaultClient = common.SaneHttpClient()
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"parseur"}) + `\b([a-f0-9]{40})\b`)
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"parseur[^il]"}) + `\b([a-f0-9]{40})\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
@@ -35,35 +37,30 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
dataStr := string(data)
|
||||
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
|
||||
for _, match := range matches {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Parseur,
|
||||
Raw: []byte(resMatch),
|
||||
}
|
||||
|
||||
if verify {
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.parseur.com/", nil)
|
||||
if err != nil {
|
||||
continue
|
||||
if s.client == nil {
|
||||
s.client = defaultClient
|
||||
}
|
||||
req.Header.Add("Authorization", fmt.Sprintf("Token %s", resMatch))
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else {
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
isVerified, verificationErr := verifyResult(ctx, s.client, resMatch)
|
||||
s1.Verified = isVerified
|
||||
s1.SetVerificationError(verificationErr, resMatch)
|
||||
}
|
||||
|
||||
if !s1.Verified {
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
@@ -73,6 +70,25 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
return results, nil
|
||||
}
|
||||
|
||||
func verifyResult(ctx context.Context, client *http.Client, token string) (bool, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://api.parseur.com/", nil)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
req.Header.Add("Authorization", fmt.Sprintf("Token %s", token))
|
||||
res, err := client.Do(req)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
return true, nil
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
|
||||
func (s Scanner) Type() detectorspb.DetectorType {
|
||||
return detectorspb.DetectorType_Parseur
|
||||
}
|
||||
|
||||
@@ -7,12 +7,117 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/kylelemons/godebug/pretty"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
func TestParseur_Pattern(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
data string
|
||||
shouldMatch bool
|
||||
match string
|
||||
}{
|
||||
// True positives
|
||||
{
|
||||
name: "valid",
|
||||
data: `const parseurToken = "6813a07afc6b4ed35518635c6fb70abf4e721962";`,
|
||||
shouldMatch: true,
|
||||
match: "6813a07afc6b4ed35518635c6fb70abf4e721962",
|
||||
},
|
||||
// This technically isn't valid but shouldn't be excluded based on the current pattern.
|
||||
{
|
||||
name: "valid",
|
||||
data: `commit 6813a07afc6b4ed35518635c6fb70abf4e721962
|
||||
Author: Stéphane Borel <[email protected]>
|
||||
Date: Thu Dec 30 13:59:59 1999 +0000
|
||||
|
||||
* Modifications de quelques erreurs sur le parseur
|
||||
|
||||
commit 2c65bd981d308d264aa0c07083b2bc914905deb3`,
|
||||
shouldMatch: true,
|
||||
match: "2c65bd981d308d264aa0c07083b2bc914905deb3",
|
||||
},
|
||||
|
||||
// False positives
|
||||
{
|
||||
name: `invalid_parseuri_package.json`,
|
||||
data: `{
|
||||
"dist": {
|
||||
"shasum": "80204a50d4dbb779bfdc6ebe2778d90e4bce320a",
|
||||
"tarball": "https://registry.npmjs.org/parseuri/-/parseuri-0.0.5.tgz"
|
||||
},
|
||||
"gitHead": "792c9a63162a4484eb6b4f95fc611ccf224a24b6",`,
|
||||
shouldMatch: false,
|
||||
},
|
||||
// https://github.com/airalab/airapkgs/blob/cb3f8021303f79345f65b5328b75117044bde852/pkgs/servers/mesh/meshviewer/yarn.nix#L6066
|
||||
{
|
||||
name: `invalid_parseuri_nix`,
|
||||
data: `
|
||||
{
|
||||
name = "parseuri-0.0.5.tgz";
|
||||
path = fetchurl {
|
||||
name = "parseuri-0.0.5.tgz";
|
||||
url = "https://registry.yarnpkg.com/parseuri/-/parseuri-0.0.5.tgz";
|
||||
sha1 = "80204a50d4dbb779bfdc6ebe2778d90e4bce320a";
|
||||
};
|
||||
}`,
|
||||
shouldMatch: false,
|
||||
},
|
||||
{
|
||||
name: `invalid_parseurl_yarn`,
|
||||
data: `parseurl@~1.3.1:
|
||||
version "1.3.1"
|
||||
resolved "https://registry.yarnpkg.com/parseurl/-/parseurl-1.3.1.tgz#c8ab8c9223ba34888aa64a297b28853bec18da56"`,
|
||||
shouldMatch: false,
|
||||
},
|
||||
// https://github.com/tolerious/django-wechat/blob/18f3f2d5d8377c7dde8700afc5977861c8488b68/django_weixin/Sample.py#L30
|
||||
{
|
||||
name: `invalid_parseurl_func`,
|
||||
data: `#sVerifyMsgSig=HttpUtils.ParseUrl("msg_signature")
|
||||
sVerifyMsgSig="5c45ff5e21c57e6ad56bac8758b79b1d9ac89fd3"`,
|
||||
shouldMatch: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
s := Scanner{}
|
||||
|
||||
results, err := s.FromData(context.Background(), false, []byte(test.data))
|
||||
if err != nil {
|
||||
t.Errorf("Parseur.FromData() error = %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
if test.shouldMatch {
|
||||
if len(results) == 0 {
|
||||
t.Errorf("%s: did not receive a match for '%v' when one was expected", test.name, test.data)
|
||||
return
|
||||
}
|
||||
expected := test.data
|
||||
if test.match != "" {
|
||||
expected = test.match
|
||||
}
|
||||
result := results[0]
|
||||
resultData := string(result.Raw)
|
||||
if resultData != expected {
|
||||
t.Errorf("%s: did not receive expected match.\n\texpected: '%s'\n\t actual: '%s'", test.name, expected, resultData)
|
||||
return
|
||||
}
|
||||
} else {
|
||||
if len(results) > 0 {
|
||||
t.Errorf("%s: received a match for '%v' when one wasn't wanted", test.name, test.data)
|
||||
return
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseur_FromChunk(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
|
||||
defer cancel()
|
||||
|
||||
@@ -0,0 +1,254 @@
|
||||
package postgres
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
_ "github.com/lib/pq" // PostgreSQL driver
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultPort = "5432"
|
||||
defaultHost = "localhost"
|
||||
)
|
||||
|
||||
var (
|
||||
_ detectors.Detector = (*Scanner)(nil)
|
||||
uriPattern = regexp.MustCompile(`\b(?i)postgresql://[\S]+\b`)
|
||||
hostnamePattern = regexp.MustCompile(`(?i)(?:host|server|address).{0,40}?(\b[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*\b)`)
|
||||
portPattern = regexp.MustCompile(`(?i)(?:port|p).{0,40}?(\b[0-9]{1,5}\b)`)
|
||||
usernamePattern = regexp.MustCompile(`(?im)(?:user|usr)\S{0,40}?[:=\s]{1,3}[ '"=]{0,1}([^:'"\s]{4,40})`)
|
||||
passwordPattern = regexp.MustCompile(`(?im)(?:pass)\S{0,40}?[:=\s]{1,3}[ '"=]{0,1}([^:'"\s]{4,40})`)
|
||||
)
|
||||
|
||||
type Scanner struct{}
|
||||
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"postgres", "psql", "pghost"}
|
||||
}
|
||||
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) ([]detectors.Result, error) {
|
||||
var results []detectors.Result
|
||||
var pgURLs []url.URL
|
||||
pgURLs = append(pgURLs, findUriMatches(string(data)))
|
||||
pgURLs = append(pgURLs, findComponentMatches(verify, string(data))...)
|
||||
|
||||
for _, pgURL := range pgURLs {
|
||||
if pgURL.User == nil {
|
||||
continue
|
||||
}
|
||||
username := pgURL.User.Username()
|
||||
password, _ := pgURL.User.Password()
|
||||
hostport := pgURL.Host
|
||||
result := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Postgres,
|
||||
Raw: []byte(hostport + username + password),
|
||||
RawV2: []byte(hostport + username + password),
|
||||
}
|
||||
|
||||
if verify {
|
||||
timeoutInSeconds := getDeadlineInSeconds(ctx)
|
||||
isVerified, verificationErr := verifyPostgres(&pgURL, timeoutInSeconds)
|
||||
result.Verified = isVerified
|
||||
result.SetVerificationError(verificationErr, password)
|
||||
}
|
||||
|
||||
if !result.Verified && detectors.IsKnownFalsePositive(password, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
results = append(results, result)
|
||||
}
|
||||
|
||||
return results, nil
|
||||
}
|
||||
|
||||
func getDeadlineInSeconds(ctx context.Context) int {
|
||||
deadline, ok := ctx.Deadline()
|
||||
if !ok {
|
||||
// Context does not have a deadline
|
||||
return 0
|
||||
}
|
||||
|
||||
duration := time.Until(deadline)
|
||||
return int(duration.Seconds())
|
||||
}
|
||||
|
||||
func findUriMatches(dataStr string) url.URL {
|
||||
var pgURL url.URL
|
||||
for _, uri := range uriPattern.FindAllString(dataStr, -1) {
|
||||
pgURL, err := url.Parse(uri)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if pgURL.User != nil {
|
||||
return *pgURL
|
||||
}
|
||||
}
|
||||
return pgURL
|
||||
}
|
||||
|
||||
// check if postgres is running
|
||||
func postgresRunning(hostname, port string) bool {
|
||||
connStr := fmt.Sprintf("host=%s port=%s sslmode=disable", hostname, port)
|
||||
db, err := sql.Open("postgres", connStr)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
defer db.Close()
|
||||
return true
|
||||
}
|
||||
|
||||
func findComponentMatches(verify bool, dataStr string) []url.URL {
|
||||
usernameMatches := usernamePattern.FindAllStringSubmatch(dataStr, -1)
|
||||
passwordMatches := passwordPattern.FindAllStringSubmatch(dataStr, -1)
|
||||
hostnameMatches := hostnamePattern.FindAllStringSubmatch(dataStr, -1)
|
||||
portMatches := portPattern.FindAllStringSubmatch(dataStr, -1)
|
||||
|
||||
var pgURLs []url.URL
|
||||
|
||||
hosts := findHosts(verify, hostnameMatches, portMatches)
|
||||
|
||||
for _, username := range dedupMatches(usernameMatches) {
|
||||
for _, password := range dedupMatches(passwordMatches) {
|
||||
for _, host := range hosts {
|
||||
hostname, port := strings.Split(host, ":")[0], strings.Split(host, ":")[1]
|
||||
if combinedLength := len(username) + len(password) + len(hostname); combinedLength > 255 {
|
||||
continue
|
||||
}
|
||||
postgresURL := url.URL{
|
||||
Scheme: "postgresql",
|
||||
User: url.UserPassword(username, password),
|
||||
Host: fmt.Sprintf("%s:%s", hostname, port),
|
||||
}
|
||||
pgURLs = append(pgURLs, postgresURL)
|
||||
}
|
||||
}
|
||||
}
|
||||
return pgURLs
|
||||
}
|
||||
|
||||
// if verification is turned on, and we can confirm that postgres is running on at least one host,
|
||||
// return only hosts where it's running. otherwise return all hosts.
|
||||
func findHosts(verify bool, hostnameMatches, portMatches [][]string) []string {
|
||||
hostnames := dedupMatches(hostnameMatches)
|
||||
ports := dedupMatches(portMatches)
|
||||
var hosts []string
|
||||
|
||||
if len(hostnames) < 1 {
|
||||
hostnames = append(hostnames, defaultHost)
|
||||
}
|
||||
|
||||
if len(ports) < 1 {
|
||||
ports = append(ports, defaultPort)
|
||||
}
|
||||
|
||||
for _, hostname := range hostnames {
|
||||
for _, port := range ports {
|
||||
hosts = append(hosts, fmt.Sprintf("%s:%s", hostname, port))
|
||||
}
|
||||
}
|
||||
|
||||
if verify {
|
||||
var verifiedHosts []string
|
||||
for _, host := range hosts {
|
||||
parts := strings.Split(host, ":")
|
||||
hostname, port := parts[0], parts[1]
|
||||
if postgresRunning(hostname, port) {
|
||||
verifiedHosts = append(verifiedHosts, host)
|
||||
}
|
||||
}
|
||||
if len(verifiedHosts) > 0 {
|
||||
return verifiedHosts
|
||||
}
|
||||
}
|
||||
|
||||
return hosts
|
||||
}
|
||||
|
||||
// deduplicate matches in order to reduce the number of verification requests
|
||||
func dedupMatches(matches [][]string) []string {
|
||||
setOfMatches := make(map[string]struct{})
|
||||
for _, match := range matches {
|
||||
if len(match) > 1 {
|
||||
setOfMatches[match[1]] = struct{}{}
|
||||
}
|
||||
}
|
||||
var results []string
|
||||
for match := range setOfMatches {
|
||||
results = append(results, match)
|
||||
}
|
||||
return results
|
||||
}
|
||||
|
||||
func verifyPostgres(pgURL *url.URL, timeoutInSeconds int) (bool, error) {
|
||||
if pgURL.User == nil {
|
||||
return false, nil
|
||||
}
|
||||
username := pgURL.User.Username()
|
||||
password, _ := pgURL.User.Password()
|
||||
|
||||
hostname, port := pgURL.Hostname(), pgURL.Port()
|
||||
if hostname == "" {
|
||||
hostname = defaultHost
|
||||
}
|
||||
if port == "" {
|
||||
port = defaultPort
|
||||
}
|
||||
|
||||
sslmode := determineSSLMode(pgURL)
|
||||
|
||||
connStr := fmt.Sprintf("user=%s password=%s host=%s port=%s sslmode=%s", username, password, hostname, port, sslmode)
|
||||
if timeoutInSeconds > 0 {
|
||||
connStr = fmt.Sprintf("%s connect_timeout=%d", connStr, timeoutInSeconds)
|
||||
}
|
||||
|
||||
db, err := sql.Open("postgres", connStr)
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "connection refused") {
|
||||
// inactive host
|
||||
return false, nil
|
||||
}
|
||||
return false, err
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second)
|
||||
defer cancel()
|
||||
|
||||
err = db.PingContext(ctx)
|
||||
if err == nil {
|
||||
return true, nil
|
||||
} else if strings.Contains(err.Error(), "password authentication failed") || // incorrect username or password
|
||||
strings.Contains(err.Error(), "connection refused") { // inactive host
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// if ssl is not enabled, manually fall-back to sslmode=disable
|
||||
if strings.Contains(err.Error(), "SSL is not enabled on the server") {
|
||||
pgURL.RawQuery = fmt.Sprintf("sslmode=%s", "disable")
|
||||
return verifyPostgres(pgURL, timeoutInSeconds)
|
||||
}
|
||||
return false, err
|
||||
}
|
||||
|
||||
func determineSSLMode(pgURL *url.URL) string {
|
||||
// default ssl mode is "prefer" per https://www.postgresql.org/docs/current/libpq-ssl.html
|
||||
// but is currently not implemented in the driver per https://github.com/lib/pq/issues/1006
|
||||
// default for the driver is "require". ideally we would use "allow" but that is also not supported by the driver.
|
||||
sslmode := "require"
|
||||
if sslQuery, ok := pgURL.Query()["sslmode"]; ok && len(sslQuery) > 0 {
|
||||
sslmode = sslQuery[0]
|
||||
}
|
||||
return sslmode
|
||||
}
|
||||
|
||||
func (s Scanner) Type() detectorspb.DetectorType {
|
||||
return detectorspb.DetectorType_Postgres
|
||||
}
|
||||
@@ -0,0 +1,340 @@
|
||||
//go:build detectors
|
||||
// +build detectors
|
||||
|
||||
package postgres
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os/exec"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/google/go-cmp/cmp/cmpopts"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
var postgresDockerHash string
|
||||
|
||||
const (
|
||||
postgresUser = "postgres"
|
||||
postgresPass = "23201dabb56ca236f3dc6736c0f9afad"
|
||||
postgresHost = "localhost"
|
||||
postgresPort = "5433"
|
||||
|
||||
inactiveUser = "inactive"
|
||||
inactivePass = "inactive"
|
||||
inactivePort = "61000"
|
||||
inactiveHost = "192.0.2.0"
|
||||
)
|
||||
|
||||
func TestPostgres_FromChunk(t *testing.T) {
|
||||
startPostgres()
|
||||
defer stopPostgres()
|
||||
|
||||
type args struct {
|
||||
ctx context.Context
|
||||
data []byte
|
||||
verify bool
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "not found",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte("You cannot find the secret within"),
|
||||
verify: true,
|
||||
},
|
||||
want: nil,
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "found with seperated credentials, verified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf(`
|
||||
POSTGRES_USER=%s
|
||||
POSTGRES_PASSWORD=%s
|
||||
POSTGRES_ADDRESS=%s
|
||||
POSTGRES_PORT=%s
|
||||
`, postgresUser, postgresPass, postgresHost, postgresPort)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Postgres,
|
||||
Verified: true,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "found with single line credentials, verified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf(`postgresql://%s:%s@%s:%s/postgres`, postgresUser, postgresPass, postgresHost, postgresPort)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Postgres,
|
||||
Verified: true,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "found with json credentials, verified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf(
|
||||
`DB_CONFIG={"user": "%s", "password": "%s", "host": "%s", "port": "%s", "database": "postgres"}`, postgresUser, postgresPass, postgresHost, postgresPort)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Postgres,
|
||||
Verified: true,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "found with seperated credentials, unverified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf(`
|
||||
POSTGRES_USER=%s
|
||||
POSTGRES_PASSWORD=%s
|
||||
POSTGRES_ADDRESS=%s
|
||||
POSTGRES_PORT=%s
|
||||
`, postgresUser, inactivePass, postgresHost, postgresPort)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Postgres,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "found with seperated credentials - no port, unverified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf(`
|
||||
POSTGRES_USER=%s
|
||||
POSTGRES_PASSWORD=%s
|
||||
POSTGRES_ADDRESS=%s
|
||||
`, postgresUser, inactivePass, postgresHost)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Postgres,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "found with single line credentials, unverified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf(`postgresql://%s:%s@%s:%s/postgres`, postgresUser, inactivePass, postgresHost, postgresPort)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Postgres,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "found with json credentials, unverified - inactive password",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf(
|
||||
`DB_CONFIG={"user": "%s", "password": "%s", "host": "%s", "port": "%s", "database": "postgres"}`, postgresUser, inactivePass, postgresHost, postgresPort)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Postgres,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "found with json credentials, unverified - inactive user",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf(
|
||||
`DB_CONFIG={"user": "%s", "password": "%s", "host": "%s", "port": "%s", "database": "postgres"}`, inactiveUser, postgresPass, postgresHost, postgresPort)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Postgres,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, unverified due to error - inactive port",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf(`postgresql://%s:%s@%s:%s/postgres`, postgresUser, postgresPass, postgresHost, inactivePort)),
|
||||
verify: true,
|
||||
},
|
||||
want: func() []detectors.Result {
|
||||
r := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Postgres,
|
||||
Verified: false,
|
||||
}
|
||||
return []detectors.Result{r}
|
||||
}(),
|
||||
wantErr: false,
|
||||
},
|
||||
// This test seems take a long time to run (70s+) even with the timeout set to 1s. It's not clear why.
|
||||
{
|
||||
name: "found, unverified due to error - inactive host",
|
||||
s: Scanner{},
|
||||
args: func() args {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
|
||||
defer cancel()
|
||||
return args{
|
||||
ctx: ctx,
|
||||
data: []byte(fmt.Sprintf(`postgresql://%s:%s@%s:%s/postgres`, postgresUser, postgresPass, inactiveHost, postgresPort)),
|
||||
verify: true,
|
||||
}
|
||||
}(),
|
||||
want: func() []detectors.Result {
|
||||
r := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Postgres,
|
||||
Verified: false,
|
||||
}
|
||||
r.SetVerificationError(errors.New("i/o timeout"))
|
||||
return []detectors.Result{r}
|
||||
}(),
|
||||
wantErr: false,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
s := Scanner{}
|
||||
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("postgres.FromData() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
for i := range got {
|
||||
if len(got[i].Raw) == 0 {
|
||||
t.Fatalf("no raw secret present: \n %+v", got[i])
|
||||
}
|
||||
gotErr := ""
|
||||
if got[i].VerificationError() != nil {
|
||||
gotErr = got[i].VerificationError().Error()
|
||||
}
|
||||
wantErr := ""
|
||||
if tt.want[i].VerificationError() != nil {
|
||||
wantErr = tt.want[i].VerificationError().Error()
|
||||
}
|
||||
if gotErr != wantErr {
|
||||
t.Fatalf("wantVerificationError = %v, verification error = %v", tt.want[i].VerificationError(), got[i].VerificationError())
|
||||
}
|
||||
}
|
||||
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError")
|
||||
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
|
||||
t.Errorf("Postgres.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func dockerLogLine(hash string, needle string) chan struct{} {
|
||||
ch := make(chan struct{}, 1)
|
||||
go func() {
|
||||
for {
|
||||
out, err := exec.Command("docker", "logs", hash).CombinedOutput()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
if strings.Contains(string(out), needle) {
|
||||
ch <- struct{}{}
|
||||
return
|
||||
}
|
||||
time.Sleep(1 * time.Second)
|
||||
}
|
||||
}()
|
||||
return ch
|
||||
}
|
||||
|
||||
func startPostgres() error {
|
||||
cmd := exec.Command(
|
||||
"docker", "run", "--rm", "-p", postgresPort+":"+defaultPort,
|
||||
"-e", "POSTGRES_PASSWORD="+postgresPass,
|
||||
"-e", "POSTGRES_USER="+postgresUser,
|
||||
"-d", "postgres",
|
||||
)
|
||||
fmt.Println(cmd.String())
|
||||
out, err := cmd.Output()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
postgresDockerHash = string(bytes.TrimSpace(out))
|
||||
select {
|
||||
case <-dockerLogLine(postgresDockerHash, "PostgreSQL init process complete; ready for start up."):
|
||||
return nil
|
||||
case <-time.After(30 * time.Second):
|
||||
stopPostgres()
|
||||
return errors.New("timeout waiting for postgres database to be ready")
|
||||
}
|
||||
}
|
||||
|
||||
func stopPostgres() {
|
||||
exec.Command("docker", "kill", postgresDockerHash).Run()
|
||||
}
|
||||
|
||||
func BenchmarkFromData(benchmark *testing.B) {
|
||||
ctx := context.Background()
|
||||
s := Scanner{}
|
||||
for name, data := range detectors.MustGetBenchmarkData() {
|
||||
benchmark.Run(name, func(b *testing.B) {
|
||||
b.ResetTimer()
|
||||
for n := 0; n < b.N; n++ {
|
||||
_, err := s.FromData(ctx, false, data)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -13,16 +13,19 @@ import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
type Scanner struct{}
|
||||
type Scanner struct {
|
||||
client *http.Client
|
||||
}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time.
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
client = common.SaneHttpClient()
|
||||
defaultClient = common.SaneHttpClient()
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"signable"}) + `\b([a-zA-Z-0-9]{32})\b`)
|
||||
tokenPat = regexp.MustCompile(detectors.PrefixRegex([]string{".{0,2}signable"}) + `\b([a-zA-Z-0-9]{32})\b`)
|
||||
keywordPat = regexp.MustCompile(`(?i)([a-z]{2})signable`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
@@ -35,41 +38,34 @@ func (s Scanner) Keywords() []string {
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
|
||||
matches := tokenPat.FindAllStringSubmatch(dataStr, -1)
|
||||
for _, match := range matches {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
|
||||
if isCommonFalsePositive(match[0]) {
|
||||
continue
|
||||
}
|
||||
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Signable,
|
||||
Raw: []byte(resMatch),
|
||||
}
|
||||
|
||||
if verify {
|
||||
data := fmt.Sprintf("%s:", resMatch)
|
||||
sEnc := b64.StdEncoding.EncodeToString([]byte(data))
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.signable.co.uk/v1/templates?offset=0&limit=5", nil)
|
||||
if err != nil {
|
||||
continue
|
||||
if s.client == nil {
|
||||
s.client = defaultClient
|
||||
}
|
||||
isVerified, verificationErr := verifyResult(ctx, s.client, resMatch)
|
||||
s1.Verified = isVerified
|
||||
s1.SetVerificationError(verificationErr, resMatch)
|
||||
}
|
||||
|
||||
req.Header.Add("Authorization", fmt.Sprintf("Basic %s", sEnc))
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else {
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if !s1.Verified && detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
|
||||
results = append(results, s1)
|
||||
@@ -78,6 +74,44 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
return results, nil
|
||||
}
|
||||
|
||||
// Eliminate the most common false positive.
|
||||
func isCommonFalsePositive(line string) bool {
|
||||
// TODO: Skip lock files altogether. (https://github.com/trufflesecurity/trufflehog/issues/1517)
|
||||
if strings.Contains(line, "helper-explode-assignable-expression") {
|
||||
return true
|
||||
}
|
||||
|
||||
// Eliminate false positives from `assignable` and `designable`.
|
||||
for _, m := range keywordPat.FindAllStringSubmatch(line, -1) {
|
||||
if strings.EqualFold(m[1], "as") || strings.EqualFold(m[1], "de") {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func verifyResult(ctx context.Context, client *http.Client, token string) (bool, error) {
|
||||
data := fmt.Sprintf("%s:", token)
|
||||
sEnc := b64.StdEncoding.EncodeToString([]byte(data))
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://api.signable.co.uk/v1/templates?offset=0&limit=5", nil)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
req.Header.Add("Authorization", fmt.Sprintf("Basic %s", sEnc))
|
||||
res, err := client.Do(req)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
return true, nil
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
|
||||
func (s Scanner) Type() detectorspb.DetectorType {
|
||||
return detectorspb.DetectorType_Signable
|
||||
}
|
||||
|
||||
@@ -10,12 +10,162 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/kylelemons/godebug/pretty"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
func TestSignable_Pattern(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
data string
|
||||
shouldMatch bool
|
||||
match string
|
||||
}{
|
||||
// True positives
|
||||
{
|
||||
name: "valid",
|
||||
data: `const signableToken = '40a1cd917bff1288f699a94a75b37a1a'`,
|
||||
shouldMatch: true,
|
||||
match: "40a1cd917bff1288f699a94a75b37a1a",
|
||||
},
|
||||
|
||||
// False positives
|
||||
{
|
||||
name: `invalid_assignable_yarn`,
|
||||
data: `" babel-helper-explode-assignable-expression@^6.24.1:
|
||||
version "6.24.1"
|
||||
resolved "https://registry.npmjs.org/babel-helper-explode-assignable-expression/-/babel-helper-explode-assignable-expression-6.24.1.tgz#f25b82cf7dc10433c55f70592d5746400ac22caa"
|
||||
dependencies:
|
||||
babel-runtime "^6.22.0"
|
||||
babel-traverse "^6.24.1"
|
||||
babel-types "^6.24.1"`,
|
||||
shouldMatch: false,
|
||||
},
|
||||
{
|
||||
name: `invalid_assignable_yarn`,
|
||||
data: `"@babel/helper-explode-assignable-expression@^7.16.7":
|
||||
version "7.16.7"
|
||||
resolved "https://registry.yarnpkg.com/@babel/helper-explode-assignable-expression/-/helper-explode-assignable-expression-7.16.7.tgz#12a6d8522fdd834f194e868af6354e8650242b7a"
|
||||
integrity sha512-KyUenhWMC8VrxzkGP0Jizjo4/Zx+1nNZhgocs+gLzyZyB8SHidhoq9KK/8Ato4anhwsivfkBLftky7gvzbZMtQ==
|
||||
dependencies:
|
||||
"@babel/types" "^7.16.7"`,
|
||||
shouldMatch: false,
|
||||
},
|
||||
// https://github.com/tbenst/purescript-nix-example/blob/558c8d6cb605742218cfa14a3fa93c062324b885/yarn.nix
|
||||
{
|
||||
name: `invalid_assignable_nix`,
|
||||
data: ` {
|
||||
name = "_babel_helper_explode_assignable_expression___helper_explode_assignable_expression_7.8.3.tgz";
|
||||
path = fetchurl {
|
||||
name = "_babel_helper_explode_assignable_expression___helper_explode_assignable_expression_7.8.3.tgz";
|
||||
url = "https://registry.yarnpkg.com/@babel/helper-explode-assignable-expression/-/helper-explode-assignable-expression-7.8.3.tgz";
|
||||
sha1 = "a728dc5b4e89e30fc2dfc7d04fa28a930653f982";
|
||||
};
|
||||
}`,
|
||||
shouldMatch: false,
|
||||
},
|
||||
{
|
||||
name: `invalid_assignable`,
|
||||
data: `<tr><td colspan="2"><br><h2>Public Member Functions</h2></td></tr>
|
||||
<tr><td class="memItemLeft" nowrap align="right" valign="top"><a class="anchor" name="b0a0dbf6ca9028bbbb2240cad5882537"></a><!-- doxytag: member="boost::gil::Assignable::constraints" ref="b0a0dbf6ca9028bbbb2240cad5882537" args="()" -->
|
||||
void </td><td class="memItemRight" valign="bottom"><b>constraints</b> ()</td></tr>
|
||||
`,
|
||||
shouldMatch: false,
|
||||
},
|
||||
{
|
||||
name: `invalid_assignable`,
|
||||
data: `File: enumIsAssignableToBuiltInEnum.kt - 6396cf8549625bfce8b8ca2511d7f347
|
||||
NL("\n")
|
||||
packageHeader`,
|
||||
shouldMatch: false,
|
||||
},
|
||||
{
|
||||
name: `invalid_assignable_php`,
|
||||
data: `'./include/SugarObjects/forms/PersonFormBase.php' => '2c1846ef127d60a40ecbab2c0b312ff5',
|
||||
'./include/SugarObjects/implements/assignable/language/en_us.lang.php' => '90f14b03e22e1eed2a1b93e10b975ef5',
|
||||
'./include/SugarObjects/implements/assignable/vardefs.php' => '358e0c47f753c5577fbdc0de08553c02',
|
||||
'./include/SugarObjects/implements/security_groups/language/en_us.lang.php' => 'ac1fd4817cb4662e3bdf973836558bdb',`,
|
||||
shouldMatch: false,
|
||||
},
|
||||
// https://github.com/past-due/warzone2100/blob/3e5637a7ed3d67ab92e439b94bf93f89f7bbea51/ChangeLog#L318
|
||||
{
|
||||
name: `invalid_designable`,
|
||||
data: ` * Fix: Prevent map selection button list from going off the form (commit:aea66eb1aa557c73d97b8019e5e66fccbb79f66e, #1347)
|
||||
* Fix: Fix odd EMP mortar pathway; Add EMP mortar to designable weapons (commit:bcf93b7fe640c09e8b1239fabfd901fde9760259, #1535)`,
|
||||
shouldMatch: false,
|
||||
},
|
||||
// https://github.com/exis-io/Exis/blob/5383174f7b52112a97aadd09e6b9ea837c2fa07b/CardsAgainstHumanityDemo/swiftCardsAgainst/Pods/Pods.xcodeproj/project.pbxproj
|
||||
{
|
||||
name: `invalid_designable`,
|
||||
data: ` 570767CBD99941F484DED46232044DC3 /* DesignableView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 93111B182BD71051B9ED0B14A9EF6EB6 /* DesignableView.swift */; };
|
||||
57140D31D50A2FDE1E26729DDE7CB762 /* M13ProgressHUD.h in Headers */ = {isa = PBXBuildFile; fileRef = ECF777CB8B4C090E8D271E62729F7DD3 /* M13ProgressHUD.h */; settings = {ATTRIBUTES = (Public, ); }; };`,
|
||||
shouldMatch: false,
|
||||
},
|
||||
{
|
||||
name: `invalid_designable`,
|
||||
data: `{"nick":"hamster88","message":"this is my gist > https://gist.github.com/thedesignable/a05f628c649a81aae757945c352a8392","date":"2016-06-19T11:05:13.776Z","type":"message"}`,
|
||||
shouldMatch: false,
|
||||
},
|
||||
{
|
||||
name: `invalid_designable`,
|
||||
data: `返回到故事板文件，选择视图(我将假设从现在起视图被选中)并打开 Identity Inspector。你会注意到一个*可设计的*状态指示器已经出现在自定义类部分。
|
||||
|
||||
`,
|
||||
shouldMatch: false,
|
||||
},
|
||||
{
|
||||
name: `invalid_designable`,
|
||||
data: `<h3 id="ibdesignable-x-paintcode:0b699a3cd6d609650a3fca90a5cd32cc">IBDesignable x PaintCode</h3>`,
|
||||
shouldMatch: false,
|
||||
},
|
||||
{
|
||||
name: `invalid_designable`,
|
||||
data: ` </designables>
|
||||
<resources>
|
||||
<image name="a932cb605eb09bff88b88fdf1c3ef8aa" width="736" height="895"/>
|
||||
<image name="plus" catalog="system" width="128" height="113"/>`,
|
||||
shouldMatch: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
s := Scanner{}
|
||||
|
||||
results, err := s.FromData(context.Background(), false, []byte(test.data))
|
||||
if err != nil {
|
||||
t.Errorf("Signable.FromData() error = %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
if test.shouldMatch {
|
||||
if len(results) == 0 {
|
||||
t.Errorf("%s: did not receive a match for '%v' when one was expected", test.name, test.data)
|
||||
return
|
||||
}
|
||||
expected := test.data
|
||||
if test.match != "" {
|
||||
expected = test.match
|
||||
}
|
||||
result := results[0]
|
||||
resultData := string(result.Raw)
|
||||
if resultData != expected {
|
||||
t.Errorf("%s: did not receive expected match.\n\texpected: '%s'\n\t actual: '%s'", test.name, expected, resultData)
|
||||
return
|
||||
}
|
||||
} else {
|
||||
if len(results) > 0 {
|
||||
t.Errorf("%s: received a match for '%v' when one wasn't wanted", test.name, test.data)
|
||||
return
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSignable_FromChunk(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
|
||||
defer cancel()
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode"
|
||||
|
||||
_ "github.com/snowflakedb/gosnowflake"
|
||||
@@ -116,6 +117,10 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
ctx = context.Background()
|
||||
}
|
||||
|
||||
// Disable pool + retries to prevent flooding the server with failed login attemps.
|
||||
db.SetConnMaxLifetime(time.Second)
|
||||
db.SetMaxOpenConns(1)
|
||||
|
||||
err = db.PingContext(ctx)
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "Incorrect username or password was specified") {
|
||||
|
||||
@@ -22,7 +22,7 @@ var _ detectors.Detector = (*Scanner)(nil)
|
||||
var (
|
||||
defaultClient = common.SaneHttpClient()
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(`\b(sgp_[a-f0-9]{40})\b`)
|
||||
keyPat = regexp.MustCompile(`\b(sgp_(?:[a-fA-F0-9]{16}|local)_[a-fA-F0-9]{40}|sgp_[a-fA-F0-9]{40}|[a-fA-F0-9]{40})\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
@@ -47,6 +47,9 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
DetectorType: detectorspb.DetectorType_Sourcegraph,
|
||||
Raw: []byte(resMatch),
|
||||
}
|
||||
s1.ExtraData = map[string]string{
|
||||
"rotation_guide": "https://howtorotate.com/docs/tutorials/sourcegraph/",
|
||||
}
|
||||
|
||||
if verify {
|
||||
client := s.client
|
||||
|
||||
@@ -25,15 +25,24 @@ func TestSourcegraph_FromChunk(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("could not get test secrets from GCP: %s", err)
|
||||
}
|
||||
secret := testSecrets.MustGetField("SOURCEGRAPH")
|
||||
inactiveSecret := testSecrets.MustGetField("SOURCEGRAPH_INACTIVE")
|
||||
|
||||
secretV1 := testSecrets.MustGetField("SOURCEGRAPH_V1")
|
||||
secretV2 := testSecrets.MustGetField("SOURCEGRAPH_V2")
|
||||
secretV3 := testSecrets.MustGetField("SOURCEGRAPH_V3")
|
||||
|
||||
inactiveSecretV1 := testSecrets.MustGetField("SOURCEGRAPH_INACTIVE_V1")
|
||||
inactiveSecretV2 := testSecrets.MustGetField("SOURCEGRAPH_INACTIVE_V2")
|
||||
inactiveSecretV3 := testSecrets.MustGetField("SOURCEGRAPH_INACTIVE_V3")
|
||||
|
||||
secrets := []string{secretV1, secretV2, secretV3, inactiveSecretV1, inactiveSecretV2, inactiveSecretV3}
|
||||
|
||||
type args struct {
|
||||
ctx context.Context
|
||||
data []byte
|
||||
verify bool
|
||||
}
|
||||
tests := []struct {
|
||||
for _, secret := range secrets {
|
||||
tests = append(tests, []struct {
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
@@ -122,6 +131,7 @@ func TestSourcegraph_FromChunk(t *testing.T) {
|
||||
wantVerificationErr: true,
|
||||
},
|
||||
}
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
@@ -137,7 +147,7 @@ func TestSourcegraph_FromChunk(t *testing.T) {
|
||||
t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())
|
||||
}
|
||||
}
|
||||
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")
|
||||
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "VerificationError", "ExtraData")
|
||||
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
|
||||
t.Errorf("Sourcegraph.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
}
|
||||
|
||||
@@ -18,7 +18,7 @@ var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
//doesn't include test keys with "sk_test"
|
||||
secretKey = regexp.MustCompile(`[rs]k_live_[a-zA-Z0-9]{20,30}`)
|
||||
secretKey = regexp.MustCompile(`[rs]k_live_[a-zA-Z0-9]{20,247}`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
|
||||
@@ -1496,6 +1496,8 @@ func DefaultDetectors() []detectors.Detector {
|
||||
speechtextai.Scanner{},
|
||||
databox.Scanner{},
|
||||
postbacks.Scanner{},
|
||||
// Too noisy, needs attention
|
||||
// postgres.Scanner{},
|
||||
collect2.Scanner{},
|
||||
uclassify.Scanner{},
|
||||
holistic.Scanner{},
|
||||
|
||||
@@ -48,3 +48,20 @@ func TestDefaultDetectorTypesImplementing(t *testing.T) {
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDefaultVersionerDetectorsHaveNonZeroVersions(t *testing.T) {
|
||||
// Loop through all our default detectors and find the ones that
|
||||
// implement Versioner. Of those, check each version is not zero.
|
||||
// This is required due to an implementation detail of filtering detectors.
|
||||
// See: https://github.com/trufflesecurity/trufflehog/blob/v3.63.7/main.go#L624-L638
|
||||
for _, detector := range DefaultDetectors() {
|
||||
v, ok := detector.(detectors.Versioner)
|
||||
if !ok || v.Version() != 0 {
|
||||
continue
|
||||
}
|
||||
t.Errorf(
|
||||
"detector %q implements Versioner that returns a zero version",
|
||||
detectorspb.DetectorType_name[int32(detector.Type())],
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
lru "github.com/hashicorp/golang-lru"
|
||||
"google.golang.org/protobuf/proto"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/cleantemp"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/config"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
@@ -425,6 +426,10 @@ func (e *Engine) Finish(ctx context.Context) error {
|
||||
close(e.results) // Detector workers are done, close the results channel and call it a day.
|
||||
e.WgNotifier.Wait() // Wait for the notifier workers to finish notifying results.
|
||||
|
||||
if err := cleantemp.CleanTempArtifacts(ctx); err != nil {
|
||||
ctx.Logger().Error(err, "error cleaning temp artifacts")
|
||||
}
|
||||
|
||||
e.metrics.ScanDuration = time.Since(e.metrics.scanStartTime)
|
||||
|
||||
return err
|
||||
|
||||
@@ -15,7 +15,9 @@ import (
|
||||
// ScanFileSystem scans a given file system.
|
||||
func (e *Engine) ScanFileSystem(ctx context.Context, c sources.FilesystemConfig) error {
|
||||
connection := &sourcespb.Filesystem{
|
||||
Paths: c.Paths,
|
||||
Paths: c.Paths,
|
||||
IncludePathsFile: c.IncludePathsFile,
|
||||
ExcludePathsFile: c.ExcludePathsFile,
|
||||
}
|
||||
var conn anypb.Any
|
||||
err := anypb.MarshalFrom(&conn, connection, proto.MarshalOptions{})
|
||||
@@ -28,7 +30,6 @@ func (e *Engine) ScanFileSystem(ctx context.Context, c sources.FilesystemConfig)
|
||||
sourceID, jobID, _ := e.sourceManager.GetIDs(ctx, sourceName, filesystem.SourceType)
|
||||
|
||||
fileSystemSource := &filesystem.Source{}
|
||||
fileSystemSource.WithFilter(c.Filter)
|
||||
if err := fileSystemSource.Init(ctx, sourceName, jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"fmt"
|
||||
"github.com/go-logr/logr"
|
||||
"io"
|
||||
"os"
|
||||
"os/exec"
|
||||
@@ -92,6 +93,7 @@ func (state ParseState) String() string {
|
||||
"BinaryFileLine",
|
||||
"HunkLineNumberLine",
|
||||
"HunkContentLine",
|
||||
"ParseFailure",
|
||||
}[state]
|
||||
}
|
||||
|
||||
@@ -313,7 +315,7 @@ func (c *Parser) FromReader(ctx context.Context, stdOut io.Reader, commitChan ch
|
||||
case isMessageLine(isStaged, latestState, line):
|
||||
latestState = MessageLine
|
||||
|
||||
currentCommit.Message.Write(line[4:])
|
||||
currentCommit.Message.Write(line[4:]) // Messages are indented with 4 spaces.
|
||||
case isMessageEndLine(isStaged, latestState, line):
|
||||
latestState = MessageEndLine
|
||||
// NoOp
|
||||
@@ -425,13 +427,14 @@ func (c *Parser) FromReader(ctx context.Context, stdOut io.Reader, commitChan ch
|
||||
|
||||
// Here be dragons...
|
||||
// Build an informative error message.
|
||||
var err error
|
||||
err := fmt.Errorf(`invalid line "%s" after state "%s"`, line, latestState)
|
||||
var logger logr.Logger
|
||||
if currentCommit != nil && currentCommit.Hash != "" {
|
||||
err = fmt.Errorf(`failed to parse line "%s" after state "%s" (commit=%s)`, line, latestState, currentCommit.Hash)
|
||||
logger = ctx.Logger().WithValues("commit", currentCommit.Hash)
|
||||
} else {
|
||||
err = fmt.Errorf(`failed to parse line "%s" after state "%s"`, line, latestState)
|
||||
logger = ctx.Logger()
|
||||
}
|
||||
ctx.Logger().V(2).Error(err, "Recovering at the latest commit or diff...\n")
|
||||
logger.Error(err, "failed to parse Git input. Recovering at the latest commit or diff...")
|
||||
|
||||
latestState = ParseFailure
|
||||
}
|
||||
@@ -612,8 +615,9 @@ func pathFromBinaryLine(line []byte) string {
|
||||
}
|
||||
|
||||
// --- a/internal/addrs/move_endpoint_module.go
|
||||
// --- /dev/null
|
||||
func isFromFileLine(isStaged bool, latestState ParseState, line []byte) bool {
|
||||
if latestState != IndexLine {
|
||||
if !(latestState == IndexLine || latestState == ModeLine) {
|
||||
return false
|
||||
}
|
||||
if len(line) >= 6 && bytes.Equal(line[:4], []byte("--- ")) {
|
||||
|
||||
@@ -323,6 +323,13 @@ func TestLineChecks(t *testing.T) {
|
||||
IndexLine,
|
||||
[]byte("--- /dev/null"),
|
||||
},
|
||||
// New file (https://github.com/trufflesecurity/trufflehog/issues/2109)
|
||||
// diff --git a/libs/Unfit-1.0 b/libs/Unfit-1.0
|
||||
// new file mode 160000
|
||||
{
|
||||
ModeLine,
|
||||
[]byte("--- /dev/null"),
|
||||
},
|
||||
// Uncommon but valid prefixes. Will these ever show up?
|
||||
// https://stackoverflow.com/a/2530012
|
||||
// https://git-scm.com/docs/git-config#Documentation/git-config.txt-diffmnemonicPrefix
|
||||
@@ -1148,7 +1155,45 @@ func TestMaxCommitSize(t *testing.T) {
|
||||
|
||||
}
|
||||
|
||||
const commitLog = `commit 4727ffb7ad6dc5130bf4b4dd166e00705abdd018 (HEAD -> master)
|
||||
const commitLog = `commit fd6e99e7a80199b76a694603be57c5ade1de18e7
|
||||
Author: Jaliborc <[email protected]>
|
||||
Date: Mon Apr 25 16:28:06 2011 +0100
|
||||
|
||||
Added Unusable coloring
|
||||
|
||||
diff --git a/components/item.lua b/components/item.lua
|
||||
index fc74534..f8d7d50 100755
|
||||
--- a/components/item.lua
|
||||
+++ b/components/item.lua
|
||||
@@ -9,6 +9,7 @@ ItemSlot:Hide()
|
||||
Bagnon.ItemSlot = ItemSlot
|
||||
|
||||
local ItemSearch = LibStub('LibItemSearch-1.0')
|
||||
+local Unfit = LibStub('Unfit-1.0')
|
||||
|
||||
local function hasBlizzQuestHighlight()
|
||||
return GetContainerItemQuestInfo and true or false
|
||||
diff --git a/embeds.xml b/embeds.xml
|
||||
index d3f4e7c..0c2df69 100755
|
||||
--- a/embeds.xml
|
||||
+++ b/embeds.xml
|
||||
@@ -6,6 +6,7 @@
|
||||
<Include file="libs\AceConsole-3.0\AceConsole-3.0.xml"/>
|
||||
<Include file="libs\AceLocale-3.0\AceLocale-3.0.xml"/>
|
||||
|
||||
+ <Script file="libs\Unfit-1.0\Unfit-1.0.lua"/>
|
||||
<Script file="libs\LibDataBroker-1.1.lua"/>
|
||||
<Script file="libs\LibItemSearch-1.0\LibItemSearch-1.0.lua"/>
|
||||
</Ui>
|
||||
\ No newline at end of file
|
||||
diff --git a/libs/Unfit-1.0 b/libs/Unfit-1.0
|
||||
new file mode 160000
|
||||
--- /dev/null
|
||||
+++ b/libs/Unfit-1.0
|
||||
@@ -0,0 +1 @@
|
||||
+Subproject commit 0000000000000000000000000000000000000000
|
||||
|
||||
commit 4727ffb7ad6dc5130bf4b4dd166e00705abdd018 (HEAD -> master)
|
||||
Author: John Smith <[email protected]>
|
||||
Date: Tue Jul 11 22:26:11 2023 -0400
|
||||
|
||||
@@ -1539,6 +1584,33 @@ index 0000000..5af88a8
|
||||
// This throws a nasty panic if it's a top-level var.
|
||||
func expectedCommits() []Commit {
|
||||
return []Commit{
|
||||
// a
|
||||
{
|
||||
Hash: "fd6e99e7a80199b76a694603be57c5ade1de18e7",
|
||||
Author: "Jaliborc <[email protected]>",
|
||||
Date: newTime("Mon Apr 25 16:28:06 2011 +0100"),
|
||||
Message: newStringBuilderValue("Added Unusable coloring\n"),
|
||||
Diffs: []Diff{
|
||||
{
|
||||
PathB: "components/item.lua",
|
||||
LineStart: 9,
|
||||
Content: *bytes.NewBuffer([]byte("\n\nlocal Unfit = LibStub('Unfit-1.0')\n\n\n")),
|
||||
IsBinary: false,
|
||||
},
|
||||
{
|
||||
PathB: "embeds.xml",
|
||||
LineStart: 6,
|
||||
Content: *bytes.NewBuffer([]byte("\n\n <Script file=\"libs\\Unfit-1.0\\Unfit-1.0.lua\"/>\n\n\n\n")),
|
||||
IsBinary: false,
|
||||
},
|
||||
{
|
||||
PathB: "libs/Unfit-1.0",
|
||||
LineStart: 1,
|
||||
Content: *bytes.NewBuffer([]byte("Subproject commit 0000000000000000000000000000000000000000\n")),
|
||||
IsBinary: false,
|
||||
},
|
||||
},
|
||||
},
|
||||
// Empty commit and message. Lord help us.
|
||||
{
|
||||
Hash: "4727ffb7ad6dc5130bf4b4dd166e00705abdd018",
|
||||
|
||||
@@ -120,7 +120,7 @@ func NormalizeOrgRepoURL(provider provider, repoURL string) (string, error) {
|
||||
func GenerateLink(repo, commit, file string, line int64) string {
|
||||
// Some paths contain '%' which breaks |url.Parse| if not encoded.
|
||||
// https://developer.mozilla.org/en-US/docs/Glossary/Percent-encoding
|
||||
file = strings.Replace(file, "%", "%25", -1)
|
||||
file = strings.ReplaceAll(file, "%", "%25")
|
||||
|
||||
switch determineProvider(repo) {
|
||||
case providerBitbucket:
|
||||
@@ -140,7 +140,7 @@ func GenerateLink(repo, commit, file string, line int64) string {
|
||||
default:
|
||||
var baseLink string
|
||||
|
||||
//Gist links are formatted differently
|
||||
// Gist links are formatted differently
|
||||
if strings.HasPrefix(repo, "https://gist.github.com") {
|
||||
baseLink = repo[:len(repo)-4] + "/"
|
||||
if commit != "" {
|
||||
|
||||
+18
-6
@@ -46,6 +46,7 @@ type Archive struct {
|
||||
size int
|
||||
currentDepth int
|
||||
skipBinaries bool
|
||||
skipArchives bool
|
||||
}
|
||||
|
||||
// New creates a new Archive handler with the provided options.
|
||||
@@ -72,6 +73,10 @@ func SetArchiveMaxTimeout(timeout time.Duration) {
|
||||
|
||||
// FromFile extracts the files from an archive.
|
||||
func (a *Archive) FromFile(originalCtx logContext.Context, data io.Reader) chan []byte {
|
||||
if a.skipArchives {
|
||||
return nil
|
||||
}
|
||||
|
||||
archiveChan := make(chan []byte, defaultBufferSize)
|
||||
go func() {
|
||||
ctx, cancel := logContext.WithTimeout(originalCtx, maxTimeout)
|
||||
@@ -91,6 +96,10 @@ func (a *Archive) FromFile(originalCtx logContext.Context, data io.Reader) chan
|
||||
|
||||
// openArchive takes a reader and extracts the contents up to the maximum depth.
|
||||
func (a *Archive) openArchive(ctx logContext.Context, depth int, reader io.Reader, archiveChan chan []byte) error {
|
||||
if common.IsDone(ctx) {
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
if depth >= maxDepth {
|
||||
return fmt.Errorf(errMaxArchiveDepthReached)
|
||||
}
|
||||
@@ -111,6 +120,9 @@ func (a *Archive) openArchive(ctx logContext.Context, depth int, reader io.Reade
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
defer compReader.Close()
|
||||
|
||||
return a.openArchive(ctx, depth+1, compReader, archiveChan)
|
||||
case archiver.Extractor:
|
||||
return archive.Extract(logContext.WithValue(ctx, depthKey, depth+1), arReader, nil, a.extractorHandler(archiveChan))
|
||||
@@ -180,6 +192,11 @@ func (a *Archive) extractorHandler(archiveChan chan []byte) func(context.Context
|
||||
return func(ctx context.Context, f archiver.File) error {
|
||||
lCtx := logContext.AddLogger(ctx)
|
||||
lCtx.Logger().V(5).Info("Handling extracted file.", "filename", f.Name())
|
||||
|
||||
if common.IsDone(ctx) {
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
depth := 0
|
||||
if ctxDepth, ok := ctx.Value(depthKey).(int); ok {
|
||||
depth = ctxDepth
|
||||
@@ -201,12 +218,7 @@ func (a *Archive) extractorHandler(archiveChan chan []byte) func(context.Context
|
||||
return nil
|
||||
}
|
||||
|
||||
fileBytes, err := a.ReadToMax(lCtx, fReader)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return a.openArchive(lCtx, depth, bytes.NewReader(fileBytes), archiveChan)
|
||||
return a.openArchive(lCtx, depth, fReader, archiveChan)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+101
-35
@@ -76,36 +76,38 @@ func TestArchiveHandler(t *testing.T) {
|
||||
}
|
||||
|
||||
for name, testCase := range tests {
|
||||
resp, err := http.Get(testCase.archiveURL)
|
||||
if err != nil || resp.StatusCode != http.StatusOK {
|
||||
t.Error(err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
archive := Archive{}
|
||||
archive.New()
|
||||
|
||||
newReader, err := diskbufferreader.New(resp.Body)
|
||||
if err != nil {
|
||||
t.Errorf("error creating reusable reader: %s", err)
|
||||
}
|
||||
archiveChan := archive.FromFile(logContext.Background(), newReader)
|
||||
|
||||
count := 0
|
||||
re := regexp.MustCompile(testCase.matchString)
|
||||
matched := false
|
||||
for chunk := range archiveChan {
|
||||
count++
|
||||
if re.Match(chunk) {
|
||||
matched = true
|
||||
t.Run(name, func(t *testing.T) {
|
||||
resp, err := http.Get(testCase.archiveURL)
|
||||
if err != nil || resp.StatusCode != http.StatusOK {
|
||||
t.Error(err)
|
||||
}
|
||||
}
|
||||
if !matched && len(testCase.matchString) > 0 {
|
||||
t.Errorf("%s: Expected string not found in archive.", name)
|
||||
}
|
||||
if count != testCase.expectedChunks {
|
||||
t.Errorf("%s: Unexpected number of chunks. Got %d, expected: %d", name, count, testCase.expectedChunks)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
archive := Archive{}
|
||||
archive.New()
|
||||
|
||||
newReader, err := diskbufferreader.New(resp.Body)
|
||||
if err != nil {
|
||||
t.Errorf("error creating reusable reader: %s", err)
|
||||
}
|
||||
archiveChan := archive.FromFile(logContext.Background(), newReader)
|
||||
|
||||
count := 0
|
||||
re := regexp.MustCompile(testCase.matchString)
|
||||
matched := false
|
||||
for chunk := range archiveChan {
|
||||
count++
|
||||
if re.Match(chunk) {
|
||||
matched = true
|
||||
}
|
||||
}
|
||||
if !matched && len(testCase.matchString) > 0 {
|
||||
t.Errorf("%s: Expected string not found in archive.", name)
|
||||
}
|
||||
if count != testCase.expectedChunks {
|
||||
t.Errorf("%s: Unexpected number of chunks. Got %d, expected: %d", name, count, testCase.expectedChunks)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -115,7 +117,9 @@ func TestHandleFile(t *testing.T) {
|
||||
// Context cancels the operation.
|
||||
canceledCtx, cancel := logContext.WithCancel(logContext.Background())
|
||||
cancel()
|
||||
assert.False(t, HandleFile(canceledCtx, strings.NewReader("file"), &sources.Chunk{}, reporter))
|
||||
reader, err := diskbufferreader.New(strings.NewReader("file"))
|
||||
assert.NoError(t, err)
|
||||
assert.False(t, HandleFile(canceledCtx, reader, &sources.Chunk{}, reporter))
|
||||
|
||||
// Only one chunk is sent on the channel.
|
||||
// TODO: Embed a zip without making an HTTP request.
|
||||
@@ -124,7 +128,7 @@ func TestHandleFile(t *testing.T) {
|
||||
defer resp.Body.Close()
|
||||
archive := Archive{}
|
||||
archive.New()
|
||||
reader, err := diskbufferreader.New(resp.Body)
|
||||
reader, err = diskbufferreader.New(resp.Body)
|
||||
assert.NoError(t, err)
|
||||
|
||||
assert.Equal(t, 0, len(reporter.Ch))
|
||||
@@ -132,6 +136,34 @@ func TestHandleFile(t *testing.T) {
|
||||
assert.Equal(t, 1, len(reporter.Ch))
|
||||
}
|
||||
|
||||
func BenchmarkHandleFile(b *testing.B) {
|
||||
file, err := os.Open("testdata/test.tgz")
|
||||
assert.Nil(b, err)
|
||||
defer file.Close()
|
||||
|
||||
archive := Archive{}
|
||||
archive.New()
|
||||
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
sourceChan := make(chan *sources.Chunk, 1)
|
||||
reader, err := diskbufferreader.New(file)
|
||||
assert.NoError(b, err)
|
||||
|
||||
b.StartTimer()
|
||||
|
||||
go func() {
|
||||
defer close(sourceChan)
|
||||
HandleFile(logContext.Background(), reader, &sources.Chunk{}, sources.ChanReporter{Ch: sourceChan})
|
||||
}()
|
||||
|
||||
for range sourceChan {
|
||||
}
|
||||
|
||||
b.StopTimer()
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleFileSkipBinaries(t *testing.T) {
|
||||
filename := createBinaryArchive(t)
|
||||
defer os.Remove(filename)
|
||||
@@ -139,13 +171,16 @@ func TestHandleFileSkipBinaries(t *testing.T) {
|
||||
file, err := os.Open(filename)
|
||||
assert.NoError(t, err)
|
||||
|
||||
reader, err := diskbufferreader.New(file)
|
||||
assert.NoError(t, err)
|
||||
|
||||
ctx, cancel := logContext.WithTimeout(logContext.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
sourceChan := make(chan *sources.Chunk, 1)
|
||||
|
||||
go func() {
|
||||
defer close(sourceChan)
|
||||
HandleFile(ctx, file, &sources.Chunk{}, sources.ChanReporter{Ch: sourceChan}, WithSkipBinaries(true))
|
||||
HandleFile(ctx, reader, &sources.Chunk{}, sources.ChanReporter{Ch: sourceChan}, WithSkipBinaries(true))
|
||||
}()
|
||||
|
||||
count := 0
|
||||
@@ -278,17 +313,45 @@ func TestExtractDebContent(t *testing.T) {
|
||||
assert.Equal(t, expectedLength, len(string(content)))
|
||||
}
|
||||
|
||||
func TestExtractTarContent(t *testing.T) {
|
||||
func TestSkipArchive(t *testing.T) {
|
||||
file, err := os.Open("testdata/test.tgz")
|
||||
assert.Nil(t, err)
|
||||
defer file.Close()
|
||||
|
||||
reader, err := diskbufferreader.New(file)
|
||||
assert.NoError(t, err)
|
||||
|
||||
ctx := logContext.Background()
|
||||
|
||||
chunkCh := make(chan *sources.Chunk)
|
||||
go func() {
|
||||
defer close(chunkCh)
|
||||
ok := HandleFile(ctx, file, &sources.Chunk{}, sources.ChanReporter{Ch: chunkCh})
|
||||
ok := HandleFile(ctx, reader, &sources.Chunk{}, sources.ChanReporter{Ch: chunkCh}, WithSkipArchives(true))
|
||||
assert.False(t, ok)
|
||||
}()
|
||||
|
||||
wantCount := 0
|
||||
count := 0
|
||||
for range chunkCh {
|
||||
count++
|
||||
}
|
||||
assert.Equal(t, wantCount, count)
|
||||
}
|
||||
|
||||
func TestExtractTarContent(t *testing.T) {
|
||||
file, err := os.Open("testdata/test.tgz")
|
||||
assert.Nil(t, err)
|
||||
defer file.Close()
|
||||
|
||||
reader, err := diskbufferreader.New(file)
|
||||
assert.NoError(t, err)
|
||||
|
||||
ctx := logContext.Background()
|
||||
|
||||
chunkCh := make(chan *sources.Chunk)
|
||||
go func() {
|
||||
defer close(chunkCh)
|
||||
ok := HandleFile(ctx, reader, &sources.Chunk{}, sources.ChanReporter{Ch: chunkCh})
|
||||
assert.True(t, ok)
|
||||
}()
|
||||
|
||||
@@ -335,13 +398,16 @@ func TestNestedDirArchive(t *testing.T) {
|
||||
assert.Nil(t, err)
|
||||
defer file.Close()
|
||||
|
||||
reader, err := diskbufferreader.New(file)
|
||||
assert.NoError(t, err)
|
||||
|
||||
ctx, cancel := logContext.WithTimeout(logContext.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
sourceChan := make(chan *sources.Chunk, 1)
|
||||
|
||||
go func() {
|
||||
defer close(sourceChan)
|
||||
HandleFile(ctx, file, &sources.Chunk{}, sources.ChanReporter{Ch: sourceChan})
|
||||
HandleFile(ctx, reader, &sources.Chunk{}, sources.ChanReporter{Ch: sourceChan})
|
||||
}()
|
||||
|
||||
count := 0
|
||||
|
||||
+15
-14
@@ -36,6 +36,15 @@ func WithSkipBinaries(skip bool) Option {
|
||||
}
|
||||
}
|
||||
|
||||
// WithSkipArchives returns a Option that configures whether to skip archive files.
|
||||
func WithSkipArchives(skip bool) Option {
|
||||
return func(h Handler) {
|
||||
if a, ok := h.(*Archive); ok {
|
||||
a.skipArchives = skip
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type Handler interface {
|
||||
FromFile(logContext.Context, io.Reader) chan []byte
|
||||
IsFiletype(logContext.Context, io.Reader) (io.Reader, bool)
|
||||
@@ -48,20 +57,11 @@ type Handler interface {
|
||||
// packages them in the provided chunk skeleton, and reports them to the chunk reporter.
|
||||
// The function returns true if processing was successful and false otherwise.
|
||||
// Context is used for cancellation, and the caller is responsible for canceling it if needed.
|
||||
func HandleFile(ctx logContext.Context, file io.Reader, chunkSkel *sources.Chunk, reporter sources.ChunkReporter, opts ...Option) bool {
|
||||
func HandleFile(ctx logContext.Context, reReader *diskbufferreader.DiskBufferReader, chunkSkel *sources.Chunk, reporter sources.ChunkReporter, opts ...Option) bool {
|
||||
for _, h := range DefaultHandlers() {
|
||||
h.New(opts...)
|
||||
|
||||
// The re-reader is used to reset the file reader after checking if the handler implements SpecializedHandler.
|
||||
// This is necessary because the archive pkg doesn't correctly determine the file type when using
|
||||
// an io.MultiReader, which is used by the SpecializedHandler.
|
||||
reReader, err := diskbufferreader.New(file)
|
||||
if err != nil {
|
||||
ctx.Logger().Error(err, "error creating reusable reader")
|
||||
return false
|
||||
}
|
||||
|
||||
if success := processHandler(ctx, h, reReader, chunkSkel, reporter); success {
|
||||
if handled := processHandler(ctx, h, reReader, chunkSkel, reporter); handled {
|
||||
return true
|
||||
}
|
||||
}
|
||||
@@ -70,9 +70,6 @@ func HandleFile(ctx logContext.Context, file io.Reader, chunkSkel *sources.Chunk
|
||||
}
|
||||
|
||||
func processHandler(ctx logContext.Context, h Handler, reReader *diskbufferreader.DiskBufferReader, chunkSkel *sources.Chunk, reporter sources.ChunkReporter) bool {
|
||||
defer reReader.Close()
|
||||
defer reReader.Stop()
|
||||
|
||||
if specialHandler, ok := h.(SpecializedHandler); ok {
|
||||
file, isSpecial, err := specialHandler.HandleSpecialized(ctx, reReader)
|
||||
if isSpecial {
|
||||
@@ -96,6 +93,10 @@ func processHandler(ctx logContext.Context, h Handler, reReader *diskbufferreade
|
||||
}
|
||||
|
||||
func handleChunks(ctx logContext.Context, handlerChan chan []byte, chunkSkel *sources.Chunk, reporter sources.ChunkReporter) bool {
|
||||
if handlerChan == nil {
|
||||
return false
|
||||
}
|
||||
|
||||
for {
|
||||
select {
|
||||
case data, open := <-handlerChan:
|
||||
|
||||
+1127
-1069
File diff suppressed because it is too large
Load Diff
+525
-441
File diff suppressed because it is too large
Load Diff
@@ -587,6 +587,8 @@ func (m *Bitbucket) validate(all bool) error {
|
||||
|
||||
// no validation rules for SkipBinaries
|
||||
|
||||
// no validation rules for SkipArchives
|
||||
|
||||
switch m.Credential.(type) {
|
||||
|
||||
case *Bitbucket_Token:
|
||||
@@ -1482,6 +1484,10 @@ func (m *Filesystem) validate(all bool) error {
|
||||
|
||||
var errors []error
|
||||
|
||||
// no validation rules for IncludePathsFile
|
||||
|
||||
// no validation rules for ExcludePathsFile
|
||||
|
||||
if len(errors) > 0 {
|
||||
return FilesystemMultiError(errors)
|
||||
}
|
||||
@@ -1806,6 +1812,8 @@ func (m *Git) validate(all bool) error {
|
||||
|
||||
// no validation rules for SkipBinaries
|
||||
|
||||
// no validation rules for SkipArchives
|
||||
|
||||
switch m.Credential.(type) {
|
||||
|
||||
case *Git_BasicAuth:
|
||||
@@ -2015,6 +2023,8 @@ func (m *GitLab) validate(all bool) error {
|
||||
|
||||
// no validation rules for SkipBinaries
|
||||
|
||||
// no validation rules for SkipArchives
|
||||
|
||||
switch m.Credential.(type) {
|
||||
|
||||
case *GitLab_Token:
|
||||
@@ -2210,6 +2220,8 @@ func (m *GitHub) validate(all bool) error {
|
||||
|
||||
// no validation rules for SkipBinaries
|
||||
|
||||
// no validation rules for SkipArchives
|
||||
|
||||
switch m.Credential.(type) {
|
||||
|
||||
case *GitHub_GithubApp:
|
||||
@@ -3594,6 +3606,8 @@ func (m *Gerrit) validate(all bool) error {
|
||||
|
||||
// no validation rules for SkipBinaries
|
||||
|
||||
// no validation rules for SkipArchives
|
||||
|
||||
switch m.Credential.(type) {
|
||||
|
||||
case *Gerrit_BasicAuth:
|
||||
@@ -4681,6 +4695,8 @@ func (m *AzureRepos) validate(all bool) error {
|
||||
|
||||
// no validation rules for SkipBinaries
|
||||
|
||||
// no validation rules for SkipArchives
|
||||
|
||||
switch m.Credential.(type) {
|
||||
|
||||
case *AzureRepos_Token:
|
||||
|
||||
+12
-5
@@ -144,15 +144,22 @@ func readInChunks(ctx context.Context, reader io.Reader, config *chunkReaderConf
|
||||
|
||||
// If there is an error other than EOF, or if we have read some bytes, send the chunk.
|
||||
// io.ReadFull will only return io.EOF when n == 0.
|
||||
if isErrAndNotEOF(err) {
|
||||
switch {
|
||||
case isErrAndNotEOF(err):
|
||||
ctx.Logger().Error(err, "error reading chunk")
|
||||
chunkRes.err = err
|
||||
chunkResultChan <- chunkRes
|
||||
} else if n > 0 {
|
||||
chunkResultChan <- chunkRes
|
||||
case n > 0:
|
||||
chunkRes.err = nil
|
||||
default:
|
||||
return
|
||||
}
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case chunkResultChan <- chunkRes:
|
||||
}
|
||||
|
||||
// Return on any type of error.
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
@@ -3,9 +3,12 @@ package sources
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
"math/rand"
|
||||
"runtime"
|
||||
"strings"
|
||||
"testing"
|
||||
"testing/iotest"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
diskbufferreader "github.com/trufflesecurity/disk-buffer-reader"
|
||||
@@ -217,3 +220,35 @@ func TestFlakyChunkReader(t *testing.T) {
|
||||
assert.NoError(t, chunk.Error())
|
||||
assert.Equal(t, a+b, string(chunk.Bytes()))
|
||||
}
|
||||
|
||||
func TestReadInChunksWithCancellation(t *testing.T) {
|
||||
largeData := strings.Repeat("large test data ", 1024*1024) // Large data string.
|
||||
|
||||
for i := 0; i < 10; i++ {
|
||||
initialGoroutines := runtime.NumGoroutine()
|
||||
|
||||
for j := 0; j < 5; j++ { // Call readInChunks multiple times
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
|
||||
reader := strings.NewReader(largeData)
|
||||
chunkReader := NewChunkReader()
|
||||
|
||||
chunkChan := chunkReader(ctx, reader)
|
||||
|
||||
if rand.Intn(2) == 0 { // Randomly decide to cancel the context
|
||||
cancel()
|
||||
} else {
|
||||
for range chunkChan {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Allow for goroutine finalization.
|
||||
time.Sleep(time.Millisecond * 100)
|
||||
|
||||
// Check for goroutine leaks.
|
||||
if runtime.NumGoroutine() > initialGoroutines {
|
||||
t.Error("Potential goroutine leak detected")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -71,11 +71,13 @@ func (s *Source) Init(aCtx context.Context, name string, jobId sources.JobID, so
|
||||
}
|
||||
s.paths = append(conn.Paths, conn.Directories...)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Source) WithFilter(filter *common.Filter) {
|
||||
filter, err := common.FilterFromFiles(conn.IncludePathsFile, conn.ExcludePathsFile)
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to create filter: %w", err)
|
||||
}
|
||||
s.filter = filter
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Chunks emits chunks of bytes over a channel.
|
||||
|
||||
@@ -97,7 +97,7 @@ func newPersistableCache(increment int, cache cache.Cache, p *sources.Progress)
|
||||
|
||||
// Set overrides the cache Set method of the cache to enable the persistence
|
||||
// of the cache contents the Progress of the source at given increments.
|
||||
func (c *persistableCache) Set(key, val string) {
|
||||
func (c *persistableCache) Set(key string, val any) {
|
||||
c.Cache.Set(key, val)
|
||||
if ok, contents := c.shouldPersist(); ok {
|
||||
c.Progress.EncodedResumeInfo = contents
|
||||
@@ -297,7 +297,14 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ .
|
||||
func (s *Source) setupCache(ctx context.Context) *persistableCache {
|
||||
var c cache.Cache
|
||||
if s.Progress.EncodedResumeInfo != "" {
|
||||
c = memory.NewWithData(ctx, strings.Split(s.Progress.EncodedResumeInfo, ","))
|
||||
keys := strings.Split(s.Progress.EncodedResumeInfo, ",")
|
||||
entries := make([]memory.CacheEntry, len(keys))
|
||||
for i, val := range keys {
|
||||
entries[i] = memory.CacheEntry{Key: val, Value: val}
|
||||
}
|
||||
|
||||
c = memory.NewWithData(entries)
|
||||
ctx.Logger().V(3).Info("Loaded cache", "num_entries", len(entries))
|
||||
} else {
|
||||
c = memory.New()
|
||||
}
|
||||
|
||||
+39
-40
@@ -5,11 +5,11 @@ import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/url"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"runtime"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
@@ -19,12 +19,13 @@ import (
|
||||
"github.com/go-git/go-git/v5/plumbing"
|
||||
"github.com/go-git/go-git/v5/plumbing/object"
|
||||
"github.com/google/go-github/v42/github"
|
||||
diskbufferreader "github.com/trufflesecurity/disk-buffer-reader"
|
||||
"golang.org/x/oauth2"
|
||||
"golang.org/x/sync/semaphore"
|
||||
"google.golang.org/protobuf/proto"
|
||||
"google.golang.org/protobuf/types/known/anypb"
|
||||
|
||||
diskbufferreader "github.com/trufflesecurity/disk-buffer-reader"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/cleantemp"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
@@ -59,6 +60,7 @@ type Git struct {
|
||||
metrics metrics
|
||||
concurrency *semaphore.Weighted
|
||||
skipBinaries bool
|
||||
skipArchives bool
|
||||
}
|
||||
|
||||
type metrics struct {
|
||||
@@ -67,6 +69,7 @@ type metrics struct {
|
||||
|
||||
func NewGit(sourceType sourcespb.SourceType, jobID sources.JobID, sourceID sources.SourceID, sourceName string, verify bool, concurrency int,
|
||||
sourceMetadataFunc func(file, email, commit, timestamp, repository string, line int64) *source_metadatapb.MetaData, skipBinaries bool,
|
||||
skipArchives bool,
|
||||
) *Git {
|
||||
return &Git{
|
||||
sourceType: sourceType,
|
||||
@@ -77,6 +80,7 @@ func NewGit(sourceType sourcespb.SourceType, jobID sources.JobID, sourceID sourc
|
||||
verify: verify,
|
||||
concurrency: semaphore.NewWeighted(int64(concurrency)),
|
||||
skipBinaries: skipBinaries,
|
||||
skipArchives: skipArchives,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -179,6 +183,7 @@ func (s *Source) Init(aCtx context.Context, name string, jobId sources.JobID, so
|
||||
}
|
||||
},
|
||||
conn.GetSkipBinaries(),
|
||||
conn.GetSkipArchives(),
|
||||
)
|
||||
return nil
|
||||
}
|
||||
@@ -447,11 +452,18 @@ func CloneRepoUsingUnauthenticated(ctx context.Context, url string, args ...stri
|
||||
}
|
||||
|
||||
// CloneRepoUsingSSH clones a repo using SSH.
|
||||
func CloneRepoUsingSSH(ctx context.Context, gitUrl string, args ...string) (string, *git.Repository, error) {
|
||||
func CloneRepoUsingSSH(ctx context.Context, gitURL string, args ...string) (string, *git.Repository, error) {
|
||||
if isCodeCommitURL(gitURL) {
|
||||
return CloneRepo(ctx, nil, gitURL, args...)
|
||||
}
|
||||
userInfo := url.User("git")
|
||||
return CloneRepo(ctx, userInfo, gitUrl, args...)
|
||||
return CloneRepo(ctx, userInfo, gitURL, args...)
|
||||
}
|
||||
|
||||
var codeCommitRE = regexp.MustCompile(`ssh://git-codecommit\.[\w-]+\.amazonaws\.com`)
|
||||
|
||||
func isCodeCommitURL(gitURL string) bool { return codeCommitRE.MatchString(gitURL) }
|
||||
|
||||
func (s *Git) CommitsScanned() uint64 {
|
||||
return atomic.LoadUint64(&s.metrics.commitsScanned)
|
||||
}
|
||||
@@ -459,7 +471,16 @@ func (s *Git) CommitsScanned() uint64 {
|
||||
const gitDirName = ".git"
|
||||
|
||||
func (s *Git) ScanCommits(ctx context.Context, repo *git.Repository, path string, scanOptions *ScanOptions, reporter sources.ChunkReporter) error {
|
||||
commitChan, err := gitparse.NewParser().RepoPath(ctx, path, scanOptions.HeadHash, scanOptions.BaseHash == "", scanOptions.ExcludeGlobs, scanOptions.Bare)
|
||||
// Get the remote URL for reporting (may be empty)
|
||||
remoteURL := getSafeRemoteURL(repo, "origin")
|
||||
var repoCtx context.Context
|
||||
if remoteURL != "" {
|
||||
repoCtx = context.WithValue(ctx, "repo", remoteURL)
|
||||
} else {
|
||||
repoCtx = context.WithValue(ctx, "repo", path)
|
||||
}
|
||||
|
||||
commitChan, err := gitparse.NewParser().RepoPath(repoCtx, path, scanOptions.HeadHash, scanOptions.BaseHash == "", scanOptions.ExcludeGlobs, scanOptions.Bare)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -467,14 +488,10 @@ func (s *Git) ScanCommits(ctx context.Context, repo *git.Repository, path string
|
||||
return nil
|
||||
}
|
||||
|
||||
// get the URL metadata for reporting (may be empty)
|
||||
urlMetadata := getSafeRemoteURL(repo, "origin")
|
||||
|
||||
var depth int64
|
||||
|
||||
gitDir := filepath.Join(path, gitDirName)
|
||||
|
||||
logger := ctx.Logger().WithValues("repo", urlMetadata)
|
||||
logger := repoCtx.Logger()
|
||||
logger.V(1).Info("scanning repo", "base", scanOptions.BaseHash, "head", scanOptions.HeadHash)
|
||||
for commit := range commitChan {
|
||||
if len(scanOptions.BaseHash) > 0 {
|
||||
@@ -507,7 +524,7 @@ func (s *Git) ScanCommits(ctx context.Context, repo *git.Repository, path string
|
||||
// Handle binary files by reading the entire file rather than using the diff.
|
||||
if diff.IsBinary {
|
||||
commitHash := plumbing.NewHash(hash)
|
||||
metadata := s.sourceMetadataFunc(fileName, email, hash, when, urlMetadata, 0)
|
||||
metadata := s.sourceMetadataFunc(fileName, email, hash, when, remoteURL, 0)
|
||||
chunkSkel := &sources.Chunk{
|
||||
SourceName: s.sourceName,
|
||||
SourceID: s.sourceID,
|
||||
@@ -523,10 +540,10 @@ func (s *Git) ScanCommits(ctx context.Context, repo *git.Repository, path string
|
||||
}
|
||||
|
||||
if diff.Content.Len() > sources.ChunkSize+sources.PeekSize {
|
||||
s.gitChunk(ctx, diff, fileName, email, hash, when, urlMetadata, reporter)
|
||||
s.gitChunk(ctx, diff, fileName, email, hash, when, remoteURL, reporter)
|
||||
continue
|
||||
}
|
||||
metadata := s.sourceMetadataFunc(fileName, email, hash, when, urlMetadata, int64(diff.LineStart))
|
||||
metadata := s.sourceMetadataFunc(fileName, email, hash, when, remoteURL, int64(diff.LineStart))
|
||||
chunk := sources.Chunk{
|
||||
SourceName: s.sourceName,
|
||||
SourceID: s.sourceID,
|
||||
@@ -1006,16 +1023,17 @@ func (s *Git) handleBinary(ctx context.Context, gitDir string, reporter sources.
|
||||
return nil
|
||||
}
|
||||
|
||||
var handlerOpts []handlers.Option
|
||||
if s.skipBinaries {
|
||||
handlerOpts = append(handlerOpts, handlers.WithSkipBinaries(true))
|
||||
if common.IsBinary(path) {
|
||||
fileCtx.Logger().V(5).Info("skipping binary file")
|
||||
return nil
|
||||
}
|
||||
fileCtx.Logger().V(5).Info("skipping binary file", "path", path)
|
||||
return nil
|
||||
}
|
||||
|
||||
var handlerOpts []handlers.Option
|
||||
|
||||
if s.skipArchives {
|
||||
handlerOpts = append(handlerOpts, handlers.WithSkipArchives(true))
|
||||
}
|
||||
|
||||
const maxSize = 1 * 1024 * 1024 * 1024 // 1GB
|
||||
cmd := exec.Command("git", "-C", gitDir, "cat-file", "blob", commitHash.String()+":"+path)
|
||||
|
||||
var stderr bytes.Buffer
|
||||
@@ -1043,31 +1061,12 @@ func (s *Git) handleBinary(ctx context.Context, gitDir string, reporter sources.
|
||||
}
|
||||
}()
|
||||
|
||||
var fileContent bytes.Buffer
|
||||
// Create a limited reader to ensure we don't read more than the max size.
|
||||
lr := io.LimitReader(fileReader, int64(maxSize))
|
||||
|
||||
// Using io.CopyBuffer for performance advantages. Though buf is mandatory
|
||||
// for the method, due to the internal implementation of io.CopyBuffer, when
|
||||
// *bytes.Buffer implements io.WriterTo or io.ReaderFrom, the provided buf
|
||||
// is simply ignored. Thus, we can pass nil for the buf parameter.
|
||||
_, err = io.CopyBuffer(&fileContent, lr, nil)
|
||||
if err != nil && !errors.Is(err, io.EOF) {
|
||||
return err
|
||||
}
|
||||
|
||||
if fileContent.Len() == maxSize {
|
||||
fileCtx.Logger().V(2).Info("Max archive size reached.")
|
||||
}
|
||||
|
||||
bufferName := cleantemp.MkFilename()
|
||||
|
||||
reader, err := diskbufferreader.New(&fileContent, diskbufferreader.WithBufferName(bufferName))
|
||||
|
||||
reader, err := diskbufferreader.New(fileReader, diskbufferreader.WithBufferName(bufferName))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
defer reader.Close()
|
||||
|
||||
if handlers.HandleFile(fileCtx, reader, chunkSkel, reporter, handlerOpts...) {
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package github
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/url"
|
||||
@@ -15,7 +16,6 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/bradleyfalzon/ghinstallation/v2"
|
||||
"github.com/go-errors/errors"
|
||||
gogit "github.com/go-git/go-git/v5"
|
||||
"github.com/go-logr/logr"
|
||||
"github.com/gobwas/glob"
|
||||
@@ -217,7 +217,7 @@ func (s *Source) Init(aCtx context.Context, name string, jobID sources.JobID, so
|
||||
var conn sourcespb.GitHub
|
||||
err := anypb.UnmarshalTo(connection, &conn, proto.UnmarshalOptions{})
|
||||
if err != nil {
|
||||
return errors.WrapPrefix(err, "error unmarshalling connection", 0)
|
||||
return fmt.Errorf("error unmarshalling connection: %w", err)
|
||||
}
|
||||
s.conn = &conn
|
||||
|
||||
@@ -277,6 +277,7 @@ func (s *Source) Init(aCtx context.Context, name string, jobID sources.JobID, so
|
||||
}
|
||||
},
|
||||
conn.GetSkipBinaries(),
|
||||
conn.GetSkipArchives(),
|
||||
)
|
||||
|
||||
return nil
|
||||
@@ -322,7 +323,7 @@ func (s *Source) Validate(ctx context.Context) []error {
|
||||
errs = append(errs, fmt.Errorf("error creating GitHub client: %+v", err))
|
||||
}
|
||||
default:
|
||||
errs = append(errs, errors.Errorf("Invalid configuration given for source. Name: %s, Type: %s", s.name, s.Type()))
|
||||
errs = append(errs, fmt.Errorf("Invalid configuration given for source. Name: %s, Type: %s", s.name, s.Type()))
|
||||
}
|
||||
|
||||
// Run a simple query to check if the client is actually valid
|
||||
@@ -356,6 +357,10 @@ func (s *Source) visibilityOf(ctx context.Context, repoURL string) (visibility s
|
||||
s.mu.Unlock()
|
||||
}()
|
||||
logger := s.log.WithValues("repo", repoURL)
|
||||
if _, unauthenticated := s.conn.GetCredential().(*sourcespb.GitHub_Unauthenticated); unauthenticated {
|
||||
logger.V(3).Info("assuming unauthenticated scan has public visibility")
|
||||
return source_metadatapb.Visibility_public
|
||||
}
|
||||
logger.V(2).Info("Checking public status")
|
||||
u, err := url.Parse(repoURL)
|
||||
if err != nil {
|
||||
@@ -378,10 +383,6 @@ func (s *Source) visibilityOf(ctx context.Context, repoURL string) (visibility s
|
||||
}
|
||||
}
|
||||
if err != nil || gist == nil {
|
||||
if _, unauthenticated := s.conn.GetCredential().(*sourcespb.GitHub_Unauthenticated); unauthenticated {
|
||||
logger.Info("Unauthenticated scans cannot determine if a repository is private.")
|
||||
visibility = source_metadatapb.Visibility_private
|
||||
}
|
||||
logger.Error(err, "Could not get Github repository")
|
||||
return
|
||||
}
|
||||
@@ -401,10 +402,6 @@ func (s *Source) visibilityOf(ctx context.Context, repoURL string) (visibility s
|
||||
}
|
||||
if err != nil || repo == nil {
|
||||
logger.Error(err, "Could not get Github repository")
|
||||
if _, unauthenticated := s.conn.GetCredential().(*sourcespb.GitHub_Unauthenticated); unauthenticated {
|
||||
logger.Info("Unauthenticated scans cannot determine if a repository is private.")
|
||||
visibility = source_metadatapb.Visibility_private
|
||||
}
|
||||
return
|
||||
}
|
||||
if *repo.Private {
|
||||
@@ -418,11 +415,13 @@ func (s *Source) visibilityOf(ctx context.Context, repoURL string) (visibility s
|
||||
return
|
||||
}
|
||||
|
||||
const cloudEndpoint = "https://api.github.com"
|
||||
|
||||
// Chunks emits chunks of bytes over a channel.
|
||||
func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, targets ...sources.ChunkingTarget) error {
|
||||
apiEndpoint := s.conn.Endpoint
|
||||
if len(apiEndpoint) == 0 || endsWithGithub.MatchString(apiEndpoint) {
|
||||
apiEndpoint = "https://api.github.com"
|
||||
apiEndpoint = cloudEndpoint
|
||||
}
|
||||
|
||||
// If targets are provided, we're only scanning the data in those targets.
|
||||
@@ -468,10 +467,18 @@ func (s *Source) enumerate(ctx context.Context, apiEndpoint string) (*github.Cli
|
||||
}
|
||||
default:
|
||||
// TODO: move this error to Init
|
||||
return nil, errors.Errorf("Invalid configuration given for source. Name: %s, Type: %s", s.name, s.Type())
|
||||
return nil, fmt.Errorf("Invalid configuration given for source. Name: %s, Type: %s", s.name, s.Type())
|
||||
}
|
||||
|
||||
s.repos = s.filteredRepoCache.Values()
|
||||
s.repos = make([]string, 0, s.filteredRepoCache.Count())
|
||||
for _, repo := range s.filteredRepoCache.Values() {
|
||||
r, ok := repo.(string)
|
||||
if !ok {
|
||||
ctx.Logger().Error(fmt.Errorf("type assertion failed"), "unexpected value in cache", "repo", repo)
|
||||
continue
|
||||
}
|
||||
s.repos = append(s.repos, r)
|
||||
}
|
||||
githubReposEnumerated.WithLabelValues(s.name).Set(float64(len(s.repos)))
|
||||
s.log.Info("Completed enumeration", "num_repos", len(s.repos), "num_orgs", s.orgsCache.Count(), "num_members", len(s.memberCache))
|
||||
|
||||
@@ -541,7 +548,6 @@ func (s *Source) enumerateWithToken(ctx context.Context, apiEndpoint, token stri
|
||||
|
||||
// If we're using public GitHub, make a regular client.
|
||||
// Otherwise, make an enterprise client.
|
||||
var isGHE bool = apiEndpoint != "https://api.github.com"
|
||||
ghClient, err := createGitHubClient(s.httpClient, apiEndpoint)
|
||||
if err != nil {
|
||||
s.log.Error(err, "error creating GitHub client")
|
||||
@@ -568,7 +574,7 @@ func (s *Source) enumerateWithToken(ctx context.Context, apiEndpoint, token stri
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return errors.New(err)
|
||||
return fmt.Errorf("error getting user: %w", err)
|
||||
}
|
||||
break
|
||||
}
|
||||
@@ -597,6 +603,7 @@ func (s *Source) enumerateWithToken(ctx context.Context, apiEndpoint, token stri
|
||||
s.log.Error(err, "error fetching repos by user")
|
||||
}
|
||||
|
||||
isGHE := !strings.EqualFold(apiEndpoint, cloudEndpoint)
|
||||
if isGHE {
|
||||
s.addAllVisibleOrgs(ctx)
|
||||
} else {
|
||||
@@ -644,12 +651,12 @@ func (s *Source) enumerateWithToken(ctx context.Context, apiEndpoint, token stri
|
||||
func (s *Source) enumerateWithApp(ctx context.Context, apiEndpoint string, app *credentialspb.GitHubApp) (installationClient *github.Client, err error) {
|
||||
installationID, err := strconv.ParseInt(app.InstallationId, 10, 64)
|
||||
if err != nil {
|
||||
return nil, errors.New(err)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
appID, err := strconv.ParseInt(app.AppId, 10, 64)
|
||||
if err != nil {
|
||||
return nil, errors.New(err)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// This client is required to create installation tokens for cloning.
|
||||
@@ -662,16 +669,16 @@ func (s *Source) enumerateWithApp(ctx context.Context, apiEndpoint string, app *
|
||||
appID,
|
||||
[]byte(app.PrivateKey))
|
||||
if err != nil {
|
||||
return nil, errors.New(err)
|
||||
return nil, err
|
||||
}
|
||||
appItr.BaseURL = apiEndpoint
|
||||
|
||||
// Does this need to be separate from |s.httpClient|?
|
||||
instHttpClient := common.RetryableHttpClientTimeout(60)
|
||||
instHttpClient.Transport = appItr
|
||||
installationClient, err = github.NewEnterpriseClient(apiEndpoint, apiEndpoint, instHttpClient)
|
||||
instHTTPClient := common.RetryableHttpClientTimeout(60)
|
||||
instHTTPClient.Transport = appItr
|
||||
installationClient, err = github.NewEnterpriseClient(apiEndpoint, apiEndpoint, instHTTPClient)
|
||||
if err != nil {
|
||||
return nil, errors.New(err)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// This client is used for most APIs.
|
||||
@@ -681,14 +688,14 @@ func (s *Source) enumerateWithApp(ctx context.Context, apiEndpoint string, app *
|
||||
installationID,
|
||||
[]byte(app.PrivateKey))
|
||||
if err != nil {
|
||||
return nil, errors.New(err)
|
||||
return nil, err
|
||||
}
|
||||
itr.BaseURL = apiEndpoint
|
||||
|
||||
s.httpClient.Transport = itr
|
||||
s.apiClient, err = github.NewEnterpriseClient(apiEndpoint, apiEndpoint, s.httpClient)
|
||||
if err != nil {
|
||||
return nil, errors.New(err)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// If no repos were provided, enumerate them.
|
||||
@@ -719,19 +726,14 @@ func (s *Source) enumerateWithApp(ctx context.Context, apiEndpoint string, app *
|
||||
return installationClient, nil
|
||||
}
|
||||
|
||||
func createGitHubClient(httpClient *http.Client, apiEndpoint string) (ghClient *github.Client, err error) {
|
||||
func createGitHubClient(httpClient *http.Client, apiEndpoint string) (*github.Client, error) {
|
||||
// If we're using public GitHub, make a regular client.
|
||||
// Otherwise, make an enterprise client.
|
||||
if apiEndpoint == "https://api.github.com" {
|
||||
ghClient = github.NewClient(httpClient)
|
||||
} else {
|
||||
ghClient, err = github.NewEnterpriseClient(apiEndpoint, apiEndpoint, httpClient)
|
||||
if err != nil {
|
||||
return nil, errors.New(err)
|
||||
}
|
||||
if strings.EqualFold(apiEndpoint, cloudEndpoint) {
|
||||
return github.NewClient(httpClient), nil
|
||||
}
|
||||
|
||||
return ghClient, err
|
||||
return github.NewEnterpriseClient(apiEndpoint, apiEndpoint, httpClient)
|
||||
}
|
||||
|
||||
func (s *Source) scan(ctx context.Context, installationClient *github.Client, chunksChan chan *sources.Chunk) error {
|
||||
@@ -960,7 +962,7 @@ func (s *Source) addAllVisibleOrgs(ctx context.Context) {
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
s.log.Error(err, "Could not list all organizations")
|
||||
s.log.Error(err, "could not list all organizations")
|
||||
return
|
||||
}
|
||||
if len(orgs) == 0 {
|
||||
@@ -972,11 +974,12 @@ func (s *Source) addAllVisibleOrgs(ctx context.Context) {
|
||||
|
||||
for _, org := range orgs {
|
||||
var name string
|
||||
if org.Name != nil {
|
||||
switch {
|
||||
case org.Name != nil:
|
||||
name = *org.Name
|
||||
} else if org.Login != nil {
|
||||
case org.Login != nil:
|
||||
name = *org.Login
|
||||
} else {
|
||||
default:
|
||||
continue
|
||||
}
|
||||
s.orgsCache.Set(name, name)
|
||||
@@ -1037,7 +1040,7 @@ func (s *Source) addMembersByOrg(ctx context.Context, org string) error {
|
||||
continue
|
||||
}
|
||||
if err != nil || len(members) == 0 {
|
||||
return errors.New("Could not list organization members: account may not have access to list organization members")
|
||||
return fmt.Errorf("could not list organization members: account may not have access to list organization members %w", err)
|
||||
}
|
||||
if res == nil {
|
||||
break
|
||||
@@ -1516,7 +1519,7 @@ func (s *Source) scanTargets(ctx context.Context, targets []sources.ChunkingTarg
|
||||
func (s *Source) scanTarget(ctx context.Context, target sources.ChunkingTarget, chunksChan chan *sources.Chunk) error {
|
||||
metaType, ok := target.QueryCriteria.GetData().(*source_metadatapb.MetaData_Github)
|
||||
if !ok {
|
||||
return fmt.Errorf("unable to cast metadata type for targetted scan")
|
||||
return fmt.Errorf("unable to cast metadata type for targeted scan")
|
||||
}
|
||||
meta := metaType.Github
|
||||
|
||||
@@ -1547,6 +1550,7 @@ func (s *Source) scanTarget(ctx context.Context, target sources.ChunkingTarget,
|
||||
SourceName: s.name,
|
||||
SourceID: s.SourceID(),
|
||||
JobID: s.JobID(),
|
||||
SecretID: target.SecretID,
|
||||
Data: []byte(res),
|
||||
SourceMetadata: &source_metadatapb.MetaData{
|
||||
Data: &source_metadatapb.MetaData_Github{Github: meta},
|
||||
|
||||
@@ -138,6 +138,7 @@ func (s *Source) Init(_ context.Context, name string, jobId sources.JobID, sourc
|
||||
}
|
||||
},
|
||||
conn.GetSkipBinaries(),
|
||||
conn.GetSkipArchives(),
|
||||
)
|
||||
|
||||
return nil
|
||||
|
||||
@@ -57,7 +57,7 @@ func TestSource_Scan(t *testing.T) {
|
||||
SourceType: sourcespb.SourceType_SOURCE_TYPE_GITLAB,
|
||||
SourceName: "test source",
|
||||
},
|
||||
wantReposScanned: 2,
|
||||
wantReposScanned: 4,
|
||||
},
|
||||
{
|
||||
name: "token auth, enumerate repo, with glob ignore",
|
||||
@@ -74,7 +74,7 @@ func TestSource_Scan(t *testing.T) {
|
||||
SourceType: sourcespb.SourceType_SOURCE_TYPE_GITLAB,
|
||||
SourceName: "test source",
|
||||
},
|
||||
wantReposScanned: 2,
|
||||
wantReposScanned: 4,
|
||||
},
|
||||
{
|
||||
name: "token auth, scoped repo",
|
||||
|
||||
@@ -229,17 +229,30 @@ func (s *SourceManager) run(ctx context.Context, source Source, report *JobProgr
|
||||
}()
|
||||
|
||||
report.TrackProgress(source.GetProgress())
|
||||
ctx = context.WithValues(ctx,
|
||||
"job_id", report.JobID,
|
||||
"source_id", report.SourceID,
|
||||
"source_name", report.SourceName,
|
||||
"source_type", source.Type().String(),
|
||||
)
|
||||
if ctx.Value("job_id") == "" {
|
||||
ctx = context.WithValue(ctx, "job_id", report.JobID)
|
||||
}
|
||||
if ctx.Value("source_id") == "" {
|
||||
ctx = context.WithValue(ctx, "source_id", report.SourceID)
|
||||
}
|
||||
if ctx.Value("source_name") == "" {
|
||||
ctx = context.WithValue(ctx, "source_name", report.SourceName)
|
||||
}
|
||||
if ctx.Value("source_type") == "" {
|
||||
ctx = context.WithValue(ctx, "source_type", source.Type().String())
|
||||
}
|
||||
|
||||
// Check for the preferred method of tracking source units.
|
||||
canUseSourceUnits := len(targets) == 0 && s.useSourceUnitsFunc != nil
|
||||
if enumChunker, ok := source.(SourceUnitEnumChunker); ok && canUseSourceUnits && s.useSourceUnitsFunc() {
|
||||
ctx.Logger().Info("running source",
|
||||
"with_units", true)
|
||||
return s.runWithUnits(ctx, enumChunker, report)
|
||||
}
|
||||
ctx.Logger().Info("running source",
|
||||
"with_units", false,
|
||||
"target_count", len(targets),
|
||||
"source_manager_units_configurable", s.useSourceUnitsFunc != nil)
|
||||
return s.runWithoutUnits(ctx, source, report, targets...)
|
||||
}
|
||||
|
||||
|
||||
@@ -24,6 +24,9 @@ type Chunk struct {
|
||||
SourceID SourceID
|
||||
// JobID is the ID of the job that the Chunk originated from.
|
||||
JobID JobID
|
||||
// SecretID is the ID of the secret, if it exists.
|
||||
// Only secrets that are being reverified will have a SecretID.
|
||||
SecretID int64
|
||||
// SourceType is the type of Source that produced the chunk.
|
||||
SourceType sourcespb.SourceType
|
||||
// SourceMetadata holds the context of where the Chunk was found.
|
||||
@@ -43,6 +46,8 @@ type Chunk struct {
|
||||
type ChunkingTarget struct {
|
||||
// QueryCriteria represents specific parameters or conditions to target the chunking process.
|
||||
QueryCriteria *source_metadatapb.MetaData
|
||||
// SecretID is the ID of the secret.
|
||||
SecretID int64
|
||||
}
|
||||
|
||||
// Source defines the interface required to implement a source chunker.
|
||||
@@ -225,8 +230,10 @@ type GitlabConfig struct {
|
||||
type FilesystemConfig struct {
|
||||
// Paths is the list of files and directories to scan.
|
||||
Paths []string
|
||||
// Filter is the filter to use to scan the source.
|
||||
Filter *common.Filter
|
||||
// IncludePathsFile is the path to a file containing a list of regexps to include in the scan.
|
||||
IncludePathsFile string
|
||||
// ExcludePathsFile is the path to a file containing a list of regexps to exclude from the scan.
|
||||
ExcludePathsFile string
|
||||
}
|
||||
|
||||
// S3Config defines the optional configuration for an S3 source.
|
||||
|
||||
@@ -303,6 +303,10 @@ func (s *Source) acceptUDPConnections(ctx context.Context, netListener net.Packe
|
||||
if common.IsDone(ctx) {
|
||||
return nil
|
||||
}
|
||||
err := netListener.SetDeadline(time.Now().Add(time.Second))
|
||||
if err != nil {
|
||||
ctx.Logger().V(2).Info("could not update connection deadline", "error", err)
|
||||
}
|
||||
input := make([]byte, 65535)
|
||||
_, remote, err := netListener.ReadFrom(input)
|
||||
if err != nil {
|
||||
|
||||
@@ -976,6 +976,18 @@ enum DetectorType {
|
||||
Overloop = 965;
|
||||
Ngrok = 966;
|
||||
Replicate = 967;
|
||||
Postgres = 968;
|
||||
AzureActiveDirectoryApplicationSecret = 969;
|
||||
AzureCacheForRedisAccessKey = 970;
|
||||
AzureCosmosDBKeyIdentifiable = 971;
|
||||
AzureDevopsPersonalAccessToken = 972;
|
||||
AzureFunctionKey = 973;
|
||||
AzureMLWebServiceClassicIdentifiableKey = 974;
|
||||
AzureSasToken = 975;
|
||||
AzureSearchAdminKey = 976;
|
||||
AzureSearchQueryKey = 977;
|
||||
AzureManagementCertificate = 978;
|
||||
AzureSQL = 979;
|
||||
}
|
||||
|
||||
message Result {
|
||||
|
||||
+21
-13
@@ -96,6 +96,7 @@ message Bitbucket {
|
||||
repeated string repositories = 5;
|
||||
repeated string ignore_repos = 6;
|
||||
bool skip_binaries = 7;
|
||||
bool skip_archives = 8;
|
||||
}
|
||||
|
||||
message CircleCI {
|
||||
@@ -156,6 +157,8 @@ message Filesystem {
|
||||
// paths when we no longer depend on the name in enterprise configs.
|
||||
repeated string directories = 1;
|
||||
repeated string paths = 2;
|
||||
string include_paths_file = 3; // path to file containing newline separated list of paths
|
||||
string exclude_paths_file = 4; // path to file containing newline separated list of paths
|
||||
}
|
||||
|
||||
message GCS {
|
||||
@@ -196,6 +199,7 @@ message Git {
|
||||
// like head, base, bare, etc.
|
||||
string uri = 13; // repository URL. https://, file://, or ssh://
|
||||
bool skip_binaries = 14;
|
||||
bool skip_archives = 15;
|
||||
}
|
||||
|
||||
message GitLab {
|
||||
@@ -208,6 +212,7 @@ message GitLab {
|
||||
repeated string repositories = 5;
|
||||
repeated string ignore_repos = 6;
|
||||
bool skip_binaries = 7;
|
||||
bool skip_archives = 8;
|
||||
}
|
||||
|
||||
message GitHub {
|
||||
@@ -224,12 +229,13 @@ message GitHub {
|
||||
bool includeForks = 8;
|
||||
string head = 9;
|
||||
string base = 10;
|
||||
repeated string ignoreRepos = 11;
|
||||
repeated string includeRepos = 12;
|
||||
bool includePullRequestComments = 14;
|
||||
bool includeIssueComments = 15;
|
||||
bool includeGistComments = 16;
|
||||
repeated string ignore_repos = 11;
|
||||
repeated string include_repos = 12;
|
||||
bool include_pull_request_comments = 14;
|
||||
bool include_issue_comments = 15;
|
||||
bool include_gist_comments = 16;
|
||||
bool skip_binaries = 17;
|
||||
bool skip_archives = 18;
|
||||
}
|
||||
|
||||
message GoogleDrive {
|
||||
@@ -282,7 +288,7 @@ message Slack {
|
||||
credentials.SlackTokens tokens = 5;
|
||||
}
|
||||
repeated string channels = 3;
|
||||
repeated string ignoreList = 4;
|
||||
repeated string ignore_list = 4;
|
||||
}
|
||||
|
||||
message Test{}
|
||||
@@ -301,6 +307,7 @@ message Gerrit {
|
||||
}
|
||||
repeated string projects = 4;
|
||||
bool skip_binaries = 5;
|
||||
bool skip_archives = 6;
|
||||
}
|
||||
|
||||
message Jenkins {
|
||||
@@ -320,13 +327,13 @@ message Teams {
|
||||
credentials.Oauth2 oauth = 7;
|
||||
}
|
||||
repeated string channels = 4;
|
||||
repeated string ignoreList = 5;
|
||||
repeated string ignore_list = 5;
|
||||
repeated string team_ids = 6;
|
||||
}
|
||||
|
||||
message Syslog {
|
||||
string protocol = 1;
|
||||
string listenAddress = 2;
|
||||
string listen_address = 2;
|
||||
string tlsCert = 3;
|
||||
string tlsKey = 4;
|
||||
string format = 5;
|
||||
@@ -363,10 +370,11 @@ message AzureRepos {
|
||||
repeated string repositories = 4;
|
||||
repeated string organizations = 5;
|
||||
repeated string projects = 6;
|
||||
bool includeForks = 7;
|
||||
repeated string ignoreRepos = 8;
|
||||
repeated string includeRepos = 9;
|
||||
repeated string includeProjects = 10;
|
||||
repeated string ignoreProjects = 11;
|
||||
bool include_forks = 7;
|
||||
repeated string ignore_repos = 8;
|
||||
repeated string include_repos = 9;
|
||||
repeated string include_projects = 10;
|
||||
repeated string ignore_projects = 11;
|
||||
bool skip_binaries = 12;
|
||||
bool skip_archives = 13;
|
||||
}
|
||||
|
||||
+35
-2
@@ -9,6 +9,7 @@ $this: download go binaries for trufflesecurity/trufflehog
|
||||
Usage: $this [-b] bindir [-d] [tag]
|
||||
-b sets bindir or installation directory, Defaults to ./bin
|
||||
-d turns on debug logging
|
||||
-v verify checksum signature. Require cosign binary to be installed.
|
||||
[tag] is a tag from
|
||||
https://github.com/trufflesecurity/trufflehog/releases
|
||||
If tag is missing, then the latest will be used.
|
||||
@@ -22,10 +23,11 @@ parse_args() {
|
||||
# over-ridden by flag below
|
||||
|
||||
BINDIR=${BINDIR:-./bin}
|
||||
while getopts "b:dh?x" arg; do
|
||||
while getopts "b:dvh?x" arg; do
|
||||
case "$arg" in
|
||||
b) BINDIR="$OPTARG" ;;
|
||||
d) log_set_priority 10 ;;
|
||||
v) VERIFY_SIGN=true;;
|
||||
h | \?) usage "$0" ;;
|
||||
x) set -x ;;
|
||||
esac
|
||||
@@ -41,8 +43,15 @@ parse_args() {
|
||||
execute() {
|
||||
tmpdir=$(mktemp -d)
|
||||
log_debug "downloading files into ${tmpdir}"
|
||||
http_download "${tmpdir}/${TARBALL}" "${TARBALL_URL}"
|
||||
http_download "${tmpdir}/${CHECKSUM}" "${CHECKSUM_URL}"
|
||||
|
||||
if [ "$VERIFY_SIGN" = true ]; then
|
||||
http_download "${tmpdir}/${CHECKSUM}.${CERT_FORMAT}" "${CHECKSUM_URL}.${CERT_FORMAT}"
|
||||
http_download "${tmpdir}/${CHECKSUM}.${SIG_FORMAT}" "${CHECKSUM_URL}.${SIG_FORMAT}"
|
||||
verify_sign "${tmpdir}/${CHECKSUM}" "${tmpdir}/${CHECKSUM}.${CERT_FORMAT}" "${tmpdir}/${CHECKSUM}.${SIG_FORMAT}"
|
||||
fi
|
||||
|
||||
http_download "${tmpdir}/${TARBALL}" "${TARBALL_URL}"
|
||||
hash_sha256_verify "${tmpdir}/${TARBALL}" "${tmpdir}/${CHECKSUM}"
|
||||
srcdir="${tmpdir}"
|
||||
(cd "${tmpdir}" && untar "${TARBALL}")
|
||||
@@ -326,6 +335,24 @@ hash_sha256_verify() {
|
||||
fi
|
||||
}
|
||||
|
||||
check_cosign_bin() {
|
||||
if [ "$VERIFY_SIGN" = true ]; then
|
||||
if [ ! -x "$(command -v "$COSIGN_BINARY")" ]; then
|
||||
log_err "Cosign binary is not installed. Follow steps from https://docs.sigstore.dev/system_config/installation/ to install it."
|
||||
return 1
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
verify_sign() {
|
||||
log_debug "Verifying artifact $1"
|
||||
${COSIGN_BINARY} verify-blob "$1" \
|
||||
--certificate "$2" \
|
||||
--signature "$3" \
|
||||
--certificate-identity-regexp "https://github\.com/${OWNER}/${REPO}/\.github/workflows/.+" \
|
||||
--certificate-oidc-issuer "https://token.actions.githubusercontent.com"
|
||||
}
|
||||
|
||||
cat /dev/null <<EOF
|
||||
------------------------------------------------------------------------
|
||||
End of functions from https://github.com/client9/shlib
|
||||
@@ -341,6 +368,10 @@ ARCH=$(uname_arch)
|
||||
PREFIX="$OWNER/$REPO"
|
||||
PLATFORM="${OS}/${ARCH}"
|
||||
GITHUB_DOWNLOAD=https://github.com/${OWNER}/${REPO}/releases/download
|
||||
COSIGN_BINARY=cosign
|
||||
VERIFY_SIGN=false
|
||||
CERT_FORMAT=pem
|
||||
SIG_FORMAT=sig
|
||||
|
||||
# use in logging routines
|
||||
log_prefix() {
|
||||
@@ -353,6 +384,8 @@ uname_arch_check "$ARCH"
|
||||
|
||||
parse_args "$@"
|
||||
|
||||
check_cosign_bin
|
||||
|
||||
get_binary
|
||||
|
||||
tag_to_version
|
||||
|
||||
Reference in New Issue
Block a user