trufflesecurity/trufflehog
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

merge main

Browse Source
This commit is contained in:
Ahrav Dutta
2024-04-19 12:43:27 -07:00
parent bae9b335b4 32652a7498
commit 418790458e
188 changed files with 12339 additions and 6663 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/codeql-analysis.yml
+24 -34
View File
@@ -13,12 +13,12 @@ name: "CodeQL"
on:
push:
branches: [ main ]
branches: [main]
pull_request:
# The branches below must be a subset of the branches above
branches: [ main ]
branches: [main]
schedule:
- cron: '35 11 * * 2'
- cron: "35 11 * * 2"
jobs:
analyze:
@@ -32,39 +32,29 @@ jobs:
strategy:
fail-fast: false
matrix:
language: [ 'go' ]
language: ["go"]
# CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
# Learn more about CodeQL language support at https://git.io/codeql-language-support
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Checkout repository
uses: actions/checkout@v4
# Initializes the CodeQL tools for scanning.
- name: Initialize CodeQL
uses: github/codeql-action/init@v2
with:
languages: ${{ matrix.language }}
# If you wish to specify custom queries, you can do so here or in a config file.
# By default, queries listed here will override any specified in a config file.
# Prefix the list here with "+" to use these queries and those in the config file.
# queries: ./path/to/local/query, your-org/your-repo/queries@main
# Autobuild attempts to build any compiled languages (C/C++, C#, or Java).
# If this step fails, then you should remove it and run the build manually (see below)
- name: Autobuild
uses: github/codeql-action/autobuild@v2
# ℹ️ Command-line programs to run using the OS shell.
# 📚 https://git.io/JvXDl
# ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
# and modify them (or add more) to build your code if your project
# uses a compiled language
#- run: |
# make bootstrap
# make release
- name: Perform CodeQL Analysis
uses: github/codeql-action/analyze@v2
# Initializes the CodeQL tools for scanning.
- name: Initialize CodeQL
uses: github/codeql-action/init@v3
with:
languages: ${{ matrix.language }}
# If you wish to specify custom queries, you can do so here or in a config file.
# By default, queries listed here will override any specified in a config file.
# Prefix the list here with "+" to use these queries and those in the config file.
# queries: ./path/to/local/query, your-org/your-repo/queries@main
- name: Install Go
uses: actions/setup-go@v5
with:
go-version: "1.22"
- name: Smoke
run: |
go build .
- name: Perform CodeQL Analysis
uses: github/codeql-action/analyze@v3
.github/workflows/detector-tests.yml
+2 -2
View File
@@ -15,9 +15,9 @@ jobs:
id-token: "write"
steps:
- name: Install Go
uses: actions/setup-go@v4
uses: actions/setup-go@v5
with:
go-version: "1.21"
go-version: "1.22"
- name: Checkout code
uses: actions/checkout@v4
- id: "auth"
.github/workflows/lint.yml
+3 -3
View File
@@ -15,12 +15,12 @@ jobs:
name: golangci-lint
runs-on: ubuntu-latest
steps:
- uses: actions/setup-go@v4
- uses: actions/setup-go@v5
with:
go-version: "1.21"
go-version: "1.22"
- uses: actions/checkout@v4
- name: golangci-lint
uses: golangci/golangci-lint-action@v3
uses: golangci/golangci-lint-action@v4
with:
# Optional: version of golangci-lint to use in form of v1.2 or v1.2.3 or `latest` to use the latest version
version: latest
.github/workflows/performance.yml
+66 -66
View File
@@ -10,83 +10,83 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Install Go
uses: actions/setup-go@v4
with:
go-version: '1.21'
- name: Install Go
uses: actions/setup-go@v5
with:
go-version: "1.22"
- name: Checkout code
uses: actions/checkout@v4
with:
fetch-depth: 0
ref: ${{ github.head_ref }}
- name: Checkout code
uses: actions/checkout@v4
with:
fetch-depth: 0
ref: ${{ github.head_ref }}
- name: Run Head
run: |
go build -o current .
repo_tmp=$(mktemp -d)
git clone https://github.com/trufflesecurity/trufflehog.git $repo_tmp
- name: Run Head
run: |
go build -o current .
repo_tmp=$(mktemp -d)
git clone https://github.com/trufflesecurity/trufflehog.git $repo_tmp
user_time_sum=0
user_time_sum=0
for i in {1..5}
do
tmpfile=$(mktemp)
/usr/bin/time -o $tmpfile $GITHUB_WORKSPACE/current git "file://$repo_tmp" --no-verification --no-update > out.txt
cat $tmpfile
time_output=$(cat $tmpfile)
rm $tmpfile
user_time=$(echo $time_output | awk '{print $1}' | sed 's/user//')
for i in {1..5}
do
tmpfile=$(mktemp)
/usr/bin/time -o $tmpfile $GITHUB_WORKSPACE/current git "file://$repo_tmp" --no-verification --no-update > out.txt
cat $tmpfile
time_output=$(cat $tmpfile)
rm $tmpfile
user_time=$(echo $time_output | awk '{print $1}' | sed 's/user//')
# Add the user time to the sum
user_time_sum=$(echo "$user_time_sum + $user_time" | bc)
done
# Add the user time to the sum
user_time_sum=$(echo "$user_time_sum + $user_time" | bc)
done
average_user_time=$(echo "scale=3; $user_time_sum / 5" | bc)
echo HEAD_TIME=$average_user_time >> $GITHUB_ENV
average_user_time=$(echo "scale=3; $user_time_sum / 5" | bc)
echo HEAD_TIME=$average_user_time >> $GITHUB_ENV
- name: Figure out previous tag
run: |
git fetch --tags
git tag -l --sort=-v:refname | head -n 1 > previous_tag.txt
echo PREVIOUS_TAG=$(cat previous_tag.txt) >> $GITHUB_ENV
- name: Figure out previous tag
run: |
git fetch --tags
git tag -l --sort=-v:refname | head -n 1 > previous_tag.txt
echo PREVIOUS_TAG=$(cat previous_tag.txt) >> $GITHUB_ENV
- name: Checkout code
uses: actions/checkout@v4
with:
fetch-depth: 0
ref: ${{ env.PREVIOUS_TAG }}
- name: Checkout code
uses: actions/checkout@v4
with:
fetch-depth: 0
ref: ${{ env.PREVIOUS_TAG }}
- name: Run Previous
run: |
go build -o previous .
repo_tmp=$(mktemp -d)
git clone https://github.com/trufflesecurity/trufflehog.git $repo_tmp
- name: Run Previous
run: |
go build -o previous .
repo_tmp=$(mktemp -d)
git clone https://github.com/trufflesecurity/trufflehog.git $repo_tmp
user_time_sum=0
user_time_sum=0
for i in {1..5}
do
tmpfile=$(mktemp)
/usr/bin/time -o $tmpfile $GITHUB_WORKSPACE/previous git "file://$repo_tmp" --no-verification --no-update > out.txt
cat $tmpfile
time_output=$(cat $tmpfile)
rm $tmpfile
user_time=$(echo $time_output | awk '{print $1}' | sed 's/user//')
for i in {1..5}
do
tmpfile=$(mktemp)
/usr/bin/time -o $tmpfile $GITHUB_WORKSPACE/previous git "file://$repo_tmp" --no-verification --no-update > out.txt
cat $tmpfile
time_output=$(cat $tmpfile)
rm $tmpfile
user_time=$(echo $time_output | awk '{print $1}' | sed 's/user//')
# Add the user time to the sum
user_time_sum=$(echo "$user_time_sum + $user_time" | bc)
done
# Add the user time to the sum
user_time_sum=$(echo "$user_time_sum + $user_time" | bc)
done
average_user_time=$(echo "scale=3; $user_time_sum / 5" | bc)
echo PREVIOUS_TIME=$average_user_time >> $GITHUB_ENV
average_user_time=$(echo "scale=3; $user_time_sum / 5" | bc)
echo PREVIOUS_TIME=$average_user_time >> $GITHUB_ENV
- name: Compare Results
run: |
echo "head ($GITHUB_SHA) avg time (n=5): $HEAD_TIME"
echo "$PREVIOUS_TAG avg time (n=5): $PREVIOUS_TIME"
if [ $(echo "$HEAD_TIME > $PREVIOUS_TIME * 1.1" | bc) -eq 1 ]
then
echo "HEAD run time is at least 10% slower than PREVIOUS run time"
exit 1
fi
- name: Compare Results
run: |
echo "head ($GITHUB_SHA) avg time (n=5): $HEAD_TIME"
echo "$PREVIOUS_TAG avg time (n=5): $PREVIOUS_TIME"
if [ $(echo "$HEAD_TIME > $PREVIOUS_TIME * 1.1" | bc) -eq 1 ]
then
echo "HEAD run time is at least 10% slower than PREVIOUS run time"
exit 1
fi
.github/workflows/release.yml
+3 -3
View File
@@ -34,11 +34,11 @@ jobs:
username: ${{ github.repository_owner }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Set up Go
uses: actions/setup-go@v4
uses: actions/setup-go@v5
with:
go-version: '1.21'
go-version: "1.22"
- name: Cosign install
uses: sigstore/cosign-installer@9614fae9e5c5eddabb09f90a270fcb487c9f7149 # v3.3.0
uses: sigstore/cosign-installer@59acb6260d9c0ba8f4a2f9d9b48431a222b68e20 # v3.5.0
- name: Run GoReleaser
uses: goreleaser/goreleaser-action@v5
with:
.github/workflows/secrets.yml
+1 -1
View File
@@ -24,4 +24,4 @@ jobs:
uses: ./
id: dogfood
with:
extra_args: --only-verified
extra_args: --results=verified,unknown
.github/workflows/smoke.yml
+11 -12
View File
@@ -1,4 +1,3 @@
name: Smoke
on:
@@ -8,14 +7,14 @@ jobs:
smoke:
runs-on: ubuntu-latest
steps:
- name: Install Go
uses: actions/setup-go@v4
with:
go-version: '1.21'
- name: Checkout code
uses: actions/checkout@v4
- name: Smoke
run: |
set -e
go run . git https://github.com/dustin-decker/secretsandstuff.git
go run . github --repo https://github.com/dustin-decker/secretsandstuff.git
- name: Install Go
uses: actions/setup-go@v5
with:
go-version: "1.22"
- name: Checkout code
uses: actions/checkout@v4
- name: Smoke
run: |
set -e
go run . git https://github.com/dustin-decker/secretsandstuff.git
go run . github --repo https://github.com/dustin-decker/secretsandstuff.git
.github/workflows/snifftest.yml
+2 -2
View File
@@ -16,9 +16,9 @@ jobs:
name: Run Snifftest
runs-on: ubuntu-latest
steps:
- uses: actions/setup-go@v4
- uses: actions/setup-go@v5
with:
go-version: '1.21'
go-version: "1.22"
- uses: actions/checkout@v4
- name: Run Snifftest
run: make snifftest
.github/workflows/test.yml
+4 -4
View File
@@ -18,9 +18,9 @@ jobs:
id-token: "write"
steps:
- name: Install Go
uses: actions/setup-go@v4
uses: actions/setup-go@v5
with:
go-version: "1.21"
go-version: "1.22"
- name: Checkout code
uses: actions/checkout@v4
- id: "auth"
@@ -60,9 +60,9 @@ jobs:
contents: "read"
steps:
- name: Install Go
uses: actions/setup-go@v4
uses: actions/setup-go@v5
with:
go-version: "1.21"
go-version: "1.22"
- name: Checkout code
uses: actions/checkout@v4
- name: Test
CONTRIBUTING.md
+2 -2
View File
@@ -1,6 +1,6 @@
# Contribution guidelines
Please create an issue to collect feedback prior to feature additions. If possible try to keep PRs scoped to one feature, and add tests for new features.
Please create an issue to collect feedback prior to feature additions. If possible try to keep PRs scoped to one feature, and add tests for new features. We use the fork-based contribution model described by [GitHub's documentation](https://docs.github.com/en/get-started/exploring-projects-on-github/contributing-to-a-project). (In short: Fork the TruffleHog repo and open a PR back from your fork into our default branch.)
When showing interest in a bug, enhancement, PR, or issue, please use the thumbs up/thumbs down emoji on the original message rather than adding comments expressing the same.
@@ -10,4 +10,4 @@ Contributors need to [sign our CLA](https://cla-assistant.io/trufflesecurity/tru
## Adding new secret detectors
We have published some [documentation and tooling to get started on adding new secret detectors](hack/docs/Adding_Detectors_external.md). Let's improve detection together!
We have published some [documentation and tooling to get started on adding new secret detectors](hack/docs/Adding_Detectors_external.md). Let's improve detection together!
Makefile
+2 -2
View File
@@ -1,4 +1,4 @@
PROTOS_IMAGE ?= trufflesecurity/protos:1.21-1
PROTOS_IMAGE ?= trufflesecurity/protos:1.22
.PHONY: check
.PHONY: lint
@@ -59,7 +59,7 @@ protos-windows:
release-protos-image:
docker buildx build --push --platform=linux/amd64,linux/arm64 \
-t trufflesecurity/protos:1.21-1 -f hack/Dockerfile.protos .
-t ${PROTOS_IMAGE} -f hack/Dockerfile.protos .
snifftest:
./hack/snifftest/snifftest.sh
README.md
+103 -23
View File
@@ -46,30 +46,68 @@ docker run --rm -it -v "$PWD:/pwd" trufflesecurity/trufflehog:latest github --or
Several options available for you:
### MacOS users
```bash
# MacOS users
brew install trufflehog
```
# Docker
### Docker:
<sub><i>*Ensure Docker engine is running before executing the following commands:*</i></sub>
#### &nbsp;&nbsp;&nbsp;&nbsp;Unix
```bash
docker run --rm -it -v "$PWD:/pwd" trufflesecurity/trufflehog:latest github --repo https://github.com/trufflesecurity/test_keys
```
# Docker for M1 and M2 Mac
#### &nbsp;&nbsp;&nbsp;&nbsp;Windows Command Prompt
```bash
docker run --rm -it -v "%cd:/=\%:/pwd" trufflesecurity/trufflehog:latest github --repo https://github.com/trufflesecurity/test_keys
```
#### &nbsp;&nbsp;&nbsp;&nbsp;Windows PowerShell
```bash
docker run --rm -it -v "${PWD}:/pwd" trufflesecurity/trufflehog github --repo https://github.com/trufflesecurity/test_keys
```
#### &nbsp;&nbsp;&nbsp;&nbsp;M1 and M2 Mac
```bash
docker run --platform linux/arm64 --rm -it -v "$PWD:/pwd" trufflesecurity/trufflehog:latest github --repo https://github.com/trufflesecurity/test_keys
```
# Binary releases
### Binary releases
```bash
Download and unpack from https://github.com/trufflesecurity/trufflehog/releases
```
# Compile from source
### Compile from source
```bash
git clone https://github.com/trufflesecurity/trufflehog.git
cd trufflehog; go install
```
# Using installation script
### Using installation script
```bash
curl -sSfL https://raw.githubusercontent.com/trufflesecurity/trufflehog/main/scripts/install.sh | sh -s -- -b /usr/local/bin
```
# Using installation script, verify checksum signature (requires cosign to be installed)
### Using installation script, verify checksum signature (requires cosign to be installed)
```bash
curl -sSfL https://raw.githubusercontent.com/trufflesecurity/trufflehog/main/scripts/install.sh | sh -s -- -v -b /usr/local/bin
```
# Using installation script to install a specific version
### Using installation script to install a specific version
```bash
curl -sSfL https://raw.githubusercontent.com/trufflesecurity/trufflehog/main/scripts/install.sh | sh -s -- -b /usr/local/bin <ReleaseTag like v3.56.0>
```
@@ -159,7 +197,7 @@ Expected output:
...
```
## 4: Scan a GitHub Repo + its Issues and Pull Requests.
## 4: Scan a GitHub Repo + its Issues and Pull Requests
```bash
trufflehog github --repo=https://github.com/trufflesecurity/test_keys --issue-comments --pr-comments
@@ -189,13 +227,13 @@ docker run --rm -v "$HOME/.ssh:/root/.ssh:ro" trufflesecurity/trufflehog:latest
trufflehog filesystem path/to/file1.txt path/to/file2.txt path/to/dir
```
## 9: Scan GCS buckets for verified secrets.
## 9: Scan GCS buckets for verified secrets
```bash
trufflehog gcs --project-id=<project-ID> --cloud-environment --only-verified
```
## 10: Scan a Docker image for verified secrets.
## 10: Scan a Docker image for verified secrets
Use the `--image` flag multiple times to scan multiple images.
@@ -211,6 +249,14 @@ Set the `--since-commit` flag to your default branch that people merge into (ex:
trufflehog git file://. --since-commit main --branch feature-1 --only-verified --fail
```
## 12: Scan a Postman workspace
Use the `--workspace`, `--collection`, `--environment` flags multiple times to scan multiple targets.
```bash
trufflehog postman --token=<postman api token> --workspace=<workspace id>
```
# :question: FAQ
- All I see is `🐷🔑🐷 TruffleHog. Unearth your secrets. 🐷🔑🐷` and the program exits, what gives?
@@ -267,9 +313,11 @@ Flags:
-j, --json Output in JSON format.
--json-legacy Use the pre-v3.0 JSON format. Only works with git, gitlab, and github sources.
--github-actions Output in GitHub Actions format.
--concurrency=8 Number of concurrent workers.
--concurrency=20 Number of concurrent workers.
--no-verification Don't verify the results.
--only-verified Only output verified results.
--allow-verification-overlap
Allow verification of similar credentials across detectors
--filter-unverified Only output first unverified result per chunk per detector if there are more than one results.
--filter-entropy=FILTER-ENTROPY
Filter unverified results with Shannon entropy. Start with 3.0.
@@ -279,6 +327,7 @@ Flags:
--no-update Don't check for updates.
--fail Exit with code 183 if results are found.
--verifier=VERIFIER ... Set custom verification endpoints.
--custom-verifiers-only Only use custom verification endpoints.
--archive-max-size=ARCHIVE-MAX-SIZE
Maximum size of archive to scan. (Byte units eg. 512B, 2KB, 4MB)
--archive-max-depth=ARCHIVE-MAX-DEPTH
@@ -308,7 +357,7 @@ Args:
For example, to scan a `git` repository, start with
```
$ trufflehog git https://github.com/trufflesecurity/trufflehog.git
trufflehog git https://github.com/trufflesecurity/trufflehog.git
```
## S3
@@ -368,7 +417,6 @@ jobs:
In the example config above, we're scanning for live secrets in all PRs and Pushes to `main`. Only code changes in the referenced commits are scanned. If you'd like to scan an entire branch, please see the "Advanced Usage" section below.
### Shallow Cloning
If you're incorporating TruffleHog into a standalone workflow and aren't running any other CI/CD tooling alongside TruffleHog, then we recommend using [Shallow Cloning](https://git-scm.com/docs/git-clone#Documentation/git-clone.txt---depthltdepthgt) to speed up your workflow. Here's an example for how to do it:
@@ -397,6 +445,12 @@ If you're incorporating TruffleHog into a standalone workflow and aren't running
Depending on the event type (push or PR), we calculate the number of commits present. Then we add 2, so that we can reference a base commit before our code changes. We pass that integer value to the `fetch-depth` flag in the checkout action in addition to the relevant branch. Now our checkout process should be much shorter.
### Canary detection
TruffleHog statically detects [https://canarytokens.org/](https://canarytokens.org/) and lets you know when they're present without setting them off. You can learn more here: [https://trufflesecurity.com/canaries](https://trufflesecurity.com/canaries)
![image](https://github.com/trufflesecurity/trufflehog/assets/52866392/74ace530-08c5-4eaf-a169-84a73e328f6f)
### Advanced Usage
```yaml
@@ -416,6 +470,7 @@ Depending on the event type (push or PR), we calculate the number of commits pre
If you'd like to specify specific `base` and `head` refs, you can use the `base` argument (`--since-commit` flag in TruffleHog CLI) and the `head` argument (`--branch` flag in the TruffleHog CLI). We only recommend using these arguments for very specific use cases, where the default behavior does not work.
#### Advanced Usage: Scan entire branch
```
- name: scan-push
uses: trufflesecurity/trufflehog@main
@@ -425,13 +480,38 @@ If you'd like to specify specific `base` and `head` refs, you can use the `base`
extra_args: --only-verified
```
## TruffleHog GitLab CI
### Example Usage
```yaml
stages:
- security
security-secrets:
stage: security
allow_failure: false
image: alpine:latest
variables:
SCAN_PATH: "." # Set the relative path in the repo to scan
before_script:
- apk add --no-cache git curl jq
- curl -sSfL https://raw.githubusercontent.com/trufflesecurity/trufflehog/main/scripts/install.sh | sh -s -- -b /usr/local/bin
script:
- trufflehog filesystem "$SCAN_PATH" --only-verified --fail --json | jq
rules:
- if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
```
In the example pipeline above, we're scanning for live secrets in all repository directories and files. This job runs only when the pipeline source is a merge request event, meaning it's triggered when a new merge request is created.
## Pre-commit Hook
Trufflehog can be used in a pre-commit hook to prevent credentials from leaking before they ever leave your computer.
TruffleHog can be used in a pre-commit hook to prevent credentials from leaking before they ever leave your computer.
**Key Usage Note:**
- **For optimal hook efficacy, execute `git add` followed by `git commit` separately.** This ensures Trufflehog analyzes all intended changes.
- **For optimal hook efficacy, execute `git add` followed by `git commit` separately.** This ensures TruffleHog analyzes all intended changes.
- **Avoid using `git commit -am`, as it might bypass pre-commit hook execution for unstaged modifications.**
An example `.pre-commit-config.yaml` is provided (see [pre-commit.com](https://pre-commit.com/) for installation).
@@ -452,13 +532,13 @@ repos:
## Regex Detector (alpha)
Trufflehog supports detection and verification of custom regular expressions.
TruffleHog supports detection and verification of custom regular expressions.
For detection, at least one **regular expression** and **keyword** is required.
A **keyword** is a fixed literal string identifier that appears in or around
the regex to be detected. To allow maximum flexibility for verification, a
webhook is used containing the regular expression matches.
Trufflehog will send a JSON POST request containing the regex matches to a
TruffleHog will send a JSON POST request containing the regex matches to a
configured webhook endpoint. If the endpoint responds with a `200 OK` response
status code, the secret is considered verified.
@@ -473,8 +553,8 @@ detectors:
keywords:
- hog
regex:
hogID: \b(HOG[0-9A-Z]{16})\b
hogToken: [^A-Za-z0-9+\/]{0,1}([A-Za-z0-9+\/]{40})[^A-Za-z0-9+\/]{0,1}
hogID: '\b(HOG[0-9A-Z]{17})\b'
hogToken: '[^A-Za-z0-9+\/]{0,1}([A-Za-z0-9+\/]{40})[^A-Za-z0-9+\/]{0,1}'
verify:
- endpoint: http://localhost:8000/
# unsafe must be set if the endpoint is HTTP
@@ -483,7 +563,6 @@ detectors:
- "Authorization: super secret authorization header"
```
```
$ trufflehog filesystem /tmp --config config.yaml --only-verified
🐷🔑🐷 TruffleHog. Unearth your secrets. 🐷🔑🐷
@@ -494,7 +573,8 @@ Decoder Type: PLAIN
Raw result: HOGAAIUNNWHAHJJWUQYR
File: /tmp/hog-facts.txt
```
Data structure sent to the custom verificaiton server:
Data structure sent to the custom verification server:
```
{
@@ -584,6 +664,6 @@ Since v3.0, TruffleHog is released under a AGPL 3 license, included in [`LICENSE
# :money_with_wings: Enterprise product
Are you interested in continuously monitoring your Git, Jira, Slack, Confluence, etc.. for credentials? We have an enterprise product that can help. Reach out here to learn more https://trufflesecurity.com/contact/
Are you interested in continuously monitoring your Git, Jira, Slack, Confluence, etc.. for credentials? We have an enterprise product that can help. Reach out here to learn more <https://trufflesecurity.com/contact/>
We take the revenue from the enterprise product to fund more awesome open source projects that the whole community can benefit from.
entrypoint.sh
+8 -1
View File
@@ -3,4 +3,11 @@
# Parse the last argument into an array of extra_args.
mapfile -t extra_args < <(bash -c "for arg in ${*: -1}; do echo \$arg; done")
/usr/bin/trufflehog "${@: 1: $#-1}" "${extra_args[@]}"
# Directories might be owned by a user other than root
git config --global --add safe.directory '*'
if [[ $# -eq 0 ]]; then
/usr/bin/trufflehog --help
else
/usr/bin/trufflehog "${@: 1: $#-1}" "${extra_args[@]}"
fi
go.mod
+127 -113
View File
@@ -1,109 +1,115 @@
module github.com/trufflesecurity/trufflehog/v3
go 1.21
go 1.22.0
replace github.com/jpillora/overseer => github.com/trufflesecurity/overseer v1.2.7
replace github.com/snowflakedb/gosnowflake => github.com/trufflesecurity/gosnowflake v0.0.1
require (
cloud.google.com/go/secretmanager v1.11.4
cloud.google.com/go/storage v1.36.0
cloud.google.com/go/secretmanager v1.12.0
cloud.google.com/go/storage v1.40.0
github.com/Azure/go-autorest/autorest/azure/auth v0.5.12
github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1
github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2
github.com/BobuSumisu/aho-corasick v1.0.3
github.com/TheZeroSlave/zapsentry v1.19.0
github.com/TheZeroSlave/zapsentry v1.23.0
github.com/adrg/strutil v0.3.1
github.com/alecthomas/kingpin/v2 v2.4.0
github.com/aws/aws-sdk-go v1.50.0
github.com/aws/aws-sdk-go v1.51.24
github.com/aymanbagabas/go-osc52 v1.2.2
github.com/bill-rich/go-syslog v0.0.0-20220413021637-49edb52a574c
github.com/bitfinexcom/bitfinex-api-go v0.0.0-20210608095005-9e0b26f200fb
github.com/bradleyfalzon/ghinstallation/v2 v2.9.0
github.com/charmbracelet/bubbles v0.16.1
github.com/charmbracelet/bubbletea v0.24.2
github.com/charmbracelet/glamour v0.6.0
github.com/charmbracelet/lipgloss v0.9.1
github.com/bradleyfalzon/ghinstallation/v2 v2.10.0
github.com/brianvoe/gofakeit/v7 v7.0.2
github.com/charmbracelet/bubbles v0.18.0
github.com/charmbracelet/bubbletea v0.25.0
github.com/charmbracelet/glamour v0.7.0
github.com/charmbracelet/lipgloss v0.10.0
github.com/coinbase/waas-client-library-go v1.0.8
github.com/couchbase/gocb/v2 v2.7.1
github.com/couchbase/gocb/v2 v2.8.1
github.com/crewjam/rfc5424 v0.1.0
github.com/denisenkom/go-mssqldb v0.12.3
github.com/envoyproxy/protoc-gen-validate v1.0.4
github.com/fatih/color v1.16.0
github.com/felixge/fgprof v0.9.3
github.com/felixge/fgprof v0.9.4
github.com/gabriel-vasile/mimetype v1.4.3
github.com/getsentry/sentry-go v0.25.0
github.com/getsentry/sentry-go v0.27.0
github.com/go-errors/errors v1.5.1
github.com/go-git/go-git/v5 v5.11.0
github.com/go-ldap/ldap/v3 v3.4.6
github.com/go-logr/logr v1.3.0
github.com/go-git/go-git/v5 v5.12.0
github.com/go-ldap/ldap/v3 v3.4.8
github.com/go-logr/logr v1.4.1
github.com/go-logr/zapr v1.3.0
github.com/go-redis/redis v6.15.9+incompatible
github.com/go-sql-driver/mysql v1.7.1
github.com/go-sql-driver/mysql v1.8.1
github.com/gobwas/glob v0.2.3
github.com/golang-jwt/jwt v3.2.2+incompatible
github.com/golang-jwt/jwt/v4 v4.5.0
github.com/google/go-cmp v0.6.0
github.com/google/go-containerregistry v0.17.0
github.com/google/go-github/v57 v57.0.0
github.com/google/uuid v1.5.0
github.com/googleapis/gax-go/v2 v2.12.0
github.com/google/go-containerregistry v0.19.1
github.com/google/go-github/v61 v61.0.0
github.com/google/uuid v1.6.0
github.com/googleapis/gax-go/v2 v2.12.3
github.com/h2non/filetype v1.1.3
github.com/hashicorp/go-retryablehttp v0.7.5
github.com/hashicorp/golang-lru v0.5.1
github.com/hashicorp/golang-lru/v2 v2.0.7
github.com/jlaffaye/ftp v0.2.0
github.com/joho/godotenv v1.5.1
github.com/jpillora/overseer v1.1.6
github.com/kylelemons/godebug v1.1.0
github.com/launchdarkly/go-server-sdk/v6 v6.1.1
github.com/launchdarkly/go-server-sdk/v7 v7.4.0
github.com/lib/pq v1.10.9
github.com/lrstanley/bubblezone v0.0.0-20230911164824-e3824f1adde9
github.com/lrstanley/bubblezone v0.0.0-20240125042004-b7bafc493195
github.com/marusama/semaphore/v2 v2.5.0
github.com/mattn/go-isatty v0.0.20
github.com/mattn/go-sqlite3 v1.14.19
github.com/mholt/archiver/v4 v4.0.0-alpha.8
github.com/microsoft/go-mssqldb v1.7.0
github.com/mitchellh/go-ps v1.0.0
github.com/muesli/reflow v0.3.0
github.com/patrickmn/go-cache v2.1.0+incompatible
github.com/paulbellamy/ratecounter v0.2.0
github.com/pkg/errors v0.9.1
github.com/prometheus/client_golang v1.17.0
github.com/prometheus/client_golang v1.19.0
github.com/rabbitmq/amqp091-go v1.9.0
github.com/sergi/go-diff v1.3.1
github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3
github.com/shuheiktgw/go-travis v0.3.1
github.com/snowflakedb/gosnowflake v1.6.25
github.com/stretchr/testify v1.8.4
github.com/snowflakedb/gosnowflake v1.8.0
github.com/stretchr/testify v1.9.0
github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502
github.com/testcontainers/testcontainers-go v0.30.0
github.com/testcontainers/testcontainers-go/modules/mssql v0.30.0
github.com/testcontainers/testcontainers-go/modules/mysql v0.30.0
github.com/testcontainers/testcontainers-go/modules/postgres v0.30.0
github.com/trufflesecurity/disk-buffer-reader v0.2.1
github.com/wasilibs/go-re2 v1.4.1
github.com/xanzy/go-gitlab v0.94.0
go.mongodb.org/mongo-driver v1.12.1
go.uber.org/mock v0.3.0
go.uber.org/zap v1.26.0
golang.org/x/crypto v0.18.0
golang.org/x/exp v0.0.0-20240119083558-1b970713d09a
golang.org/x/net v0.20.0
golang.org/x/oauth2 v0.15.0
golang.org/x/sync v0.6.0
github.com/wasilibs/go-re2 v1.5.2
github.com/xanzy/go-gitlab v0.103.0
go.mongodb.org/mongo-driver v1.15.0
go.uber.org/mock v0.4.0
go.uber.org/zap v1.27.0
golang.org/x/crypto v0.22.0
golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f
golang.org/x/net v0.24.0
golang.org/x/oauth2 v0.19.0
golang.org/x/sync v0.7.0
golang.org/x/text v0.14.0
google.golang.org/api v0.152.0
google.golang.org/protobuf v1.32.0
google.golang.org/api v0.174.0
google.golang.org/protobuf v1.33.0
gopkg.in/h2non/gock.v1 v1.1.2
pgregory.net/rapid v1.1.0
sigs.k8s.io/yaml v1.4.0
)
require (
cloud.google.com/go v0.111.0 // indirect
cloud.google.com/go/compute v1.23.3 // indirect
cloud.google.com/go/compute/metadata v0.2.3 // indirect
cloud.google.com/go/iam v1.1.5 // indirect
cloud.google.com/go/longrunning v0.5.4 // indirect
cloud.google.com/go v0.112.1 // indirect
cloud.google.com/go/auth v0.2.0 // indirect
cloud.google.com/go/auth/oauth2adapt v0.2.0 // indirect
cloud.google.com/go/compute v1.24.0 // indirect
cloud.google.com/go/compute/metadata v0.3.0 // indirect
cloud.google.com/go/iam v1.1.7 // indirect
cloud.google.com/go/longrunning v0.5.5 // indirect
dario.cat/mergo v1.0.0 // indirect
filippo.io/edwards25519 v1.1.0 // indirect
github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 // indirect
github.com/99designs/keyring v1.2.2 // indirect
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.4.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2 // indirect
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 // indirect
github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
github.com/Azure/go-autorest v14.2.0+incompatible // indirect
github.com/Azure/go-autorest/autorest v0.11.24 // indirect
github.com/Azure/go-autorest/autorest/adal v0.9.18 // indirect
@@ -114,77 +120,76 @@ require (
github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect
github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c // indirect
github.com/Microsoft/go-winio v0.6.1 // indirect
github.com/ProtonMail/go-crypto v0.0.0-20230828082145-3c4c8a2d2371 // indirect
github.com/alecthomas/chroma v0.10.0 // indirect
github.com/Microsoft/hcsshim v0.11.4 // indirect
github.com/ProtonMail/go-crypto v1.0.0 // indirect
github.com/alecthomas/chroma/v2 v2.8.0 // indirect
github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect
github.com/andybalholm/brotli v1.0.6 // indirect
github.com/apache/arrow/go/v12 v12.0.1 // indirect
github.com/apache/thrift v0.16.0 // indirect
github.com/apache/arrow/go/v14 v14.0.2 // indirect
github.com/atotto/clipboard v0.1.4 // indirect
github.com/aws/aws-sdk-go-v2 v1.17.7 // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 // indirect
github.com/aws/aws-sdk-go-v2/credentials v1.13.18 // indirect
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.59 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.31 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.25 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.23 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.26 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.25 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.14.0 // indirect
github.com/aws/aws-sdk-go-v2/service/s3 v1.31.0 // indirect
github.com/aws/smithy-go v1.13.5 // indirect
github.com/aws/smithy-go v1.20.1 // indirect
github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
github.com/aymerick/douceur v0.2.0 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/bodgit/plumbing v1.3.0 // indirect
github.com/bodgit/sevenzip v1.4.5 // indirect
github.com/bodgit/windows v1.0.1 // indirect
github.com/cenkalti/backoff/v4 v4.2.1 // indirect
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/cloudflare/circl v1.3.7 // indirect
github.com/containerd/console v1.0.4-0.20230313162750-1ae8d489ac81 // indirect
github.com/containerd/containerd v1.7.12 // indirect
github.com/containerd/log v0.1.0 // indirect
github.com/containerd/stargz-snapshotter/estargz v0.14.3 // indirect
github.com/couchbase/gocbcore/v10 v10.3.1 // indirect
github.com/couchbase/gocbcoreps v0.1.1 // indirect
github.com/couchbase/goprotostellar v1.0.1 // indirect
github.com/couchbase/gocbcore/v10 v10.4.1 // indirect
github.com/couchbase/gocbcoreps v0.1.2 // indirect
github.com/couchbase/goprotostellar v1.0.2 // indirect
github.com/couchbaselabs/gocbconnstr/v2 v2.0.0-20230515165046-68b522a21131 // indirect
github.com/cpuguy83/dockercfg v0.3.1 // indirect
github.com/cyphar/filepath-securejoin v0.2.4 // indirect
github.com/danieljoos/wincred v1.1.2 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/dimchansky/utfbom v1.1.1 // indirect
github.com/distribution/reference v0.5.0 // indirect
github.com/dlclark/regexp2 v1.4.0 // indirect
github.com/docker/cli v24.0.0+incompatible // indirect
github.com/docker/distribution v2.8.2+incompatible // indirect
github.com/docker/docker v24.0.7+incompatible // indirect
github.com/docker/docker v25.0.5+incompatible // indirect
github.com/docker/docker-credential-helpers v0.7.0 // indirect
github.com/docker/go-connections v0.5.0 // indirect
github.com/docker/go-units v0.5.0 // indirect
github.com/dsnet/compress v0.0.1 // indirect
github.com/dvsekhvalnov/jose2go v1.6.0 // indirect
github.com/emirpasic/gods v1.18.1 // indirect
github.com/felixge/httpsnoop v1.0.4 // indirect
github.com/form3tech-oss/jwt-go v3.2.5+incompatible // indirect
github.com/go-asn1-ber/asn1-ber v1.5.5 // indirect
github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
github.com/go-git/go-billy/v5 v5.5.0 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/go-ole/go-ole v1.2.6 // indirect
github.com/goccy/go-json v0.10.0 // indirect
github.com/goccy/go-json v0.10.2 // indirect
github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 // indirect
github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang-jwt/jwt/v5 v5.2.1 // indirect
github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9 // indirect
github.com/golang-sql/sqlexp v0.1.0 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/protobuf v1.5.3 // indirect
github.com/golang/protobuf v1.5.4 // indirect
github.com/golang/snappy v0.0.4 // indirect
github.com/google/flatbuffers v23.1.21+incompatible // indirect
github.com/google/flatbuffers v23.5.26+incompatible // indirect
github.com/google/go-github/v60 v60.0.0 // indirect
github.com/google/go-querystring v1.1.0 // indirect
github.com/google/pprof v0.0.0-20211214055906-6f57359322fd // indirect
github.com/google/pprof v0.0.0-20240227163752-401108e1b7e7 // indirect
github.com/google/s2a-go v0.1.7 // indirect
github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect
github.com/gorilla/css v1.0.0 // indirect
github.com/gregjones/httpcache v0.0.0-20171119193500-2bcd89a1743f // indirect
github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.15.2 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 // indirect
github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c // indirect
github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 // indirect
github.com/hashicorp/errwrap v1.0.0 // indirect
github.com/hashicorp/errwrap v1.1.0 // indirect
github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
github.com/hashicorp/go-multierror v1.1.1 // indirect
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
@@ -192,29 +197,32 @@ require (
github.com/josharian/intern v1.0.0 // indirect
github.com/jpillora/s3 v1.1.4 // indirect
github.com/kevinburke/ssh_config v1.2.0 // indirect
github.com/klauspost/asmfmt v1.3.2 // indirect
github.com/klauspost/compress v1.17.4 // indirect
github.com/klauspost/cpuid/v2 v2.2.3 // indirect
github.com/klauspost/cpuid/v2 v2.2.5 // indirect
github.com/klauspost/pgzip v1.2.5 // indirect
github.com/launchdarkly/ccache v1.1.0 // indirect
github.com/launchdarkly/eventsource v1.6.2 // indirect
github.com/launchdarkly/go-jsonstream/v3 v3.0.0 // indirect
github.com/launchdarkly/go-sdk-common/v3 v3.0.1 // indirect
github.com/launchdarkly/go-sdk-events/v2 v2.0.2 // indirect
github.com/launchdarkly/go-sdk-common/v3 v3.1.0 // indirect
github.com/launchdarkly/go-sdk-events/v3 v3.2.0 // indirect
github.com/launchdarkly/go-semver v1.0.2 // indirect
github.com/launchdarkly/go-server-sdk-evaluation/v2 v2.0.2 // indirect
github.com/launchdarkly/go-server-sdk-evaluation/v3 v3.0.0 // indirect
github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
github.com/magefile/mage v1.14.0 // indirect
github.com/magiconair/properties v1.8.7 // indirect
github.com/mailru/easyjson v0.7.7 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-localereader v0.0.1 // indirect
github.com/mattn/go-runewidth v0.0.15 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
github.com/microcosm-cc/bluemonday v1.0.23 // indirect
github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 // indirect
github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 // indirect
github.com/microcosm-cc/bluemonday v1.0.25 // indirect
github.com/mitchellh/go-homedir v1.1.0 // indirect
github.com/moby/patternmatcher v0.6.0 // indirect
github.com/moby/sys/sequential v0.5.0 // indirect
github.com/moby/sys/user v0.1.0 // indirect
github.com/moby/term v0.5.0 // indirect
github.com/montanaflynn/stats v0.7.0 // indirect
github.com/morikuni/aec v1.0.0 // indirect
github.com/mtibben/percent v0.2.1 // indirect
github.com/muesli/ansi v0.0.0-20211031195517-c9f0611b6c70 // indirect
github.com/muesli/cancelreader v0.2.2 // indirect
@@ -223,21 +231,25 @@ require (
github.com/olekukonko/tablewriter v0.0.5 // indirect
github.com/onsi/ginkgo v1.16.5 // indirect
github.com/opencontainers/go-digest v1.0.0 // indirect
github.com/opencontainers/image-spec v1.1.0-rc3 // indirect
github.com/opencontainers/image-spec v1.1.0 // indirect
github.com/pierrec/lz4/v4 v4.1.19 // indirect
github.com/pjbgf/sha1cd v0.3.0 // indirect
github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect
github.com/pkg/diff v0.0.0-20200914180035-5b29258ca4f7 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 // indirect
github.com/prometheus/common v0.44.0 // indirect
github.com/prometheus/procfs v0.11.1 // indirect
github.com/rivo/uniseg v0.4.2 // indirect
github.com/sahilm/fuzzy v0.1.0 // indirect
github.com/sirupsen/logrus v1.9.1 // indirect
github.com/skeema/knownhosts v1.2.1 // indirect
github.com/tetratelabs/wazero v1.5.0 // indirect
github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
github.com/prometheus/client_model v0.5.0 // indirect
github.com/prometheus/common v0.48.0 // indirect
github.com/prometheus/procfs v0.12.0 // indirect
github.com/rivo/uniseg v0.4.7 // indirect
github.com/sahilm/fuzzy v0.1.1-0.20230530133925-c48e322e2a8f // indirect
github.com/shirou/gopsutil/v3 v3.23.12 // indirect
github.com/shoenig/go-m1cpu v0.1.6 // indirect
github.com/sirupsen/logrus v1.9.3 // indirect
github.com/skeema/knownhosts v1.2.2 // indirect
github.com/tetratelabs/wazero v1.7.0 // indirect
github.com/therootcompany/xz v1.0.1 // indirect
github.com/tklauser/go-sysconf v0.3.12 // indirect
github.com/tklauser/numcpus v0.6.1 // indirect
github.com/ulikunitz/xz v0.5.11 // indirect
github.com/vbatts/tar-split v0.11.3 // indirect
github.com/xanzy/ssh-agent v0.3.3 // indirect
@@ -246,28 +258,30 @@ require (
github.com/xdg-go/stringprep v1.0.4 // indirect
github.com/xhit/go-str2duration/v2 v2.1.0 // indirect
github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect
github.com/yuin/goldmark v1.5.2 // indirect
github.com/yuin/goldmark-emoji v1.0.1 // indirect
github.com/yusufpapurcu/wmi v1.2.2 // indirect
github.com/yuin/goldmark v1.5.4 // indirect
github.com/yuin/goldmark-emoji v1.0.2 // indirect
github.com/yusufpapurcu/wmi v1.2.3 // indirect
github.com/zeebo/xxh3 v1.0.2 // indirect
go.einride.tech/aip v0.60.0 // indirect
go.opencensus.io v0.24.0 // indirect
go.opentelemetry.io/otel v1.19.0 // indirect
go.opentelemetry.io/otel/metric v1.19.0 // indirect
go.opentelemetry.io/otel/trace v1.19.0 // indirect
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect
go.opentelemetry.io/otel v1.24.0 // indirect
go.opentelemetry.io/otel/metric v1.24.0 // indirect
go.opentelemetry.io/otel/trace v1.24.0 // indirect
go.uber.org/multierr v1.11.0 // indirect
go4.org v0.0.0-20200411211856-f5505b9728dd // indirect
golang.org/x/mod v0.14.0 // indirect
golang.org/x/sys v0.16.0 // indirect
golang.org/x/term v0.16.0 // indirect
golang.org/x/mod v0.17.0 // indirect
golang.org/x/sys v0.19.0 // indirect
golang.org/x/term v0.19.0 // indirect
golang.org/x/time v0.5.0 // indirect
golang.org/x/tools v0.17.0 // indirect
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
golang.org/x/tools v0.20.0 // indirect
golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect
google.golang.org/appengine v1.6.8 // indirect
google.golang.org/genproto v0.0.0-20240102182953-50ed04b92917 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20231212172506-995d672761c0 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20240108191215-35c7eff3a6b1 // indirect
google.golang.org/grpc v1.60.1 // indirect
google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20240314234333-6e1732d8331c // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20240415180920-8c6c420018be // indirect
google.golang.org/grpc v1.63.2 // indirect
gopkg.in/square/go-jose.v2 v2.6.0 // indirect
gopkg.in/warnings.v0 v0.1.2 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
go.sum
+413 -267
View File
File diff suppressed because it is too large Load Diff
hack/Dockerfile.protos
+21 -10
View File
@@ -1,24 +1,35 @@
# trufflesecurity/protos:1.18-0
# trufflesecurity/protos:1.22
FROM golang:1.21-bullseye
FROM golang:1.22-bullseye
ARG TARGETARCH
ARG TARGETOS
ENV PROTOC_VER=3.20.0
ENV GORELEASER_VER=v1.7.0
ENV SQLBOILER_VER=v4.9.0
ENV PROTOC_GEN_GO_VER=v1.5.2
ENV PROTOC_VER=25.3
ENV PROTOC_GEN_GO_VER=v1.5.4
ENV PROTOC_GEN_VALIDATE_VER=v1.0.4
ENV GORELEASER_VERSION=1.19.2
RUN echo "building $TARGETARCH"
RUN apt-get update; apt-get install -y --no-install-recommends git unzip && rm -rf /var/lib/apt/lists/*
RUN go install github.com/dustin-decker/quill/cmd/[email protected]
RUN apt-get update; apt-get install apt-transport-https ca-certificates curl gnupg lsb-release -y; \
curl -fsSL https://download.docker.com/linux/debian/gpg | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg; \
echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/debian \
$(lsb_release -cs) stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null; \
apt-get update; apt-get install -y --no-install-recommends python3-pip docker-ce-cli docker-buildx-plugin docker-compose-plugin \
git netbase wget upx unzip && rm -rf /var/lib/apt/lists/*
RUN pip3 install --upgrade setuptools pip
RUN set -e; \
arch=$(echo $TARGETARCH | sed -e s/amd64/x86_64/ -e s/arm64/aarch_64/); \
wget -q https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VER}/protoc-${PROTOC_VER}-${TARGETOS}-${arch}.zip && unzip protoc-${PROTOC_VER}-${TARGETOS}-${arch}.zip -d /usr/local
RUN curl -fsSL https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add -; echo "deb https://packages.cloud.google.com/apt cloud-sdk main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list; apt-get update; apt-get install -y google-cloud-cli
RUN echo 'deb [trusted=yes] https://repo.goreleaser.com/apt/ /' | tee /etc/apt/sources.list.d/goreleaser.list; \
apt-get update; apt-get install -y goreleaser=${GORELEASER_VERSION} && rm -rf /var/lib/apt/lists/*
RUN go install "github.com/golang/protobuf/protoc-gen-go@${PROTOC_GEN_GO_VER}"
RUN GO111MODULE=off go get -d "github.com/envoyproxy/protoc-gen-validate" && \
RUN go install gotest.tools/gotestsum@latest
RUN git clone https://github.com/envoyproxy/protoc-gen-validate $GOPATH/src/github.com/envoyproxy/protoc-gen-validate && \
cd $GOPATH/src/github.com/envoyproxy/protoc-gen-validate && \
git checkout v0.6.7 && \
git checkout ${PROTOC_GEN_VALIDATE_VER} && \
ln -s /usr/local/protoc/include/google google && \
make build
RUN go install github.com/chrusty/protoc-gen-jsonschema/cmd/protoc-gen-jsonschema@latest
CMD ["bash"]
hack/docs/Adding_Detectors_Internal.md
+12 -6
View File
@@ -46,15 +46,20 @@ If you think that something should be included outside of these guidelines, plea
### Adding New Token Formats to an Existing Scanner
In some instances, services will update their token format, requiring a new regex to properly detect secrets in addition to supporting the previous token format. Accomodating this can be done without adding a net-new detector. [We provide a `Versioner` interface](https://github.com/trufflesecurity/trufflehog/blob/e18cfd5e0af1469a9f05b8d5732bcc94c39da49c/pkg/detectors/detectors.go#L30) that can be implemented.
In some instances, services will update their token format, requiring a new regex to properly detect secrets in addition to supporting the previous token format. Accommodating this can be done without adding a net-new detector. [We provide a `Versioner` interface](https://github.com/trufflesecurity/trufflehog/blob/e18cfd5e0af1469a9f05b8d5732bcc94c39da49c/pkg/detectors/detectors.go#L30) that can be implemented.
1. Create a copy of the package and append `_v2` to the package and file names. Ex: `<packagename>/` -> `<packagename>_v2`, `<packagename>.go` -> `<packagename>_v2.go`
1. Create two new directories `v1` and `v2`. Move the existing detector and tests into `v1`, and add new files to `v2`.
Ex: `<packagename>/<old_files>` -> `<packagename>/v1/<old_files>`, `<packagename>/v2/<new_files>`
Note: Be sure to update the tests to reference the new secret values in GSM, or the tests will fail.
2. Implement the `Versioner` interface. [GitHub example implementation.](/pkg/detectors/github_old/github_old.go#L22)
2. Implement the `Versioner` interface. [GitHub example implementation.](/pkg/detectors/github/v1/github_old.go#L23)
3. Proceed from step 3 of [Creating a new Secret Scanner](#creating-a-new-secret-scanner)
3. Add a 'version' field in ExtraData for both existing and new detector versions.
4. Update the existing detector in DefaultDetectors in `/pkg/engine/defaults.go`
5. Proceed from step 3 of [Creating a new Secret Scanner](#creating-a-new-secret-scanner)
### Creating a new Secret Scanner
@@ -82,7 +87,8 @@ Note: Be sure to update the tests to reference the new secret values in GSM, or
4. Found and unverified (indeterminately due to an unexpected API response)
5. Not found
6. Any false positive cases that you come across
5. Create a merge request for review. CI tests must be passing.
5. Add your new detector to DefaultDetectors in `/pkg/engine/defaults.go`
6. Create a merge request for review. CI tests must be passing.
## Addendum
@@ -92,7 +98,7 @@ There are two types of reasons that secret verification can fail:
* The candidate secret is not actually a valid secret.
* Something went wrong in the process unrelated to the candidate secret, such as a transient network error or an unexpected API response.
In Trufflehog parlance, the first type of verification response is called _determinate_ and the second type is called _indeterminate_. Verification code should distinguish between the two by returning an error object in the result struct **only** for indeterminate failures. In general, a verifier should return an error (indicating an indeterminate failure) in all cases that haven't been explicitly identified as determinate failure states.
In TruffleHog parlance, the first type of verification response is called _determinate_ and the second type is called _indeterminate_. Verification code should distinguish between the two by returning an error object in the result struct **only** for indeterminate failures. In general, a verifier should return an error (indicating an indeterminate failure) in all cases that haven't been explicitly identified as determinate failure states.
For example, consider a hypothetical authentication endpoint that returns `200 OK` for valid credentials and `403 Forbidden` for invalid credentials. The verifier for this endpoint could make an HTTP request and use the response status code to decide what to return:
* A `200` response would indicate that verification succeeded. (Or maybe any `2xx` response.)
hack/docs/Adding_Detectors_external.md
+13 -7
View File
@@ -42,15 +42,20 @@ If you think that something should be included outside of these guidelines, plea
### Adding New Token Formats to an Existing Scanner
In some instances, services will update their token format, requiring a new regex to properly detect secrets in addition to supporting the previous token format. Accomodating this can be done without adding a net-new detector. [We provide a `Versioner` interface](https://github.com/trufflesecurity/trufflehog/blob/e18cfd5e0af1469a9f05b8d5732bcc94c39da49c/pkg/detectors/detectors.go#L30) that can be implemented.
In some instances, services will update their token format, requiring a new regex to properly detect secrets in addition to supporting the previous token format. Accommodating this can be done without adding a net-new detector. [We provide a `Versioner` interface](https://github.com/trufflesecurity/trufflehog/blob/e18cfd5e0af1469a9f05b8d5732bcc94c39da49c/pkg/detectors/detectors.go#L30) that can be implemented.
1. Create a copy of the package and append `_v2` to the package and file names. Ex: `<packagename>/` -> `<packagename>_v2`, `<packagename>.go` -> `<packagename>_v2.go`
1. Create two new directories `v1` and `v2`. Move the existing detector and tests into `v1`, and add new files to `v2`.
Ex: `<packagename>/<old_files>` -> `<packagename>/v1/<old_files>`, `<packagename>/v2/<new_files>`
Note: Be sure to update the tests to reference the new secret values in GSM, or the tests will fail.
2. Implement the `Versioner` interface. [GitHub example implementation.](/pkg/detectors/github_old/github_old.go#L22)
2. Implement the `Versioner` interface. [GitHub example implementation.](/pkg/detectors/github_old/github_old.go#L23)
3. Proceed from step 3 of [Creating a new Secret Scanner](#creating-a-new-secret-scanner)
3. Add a 'version' field in ExtraData for both existing and new detector versions.
4. Update the existing detector in DefaultDetectors in `/pkg/engine/defaults.go`
5. Proceed from step 3 of [Creating a new Secret Scanner](#creating-a-new-secret-scanner)
### Creating a new Secret Detector
@@ -77,8 +82,9 @@ Note: Be sure to update the tests to reference the new secret values in GSM, or
1. Update the pattern regex and keywords. Try iterating with [regex101.com](http://regex101.com/).
2. Update the verifier code to use a non-destructive API call that can determine whether the secret is valid or not.
* Make sure you understand [verification indeterminacy](#verification-indeterminacy).
3. Create a [test for the detector](#testing-the-detector)
4. Create a pull request for review.
3. Create a [test for the detector](#testing-the-detector).
4. Add your new detector to DefaultDetectors in `/pkg/engine/defaults.go`.
5. Create a pull request for review.
### Testing the Detector
To ensure the quality of your PR, make sure your tests are passing with verified credentials.
@@ -132,7 +138,7 @@ There are two types of reasons that secret verification can fail:
* The candidate secret is not actually a valid secret.
* Something went wrong in the process unrelated to the candidate secret, such as a transient network error or an unexpected API response.
In Trufflehog parlance, the first type of verification response is called _determinate_ and the second type is called _indeterminate_. Verification code should distinguish between the two by returning an error object in the result struct **only** for indeterminate failures. In general, a verifier should return an error (indicating an indeterminate failure) in all cases that haven't been explicitly identified as determinate failure states.
In TruffleHog parlance, the first type of verification response is called _determinate_ and the second type is called _indeterminate_. Verification code should distinguish between the two by returning an error object in the result struct **only** for indeterminate failures. In general, a verifier should return an error (indicating an indeterminate failure) in all cases that haven't been explicitly identified as determinate failure states.
For example, consider a hypothetical authentication endpoint that returns `200 OK` for valid credentials and `403 Forbidden` for invalid credentials. The verifier for this endpoint could make an HTTP request and use the response status code to decide what to return:
* A `200` response would indicate that verification succeeded. (Or maybe any `2xx` response.)
main.go
+88 -15
View File
@@ -38,18 +38,20 @@ import (
)
var (
cli = kingpin.New("TruffleHog", "TruffleHog is a tool for finding credentials.")
cmd string
debug = cli.Flag("debug", "Run in debug mode.").Bool()
trace = cli.Flag("trace", "Run in trace mode.").Bool()
profile = cli.Flag("profile", "Enables profiling and sets a pprof and fgprof server on :18066.").Bool()
localDev = cli.Flag("local-dev", "Hidden feature to disable overseer for local dev.").Hidden().Bool()
jsonOut = cli.Flag("json", "Output in JSON format.").Short('j').Bool()
jsonLegacy = cli.Flag("json-legacy", "Use the pre-v3.0 JSON format. Only works with git, gitlab, and github sources.").Bool()
gitHubActionsFormat = cli.Flag("github-actions", "Output in GitHub Actions format.").Bool()
concurrency = cli.Flag("concurrency", "Number of concurrent workers.").Default(strconv.Itoa(runtime.NumCPU())).Int()
noVerification = cli.Flag("no-verification", "Don't verify the results.").Bool()
onlyVerified = cli.Flag("only-verified", "Only output verified results.").Bool()
cli = kingpin.New("TruffleHog", "TruffleHog is a tool for finding credentials.")
cmd string
debug = cli.Flag("debug", "Run in debug mode.").Bool()
trace = cli.Flag("trace", "Run in trace mode.").Bool()
profile = cli.Flag("profile", "Enables profiling and sets a pprof and fgprof server on :18066.").Bool()
localDev = cli.Flag("local-dev", "Hidden feature to disable overseer for local dev.").Hidden().Bool()
jsonOut = cli.Flag("json", "Output in JSON format.").Short('j').Bool()
jsonLegacy = cli.Flag("json-legacy", "Use the pre-v3.0 JSON format. Only works with git, gitlab, and github sources.").Bool()
gitHubActionsFormat = cli.Flag("github-actions", "Output in GitHub Actions format.").Bool()
concurrency = cli.Flag("concurrency", "Number of concurrent workers.").Default(strconv.Itoa(runtime.NumCPU())).Int()
noVerification = cli.Flag("no-verification", "Don't verify the results.").Bool()
onlyVerified = cli.Flag("only-verified", "Only output verified results.").Bool()
results = cli.Flag("results", "Specifies which type(s) of results to output: verified, unknown, unverified. Defaults to all types.").Hidden().String()
allowVerificationOverlap = cli.Flag("allow-verification-overlap", "Allow verification of similar credentials across detectors").Bool()
filterUnverified = cli.Flag("filter-unverified", "Only output first unverified result per chunk per detector if there are more than one results.").Bool()
filterEntropy = cli.Flag("filter-entropy", "Filter unverified results with Shannon entropy. Start with 3.0.").Float64()
@@ -120,7 +122,8 @@ var (
s3ScanSecret = s3Scan.Flag("secret", "S3 secret used to authenticate. Can be provided with environment variable AWS_SECRET_ACCESS_KEY.").Envar("AWS_SECRET_ACCESS_KEY").String()
s3ScanSessionToken = s3Scan.Flag("session-token", "S3 session token used to authenticate temporary credentials. Can be provided with environment variable AWS_SESSION_TOKEN.").Envar("AWS_SESSION_TOKEN").String()
s3ScanCloudEnv = s3Scan.Flag("cloud-environment", "Use IAM credentials in cloud environment.").Bool()
s3ScanBuckets = s3Scan.Flag("bucket", "Name of S3 bucket to scan. You can repeat this flag.").Strings()
s3ScanBuckets = s3Scan.Flag("bucket", "Name of S3 bucket to scan. You can repeat this flag. Incompatible with --ignore-bucket.").Strings()
s3ScanIgnoreBuckets = s3Scan.Flag("ignore-bucket", "Name of S3 bucket to ignore. You can repeat this flag. Incompatible with --bucket.").Strings()
s3ScanMaxObjectSize = s3Scan.Flag("max-object-size", "Maximum size of objects to scan. Objects larger than this will be skipped. (Byte units eg. 512B, 2KB, 4MB)").Default("250MB").Bytes()
gcsScan = cli.Command("gcs", "Find credentials in GCS buckets.")
@@ -150,6 +153,20 @@ var (
travisCiScan = cli.Command("travisci", "Scan TravisCI")
travisCiScanToken = travisCiScan.Flag("token", "TravisCI token. Can also be provided with environment variable").Envar("TRAVISCI_TOKEN").Required().String()
// Postman is hidden for now until we get more feedback from the community.
postmanScan = cli.Command("postman", "Scan Postman")
postmanToken = postmanScan.Flag("token", "Postman token. Can also be provided with environment variable").Envar("POSTMAN_TOKEN").String()
postmanWorkspaces = postmanScan.Flag("workspace", "Postman workspace to scan. You can repeat this flag.").Strings()
postmanCollections = postmanScan.Flag("collection", "Postman collection to scan. You can repeat this flag.").Strings()
postmanEnvironments = postmanScan.Flag("environment", "Postman environment to scan. You can repeat this flag.").Strings()
postmanIncludeCollections = postmanScan.Flag("include-collections", "Collections to include in scan. You can repeat this flag.").Strings()
postmanIncludeEnvironments = postmanScan.Flag("include-environments", "Environments to include in scan. You can repeat this flag.").Strings()
postmanExcludeCollections = postmanScan.Flag("exclude-collections", "Collections to exclude from scan. You can repeat this flag.").Strings()
postmanExcludeEnvironments = postmanScan.Flag("exclude-environments", "Environments to exclude from scan. You can repeat this flag.").Strings()
postmanWorkspacePaths = postmanScan.Flag("workspace-paths", "Path to Postman workspaces.").Strings()
postmanCollectionPaths = postmanScan.Flag("collection-paths", "Path to Postman collections.").Strings()
postmanEnvironmentPaths = postmanScan.Flag("environment-paths", "Path to Postman environments.").Strings()
)
func init() {
@@ -404,8 +421,19 @@ func run(state overseer.State) {
if *jobReportFile != nil {
jobReportWriter = *jobReportFile
}
// Parse --results flag.
if *onlyVerified {
r := "verified"
results = &r
}
parsedResults, err := parseResults(results)
if err != nil {
logFatal(err, "failed to configure results flag")
}
e, err := engine.Start(ctx,
engine.WithConcurrency(uint8(*concurrency)),
engine.WithConcurrency(*concurrency),
engine.WithDecoders(decoders.DefaultDecoders()...),
engine.WithDetectors(engine.DefaultDetectors()...),
engine.WithDetectors(conf.Detectors...),
@@ -414,7 +442,7 @@ func run(state overseer.State) {
engine.WithFilterDetectors(excludeFilter),
engine.WithFilterDetectors(endpointCustomizer),
engine.WithFilterUnverified(*filterUnverified),
engine.WithOnlyVerified(*onlyVerified),
engine.WithResults(parsedResults),
engine.WithPrintAvgDetectorTime(*printAvgDetectorTime),
engine.WithPrinter(printer),
engine.WithFilterEntropy(*filterEntropy),
@@ -504,6 +532,7 @@ func run(state overseer.State) {
Secret: *s3ScanSecret,
SessionToken: *s3ScanSessionToken,
Buckets: *s3ScanBuckets,
IgnoreBuckets: *s3ScanIgnoreBuckets,
Roles: *s3ScanRoleArns,
CloudCred: *s3ScanCloudEnv,
MaxObjectSize: int64(*s3ScanMaxObjectSize),
@@ -562,12 +591,32 @@ func run(state overseer.State) {
if err := e.ScanDocker(ctx, anyConn); err != nil {
logFatal(err, "Failed to scan Docker.")
}
case postmanScan.FullCommand():
cfg := sources.PostmanConfig{
Token: *postmanToken,
Workspaces: *postmanWorkspaces,
Collections: *postmanCollections,
Environments: *postmanEnvironments,
IncludeCollections: *postmanIncludeCollections,
IncludeEnvironments: *postmanIncludeEnvironments,
ExcludeCollections: *postmanExcludeCollections,
ExcludeEnvironments: *postmanExcludeEnvironments,
CollectionPaths: *postmanCollectionPaths,
WorkspacePaths: *postmanWorkspacePaths,
EnvironmentPaths: *postmanEnvironmentPaths,
}
if err := e.ScanPostman(ctx, cfg); err != nil {
logFatal(err, "Failed to scan Postman.")
}
}
// Wait for all workers to finish.
if err = e.Finish(ctx); err != nil {
logFatal(err, "engine failed to finish execution")
}
if err := cleantemp.CleanTempArtifacts(ctx); err != nil {
ctx.Logger().Error(err, "error cleaning temp artifacts")
}
metrics := e.GetMetrics()
// Print results.
@@ -589,6 +638,30 @@ func run(state overseer.State) {
}
}
// parseResults ensures that users provide valid CSV input to `--results`.
//
// This is a work-around to kingpin not supporting CSVs.
// See: https://github.com/trufflesecurity/trufflehog/pull/2372#issuecomment-1983868917
func parseResults(input *string) (map[string]struct{}, error) {
if *input == "" {
return nil, nil
}
var (
values = strings.Split(strings.ToLower(*input), ",")
results = make(map[string]struct{}, 3)
)
for _, value := range values {
switch value {
case "verified", "unknown", "unverified":
results[value] = struct{}{}
default:
return nil, fmt.Errorf("invalid value '%s', valid values are 'verified,unknown,unverified'", value)
}
}
return results, nil
}
// logFatalFunc returns a log.Fatal style function. Calling the returned
// function will terminate the program without cleanup.
func logFatalFunc(logger logr.Logger) func(error, string, ...any) {
pkg/common/http.go
+43 -3
View File
@@ -115,6 +115,39 @@ func ConstantResponseHttpClient(statusCode int, body string) *http.Client {
}
}
// ClientOption configures how we set up the client.
type ClientOption func(*retryablehttp.Client)
// WithCheckRetry allows setting a custom CheckRetry policy.
func WithCheckRetry(cr retryablehttp.CheckRetry) ClientOption {
return func(c *retryablehttp.Client) { c.CheckRetry = cr }
}
// WithBackoff allows setting a custom backoff policy.
func WithBackoff(b retryablehttp.Backoff) ClientOption {
return func(c *retryablehttp.Client) { c.Backoff = b }
}
// WithTimeout allows setting a custom timeout.
func WithTimeout(timeout time.Duration) ClientOption {
return func(c *retryablehttp.Client) { c.HTTPClient.Timeout = timeout }
}
// WithMaxRetries allows setting a custom maximum number of retries.
func WithMaxRetries(retries int) ClientOption {
return func(c *retryablehttp.Client) { c.RetryMax = retries }
}
// WithRetryWaitMin allows setting a custom minimum retry wait.
func WithRetryWaitMin(wait time.Duration) ClientOption {
return func(c *retryablehttp.Client) { c.RetryWaitMin = wait }
}
// WithRetryWaitMax allows setting a custom maximum retry wait.
func WithRetryWaitMax(wait time.Duration) ClientOption {
return func(c *retryablehttp.Client) { c.RetryWaitMax = wait }
}
func PinnedRetryableHttpClient() *http.Client {
httpClient := retryablehttp.NewClient()
httpClient.Logger = nil
@@ -136,21 +169,28 @@ func PinnedRetryableHttpClient() *http.Client {
return httpClient.StandardClient()
}
func RetryableHttpClient() *http.Client {
func RetryableHTTPClient(opts ...ClientOption) *http.Client {
httpClient := retryablehttp.NewClient()
httpClient.RetryMax = 3
httpClient.Logger = nil
httpClient.HTTPClient.Timeout = 3 * time.Second
httpClient.HTTPClient.Transport = NewCustomTransport(nil)
for _, opt := range opts {
opt(httpClient)
}
return httpClient.StandardClient()
}
func RetryableHttpClientTimeout(timeOutSeconds int64) *http.Client {
func RetryableHTTPClientTimeout(timeOutSeconds int64, opts ...ClientOption) *http.Client {
httpClient := retryablehttp.NewClient()
httpClient.RetryMax = 3
httpClient.Logger = nil
httpClient.HTTPClient.Timeout = time.Duration(timeOutSeconds) * time.Second
httpClient.HTTPClient.Transport = NewCustomTransport(nil)
for _, opt := range opts {
opt(httpClient)
}
return httpClient.StandardClient()
}
pkg/common/http_test.go
+237
View File
@@ -0,0 +1,237 @@
package common
import (
"context"
"math"
"net/http"
"net/http/httptest"
"testing"
"time"
"github.com/hashicorp/go-retryablehttp"
"github.com/stretchr/testify/assert"
"golang.org/x/exp/slices"
)
func TestRetryableHTTPClientCheckRetry(t *testing.T) {
testCases := []struct {
name string
responseStatus int
checkRetry retryablehttp.CheckRetry
expectedRetries int
}{
{
name: "Retry on 500 status, give up after 3 retries",
responseStatus: http.StatusInternalServerError, // Server error status
checkRetry: func(ctx context.Context, resp *http.Response, err error) (bool, error) {
// The underlying transport will retry on 500 status.
if resp.StatusCode == http.StatusInternalServerError {
return true, nil
}
return false, nil
},
expectedRetries: 3,
},
{
name: "No retry on 400 status",
responseStatus: http.StatusBadRequest, // Client error status
checkRetry: func(ctx context.Context, resp *http.Response, err error) (bool, error) {
// Do not retry on client errors.
return false, nil
},
expectedRetries: 0,
},
{
name: "Retry on 429 status, give up after 3 retries",
responseStatus: http.StatusTooManyRequests,
checkRetry: func(ctx context.Context, resp *http.Response, err error) (bool, error) {
// The underlying transport will retry on 429 status.
if resp.StatusCode == http.StatusTooManyRequests {
return true, nil
}
return false, nil
},
expectedRetries: 3,
},
}
for _, tc := range testCases {
tc := tc
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
var retryCount int
// Do not count the initial request as a retry.
i := 0
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if i != 0 {
retryCount++
}
i++
w.WriteHeader(tc.responseStatus)
}))
defer server.Close()
ctx := context.Background()
client := RetryableHTTPClient(WithCheckRetry(tc.checkRetry), WithTimeout(10*time.Millisecond), WithRetryWaitMin(1*time.Millisecond))
req, err := http.NewRequestWithContext(ctx, http.MethodGet, server.URL, nil)
assert.NoError(t, err)
// Bad linter, there is no body to close.
_, err = client.Do(req) //nolint:bodyclose
if err != nil && slices.Contains([]int{http.StatusInternalServerError, http.StatusTooManyRequests}, tc.responseStatus) {
// The underlying transport will retry on 500 and 429 status.
assert.Error(t, err)
}
assert.Equal(t, tc.expectedRetries, retryCount, "Retry count does not match expected")
})
}
}
func TestRetryableHTTPClientMaxRetry(t *testing.T) {
testCases := []struct {
name string
responseStatus int
maxRetries int
expectedRetries int
}{
{
name: "Max retries with 500 status",
responseStatus: http.StatusInternalServerError,
maxRetries: 2,
expectedRetries: 2,
},
{
name: "Max retries with 429 status",
responseStatus: http.StatusTooManyRequests,
maxRetries: 1,
expectedRetries: 1,
},
{
name: "Max retries with 200 status",
responseStatus: http.StatusOK,
maxRetries: 3,
expectedRetries: 0,
},
{
name: "Max retries with 400 status",
responseStatus: http.StatusBadRequest,
maxRetries: 3,
expectedRetries: 0,
},
{
name: "Max retries with 401 status",
responseStatus: http.StatusUnauthorized,
maxRetries: 3,
expectedRetries: 0,
},
}
for _, tc := range testCases {
tc := tc
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
var retryCount int
i := 0
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if i != 0 {
retryCount++
}
i++
w.WriteHeader(tc.responseStatus)
}))
defer server.Close()
client := RetryableHTTPClient(
WithMaxRetries(tc.maxRetries),
WithTimeout(10*time.Millisecond),
WithRetryWaitMin(1*time.Millisecond),
)
ctx := context.Background()
req, err := http.NewRequestWithContext(ctx, http.MethodGet, server.URL, nil)
assert.NoError(t, err)
// Bad linter, there is no body to close.
_, err = client.Do(req) //nolint:bodyclose
if err != nil && tc.responseStatus == http.StatusOK {
assert.Error(t, err)
}
assert.Equal(t, tc.expectedRetries, retryCount, "Retry count does not match expected")
})
}
}
func TestRetryableHTTPClientBackoff(t *testing.T) {
testCases := []struct {
name string
responseStatus int
expectedRetries int
backoffPolicy retryablehttp.Backoff
expectedBackoffs []time.Duration
}{
{
name: "Custom backoff on 500 status",
responseStatus: http.StatusInternalServerError,
expectedRetries: 3,
backoffPolicy: func(min, max time.Duration, attemptNum int, resp *http.Response) time.Duration {
switch attemptNum {
case 1:
return 1 * time.Millisecond
case 2:
return 2 * time.Millisecond
case 3:
return 4 * time.Millisecond
default:
return max
}
},
expectedBackoffs: []time.Duration{1 * time.Millisecond, 2 * time.Millisecond, 4 * time.Millisecond},
},
}
for _, tc := range testCases {
tc := tc
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
var actualBackoffs []time.Duration
var lastTime time.Time
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
now := time.Now()
if !lastTime.IsZero() {
actualBackoffs = append(actualBackoffs, now.Sub(lastTime))
}
lastTime = now
w.WriteHeader(tc.responseStatus)
}))
defer server.Close()
ctx := context.Background()
client := RetryableHTTPClient(
WithBackoff(tc.backoffPolicy),
WithTimeout(10*time.Millisecond),
WithRetryWaitMin(1*time.Millisecond),
WithRetryWaitMax(10*time.Millisecond),
)
req, err := http.NewRequestWithContext(ctx, http.MethodGet, server.URL, nil)
assert.NoError(t, err)
_, err = client.Do(req) //nolint:bodyclose
assert.Error(t, err, "Expected error due to 500 status")
assert.Len(t, actualBackoffs, tc.expectedRetries, "Unexpected number of backoffs")
for i, expectedBackoff := range tc.expectedBackoffs {
if i < len(actualBackoffs) {
// Allow some deviation in timing due to processing delays.
assert.Less(t, math.Abs(float64(actualBackoffs[i]-expectedBackoff)), float64(15*time.Millisecond), "Unexpected backoff duration")
}
}
})
}
}
pkg/decoders/base64.go
+13 -6
View File
@@ -3,6 +3,7 @@ package decoders
import (
"bytes"
"encoding/base64"
"unicode"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
@@ -33,15 +34,12 @@ func (d *Base64) FromChunk(chunk *sources.Chunk) *DecodableChunk {
for _, str := range encodedSubstrings {
dec, err := base64.StdEncoding.DecodeString(str)
if err == nil {
if len(dec) > 0 {
decodedSubstrings[str] = dec
}
continue
if err == nil && len(dec) > 0 && isASCII(dec) {
decodedSubstrings[str] = dec
}
dec, err = base64.RawURLEncoding.DecodeString(str)
if err == nil && len(dec) > 0 {
if err == nil && len(dec) > 0 && isASCII(dec) {
decodedSubstrings[str] = dec
}
}
@@ -69,6 +67,15 @@ func (d *Base64) FromChunk(chunk *sources.Chunk) *DecodableChunk {
return nil
}
func isASCII(b []byte) bool {
for i := 0; i < len(b); i++ {
if b[i] > unicode.MaxASCII {
return false
}
}
return true
}
func getSubstringsOfCharacterSet(data []byte, threshold int, charsetMapping [128]bool, endChars string) []string {
if len(data) == 0 {
return nil
pkg/decoders/base64_test.go
+7
View File
@@ -123,7 +123,14 @@ func TestBase64_FromChunk(t *testing.T) {
Data: []byte(`b64urlsafe-test-secret-underscores??`),
},
},
{
name: "invalid base64 string",
chunk: &sources.Chunk{
Data: []byte(`a3d3fa7c2bb99e469ba55e5834ce79ee4853a8a3`),
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
d := &Base64{}
pkg/decoders/decoders.go
+1
View File
@@ -11,6 +11,7 @@ func DefaultDecoders() []Decoder {
&UTF8{},
&Base64{},
&UTF16{},
&EscapedUnicode{},
}
}
pkg/decoders/escaped_unicode.go
+127
View File
@@ -0,0 +1,127 @@
package decoders
import (
"regexp"
"strconv"
"unicode/utf8"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
)
type EscapedUnicode struct{}
var _ Decoder = (*EscapedUnicode)(nil)
// It might be advantageous to limit these to a subset of acceptable characters, similar to base64.
// https://dencode.com/en/string/unicode-escape
var (
// Standard Unicode notation.
//https://unicode.org/standard/principles.html
codePointPat = regexp.MustCompile(`\bU\+([a-fA-F0-9]{4}).?`)
// Common escape sequence used in programming languages.
escapePat = regexp.MustCompile(`(?i:\\{1,2}u)([a-fA-F0-9]{4})`)
)
func (d *EscapedUnicode) FromChunk(chunk *sources.Chunk) *DecodableChunk {
if chunk == nil || len(chunk.Data) == 0 {
return nil
}
matched := false
if codePointPat.Match(chunk.Data) {
matched = true
chunk.Data = decodeCodePoint(chunk.Data)
}
if escapePat.Match(chunk.Data) {
matched = true
chunk.Data = decodeEscaped(chunk.Data)
}
if matched {
decodableChunk := &DecodableChunk{
DecoderType: detectorspb.DecoderType_ESCAPED_UNICODE,
Chunk: chunk,
}
return decodableChunk
} else {
return nil
}
}
// Unicode characters are encoded as 1 to 4 bytes per rune.
const maxBytesPerRune = 4
const spaceChar = byte(' ')
func decodeCodePoint(input []byte) []byte {
// Find all Unicode escape sequences in the input byte slice
indices := codePointPat.FindAllSubmatchIndex(input, -1)
// Iterate over found indices in reverse order to avoid modifying the slice length
utf8Bytes := make([]byte, maxBytesPerRune)
for i := len(indices) - 1; i >= 0; i-- {
matches := indices[i]
startIndex := matches[0]
endIndex := matches[1]
hexStartIndex := matches[2]
hexEndIndex := matches[3]
// If the input is like `U+1234 U+5678` we should replace `U+1234 `.
// Otherwise, we should only replace `U+1234`.
if endIndex != hexEndIndex && input[endIndex-1] != spaceChar {
endIndex = endIndex - 1
}
// Extract the hexadecimal value from the escape sequence
hexValue := string(input[hexStartIndex:hexEndIndex])
// Parse the hexadecimal value to an integer
unicodeInt, err := strconv.ParseInt(hexValue, 16, 32)
if err != nil {
// If there's an error, continue to the next escape sequence
continue
}
// Convert the Unicode code point to a UTF-8 representation
utf8Len := utf8.EncodeRune(utf8Bytes, rune(unicodeInt))
// Replace the escape sequence with the UTF-8 representation
input = append(input[:startIndex], append(utf8Bytes[:utf8Len], input[endIndex:]...)...)
}
return input
}
func decodeEscaped(input []byte) []byte {
// Find all Unicode escape sequences in the input byte slice
indices := escapePat.FindAllSubmatchIndex(input, -1)
// Iterate over found indices in reverse order to avoid modifying the slice length
utf8Bytes := make([]byte, maxBytesPerRune)
for i := len(indices) - 1; i >= 0; i-- {
matches := indices[i]
startIndex := matches[0]
hexStartIndex := matches[2]
endIndex := matches[3]
// Extract the hexadecimal value from the escape sequence
hexValue := string(input[hexStartIndex:endIndex])
// Parse the hexadecimal value to an integer
unicodeInt, err := strconv.ParseInt(hexValue, 16, 32)
if err != nil {
// If there's an error, continue to the next escape sequence
continue
}
// Convert the Unicode code point to a UTF-8 representation
utf8Len := utf8.EncodeRune(utf8Bytes, rune(unicodeInt))
// Replace the escape sequence with the UTF-8 representation
input = append(input[:startIndex], append(utf8Bytes[:utf8Len], input[endIndex:]...)...)
}
return input
}
pkg/decoders/escaped_unicode_test.go
+86
View File
@@ -0,0 +1,86 @@
package decoders
import (
"testing"
"github.com/kylelemons/godebug/pretty"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
)
func TestUnicodeEscape_FromChunk(t *testing.T) {
tests := []struct {
name string
chunk *sources.Chunk
want *sources.Chunk
wantErr bool
}{
// U+1234
{
name: "[notation] all escaped",
chunk: &sources.Chunk{
Data: []byte("U+0074 U+006f U+006b U+0065 U+006e U+003a U+0020 U+0022 U+0067 U+0068 U+0070 U+005f U+0049 U+0077 U+0064 U+004d U+0078 U+0039 U+0057 U+0046 U+0057 U+0052 U+0052 U+0066 U+004d U+0068 U+0054 U+0059 U+0069 U+0061 U+0056 U+006a U+005a U+0037 U+0038 U+004a U+0066 U+0075 U+0061 U+006d U+0076 U+006e U+0030 U+0059 U+0057 U+0052 U+004d U+0030 U+0022"),
},
want: &sources.Chunk{
Data: []byte("token: \"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\""),
},
},
// \u1234
{
name: "[slash] all escaped",
chunk: &sources.Chunk{
Data: []byte("\\u0074\\u006f\\u006b\\u0065\\u006e\\u003a\\u0020\\u0022\\u0067\\u0068\\u0070\\u005f\\u0049\\u0077\\u0064\\u004d\\u0078\\u0039\\u0057\\u0046\\u0057\\u0052\\u0052\\u0066\\u004d\\u0068\\u0054\\u0059\\u0069\\u0061\\u0056\\u006a\\u005a\\u0037\\u0038\\u004a\\u0066\\u0075\\u0061\\u006d\\u0076\\u006e\\u0030\\u0059\\u0057\\u0052\\u004d\\u0030\\u0022"),
},
want: &sources.Chunk{
Data: []byte("token: \"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\""),
},
},
{
name: "[slash] mixed content",
chunk: &sources.Chunk{
Data: []byte("npm config set @trufflesec:registry=https://npm.pkg.github.com\nnpm config set //npm.pkg.github.com:_authToken=$'\\u0067hp_9ovSHEBCq0drG42yjoam76iNybtqLN25CgSf'"),
},
want: &sources.Chunk{
Data: []byte("npm config set @trufflesec:registry=https://npm.pkg.github.com\nnpm config set //npm.pkg.github.com:_authToken=$'ghp_9ovSHEBCq0drG42yjoam76iNybtqLN25CgSf'"),
},
},
{
name: "[slash] multiple slashes",
chunk: &sources.Chunk{
Data: []byte(`SameValue("hello","\\u0068el\\u006co"); // true`),
},
want: &sources.Chunk{
Data: []byte(`SameValue("hello","hello"); // true`),
},
},
// nothing
{
name: "no escaped",
chunk: &sources.Chunk{
Data: []byte(`-//npm.fontawesome.com/:_authToken=12345678-2323-1111-1111-12345670B312
+//npm.fontawesome.com/:_authToken=REMOVED_TOKEN`),
},
want: nil,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
d := &EscapedUnicode{}
got := d.FromChunk(tt.chunk)
if tt.want != nil {
if got == nil {
t.Fatal("got nil, did not want nil")
}
if diff := pretty.Compare(string(tt.want.Data), string(got.Data)); diff != "" {
t.Errorf("UnicodeEscape.FromChunk() %s diff: (-want +got)\n%s", tt.name, diff)
}
} else {
if got != nil {
t.Error("Expected nil chunk")
}
}
})
}
}
pkg/detectors/alchemy/alchemy.go
+5 -5
View File
@@ -85,15 +85,15 @@ func verifyMatch(ctx context.Context, client *http.Client, token string) (bool,
_ = res.Body.Close()
}()
if res.StatusCode >= 200 && res.StatusCode < 300 {
switch res.StatusCode {
case http.StatusOK:
// If the endpoint returns useful information, we can return it as a map.
return true, nil, nil
} else if res.StatusCode == 401 {
case http.StatusUnauthorized:
// The secret is determinately not verified (nothing to do)
return false, nil, nil
} else {
err = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
return false, nil, err
default:
return false, nil, fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
}
}
pkg/detectors/anthropic/anthropic.go
+59 -50
View File
@@ -5,10 +5,11 @@ import (
"context"
"encoding/json"
"fmt"
regexp "github.com/wasilibs/go-re2"
"net/http"
"strings"
regexp "github.com/wasilibs/go-re2"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
@@ -33,13 +34,6 @@ func (s Scanner) Keywords() []string {
return []string{"sk-ant-api03"}
}
type response struct {
Error struct {
Type string `json:"type"`
Message string `json:"message"`
} `json:"error"`
}
// FromData will find and optionally verify Anthropic secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
@@ -62,59 +56,74 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
if client == nil {
client = defaultClient
}
body := map[string]interface{}{
"prompt": "test",
}
bodyBytes, err := json.Marshal(body)
if err != nil {
continue
}
req, err := http.NewRequestWithContext(ctx, "POST", "https://api.anthropic.com/v1/complete", bytes.NewReader(bodyBytes))
if err != nil {
continue
}
req.Header.Set("x-api-key", resMatch)
req.Header.Set("Content-Type", "application/json")
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode == http.StatusBadRequest {
var resp response
if err = json.NewDecoder(res.Body).Decode(&resp); err != nil {
err = fmt.Errorf("unexpected HTTP response body: %w", err)
s1.SetVerificationError(err, resMatch)
continue
}
if resp.Error.Message == "max_tokens_to_sample: field required" {
// The secret is verified
// Anthropic returns 400 on a request containing a valid API key,
// when not containing a valid model field
s1.Verified = true
}
} else if res.StatusCode == 401 {
// The secret is determinately not verified (nothing to do)
// Anthropic returns 401 on all requests not containing a valid x-api-key header
} else {
err = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
s1.SetVerificationError(err, resMatch)
}
} else {
s1.SetVerificationError(err, resMatch)
}
isVerified, err := verifyToken(ctx, client, resMatch)
s1.Verified = isVerified
s1.SetVerificationError(err, resMatch)
}
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if !s1.Verified && detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
results = append(results, s1)
}
return results, nil
}
type response struct {
Error struct {
Message string `json:"message"`
} `json:"error"`
}
func verifyToken(ctx context.Context, client *http.Client, apiKey string) (bool, error) {
body := map[string]any{
"model": "claude-3-opus-20240229",
"max_tokens": 1024,
"messages": []map[string]string{
{"role": "user", "content": "Hello, world"},
},
}
bodyBytes, err := json.Marshal(body)
if err != nil {
return false, nil
}
// https://docs.anthropic.com/claude/reference/messages_post
req, err := http.NewRequestWithContext(ctx, http.MethodPost, "https://api.anthropic.com/v1/messages", bytes.NewReader(bodyBytes))
if err != nil {
return false, nil
}
req.Header.Set("x-api-key", apiKey)
req.Header.Set("Content-Type", "application/json")
req.Header.Set("anthropic-version", "2023-06-01")
res, err := client.Do(req)
if err != nil {
return false, err
}
defer res.Body.Close()
switch res.StatusCode {
case http.StatusOK:
return true, nil
case http.StatusBadRequest:
var resp response
if err = json.NewDecoder(res.Body).Decode(&resp); err != nil {
return false, fmt.Errorf("unexpected HTTP response body: %w", err)
}
return true, nil
case http.StatusUnauthorized:
return false, nil
default:
return false, fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
}
}
func (s Scanner) Type() detectorspb.DetectorType {
return detectorspb.DetectorType_Anthropic
}
pkg/detectors/atera/atera.go
+1 -1
View File
@@ -20,7 +20,7 @@ var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"atera"}) + `\b([[0-9a-z]{32})\b`)
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"atera"}) + `\b([0-9a-z]{32})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
pkg/detectors/aws/aws.go
+108 -10
View File
@@ -16,6 +16,11 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/sns"
)
type scanner struct {
@@ -39,6 +44,39 @@ var resourceTypes = map[string]string{
"ASIA": "Temporary (AWS STS) access key IDs",
}
var thinkstCanaryList = map[string]struct{}{
"052310077262": {},
"171436882533": {},
"534261010715": {},
"595918472158": {},
"717712589309": {},
"819147034852": {},
"992382622183": {},
}
const thinkstMessage = "This is an AWS canary token generated at canarytokens.org, and was not set off; learn more here: https://trufflesecurity.com/canaries"
var thinkstKnockoffsCanaryList = map[string]struct{}{
"044858866125": {},
"251535659677": {},
"344043088457": {},
"351906852752": {},
"390477818340": {},
"426127672474": {},
"427150556519": {},
"439872796651": {},
"445142720921": {},
"465867158099": {},
"637958123769": {},
"693412236332": {},
"732624840810": {},
"735421457923": {},
"959235150393": {},
"982842642351": {},
}
const thinkstKnockoffsMessage = "This is an off brand AWS Canary inspired by canarytokens.org. It wasn't set off; learn more here: https://trufflesecurity.com/canaries"
func New(opts ...func(*scanner)) *scanner {
scanner := &scanner{
skipIDs: map[string]struct{}{},
@@ -85,7 +123,6 @@ func (s scanner) Keywords() []string {
"AKIA",
"ABIA",
"ACCA",
"ASIA",
}
}
@@ -137,7 +174,44 @@ func (s scanner) FromData(ctx context.Context, verify bool, data []byte) (result
},
}
if verify {
account, err := common.GetAccountNumFromAWSID(resIDMatch)
if err == nil {
s1.ExtraData["account"] = account
}
if _, ok := thinkstCanaryList[account]; ok {
s1.ExtraData["is_canary"] = "true"
s1.ExtraData["message"] = thinkstMessage
if verify {
verified, arn, err := s.verifyCanary(resIDMatch, resSecretMatch)
if verified {
s1.Verified = true
}
if arn != "" {
s1.ExtraData["arn"] = arn
}
if err != nil {
s1.SetVerificationError(err, resSecretMatch)
}
}
}
if _, ok := thinkstKnockoffsCanaryList[account]; ok {
s1.ExtraData["is_canary"] = "true"
s1.ExtraData["message"] = thinkstKnockoffsMessage
if verify {
verified, arn, err := s.verifyCanary(resIDMatch, resSecretMatch)
if verified {
s1.Verified = true
}
if arn != "" {
s1.ExtraData["arn"] = arn
}
if err != nil {
s1.SetVerificationError(err, resSecretMatch)
}
}
}
if verify && (s1.ExtraData["is_canary"] != "true") {
isVerified, extraData, verificationErr := s.verifyMatch(ctx, resIDMatch, resSecretMatch, true)
s1.Verified = isVerified
// It'd be good to log when calculated account value does not match
@@ -165,14 +239,6 @@ func (s scanner) FromData(ctx context.Context, verify bool, data []byte) (result
}
}
// If we haven't already found an account number for this ID (via API), calculate one.
if _, ok := s1.ExtraData["account"]; !ok {
account, err := common.GetAccountNumFromAWSID(resIDMatch)
if err == nil {
s1.ExtraData["account"] = account
}
}
results = append(results, s1)
// If we've found a verified match with this ID, we don't need to look for any more. So move on to the next ID.
if s1.Verified {
@@ -302,6 +368,38 @@ func (s scanner) verifyMatch(ctx context.Context, resIDMatch, resSecretMatch str
}
}
func (s scanner) verifyCanary(resIDMatch, resSecretMatch string) (bool, string, error) {
// Prep AWS Creds for SNS
sess := session.Must(session.NewSession(&aws.Config{
Region: aws.String("us-east-1"), // any region seems to work
Credentials: credentials.NewStaticCredentials(
resIDMatch,
resSecretMatch,
"",
),
HTTPClient: s.verificationClient,
}))
svc := sns.New(sess)
// Prep vars and Publish to SNS
_, err := svc.Publish(&sns.PublishInput{
Message: aws.String("foo"),
PhoneNumber: aws.String("1"),
})
if strings.Contains(err.Error(), "not authorized to perform") {
arn := strings.Split(err.Error(), "User: ")[1]
arn = strings.Split(arn, " is not authorized to perform: ")[0]
return true, arn, nil
} else if strings.Contains(err.Error(), "does not match the signature you provided") {
return false, "", nil
} else if strings.Contains(err.Error(), "status code: 403") {
return false, "", nil
} else {
return false, "", err
}
}
func awsCustomCleanResults(results []detectors.Result) []detectors.Result {
if len(results) == 0 {
return results
pkg/detectors/aws/aws_test.go
+85 -30
View File
@@ -17,6 +17,8 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
const canaryAccessKeyID = "AKIASP2TPHJSQH3FJRUX"
var unverifiedSecretClient = common.ConstantResponseHttpClient(403, `{"Error": {"Code": "InvalidClientTokenId"} }`)
func TestAWS_FromChunk(t *testing.T) {
@@ -61,11 +63,11 @@ func TestAWS_FromChunk(t *testing.T) {
Verified: true,
Redacted: "AKIASP2TPHJSQH3FJRUX",
ExtraData: map[string]string{
"resource_type": "Access key",
"rotation_guide": "https://howtorotate.com/docs/tutorials/aws/",
"account": "171436882533",
"arn": "arn:aws:iam::171436882533:user/canarytokens.com@@4dxkh0pdeop3bzu9zx5wob793",
"user_id": "AIDASP2TPHJSUFRSTTZX4",
"resource_type": "Access key",
"account": "171436882533",
"arn": "arn:aws:iam::171436882533:user/canarytokens.com@@4dxkh0pdeop3bzu9zx5wob793",
"is_canary": "true",
"message": "This is an AWS canary token generated at canarytokens.org, and was not set off; learn more here: https://trufflesecurity.com/canaries",
},
},
},
@@ -87,6 +89,8 @@ func TestAWS_FromChunk(t *testing.T) {
ExtraData: map[string]string{
"resource_type": "Access key",
"account": "171436882533",
"is_canary": "true",
"message": "This is an AWS canary token generated at canarytokens.org, and was not set off; learn more here: https://trufflesecurity.com/canaries",
},
},
},
@@ -119,6 +123,8 @@ func TestAWS_FromChunk(t *testing.T) {
ExtraData: map[string]string{
"resource_type": "Access key",
"account": "171436882533",
"is_canary": "true",
"message": "This is an AWS canary token generated at canarytokens.org, and was not set off; learn more here: https://trufflesecurity.com/canaries",
},
},
{
@@ -126,11 +132,11 @@ func TestAWS_FromChunk(t *testing.T) {
Verified: true,
Redacted: "AKIASP2TPHJSQH3FJRUX",
ExtraData: map[string]string{
"resource_type": "Access key",
"rotation_guide": "https://howtorotate.com/docs/tutorials/aws/",
"account": "171436882533",
"arn": "arn:aws:iam::171436882533:user/canarytokens.com@@4dxkh0pdeop3bzu9zx5wob793",
"user_id": "AIDASP2TPHJSUFRSTTZX4",
"resource_type": "Access key",
"account": "171436882533",
"arn": "arn:aws:iam::171436882533:user/canarytokens.com@@4dxkh0pdeop3bzu9zx5wob793",
"is_canary": "true",
"message": "This is an AWS canary token generated at canarytokens.org, and was not set off; learn more here: https://trufflesecurity.com/canaries",
},
},
},
@@ -161,24 +167,32 @@ func TestAWS_FromChunk(t *testing.T) {
Verified: true,
Redacted: "AKIASP2TPHJSQH3FJRUX",
ExtraData: map[string]string{
"resource_type": "Access key",
"rotation_guide": "https://howtorotate.com/docs/tutorials/aws/",
"account": "171436882533",
"arn": "arn:aws:iam::171436882533:user/canarytokens.com@@4dxkh0pdeop3bzu9zx5wob793",
"user_id": "AIDASP2TPHJSUFRSTTZX4",
"resource_type": "Access key",
"account": "171436882533",
"arn": "arn:aws:iam::171436882533:user/canarytokens.com@@4dxkh0pdeop3bzu9zx5wob793",
"is_canary": "true",
"message": "This is an AWS canary token generated at canarytokens.org, and was not set off; learn more here: https://trufflesecurity.com/canaries",
},
},
{
DetectorType: detectorspb.DetectorType_AWS,
Verified: false,
Redacted: inactiveID,
ExtraData: map[string]string{
"account": "171436882533",
"resource_type": "Access key",
"is_canary": "true",
"message": "This is an AWS canary token generated at canarytokens.org, and was not set off; learn more here: https://trufflesecurity.com/canaries",
},
},
},
wantErr: false,
},
{
name: "found, unverified, with leading +",
s: scanner{verificationClient: unverifiedSecretClient},
s: scanner{
verificationClient: unverifiedSecretClient,
},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a aws secret %s within aws %s but not valid", "+HaNv9cTwheDKGJaws/+BMF2GgybQgBWdhcOOdfF", id)), // the secret would satisfy the regex but not pass validation
@@ -192,6 +206,8 @@ func TestAWS_FromChunk(t *testing.T) {
ExtraData: map[string]string{
"resource_type": "Access key",
"account": "171436882533",
"is_canary": "true",
"message": "This is an AWS canary token generated at canarytokens.org, and was not set off; learn more here: https://trufflesecurity.com/canaries",
},
},
},
@@ -213,7 +229,9 @@ func TestAWS_FromChunk(t *testing.T) {
},
{
name: "found, would be verified if not for http timeout",
s: scanner{verificationClient: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
s: scanner{
verificationClient: common.SaneHttpClientTimeOut(1 * time.Microsecond),
},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a aws secret %s within aws %s", secret, id)),
@@ -227,6 +245,8 @@ func TestAWS_FromChunk(t *testing.T) {
ExtraData: map[string]string{
"resource_type": "Access key",
"account": "171436882533",
"is_canary": "true",
"message": "This is an AWS canary token generated at canarytokens.org, and was not set off; learn more here: https://trufflesecurity.com/canaries",
},
},
},
@@ -235,7 +255,9 @@ func TestAWS_FromChunk(t *testing.T) {
},
{
name: "found, unverified due to unexpected http response status",
s: scanner{verificationClient: common.ConstantResponseHttpClient(500, "internal server error")},
s: scanner{
verificationClient: common.ConstantResponseHttpClient(500, "internal server error"),
},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a aws secret %s within aws %s", secret, id)),
@@ -249,6 +271,8 @@ func TestAWS_FromChunk(t *testing.T) {
ExtraData: map[string]string{
"resource_type": "Access key",
"account": "171436882533",
"is_canary": "true",
"message": "This is an AWS canary token generated at canarytokens.org, and was not set off; learn more here: https://trufflesecurity.com/canaries",
},
},
},
@@ -256,26 +280,52 @@ func TestAWS_FromChunk(t *testing.T) {
wantVerificationError: true,
},
{
name: "found, unverified due to unexpected 403 response reason",
s: scanner{verificationClient: common.ConstantResponseHttpClient(403, `{"Error": {"Code": "SignatureDoesNotMatch"} }`)},
name: "found, unverified due to invalid aws_secret with valid canary access_key_id",
s: scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a aws secret %s within aws %s", secret, id)),
data: []byte(fmt.Sprintf("You can find a aws secret %s within aws %s", inactiveSecret, canaryAccessKeyID)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_AWS,
Verified: false,
Redacted: "AKIASP2TPHJSQH3FJRUX",
Redacted: canaryAccessKeyID,
ExtraData: map[string]string{
"resource_type": "Access key",
"account": "171436882533",
"is_canary": "true",
"message": "This is an AWS canary token generated at canarytokens.org, and was not set off; learn more here: https://trufflesecurity.com/canaries",
},
},
},
wantErr: false,
wantVerificationError: true,
wantVerificationError: false,
},
{
name: "found, valid canary token with no verification",
s: scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a aws secret %s within aws %s", secret, canaryAccessKeyID)),
verify: false,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_AWS,
Verified: false,
Redacted: canaryAccessKeyID,
ExtraData: map[string]string{
"resource_type": "Access key",
"account": "171436882533",
"is_canary": "true",
"message": "This is an AWS canary token generated at canarytokens.org, and was not set off; learn more here: https://trufflesecurity.com/canaries",
},
},
},
wantErr: false,
wantVerificationError: false,
},
{
name: "verified secret checked directly after unverified secret with same key id",
@@ -291,11 +341,11 @@ func TestAWS_FromChunk(t *testing.T) {
Verified: true,
Redacted: "AKIASP2TPHJSQH3FJRUX",
ExtraData: map[string]string{
"resource_type": "Access key",
"rotation_guide": "https://howtorotate.com/docs/tutorials/aws/",
"account": "171436882533",
"arn": "arn:aws:iam::171436882533:user/canarytokens.com@@4dxkh0pdeop3bzu9zx5wob793",
"user_id": "AIDASP2TPHJSUFRSTTZX4",
"resource_type": "Access key",
"account": "171436882533",
"arn": "arn:aws:iam::171436882533:user/canarytokens.com@@4dxkh0pdeop3bzu9zx5wob793",
"is_canary": "true",
"message": "This is an AWS canary token generated at canarytokens.org, and was not set off; learn more here: https://trufflesecurity.com/canaries",
},
},
},
@@ -318,8 +368,13 @@ func TestAWS_FromChunk(t *testing.T) {
t.Fatalf("wantVerificationError %v, verification error = %v", tt.wantVerificationError, got[i].VerificationError())
}
}
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "RawV2", "Raw", "verificationError")
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
ignoreOpts := []cmp.Option{
cmpopts.IgnoreFields(detectors.Result{}, "RawV2", "Raw", "verificationError"),
cmpopts.SortSlices(func(x, y detectors.Result) bool {
return x.Redacted < y.Redacted
}),
}
if diff := cmp.Diff(got, tt.want, ignoreOpts...); diff != "" {
t.Errorf("AWS.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
}
})
pkg/detectors/cloudplan/cloudplan.go
+2 -1
View File
@@ -2,10 +2,11 @@ package cloudplan
import (
"context"
regexp "github.com/wasilibs/go-re2"
"net/http"
"strings"
regexp "github.com/wasilibs/go-re2"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
pkg/detectors/coinbase_waas/coinbase_waas_test.go
+1 -1
View File
@@ -75,7 +75,7 @@ func TestCoinbaseWaaS_Pattern(t *testing.T) {
shouldMatch: true,
match: "organizations/7eead2d5-fa48-4423-8f40-c70d8ce398ae/apiKeys/7b9516b6-d82e-44e8-bed5-89b160452ed9:-----BEGIN ECDSA PRIVATE KEY-----\nMHcCAQEEINQdZMbF2r07KF0mxfLYt9Y1PNaC0C6UpZ31MxD4NEE8oAoGCCqGSM49\nAwEHoUQDQgAEeRFgMrQEHI/APWaziRH90jN7EozjdbPVxvzc1F4zqWTeCtLASwqA\nqnMugYX2epqsFhGn82xNXu2NwgORc6embQ==\n-----END ECDSA PRIVATE KEY-----\n",
},
// TODO: Is it worth supporting case-insensitve headers?
// TODO: Is it worth supporting case-insensitive headers?
// https://github.com/coinbase/waas-sdk-react-native/blob/bbaf597e73d02ecaf64161061e71b85d9eeeb9d6/example/src/.coinbase_cloud_api_key.json#L4
// {
// name: "valid_result_case_insensitive",
pkg/detectors/coinmarketcap/coinmarketcap.go
-78
View File
@@ -1,78 +0,0 @@
package coinmarketcap
import (
"context"
regexp "github.com/wasilibs/go-re2"
"net/http"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
type Scanner struct{}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"coinmarketcap", "CMC"}) + `\b([0-9Aa-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"coinmarketcap", "CMC"}
}
// FromData will find and optionally verify Coinmarketcap secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
for _, match := range matches {
if len(match) != 2 {
continue
}
resMatch := strings.TrimSpace(match[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_CoinMarketCap,
Raw: []byte(resMatch),
}
if verify {
req, err := http.NewRequestWithContext(ctx, "GET", "https://pro-api.coinmarketcap.com/v1/cryptocurrency/listings/latest?start=1&limit=5000&convert=USD", nil)
if err != nil {
continue
}
req.Header.Add("Accept", "application/json")
req.Header.Add("X-CMC_PRO_API_KEY", resMatch)
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
}
}
}
results = append(results, s1)
}
return results, nil
}
func (s Scanner) Type() detectorspb.DetectorType {
return detectorspb.DetectorType_CoinMarketCap
}
pkg/detectors/crossbrowsertesting/crossbrowsertesting.go
-86
View File
@@ -1,86 +0,0 @@
package crossbrowsertesting
import (
"context"
regexp "github.com/wasilibs/go-re2"
"net/http"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
type Scanner struct{}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"crossbrowsertesting"}) + `\b([0-9a-z]{16})\b`)
idPat = regexp.MustCompile(detectors.PrefixRegex([]string{"crossbrowsertesting"}) + `\b([a-z0-9]{4,25}@[a-zA-Z0-9]{2,12}.[a-zA-Z0-9]{2,6})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"crossbrowsertesting"}
}
// FromData will find and optionally verify CrossBrowserTesting secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
idmatches := idPat.FindAllStringSubmatch(dataStr, -1)
for _, match := range matches {
if len(match) != 2 {
continue
}
resMatch := strings.TrimSpace(match[1])
for _, idmatch := range idmatches {
if len(idmatch) != 2 {
continue
}
resIdMatch := strings.TrimSpace(idmatch[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_CrossBrowserTesting,
Raw: []byte(resMatch),
RawV2: []byte(resMatch + resIdMatch),
}
if verify {
req, err := http.NewRequestWithContext(ctx, "GET", "https://crossbrowsertesting.com/api/v3/livetests", nil)
if err != nil {
continue
}
req.SetBasicAuth(resIdMatch, resMatch)
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
}
}
}
results = append(results, s1)
}
}
return results, nil
}
func (s Scanner) Type() detectorspb.DetectorType {
return detectorspb.DetectorType_CrossBrowserTesting
}
pkg/detectors/crossbrowsertesting/crossbrowsertesting_test.go
-121
View File
@@ -1,121 +0,0 @@
//go:build detectors
// +build detectors
package crossbrowsertesting
import (
"context"
"fmt"
"testing"
"time"
"github.com/kylelemons/godebug/pretty"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
func TestCrossBrowserTesting_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
secret := testSecrets.MustGetField("CROSSBROWSERTESTING")
id := testSecrets.MustGetField("CROSSBROWSERTESTING_USER")
inactiveSecret := testSecrets.MustGetField("CROSSBROWSERTESTING_INACTIVE")
type args struct {
ctx context.Context
data []byte
verify bool
}
tests := []struct {
name string
s Scanner
args args
want []detectors.Result
wantErr bool
}{
{
name: "found, verified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a crossbrowsertesting secret %s within crossbrowsertesting %s", secret, id)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_CrossBrowserTesting,
Verified: true,
},
},
wantErr: false,
},
{
name: "found, unverified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a crossbrowsertesting secret %s within crossbrowsertesting %s but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_CrossBrowserTesting,
Verified: false,
},
},
wantErr: false,
},
{
name: "not found",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte("You cannot find the secret within"),
verify: true,
},
want: nil,
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
s := Scanner{}
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
if (err != nil) != tt.wantErr {
t.Errorf("CrossBrowserTesting.FromData() error = %v, wantErr %v", err, tt.wantErr)
return
}
for i := range got {
if len(got[i].Raw) == 0 {
t.Fatalf("no raw secret present: \n %+v", got[i])
}
got[i].Raw = nil
}
if diff := pretty.Compare(got, tt.want); diff != "" {
t.Errorf("CrossBrowserTesting.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
}
})
}
}
func BenchmarkFromData(benchmark *testing.B) {
ctx := context.Background()
s := Scanner{}
for name, data := range detectors.MustGetBenchmarkData() {
benchmark.Run(name, func(b *testing.B) {
b.ResetTimer()
for n := 0; n < b.N; n++ {
_, err := s.FromData(ctx, false, data)
if err != nil {
b.Fatal(err)
}
}
})
}
}
pkg/detectors/datadogtoken/datadogtoken.go
+43 -34
View File
@@ -2,19 +2,25 @@ package datadogtoken
import (
"context"
regexp "github.com/wasilibs/go-re2"
"net/http"
"strings"
regexp "github.com/wasilibs/go-re2"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
type Scanner struct{}
type Scanner struct {
detectors.EndpointSetter
}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var _ detectors.EndpointCustomizer = (*Scanner)(nil)
func (Scanner) DefaultEndpoint() string { return "https://api.datadoghq.com" }
var (
client = common.SaneHttpClient()
@@ -59,23 +65,24 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
}
if verify {
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.datadoghq.com/api/v2/users", nil)
if err != nil {
continue
}
req.Header.Add("Content-Type", "application/json")
req.Header.Add("DD-API-KEY", resApiMatch)
req.Header.Add("DD-APPLICATION-KEY", resAppMatch)
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(resApiMatch, detectors.DefaultFalsePositives, true) {
continue
for _, baseURL := range s.Endpoints(s.DefaultEndpoint()) {
req, err := http.NewRequestWithContext(ctx, "GET", baseURL+"/api/v2/users", nil)
if err != nil {
continue
}
req.Header.Add("Content-Type", "application/json")
req.Header.Add("DD-API-KEY", resApiMatch)
req.Header.Add("DD-APPLICATION-KEY", resAppMatch)
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(resApiMatch, detectors.DefaultFalsePositives, true) {
continue
}
}
}
}
@@ -96,21 +103,23 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
if verify {
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.datadoghq.com/api/v1/validate", nil)
if err != nil {
continue
}
req.Header.Add("Content-Type", "application/json")
req.Header.Add("DD-API-KEY", resApiMatch)
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(resApiMatch, detectors.DefaultFalsePositives, true) {
continue
for _, baseURL := range s.Endpoints(s.DefaultEndpoint()) {
req, err := http.NewRequestWithContext(ctx, "GET", baseURL+"/api/v1/validate", nil)
if err != nil {
continue
}
req.Header.Add("Content-Type", "application/json")
req.Header.Add("DD-API-KEY", resApiMatch)
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(resApiMatch, detectors.DefaultFalsePositives, true) {
continue
}
}
}
}
pkg/detectors/detectors.go
+1 -1
View File
@@ -159,7 +159,7 @@ func CleanResults(results []Result) []Result {
func PrefixRegex(keywords []string) string {
pre := `(?i)(?:`
middle := strings.Join(keywords, "|")
post := `)(?:.|[\n\r]){0,40}`
post := `)(?:.|[\n\r]){0,40}?`
return pre + middle + post
}
pkg/detectors/dockerhub/v1/dockerhub.go
+179
View File
@@ -0,0 +1,179 @@
package dockerhubv2
import (
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"strings"
regexp "github.com/wasilibs/go-re2"
"github.com/golang-jwt/jwt/v4"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
type Scanner struct {
client *http.Client
}
func (s Scanner) Version() int { return 1 }
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var _ detectors.Versioner = (*Scanner)(nil)
var (
// Can use email or username for login.
usernamePat = regexp.MustCompile(`(?im)(?:user|usr|-u|id)\S{0,40}?[:=\s]{1,3}[ '"=]?([a-zA-Z0-9]{4,40})\b`)
emailPat = regexp.MustCompile(`(` + common.EmailPattern + `)`)
// Can use password or personal access token (PAT) for login, but this scanner will only check for PATs.
accessTokenPat = regexp.MustCompile(detectors.PrefixRegex([]string{"docker"}) + `\b([a-z0-9]{8}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{12})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"docker"}
}
// FromData will find and optionally verify Dockerhub secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
// Deduplicate results.
tokens := make(map[string]struct{})
for _, matches := range accessTokenPat.FindAllStringSubmatch(dataStr, -1) {
tokens[matches[1]] = struct{}{}
}
if len(tokens) == 0 {
return
}
usernames := make(map[string]struct{})
for _, matches := range usernamePat.FindAllStringSubmatch(dataStr, -1) {
usernames[matches[1]] = struct{}{}
}
for _, matches := range emailPat.FindAllStringSubmatch(dataStr, -1) {
usernames[matches[1]] = struct{}{}
}
// Process results.
for token := range tokens {
s1 := detectors.Result{
DetectorType: s.Type(),
Raw: []byte(token),
}
for username := range usernames {
s1.RawV2 = []byte(fmt.Sprintf("%s:%s", username, token))
if verify {
if s.client == nil {
s.client = common.SaneHttpClient()
}
isVerified, extraData, verificationErr := s.verifyMatch(ctx, username, token)
s1.Verified = isVerified
s1.ExtraData = extraData
s1.SetVerificationError(verificationErr)
}
results = append(results, s1)
if s1.Verified {
break
}
}
// PAT matches without usernames cannot be verified but might still be useful.
if len(usernames) == 0 {
results = append(results, s1)
}
}
return
}
func (s Scanner) verifyMatch(ctx context.Context, username string, password string) (bool, map[string]string, error) {
payload := strings.NewReader(fmt.Sprintf(`{"username": "%s", "password": "%s"}`, username, password))
req, err := http.NewRequestWithContext(ctx, http.MethodPost, "https://hub.docker.com/v2/users/login", payload)
if err != nil {
return false, nil, err
}
req.Header.Add("Content-Type", "application/json")
res, err := s.client.Do(req)
if err != nil {
return false, nil, err
}
defer res.Body.Close()
body, err := io.ReadAll(res.Body)
if err != nil {
return false, nil, err
}
if res.StatusCode == http.StatusOK {
var tokenRes tokenResponse
if err := json.Unmarshal(body, &tokenRes); (err != nil || tokenRes == tokenResponse{}) {
return false, nil, err
}
parser := jwt.NewParser()
token, _, err := parser.ParseUnverified(tokenRes.Token, &hubJwtClaims{})
if err != nil {
return true, nil, err
}
if claims, ok := token.Claims.(*hubJwtClaims); ok {
extraData := map[string]string{
"hub_username": username,
"hub_email": claims.HubClaims.Email,
"hub_scope": claims.Scope,
}
return true, extraData, nil
}
return true, nil, nil
} else if res.StatusCode == http.StatusUnauthorized {
// Valid credentials can still return a 401 status code if 2FA is enabled
var mfaRes mfaRequiredResponse
if err := json.Unmarshal(body, &mfaRes); err != nil || mfaRes.MfaToken == "" {
return false, nil, nil
}
extraData := map[string]string{
"hub_username": username,
"2fa_required": "true",
}
return true, extraData, nil
} else {
return false, nil, fmt.Errorf("unexpected response status %d", res.StatusCode)
}
}
type tokenResponse struct {
Token string `json:"token"`
}
type userClaims struct {
Username string `json:"username"`
Email string `json:"email"`
}
type hubJwtClaims struct {
Scope string `json:"scope"`
HubClaims userClaims `json:"https://hub.docker.com"` // not sure why this is a key, further investigation required.
jwt.RegisteredClaims
}
type mfaRequiredResponse struct {
MfaToken string `json:"login_2fa_token"`
}
func (s Scanner) Type() detectorspb.DetectorType {
return detectorspb.DetectorType_Dockerhub
}
pkg/detectors/coinmarketcap/coinmarketcap_test.go → pkg/detectors/dockerhub/v1/dockerhub_test.go
+31 -10
View File
@@ -1,7 +1,7 @@
//go:build detectors
// +build detectors
package coinmarketcap
package dockerhubv2
import (
"context"
@@ -10,21 +10,24 @@ import (
"time"
"github.com/kylelemons/godebug/pretty"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
func TestCoinmarketcap_FromChunk(t *testing.T) {
func TestDockerhub_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
secret := testSecrets.MustGetField("COINMARKETCAP")
inactiveSecret := testSecrets.MustGetField("COINMARKETCAP_INACTIVE")
username := testSecrets.MustGetField("DOCKERHUB_USERNAME")
email := testSecrets.MustGetField("DOCKERHUB_EMAIL")
pat := testSecrets.MustGetField("DOCKERHUB_PAT")
inactivePat := testSecrets.MustGetField("DOCKERHUB_INACTIVE_PAT")
type args struct {
ctx context.Context
@@ -43,12 +46,28 @@ func TestCoinmarketcap_FromChunk(t *testing.T) {
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a coinmarketcap secret %s within", secret)),
data: []byte(fmt.Sprintf("docker login -u %s -p %s", username, pat)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_CoinMarketCap,
DetectorType: detectorspb.DetectorType_Dockerhub,
Verified: true,
},
},
wantErr: false,
},
{
name: "found, verified (email)",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("docker login -u %s -p %s", email, pat)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_Dockerhub,
Verified: true,
},
},
@@ -59,12 +78,12 @@ func TestCoinmarketcap_FromChunk(t *testing.T) {
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a coinmarketcap secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
data: []byte(fmt.Sprintf("docker login -u %s -p %s", username, inactivePat)), // the secret would satisfy the regex but not pass validation
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_CoinMarketCap,
DetectorType: detectorspb.DetectorType_Dockerhub,
Verified: false,
},
},
@@ -87,7 +106,7 @@ func TestCoinmarketcap_FromChunk(t *testing.T) {
s := Scanner{}
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
if (err != nil) != tt.wantErr {
t.Errorf("Coinmarketcap.FromData() error = %v, wantErr %v", err, tt.wantErr)
t.Errorf("Dockerhub.FromData() error = %v, wantErr %v", err, tt.wantErr)
return
}
for i := range got {
@@ -95,9 +114,11 @@ func TestCoinmarketcap_FromChunk(t *testing.T) {
t.Fatalf("no raw secret present: \n %+v", got[i])
}
got[i].Raw = nil
got[i].RawV2 = nil
got[i].ExtraData = nil
}
if diff := pretty.Compare(got, tt.want); diff != "" {
t.Errorf("Coinmarketcap.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
t.Errorf("Dockerhub.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
}
})
}
pkg/detectors/dockerhub/dockerhub.go → pkg/detectors/dockerhub/v2/dockerhub.go
+4 -1
View File
@@ -21,12 +21,15 @@ type Scanner struct {
client *http.Client
}
func (s Scanner) Version() int { return 2 }
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var _ detectors.Versioner = (*Scanner)(nil)
var (
// Can use email or username for login.
usernamePat = regexp.MustCompile(`(?im)(?:user|usr|-u)\S{0,40}?[:=\s]{1,3}[ '"=]?([a-zA-Z0-9]{4,40})\b`)
usernamePat = regexp.MustCompile(`(?im)(?:user|usr|-u|id)\S{0,40}?[:=\s]{1,3}[ '"=]?([a-zA-Z0-9]{4,40})\b`)
emailPat = regexp.MustCompile(`(` + common.EmailPattern + `)`)
// Can use password or personal access token (PAT) for login, but this scanner will only check for PATs.
pkg/detectors/dockerhub/dockerhub_test.go → pkg/detectors/dockerhub/v2/dockerhub_test.go
View File
pkg/detectors/etsyapikey/etsyapikey.go
-76
View File
@@ -1,76 +0,0 @@
package etsyapikey
import (
"context"
regexp "github.com/wasilibs/go-re2"
"net/http"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
type Scanner struct{}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"etsy"}) + `\b([a-zA-Z-0-9]{24})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"etsy"}
}
// FromData will find and optionally verify EtsyApiKey secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
for _, match := range matches {
if len(match) != 2 {
continue
}
resMatch := strings.TrimSpace(match[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_EtsyApiKey,
Raw: []byte(resMatch),
}
if verify {
req, err := http.NewRequestWithContext(ctx, "GET", "https://openapi.etsy.com/v2/listings/active?api_key="+resMatch, nil)
if err != nil {
continue
}
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
}
}
}
results = append(results, s1)
}
return results, nil
}
func (s Scanner) Type() detectorspb.DetectorType {
return detectorspb.DetectorType_EtsyApiKey
}
pkg/detectors/etsyapikey/etsyapikey_test.go
-120
View File
@@ -1,120 +0,0 @@
//go:build detectors
// +build detectors
package etsyapikey
import (
"context"
"fmt"
"testing"
"time"
"github.com/kylelemons/godebug/pretty"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
func TestEtsyApiKey_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
secret := testSecrets.MustGetField("ETSYAPIKEY_TOKEN")
inactiveSecret := testSecrets.MustGetField("ETSYAPIKEY_INACTIVE")
type args struct {
ctx context.Context
data []byte
verify bool
}
tests := []struct {
name string
s Scanner
args args
want []detectors.Result
wantErr bool
}{
{
name: "found, verified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a etsy secret %s within", secret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_EtsyApiKey,
Verified: true,
},
},
wantErr: false,
},
{
name: "found, unverified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a etsy secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_EtsyApiKey,
Verified: false,
},
},
wantErr: false,
},
{
name: "not found",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte("You cannot find the secret within"),
verify: true,
},
want: nil,
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
s := Scanner{}
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
if (err != nil) != tt.wantErr {
t.Errorf("EtsyApiKey.FromData() error = %v, wantErr %v", err, tt.wantErr)
return
}
for i := range got {
if len(got[i].Raw) == 0 {
t.Fatalf("no raw secret present: \n %+v", got[i])
}
got[i].Raw = nil
}
if diff := pretty.Compare(got, tt.want); diff != "" {
t.Errorf("EtsyApiKey.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
}
})
}
}
func BenchmarkFromData(benchmark *testing.B) {
ctx := context.Background()
s := Scanner{}
for name, data := range detectors.MustGetBenchmarkData() {
benchmark.Run(name, func(b *testing.B) {
b.ResetTimer()
for n := 0; n < b.N; n++ {
_, err := s.FromData(ctx, false, data)
if err != nil {
b.Fatal(err)
}
}
})
}
}
pkg/detectors/figmapersonalaccesstoken/v1/figmapersonalaccesstoken.go
+5 -1
View File
@@ -3,10 +3,11 @@ package figmapersonalaccesstoken
import (
"context"
"fmt"
regexp "github.com/wasilibs/go-re2"
"net/http"
"strings"
regexp "github.com/wasilibs/go-re2"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
@@ -49,6 +50,9 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_FigmaPersonalAccessToken,
Raw: []byte(resMatch),
ExtraData: map[string]string{
"version": fmt.Sprintf("%d", s.Version()),
},
}
if verify {
pkg/detectors/figmapersonalaccesstoken/v1/figmapersonalaccesstoken_test.go
+12
View File
@@ -51,6 +51,9 @@ func TestFigmaPersonalAccessToken_FromChunk(t *testing.T) {
{
DetectorType: detectorspb.DetectorType_FigmaPersonalAccessToken,
Verified: true,
ExtraData: map[string]string{
"version": "1",
},
},
},
wantErr: false,
@@ -67,6 +70,9 @@ func TestFigmaPersonalAccessToken_FromChunk(t *testing.T) {
{
DetectorType: detectorspb.DetectorType_FigmaPersonalAccessToken,
Verified: false,
ExtraData: map[string]string{
"version": "1",
},
},
},
wantErr: false,
@@ -83,6 +89,9 @@ func TestFigmaPersonalAccessToken_FromChunk(t *testing.T) {
{
DetectorType: detectorspb.DetectorType_FigmaPersonalAccessToken,
Verified: false,
ExtraData: map[string]string{
"version": "1",
},
},
},
wantErr: false,
@@ -100,6 +109,9 @@ func TestFigmaPersonalAccessToken_FromChunk(t *testing.T) {
{
DetectorType: detectorspb.DetectorType_FigmaPersonalAccessToken,
Verified: false,
ExtraData: map[string]string{
"version": "1",
},
},
},
wantErr: false,
pkg/detectors/figmapersonalaccesstoken/v2/figmapersonalaccesstoken_v2.go
+5 -1
View File
@@ -3,10 +3,11 @@ package figmapersonalaccesstoken
import (
"context"
"fmt"
regexp "github.com/wasilibs/go-re2"
"net/http"
"strings"
regexp "github.com/wasilibs/go-re2"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
@@ -48,6 +49,9 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_FigmaPersonalAccessToken,
Raw: []byte(resMatch),
ExtraData: map[string]string{
"version": fmt.Sprintf("%d", s.Version()),
},
}
if verify {
pkg/detectors/figmapersonalaccesstoken/v2/figmapersonalaccesstoken_v2_test.go
+12
View File
@@ -51,6 +51,9 @@ func TestFigmaPersonalAccessToken_FromChunk(t *testing.T) {
{
DetectorType: detectorspb.DetectorType_FigmaPersonalAccessToken,
Verified: true,
ExtraData: map[string]string{
"version": "2",
},
},
},
wantErr: false,
@@ -67,6 +70,9 @@ func TestFigmaPersonalAccessToken_FromChunk(t *testing.T) {
{
DetectorType: detectorspb.DetectorType_FigmaPersonalAccessToken,
Verified: false,
ExtraData: map[string]string{
"version": "2",
},
},
},
wantErr: false,
@@ -83,6 +89,9 @@ func TestFigmaPersonalAccessToken_FromChunk(t *testing.T) {
{
DetectorType: detectorspb.DetectorType_FigmaPersonalAccessToken,
Verified: false,
ExtraData: map[string]string{
"version": "2",
},
},
},
wantErr: false,
@@ -100,6 +109,9 @@ func TestFigmaPersonalAccessToken_FromChunk(t *testing.T) {
{
DetectorType: detectorspb.DetectorType_FigmaPersonalAccessToken,
Verified: false,
ExtraData: map[string]string{
"version": "2",
},
},
},
wantErr: false,
pkg/detectors/flowdash/flowdash.go
-79
View File
@@ -1,79 +0,0 @@
package flowdash
import (
"context"
"fmt"
regexp "github.com/wasilibs/go-re2"
"net/http"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
type Scanner struct{}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"flowdash"}) + `\b([0-9a-zA-Z]{24})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"flowdash"}
}
// FromData will find and optionally verify Flowdash secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
for _, match := range matches {
if len(match) != 2 {
continue
}
resMatch := strings.TrimSpace(match[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_Flowdash,
Raw: []byte(resMatch),
}
if verify {
req, err := http.NewRequestWithContext(ctx, "GET", "https://app.flowdash.com/api/v1/workspace", nil)
if err != nil {
continue
}
req.Header.Add("Content-Type", "application/json")
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
}
}
}
results = append(results, s1)
}
return results, nil
}
func (s Scanner) Type() detectorspb.DetectorType {
return detectorspb.DetectorType_Flowdash
}
pkg/detectors/flowdash/flowdash_test.go
-120
View File
@@ -1,120 +0,0 @@
//go:build detectors
// +build detectors
package flowdash
import (
"context"
"fmt"
"testing"
"time"
"github.com/kylelemons/godebug/pretty"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
func TestFlowdash_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
secret := testSecrets.MustGetField("FLOWDASH")
inactiveSecret := testSecrets.MustGetField("FLOWDASH_INACTIVE")
type args struct {
ctx context.Context
data []byte
verify bool
}
tests := []struct {
name string
s Scanner
args args
want []detectors.Result
wantErr bool
}{
{
name: "found, verified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a flowdash secret %s within", secret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_Flowdash,
Verified: true,
},
},
wantErr: false,
},
{
name: "found, unverified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a flowdash secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_Flowdash,
Verified: false,
},
},
wantErr: false,
},
{
name: "not found",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte("You cannot find the secret within"),
verify: true,
},
want: nil,
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
s := Scanner{}
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
if (err != nil) != tt.wantErr {
t.Errorf("Flowdash.FromData() error = %v, wantErr %v", err, tt.wantErr)
return
}
for i := range got {
if len(got[i].Raw) == 0 {
t.Fatalf("no raw secret present: \n %+v", got[i])
}
got[i].Raw = nil
}
if diff := pretty.Compare(got, tt.want); diff != "" {
t.Errorf("Flowdash.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
}
})
}
}
func BenchmarkFromData(benchmark *testing.B) {
ctx := context.Background()
s := Scanner{}
for name, data := range detectors.MustGetBenchmarkData() {
benchmark.Run(name, func(b *testing.B) {
b.ResetTimer()
for n := 0; n < b.N; n++ {
_, err := s.FromData(ctx, false, data)
if err != nil {
b.Fatal(err)
}
}
})
}
}
pkg/detectors/fullstory/fullstory.go → pkg/detectors/fullstory/v1/fullstory.go
+5 -1
View File
@@ -3,10 +3,11 @@ package fullstory
import (
"context"
"fmt"
regexp "github.com/wasilibs/go-re2"
"net/http"
"strings"
regexp "github.com/wasilibs/go-re2"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
@@ -48,6 +49,9 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_Fullstory,
Raw: []byte(resMatch),
ExtraData: map[string]string{
"version": fmt.Sprintf("%d", s.Version()),
},
}
if verify {
pkg/detectors/fullstory/fullstory_test.go → pkg/detectors/fullstory/v1/fullstory_test.go
+6
View File
@@ -50,6 +50,9 @@ func TestFullstory_FromChunk(t *testing.T) {
{
DetectorType: detectorspb.DetectorType_Fullstory,
Verified: true,
ExtraData: map[string]string{
"version": "1",
},
},
},
wantErr: false,
@@ -66,6 +69,9 @@ func TestFullstory_FromChunk(t *testing.T) {
{
DetectorType: detectorspb.DetectorType_Fullstory,
Verified: false,
ExtraData: map[string]string{
"version": "1",
},
},
},
wantErr: false,
pkg/detectors/fullstory_v2/fullstory_v2.go → pkg/detectors/fullstory/v2/fullstory_v2.go
+6 -2
View File
@@ -1,12 +1,13 @@
package fullstory_v2
package fullstory
import (
"context"
"fmt"
regexp "github.com/wasilibs/go-re2"
"net/http"
"strings"
regexp "github.com/wasilibs/go-re2"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
@@ -48,6 +49,9 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_Fullstory,
Raw: []byte(resMatch),
ExtraData: map[string]string{
"version": fmt.Sprintf("%d", s.Version()),
},
}
if verify {
pkg/detectors/fullstory_v2/fullstory_v2_test.go → pkg/detectors/fullstory/v2/fullstory_v2_test.go
+7 -1
View File
@@ -1,7 +1,7 @@
//go:build detectors
// +build detectors
package fullstory_v2
package fullstory
import (
"context"
@@ -50,6 +50,9 @@ func TestFullstory_FromChunk(t *testing.T) {
{
DetectorType: detectorspb.DetectorType_Fullstory,
Verified: true,
ExtraData: map[string]string{
"version": "2",
},
},
},
wantErr: false,
@@ -66,6 +69,9 @@ func TestFullstory_FromChunk(t *testing.T) {
{
DetectorType: detectorspb.DetectorType_Fullstory,
Verified: false,
ExtraData: map[string]string{
"version": "2",
},
},
},
wantErr: false,
pkg/detectors/fusebill/fusebill.go
-79
View File
@@ -1,79 +0,0 @@
package fusebill
import (
"context"
"fmt"
regexp "github.com/wasilibs/go-re2"
"net/http"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
type Scanner struct{}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"fusebill"}) + `\b([a-zA-Z0-9]{88})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"fusebill"}
}
// FromData will find and optionally verify Fusebill secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
for _, match := range matches {
if len(match) != 2 {
continue
}
resMatch := strings.TrimSpace(match[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_Fusebill,
Raw: []byte(resMatch),
}
if verify {
req, err := http.NewRequestWithContext(ctx, "GET", "https://secure.fusebill.com/v1/customers?pageSize=100&pageNumber=0", nil)
if err != nil {
continue
}
req.Header.Add("Authorization", fmt.Sprintf("Basic %s", resMatch))
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
}
}
}
results = append(results, s1)
}
return results, nil
}
func (s Scanner) Type() detectorspb.DetectorType {
return detectorspb.DetectorType_Fusebill
}
pkg/detectors/fusebill/fusebill_test.go
-120
View File
@@ -1,120 +0,0 @@
//go:build detectors
// +build detectors
package fusebill
import (
"context"
"fmt"
"testing"
"time"
"github.com/kylelemons/godebug/pretty"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
func TestFusebill_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
secret := testSecrets.MustGetField("FUSEBILL")
inactiveSecret := testSecrets.MustGetField("FUSEBILL_INACTIVE")
type args struct {
ctx context.Context
data []byte
verify bool
}
tests := []struct {
name string
s Scanner
args args
want []detectors.Result
wantErr bool
}{
{
name: "found, verified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a fusebill secret %s within", secret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_Fusebill,
Verified: true,
},
},
wantErr: false,
},
{
name: "found, unverified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a fusebill secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_Fusebill,
Verified: false,
},
},
wantErr: false,
},
{
name: "not found",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte("You cannot find the secret within"),
verify: true,
},
want: nil,
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
s := Scanner{}
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
if (err != nil) != tt.wantErr {
t.Errorf("Fusebill.FromData() error = %v, wantErr %v", err, tt.wantErr)
return
}
for i := range got {
if len(got[i].Raw) == 0 {
t.Fatalf("no raw secret present: \n %+v", got[i])
}
got[i].Raw = nil
}
if diff := pretty.Compare(got, tt.want); diff != "" {
t.Errorf("Fusebill.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
}
})
}
}
func BenchmarkFromData(benchmark *testing.B) {
ctx := context.Background()
s := Scanner{}
for name, data := range detectors.MustGetBenchmarkData() {
benchmark.Run(name, func(b *testing.B) {
b.ResetTimer()
for n := 0; n < b.N; n++ {
_, err := s.FromData(ctx, false, data)
if err != nil {
b.Fatal(err)
}
}
})
}
}
pkg/detectors/gcp/gcp.go
+10 -2
View File
@@ -1,14 +1,17 @@
package gcp
import (
"bytes"
"context"
"encoding/json"
regexp "github.com/wasilibs/go-re2"
"strings"
regexp "github.com/wasilibs/go-re2"
"golang.org/x/oauth2/google"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
"golang.org/x/oauth2/google"
)
type Scanner struct{}
@@ -78,6 +81,11 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
if len(raw) == 0 {
raw = []byte(key)
}
// This is an unprivileged service account used in Kubernetes' tests. It is intentionally public.
// https://github.com/kubernetes/kubernetes/blob/10a06602223eab17e02e197d1da591727c756d32/test/e2e_node/runtime_conformance_test.go#L50
if bytes.Equal(raw, []byte("image-pulling@authenticated-image-pulling.iam.gserviceaccount.com")) {
continue
}
credBytes, _ := json.Marshal(creds)
pkg/detectors/gcpapplicationdefaultcredentials/gcpapplicationdefaultcredentials.go
+164
View File
@@ -0,0 +1,164 @@
package gcpapplicationdefaultcredentials
import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"strings"
regexp "github.com/wasilibs/go-re2"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
"golang.org/x/oauth2"
"golang.org/x/oauth2/google"
)
type Scanner struct {
client *http.Client
}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var (
defaultClient = common.SaneHttpClient()
keyPat = regexp.MustCompile(`\{[^{]+client_secret[^}]+\}`)
)
type gcpApplicationDefaultCredentials struct {
ClientID string `json:"client_id"`
ClientSecret string `json:"client_secret"`
RefreshToken string `json:"refresh_token"`
Type string `json:"type"`
}
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{".apps.googleusercontent.com"}
}
// FromData will find and optionally verify Gcpapplicationdefaultcredentials secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
matches := keyPat.FindAllString(dataStr, -1)
for _, match := range matches {
key := match
// Detect keys by unmarshalling the data.
creds := gcpApplicationDefaultCredentials{}
err := json.Unmarshal([]byte(key), &creds)
if err != nil {
continue
}
// Trim prefix (".apps.googleusercontent.com") because it will be labeled as false positive
detectedClientID, _, _ := strings.Cut(creds.ClientID, ".")
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_GCPApplicationDefaultCredentials,
Raw: []byte(detectedClientID),
}
if verify {
client := s.client
if client == nil {
client = defaultClient
}
// Use marshalled credential to verify if the found key is active
credBytes, _ := json.Marshal(creds)
isVerified, extraData, verificationErr := verifyMatch(ctx, client, string(credBytes))
s1.Verified = isVerified
s1.ExtraData = extraData
s1.SetVerificationError(verificationErr, match)
}
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if !s1.Verified && detectors.IsKnownFalsePositive(string(s1.Raw), detectors.DefaultFalsePositives, true) {
continue
}
results = append(results, s1)
}
return
}
func verifyMatch(ctx context.Context, client *http.Client, token string) (bool, map[string]string, error) {
// First load the credential from the found key
credentials, err := google.CredentialsFromJSON(ctx, []byte(token), "https://www.googleapis.com/auth/cloud-platform")
if err != nil {
return false, nil, err
}
// Credential not loaded. Not sure this can happened but it should be labeled unverified.
if credentials == nil {
return false, nil, nil
}
// Get token from the credentials
gcpToken, err := credentials.TokenSource.Token()
if err != nil {
// Return verification error if the error is temporary
// See https://pkg.go.dev/golang.org/x/oauth2/google#AuthenticationError.Temporary for details
var temporaryError *(google.AuthenticationError)
if errors.As(err, &temporaryError) {
if err.(*google.AuthenticationError).Temporary() {
return false, nil, err
}
}
return false, nil, nil
}
// Return verification error if the retrieved token is invalid
if !gcpToken.Valid() {
return false, nil, err
}
// Build request to call an IAM endpoint
req, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://iam.googleapis.com/v1/roles", nil)
if err != nil {
return false, nil, nil
}
// If we are not using a faketransport, leave it as is because the test wants to modify the response. Otherwise, set the retrieved token to the client.
if _, ok := client.Transport.(common.FakeTransport); !ok {
client.Transport = &oauth2.Transport{
Source: credentials.TokenSource,
}
}
res, err := client.Do(req)
if err != nil {
return false, nil, err
}
defer func() {
_, _ = io.Copy(io.Discard, res.Body)
_ = res.Body.Close()
}()
if res.StatusCode >= 200 && res.StatusCode < 300 {
// If the endpoint returns useful information, we can return it as a map.
return true, nil, nil
} else if res.StatusCode == 401 {
// The secret is determinately not verified (nothing to do)
return false, nil, nil
} else {
err = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
return false, nil, err
}
}
func (s Scanner) Type() detectorspb.DetectorType {
return detectorspb.DetectorType_GCPApplicationDefaultCredentials
}
pkg/detectors/gcpapplicationdefaultcredentials/gcpapplicationdefaultcredentials_test.go
+230
View File
@@ -0,0 +1,230 @@
//go:build detectors
// +build detectors
package gcpapplicationdefaultcredentials
import (
"context"
"fmt"
"testing"
"time"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
var (
validTestString = `{'client_id': '191375729402-oiuj2498ry3497gjveoierj8294jfj41.apps.googleusercontent.com', 'client_secret': 'z-OIFJWEOIJGWER91834325R', 'refresh_token': '1//0_joijgor3i4ut98579862709342j3kjJOIE02834jijfewoifjowiejfhghyzznfoiwejfwnvuhewiufnwinciwu_-o2i3jjfcc', 'type': 'authorized_user'}`
)
func TestGcpapplicationdefaultcredentials_Pattern(t *testing.T) {
d := Scanner{}
ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
name string
input string
want []string
}{
{
name: "typical pattern",
input: `{
"client_id": "191375729402-oiuj2498ry3497gjveoierj8294jfj41.apps.googleusercontent.com",
"client_secret": "z-OIFJWEOIJGWER91834325R",
"refresh_token": "1//0_joijgor3i4ut98579862709342j3kjJOIE02834jijfewoifjowiejfhghyzznfoiwejfwnvuhewiufnwinciwu_-o2i3jjfcc",
"type": "authorized_user"
}`,
want: []string{"191375729402-oiuj2498ry3497gjveoierj8294jfj41"},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
chunkSpecificDetectors := make(map[ahocorasick.DetectorKey]detectors.Detector, 2)
ahoCorasickCore.PopulateMatchingDetectors(test.input, chunkSpecificDetectors)
if len(chunkSpecificDetectors) == 0 {
t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
results, err := d.FromData(context.Background(), false, []byte(test.input))
if err != nil {
t.Errorf("error = %v", err)
return
}
if len(results) != len(test.want) {
if len(results) == 0 {
t.Errorf("did not receive result")
} else {
t.Errorf("expected %d results, only received %d", len(test.want), len(results))
}
return
}
actual := make(map[string]struct{}, len(results))
for _, r := range results {
if len(r.RawV2) > 0 {
actual[string(r.RawV2)] = struct{}{}
} else {
actual[string(r.Raw)] = struct{}{}
}
}
expected := make(map[string]struct{}, len(test.want))
for _, v := range test.want {
expected[v] = struct{}{}
}
if diff := cmp.Diff(expected, actual); diff != "" {
t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
}
})
}
}
func TestGcpapplicationdefaultcredentials_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
secret := testSecrets.MustGetField("GCPAPPLICATIONDEFAULTCREDENTIALS")
inactiveSecret := testSecrets.MustGetField("GCPAPPLICATIONDEFAULTCREDENTIALS_INACTIVE")
type args struct {
ctx context.Context
data []byte
verify bool
}
tests := []struct {
name string
s Scanner
args args
want []detectors.Result
wantErr bool
wantVerificationErr bool
}{
{
name: "found, verified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a gcpapplicationdefaultcredentials secret %s within", secret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_GCPApplicationDefaultCredentials,
Verified: true,
},
},
wantErr: false,
wantVerificationErr: false,
},
{
name: "found, unverified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a gcpapplicationdefaultcredentials secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_GCPApplicationDefaultCredentials,
Verified: false,
},
},
wantErr: false,
wantVerificationErr: false,
},
{
name: "not found",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte("You cannot find the secret within"),
verify: true,
},
want: nil,
wantErr: false,
wantVerificationErr: false,
},
{
name: "found, would be verified if not for timeout",
s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a gcpapplicationdefaultcredentials secret %s within", secret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_GCPApplicationDefaultCredentials,
Verified: false,
},
},
wantErr: false,
wantVerificationErr: true,
},
{
name: "found, verified but unexpected api surface",
s: Scanner{client: common.ConstantResponseHttpClient(404, "")},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a gcpapplicationdefaultcredentials secret %s within", secret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_GCPApplicationDefaultCredentials,
Verified: false,
},
},
wantErr: false,
wantVerificationErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
if (err != nil) != tt.wantErr {
t.Errorf("Gcpapplicationdefaultcredentials.FromData() error = %v, wantErr %v", err, tt.wantErr)
return
}
for i := range got {
if len(got[i].Raw) == 0 {
t.Fatalf("no raw secret present: \n %+v", got[i])
}
if (got[i].VerificationError() != nil) != tt.wantVerificationErr {
t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())
}
}
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
t.Errorf("Gcpapplicationdefaultcredentials.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
}
})
}
}
func BenchmarkFromData(benchmark *testing.B) {
ctx := context.Background()
s := Scanner{}
for name, data := range detectors.MustGetBenchmarkData() {
benchmark.Run(name, func(b *testing.B) {
b.ResetTimer()
for n := 0; n < b.N; n++ {
_, err := s.FromData(ctx, false, data)
if err != nil {
b.Fatal(err)
}
}
})
}
}
pkg/detectors/github/github.go
-142
View File
@@ -1,142 +0,0 @@
package github
import (
"context"
"encoding/json"
"fmt"
regexp "github.com/wasilibs/go-re2"
"net/http"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
)
type Scanner struct{ detectors.EndpointSetter }
// Ensure the Scanner satisfies the interfaces at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var _ detectors.Versioner = (*Scanner)(nil)
var _ detectors.EndpointCustomizer = (*Scanner)(nil)
func (Scanner) Version() int { return 2 }
func (Scanner) DefaultEndpoint() string { return "https://api.github.com" }
var (
// Oauth token
// https://developer.github.com/v3/#oauth2-token-sent-in-a-header
// Token type list:
// https://github.blog/2021-04-05-behind-githubs-new-authentication-token-formats/
// https://github.blog/changelog/2022-10-18-introducing-fine-grained-personal-access-tokens/
keyPat = regexp.MustCompile(`\b((?:ghp|gho|ghu|ghs|ghr|github_pat)_[a-zA-Z0-9_]{36,255})\b`)
// TODO: Oauth2 client_id and client_secret
// https://developer.github.com/v3/#oauth2-keysecret
)
// TODO: Add secret context?? Information about access, ownership etc
type userRes struct {
Login string `json:"login"`
Type string `json:"type"`
SiteAdmin bool `json:"site_admin"`
Name string `json:"name"`
Company string `json:"company"`
UserURL string `json:"html_url"`
// Included in GitHub Enterprise Server.
LdapDN string `json:"ldap_dn"`
}
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"ghp_", "gho_", "ghu_", "ghs_", "ghr_", "github_pat_"}
}
// FromData will find and optionally verify GitHub secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
for _, match := range matches {
// First match is entire regex, second is the first group.
if len(match) != 2 {
continue
}
token := match[1]
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_Github,
Raw: []byte(token),
ExtraData: map[string]string{
"rotation_guide": "https://howtorotate.com/docs/tutorials/github/",
},
}
if verify {
client := common.SaneHttpClient()
// https://developer.github.com/v3/users/#get-the-authenticated-user
for _, url := range s.Endpoints(s.DefaultEndpoint()) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, fmt.Sprintf("%s/user", url), nil)
if err != nil {
continue
}
req.Header.Set("Content-Type", "application/json; charset=utf-8")
req.Header.Set("Authorization", fmt.Sprintf("token %s", token))
res, err := client.Do(req)
if err == nil {
if res.StatusCode >= 200 && res.StatusCode < 300 {
var userResponse userRes
err = json.NewDecoder(res.Body).Decode(&userResponse)
res.Body.Close()
if err == nil {
s1.Verified = true
s1.ExtraData["username"] = userResponse.Login
s1.ExtraData["url"] = userResponse.UserURL
s1.ExtraData["account_type"] = userResponse.Type
if userResponse.SiteAdmin {
s1.ExtraData["site_admin"] = "true"
}
if userResponse.Name != "" {
s1.ExtraData["name"] = userResponse.Name
}
if userResponse.Company != "" {
s1.ExtraData["company"] = userResponse.Company
}
if userResponse.LdapDN != "" {
s1.ExtraData["ldap_dn"] = userResponse.LdapDN
}
// GitHub does not seem to consistently return this header.
scopes := res.Header.Get("X-OAuth-Scopes")
if scopes != "" {
s1.ExtraData["scopes"] = scopes
}
expiry := res.Header.Get("github-authentication-token-expiration")
if expiry != "" {
s1.ExtraData["expires_at"] = expiry
}
}
}
} else {
s1.SetVerificationError(err, token)
}
}
}
if !s1.Verified && detectors.IsKnownFalsePositive(string(s1.Raw), detectors.DefaultFalsePositives, true) {
continue
}
results = append(results, s1)
}
return
}
func (s Scanner) Type() detectorspb.DetectorType {
return detectorspb.DetectorType_Github
}
pkg/detectors/github_old/github_old.go → pkg/detectors/github/v1/github_old.go
+83 -52
View File
@@ -1,12 +1,13 @@
package github_old
package github
import (
"context"
"encoding/json"
"fmt"
regexp "github.com/wasilibs/go-re2"
"net/http"
regexp "github.com/wasilibs/go-re2"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
@@ -32,7 +33,7 @@ var (
)
// TODO: Add secret context?? Information about access, ownership etc
type userRes struct {
type UserRes struct {
Login string `json:"login"`
Type string `json:"type"`
SiteAdmin bool `json:"site_admin"`
@@ -43,6 +44,11 @@ type userRes struct {
LdapDN string `json:"ldap_dn"`
}
type HeaderInfo struct {
Scopes string `json:"X-OAuth-Scopes"`
Expiry string `json:"github-authentication-token-expiration"`
}
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
@@ -71,61 +77,24 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_Github,
Raw: []byte(token),
}
s1.ExtraData = map[string]string{
"rotation_guide": "https://howtorotate.com/docs/tutorials/github/",
ExtraData: map[string]string{
"rotation_guide": "https://howtorotate.com/docs/tutorials/github/",
"version": fmt.Sprintf("%d", s.Version()),
},
}
if verify {
client := common.SaneHttpClient()
// https://developer.github.com/v3/users/#get-the-authenticated-user
for _, url := range s.Endpoints(s.DefaultEndpoint()) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, fmt.Sprintf("%s/user", url), nil)
if err != nil {
continue
}
req.Header.Set("Content-Type", "application/json; charset=utf-8")
req.Header.Set("Authorization", fmt.Sprintf("token %s", token))
res, err := client.Do(req)
if err == nil {
if res.StatusCode >= 200 && res.StatusCode < 300 {
var userResponse userRes
err = json.NewDecoder(res.Body).Decode(&userResponse)
res.Body.Close()
if err == nil {
s1.Verified = true
if err == nil {
s1.Verified = true
s1.ExtraData["username"] = userResponse.Login
s1.ExtraData["url"] = userResponse.UserURL
s1.ExtraData["account_type"] = userResponse.Type
if userResponse.SiteAdmin {
s1.ExtraData["site_admin"] = "true"
}
if userResponse.Name != "" {
s1.ExtraData["name"] = userResponse.Name
}
if userResponse.Company != "" {
s1.ExtraData["company"] = userResponse.Company
}
if userResponse.LdapDN != "" {
s1.ExtraData["ldap_dn"] = userResponse.LdapDN
}
isVerified, userResponse, headers, err := s.VerifyGithub(ctx, client, token)
s1.Verified = isVerified
s1.SetVerificationError(err, token)
// GitHub does not seem to consistently return this header.
scopes := res.Header.Get("X-OAuth-Scopes")
if scopes != "" {
s1.ExtraData["scopes"] = scopes
}
expiry := res.Header.Get("github-authentication-token-expiration")
if expiry != "" {
s1.ExtraData["expires_at"] = expiry
}
}
}
}
}
if userResponse != nil {
SetUserResponse(userResponse, &s1)
}
if headers != nil {
SetHeaderInfo(headers, &s1)
}
}
@@ -139,6 +108,68 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
return results, nil
}
func (s Scanner) VerifyGithub(ctx context.Context, client *http.Client, token string) (bool, *UserRes, *HeaderInfo, error) {
// https://developer.github.com/v3/users/#get-the-authenticated-user
var requestErr error
for _, url := range s.Endpoints(s.DefaultEndpoint()) {
requestErr = nil
req, err := http.NewRequestWithContext(ctx, http.MethodGet, fmt.Sprintf("%s/user", url), nil)
if err != nil {
continue
}
req.Header.Set("Content-Type", "application/json; charset=utf-8")
req.Header.Set("Authorization", fmt.Sprintf("token %s", token))
res, err := client.Do(req)
if err != nil {
requestErr = err
continue
}
if res.StatusCode >= 200 && res.StatusCode < 300 {
var userResponse UserRes
err = json.NewDecoder(res.Body).Decode(&userResponse)
res.Body.Close()
if err == nil {
// GitHub does not seem to consistently return this header.
scopes := res.Header.Get("X-OAuth-Scopes")
expiry := res.Header.Get("github-authentication-token-expiration")
return true, &userResponse, &HeaderInfo{Scopes: scopes, Expiry: expiry}, nil
}
}
}
return false, nil, nil, requestErr
}
func SetUserResponse(userResponse *UserRes, s1 *detectors.Result) {
s1.ExtraData["username"] = userResponse.Login
s1.ExtraData["url"] = userResponse.UserURL
s1.ExtraData["account_type"] = userResponse.Type
if userResponse.SiteAdmin {
s1.ExtraData["site_admin"] = "true"
}
if userResponse.Name != "" {
s1.ExtraData["name"] = userResponse.Name
}
if userResponse.Company != "" {
s1.ExtraData["company"] = userResponse.Company
}
if userResponse.LdapDN != "" {
s1.ExtraData["ldap_dn"] = userResponse.LdapDN
}
}
func SetHeaderInfo(headers *HeaderInfo, s1 *detectors.Result) {
if headers.Scopes != "" {
s1.ExtraData["scopes"] = headers.Scopes
}
if headers.Expiry != "" {
s1.ExtraData["expiry"] = headers.Expiry
}
}
func (s Scanner) Type() detectorspb.DetectorType {
return detectorspb.DetectorType_Github
}
pkg/detectors/github_old/github_old_test.go → pkg/detectors/github/v1/github_old_test.go
+11 -1
View File
@@ -1,7 +1,7 @@
//go:build detectors
// +build detectors
package github_old
package github
import (
"context"
@@ -16,6 +16,8 @@ import (
)
func TestGitHub_FromChunk(t *testing.T) {
t.Skip("old tokens no longer valid, and no way to generate new ones")
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
@@ -48,6 +50,10 @@ func TestGitHub_FromChunk(t *testing.T) {
{
DetectorType: detectorspb.DetectorType_Github,
Verified: true,
ExtraData: map[string]string{
"rotation_guide": "https://howtorotate.com/docs/tutorials/github/",
"version": "1",
},
},
},
wantErr: false,
@@ -64,6 +70,10 @@ func TestGitHub_FromChunk(t *testing.T) {
{
DetectorType: detectorspb.DetectorType_Github,
Verified: false,
ExtraData: map[string]string{
"rotation_guide": "https://howtorotate.com/docs/tutorials/github/",
"version": "1",
},
},
},
wantErr: false,
pkg/detectors/github/v2/github.go
+97
View File
@@ -0,0 +1,97 @@
package github
import (
"context"
"fmt"
regexp "github.com/wasilibs/go-re2"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
v1 "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/github/v1"
)
type Scanner struct {
v1.Scanner
}
// Ensure the Scanner satisfies the interfaces at compile time.
var _ detectors.Detector = (*v1.Scanner)(nil)
var _ detectors.Versioner = (*v1.Scanner)(nil)
var _ detectors.EndpointCustomizer = (*v1.Scanner)(nil)
func (s Scanner) Version() int {
return 2
}
func (Scanner) DefaultEndpoint() string { return "https://api.github.com" }
var (
// Oauth token
// https://developer.github.com/v3/#oauth2-token-sent-in-a-header
// Token type list:
// https://github.blog/2021-04-05-behind-githubs-new-authentication-token-formats/
// https://github.blog/changelog/2022-10-18-introducing-fine-grained-personal-access-tokens/
keyPat = regexp.MustCompile(`\b((?:ghp|gho|ghu|ghs|ghr|github_pat)_[a-zA-Z0-9_]{36,255})\b`)
// TODO: Oauth2 client_id and client_secret
// https://developer.github.com/v3/#oauth2-keysecret
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"ghp_", "gho_", "ghu_", "ghs_", "ghr_", "github_pat_"}
}
// FromData will find and optionally verify GitHub secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
for _, match := range matches {
// First match is entire regex, second is the first group.
if len(match) != 2 {
continue
}
token := match[1]
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_Github,
Raw: []byte(token),
ExtraData: map[string]string{
"rotation_guide": "https://howtorotate.com/docs/tutorials/github/",
"version": fmt.Sprintf("%d", s.Version()),
},
}
if verify {
client := common.SaneHttpClient()
isVerified, userResponse, headers, err := s.VerifyGithub(ctx, client, token)
s1.Verified = isVerified
s1.SetVerificationError(err, token)
if userResponse != nil {
v1.SetUserResponse(userResponse, &s1)
}
if headers != nil {
v1.SetHeaderInfo(headers, &s1)
}
}
if !s1.Verified && detectors.IsKnownFalsePositive(string(s1.Raw), detectors.DefaultFalsePositives, true) {
continue
}
results = append(results, s1)
}
return
}
func (s Scanner) Type() detectorspb.DetectorType {
return detectorspb.DetectorType_Github
}
pkg/detectors/github/github_test.go → pkg/detectors/github/v2/github_test.go
+13 -6
View File
@@ -55,14 +55,15 @@ func TestGitHub_FromChunk(t *testing.T) {
DetectorType: detectorspb.DetectorType_Github,
Verified: true,
ExtraData: map[string]string{
"account_type": "User",
"company": "",
"name": "",
"account_type": "User",
// "company": "", // not present in test verifiedGhp
// "name": "", // not present in test verifiedGhp
"rotation_guide": "https://howtorotate.com/docs/tutorials/github/",
"scopes": "notifications",
"site_admin": "false",
"url": "https://github.com/truffle-sandbox",
"username": "truffle-sandbox",
// "site_admin": "false", // not present in test verifiedGhp
"url": "https://github.com/truffle-sandbox",
"username": "truffle-sandbox",
"version": "2",
},
},
},
@@ -82,6 +83,7 @@ func TestGitHub_FromChunk(t *testing.T) {
Verified: false,
ExtraData: map[string]string{
"rotation_guide": "https://howtorotate.com/docs/tutorials/github/",
"version": "2",
},
},
},
@@ -101,6 +103,7 @@ func TestGitHub_FromChunk(t *testing.T) {
Verified: false,
ExtraData: map[string]string{
"rotation_guide": "https://howtorotate.com/docs/tutorials/github/",
"version": "2",
},
},
},
@@ -120,6 +123,7 @@ func TestGitHub_FromChunk(t *testing.T) {
Verified: false,
ExtraData: map[string]string{
"rotation_guide": "https://howtorotate.com/docs/tutorials/github/",
"version": "2",
},
},
},
@@ -139,6 +143,7 @@ func TestGitHub_FromChunk(t *testing.T) {
Verified: false,
ExtraData: map[string]string{
"rotation_guide": "https://howtorotate.com/docs/tutorials/github/",
"version": "2",
},
},
},
@@ -158,6 +163,7 @@ func TestGitHub_FromChunk(t *testing.T) {
Verified: false,
ExtraData: map[string]string{
"rotation_guide": "https://howtorotate.com/docs/tutorials/github/",
"version": "2",
},
},
},
@@ -177,6 +183,7 @@ func TestGitHub_FromChunk(t *testing.T) {
Verified: false,
ExtraData: map[string]string{
"rotation_guide": "https://howtorotate.com/docs/tutorials/github/",
"version": "2",
},
},
},
pkg/detectors/gitlab/gitlab.go → pkg/detectors/gitlab/v1/gitlab.go
+3 -1
View File
@@ -4,11 +4,12 @@ import (
"context"
"encoding/json"
"fmt"
regexp "github.com/wasilibs/go-re2"
"io"
"net/http"
"strings"
regexp "github.com/wasilibs/go-re2"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
@@ -61,6 +62,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
}
s1.ExtraData = map[string]string{
"rotation_guide": "https://howtorotate.com/docs/tutorials/gitlab/",
"version": fmt.Sprintf("%d", s.Version()),
}
if verify {
pkg/detectors/gitlab/gitlab_v1_test.go → pkg/detectors/gitlab/v1/gitlab_v1_test.go
+24
View File
@@ -51,6 +51,10 @@ func TestGitlab_FromChunk(t *testing.T) {
{
DetectorType: detectorspb.DetectorType_Gitlab,
Verified: true,
ExtraData: map[string]string{
"rotation_guide": "https://howtorotate.com/docs/tutorials/gitlab/",
"version": "1",
},
},
},
wantErr: false,
@@ -67,6 +71,10 @@ func TestGitlab_FromChunk(t *testing.T) {
{
DetectorType: detectorspb.DetectorType_Gitlab,
Verified: true,
ExtraData: map[string]string{
"rotation_guide": "https://howtorotate.com/docs/tutorials/gitlab/",
"version": "1",
},
},
},
wantErr: false,
@@ -83,6 +91,10 @@ func TestGitlab_FromChunk(t *testing.T) {
{
DetectorType: detectorspb.DetectorType_Gitlab,
Verified: false,
ExtraData: map[string]string{
"rotation_guide": "https://howtorotate.com/docs/tutorials/gitlab/",
"version": "1",
},
},
},
wantErr: false,
@@ -99,6 +111,10 @@ func TestGitlab_FromChunk(t *testing.T) {
{
DetectorType: detectorspb.DetectorType_Gitlab,
Verified: false,
ExtraData: map[string]string{
"rotation_guide": "https://howtorotate.com/docs/tutorials/gitlab/",
"version": "1",
},
},
},
wantErr: false,
@@ -116,6 +132,10 @@ func TestGitlab_FromChunk(t *testing.T) {
{
DetectorType: detectorspb.DetectorType_Gitlab,
Verified: false,
ExtraData: map[string]string{
"rotation_guide": "https://howtorotate.com/docs/tutorials/gitlab/",
"version": "1",
},
},
},
wantErr: false,
@@ -133,6 +153,10 @@ func TestGitlab_FromChunk(t *testing.T) {
{
DetectorType: detectorspb.DetectorType_Gitlab,
Verified: true,
ExtraData: map[string]string{
"rotation_guide": "https://howtorotate.com/docs/tutorials/gitlab/",
"version": "1",
},
},
},
wantErr: false,
pkg/detectors/gitlab/gitlab_v2_test.go → pkg/detectors/gitlab/v1/gitlab_v2_test.go
View File
pkg/detectors/gitlabv2/gitlab_v1_test.go → pkg/detectors/gitlab/v2/gitlab_v1_test.go
+1 -1
View File
@@ -1,7 +1,7 @@
//go:build detectors
// +build detectors
package gitlabv2
package gitlab
import (
"context"
pkg/detectors/gitlabv2/gitlab_v2.go → pkg/detectors/gitlab/v2/gitlab_v2.go
+4 -2
View File
@@ -1,12 +1,13 @@
package gitlabv2
package gitlab
import (
"context"
"fmt"
regexp "github.com/wasilibs/go-re2"
"net/http"
"strings"
regexp "github.com/wasilibs/go-re2"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
@@ -53,6 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
}
s1.ExtraData = map[string]string{
"rotation_guide": "https://howtorotate.com/docs/tutorials/gitlab/",
"version": fmt.Sprintf("%d", s.Version()),
}
if verify {
pkg/detectors/gitlabv2/gitlab_v2_test.go → pkg/detectors/gitlab/v2/gitlab_v2_test.go
+21 -1
View File
@@ -1,7 +1,7 @@
//go:build detectors
// +build detectors
package gitlabv2
package gitlab
import (
"context"
@@ -51,6 +51,10 @@ func TestGitlabV2_FromChunk(t *testing.T) {
{
DetectorType: detectorspb.DetectorType_Gitlab,
Verified: true,
ExtraData: map[string]string{
"rotation_guide": "https://howtorotate.com/docs/tutorials/gitlab/",
"version": "2",
},
},
},
wantErr: false,
@@ -67,6 +71,10 @@ func TestGitlabV2_FromChunk(t *testing.T) {
{
DetectorType: detectorspb.DetectorType_Gitlab,
Verified: false,
ExtraData: map[string]string{
"rotation_guide": "https://howtorotate.com/docs/tutorials/gitlab/",
"version": "2",
},
},
},
wantErr: false,
@@ -83,6 +91,10 @@ func TestGitlabV2_FromChunk(t *testing.T) {
{
DetectorType: detectorspb.DetectorType_Gitlab,
Verified: false,
ExtraData: map[string]string{
"rotation_guide": "https://howtorotate.com/docs/tutorials/gitlab/",
"version": "2",
},
},
},
wantErr: false,
@@ -100,6 +112,10 @@ func TestGitlabV2_FromChunk(t *testing.T) {
{
DetectorType: detectorspb.DetectorType_Gitlab,
Verified: false,
ExtraData: map[string]string{
"rotation_guide": "https://howtorotate.com/docs/tutorials/gitlab/",
"version": "2",
},
},
},
wantErr: false,
@@ -117,6 +133,10 @@ func TestGitlabV2_FromChunk(t *testing.T) {
{
DetectorType: detectorspb.DetectorType_Gitlab,
Verified: true,
ExtraData: map[string]string{
"rotation_guide": "https://howtorotate.com/docs/tutorials/gitlab/",
"version": "2",
},
},
},
wantErr: false,
pkg/detectors/googleoauth2/googleoauth2_access_token.go
+1 -1
View File
@@ -122,7 +122,7 @@ func (s Scanner) verify(ctx context.Context, token string) (bool, map[string]str
return false, nil, fmt.Errorf("unexpected error description '%s' for %s", errInfo.Error, req.URL)
}
} else {
return false, nil, fmt.Errorf("unexpected respones %d for %s", res.StatusCode, req.URL)
return false, nil, fmt.Errorf("unexpected response %d for %s", res.StatusCode, req.URL)
}
}
pkg/detectors/guardianapi/guardianapi.go
+3 -2
View File
@@ -2,10 +2,11 @@ package guardianapi
import (
"context"
regexp "github.com/wasilibs/go-re2"
"net/http"
"strings"
regexp "github.com/wasilibs/go-re2"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
@@ -20,7 +21,7 @@ var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(`\b([0-9Aa-z]{8}-[0-9a-z]{4}-[0-9a-z]{4}-[0-9a-z]{4}-[0-9a-z]{12})\b`)
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"guardian"}) + `\b([0-9Aa-z]{8}-[0-9a-z]{4}-[0-9a-z]{4}-[0-9a-z]{4}-[0-9a-z]{12})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
pkg/detectors/ipify/ipify.go
-80
View File
@@ -1,80 +0,0 @@
package ipify
import (
"context"
"fmt"
regexp "github.com/wasilibs/go-re2"
"net/http"
"strings"
"time"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
type Scanner struct{}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"ipify"}) + `\b([a-zA-Z0-9_-]{32})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"ipify"}
}
// FromData will find and optionally verify Ipify secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
for _, match := range matches {
if len(match) != 2 {
continue
}
resMatch := strings.TrimSpace(match[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_Ipify,
Raw: []byte(resMatch),
}
if verify {
timeout := 10 * time.Second
client.Timeout = timeout
req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://geo.ipify.org/api/v2/country?apiKey=%s&ipAddress=8.8.8.8", resMatch), nil)
if err != nil {
continue
}
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
}
}
}
results = append(results, s1)
}
return results, nil
}
func (s Scanner) Type() detectorspb.DetectorType {
return detectorspb.DetectorType_Ipify
}
pkg/detectors/ipify/ipify_test.go
-120
View File
@@ -1,120 +0,0 @@
//go:build detectors
// +build detectors
package ipify
import (
"context"
"fmt"
"testing"
"time"
"github.com/kylelemons/godebug/pretty"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
func TestIpify_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
secret := testSecrets.MustGetField("IPIFY")
inactiveSecret := testSecrets.MustGetField("IPIFY_INACTIVE")
type args struct {
ctx context.Context
data []byte
verify bool
}
tests := []struct {
name string
s Scanner
args args
want []detectors.Result
wantErr bool
}{
{
name: "found, verified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a ipify secret %s within", secret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_Ipify,
Verified: true,
},
},
wantErr: false,
},
{
name: "found, unverified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a ipify secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_Ipify,
Verified: false,
},
},
wantErr: false,
},
{
name: "not found",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte("You cannot find the secret within"),
verify: true,
},
want: nil,
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
s := Scanner{}
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
if (err != nil) != tt.wantErr {
t.Errorf("Ipify.FromData() error = %v, wantErr %v", err, tt.wantErr)
return
}
for i := range got {
if len(got[i].Raw) == 0 {
t.Fatalf("no raw secret present: \n %+v", got[i])
}
got[i].Raw = nil
}
if diff := pretty.Compare(got, tt.want); diff != "" {
t.Errorf("Ipify.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
}
})
}
}
func BenchmarkFromData(benchmark *testing.B) {
ctx := context.Background()
s := Scanner{}
for name, data := range detectors.MustGetBenchmarkData() {
benchmark.Run(name, func(b *testing.B) {
b.ResetTimer()
for n := 0; n < b.N; n++ {
_, err := s.FromData(ctx, false, data)
if err != nil {
b.Fatal(err)
}
}
})
}
}
pkg/detectors/jdbc/jdbc.go
-1
View File
@@ -200,7 +200,6 @@ func tryRedactRegex(conn string) (string, bool) {
}
var supportedSubprotocols = map[string]func(string) (jdbc, error){
"sqlite": parseSqlite,
"mysql": parseMySQL,
"postgresql": parsePostgres,
"sqlserver": parseSqlServer,
pkg/detectors/jdbc/jdbc_integration_test.go
+97 -53
View File
@@ -7,61 +7,99 @@ import (
"context"
"fmt"
"os"
"os/exec"
"strings"
"testing"
"time"
"github.com/brianvoe/gofakeit/v7"
"github.com/kylelemons/godebug/pretty"
"github.com/stretchr/testify/assert"
"github.com/testcontainers/testcontainers-go"
"github.com/testcontainers/testcontainers-go/modules/mssql"
"github.com/testcontainers/testcontainers-go/modules/mysql"
"github.com/testcontainers/testcontainers-go/modules/postgres"
"github.com/testcontainers/testcontainers-go/wait"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
func TestMain(m *testing.M) {
code, err := runMain(m)
if err != nil {
panic(err)
}
os.Exit(code)
}
func runMain(m *testing.M) (int, error) {
for _, ctrl := range []struct {
start func() error
stop func()
}{
{startPostgres, stopPostgres},
{startMySQL, stopMySQL},
{startSqlServer, stopSqlServer},
} {
if err := ctrl.start(); err != nil {
return 0, err
}
defer ctrl.stop()
}
return m.Run(), nil
}
func dockerLogLine(hash string, needle string) chan struct{} {
ch := make(chan struct{}, 1)
go func() {
for {
out, err := exec.Command("docker", "logs", hash).CombinedOutput()
if err != nil {
panic(err)
}
if strings.Contains(string(out), needle) {
ch <- struct{}{}
return
}
time.Sleep(1 * time.Second)
}
}()
return ch
}
func TestJdbcVerified(t *testing.T) {
ctx := context.Background()
postgresUser := gofakeit.Username()
postgresPass := gofakeit.Password(true, true, true, false, false, 10)
postgresDB := gofakeit.Word()
postgresContainer, err := postgres.RunContainer(ctx,
testcontainers.WithImage("postgres:13-alpine"),
postgres.WithDatabase(postgresDB),
postgres.WithUsername(postgresUser),
postgres.WithPassword(postgresPass),
testcontainers.WithWaitStrategy(
wait.ForLog("database system is ready to accept connections").
WithOccurrence(2).
WithStartupTimeout(5*time.Second)),
)
if err != nil {
t.Fatal(err)
}
defer postgresContainer.Terminate(ctx)
postgresHost, err := postgresContainer.Host(ctx)
if err != nil {
t.Fatal(err)
}
postgresPort, err := postgresContainer.MappedPort(ctx, "5432")
if err != nil {
t.Fatal(err)
}
mysqlUser := gofakeit.Username()
mysqlPass := gofakeit.Password(true, true, true, false, false, 10)
mysqlDatabase := gofakeit.Word()
mysqlC, err := mysql.RunContainer(ctx,
mysql.WithDatabase(mysqlDatabase),
mysql.WithUsername(mysqlUser),
mysql.WithPassword(mysqlPass),
)
if err != nil {
t.Fatal(err)
}
defer mysqlC.Terminate(ctx)
mysqlHost, err := mysqlC.Host(ctx)
if err != nil {
t.Fatal(err)
}
mysqlPort, err := mysqlC.MappedPort(ctx, "3306")
if err != nil {
t.Fatal(err)
}
sqlServerPass := gofakeit.Password(true, true, true, false, false, 10)
sqlServerDatabase := "master"
mssqlContainer, err := mssql.RunContainer(ctx,
testcontainers.WithImage("mcr.microsoft.com/mssql/server:2022-RTM-GDR1-ubuntu-20.04"),
mssql.WithAcceptEULA(),
mssql.WithPassword(sqlServerPass),
)
if err != nil {
t.Fatal(err)
}
defer mssqlContainer.Terminate(ctx)
sqlServerHost, err := mssqlContainer.Host(ctx)
if err != nil {
t.Fatal(err)
}
sqlServerPort, err := mssqlContainer.MappedPort(ctx, "1433")
if err != nil {
t.Fatal(err)
}
type args struct {
ctx context.Context
data []byte
@@ -76,15 +114,17 @@ func TestJdbcVerified(t *testing.T) {
{
name: "postgres verified",
args: args{
ctx: context.Background(),
data: []byte(`jdbc connection string: jdbc:postgresql://localhost:5432/foo?sslmode=disable&password=` + postgresPass),
ctx: context.Background(),
data: []byte(fmt.Sprintf("jdbc connection string: jdbc:postgresql://%s:%s/%s?sslmode=disable&password=%s&user=%s",
postgresHost, postgresPort.Port(), postgresDB, postgresPass, postgresUser)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_JDBC,
Verified: true,
Redacted: "jdbc:postgresql://localhost:5432/foo?sslmode=disable&password=" + strings.Repeat("*", len(postgresPass)),
Redacted: fmt.Sprintf("jdbc:postgresql://%s:%s/%s?sslmode=disable&password=%s&user=%s",
postgresHost, postgresPort.Port(), postgresDB, strings.Repeat("*", len(postgresPass)), postgresUser),
},
},
wantErr: false,
@@ -92,15 +132,17 @@ func TestJdbcVerified(t *testing.T) {
{
name: "mysql verified",
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf(`CONN="jdbc:mysql://%s:%s@tcp(127.0.0.1:3306)/%s"`, mysqlUser, mysqlPass, mysqlDatabase)),
ctx: context.Background(),
data: []byte(fmt.Sprintf(`CONN="jdbc:mysql://%s:%s@tcp(%s:%s)/%s"`,
mysqlUser, mysqlPass, mysqlHost, mysqlPort.Port(), mysqlDatabase)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_JDBC,
Verified: true,
Redacted: fmt.Sprintf(`jdbc:mysql://%s:%s@tcp(127.0.0.1:3306)/%s`, mysqlUser, strings.Repeat("*", len(mysqlPass)), mysqlDatabase),
Redacted: fmt.Sprintf(`jdbc:mysql://%s:%s@tcp(%s:%s)/%s`,
mysqlUser, strings.Repeat("*", len(mysqlPass)), mysqlHost, mysqlPort.Port(), mysqlDatabase),
},
},
wantErr: false,
@@ -108,15 +150,17 @@ func TestJdbcVerified(t *testing.T) {
{
name: "sql server verified",
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("jdbc:sqlserver://odbc:server=localhost;database=%s;password=%s", sqlServerDatabase, sqlServerPass)),
ctx: context.Background(),
data: []byte(fmt.Sprintf("jdbc:sqlserver://odbc:server=%s;port=%s;database=%s;password=%s",
sqlServerHost, sqlServerPort.Port(), sqlServerDatabase, sqlServerPass)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_JDBC,
Verified: true,
Redacted: "jdbc:sqlserver://odbc:server=localhost;database=master;password=**************",
Redacted: fmt.Sprintf("jdbc:sqlserver://odbc:server=%s;port=%s;database=%s;password=%s",
sqlServerHost, sqlServerPort.Port(), sqlServerDatabase, strings.Repeat("*", len(sqlServerPass))),
},
},
wantErr: false,
pkg/detectors/jdbc/jdbc_test.go
+2 -17
View File
@@ -5,10 +5,11 @@ package jdbc
import (
"context"
"github.com/stretchr/testify/assert"
"os"
"testing"
"github.com/stretchr/testify/assert"
"github.com/kylelemons/godebug/pretty"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
@@ -68,22 +69,6 @@ func TestJdbc_FromChunk(t *testing.T) {
want: nil,
wantErr: false,
},
{
name: "sqlite unverified",
args: args{
ctx: context.Background(),
data: []byte("jdbc:sqlite::memory:"),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_JDBC,
Verified: false,
Redacted: "jdbc:sqlite::memory:",
},
},
wantErr: false,
},
{
name: "found double quoted string, unverified",
args: args{
pkg/detectors/jdbc/mysql.go
+44 -18
View File
@@ -3,22 +3,22 @@ package jdbc
import (
"context"
"errors"
"github.com/go-sql-driver/mysql"
"fmt"
"net/url"
"strings"
"github.com/go-sql-driver/mysql"
)
type mysqlJDBC struct {
conn string
userPass string
host string
database string
params string
}
func (s *mysqlJDBC) ping(ctx context.Context) pingResult {
return ping(ctx, "mysql", isMySQLErrorDeterminate,
s.conn,
buildMySQLConnectionString(s.host, s.database, s.userPass, s.params),
buildMySQLConnectionString(s.host, "", s.userPass, s.params))
}
@@ -51,25 +51,51 @@ func isMySQLErrorDeterminate(err error) bool {
func parseMySQL(subname string) (jdbc, error) {
// expected form: [subprotocol:]//[user:password@]HOST[/DB][?key=val[&key=val]]
hostAndDB, params, _ := strings.Cut(subname, "?")
if !strings.HasPrefix(hostAndDB, "//") {
if !strings.HasPrefix(subname, "//") {
return nil, errors.New("expected host to start with //")
}
userPassAndHostAndDB := strings.TrimPrefix(hostAndDB, "//")
userPass, hostAndDB, found := strings.Cut(userPassAndHostAndDB, "@")
if !found {
hostAndDB = userPass
userPass = ""
// need for hostnames that have tcp(host:port) format required by this database driver
cfg, err := mysql.ParseDSN(strings.TrimPrefix(subname, "//"))
if err == nil {
return &mysqlJDBC{
conn: subname[2:],
userPass: cfg.User + ":" + cfg.Passwd,
host: fmt.Sprintf("tcp(%s)", cfg.Addr),
params: "timeout=5s",
}, nil
}
host, database, found := strings.Cut(hostAndDB, "/")
if !found {
return nil, errors.New("expected host and database to be separated by /")
// for standard URI format, which is all i've seen for JDBC
u, err := url.Parse(subname)
if err != nil {
return nil, err
}
user := "root"
pass := ""
if u.User != nil {
user = u.User.Username()
pass, _ = u.User.Password()
}
if v := u.Query().Get("user"); v != "" {
user = v
}
if v := u.Query().Get("password"); v != "" {
pass = v
}
userAndPass := user
if pass != "" {
userAndPass = userAndPass + ":" + pass
}
return &mysqlJDBC{
conn: subname[2:],
userPass: userPass,
host: host,
database: database,
params: params,
userPass: userAndPass,
host: fmt.Sprintf("tcp(%s)", u.Host),
params: "timeout=5s",
}, nil
}
pkg/detectors/jdbc/mysql_integration_test.go
+63 -57
View File
@@ -4,55 +4,87 @@
package jdbc
import (
"bytes"
"context"
"errors"
"os/exec"
"fmt"
"testing"
"time"
"github.com/brianvoe/gofakeit/v7"
"github.com/google/go-cmp/cmp"
"github.com/stretchr/testify/assert"
)
const (
mysqlUser = "coolGuy"
mysqlPass = "23201dabb56ca236f3dc6736c0f9afad"
mysqlDatabase = "stuff"
"github.com/testcontainers/testcontainers-go/modules/mysql"
)
func TestMySQL(t *testing.T) {
mysqlUser := gofakeit.Username()
mysqlPass := gofakeit.Password(true, true, true, false, false, 10)
mysqlDatabase := gofakeit.Word()
ctx := context.Background()
mysqlC, err := mysql.RunContainer(ctx,
mysql.WithDatabase(mysqlDatabase),
mysql.WithUsername(mysqlUser),
mysql.WithPassword(mysqlPass),
)
if err != nil {
t.Fatal(err)
}
defer mysqlC.Terminate(ctx)
host, err := mysqlC.Host(ctx)
if err != nil {
t.Fatal(err)
}
port, err := mysqlC.MappedPort(ctx, "3306")
if err != nil {
t.Fatal(err)
}
type result struct {
parseErr bool
pingOk bool
pingDeterminate bool
ParseErr bool
PingOk bool
PingDeterminate bool
}
tests := []struct {
name string
input string
want result
}{
{
name: "empty",
input: "",
want: result{parseErr: true},
want: result{ParseErr: true},
},
{
input: "//" + mysqlUser + ":" + mysqlPass + "@tcp(127.0.0.1:3306)/" + mysqlDatabase,
want: result{pingOk: true, pingDeterminate: true},
name: "all good",
input: fmt.Sprintf("//%s:%s@tcp(%s:%s)/%s", mysqlUser, mysqlPass, host, port.Port(), mysqlDatabase),
want: result{PingOk: true, PingDeterminate: true},
},
{
input: "//wrongUser:wrongPass@tcp(127.0.0.1:3306)/" + mysqlDatabase,
want: result{pingOk: false, pingDeterminate: true},
name: "wrong creds",
input: fmt.Sprintf("//wrongUser:wrongPassword@tcp(%s:%s)/%s", host, port.Port(), mysqlDatabase),
want: result{PingOk: false, PingDeterminate: true},
},
{
input: "//" + mysqlUser + ":wrongPass@tcp(127.0.0.1:3306)/" + mysqlDatabase,
want: result{pingOk: false, pingDeterminate: true},
name: "wrong pass",
input: fmt.Sprintf("//%s:wrongPass@tcp(%s:%s)/%s", mysqlUser, host, port.Port(), mysqlDatabase),
want: result{PingOk: false, PingDeterminate: true},
},
{
input: "//" + mysqlUser + ":" + mysqlPass + "@tcp(127.0.0.1:3306)/",
want: result{pingOk: true, pingDeterminate: true},
name: "no db",
input: fmt.Sprintf("//%s:%s@tcp(%s:%s)/", mysqlUser, mysqlPass, host, port.Port()),
want: result{PingOk: true, PingDeterminate: true},
},
{
input: "//" + mysqlUser + ":" + mysqlPass + "@tcp(127.0.0.1:3306)/wrongDB",
want: result{pingOk: true, pingDeterminate: true},
name: "wrong db",
input: fmt.Sprintf("//%s:%s@tcp(%s:%s)/wrongDB", mysqlUser, mysqlPass, host, port.Port()),
want: result{PingOk: true, PingDeterminate: true},
},
{
name: "jdbc format",
input: fmt.Sprintf("//%s:%s@%s:%s/%s", mysqlUser, mysqlPass, host, port.Port(), mysqlDatabase),
want: result{PingOk: true, PingDeterminate: true},
},
}
for _, tt := range tests {
@@ -60,45 +92,19 @@ func TestMySQL(t *testing.T) {
j, err := parseMySQL(tt.input)
if err != nil {
got := result{parseErr: true}
got := result{ParseErr: true}
assert.Equal(t, tt.want, got)
return
}
pr := j.ping(context.Background())
got := result{pingOk: pr.err == nil, pingDeterminate: pr.determinate}
assert.Equal(t, tt.want, got)
got := result{PingOk: pr.err == nil, PingDeterminate: pr.determinate}
if diff := cmp.Diff(tt.want, got); diff != "" {
t.Errorf("%s: (-want +got)\n%s", tt.name, diff)
t.Errorf("error is: %v", pr.err)
}
})
}
}
var mysqlDockerHash string
func startMySQL() error {
cmd := exec.Command(
"docker", "run", "--rm", "-p", "3306:3306",
"-e", "MYSQL_ROOT_PASSWORD=403a96cff2a323f74bfb1c16992895be",
"-e", "MYSQL_USER="+mysqlUser,
"-e", "MYSQL_PASSWORD="+mysqlPass,
"-e", "MYSQL_DATABASE="+mysqlDatabase,
"-e", "MYSQL_ROOT_HOST=%",
"-d", "mysql",
)
out, err := cmd.Output()
if err != nil {
return err
}
mysqlDockerHash = string(bytes.TrimSpace(out))
select {
case <-dockerLogLine(mysqlDockerHash, "socket: '/var/run/mysqld/mysqld.sock' port: 3306"):
return nil
case <-time.After(30 * time.Second):
stopMySQL()
return errors.New("timeout waiting for mysql database to be ready")
}
}
func stopMySQL() {
exec.Command("docker", "kill", mysqlDockerHash).Run()
}
pkg/detectors/jdbc/postgres.go
+44 -27
View File
@@ -4,6 +4,7 @@ import (
"context"
"errors"
"fmt"
"net/url"
"strings"
"github.com/lib/pq"
@@ -23,8 +24,7 @@ func (s *postgresJDBC) ping(ctx context.Context) pingResult {
return ping(ctx, "postgres", isPostgresErrorDeterminate,
buildPostgresConnectionString(s.params, true),
buildPostgresConnectionString(s.params, false),
s.conn,
"postgres://"+s.conn)
)
}
func isPostgresErrorDeterminate(err error) bool {
@@ -59,36 +59,50 @@ func joinKeyValues(m map[string]string, sep string) string {
func parsePostgres(subname string) (jdbc, error) {
// expected form: [subprotocol:]//[user:password@]HOST[/DB][?key=val[&key=val]]
hostAndDB, paramString, _ := strings.Cut(subname, "?")
if !strings.HasPrefix(hostAndDB, "//") {
if !strings.HasPrefix(subname, "//") {
return nil, errors.New("expected host to start with //")
}
userPassAndHostAndDB := strings.TrimPrefix(hostAndDB, "//")
userPass, hostAndDB, found := strings.Cut(userPassAndHostAndDB, "@")
var user, pass string
if found {
user, pass, _ = strings.Cut(userPass, ":")
} else {
hostAndDB = userPass
u, err := url.Parse(subname)
if err != nil {
return nil, err
}
host, database, found := strings.Cut(hostAndDB, "/")
if !found {
return nil, errors.New("expected host and database to be separated by /")
dbName := strings.TrimPrefix(u.Path, "/")
if dbName == "" {
dbName = "postgres"
}
params := map[string]string{
"host": host,
"dbname": database,
"host": u.Host,
"dbname": dbName,
"connect_timeout": "5",
}
if len(user) > 0 {
params["user"] = user
if u.User != nil {
params["user"] = u.User.Username()
pass, set := u.User.Password()
if set {
params["password"] = pass
}
}
if len(pass) > 0 {
params["password"] = pass
if v := u.Query()["sslmode"]; len(v) > 0 {
switch v[0] {
// https://www.postgresql.org/docs/current/libpq-ssl.html#LIBPQ-SSL-PROTECTION
case "disable", "allow", "prefer",
"require", "verify-ca", "verify-full":
params["sslmode"] = v[0]
}
}
for _, param := range strings.Split(paramString, "&") {
key, val, _ := strings.Cut(param, "=")
params[key] = val
if v := u.Query().Get("user"); v != "" {
params["user"] = v
}
if v := u.Query().Get("password"); v != "" {
params["password"] = v
}
return &postgresJDBC{subname[2:], params}, nil
@@ -107,11 +121,14 @@ func buildPostgresConnectionString(params map[string]string, includeDbName bool)
continue
}
}
if key == "dbname" && !includeDbName {
continue
}
data[key] = val
}
return joinKeyValues(data, " ")
if !includeDbName {
data["dbname"] = "postgres"
}
connStr := joinKeyValues(data, " ")
return connStr
}
pkg/detectors/jdbc/postgres_integration_test.go
+94 -68
View File
@@ -4,114 +4,140 @@
package jdbc
import (
"bytes"
"context"
"errors"
"fmt"
"os/exec"
"log"
"testing"
"time"
"github.com/stretchr/testify/assert"
)
const (
postgresUser = "postgres"
postgresPass = "23201dabb56ca236f3dc6736c0f9afad"
"github.com/brianvoe/gofakeit/v7"
"github.com/google/go-cmp/cmp"
"github.com/testcontainers/testcontainers-go"
"github.com/testcontainers/testcontainers-go/modules/postgres"
"github.com/testcontainers/testcontainers-go/wait"
)
func TestPostgres(t *testing.T) {
type result struct {
parseErr bool
pingOk bool
pingDeterminate bool
ParseErr bool
PingOk bool
PingDeterminate bool
}
user := gofakeit.Username()
pass := gofakeit.Password(true, true, true, false, false, 32)
dbName := gofakeit.Word()
t.Log("user: ", user)
t.Log("dbName: ", dbName)
ctx := context.Background()
postgresContainer, err := postgres.RunContainer(ctx,
testcontainers.WithImage("postgres:13-alpine"),
postgres.WithDatabase(dbName),
postgres.WithUsername(user),
postgres.WithPassword(pass),
testcontainers.WithWaitStrategy(
wait.ForLog("database system is ready to accept connections").
WithOccurrence(2).
WithStartupTimeout(5*time.Second)),
)
if err != nil {
t.Fatal(err)
}
host, err := postgresContainer.Host(ctx)
if err != nil {
t.Fatal(err)
}
port, err := postgresContainer.MappedPort(ctx, "5432")
if err != nil {
t.Fatal(err)
}
if err != nil {
log.Fatalf("failed to start container: %s", err)
}
defer postgresContainer.Terminate(ctx)
tests := []struct {
name string
input string
want result
}{
{
input: "//localhost:5432/foo?sslmode=disable&password=" + postgresPass,
want: result{pingOk: true, pingDeterminate: true},
name: "invalid password",
input: fmt.Sprintf("//%s:%s/foo?sslmode=disable&password=foo", host, port.Port()),
want: result{PingOk: false, PingDeterminate: true},
},
{
input: fmt.Sprintf("//postgres:%s@localhost:5432/foo?sslmode=disable", postgresPass),
want: result{pingOk: true, pingDeterminate: true},
name: "valid password - no user name",
input: fmt.Sprintf("//%s:%s/foo?sslmode=disable&password=%s", host, port.Port(), pass),
want: result{PingOk: false, PingDeterminate: true},
},
{
input: "//localhost:5432/foo?sslmode=disable&user=" + postgresUser + "&password=" + postgresPass,
want: result{pingOk: true, pingDeterminate: true},
name: "exact username and password, wrong db",
input: fmt.Sprintf("//%s:%s@%s:%s/foo?sslmode=disable", user, pass, host, port.Port()),
want: result{PingOk: true, PingDeterminate: true},
},
{
input: fmt.Sprintf("//%s:%s@localhost:5432/foo?sslmode=disable", postgresUser, postgresPass),
want: result{pingOk: true, pingDeterminate: true},
name: "exact username and password, no db",
input: fmt.Sprintf("//%s:%s@%s:%s?sslmode=disable", user, pass, host, port.Port()),
want: result{PingOk: true, PingDeterminate: true},
},
{
input: "//localhost/foo?sslmode=disable&port=5432&password=" + postgresPass,
want: result{pingOk: true, pingDeterminate: true},
name: "invalid user name",
input: fmt.Sprintf("//%s:%s/foo?sslmode=disable&user=foo&password=%s", host, port.Port(), pass),
want: result{PingOk: false, PingDeterminate: true},
},
{
input: "//localhost:5432/foo?password=" + postgresPass,
want: result{pingOk: false, pingDeterminate: false},
name: "invalid hostname",
input: fmt.Sprintf("//badhost:%s/foo?sslmode=disable&user=foo&password=%s", port.Port(), pass),
want: result{PingOk: false, PingDeterminate: false},
},
{
input: "//localhost:5432/foo?sslmode=disable&password=foo",
want: result{pingOk: false, pingDeterminate: true},
name: "no username, password",
input: fmt.Sprintf("//%s:%s/foo?password=%s", host, port.Port(), pass),
want: result{PingOk: false, PingDeterminate: false},
},
{
input: "//localhost:5432/foo?sslmode=disable&user=foo&password=" + postgresPass,
want: result{pingOk: false, pingDeterminate: true},
},
{
input: "//badhost:5432/foo?sslmode=disable&user=foo&password=" + postgresPass,
want: result{pingOk: false, pingDeterminate: false},
name: "db, password - no username",
input: fmt.Sprintf("//%s:%s/foo?sslmode=disable&password=%s", host, port.Port(), pass),
want: result{PingOk: false, PingDeterminate: true},
},
{
name: "invalid format",
input: "invalid",
want: result{parseErr: true},
want: result{ParseErr: true},
},
{
name: "normal connect with generated username and password",
input: fmt.Sprintf("//%s:%s@%s:%s/%s?sslmode=disable", user, pass, host, port.Port(), dbName),
want: result{PingOk: true, PingDeterminate: true},
},
}
for _, tt := range tests {
t.Run(tt.input, func(t *testing.T) {
j, err := parsePostgres(tt.input)
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
j, err := parsePostgres(tt.input)
if err != nil {
got := result{parseErr: true}
assert.Equal(t, tt.want, got)
got := result{ParseErr: true}
if diff := cmp.Diff(tt.want, got); diff != "" {
t.Errorf("%s: (-want +got)\n%s", tt.name, diff)
t.Errorf("error is: %v", err)
}
return
}
pr := j.ping(context.Background())
pr := j.ping(ctx)
got := result{pingOk: pr.err == nil, pingDeterminate: pr.determinate}
assert.Equal(t, tt.want, got)
got := result{PingOk: pr.err == nil, PingDeterminate: pr.determinate}
if diff := cmp.Diff(tt.want, got); diff != "" {
t.Errorf("%s: (-want +got)\n%s", tt.name, diff)
t.Errorf("error is: %v", pr.err)
}
})
}
}
var postgresDockerHash string
func startPostgres() error {
cmd := exec.Command(
"docker", "run", "--rm", "-p", "5432:5432",
"-e", "POSTGRES_PASSWORD="+postgresPass,
"-e", "POSTGRES_USER="+postgresUser,
"-d", "postgres",
)
out, err := cmd.Output()
if err != nil {
return err
}
postgresDockerHash = string(bytes.TrimSpace(out))
select {
case <-dockerLogLine(postgresDockerHash, "PostgreSQL init process complete; ready for start up."):
return nil
case <-time.After(30 * time.Second):
stopPostgres()
return errors.New("timeout waiting for postgres database to be ready")
}
}
func stopPostgres() {
exec.Command("docker", "kill", postgresDockerHash).Run()
}
pkg/detectors/jdbc/sqlite.go
-43
View File
@@ -1,43 +0,0 @@
package jdbc
import (
"context"
"errors"
"strings"
_ "github.com/mattn/go-sqlite3"
)
type sqliteJDBC struct {
filename string
params map[string]string
testing bool
}
var cannotVerifySqliteError error = errors.New("sqlite credentials cannot be verified")
func (s *sqliteJDBC) ping(ctx context.Context) pingResult {
if !s.testing {
// sqlite is not a networked database, so we cannot verify
return pingResult{cannotVerifySqliteError, true}
}
return ping(ctx, "sqlite3", isSqliteErrorDeterminate, s.filename)
}
func isSqliteErrorDeterminate(err error) bool {
return true
}
func parseSqlite(subname string) (jdbc, error) {
filename, params, _ := strings.Cut(subname, "?")
if filename == "" {
return nil, errors.New("empty filename")
}
j := &sqliteJDBC{filename: filename, params: map[string]string{}}
for _, keyVal := range strings.Split(params, "&") {
if key, val, found := strings.Cut(keyVal, "="); found {
j.params[key] = val
}
}
return j, nil
}
pkg/detectors/jdbc/sqlite_test.go
-46
View File
@@ -1,46 +0,0 @@
//go:build detectors
// +build detectors
package jdbc
import (
"context"
"testing"
"github.com/stretchr/testify/assert"
)
func parseSqliteTest(subname string) (jdbc, error) {
j, err := parseSqlite(subname)
if err != nil {
return j, err
}
j.(*sqliteJDBC).testing = true
return j, err
}
func TestParseSqlite(t *testing.T) {
type testStruct struct {
input string
wantErr bool
}
tests := []struct {
input string
wantErr bool
}{
{input: "", wantErr: true},
{input: ":memory:"},
}
for _, test := range tests {
t.Run(test.input, func(t *testing.T) {
j, err := parseSqliteTest(test.input)
if test.wantErr {
assert.Error(t, err)
} else {
assert.NoError(t, err)
assert.True(t, j.ping(context.Background()).err == nil)
}
})
}
}
pkg/detectors/jdbc/sqlserver.go
+29 -17
View File
@@ -3,20 +3,19 @@ package jdbc
import (
"context"
"errors"
mssql "github.com/denisenkom/go-mssqldb"
"fmt"
"strings"
mssql "github.com/microsoft/go-mssqldb"
)
type sqlServerJDBC struct {
conn string
params map[string]string
connStr string
}
func (s *sqlServerJDBC) ping(ctx context.Context) pingResult {
return ping(ctx, "mssql", isSqlServerErrorDeterminate,
joinKeyValues(s.params, ";"),
s.conn,
"sqlserver://"+s.conn)
s.connStr)
}
func isSqlServerErrorDeterminate(err error) bool {
@@ -38,22 +37,35 @@ func parseSqlServer(subname string) (jdbc, error) {
return nil, errors.New("expected connection to start with //")
}
conn := strings.TrimPrefix(subname, "//")
params := map[string]string{
"user id": "sa",
"database": "master",
}
for _, param := range strings.Split(conn, ";") {
port := "1433"
var password, host string
for i, param := range strings.Split(conn, ";") {
key, value, found := strings.Cut(param, "=")
if !found {
if !found && i == 0 {
// String connectionUrl = "jdbc:sqlserver://<server>:<port>;encrypt=true;databaseName=AdventureWorks;user=<user>;password=<password>";
if split := strings.Split(param, ":"); len(split) > 1 {
host = split[0]
port = split[1]
} else {
host = param
}
continue
}
params[key] = value
if key != "password" && strings.Contains(strings.ToLower(key), "password") {
params["password"] = value
switch strings.ToLower(key) {
case "password":
password = value
case "spring.datasource.password":
password = value
case "server":
host = value
case "port":
port = value
}
}
return &sqlServerJDBC{
conn: strings.TrimPrefix(subname, "//"),
params: params,
connStr: fmt.Sprintf("sqlserver://sa:%s@%s:%s?database=master&connection+timeout=5", password, host, port),
}, nil
}
pkg/detectors/jdbc/sqlserver_integration_test.go
+44 -43
View File
@@ -4,23 +4,44 @@
package jdbc
import (
"bytes"
"context"
"errors"
"os/exec"
"fmt"
"testing"
"time"
"github.com/brianvoe/gofakeit/v7"
"github.com/stretchr/testify/assert"
)
const (
sqlServerPass = "Secr3tP@s5w0rd"
sqlServerUser = "sa"
sqlServerDatabase = "master"
"github.com/testcontainers/testcontainers-go"
"github.com/testcontainers/testcontainers-go/modules/mssql"
)
func TestSqlServer(t *testing.T) {
ctx := context.Background()
sqlServerUser := "sa"
sqlServerPass := gofakeit.Password(true, true, true, false, false, 10)
sqlServerDB := "master"
mssqlContainer, err := mssql.RunContainer(ctx,
testcontainers.WithImage("mcr.microsoft.com/mssql/server:2022-RTM-GDR1-ubuntu-20.04"),
mssql.WithAcceptEULA(),
mssql.WithPassword(sqlServerPass),
)
if err != nil {
t.Fatal(err)
}
defer mssqlContainer.Terminate(ctx)
mssqlHost, err := mssqlContainer.Host(ctx)
if err != nil {
t.Fatal(err)
}
mssqlPort, err := mssqlContainer.MappedPort(ctx, "1433")
if err != nil {
t.Fatal(err)
}
type result struct {
parseErr bool
pingOk bool
@@ -35,21 +56,28 @@ func TestSqlServer(t *testing.T) {
want: result{parseErr: true},
},
{
input: "//server=localhost;user id=sa;database=master;password=" + sqlServerPass,
want: result{pingOk: true, pingDeterminate: true},
input: fmt.Sprintf("//server=%s;port=%s;user id=%s;database=%s;password=%s",
mssqlHost, mssqlPort.Port(), sqlServerUser, sqlServerDB, sqlServerPass),
want: result{pingOk: true, pingDeterminate: true},
},
{
input: "//server=badhost;user id=sa;database=master;password=" + sqlServerPass,
input: "//server=badhost;user id=sa;database=master;password=",
want: result{pingOk: false, pingDeterminate: false},
},
{
input: "//localhost;database=master;spring.datasource.password=" + sqlServerPass,
want: result{pingOk: true, pingDeterminate: true},
input: fmt.Sprintf("//%s;database=master;spring.datasource.password=%s;port=%s",
mssqlHost, sqlServerPass, mssqlPort.Port()),
want: result{pingOk: true, pingDeterminate: true},
},
{
input: "//localhost;database=master;spring.datasource.password=badpassword",
input: fmt.Sprintf("//%s;database=master;spring.datasource.password=badpassword;port=%s", mssqlHost, mssqlPort.Port()),
want: result{pingOk: false, pingDeterminate: true},
},
{
input: fmt.Sprintf("//%s:%s;databaseName=master;user=%s;password=%s",
mssqlHost, mssqlPort.Port(), sqlServerUser, sqlServerPass),
want: result{pingOk: true, pingDeterminate: true},
},
}
for _, tt := range tests {
t.Run(tt.input, func(t *testing.T) {
@@ -68,30 +96,3 @@ func TestSqlServer(t *testing.T) {
})
}
}
var sqlServerDockerHash string
func startSqlServer() error {
cmd := exec.Command(
"docker", "run", "--rm", "-p", "1433:1433",
"-e", "ACCEPT_EULA=1",
"-e", "MSSQL_SA_PASSWORD="+sqlServerPass,
"-d", "mcr.microsoft.com/azure-sql-edge",
)
out, err := cmd.Output()
if err != nil {
return err
}
sqlServerDockerHash = string(bytes.TrimSpace(out))
select {
case <-dockerLogLine(sqlServerDockerHash, "EdgeTelemetry starting up"):
return nil
case <-time.After(30 * time.Second):
stopSqlServer()
return errors.New("timeout waiting for sql server database to be ready")
}
}
func stopSqlServer() {
exec.Command("docker", "kill", sqlServerDockerHash).Run()
}
pkg/detectors/jiratoken/jiratoken.go → pkg/detectors/jiratoken/v1/jiratoken.go
+6 -4
View File
@@ -4,10 +4,11 @@ import (
"context"
b64 "encoding/base64"
"fmt"
regexp "github.com/wasilibs/go-re2"
"net/http"
"strings"
regexp "github.com/wasilibs/go-re2"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
@@ -72,9 +73,10 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
DetectorType: detectorspb.DetectorType_JiraToken,
Raw: []byte(resToken),
RawV2: []byte(fmt.Sprintf("%s:%s:%s", resEmail, resToken, resDomain)),
}
s1.ExtraData = map[string]string{
"rotation_guide": "https://howtorotate.com/docs/tutorials/atlassian/",
ExtraData: map[string]string{
"rotation_guide": "https://howtorotate.com/docs/tutorials/atlassian/",
"version": fmt.Sprintf("%d", s.Version()),
},
}
if verify {
pkg/detectors/jiratoken/jiratoken_test.go → pkg/detectors/jiratoken/v1/jiratoken_test.go
+16
View File
@@ -54,6 +54,10 @@ func TestJiraToken_FromChunk(t *testing.T) {
{
DetectorType: detectorspb.DetectorType_JiraToken,
Verified: true,
ExtraData: map[string]string{
"rotation_guide": "https://howtorotate.com/docs/tutorials/atlassian/",
"version": "1",
},
},
},
wantErr: false,
@@ -71,6 +75,10 @@ func TestJiraToken_FromChunk(t *testing.T) {
{
DetectorType: detectorspb.DetectorType_JiraToken,
Verified: false,
ExtraData: map[string]string{
"rotation_guide": "https://howtorotate.com/docs/tutorials/atlassian/",
"version": "1",
},
},
},
wantErr: false,
@@ -100,6 +108,10 @@ func TestJiraToken_FromChunk(t *testing.T) {
{
DetectorType: detectorspb.DetectorType_JiraToken,
Verified: false,
ExtraData: map[string]string{
"rotation_guide": "https://howtorotate.com/docs/tutorials/atlassian/",
"version": "1",
},
},
},
wantErr: false,
@@ -117,6 +129,10 @@ func TestJiraToken_FromChunk(t *testing.T) {
{
DetectorType: detectorspb.DetectorType_JiraToken,
Verified: false,
ExtraData: map[string]string{
"rotation_guide": "https://howtorotate.com/docs/tutorials/atlassian/",
"version": "1",
},
},
},
wantErr: false,
pkg/detectors/jiratoken_v2/jiratoken_v2.go → pkg/detectors/jiratoken/v2/jiratoken_v2.go
+7 -2
View File
@@ -1,13 +1,14 @@
package jiratoken_v2
package jiratoken
import (
"context"
b64 "encoding/base64"
"fmt"
regexp "github.com/wasilibs/go-re2"
"net/http"
"strings"
regexp "github.com/wasilibs/go-re2"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
@@ -73,6 +74,10 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
DetectorType: detectorspb.DetectorType_JiraToken,
Raw: []byte(resToken),
RawV2: []byte(fmt.Sprintf("%s:%s:%s", resEmail, resToken, resDomain)),
ExtraData: map[string]string{
"rotation_guide": "https://howtorotate.com/docs/tutorials/atlassian/",
"version": fmt.Sprintf("%d", s.Version()),
},
}
if verify {
pkg/detectors/jiratoken_v2/jiratoken_v2_test.go → pkg/detectors/jiratoken/v2/jiratoken_v2_test.go
+17 -1
View File
@@ -1,7 +1,7 @@
//go:build detectors
// +build detectors
package jiratoken_v2
package jiratoken
import (
"context"
@@ -54,6 +54,10 @@ func TestJiraToken_FromChunk(t *testing.T) {
{
DetectorType: detectorspb.DetectorType_JiraToken,
Verified: true,
ExtraData: map[string]string{
"rotation_guide": "https://howtorotate.com/docs/tutorials/atlassian/",
"version": "2",
},
},
},
wantErr: false,
@@ -71,6 +75,10 @@ func TestJiraToken_FromChunk(t *testing.T) {
{
DetectorType: detectorspb.DetectorType_JiraToken,
Verified: false,
ExtraData: map[string]string{
"rotation_guide": "https://howtorotate.com/docs/tutorials/atlassian/",
"version": "2",
},
},
},
wantErr: false,
@@ -100,6 +108,10 @@ func TestJiraToken_FromChunk(t *testing.T) {
{
DetectorType: detectorspb.DetectorType_JiraToken,
Verified: false,
ExtraData: map[string]string{
"rotation_guide": "https://howtorotate.com/docs/tutorials/atlassian/",
"version": "2",
},
},
},
wantErr: false,
@@ -117,6 +129,10 @@ func TestJiraToken_FromChunk(t *testing.T) {
{
DetectorType: detectorspb.DetectorType_JiraToken,
Verified: false,
ExtraData: map[string]string{
"rotation_guide": "https://howtorotate.com/docs/tutorials/atlassian/",
"version": "2",
},
},
},
wantErr: false,
pkg/detectors/jupiterone/jupiterone.go
+99
View File
@@ -0,0 +1,99 @@
package jupiterone
import (
"context"
"fmt"
"net/http"
"strings"
regexp "github.com/wasilibs/go-re2"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
type Scanner struct {
client *http.Client
}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var (
defaultClient = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"jupiterone"}) + `\b([0-9a-zA-Z]{76})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"jupiterone"}
}
// FromData will find and optionally verify Jupiterone secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
for _, match := range matches {
if len(match) != 2 {
continue
}
resMatch := strings.TrimSpace(match[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_JupiterOne,
Raw: []byte(resMatch),
}
if verify {
client := s.client
if client == nil {
client = defaultClient
}
payload := strings.NewReader(`{
"query": "query J1QL($query: String! = \"find jupiterone_account\", $variables: JSON, $cursor: String, $scopeFilters: [JSON!], $flags: QueryV1Flags) { queryV1(query: $query, variables: $variables, cursor: $cursor, scopeFilters: $scopeFilters, flags: $flags) { type data cursor }}"
}`,
)
req, err := http.NewRequestWithContext(ctx, "POST", "https://graphql.us.jupiterone.io/", payload)
if err != nil {
continue
}
req.Header.Add("Authorization", "Bearer "+resMatch)
req.Header.Add("JupiterOne-Account", "12345678-1234-1234-1234-123412341234") // dummy account number
req.Header.Add("Content-Type", "application/json")
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode == 200 {
s1.Verified = true
} else if res.StatusCode == 401 {
// The secret is determinately not verified (nothing to do)
} else {
s1.SetVerificationError(fmt.Errorf("unexpected HTTP response status %d", res.StatusCode), resMatch)
}
} else {
s1.SetVerificationError(err, resMatch)
}
}
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if !s1.Verified && detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
results = append(results, s1)
}
return results, nil
}
func (s Scanner) Type() detectorspb.DetectorType {
return detectorspb.DetectorType_JupiterOne
}
pkg/detectors/jupiterone/jupiterone_test.go
+162
View File
@@ -0,0 +1,162 @@
//go:build detectors
// +build detectors
package jupiterone
import (
"context"
"fmt"
"testing"
"time"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
func TestJupiterone_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
secret := testSecrets.MustGetField("JUPITERONE")
inactiveSecret := testSecrets.MustGetField("JUPITERONE_INACTIVE")
type args struct {
ctx context.Context
data []byte
verify bool
}
tests := []struct {
name string
s Scanner
args args
want []detectors.Result
wantErr bool
wantVerificationErr bool
}{
{
name: "found, verified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a jupiterone secret %s within", secret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_JupiterOne,
Verified: true,
},
},
wantErr: false,
wantVerificationErr: false,
},
{
name: "found, unverified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a jupiterone secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_JupiterOne,
Verified: false,
},
},
wantErr: false,
wantVerificationErr: false,
},
{
name: "not found",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte("You cannot find the secret within"),
verify: true,
},
want: nil,
wantErr: false,
wantVerificationErr: false,
},
{
name: "found, would be verified if not for timeout",
s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a jupiterone secret %s within", secret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_JupiterOne,
Verified: false,
},
},
wantErr: false,
wantVerificationErr: true,
},
{
name: "found, verified but unexpected api surface",
s: Scanner{client: common.ConstantResponseHttpClient(404, "")},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a jupiterone secret %s within", secret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_JupiterOne,
Verified: false,
},
},
wantErr: false,
wantVerificationErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
if (err != nil) != tt.wantErr {
t.Errorf("Jupiterone.FromData() error = %v, wantErr %v", err, tt.wantErr)
return
}
for i := range got {
if len(got[i].Raw) == 0 {
t.Fatalf("no raw secret present: \n %+v", got[i])
}
if (got[i].VerificationError() != nil) != tt.wantVerificationErr {
t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())
}
}
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
t.Errorf("Jupiterone.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
}
})
}
}
func BenchmarkFromData(benchmark *testing.B) {
ctx := context.Background()
s := Scanner{}
for name, data := range detectors.MustGetBenchmarkData() {
benchmark.Run(name, func(b *testing.B) {
b.ResetTimer()
for n := 0; n < b.N; n++ {
_, err := s.FromData(ctx, false, data)
if err != nil {
b.Fatal(err)
}
}
})
}
}
pkg/detectors/lastfm/lastfm.go
-76
View File
@@ -1,76 +0,0 @@
package lastfm
import (
"context"
regexp "github.com/wasilibs/go-re2"
"net/http"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
type Scanner struct{}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"lastfm"}) + `\b([0-9a-z]{32})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"lastfm"}
}
// FromData will find and optionally verify Lastfm secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
for _, match := range matches {
if len(match) != 2 {
continue
}
resMatch := strings.TrimSpace(match[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_Lastfm,
Raw: []byte(resMatch),
}
if verify {
req, err := http.NewRequestWithContext(ctx, "GET", "https://ws.audioscrobbler.com/2.0/?method=artist.getSimilar&api_key="+resMatch, nil)
if err != nil {
continue
}
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
}
}
}
results = append(results, s1)
}
return results, nil
}
func (s Scanner) Type() detectorspb.DetectorType {
return detectorspb.DetectorType_Lastfm
}
pkg/detectors/lastfm/lastfm_test.go
-120
View File
@@ -1,120 +0,0 @@
//go:build detectors
// +build detectors
package lastfm
import (
"context"
"fmt"
"testing"
"time"
"github.com/kylelemons/godebug/pretty"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
func TestLastfm_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
secret := testSecrets.MustGetField("LASTFM")
inactiveSecret := testSecrets.MustGetField("LASTFM_INACTIVE")
type args struct {
ctx context.Context
data []byte
verify bool
}
tests := []struct {
name string
s Scanner
args args
want []detectors.Result
wantErr bool
}{
{
name: "found, verified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a lastfm secret %s within", secret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_Lastfm,
Verified: true,
},
},
wantErr: false,
},
{
name: "found, unverified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a lastfm secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_Lastfm,
Verified: false,
},
},
wantErr: false,
},
{
name: "not found",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte("You cannot find the secret within"),
verify: true,
},
want: nil,
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
s := Scanner{}
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
if (err != nil) != tt.wantErr {
t.Errorf("Lastfm.FromData() error = %v, wantErr %v", err, tt.wantErr)
return
}
for i := range got {
if len(got[i].Raw) == 0 {
t.Fatalf("no raw secret present: \n %+v", got[i])
}
got[i].Raw = nil
}
if diff := pretty.Compare(got, tt.want); diff != "" {
t.Errorf("Lastfm.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
}
})
}
}
func BenchmarkFromData(benchmark *testing.B) {
ctx := context.Background()
s := Scanner{}
for name, data := range detectors.MustGetBenchmarkData() {
benchmark.Run(name, func(b *testing.B) {
b.ResetTimer()
for n := 0; n < b.N; n++ {
_, err := s.FromData(ctx, false, data)
if err != nil {
b.Fatal(err)
}
}
})
}
}
pkg/detectors/launchdarkly/launchdarkly.go
+4 -3
View File
@@ -4,17 +4,18 @@ import (
"context"
"errors"
"fmt"
regexp "github.com/wasilibs/go-re2"
"net/http"
"strings"
"time"
regexp "github.com/wasilibs/go-re2"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
ldclient "github.com/launchdarkly/go-server-sdk/v6"
"github.com/launchdarkly/go-server-sdk/v6/ldcomponents"
ldclient "github.com/launchdarkly/go-server-sdk/v7"
"github.com/launchdarkly/go-server-sdk/v7/ldcomponents"
)
type Scanner struct {
pkg/detectors/ldap/ldap.go
+5 -3
View File
@@ -4,13 +4,14 @@ import (
"context"
"crypto/tls"
"fmt"
regexp "github.com/wasilibs/go-re2"
"net"
"net/url"
"strings"
"time"
"github.com/go-ldap/ldap/v3"
regexp "github.com/wasilibs/go-re2"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
@@ -151,16 +152,17 @@ func verifyLDAP(username, password string, ldapURL *url.URL) error {
return l.Bind(username, password)
case "ldaps":
// TLS dial
l, err := ldap.DialTLS("tcp", uri, &tls.Config{InsecureSkipVerify: true})
l, err := ldap.DialURL(uri, ldap.DialWithTLSConfig(&tls.Config{InsecureSkipVerify: true}))
if err != nil {
return err
}
defer l.Close()
// TLS verify
return l.Bind(username, password)
default:
return fmt.Errorf("unknown ldap scheme %q", ldapURL.Scheme)
}
return fmt.Errorf("unknown ldap scheme %q", ldapURL.Scheme)
}
func (s Scanner) Type() detectorspb.DetectorType {
pkg/detectors/macaddress/macaddress.go
-75
View File
@@ -1,75 +0,0 @@
package macaddress
import (
"context"
"fmt"
regexp "github.com/wasilibs/go-re2"
"net/http"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
type Scanner struct{}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"macaddress"}) + `\b([a-zA-Z0-9_]{32})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"macaddress"}
}
// FromData will find and optionally verify Macaddress secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
for _, match := range matches {
if len(match) != 2 {
continue
}
resMatch := strings.TrimSpace(match[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_Macaddress,
Raw: []byte(resMatch),
}
if verify {
req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://api.macaddress.io/v1?apiKey=%s&output=json&search=44:38:39:ff:ef:51", resMatch), nil)
if err != nil {
continue
}
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
}
}
}
results = append(results, s1)
}
return results, nil
}
func (s Scanner) Type() detectorspb.DetectorType {
return detectorspb.DetectorType_Macaddress
}
pkg/detectors/macaddress/macaddress_test.go
-120
View File
@@ -1,120 +0,0 @@
//go:build detectors
// +build detectors
package macaddress
import (
"context"
"fmt"
"testing"
"time"
"github.com/kylelemons/godebug/pretty"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
func TestMacaddress_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
secret := testSecrets.MustGetField("MACADDRESS")
inactiveSecret := testSecrets.MustGetField("MACADDRESS_INACTIVE")
type args struct {
ctx context.Context
data []byte
verify bool
}
tests := []struct {
name string
s Scanner
args args
want []detectors.Result
wantErr bool
}{
{
name: "found, verified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a macaddress secret %s within", secret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_Macaddress,
Verified: true,
},
},
wantErr: false,
},
{
name: "found, unverified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a macaddress secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_Macaddress,
Verified: false,
},
},
wantErr: false,
},
{
name: "not found",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte("You cannot find the secret within"),
verify: true,
},
want: nil,
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
s := Scanner{}
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
if (err != nil) != tt.wantErr {
t.Errorf("Macaddress.FromData() error = %v, wantErr %v", err, tt.wantErr)
return
}
for i := range got {
if len(got[i].Raw) == 0 {
t.Fatalf("no raw secret present: \n %+v", got[i])
}
got[i].Raw = nil
}
if diff := pretty.Compare(got, tt.want); diff != "" {
t.Errorf("Macaddress.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
}
})
}
}
func BenchmarkFromData(benchmark *testing.B) {
ctx := context.Background()
s := Scanner{}
for name, data := range detectors.MustGetBenchmarkData() {
benchmark.Run(name, func(b *testing.B) {
b.ResetTimer()
for n := 0; n < b.N; n++ {
_, err := s.FromData(ctx, false, data)
if err != nil {
b.Fatal(err)
}
}
})
}
}
pkg/detectors/maxmindlicense/maxmindlicense.go → pkg/detectors/maxmindlicense/v1/maxmindlicense.go
+7 -4
View File
@@ -2,10 +2,12 @@ package maxmindlicense
import (
"context"
regexp "github.com/wasilibs/go-re2"
"fmt"
"net/http"
"strings"
regexp "github.com/wasilibs/go-re2"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
@@ -51,9 +53,10 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
DetectorType: detectorspb.DetectorType_MaxMindLicense,
Redacted: idRes,
Raw: []byte(keyRes),
}
s1.ExtraData = map[string]string{
"rotation_guide": "https://howtorotate.com/docs/tutorials/maxmind/",
ExtraData: map[string]string{
"rotation_guide": "https://howtorotate.com/docs/tutorials/maxmind/",
"version": fmt.Sprintf("%d", s.Version()),
},
}
if verify {
pkg/detectors/maxmindlicense/maxmindlicense_test.go → pkg/detectors/maxmindlicense/v1/maxmindlicense_test.go
+8
View File
@@ -52,6 +52,10 @@ func TestMaxMindLicense_FromChunk(t *testing.T) {
DetectorType: detectorspb.DetectorType_MaxMindLicense,
Redacted: "510124",
Verified: true,
ExtraData: map[string]string{
"rotation_guide": "https://howtorotate.com/docs/tutorials/maxmind/",
"version": "1",
},
},
},
wantErr: false,
@@ -69,6 +73,10 @@ func TestMaxMindLicense_FromChunk(t *testing.T) {
DetectorType: detectorspb.DetectorType_MaxMindLicense,
Redacted: "510124",
Verified: false,
ExtraData: map[string]string{
"rotation_guide": "https://howtorotate.com/docs/tutorials/maxmind/",
"version": "1",
},
},
},
wantErr: false,
pkg/detectors/maxmindlicense_v2/maxmindlicense_v2.go → pkg/detectors/maxmindlicense/v2/maxmindlicense_v2.go
+31 -35
View File
@@ -1,8 +1,10 @@
package maxmindlicense_v2
package maxmindlicense
import (
"context"
"fmt"
"net/http"
"net/url"
"regexp"
"strings"
@@ -19,7 +21,6 @@ var _ detectors.Detector = (*Scanner)(nil)
var (
client = common.SaneHttpClient()
idPat = regexp.MustCompile(detectors.PrefixRegex([]string{"maxmind", "geoip"}) + `\b([0-9]{2,7})\b`)
keyPat = regexp.MustCompile(`\b([0-9A-Za-z]{6}_[0-9A-Za-z]{29}_mmk)\b`)
)
@@ -36,48 +37,43 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
dataStr := string(data)
keyMatches := keyPat.FindAllStringSubmatch(dataStr, -1)
idMatches := idPat.FindAllStringSubmatch(dataStr, -1)
for _, keyMatch := range keyMatches {
keyRes := strings.TrimSpace(keyMatch[1])
keyRes := keyMatch[1]
for _, idMatch := range idMatches {
if len(idMatch) != 2 {
r := detectors.Result{
DetectorType: detectorspb.DetectorType_MaxMindLicense,
Raw: []byte(keyRes),
ExtraData: map[string]string{
"rotation_guide": "https://howtorotate.com/docs/tutorials/maxmind/",
"version": fmt.Sprintf("%d", s.Version()),
},
}
if verify {
data := url.Values{}
data.Add("license_key", keyRes)
req, err := http.NewRequestWithContext(
ctx, "POST", "https://secret-scanning.maxmind.com/secrets/validate-license-key",
strings.NewReader(data.Encode()))
if err != nil {
r.SetVerificationError(err)
results = append(results, r)
continue
}
idRes := strings.TrimSpace(idMatch[1])
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
s := detectors.Result{
DetectorType: detectorspb.DetectorType_MaxMindLicense,
Redacted: idRes,
Raw: []byte(keyRes),
RawV2: []byte(keyRes + idRes),
res, err := client.Do(req)
if err != nil {
r.SetVerificationError(err)
}
s.ExtraData = map[string]string{
"rotation_guide": "https://howtorotate.com/docs/tutorials/maxmind/",
defer res.Body.Close()
if err == nil && res.StatusCode >= 200 && res.StatusCode < 300 {
r.Verified = true
}
if verify {
req, err := http.NewRequestWithContext(ctx, "GET", "https://geoip.maxmind.com/geoip/v2.1/country/8.8.8.8", nil)
if err != nil {
continue
}
req.SetBasicAuth(idRes, keyRes)
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s.Verified = true
} else {
if detectors.IsKnownFalsePositive(keyRes, detectors.DefaultFalsePositives, true) {
continue
}
}
}
}
results = append(results, s)
}
results = append(results, r)
}
return results, nil

Some files were not shown because too many files have changed in this diff Show More