5701963045
Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> Co-authored-by: Bryan Beverly <[email protected]>
267 lines
11 KiB
YAML
267 lines
11 KiB
YAML
name: Corpora Test
|
|
|
|
on:
|
|
workflow_dispatch:
|
|
pull_request:
|
|
types: [opened]
|
|
paths:
|
|
- 'pkg/detectors/**'
|
|
- 'pkg/engine/defaults/defaults.go'
|
|
- '.github/workflows/detector-corpora-test.yml'
|
|
- 'scripts/test/detector_corpora_test.sh'
|
|
- 'scripts/test/diff_corpora_results.py'
|
|
- 'scripts/test/detect_changed_detectors.sh'
|
|
|
|
env:
|
|
DATASETS: |
|
|
s3://trufflehog-corpora-datasets/contents.2025-11-04.jsonl.zstd
|
|
|
|
jobs:
|
|
corpora-test:
|
|
if: ${{ github.repository == 'trufflesecurity/trufflehog' && !github.event.pull_request.head.repo.fork }}
|
|
runs-on: ubuntu-latest
|
|
permissions:
|
|
contents: read
|
|
pull-requests: write
|
|
steps:
|
|
- name: Checkout code
|
|
uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
|
|
with:
|
|
fetch-depth: 0
|
|
persist-credentials: false
|
|
|
|
- name: Install Go
|
|
uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6
|
|
with:
|
|
go-version: "1.27"
|
|
|
|
- name: Install dependencies
|
|
run: sudo apt-get install -y zstd jq
|
|
|
|
- name: Resolve merge-base
|
|
id: merge_base
|
|
shell: bash
|
|
run: |
|
|
set -o pipefail
|
|
git fetch --no-tags --prune origin main
|
|
MERGE_BASE=$(git merge-base origin/main HEAD)
|
|
echo "Merge base: $MERGE_BASE"
|
|
echo "sha=$MERGE_BASE" >> "$GITHUB_OUTPUT"
|
|
|
|
# Determine which detectors changed in this PR. The PR build scopes its
|
|
# scan to the full set; the main build excludes detectors that don't
|
|
# exist there yet (new detectors). If the set is empty, the workflow
|
|
# short-circuits with a skip comment — scoping is the entire point of
|
|
# Phase 2, falling back to scan-all defeats it.
|
|
- name: Detect changed detectors
|
|
id: detect
|
|
shell: bash
|
|
env:
|
|
BASE_REF: ${{ steps.merge_base.outputs.sha }}
|
|
run: |
|
|
set -o pipefail
|
|
chmod +x scripts/test/detect_changed_detectors.sh
|
|
PR_CSV=$(./scripts/test/detect_changed_detectors.sh --pr-csv || true)
|
|
MAIN_CSV=$(./scripts/test/detect_changed_detectors.sh --main-csv || true)
|
|
NEW_LIST=$(./scripts/test/detect_changed_detectors.sh --new-only || true)
|
|
NEW_CSV=$(echo "$NEW_LIST" | paste -sd, -)
|
|
echo "PR detectors: $PR_CSV"
|
|
echo "Main detectors: $MAIN_CSV"
|
|
echo "New detectors: $NEW_CSV"
|
|
echo "pr_csv=$PR_CSV" >> "$GITHUB_OUTPUT"
|
|
echo "main_csv=$MAIN_CSV" >> "$GITHUB_OUTPUT"
|
|
echo "new_csv=$NEW_CSV" >> "$GITHUB_OUTPUT"
|
|
if [[ -n "$PR_CSV" ]]; then
|
|
echo "any_changed=true" >> "$GITHUB_OUTPUT"
|
|
else
|
|
echo "any_changed=false" >> "$GITHUB_OUTPUT"
|
|
fi
|
|
|
|
# Sticky comment: find any prior detector-bench comment on the PR by
|
|
# the marker substring and update it in place. The marker — kept in
|
|
# sync with STICKY_COMMENT_MARKER in scripts/test/diff_corpora_results.py —
|
|
# has to appear in BOTH the skip body and the diff body so the same
|
|
# comment flips between them as iterative pushes change which path
|
|
# fires. Skip body is only posted on pull_request events; workflow_dispatch
|
|
# runs with no changed detectors silently finish without posting.
|
|
- name: Find existing skip comment
|
|
if: steps.detect.outputs.any_changed != 'true' && github.event_name == 'pull_request'
|
|
id: find_skip_comment
|
|
uses: peter-evans/find-comment@b30e6a3c0ed37e7c023ccd3f1db5c6c0b0c23aad # v4
|
|
with:
|
|
issue-number: ${{ github.event.pull_request.number }}
|
|
comment-author: 'github-actions[bot]'
|
|
body-includes: '<!-- detector-bench -->'
|
|
|
|
- name: Post or update skip comment
|
|
if: steps.detect.outputs.any_changed != 'true' && github.event_name == 'pull_request'
|
|
uses: peter-evans/create-or-update-comment@e8674b075228eee787fea43ef493e45ece1004c9 # v5
|
|
with:
|
|
comment-id: ${{ steps.find_skip_comment.outputs.comment-id }}
|
|
issue-number: ${{ github.event.pull_request.number }}
|
|
edit-mode: replace
|
|
body: |
|
|
<!-- detector-bench -->
|
|
## Corpora Test Results
|
|
|
|
No detector regex or keyword changes in this PR. Bench skipped.
|
|
|
|
- name: Configure AWS credentials
|
|
if: steps.detect.outputs.any_changed == 'true'
|
|
uses: aws-actions/configure-aws-credentials@cbe3b392738ccf3f987d68400dafcf4b0624a56c # v6
|
|
with:
|
|
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY }}
|
|
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
|
aws-region: us-east-1
|
|
|
|
# Cache the main scan results by merge-base + scoped detector set.
|
|
# On subsequent pushes to the same PR without a rebase, both are
|
|
# identical, so the main scan (35 GB of S3 streaming + trufflehog) is
|
|
# skipped entirely.
|
|
- name: Restore main scan cache
|
|
id: main_scan_cache
|
|
if: steps.detect.outputs.any_changed == 'true' && steps.detect.outputs.main_csv != ''
|
|
uses: actions/cache/restore@caa296126883cff596d87d8935842f9db880ef25 # v5
|
|
with:
|
|
path: /tmp/results-main.jsonl
|
|
key: main-scan-v1-${{ steps.merge_base.outputs.sha }}-${{ steps.detect.outputs.main_csv }}
|
|
|
|
# Two independent builds run in parallel:
|
|
# A) prepare main worktree → build main binary (git I/O then CPU)
|
|
# Skipped on main scan cache hit or when main_csv is empty
|
|
# (all changed detectors are new — no baseline needed).
|
|
# B) build PR binary (CPU, no dependencies)
|
|
- name: Build binaries
|
|
if: steps.detect.outputs.any_changed == 'true'
|
|
shell: bash
|
|
env:
|
|
MERGE_BASE: ${{ steps.merge_base.outputs.sha }}
|
|
MAIN_CSV: ${{ steps.detect.outputs.main_csv }}
|
|
MAIN_SCAN_CACHE_HIT: ${{ steps.main_scan_cache.outputs.cache-hit }}
|
|
run: |
|
|
set -o pipefail
|
|
|
|
# Chain A: prepare worktree, then build main binary.
|
|
# Skipped when main scan results are already cached, or when all
|
|
# changed detectors are new (main_csv empty — no baseline needed).
|
|
if [[ -n "$MAIN_CSV" && "$MAIN_SCAN_CACHE_HIT" != 'true' ]]; then
|
|
(
|
|
git worktree add /tmp/trufflehog-main-src "$MERGE_BASE"
|
|
cd /tmp/trufflehog-main-src
|
|
CGO_ENABLED=0 go build -o /tmp/trufflehog-main .
|
|
) &
|
|
PID_MAIN_BUILD=$!
|
|
fi
|
|
|
|
# Chain B: build PR binary (no dependencies).
|
|
CGO_ENABLED=0 go build -o /tmp/trufflehog-pr . &
|
|
PID_PR_BUILD=$!
|
|
|
|
[[ -n "${PID_MAIN_BUILD:-}" ]] && { wait $PID_MAIN_BUILD || { echo "Main binary build failed" >&2; exit 1; }; }
|
|
wait $PID_PR_BUILD || { echo "PR binary build failed" >&2; exit 1; }
|
|
|
|
# PR and main scans share a single S3 stream per dataset file, teed to
|
|
# both binaries simultaneously. The main side is skipped on a cache hit
|
|
# (results already in /tmp/results-main.jsonl) or when main_csv is empty
|
|
# (PR adds only new detectors — no overlap with main).
|
|
- name: Run corpora tests
|
|
if: steps.detect.outputs.any_changed == 'true'
|
|
shell: bash
|
|
env:
|
|
PR_CSV: ${{ steps.detect.outputs.pr_csv }}
|
|
MAIN_CSV: ${{ steps.detect.outputs.main_csv }}
|
|
MAIN_SCAN_CACHE_HIT: ${{ steps.main_scan_cache.outputs.cache-hit }}
|
|
run: |
|
|
set -o pipefail
|
|
files=()
|
|
while IFS= read -r dataset; do
|
|
[[ -z "$dataset" ]] && continue
|
|
files+=("$dataset")
|
|
done <<< "$DATASETS"
|
|
|
|
export TRUFFLEHOG_BIN=/tmp/trufflehog-pr
|
|
export OUTPUT_JSONL=/tmp/results-pr.jsonl
|
|
export INCLUDE_DETECTORS="$PR_CSV"
|
|
|
|
if [[ -n "$MAIN_CSV" && "$MAIN_SCAN_CACHE_HIT" != 'true' ]]; then
|
|
# Dual-binary: single S3 download teed to both PR and main binaries.
|
|
export TRUFFLEHOG_BIN_MAIN=/tmp/trufflehog-main
|
|
export OUTPUT_JSONL_MAIN=/tmp/results-main.jsonl
|
|
export INCLUDE_DETECTORS_MAIN="$MAIN_CSV"
|
|
elif [[ -z "$MAIN_CSV" ]]; then
|
|
echo "No overlapping detectors in main; skipping main scan."
|
|
: > /tmp/results-main.jsonl
|
|
else
|
|
echo "Main scan cache hit; skipping main scan."
|
|
fi
|
|
|
|
./scripts/test/detector_corpora_test.sh "${files[@]}" \
|
|
|| { echo "Corpora scan failed" >&2; exit 1; }
|
|
|
|
- name: Save main scan cache
|
|
if: steps.detect.outputs.any_changed == 'true' && steps.detect.outputs.main_csv != '' && steps.main_scan_cache.outputs.cache-hit != 'true'
|
|
uses: actions/cache/save@caa296126883cff596d87d8935842f9db880ef25 # v5
|
|
with:
|
|
path: /tmp/results-main.jsonl
|
|
key: main-scan-v1-${{ steps.merge_base.outputs.sha }}-${{ steps.detect.outputs.main_csv }}
|
|
|
|
- name: Diff results
|
|
if: steps.detect.outputs.any_changed == 'true'
|
|
shell: bash
|
|
env:
|
|
CHANGED: ${{ steps.detect.outputs.pr_csv }}
|
|
NEW_DETECTORS: ${{ steps.detect.outputs.new_csv }}
|
|
run: |
|
|
set -o pipefail
|
|
python3 scripts/test/diff_corpora_results.py \
|
|
/tmp/results-main.jsonl /tmp/results-pr.jsonl \
|
|
--changed-detectors="$CHANGED" \
|
|
--new-detectors="$NEW_DETECTORS" \
|
|
> /tmp/diff-report.md
|
|
cat /tmp/diff-report.md
|
|
|
|
# workflow_dispatch runs don't carry an issue context, so resolve the
|
|
# PR number by branch lookup. pull_request events fall through to the
|
|
# event's issue number. Output feeds the find/update pair below.
|
|
- name: Resolve PR number
|
|
if: steps.detect.outputs.any_changed == 'true'
|
|
id: resolve_pr
|
|
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9
|
|
with:
|
|
script: |
|
|
let issue_number;
|
|
if (context.eventName === 'workflow_dispatch') {
|
|
const pulls = await github.rest.pulls.list({
|
|
owner: context.repo.owner,
|
|
repo: context.repo.repo,
|
|
head: `${context.repo.owner}:${context.ref.replace('refs/heads/', '')}`,
|
|
state: 'open',
|
|
});
|
|
if (pulls.data.length === 0) {
|
|
core.setFailed(`No open PR found for branch ${context.ref}`);
|
|
return;
|
|
}
|
|
issue_number = pulls.data[0].number;
|
|
} else {
|
|
issue_number = context.issue.number;
|
|
}
|
|
core.setOutput('issue_number', issue_number);
|
|
|
|
- name: Find existing diff comment
|
|
if: steps.detect.outputs.any_changed == 'true'
|
|
id: find_diff_comment
|
|
uses: peter-evans/find-comment@b30e6a3c0ed37e7c023ccd3f1db5c6c0b0c23aad # v4
|
|
with:
|
|
issue-number: ${{ steps.resolve_pr.outputs.issue_number }}
|
|
comment-author: 'github-actions[bot]'
|
|
body-includes: '<!-- detector-bench -->'
|
|
|
|
- name: Post or update diff comment
|
|
if: steps.detect.outputs.any_changed == 'true'
|
|
uses: peter-evans/create-or-update-comment@e8674b075228eee787fea43ef493e45ece1004c9 # v5
|
|
with:
|
|
comment-id: ${{ steps.find_diff_comment.outputs.comment-id }}
|
|
issue-number: ${{ steps.resolve_pr.outputs.issue_number }}
|
|
edit-mode: replace
|
|
body-path: /tmp/diff-report.md
|