joeleonjrandDustin Decker
0bbb68931c
* verify canaries against SNS; get ARN * clean comments * Update tests and logic * added test for invalid canary secret * added verify logic for canaries * go mod tidy --------- Co-authored-by: Dustin Decker <[email protected]>
399 lines
13 KiB
Go
399 lines
13 KiB
Go
//go:build detectors
|
|
// +build detectors
|
|
|
|
package aws
|
|
|
|
import (
|
|
"context"
|
|
"crypto/sha256"
|
|
"fmt"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/google/go-cmp/cmp"
|
|
"github.com/google/go-cmp/cmp/cmpopts"
|
|
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
|
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
|
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
|
)
|
|
|
|
const canaryAccessKeyID = "AKIASP2TPHJSQH3FJRUX"
|
|
|
|
var unverifiedSecretClient = common.ConstantResponseHttpClient(403, `{"Error": {"Code": "InvalidClientTokenId"} }`)
|
|
|
|
func TestAWS_FromChunk(t *testing.T) {
|
|
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
|
|
defer cancel()
|
|
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4")
|
|
if err != nil {
|
|
t.Fatalf("could not get test secrets from GCP: %s", err)
|
|
}
|
|
secret := testSecrets.MustGetField("AWS")
|
|
id := testSecrets.MustGetField("AWS_ID")
|
|
inactiveSecret := testSecrets.MustGetField("AWS_INACTIVE")
|
|
inactiveID := id[:len(id)-3] + "XYZ"
|
|
hasher := sha256.New()
|
|
hasher.Write([]byte(inactiveSecret))
|
|
hash := string(hasher.Sum(nil))
|
|
|
|
type args struct {
|
|
ctx context.Context
|
|
data []byte
|
|
verify bool
|
|
}
|
|
tests := []struct {
|
|
name string
|
|
s scanner
|
|
args args
|
|
want []detectors.Result
|
|
wantErr bool
|
|
wantVerificationError bool
|
|
}{
|
|
{
|
|
name: "found, verified",
|
|
s: scanner{},
|
|
args: args{
|
|
ctx: context.Background(),
|
|
data: []byte(fmt.Sprintf("You can find a aws secret %s within aws %s", secret, id)),
|
|
verify: true,
|
|
},
|
|
want: []detectors.Result{
|
|
{
|
|
DetectorType: detectorspb.DetectorType_AWS,
|
|
Verified: true,
|
|
Redacted: "AKIASP2TPHJSQH3FJRUX",
|
|
ExtraData: map[string]string{
|
|
"resource_type": "Access key",
|
|
"account": "171436882533",
|
|
"arn": "arn:aws:iam::171436882533:user/canarytokens.com@@4dxkh0pdeop3bzu9zx5wob793",
|
|
"is_canary": "true",
|
|
"message": "This is an AWS canary token generated at canarytokens.org, and was not set off; learn more here: https://trufflesecurity.com/canaries",
|
|
},
|
|
},
|
|
},
|
|
wantErr: false,
|
|
},
|
|
{
|
|
name: "found, unverified",
|
|
s: scanner{verificationClient: unverifiedSecretClient},
|
|
args: args{
|
|
ctx: context.Background(),
|
|
data: []byte(fmt.Sprintf("You can find a aws secret %s within aws %s but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation
|
|
verify: true,
|
|
},
|
|
want: []detectors.Result{
|
|
{
|
|
DetectorType: detectorspb.DetectorType_AWS,
|
|
Verified: false,
|
|
Redacted: "AKIASP2TPHJSQH3FJRUX",
|
|
ExtraData: map[string]string{
|
|
"resource_type": "Access key",
|
|
"account": "171436882533",
|
|
"is_canary": "true",
|
|
"message": "This is an AWS canary token generated at canarytokens.org, and was not set off; learn more here: https://trufflesecurity.com/canaries",
|
|
},
|
|
},
|
|
},
|
|
wantErr: false,
|
|
},
|
|
{
|
|
name: "not found",
|
|
s: scanner{},
|
|
args: args{
|
|
ctx: context.Background(),
|
|
data: []byte("You cannot find the secret within"),
|
|
verify: true,
|
|
},
|
|
want: nil,
|
|
wantErr: false,
|
|
},
|
|
{
|
|
name: "found two, one included for every ID found",
|
|
s: scanner{},
|
|
args: args{
|
|
ctx: context.Background(),
|
|
data: []byte(fmt.Sprintf("The verified ID is %s with a secret of %s, but the unverified ID is %s and this is the secret %s", id, secret, inactiveID, inactiveSecret)),
|
|
verify: true,
|
|
},
|
|
want: []detectors.Result{
|
|
{
|
|
DetectorType: detectorspb.DetectorType_AWS,
|
|
Verified: false,
|
|
Redacted: "AKIASP2TPHJSQH3FJXYZ",
|
|
ExtraData: map[string]string{
|
|
"resource_type": "Access key",
|
|
"account": "171436882533",
|
|
"is_canary": "true",
|
|
"message": "This is an AWS canary token generated at canarytokens.org, and was not set off; learn more here: https://trufflesecurity.com/canaries",
|
|
},
|
|
},
|
|
{
|
|
DetectorType: detectorspb.DetectorType_AWS,
|
|
Verified: true,
|
|
Redacted: "AKIASP2TPHJSQH3FJRUX",
|
|
ExtraData: map[string]string{
|
|
"resource_type": "Access key",
|
|
"account": "171436882533",
|
|
"arn": "arn:aws:iam::171436882533:user/canarytokens.com@@4dxkh0pdeop3bzu9zx5wob793",
|
|
"is_canary": "true",
|
|
"message": "This is an AWS canary token generated at canarytokens.org, and was not set off; learn more here: https://trufflesecurity.com/canaries",
|
|
},
|
|
},
|
|
},
|
|
wantErr: false,
|
|
},
|
|
{
|
|
name: "not found, because unverified secret was a hash",
|
|
s: scanner{},
|
|
args: args{
|
|
ctx: context.Background(),
|
|
data: []byte(fmt.Sprintf("You can find a aws secret %s within aws %s but not valid", hash, id)), // The secret would satisfy the regex but be filtered out after not passing validation.
|
|
verify: true,
|
|
},
|
|
want: nil,
|
|
wantErr: false,
|
|
},
|
|
{
|
|
name: "found two, returned both because the active secret for one paired with the inactive ID, despite the hash",
|
|
s: scanner{},
|
|
args: args{
|
|
ctx: context.Background(),
|
|
data: []byte(fmt.Sprintf("The verified ID is %s with a secret of %s, but the unverified ID is %s and the secret is this hash %s", id, secret, inactiveID, hash)),
|
|
verify: true,
|
|
},
|
|
want: []detectors.Result{
|
|
{
|
|
DetectorType: detectorspb.DetectorType_AWS,
|
|
Verified: true,
|
|
Redacted: "AKIASP2TPHJSQH3FJRUX",
|
|
ExtraData: map[string]string{
|
|
"resource_type": "Access key",
|
|
"account": "171436882533",
|
|
"arn": "arn:aws:iam::171436882533:user/canarytokens.com@@4dxkh0pdeop3bzu9zx5wob793",
|
|
"is_canary": "true",
|
|
"message": "This is an AWS canary token generated at canarytokens.org, and was not set off; learn more here: https://trufflesecurity.com/canaries",
|
|
},
|
|
},
|
|
{
|
|
DetectorType: detectorspb.DetectorType_AWS,
|
|
Verified: false,
|
|
Redacted: inactiveID,
|
|
ExtraData: map[string]string{
|
|
"account": "171436882533",
|
|
"resource_type": "Access key",
|
|
"is_canary": "true",
|
|
"message": "This is an AWS canary token generated at canarytokens.org, and was not set off; learn more here: https://trufflesecurity.com/canaries",
|
|
},
|
|
},
|
|
},
|
|
wantErr: false,
|
|
},
|
|
{
|
|
name: "found, unverified, with leading +",
|
|
s: scanner{
|
|
verificationClient: unverifiedSecretClient,
|
|
},
|
|
args: args{
|
|
ctx: context.Background(),
|
|
data: []byte(fmt.Sprintf("You can find a aws secret %s within aws %s but not valid", "+HaNv9cTwheDKGJaws/+BMF2GgybQgBWdhcOOdfF", id)), // the secret would satisfy the regex but not pass validation
|
|
verify: true,
|
|
},
|
|
want: []detectors.Result{
|
|
{
|
|
DetectorType: detectorspb.DetectorType_AWS,
|
|
Verified: false,
|
|
Redacted: "AKIASP2TPHJSQH3FJRUX",
|
|
ExtraData: map[string]string{
|
|
"resource_type": "Access key",
|
|
"account": "171436882533",
|
|
"is_canary": "true",
|
|
"message": "This is an AWS canary token generated at canarytokens.org, and was not set off; learn more here: https://trufflesecurity.com/canaries",
|
|
},
|
|
},
|
|
},
|
|
wantErr: false,
|
|
},
|
|
{
|
|
name: "skipped",
|
|
s: scanner{
|
|
skipIDs: map[string]struct{}{
|
|
"AKIASP2TPHJSQH3FJRUX": {},
|
|
},
|
|
},
|
|
args: args{
|
|
ctx: context.Background(),
|
|
data: []byte(fmt.Sprintf("You can find a aws secret %s within aws %s but not valid", "+HaNv9cTwheDKGJaws/+BMF2GgybQgBWdhcOOdfF", id)), // the secret would satisfy the regex but not pass validation
|
|
verify: true,
|
|
},
|
|
wantErr: false,
|
|
},
|
|
{
|
|
name: "found, would be verified if not for http timeout",
|
|
s: scanner{
|
|
verificationClient: common.SaneHttpClientTimeOut(1 * time.Microsecond),
|
|
},
|
|
args: args{
|
|
ctx: context.Background(),
|
|
data: []byte(fmt.Sprintf("You can find a aws secret %s within aws %s", secret, id)),
|
|
verify: true,
|
|
},
|
|
want: []detectors.Result{
|
|
{
|
|
DetectorType: detectorspb.DetectorType_AWS,
|
|
Verified: false,
|
|
Redacted: "AKIASP2TPHJSQH3FJRUX",
|
|
ExtraData: map[string]string{
|
|
"resource_type": "Access key",
|
|
"account": "171436882533",
|
|
"is_canary": "true",
|
|
"message": "This is an AWS canary token generated at canarytokens.org, and was not set off; learn more here: https://trufflesecurity.com/canaries",
|
|
},
|
|
},
|
|
},
|
|
wantErr: false,
|
|
wantVerificationError: true,
|
|
},
|
|
{
|
|
name: "found, unverified due to unexpected http response status",
|
|
s: scanner{
|
|
verificationClient: common.ConstantResponseHttpClient(500, "internal server error"),
|
|
},
|
|
args: args{
|
|
ctx: context.Background(),
|
|
data: []byte(fmt.Sprintf("You can find a aws secret %s within aws %s", secret, id)),
|
|
verify: true,
|
|
},
|
|
want: []detectors.Result{
|
|
{
|
|
DetectorType: detectorspb.DetectorType_AWS,
|
|
Verified: false,
|
|
Redacted: "AKIASP2TPHJSQH3FJRUX",
|
|
ExtraData: map[string]string{
|
|
"resource_type": "Access key",
|
|
"account": "171436882533",
|
|
"is_canary": "true",
|
|
"message": "This is an AWS canary token generated at canarytokens.org, and was not set off; learn more here: https://trufflesecurity.com/canaries",
|
|
},
|
|
},
|
|
},
|
|
wantErr: false,
|
|
wantVerificationError: true,
|
|
},
|
|
{
|
|
name: "found, unverified due to invalid aws_secret with valid canary access_key_id",
|
|
s: scanner{},
|
|
args: args{
|
|
ctx: context.Background(),
|
|
data: []byte(fmt.Sprintf("You can find a aws secret %s within aws %s", inactiveSecret, canaryAccessKeyID)),
|
|
verify: true,
|
|
},
|
|
want: []detectors.Result{
|
|
{
|
|
DetectorType: detectorspb.DetectorType_AWS,
|
|
Verified: false,
|
|
Redacted: canaryAccessKeyID,
|
|
ExtraData: map[string]string{
|
|
"resource_type": "Access key",
|
|
"account": "171436882533",
|
|
"is_canary": "true",
|
|
"message": "This is an AWS canary token generated at canarytokens.org, and was not set off; learn more here: https://trufflesecurity.com/canaries",
|
|
},
|
|
},
|
|
},
|
|
wantErr: false,
|
|
wantVerificationError: false,
|
|
},
|
|
{
|
|
name: "found, valid canary token with no verification",
|
|
s: scanner{},
|
|
args: args{
|
|
ctx: context.Background(),
|
|
data: []byte(fmt.Sprintf("You can find a aws secret %s within aws %s", secret, canaryAccessKeyID)),
|
|
verify: false,
|
|
},
|
|
want: []detectors.Result{
|
|
{
|
|
DetectorType: detectorspb.DetectorType_AWS,
|
|
Verified: false,
|
|
Redacted: canaryAccessKeyID,
|
|
ExtraData: map[string]string{
|
|
"resource_type": "Access key",
|
|
"account": "171436882533",
|
|
"is_canary": "true",
|
|
"message": "This is an AWS canary token generated at canarytokens.org, and was not set off; learn more here: https://trufflesecurity.com/canaries",
|
|
},
|
|
},
|
|
},
|
|
wantErr: false,
|
|
wantVerificationError: false,
|
|
},
|
|
{
|
|
name: "verified secret checked directly after unverified secret with same key id",
|
|
s: scanner{},
|
|
args: args{
|
|
ctx: context.Background(),
|
|
data: []byte(fmt.Sprintf("%s\n%s\n%s", inactiveSecret, id, secret)),
|
|
verify: true,
|
|
},
|
|
want: []detectors.Result{
|
|
{
|
|
DetectorType: detectorspb.DetectorType_AWS,
|
|
Verified: true,
|
|
Redacted: "AKIASP2TPHJSQH3FJRUX",
|
|
ExtraData: map[string]string{
|
|
"resource_type": "Access key",
|
|
"account": "171436882533",
|
|
"arn": "arn:aws:iam::171436882533:user/canarytokens.com@@4dxkh0pdeop3bzu9zx5wob793",
|
|
"is_canary": "true",
|
|
"message": "This is an AWS canary token generated at canarytokens.org, and was not set off; learn more here: https://trufflesecurity.com/canaries",
|
|
},
|
|
},
|
|
},
|
|
wantErr: false,
|
|
},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
s := tt.s
|
|
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
|
if (err != nil) != tt.wantErr {
|
|
t.Errorf("AWS.FromData() error = %v, wantErr %v", err, tt.wantErr)
|
|
return
|
|
}
|
|
for i := range got {
|
|
if len(got[i].Raw) == 0 {
|
|
t.Fatalf("no raw secret present: \n %+v", got[i])
|
|
}
|
|
if (got[i].VerificationError() != nil) != tt.wantVerificationError {
|
|
t.Fatalf("wantVerificationError %v, verification error = %v", tt.wantVerificationError, got[i].VerificationError())
|
|
}
|
|
}
|
|
ignoreOpts := []cmp.Option{
|
|
cmpopts.IgnoreFields(detectors.Result{}, "RawV2", "Raw", "verificationError"),
|
|
cmpopts.SortSlices(func(x, y detectors.Result) bool {
|
|
return x.Redacted < y.Redacted
|
|
}),
|
|
}
|
|
if diff := cmp.Diff(got, tt.want, ignoreOpts...); diff != "" {
|
|
t.Errorf("AWS.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func BenchmarkFromData(benchmark *testing.B) {
|
|
ctx := context.Background()
|
|
s := scanner{}
|
|
for name, data := range detectors.MustGetBenchmarkData() {
|
|
benchmark.Run(name, func(b *testing.B) {
|
|
b.ResetTimer()
|
|
for n := 0; n < b.N; n++ {
|
|
_, err := s.FromData(ctx, false, data)
|
|
if err != nil {
|
|
b.Fatal(err)
|
|
}
|
|
}
|
|
})
|
|
}
|
|
}
|