From e217e2fbfd77e63d96caff6634477562e13234ca Mon Sep 17 00:00:00 2001 From: Dustin Decker Date: Thu, 20 Apr 2023 15:07:59 -0700 Subject: [PATCH] Ensure multipart credentials are deduplicated correctly (#1271) * Ensure multipart credentials are deduplicated correctly * update tests --- pkg/detectors/aws/aws_test.go | 40 +++++++++++------------ pkg/detectors/gcp/gcp.go | 4 ++- pkg/detectors/jiratoken/jiratoken.go | 1 + pkg/detectors/jiratoken/jiratoken_test.go | 1 + pkg/detectors/okta/okta.go | 1 + pkg/detectors/onelogin/onelogin.go | 1 + pkg/detectors/uri/uri.go | 3 ++ pkg/detectors/uri/uri_test.go | 7 ++-- 8 files changed, 34 insertions(+), 24 deletions(-) diff --git a/pkg/detectors/aws/aws_test.go b/pkg/detectors/aws/aws_test.go index f289451b8..3628a9f58 100644 --- a/pkg/detectors/aws/aws_test.go +++ b/pkg/detectors/aws/aws_test.go @@ -56,11 +56,11 @@ func TestAWS_FromChunk(t *testing.T) { { DetectorType: detectorspb.DetectorType_AWS, Verified: true, - Redacted: "AKIAWARWQKZNHMZBLY4I", + Redacted: "AKIASP2TPHJSQH3FJRUX", ExtraData: map[string]string{ - "account": "413504919130", - "arn": "arn:aws:iam::413504919130:root", - "user_id": "413504919130", + "account": "171436882533", + "arn": "arn:aws:iam::171436882533:user/canarytokens.com@@4dxkh0pdeop3bzu9zx5wob793", + "user_id": "AIDASP2TPHJSUFRSTTZX4", }, }, }, @@ -78,7 +78,7 @@ func TestAWS_FromChunk(t *testing.T) { { DetectorType: detectorspb.DetectorType_AWS, Verified: false, - Redacted: "AKIAWARWQKZNHMZBLY4I", + Redacted: "AKIASP2TPHJSQH3FJRUX", ExtraData: nil, }, }, @@ -106,18 +106,18 @@ func TestAWS_FromChunk(t *testing.T) { want: []detectors.Result{ { DetectorType: detectorspb.DetectorType_AWS, - Verified: true, - Redacted: id, - ExtraData: map[string]string{ - "account": "413504919130", - "arn": "arn:aws:iam::413504919130:root", - "user_id": "413504919130", - }, + Verified: false, + Redacted: "AKIASP2TPHJSQH3FJXYZ", }, { DetectorType: detectorspb.DetectorType_AWS, - Verified: false, - Redacted: inactiveID, + Verified: true, + Redacted: "AKIASP2TPHJSQH3FJRUX", + ExtraData: map[string]string{ + "account": "171436882533", + "arn": "arn:aws:iam::171436882533:user/canarytokens.com@@4dxkh0pdeop3bzu9zx5wob793", + "user_id": "AIDASP2TPHJSUFRSTTZX4", + }, }, }, wantErr: false, @@ -145,11 +145,11 @@ func TestAWS_FromChunk(t *testing.T) { { DetectorType: detectorspb.DetectorType_AWS, Verified: true, - Redacted: id, + Redacted: "AKIASP2TPHJSQH3FJRUX", ExtraData: map[string]string{ - "account": "413504919130", - "arn": "arn:aws:iam::413504919130:root", - "user_id": "413504919130", + "account": "171436882533", + "arn": "arn:aws:iam::171436882533:user/canarytokens.com@@4dxkh0pdeop3bzu9zx5wob793", + "user_id": "AIDASP2TPHJSUFRSTTZX4", }, }, { @@ -172,7 +172,7 @@ func TestAWS_FromChunk(t *testing.T) { { DetectorType: detectorspb.DetectorType_AWS, Verified: false, - Redacted: "AKIAWARWQKZNHMZBLY4I", + Redacted: "AKIASP2TPHJSQH3FJRUX", }, }, wantErr: false, @@ -181,7 +181,7 @@ func TestAWS_FromChunk(t *testing.T) { name: "skipped", s: scanner{ skipIDs: map[string]struct{}{ - "AKIAWARWQKZNHMZBLY4I": {}, + "AKIASP2TPHJSQH3FJRUX": {}, }, }, args: args{ diff --git a/pkg/detectors/gcp/gcp.go b/pkg/detectors/gcp/gcp.go index b026f99ea..5ec70a943 100644 --- a/pkg/detectors/gcp/gcp.go +++ b/pkg/detectors/gcp/gcp.go @@ -79,14 +79,16 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result raw = []byte(key) } + credBytes, _ := json.Marshal(creds) + s := detectors.Result{ DetectorType: detectorspb.DetectorType_GCP, Raw: raw, + RawV2: credBytes, Redacted: creds.ClientEmail, } if verify { - credBytes, _ := json.Marshal(creds) credentials, err := google.CredentialsFromJSON(ctx, credBytes, "https://www.googleapis.com/auth/cloud-platform") if err != nil { continue diff --git a/pkg/detectors/jiratoken/jiratoken.go b/pkg/detectors/jiratoken/jiratoken.go index a04a58c98..678d28511 100644 --- a/pkg/detectors/jiratoken/jiratoken.go +++ b/pkg/detectors/jiratoken/jiratoken.go @@ -60,6 +60,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result s1 := detectors.Result{ DetectorType: detectorspb.DetectorType_JiraToken, Raw: []byte(resToken), + RawV2: []byte(fmt.Sprintf("%s:%s:%s", resEmail, resToken, resDomain)), } if verify { diff --git a/pkg/detectors/jiratoken/jiratoken_test.go b/pkg/detectors/jiratoken/jiratoken_test.go index aaddcd03b..2c0486124 100644 --- a/pkg/detectors/jiratoken/jiratoken_test.go +++ b/pkg/detectors/jiratoken/jiratoken_test.go @@ -97,6 +97,7 @@ func TestJiraToken_FromChunk(t *testing.T) { t.Fatalf("no raw secret present: \n %+v", got[i]) } got[i].Raw = nil + got[i].RawV2 = nil } if diff := pretty.Compare(got, tt.want); diff != "" { t.Errorf("JiraToken.FromData() %s diff: (-got +want)\n%s", tt.name, diff) diff --git a/pkg/detectors/okta/okta.go b/pkg/detectors/okta/okta.go index 5c85e384b..ecaef8a27 100644 --- a/pkg/detectors/okta/okta.go +++ b/pkg/detectors/okta/okta.go @@ -41,6 +41,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result s := detectors.Result{ DetectorType: detectorspb.DetectorType_Okta, Raw: []byte(token), + RawV2: []byte(fmt.Sprintf("%s:%s", domain, token)), } if verify { diff --git a/pkg/detectors/onelogin/onelogin.go b/pkg/detectors/onelogin/onelogin.go index 0ef614d13..0212a0681 100644 --- a/pkg/detectors/onelogin/onelogin.go +++ b/pkg/detectors/onelogin/onelogin.go @@ -50,6 +50,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result s := detectors.Result{ DetectorType: detectorspb.DetectorType_OneLogin, Raw: []byte(clientID[1]), + RawV2: []byte(fmt.Sprintf("%s:%s", clientID[1], clientSecret[1])), Redacted: clientID[1], } diff --git a/pkg/detectors/uri/uri.go b/pkg/detectors/uri/uri.go index 203b048ae..05d00f7ce 100644 --- a/pkg/detectors/uri/uri.go +++ b/pkg/detectors/uri/uri.go @@ -66,12 +66,15 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } rawURL, _ := url.Parse(urlMatch) + rawURLStr := rawURL.String() + // Removing the path causes possible deduplication issues if some paths have basic auth and some do not. rawURL.Path = "" redact := strings.TrimSpace(strings.Replace(rawURL.String(), password, "********", -1)) s := detectors.Result{ DetectorType: detectorspb.DetectorType_URI, Raw: []byte(rawURL.String()), + RawV2: []byte(rawURLStr), Redacted: redact, } diff --git a/pkg/detectors/uri/uri_test.go b/pkg/detectors/uri/uri_test.go index 1cd024e3f..ba090c0e2 100644 --- a/pkg/detectors/uri/uri_test.go +++ b/pkg/detectors/uri/uri_test.go @@ -38,7 +38,7 @@ func TestURI_FromChunk(t *testing.T) { { DetectorType: detectorspb.DetectorType_URI, Verified: false, - Redacted: "https://user:****@www.httpwatch.com", + Redacted: "https://user:********@www.httpwatch.com", }, }, wantErr: false, @@ -55,7 +55,7 @@ func TestURI_FromChunk(t *testing.T) { { DetectorType: detectorspb.DetectorType_URI, Verified: true, - Redacted: "https://httpwatch:****@www.httpwatch.com", + Redacted: "https://httpwatch:********@www.httpwatch.com", }, }, wantErr: false, @@ -72,7 +72,7 @@ func TestURI_FromChunk(t *testing.T) { { DetectorType: detectorspb.DetectorType_URI, Verified: true, - Redacted: "https://httpwatch:****@www.httpwatch.com", + Redacted: "https://httpwatch:********@www.httpwatch.com", }, }, wantErr: false, @@ -101,6 +101,7 @@ func TestURI_FromChunk(t *testing.T) { // } for i := range got { got[i].Raw = nil + got[i].RawV2 = nil } if diff := pretty.Compare(got, tt.want); diff != "" { t.Errorf("URI.FromData() %s diff: (-got +want)\n%s", tt.name, diff)