From 47d5ddebf2b94f65e3a87a0fa4000214ebbda8ea Mon Sep 17 00:00:00 2001 From: ahrav Date: Tue, 19 Sep 2023 15:39:13 -0700 Subject: [PATCH] Ability to update line number in link (#1788) * Add functionality to update a source's link in the metadata with the updated line number. * update comment. * add logic to engine. * only update link for non empty links. * add tests for bb. --- pkg/engine/engine.go | 37 ++++++++++++----- pkg/engine/engine_test.go | 58 +++++++++++++++++++++++++++ pkg/giturl/giturl.go | 43 ++++++++++++++++++++ pkg/giturl/giturl_test.go | 83 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 212 insertions(+), 9 deletions(-) diff --git a/pkg/engine/engine.go b/pkg/engine/engine.go index ff6167905..04979d3ac 100644 --- a/pkg/engine/engine.go +++ b/pkg/engine/engine.go @@ -19,6 +19,7 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/context" "github.com/trufflesecurity/trufflehog/v3/pkg/decoders" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/giturl" "github.com/trufflesecurity/trufflehog/v3/pkg/output" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb" @@ -532,12 +533,12 @@ func (e *Engine) detectChunk(ctx context.Context, data detectableChunk) { } for _, res := range results { - e.processResult(data, res) + e.processResult(ctx, data, res) } data.wgDoneFn() } -func (e *Engine) processResult(data detectableChunk, res detectors.Result) { +func (e *Engine) processResult(ctx context.Context, data detectableChunk, res detectors.Result) { ignoreLinePresent := false if SupportsLineNumbers(data.chunk.SourceType) { copyChunk := data.chunk @@ -545,8 +546,11 @@ func (e *Engine) processResult(data detectableChunk, res detectors.Result) { if copyMetaData, ok := copyMetaDataClone.(*source_metadatapb.MetaData); ok { copyChunk.SourceMetadata = copyMetaData } - fragStart, mdLine := FragmentFirstLine(©Chunk) + fragStart, mdLine, link := FragmentFirstLineAndLink(©Chunk) ignoreLinePresent = SetResultLineNumber(©Chunk, &res, fragStart, mdLine) + if link != "" { + copyChunk.SourceMetadata.GetGithub().Link = giturl.UpdateLinkLineNumber(ctx, link, *mdLine) + } data.chunk = copyChunk } if ignoreLinePresent { @@ -625,27 +629,42 @@ func FragmentLineOffset(chunk *sources.Chunk, result *detectors.Result) (int64, return lineNumber, false } -// FragmentFirstLine returns the first line number of a fragment along with a pointer to the value to update in the -// chunk metadata. -func FragmentFirstLine(chunk *sources.Chunk) (int64, *int64) { - var fragmentStart *int64 +// FragmentFirstLineAndLink extracts the first line number and the link from the chunk metadata. +// It returns: +// - The first line number of the fragment. +// - A pointer to the line number, facilitating direct updates. +// - The link associated with the fragment. This link may be updated in the chunk metadata +// if there's a change in the line number. +func FragmentFirstLineAndLink(chunk *sources.Chunk) (int64, *int64, string) { + if chunk.SourceMetadata == nil { + return 0, nil, "" + } + + var ( + fragmentStart *int64 + link string + ) switch metadata := chunk.SourceMetadata.GetData().(type) { case *source_metadatapb.MetaData_Git: fragmentStart = &metadata.Git.Line case *source_metadatapb.MetaData_Github: fragmentStart = &metadata.Github.Line + link = metadata.Github.Link case *source_metadatapb.MetaData_Gitlab: fragmentStart = &metadata.Gitlab.Line + link = metadata.Gitlab.Link case *source_metadatapb.MetaData_Bitbucket: fragmentStart = &metadata.Bitbucket.Line + link = metadata.Bitbucket.Link case *source_metadatapb.MetaData_Gerrit: fragmentStart = &metadata.Gerrit.Line case *source_metadatapb.MetaData_Filesystem: fragmentStart = &metadata.Filesystem.Line + link = metadata.Filesystem.Link default: - return 0, nil + return 0, nil, "" } - return *fragmentStart, fragmentStart + return *fragmentStart, fragmentStart, link } // SetResultLineNumber sets the line number in the provided result. diff --git a/pkg/engine/engine_test.go b/pkg/engine/engine_test.go index 948d7f947..6c3affcf3 100644 --- a/pkg/engine/engine_test.go +++ b/pkg/engine/engine_test.go @@ -11,6 +11,7 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/context" "github.com/trufflesecurity/trufflehog/v3/pkg/decoders" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb" "github.com/trufflesecurity/trufflehog/v3/pkg/sources" ) @@ -205,3 +206,60 @@ func TestEngine_DuplicatSecrets(t *testing.T) { want := uint64(5) assert.Equal(t, want, e.GetMetrics().UnverifiedSecretsFound) } + +func TestFragmentFirstLineAndLink(t *testing.T) { + tests := []struct { + name string + chunk *sources.Chunk + expectedLine int64 + expectedLink string + }{ + { + name: "Test Git Metadata", + chunk: &sources.Chunk{ + SourceMetadata: &source_metadatapb.MetaData{ + Data: &source_metadatapb.MetaData_Git{ + Git: &source_metadatapb.Git{ + Line: 10, + }, + }, + }, + }, + expectedLine: 10, + expectedLink: "", // Git doesn't support links + }, + { + name: "Test Github Metadata", + chunk: &sources.Chunk{ + SourceMetadata: &source_metadatapb.MetaData{ + Data: &source_metadatapb.MetaData_Github{ + Github: &source_metadatapb.Github{ + Line: 5, + Link: "https://example.github.com", + }, + }, + }, + }, + expectedLine: 5, + expectedLink: "https://example.github.com", + }, + { + name: "Unsupported Type", + chunk: &sources.Chunk{}, + expectedLine: 0, + expectedLink: "", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + line, linePtr, link := FragmentFirstLineAndLink(tt.chunk) + assert.Equal(t, tt.expectedLink, link, "Mismatch in link") + assert.Equal(t, tt.expectedLine, line, "Mismatch in line") + + if linePtr != nil { + assert.Equal(t, tt.expectedLine, *linePtr, "Mismatch in linePtr value") + } + }) + } +} diff --git a/pkg/giturl/giturl.go b/pkg/giturl/giturl.go index b9c1bcec2..bf7adee04 100644 --- a/pkg/giturl/giturl.go +++ b/pkg/giturl/giturl.go @@ -2,10 +2,13 @@ package giturl import ( "net/url" + "regexp" "strconv" "strings" "github.com/pkg/errors" + + "github.com/trufflesecurity/trufflehog/v3/pkg/context" ) type provider string @@ -143,3 +146,43 @@ func GenerateLink(repo, commit, file string, line int64) string { return baseLink } } + +var linePattern = regexp.MustCompile(`L\d+`) + +// UpdateLinkLineNumber updates the line number in a repository link. +// Used post-link generation to refine reported issue locations within large scanned blocks. +func UpdateLinkLineNumber(ctx context.Context, link string, newLine int64) string { + parsedURL, err := url.Parse(link) + if err != nil { + ctx.Logger().Error(err, "unable to parse link to update line number", "link", link) + return link + } + + switch determineProvider(link) { + case providerBitbucket: + // For Bitbucket, it doesn't support line links (based on the GenerateLink function). + // So we don't need to change anything. + return link + + case providerAzure: + // For Azure, line numbers are appended as ?line=. + query := parsedURL.Query() + query.Set("line", strconv.FormatInt(newLine, 10)) + parsedURL.RawQuery = query.Encode() + + case providerGithub, providerGitlab: + // If the provider name isn't one of the cloud defaults, it is probably an on-prem github or gitlab. + // So do the same thing. + fallthrough + default: + // Assumed format: .../blob//file.go#L + fragment := "L" + strconv.FormatInt(newLine, 10) + if linePattern.MatchString(parsedURL.Fragment) { + parsedURL.Fragment = linePattern.ReplaceAllString(parsedURL.Fragment, fragment) + } else { + parsedURL.Fragment += fragment + } + } + + return parsedURL.String() +} diff --git a/pkg/giturl/giturl_test.go b/pkg/giturl/giturl_test.go index a59c284d0..1ccc2fd74 100644 --- a/pkg/giturl/giturl_test.go +++ b/pkg/giturl/giturl_test.go @@ -4,6 +4,8 @@ import ( "testing" "github.com/pkg/errors" + + "github.com/trufflesecurity/trufflehog/v3/pkg/context" ) func Test_NormalizeOrgRepoURL(t *testing.T) { @@ -187,3 +189,84 @@ func TestGenerateLink(t *testing.T) { }) } } + +func TestUpdateLinkLineNumber(t *testing.T) { + t.Parallel() + + type args struct { + link string + newLine int64 + } + tests := []struct { + name string + args args + want string + wantErr bool + }{ + { + name: "Update bitbucket, no line number supported", + args: args{ + link: "https://bitbucket.org/org/repo/blob/xyz123/main.go", + newLine: int64(10), + }, + want: "https://bitbucket.org/org/repo/blob/xyz123/main.go", + }, + { + name: "Update github link with line", + args: args{ + link: "https://github.com/trufflesec-julian/confluence-go-api/blob/047b4a2ba42fc5b6c0bd535c5307434a666db5ec/.gitignore#L4", + newLine: int64(10), + }, + want: "https://github.com/trufflesec-julian/confluence-go-api/blob/047b4a2ba42fc5b6c0bd535c5307434a666db5ec/.gitignore#L10", + }, + { + name: "Update Azure link with line", + args: args{ + link: "https://dev.azure.com/org/project/_git/repo/commit/abcdef/main.go?line=20", + newLine: int64(40), + }, + want: "https://dev.azure.com/org/project/_git/repo/commit/abcdef/main.go?line=40", + }, + { + name: "Add line to github link without line", + args: args{ + link: "https://github.com/trufflesec-julian/confluence-go-api/blob/047b4a2ba42fc5b6c0bd535c5307434a666db5ec/.gitignore", + newLine: int64(7), + }, + want: "https://github.com/trufflesec-julian/confluence-go-api/blob/047b4a2ba42fc5b6c0bd535c5307434a666db5ec/.gitignore#L7", + }, + { + name: "Update Unknown provider on-prem instance with line", + args: args{ + link: "https://onprem.customdomain.com/org/repo/blob/xyz123/main.go#L30", + newLine: int64(50), + }, + want: "https://onprem.customdomain.com/org/repo/blob/xyz123/main.go#L50", + }, + { + name: "Update Unknown provider on-prem instance without line", + args: args{ + link: "https://onprem.customdomain.com/org/repo/commit/xyz123", + newLine: int64(50), + }, + want: "https://onprem.customdomain.com/org/repo/commit/xyz123#L50", + }, + { + name: "Invalid link", + args: args{ + link: "definitely not a link", + newLine: int64(50), + }, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := UpdateLinkLineNumber(context.Background(), tt.args.link, tt.args.newLine) + if got != tt.want && !tt.wantErr { + t.Errorf("UpdateLinkLineNumber() = %v, want %v", got, tt.want) + } + }) + } +}