trufflesecurity/trufflehog
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Ability to update line number in link (#1788)

Browse Source 
* Add functionality to update a source's link in the metadata with the updated line number.

* update comment.

* add logic to engine.

* only update link for non empty links.

* add tests for bb.
This commit is contained in:
ahrav
2023-09-19 15:39:13 -07:00
committed by GitHub
parent 63eaccd208
commit 47d5ddebf2
4 changed files with 212 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/engine/engine.go
+28 -9
View File
@@ -19,6 +19,7 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
"github.com/trufflesecurity/trufflehog/v3/pkg/decoders"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/giturl"
"github.com/trufflesecurity/trufflehog/v3/pkg/output"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
@@ -532,12 +533,12 @@ func (e *Engine) detectChunk(ctx context.Context, data detectableChunk) {
}
for _, res := range results {
e.processResult(data, res)
e.processResult(ctx, data, res)
}
data.wgDoneFn()
}
func (e *Engine) processResult(data detectableChunk, res detectors.Result) {
func (e *Engine) processResult(ctx context.Context, data detectableChunk, res detectors.Result) {
ignoreLinePresent := false
if SupportsLineNumbers(data.chunk.SourceType) {
copyChunk := data.chunk
@@ -545,8 +546,11 @@ func (e *Engine) processResult(data detectableChunk, res detectors.Result) {
if copyMetaData, ok := copyMetaDataClone.(*source_metadatapb.MetaData); ok {
copyChunk.SourceMetadata = copyMetaData
}
fragStart, mdLine := FragmentFirstLine(&copyChunk)
fragStart, mdLine, link := FragmentFirstLineAndLink(&copyChunk)
ignoreLinePresent = SetResultLineNumber(&copyChunk, &res, fragStart, mdLine)
if link != "" {
copyChunk.SourceMetadata.GetGithub().Link = giturl.UpdateLinkLineNumber(ctx, link, *mdLine)
}
data.chunk = copyChunk
}
if ignoreLinePresent {
@@ -625,27 +629,42 @@ func FragmentLineOffset(chunk *sources.Chunk, result *detectors.Result) (int64,
return lineNumber, false
}
// FragmentFirstLine returns the first line number of a fragment along with a pointer to the value to update in the
// chunk metadata.
func FragmentFirstLine(chunk *sources.Chunk) (int64, *int64) {
var fragmentStart *int64
// FragmentFirstLineAndLink extracts the first line number and the link from the chunk metadata.
// It returns:
// - The first line number of the fragment.
// - A pointer to the line number, facilitating direct updates.
// - The link associated with the fragment. This link may be updated in the chunk metadata
// if there's a change in the line number.
func FragmentFirstLineAndLink(chunk *sources.Chunk) (int64, *int64, string) {
if chunk.SourceMetadata == nil {
return 0, nil, ""
}
var (
fragmentStart *int64
link string
)
switch metadata := chunk.SourceMetadata.GetData().(type) {
case *source_metadatapb.MetaData_Git:
fragmentStart = &metadata.Git.Line
case *source_metadatapb.MetaData_Github:
fragmentStart = &metadata.Github.Line
link = metadata.Github.Link
case *source_metadatapb.MetaData_Gitlab:
fragmentStart = &metadata.Gitlab.Line
link = metadata.Gitlab.Link
case *source_metadatapb.MetaData_Bitbucket:
fragmentStart = &metadata.Bitbucket.Line
link = metadata.Bitbucket.Link
case *source_metadatapb.MetaData_Gerrit:
fragmentStart = &metadata.Gerrit.Line
case *source_metadatapb.MetaData_Filesystem:
fragmentStart = &metadata.Filesystem.Line
link = metadata.Filesystem.Link
default:
return 0, nil
return 0, nil, ""
}
return *fragmentStart, fragmentStart
return *fragmentStart, fragmentStart, link
}
// SetResultLineNumber sets the line number in the provided result.
pkg/engine/engine_test.go
+58
View File
@@ -11,6 +11,7 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
"github.com/trufflesecurity/trufflehog/v3/pkg/decoders"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
)
@@ -205,3 +206,60 @@ func TestEngine_DuplicatSecrets(t *testing.T) {
want := uint64(5)
assert.Equal(t, want, e.GetMetrics().UnverifiedSecretsFound)
}
func TestFragmentFirstLineAndLink(t *testing.T) {
tests := []struct {
name string
chunk *sources.Chunk
expectedLine int64
expectedLink string
}{
{
name: "Test Git Metadata",
chunk: &sources.Chunk{
SourceMetadata: &source_metadatapb.MetaData{
Data: &source_metadatapb.MetaData_Git{
Git: &source_metadatapb.Git{
Line: 10,
},
},
},
},
expectedLine: 10,
expectedLink: "", // Git doesn't support links
},
{
name: "Test Github Metadata",
chunk: &sources.Chunk{
SourceMetadata: &source_metadatapb.MetaData{
Data: &source_metadatapb.MetaData_Github{
Github: &source_metadatapb.Github{
Line: 5,
Link: "https://example.github.com",
},
},
},
},
expectedLine: 5,
expectedLink: "https://example.github.com",
},
{
name: "Unsupported Type",
chunk: &sources.Chunk{},
expectedLine: 0,
expectedLink: "",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
line, linePtr, link := FragmentFirstLineAndLink(tt.chunk)
assert.Equal(t, tt.expectedLink, link, "Mismatch in link")
assert.Equal(t, tt.expectedLine, line, "Mismatch in line")
if linePtr != nil {
assert.Equal(t, tt.expectedLine, *linePtr, "Mismatch in linePtr value")
}
})
}
}
pkg/giturl/giturl.go
+43
View File
@@ -2,10 +2,13 @@ package giturl
import (
"net/url"
"regexp"
"strconv"
"strings"
"github.com/pkg/errors"
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
)
type provider string
@@ -143,3 +146,43 @@ func GenerateLink(repo, commit, file string, line int64) string {
return baseLink
}
}
var linePattern = regexp.MustCompile(`L\d+`)
// UpdateLinkLineNumber updates the line number in a repository link.
// Used post-link generation to refine reported issue locations within large scanned blocks.
func UpdateLinkLineNumber(ctx context.Context, link string, newLine int64) string {
parsedURL, err := url.Parse(link)
if err != nil {
ctx.Logger().Error(err, "unable to parse link to update line number", "link", link)
return link
}
switch determineProvider(link) {
case providerBitbucket:
// For Bitbucket, it doesn't support line links (based on the GenerateLink function).
// So we don't need to change anything.
return link
case providerAzure:
// For Azure, line numbers are appended as ?line=<number>.
query := parsedURL.Query()
query.Set("line", strconv.FormatInt(newLine, 10))
parsedURL.RawQuery = query.Encode()
case providerGithub, providerGitlab:
// If the provider name isn't one of the cloud defaults, it is probably an on-prem github or gitlab.
// So do the same thing.
fallthrough
default:
// Assumed format: .../blob/<commit>/file.go#L<number>
fragment := "L" + strconv.FormatInt(newLine, 10)
if linePattern.MatchString(parsedURL.Fragment) {
parsedURL.Fragment = linePattern.ReplaceAllString(parsedURL.Fragment, fragment)
} else {
parsedURL.Fragment += fragment
}
}
return parsedURL.String()
}
pkg/giturl/giturl_test.go
+83
View File
@@ -4,6 +4,8 @@ import (
"testing"
"github.com/pkg/errors"
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
)
func Test_NormalizeOrgRepoURL(t *testing.T) {
@@ -187,3 +189,84 @@ func TestGenerateLink(t *testing.T) {
})
}
}
func TestUpdateLinkLineNumber(t *testing.T) {
t.Parallel()
type args struct {
link string
newLine int64
}
tests := []struct {
name string
args args
want string
wantErr bool
}{
{
name: "Update bitbucket, no line number supported",
args: args{
link: "https://bitbucket.org/org/repo/blob/xyz123/main.go",
newLine: int64(10),
},
want: "https://bitbucket.org/org/repo/blob/xyz123/main.go",
},
{
name: "Update github link with line",
args: args{
link: "https://github.com/trufflesec-julian/confluence-go-api/blob/047b4a2ba42fc5b6c0bd535c5307434a666db5ec/.gitignore#L4",
newLine: int64(10),
},
want: "https://github.com/trufflesec-julian/confluence-go-api/blob/047b4a2ba42fc5b6c0bd535c5307434a666db5ec/.gitignore#L10",
},
{
name: "Update Azure link with line",
args: args{
link: "https://dev.azure.com/org/project/_git/repo/commit/abcdef/main.go?line=20",
newLine: int64(40),
},
want: "https://dev.azure.com/org/project/_git/repo/commit/abcdef/main.go?line=40",
},
{
name: "Add line to github link without line",
args: args{
link: "https://github.com/trufflesec-julian/confluence-go-api/blob/047b4a2ba42fc5b6c0bd535c5307434a666db5ec/.gitignore",
newLine: int64(7),
},
want: "https://github.com/trufflesec-julian/confluence-go-api/blob/047b4a2ba42fc5b6c0bd535c5307434a666db5ec/.gitignore#L7",
},
{
name: "Update Unknown provider on-prem instance with line",
args: args{
link: "https://onprem.customdomain.com/org/repo/blob/xyz123/main.go#L30",
newLine: int64(50),
},
want: "https://onprem.customdomain.com/org/repo/blob/xyz123/main.go#L50",
},
{
name: "Update Unknown provider on-prem instance without line",
args: args{
link: "https://onprem.customdomain.com/org/repo/commit/xyz123",
newLine: int64(50),
},
want: "https://onprem.customdomain.com/org/repo/commit/xyz123#L50",
},
{
name: "Invalid link",
args: args{
link: "definitely not a link",
newLine: int64(50),
},
wantErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := UpdateLinkLineNumber(context.Background(), tt.args.link, tt.args.newLine)
if got != tt.want && !tt.wantErr {
t.Errorf("UpdateLinkLineNumber() = %v, want %v", got, tt.want)
}
})
}
}