trufflesecurity/trufflehog
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[oc-147] - Add context to all git methods (#901)

Browse Source 
* Add context to all git methods.

* remove logrus.

* Add ctx.

* Address comments.

* Add error to clone failing.

* Return error.
This commit is contained in:
ahrav
2022-11-03 16:36:52 -07:00
committed by GitHub
parent 3a143f095b
commit dd141fb55f
9 changed files with 96 additions and 88 deletions
Download Patch File Download Diff File Expand all files Collapse all files
hack/snifftest/main.go
+1 -1
View File
@@ -162,7 +162,7 @@ func main() {
defer sem.Release(1) defer sem.Release(1)
defer wgChunkers.Done() defer wgChunkers.Done()
log.Infof("cloning %s", r) log.Infof("cloning %s", r)
path, repo, err := git.CloneRepoUsingUnauthenticated(r) path, repo, err := git.CloneRepoUsingUnauthenticated(ctx, r)
if err != nil { if err != nil {
log.Fatal(err) log.Fatal(err)
} }
main.go
+2 -2
View File
@@ -185,7 +185,7 @@ func run(state overseer.State) {
var remote bool var remote bool
switch cmd { switch cmd {
case gitScan.FullCommand(): case gitScan.FullCommand():
repoPath, remote, err = git.PrepareRepoSinceCommit(*gitScanURI, *gitScanSinceCommit) repoPath, remote, err = git.PrepareRepoSinceCommit(ctx, *gitScanURI, *gitScanSinceCommit)
if err != nil || repoPath == "" { if err != nil || repoPath == "" {
logrus.WithError(err).Fatal("error preparing git repo for scanning") logrus.WithError(err).Fatal("error preparing git repo for scanning")
} }
@@ -282,7 +282,7 @@ func run(state overseer.State) {
switch { switch {
case *jsonLegacy: case *jsonLegacy:
output.PrintLegacyJSON(&r) output.PrintLegacyJSON(ctx, &r)
case *jsonOut: case *jsonOut:
output.PrintJSON(&r) output.PrintJSON(&r)
default: default:
pkg/engine/git_test.go
+4 -2
View File
@@ -18,8 +18,9 @@ type expResult struct {
} }
func TestGitEngine(t *testing.T) { func TestGitEngine(t *testing.T) {
ctx := context.Background()
repoUrl := "https://github.com/dustin-decker/secretsandstuff.git" repoUrl := "https://github.com/dustin-decker/secretsandstuff.git"
path, _, err := git.PrepareRepo(repoUrl) path, _, err := git.PrepareRepo(ctx, repoUrl)
if err != nil { if err != nil {
t.Error(err) t.Error(err)
} }
@@ -89,8 +90,9 @@ func TestGitEngine(t *testing.T) {
} }
func BenchmarkGitEngine(b *testing.B) { func BenchmarkGitEngine(b *testing.B) {
ctx := context.Background()
repoUrl := "https://github.com/dustin-decker/secretsandstuff.git" repoUrl := "https://github.com/dustin-decker/secretsandstuff.git"
path, _, err := git.PrepareRepo(repoUrl) path, _, err := git.PrepareRepo(ctx, repoUrl)
if err != nil { if err != nil {
b.Error(err) b.Error(err)
} }
pkg/output/legacy_json.go
+4 -2
View File
@@ -13,12 +13,14 @@ import (
"github.com/go-git/go-git/v5/plumbing/object" "github.com/go-git/go-git/v5/plumbing/object"
"github.com/sergi/go-diff/diffmatchpatch" "github.com/sergi/go-diff/diffmatchpatch"
"github.com/sirupsen/logrus" "github.com/sirupsen/logrus"
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources/git" "github.com/trufflesecurity/trufflehog/v3/pkg/sources/git"
) )
func PrintLegacyJSON(r *detectors.ResultWithMetadata) { func PrintLegacyJSON(ctx context.Context, r *detectors.ResultWithMetadata) {
var repo string var repo string
switch r.SourceType { switch r.SourceType {
case sourcespb.SourceType_SOURCE_TYPE_GIT: case sourcespb.SourceType_SOURCE_TYPE_GIT:
@@ -32,7 +34,7 @@ func PrintLegacyJSON(r *detectors.ResultWithMetadata) {
} }
// cloning the repo again here is not great and only works with unauthed repos // cloning the repo again here is not great and only works with unauthed repos
repoPath, remote, err := git.PrepareRepo(repo) repoPath, remote, err := git.PrepareRepo(ctx, repo)
if err != nil || repoPath == "" { if err != nil || repoPath == "" {
logrus.WithError(err).Fatal("error preparing git repo for scanning") logrus.WithError(err).Fatal("error preparing git repo for scanning")
} }
pkg/sources/git/git.go
+72 -70
View File
@@ -20,8 +20,6 @@ import (
"github.com/go-git/go-git/v5/plumbing" "github.com/go-git/go-git/v5/plumbing"
"github.com/go-git/go-git/v5/plumbing/object" "github.com/go-git/go-git/v5/plumbing/object"
"github.com/google/go-github/v42/github" "github.com/google/go-github/v42/github"
"github.com/rs/zerolog"
log "github.com/sirupsen/logrus"
"golang.org/x/oauth2" "golang.org/x/oauth2"
"golang.org/x/sync/semaphore" "golang.org/x/sync/semaphore"
"google.golang.org/protobuf/proto" "google.golang.org/protobuf/proto"
@@ -140,7 +138,7 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
continue continue
} }
err := func(repoURI string) error { err := func(repoURI string) error {
path, repo, err := CloneRepoUsingToken(token, repoURI, user) path, repo, err := CloneRepoUsingToken(ctx, token, repoURI, user)
defer os.RemoveAll(path) defer os.RemoveAll(path)
if err != nil { if err != nil {
return err return err
@@ -158,7 +156,7 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
continue continue
} }
err := func(repoURI string) error { err := func(repoURI string) error {
path, repo, err := CloneRepoUsingUnauthenticated(repoURI) path, repo, err := CloneRepoUsingUnauthenticated(ctx, repoURI)
defer os.RemoveAll(path) defer os.RemoveAll(path)
if err != nil { if err != nil {
return err return err
@@ -176,7 +174,7 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
continue continue
} }
err := func(repoURI string) error { err := func(repoURI string) error {
path, repo, err := CloneRepoUsingSSH(repoURI) path, repo, err := CloneRepoUsingSSH(ctx, repoURI)
defer os.RemoveAll(path) defer os.RemoveAll(path)
if err != nil { if err != nil {
return err return err
@@ -217,6 +215,8 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
} }
} }
ctx.Logger().V(1).Info("Git source finished scanning", "repo-count", len(s.conn.Repositories))
s.SetProgressComplete(len(s.conn.Repositories), len(s.conn.Repositories), fmt.Sprintf("Completed scanning source %s", s.name), "") s.SetProgressComplete(len(s.conn.Repositories), len(s.conn.Repositories), fmt.Sprintf("Completed scanning source %s", s.name), "")
return nil return nil
} }
@@ -246,14 +246,13 @@ func gitURLParse(gitURL string) (*url.URL, error) {
return parsedURL, nil return parsedURL, nil
} }
func CloneRepo(userInfo *url.Userinfo, gitUrl string, args ...string) (clonePath string, repo *git.Repository, err error) { func CloneRepo(ctx context.Context, userInfo *url.Userinfo, gitUrl string, args ...string) (string, *git.Repository, error) {
if err = GitCmdCheck(); err != nil { if err := gitCmdCheck(); err != nil {
return return "", nil, err
} }
clonePath, err = ioutil.TempDir(os.TempDir(), "trufflehog") clonePath, err := ioutil.TempDir(os.TempDir(), "trufflehog")
if err != nil { if err != nil {
err = errors.New(err) return "", nil, err
return
} }
defer CleanOnError(&err, clonePath) defer CleanOnError(&err, clonePath)
cloneURL, err := gitURLParse(gitUrl) cloneURL, err := gitURLParse(gitUrl)
@@ -279,38 +278,40 @@ func CloneRepo(userInfo *url.Userinfo, gitUrl string, args ...string) (clonePath
if cloneCmd.ProcessState != nil && cloneCmd.ProcessState.ExitCode() != 0 { if cloneCmd.ProcessState != nil && cloneCmd.ProcessState.ExitCode() != 0 {
safeUrl, err := stripPassword(gitUrl) safeUrl, err := stripPassword(gitUrl)
if err != nil { if err != nil {
log.WithError(err).Errorf("failed to strip credentials from git url") ctx.Logger().V(1).Info("error stripping password from git url", "error", err)
} }
log.WithField("exit_code", cloneCmd.ProcessState.ExitCode()).WithField("repo", safeUrl).WithField("output", string(output)).Errorf("failed to clone repo") ctx.Logger().V(1).Info("git clone failed", "error", err, "repo", safeUrl, "output", string(output))
return "", nil, fmt.Errorf("could not clone repo: %s", safeUrl) return "", nil, fmt.Errorf("could not clone repo: %s, %w", safeUrl, err)
} }
repo, err = git.PlainOpen(clonePath)
repo, err := git.PlainOpen(clonePath)
if err != nil { if err != nil {
err = errors.WrapPrefix(err, "could not open cloned repo", 0) return "", nil, fmt.Errorf("could not open cloned repo: %w", err)
return
} }
log.WithField("clone_path", clonePath).WithField("repo", gitUrl).Debug("cloned repo")
return ctx.Logger().V(1).Info("cloned repo", "repo", gitUrl, "clone-path", clonePath)
return clonePath, repo, nil
} }
// CloneRepoUsingToken clones a repo using a provided token. // CloneRepoUsingToken clones a repo using a provided token.
func CloneRepoUsingToken(token, gitUrl, user string, args ...string) (string, *git.Repository, error) { func CloneRepoUsingToken(ctx context.Context, token, gitUrl, user string, args ...string) (string, *git.Repository, error) {
userInfo := url.UserPassword(user, token) userInfo := url.UserPassword(user, token)
return CloneRepo(userInfo, gitUrl, args...) return CloneRepo(ctx, userInfo, gitUrl, args...)
} }
// CloneRepoUsingUnauthenticated clones a repo with no authentication required. // CloneRepoUsingUnauthenticated clones a repo with no authentication required.
func CloneRepoUsingUnauthenticated(url string, args ...string) (string, *git.Repository, error) { func CloneRepoUsingUnauthenticated(ctx context.Context, url string, args ...string) (string, *git.Repository, error) {
return CloneRepo(nil, url, args...) return CloneRepo(ctx, nil, url, args...)
} }
// CloneRepoUsingUnauthenticated clones a repo with no authentication required. // CloneRepoUsingSSH clones a repo using SSH.
func CloneRepoUsingSSH(gitUrl string, args ...string) (string, *git.Repository, error) { func CloneRepoUsingSSH(ctx context.Context, gitUrl string, args ...string) (string, *git.Repository, error) {
userInfo := url.User("git") userInfo := url.User("git")
return CloneRepo(userInfo, gitUrl, args...) return CloneRepo(ctx, userInfo, gitUrl, args...)
} }
func GitCmdCheck() error { // gitCmdCheck checks if git is installed.
func gitCmdCheck() error {
if errors.Is(exec.Command("git").Run(), exec.ErrNotFound) { if errors.Is(exec.Command("git").Run(), exec.ErrNotFound) {
return fmt.Errorf("'git' command not found in $PATH. Make sure git is installed and included in $PATH") return fmt.Errorf("'git' command not found in $PATH. Make sure git is installed and included in $PATH")
} }
@@ -318,12 +319,9 @@ func GitCmdCheck() error {
} }
func (s *Git) ScanCommits(ctx context.Context, repo *git.Repository, path string, scanOptions *ScanOptions, chunksChan chan *sources.Chunk) error { func (s *Git) ScanCommits(ctx context.Context, repo *git.Repository, path string, scanOptions *ScanOptions, chunksChan chan *sources.Chunk) error {
if err := GitCmdCheck(); err != nil { if err := gitCmdCheck(); err != nil {
return err return err
} }
if log.GetLevel() < log.DebugLevel {
zerolog.SetGlobalLevel(zerolog.Disabled)
}
commitChan, err := gitparse.RepoPath(ctx, path, scanOptions.HeadHash) commitChan, err := gitparse.RepoPath(ctx, path, scanOptions.HeadHash)
if err != nil { if err != nil {
@@ -338,11 +336,12 @@ func (s *Git) ScanCommits(ctx context.Context, repo *git.Repository, path string
var depth int64 var depth int64
var reachedBase = false var reachedBase = false
log.WithField("repo", urlMetadata).Debugf("Scanning repo")
ctx.Logger().V(1).Info("scanning repo", "repo", urlMetadata, "base", scanOptions.BaseHash, "head", scanOptions.HeadHash)
for commit := range commitChan { for commit := range commitChan {
log.Tracef("Scanning commit %s", commit.Hash) ctx.Logger().V(5).Info("scanning commit", "commit", commit.Hash, "message", commit.Message)
if scanOptions.MaxDepth > 0 && depth >= scanOptions.MaxDepth { if scanOptions.MaxDepth > 0 && depth >= scanOptions.MaxDepth {
log.Debugf("reached max depth") ctx.Logger().V(1).Info("reached max depth", "depth", depth)
break break
} }
depth++ depth++
@@ -351,13 +350,11 @@ func (s *Git) ScanCommits(ctx context.Context, repo *git.Repository, path string
} }
if len(scanOptions.BaseHash) > 0 { if len(scanOptions.BaseHash) > 0 {
if commit.Hash == scanOptions.BaseHash { if commit.Hash == scanOptions.BaseHash {
log.Debugf("Reached base commit. Finishing scanning files.") ctx.Logger().V(1).Info("reached base commit", "commit", commit.Hash)
reachedBase = true reachedBase = true
} }
} }
for _, diff := range commit.Diffs { for _, diff := range commit.Diffs {
log.WithField("commit", commit.Hash).WithField("file", diff.PathB).Trace("Scanning file from git")
if !scanOptions.Filter.Pass(diff.PathB) { if !scanOptions.Filter.Pass(diff.PathB) {
continue continue
} }
@@ -383,13 +380,13 @@ func (s *Git) ScanCommits(ctx context.Context, repo *git.Repository, path string
Verify: s.verify, Verify: s.verify,
} }
if err := handleBinary(ctx, repo, chunksChan, chunkSkel, commitHash, fileName); err != nil { if err := handleBinary(ctx, repo, chunksChan, chunkSkel, commitHash, fileName); err != nil {
log.WithError(err).WithField("file", fileName).Debug("Error handling binary file") ctx.Logger().V(1).Info("error handling binary file", "error", err, "filename", fileName, "commit", commitHash, "file", diff.PathB)
} }
continue continue
} }
if diff.Content.Len() > sources.ChunkSize+sources.PeekSize { if diff.Content.Len() > sources.ChunkSize+sources.PeekSize {
s.gitChunk(diff, fileName, email, hash, when, urlMetadata, chunksChan) s.gitChunk(ctx, diff, fileName, email, hash, when, urlMetadata, chunksChan)
continue continue
} }
metadata := s.sourceMetadataFunc(fileName, email, hash, when, urlMetadata, int64(diff.LineStart)) metadata := s.sourceMetadataFunc(fileName, email, hash, when, urlMetadata, int64(diff.LineStart))
@@ -406,7 +403,7 @@ func (s *Git) ScanCommits(ctx context.Context, repo *git.Repository, path string
return nil return nil
} }
func (s *Git) gitChunk(diff gitparse.Diff, fileName, email, hash, when, urlMetadata string, chunksChan chan *sources.Chunk) { func (s *Git) gitChunk(ctx context.Context, diff gitparse.Diff, fileName, email, hash, when, urlMetadata string, chunksChan chan *sources.Chunk) {
originalChunk := bufio.NewScanner(&diff.Content) originalChunk := bufio.NewScanner(&diff.Content)
newChunkBuffer := bytes.Buffer{} newChunkBuffer := bytes.Buffer{}
lastOffset := 0 lastOffset := 0
@@ -443,9 +440,8 @@ func (s *Git) gitChunk(diff gitparse.Diff, fileName, email, hash, when, urlMetad
} }
} }
_, err := newChunkBuffer.Write(line) if _, err := newChunkBuffer.Write(line); err != nil {
if err != nil { ctx.Logger().Error(err, "error writing to chunk buffer", "filename", fileName, "commit", hash, "file", diff.PathB)
log.WithError(err).Error("Could not write line to git diff buffer.")
} }
} }
// Send anything still in the new chunk buffer // Send anything still in the new chunk buffer
@@ -477,12 +473,15 @@ func (s *Git) ScanUnstaged(ctx context.Context, repo *git.Repository, path strin
var depth int64 var depth int64
var reachedBase = false var reachedBase = false
log.Debugf("Scanning repo")
ctx.Logger().V(1).Info("scanning unstaged changes", "path", path)
for commit := range commitChan { for commit := range commitChan {
for _, diff := range commit.Diffs { for _, diff := range commit.Diffs {
log.WithField("commit", commit.Hash).WithField("file", diff.PathB).Trace("Scanning file from git") logger := ctx.Logger().WithValues("filename", diff.PathB, "commit", commit.Hash, "file", diff.PathB)
logger.V(2).Info("scanning unstaged changes from git")
if scanOptions.MaxDepth > 0 && depth >= scanOptions.MaxDepth { if scanOptions.MaxDepth > 0 && depth >= scanOptions.MaxDepth {
log.Debugf("reached max depth") logger.V(1).Info("reached max depth")
break break
} }
depth++ depth++
@@ -491,7 +490,7 @@ func (s *Git) ScanUnstaged(ctx context.Context, repo *git.Repository, path strin
} }
if len(scanOptions.BaseHash) > 0 { if len(scanOptions.BaseHash) > 0 {
if commit.Hash == scanOptions.BaseHash { if commit.Hash == scanOptions.BaseHash {
log.Debugf("Reached base commit. Finishing scanning files.") logger.V(1).Info("reached base hash, finishing scanning files")
reachedBase = true reachedBase = true
} }
} }
@@ -521,7 +520,7 @@ func (s *Git) ScanUnstaged(ctx context.Context, repo *git.Repository, path strin
Verify: s.verify, Verify: s.verify,
} }
if err := handleBinary(ctx, repo, chunksChan, chunkSkel, commitHash, fileName); err != nil { if err := handleBinary(ctx, repo, chunksChan, chunkSkel, commitHash, fileName); err != nil {
log.WithError(err).WithField("file", fileName).Debug("Error handling binary file") logger.V(1).Info("error handling binary file", "error", err, "filename", fileName)
} }
continue continue
} }
@@ -549,10 +548,11 @@ func (s *Git) ScanRepo(ctx context.Context, repo *git.Repository, repoPath strin
return err return err
} }
if err := s.ScanUnstaged(ctx, repo, repoPath, scanOptions, chunksChan); err != nil { if err := s.ScanUnstaged(ctx, repo, repoPath, scanOptions, chunksChan); err != nil {
log.WithError(err).Error("Error scanning unstaged changes") ctx.Logger().V(1).Info("error scanning unstaged changes", "error", err)
} }
scanTime := time.Now().UnixNano() - start scanTime := time.Now().UnixNano() - start
log.Debugf("Scanning complete. Scan time: %f", time.Duration(scanTime).Seconds()) ctx.Logger().V(1).Info("scanning git repo complete", "path", repoPath, "time", scanTime)
return nil return nil
} }
@@ -656,9 +656,9 @@ func TryAdditionalBaseRefs(repo *git.Repository, base string) (*plumbing.Hash, e
} }
// PrepareRepoSinceCommit clones a repo starting at the given commitHash and returns the cloned repo path. // PrepareRepoSinceCommit clones a repo starting at the given commitHash and returns the cloned repo path.
func PrepareRepoSinceCommit(uriString, commitHash string) (string, bool, error) { func PrepareRepoSinceCommit(ctx context.Context, uriString, commitHash string) (string, bool, error) {
if commitHash == "" { if commitHash == "" {
return PrepareRepo(uriString) return PrepareRepo(ctx, uriString)
} }
// TODO: refactor with PrepareRepo to remove duplicated logic // TODO: refactor with PrepareRepo to remove duplicated logic
@@ -673,13 +673,13 @@ func PrepareRepoSinceCommit(uriString, commitHash string) (string, bool, error)
} }
if uri.Scheme == "file" || uri.Host != "github.com" { if uri.Scheme == "file" || uri.Host != "github.com" {
return PrepareRepo(uriString) return PrepareRepo(ctx, uriString)
} }
uriPath := strings.TrimPrefix(uri.Path, "/") uriPath := strings.TrimPrefix(uri.Path, "/")
owner, repoName, found := strings.Cut(uriPath, "/") owner, repoName, found := strings.Cut(uriPath, "/")
if !found { if !found {
return PrepareRepo(uriString) return PrepareRepo(ctx, uriString)
} }
client := github.NewClient(nil) client := github.NewClient(nil)
@@ -693,13 +693,13 @@ func PrepareRepoSinceCommit(uriString, commitHash string) (string, bool, error)
commit, _, err := client.Git.GetCommit(context.Background(), owner, repoName, commitHash) commit, _, err := client.Git.GetCommit(context.Background(), owner, repoName, commitHash)
if err != nil { if err != nil {
return PrepareRepo(uriString) return PrepareRepo(ctx, uriString)
} }
var timestamp string var timestamp string
{ {
author := commit.GetAuthor() author := commit.GetAuthor()
if author == nil { if author == nil {
return PrepareRepo(uriString) return PrepareRepo(ctx, uriString)
} }
timestamp = author.GetDate().Format(time.RFC3339) timestamp = author.GetDate().Format(time.RFC3339)
} }
@@ -708,28 +708,29 @@ func PrepareRepoSinceCommit(uriString, commitHash string) (string, bool, error)
var path string var path string
switch { switch {
case uri.User != nil: case uri.User != nil:
log.Debugf("Cloning remote Git repo with authentication") ctx.Logger().V(1).Info("cloning repo with authentication", "uri", uri)
password, ok := uri.User.Password() password, ok := uri.User.Password()
if !ok { if !ok {
return "", true, fmt.Errorf("password must be included in Git repo URL when username is provided") return "", true, fmt.Errorf("password must be included in Git repo URL when username is provided")
} }
path, _, err = CloneRepoUsingToken(password, remotePath, uri.User.Username(), "--shallow-since", timestamp) path, _, err = CloneRepoUsingToken(ctx, password, remotePath, uri.User.Username(), "--shallow-since", timestamp)
if err != nil { if err != nil {
return path, true, fmt.Errorf("failed to clone authenticated Git repo (%s): %s", remotePath, err) return path, true, fmt.Errorf("failed to clone authenticated Git repo (%s): %s", remotePath, err)
} }
default: default:
log.Debugf("Cloning remote Git repo without authentication") ctx.Logger().V(1).Info("cloning repo without authentication", "uri", uri)
path, _, err = CloneRepoUsingUnauthenticated(remotePath, "--shallow-since", timestamp) path, _, err = CloneRepoUsingUnauthenticated(ctx, remotePath, "--shallow-since", timestamp)
if err != nil { if err != nil {
return path, true, fmt.Errorf("failed to clone unauthenticated Git repo (%s): %s", remotePath, err) return path, true, fmt.Errorf("failed to clone unauthenticated Git repo (%s): %s", remotePath, err)
} }
} }
log.Debugf("Git repo local path: %s", path)
ctx.Logger().V(1).Info("cloned repo", "path", path)
return path, true, nil return path, true, nil
} }
// PrepareRepo clones a repo if possible and returns the cloned repo path. // PrepareRepo clones a repo if possible and returns the cloned repo path.
func PrepareRepo(uriString string) (string, bool, error) { func PrepareRepo(ctx context.Context, uriString string) (string, bool, error) {
var path string var path string
uri, err := gitURLParse(uriString) uri, err := gitURLParse(uriString)
if err != nil { if err != nil {
@@ -745,18 +746,18 @@ func PrepareRepo(uriString string) (string, bool, error) {
remote = true remote = true
switch { switch {
case uri.User != nil: case uri.User != nil:
log.Debugf("Cloning remote Git repo with authentication") ctx.Logger().V(1).Info("cloning repo with authentication", "uri", uri)
password, ok := uri.User.Password() password, ok := uri.User.Password()
if !ok { if !ok {
return "", remote, fmt.Errorf("password must be included in Git repo URL when username is provided") return "", remote, fmt.Errorf("password must be included in Git repo URL when username is provided")
} }
path, _, err = CloneRepoUsingToken(password, remotePath, uri.User.Username()) path, _, err = CloneRepoUsingToken(ctx, password, remotePath, uri.User.Username())
if err != nil { if err != nil {
return path, remote, fmt.Errorf("failed to clone authenticated Git repo (%s): %s", remotePath, err) return path, remote, fmt.Errorf("failed to clone authenticated Git repo (%s): %s", remotePath, err)
} }
default: default:
log.Debugf("Cloning remote Git repo without authentication") ctx.Logger().V(1).Info("cloning repo without authentication", "uri", uri)
path, _, err = CloneRepoUsingUnauthenticated(remotePath) path, _, err = CloneRepoUsingUnauthenticated(ctx, remotePath)
if err != nil { if err != nil {
return path, remote, fmt.Errorf("failed to clone unauthenticated Git repo (%s): %s", remotePath, err) return path, remote, fmt.Errorf("failed to clone unauthenticated Git repo (%s): %s", remotePath, err)
} }
@@ -764,14 +765,15 @@ func PrepareRepo(uriString string) (string, bool, error) {
case "ssh": case "ssh":
remotePath := uri.String() remotePath := uri.String()
remote = true remote = true
path, _, err = CloneRepoUsingSSH(remotePath) path, _, err = CloneRepoUsingSSH(ctx, remotePath)
if err != nil { if err != nil {
return path, remote, fmt.Errorf("failed to clone unauthenticated Git repo (%s): %s", remotePath, err) return path, remote, fmt.Errorf("failed to clone unauthenticated Git repo (%s): %s", remotePath, err)
} }
default: default:
return "", remote, fmt.Errorf("unsupported Git URI: %s", uriString) return "", remote, fmt.Errorf("unsupported Git URI: %s", uriString)
} }
log.Debugf("Git repo local path: %s", path)
ctx.Logger().V(1).Info("cloned repo", "path", path)
return path, remote, nil return path, remote, nil
} }
@@ -799,7 +801,7 @@ func getSafeRemoteURL(repo *git.Repository, preferred string) string {
} }
func handleBinary(ctx context.Context, repo *git.Repository, chunksChan chan *sources.Chunk, chunkSkel *sources.Chunk, commitHash plumbing.Hash, path string) error { func handleBinary(ctx context.Context, repo *git.Repository, chunksChan chan *sources.Chunk, chunkSkel *sources.Chunk, commitHash plumbing.Hash, path string) error {
log.WithField("path", path).Trace("Binary file found in repository.") ctx.Logger().V(5).Info("handling binary file", "path", path)
commit, err := repo.CommitObject(commitHash) commit, err := repo.CommitObject(commitHash)
if err != nil { if err != nil {
return err return err
@@ -825,7 +827,7 @@ func handleBinary(ctx context.Context, repo *git.Repository, chunksChan chan *so
return nil return nil
} }
log.WithField("path", path).Trace("Binary file is not recognized by file handlers. Chunking raw.") ctx.Logger().V(1).Info("binary file not handled, chunking raw", "path", path)
if err := reader.Reset(); err != nil { if err := reader.Reset(); err != nil {
return err return err
} }
pkg/sources/git/git_test.go
+6 -4
View File
@@ -210,7 +210,7 @@ func TestSource_Chunks_Integration(t *testing.T) {
tests := []struct { tests := []struct {
name string name string
init init init init
//verified // verified
repoURL string repoURL string
expectedChunkData map[string]*byteCompare expectedChunkData map[string]*byteCompare
scanOptions ScanOptions scanOptions ScanOptions
@@ -285,7 +285,7 @@ func TestSource_Chunks_Integration(t *testing.T) {
chunksCh := make(chan *sources.Chunk, 1) chunksCh := make(chan *sources.Chunk, 1)
go func() { go func() {
defer close(chunksCh) defer close(chunksCh)
repoPath, repo, err := CloneRepoUsingUnauthenticated(tt.repoURL) repoPath, repo, err := CloneRepoUsingUnauthenticated(ctx, tt.repoURL)
if err != nil { if err != nil {
panic(err) panic(err)
} }
@@ -476,7 +476,8 @@ func TestPrepareRepo(t *testing.T) {
} }
for _, tt := range tests { for _, tt := range tests {
repo, b, err := PrepareRepo(tt.uri) ctx := context.Background()
repo, b, err := PrepareRepo(ctx, tt.uri)
var repoLen bool var repoLen bool
if len(repo) > 0 { if len(repo) > 0 {
repoLen = true repoLen = true
@@ -491,8 +492,9 @@ func TestPrepareRepo(t *testing.T) {
func BenchmarkPrepareRepo(b *testing.B) { func BenchmarkPrepareRepo(b *testing.B) {
uri := "https://github.com/dustin-decker/secretsandstuff.git" uri := "https://github.com/dustin-decker/secretsandstuff.git"
ctx := context.Background()
for i := 0; i < b.N; i++ { for i := 0; i < b.N; i++ {
_, _, _ = PrepareRepo(uri) _, _, _ = PrepareRepo(ctx, uri)
} }
} }
pkg/sources/github/github.go
+3 -3
View File
@@ -570,7 +570,7 @@ func (s *Source) cloneRepo(ctx context.Context, repoURL string, installationClie
switch s.conn.GetCredential().(type) { switch s.conn.GetCredential().(type) {
case *sourcespb.GitHub_Unauthenticated: case *sourcespb.GitHub_Unauthenticated:
path, repo, err = git.CloneRepoUsingUnauthenticated(repoURL) path, repo, err = git.CloneRepoUsingUnauthenticated(ctx, repoURL)
if err != nil { if err != nil {
return "", nil, fmt.Errorf("error cloning repo %s: %w", repoURL, err) return "", nil, fmt.Errorf("error cloning repo %s: %w", repoURL, err)
} }
@@ -581,7 +581,7 @@ func (s *Source) cloneRepo(ctx context.Context, repoURL string, installationClie
return "", nil, fmt.Errorf("error getting token for repo %s: %w", repoURL, err) return "", nil, fmt.Errorf("error getting token for repo %s: %w", repoURL, err)
} }
path, repo, err = git.CloneRepoUsingToken(s.githubToken, repoURL, s.githubUser) path, repo, err = git.CloneRepoUsingToken(ctx, s.githubToken, repoURL, s.githubUser)
if err != nil { if err != nil {
return "", nil, fmt.Errorf("error cloning repo %s: %w", repoURL, err) return "", nil, fmt.Errorf("error cloning repo %s: %w", repoURL, err)
} }
@@ -594,7 +594,7 @@ func (s *Source) cloneRepo(ctx context.Context, repoURL string, installationClie
return "", nil, fmt.Errorf("error getting token for repo %s: %w", repoURL, err) return "", nil, fmt.Errorf("error getting token for repo %s: %w", repoURL, err)
} }
} }
path, repo, err = git.CloneRepoUsingToken(s.githubToken, repoURL, s.githubUser) path, repo, err = git.CloneRepoUsingToken(ctx, s.githubToken, repoURL, s.githubUser)
if err != nil { if err != nil {
return "", nil, fmt.Errorf("error cloning repo %s: %w", repoURL, err) return "", nil, fmt.Errorf("error cloning repo %s: %w", repoURL, err)
} }
pkg/sources/github/github_integration_test.go
+2 -2
View File
@@ -68,11 +68,11 @@ func TestSource_Token(t *testing.T) {
assert.NoError(t, err) assert.NoError(t, err)
// user provided // user provided
_, _, err = git.CloneRepoUsingToken(token, "https://github.com/trufflesecurity/trufflehog-updater.git", user) _, _, err = git.CloneRepoUsingToken(ctx, token, "https://github.com/trufflesecurity/trufflehog-updater.git", user)
assert.NoError(t, err) assert.NoError(t, err)
// no user provided // no user provided
_, _, err = git.CloneRepoUsingToken(token, "https://github.com/trufflesecurity/trufflehog-updater.git", "") _, _, err = git.CloneRepoUsingToken(ctx, token, "https://github.com/trufflesecurity/trufflehog-updater.git", "")
assert.Error(t, err) assert.Error(t, err)
_, _, err = s.cloneRepo(ctx, "https://github.com/trufflesecurity/trufflehog-updater.git", installationClient) _, _, err = s.cloneRepo(ctx, "https://github.com/trufflesecurity/trufflehog-updater.git", installationClient)
pkg/sources/gitlab/gitlab.go
+2 -2
View File
@@ -307,7 +307,7 @@ func (s *Source) scanRepos(ctx context.Context, chunksChan chan *sources.Chunk)
var repo *gogit.Repository var repo *gogit.Repository
var err error var err error
if s.authMethod == "UNAUTHENTICATED" { if s.authMethod == "UNAUTHENTICATED" {
path, repo, err = git.CloneRepoUsingUnauthenticated(repoURL) path, repo, err = git.CloneRepoUsingUnauthenticated(ctx, repoURL)
} else { } else {
// If a username is not provided we need to use a default one in order to clone a private repo. // If a username is not provided we need to use a default one in order to clone a private repo.
// Not setting "placeholder" as s.user on purpose in case any downstream services rely on a "" value for s.user. // Not setting "placeholder" as s.user on purpose in case any downstream services rely on a "" value for s.user.
@@ -315,7 +315,7 @@ func (s *Source) scanRepos(ctx context.Context, chunksChan chan *sources.Chunk)
if user == "" { if user == "" {
user = "placeholder" user = "placeholder"
} }
path, repo, err = git.CloneRepoUsingToken(s.token, repoURL, user) path, repo, err = git.CloneRepoUsingToken(ctx, s.token, repoURL, user)
} }
defer os.RemoveAll(path) defer os.RemoveAll(path)
if err != nil { if err != nil {