Add common sentry recover library and add into goroutines (#738)
* Add common sentry recover library and add into goroutines * fix nits
This commit is contained in:
@@ -47,7 +47,6 @@ require (
|
||||
github.com/xanzy/go-gitlab v0.73.1
|
||||
go.uber.org/zap v1.22.0
|
||||
golang.org/x/crypto v0.0.0-20211215165025-cf75a172585e
|
||||
golang.org/x/net v0.0.0-20220805013720-a33c5aa5df48
|
||||
golang.org/x/oauth2 v0.0.0-20220722155238-128564f6959c
|
||||
golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f
|
||||
google.golang.org/genproto v0.0.0-20220617124728-180714bec0ad
|
||||
@@ -114,6 +113,7 @@ require (
|
||||
go.uber.org/atomic v1.7.0 // indirect
|
||||
go.uber.org/multierr v1.6.0 // indirect
|
||||
golang.org/x/mod v0.5.0 // indirect
|
||||
golang.org/x/net v0.0.0-20220805013720-a33c5aa5df48 // indirect
|
||||
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab // indirect
|
||||
golang.org/x/text v0.3.7 // indirect
|
||||
golang.org/x/time v0.0.0-20220722155302-e5dcc9cfc0b9 // indirect
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"reflect"
|
||||
@@ -16,6 +15,7 @@ import (
|
||||
"golang.org/x/sync/semaphore"
|
||||
"gopkg.in/alecthomas/kingpin.v2"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/decoders"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/engine"
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"net/http"
|
||||
@@ -19,15 +18,15 @@ import (
|
||||
"github.com/sirupsen/logrus"
|
||||
"gopkg.in/alecthomas/kingpin.v2"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/updater"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/version"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/decoders"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/engine"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/output"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources/git"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/updater"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/version"
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -264,7 +263,7 @@ func run(state overseer.State) {
|
||||
}
|
||||
}
|
||||
// asynchronously wait for scanning to finish and cleanup
|
||||
go e.Finish()
|
||||
go e.Finish(ctx)
|
||||
|
||||
if !*jsonLegacy && !*jsonOut {
|
||||
fmt.Fprintf(os.Stderr, "🐷🔑🐷 TruffleHog. Unearth your secrets. 🐷🔑🐷\n\n")
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
package common
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"runtime/debug"
|
||||
"time"
|
||||
|
||||
"github.com/getsentry/sentry-go"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
)
|
||||
|
||||
// Recover handles panics and reports to Sentry before exiting.
|
||||
func Recover(ctx context.Context) {
|
||||
if err := recover(); err != nil {
|
||||
panicStack := string(debug.Stack())
|
||||
if eventID := sentry.CurrentHub().Recover(err); eventID != nil {
|
||||
ctx.Logger().Info("panic captured", "event_id", *eventID)
|
||||
}
|
||||
fmt.Fprint(os.Stderr, panicStack)
|
||||
if !sentry.Flush(time.Second * 5) {
|
||||
ctx.Logger().Info("sentry flush failed")
|
||||
}
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
@@ -2,7 +2,6 @@ package engine
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"runtime"
|
||||
"strings"
|
||||
"sync"
|
||||
@@ -11,6 +10,8 @@ import (
|
||||
|
||||
"github.com/sirupsen/logrus"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/decoders"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
|
||||
@@ -97,6 +98,7 @@ func Start(ctx context.Context, options ...EngineOption) *Engine {
|
||||
for i := 0; i < e.concurrency; i++ {
|
||||
e.workersWg.Add(1)
|
||||
go func() {
|
||||
defer common.Recover(ctx)
|
||||
defer e.workersWg.Done()
|
||||
e.detectorWorker(ctx)
|
||||
}()
|
||||
@@ -108,7 +110,8 @@ func Start(ctx context.Context, options ...EngineOption) *Engine {
|
||||
// Finish waits for running sources to complete and workers to finish scanning
|
||||
// chunks before closing their respective channels. Once Finish is called, no
|
||||
// more sources may be scanned by the engine.
|
||||
func (e *Engine) Finish() {
|
||||
func (e *Engine) Finish(ctx context.Context) {
|
||||
defer common.Recover(ctx)
|
||||
// wait for the sources to finish putting chunks onto the chunks channel
|
||||
e.sourcesWg.Wait()
|
||||
close(e.chunks)
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
package engine
|
||||
|
||||
import (
|
||||
"context"
|
||||
"runtime"
|
||||
|
||||
"github.com/go-errors/errors"
|
||||
@@ -9,6 +8,8 @@ import (
|
||||
"google.golang.org/protobuf/proto"
|
||||
"google.golang.org/protobuf/types/known/anypb"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources/filesystem"
|
||||
@@ -33,6 +34,7 @@ func (e *Engine) ScanFileSystem(ctx context.Context, c sources.Config) error {
|
||||
}
|
||||
e.sourcesWg.Add(1)
|
||||
go func() {
|
||||
defer common.Recover(ctx)
|
||||
defer e.sourcesWg.Done()
|
||||
err := fileSystemSource.Chunks(ctx, e.ChunksChan())
|
||||
if err != nil {
|
||||
|
||||
+3
-1
@@ -1,7 +1,6 @@
|
||||
package engine
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"runtime"
|
||||
|
||||
@@ -11,6 +10,8 @@ import (
|
||||
"github.com/go-git/go-git/v5/plumbing/object"
|
||||
"github.com/sirupsen/logrus"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
@@ -106,6 +107,7 @@ func (e *Engine) ScanGit(ctx context.Context, c sources.Config) error {
|
||||
|
||||
e.sourcesWg.Add(1)
|
||||
go func() {
|
||||
defer common.Recover(ctx)
|
||||
defer e.sourcesWg.Done()
|
||||
err := gitSource.ScanRepo(ctx, repo, c.RepoPath, scanOptions, e.ChunksChan())
|
||||
if err != nil {
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
package engine
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/decoders"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
@@ -67,7 +67,7 @@ func TestGitEngine(t *testing.T) {
|
||||
if err := e.ScanGit(ctx, cfg); err != nil {
|
||||
return
|
||||
}
|
||||
go e.Finish()
|
||||
go e.Finish(ctx)
|
||||
resultCount := 0
|
||||
for result := range e.ResultsChan() {
|
||||
switch meta := result.SourceMetadata.GetData().(type) {
|
||||
@@ -122,5 +122,5 @@ func BenchmarkGitEngine(b *testing.B) {
|
||||
return
|
||||
}
|
||||
}
|
||||
e.Finish()
|
||||
e.Finish(ctx)
|
||||
}
|
||||
|
||||
@@ -1,12 +1,12 @@
|
||||
package engine
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/sirupsen/logrus"
|
||||
"google.golang.org/protobuf/proto"
|
||||
"google.golang.org/protobuf/types/known/anypb"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources/github"
|
||||
@@ -43,6 +43,7 @@ func (e *Engine) ScanGitHub(ctx context.Context, c sources.Config) error {
|
||||
|
||||
e.sourcesWg.Add(1)
|
||||
go func() {
|
||||
defer common.Recover(ctx)
|
||||
defer e.sourcesWg.Done()
|
||||
err := source.Chunks(ctx, e.ChunksChan())
|
||||
if err != nil {
|
||||
|
||||
@@ -6,10 +6,11 @@ import (
|
||||
|
||||
"github.com/go-errors/errors"
|
||||
"github.com/sirupsen/logrus"
|
||||
"golang.org/x/net/context"
|
||||
"google.golang.org/protobuf/proto"
|
||||
"google.golang.org/protobuf/types/known/anypb"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources/gitlab"
|
||||
@@ -51,6 +52,7 @@ func (e *Engine) ScanGitLab(ctx context.Context, c sources.Config) error {
|
||||
|
||||
e.sourcesWg.Add(1)
|
||||
go func() {
|
||||
defer common.Recover(ctx)
|
||||
defer e.sourcesWg.Done()
|
||||
err := gitlabSource.Chunks(ctx, e.ChunksChan())
|
||||
if err != nil {
|
||||
|
||||
+3
-1
@@ -1,7 +1,6 @@
|
||||
package engine
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"runtime"
|
||||
|
||||
@@ -10,6 +9,8 @@ import (
|
||||
"google.golang.org/protobuf/proto"
|
||||
"google.golang.org/protobuf/types/known/anypb"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/credentialspb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
@@ -53,6 +54,7 @@ func (e *Engine) ScanS3(ctx context.Context, c sources.Config) error {
|
||||
|
||||
e.sourcesWg.Add(1)
|
||||
go func() {
|
||||
defer common.Recover(ctx)
|
||||
defer e.sourcesWg.Done()
|
||||
err := s3Source.Chunks(ctx, e.ChunksChan())
|
||||
if err != nil {
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
package engine
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
|
||||
"github.com/go-errors/errors"
|
||||
@@ -9,6 +8,8 @@ import (
|
||||
"google.golang.org/protobuf/proto"
|
||||
"google.golang.org/protobuf/types/known/anypb"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources/syslog"
|
||||
@@ -51,6 +52,7 @@ func (e *Engine) ScanSyslog(ctx context.Context, c sources.Config) error {
|
||||
|
||||
e.sourcesWg.Add(1)
|
||||
go func() {
|
||||
defer common.Recover(ctx)
|
||||
defer e.sourcesWg.Done()
|
||||
err := source.Chunks(ctx, e.ChunksChan())
|
||||
if err != nil {
|
||||
|
||||
@@ -11,6 +11,9 @@ import (
|
||||
"time"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
)
|
||||
|
||||
// DateFormat is the standard date format for git.
|
||||
@@ -34,7 +37,7 @@ type Diff struct {
|
||||
}
|
||||
|
||||
// RepoPath parses the output of the `git log` command for the `source` path.
|
||||
func RepoPath(source string, head string) (chan Commit, error) {
|
||||
func RepoPath(ctx context.Context, source string, head string) (chan Commit, error) {
|
||||
commitChan := make(chan Commit)
|
||||
|
||||
args := []string{"-C", source, "log", "-p", "-U0", "--full-history", "--diff-filter=AM", "--date=format:%a %b %d %H:%M:%S %Y %z"}
|
||||
@@ -77,6 +80,7 @@ func RepoPath(source string, head string) (chan Commit, error) {
|
||||
}()
|
||||
|
||||
go func() {
|
||||
defer common.Recover(ctx)
|
||||
for {
|
||||
line, err := outReader.ReadBytes([]byte("\n")[0])
|
||||
if err != nil && len(line) == 0 {
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
package filesystem
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
@@ -15,6 +14,7 @@ import (
|
||||
"google.golang.org/protobuf/types/known/anypb"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/handlers"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
package filesystem
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -9,6 +8,7 @@ import (
|
||||
log "github.com/sirupsen/logrus"
|
||||
"google.golang.org/protobuf/types/known/anypb"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
|
||||
@@ -2,7 +2,6 @@ package git
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
@@ -27,6 +26,7 @@ import (
|
||||
"google.golang.org/protobuf/types/known/anypb"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/gitparse"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/handlers"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
|
||||
@@ -279,7 +279,7 @@ func GitCmdCheck() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Git) ScanCommits(repo *git.Repository, path string, scanOptions *ScanOptions, chunksChan chan *sources.Chunk) error {
|
||||
func (s *Git) ScanCommits(ctx context.Context, repo *git.Repository, path string, scanOptions *ScanOptions, chunksChan chan *sources.Chunk) error {
|
||||
if err := GitCmdCheck(); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -287,7 +287,7 @@ func (s *Git) ScanCommits(repo *git.Repository, path string, scanOptions *ScanOp
|
||||
zerolog.SetGlobalLevel(zerolog.Disabled)
|
||||
}
|
||||
|
||||
commitChan, err := gitparse.RepoPath(path, scanOptions.HeadHash)
|
||||
commitChan, err := gitparse.RepoPath(ctx, path, scanOptions.HeadHash)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -412,9 +412,9 @@ func (s *Git) ScanUnstaged(repo *git.Repository, scanOptions *ScanOptions, chunk
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Git) ScanRepo(_ context.Context, repo *git.Repository, repoPath string, scanOptions *ScanOptions, chunksChan chan *sources.Chunk) error {
|
||||
func (s *Git) ScanRepo(ctx context.Context, repo *git.Repository, repoPath string, scanOptions *ScanOptions, chunksChan chan *sources.Chunk) error {
|
||||
start := time.Now().UnixNano()
|
||||
if err := s.ScanCommits(repo, repoPath, scanOptions, chunksChan); err != nil {
|
||||
if err := s.ScanCommits(ctx, repo, repoPath, scanOptions, chunksChan); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := s.ScanUnstaged(repo, scanOptions, chunksChan); err != nil {
|
||||
@@ -591,9 +591,9 @@ func PrepareRepo(uriString string) (string, bool, error) {
|
||||
remotePath := uri.String()
|
||||
remote = true
|
||||
path, _, err = CloneRepoUsingSSH(remotePath)
|
||||
if err != nil {
|
||||
return path, remote, fmt.Errorf("failed to clone unauthenticated Git repo (%s): %s", remotePath, err)
|
||||
}
|
||||
if err != nil {
|
||||
return path, remote, fmt.Errorf("failed to clone unauthenticated Git repo (%s): %s", remotePath, err)
|
||||
}
|
||||
default:
|
||||
return "", remote, fmt.Errorf("unsupported Git URI: %s", uriString)
|
||||
}
|
||||
|
||||
@@ -2,19 +2,20 @@ package git
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/kylelemons/godebug/pretty"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"google.golang.org/protobuf/types/known/anypb"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/credentialspb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
"google.golang.org/protobuf/types/known/anypb"
|
||||
)
|
||||
|
||||
func TestSource_Scan(t *testing.T) {
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
package github
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
@@ -25,6 +24,7 @@ import (
|
||||
"google.golang.org/protobuf/types/known/anypb"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/giturl"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/credentialspb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
|
||||
|
||||
@@ -4,7 +4,6 @@
|
||||
package github
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
"os"
|
||||
@@ -19,6 +18,7 @@ import (
|
||||
"google.golang.org/protobuf/types/known/anypb"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/credentialspb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
|
||||
|
||||
@@ -2,7 +2,6 @@ package github
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"crypto/rsa"
|
||||
"crypto/x509"
|
||||
@@ -21,6 +20,7 @@ import (
|
||||
"google.golang.org/protobuf/types/known/anypb"
|
||||
"gopkg.in/h2non/gock.v1"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/credentialspb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
|
||||
)
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
package gitlab
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"os"
|
||||
@@ -11,6 +10,7 @@ import (
|
||||
"sync"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/giturl"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
package gitlab
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"reflect"
|
||||
@@ -12,6 +11,7 @@ import (
|
||||
"google.golang.org/protobuf/types/known/anypb"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/credentialspb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
package s3
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
@@ -15,15 +14,17 @@ import (
|
||||
diskbufferreader "github.com/bill-rich/disk-buffer-reader"
|
||||
"github.com/go-errors/errors"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"golang.org/x/sync/semaphore"
|
||||
"google.golang.org/protobuf/proto"
|
||||
"google.golang.org/protobuf/types/known/anypb"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/handlers"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sanitizer"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
"golang.org/x/sync/semaphore"
|
||||
"google.golang.org/protobuf/proto"
|
||||
"google.golang.org/protobuf/types/known/anypb"
|
||||
)
|
||||
|
||||
type Source struct {
|
||||
@@ -193,6 +194,7 @@ func (s *Source) pageChunker(ctx context.Context, client *s3.S3, chunksChan chan
|
||||
}
|
||||
wg.Add(1)
|
||||
go func(ctx context.Context, wg *sync.WaitGroup, sem *semaphore.Weighted, obj *s3.Object) {
|
||||
defer common.Recover(ctx)
|
||||
defer sem.Release(1)
|
||||
defer wg.Done()
|
||||
//defer log.Debugf("DONE - %s", *obj.Key)
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
package s3
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
"os"
|
||||
@@ -14,6 +13,7 @@ import (
|
||||
"google.golang.org/protobuf/types/known/anypb"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/credentialspb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
|
||||
@@ -1,12 +1,12 @@
|
||||
package sources
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
|
||||
"google.golang.org/protobuf/types/known/anypb"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
|
||||
)
|
||||
|
||||
@@ -1,10 +1,8 @@
|
||||
package syslog
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"fmt"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"io"
|
||||
"net"
|
||||
"runtime"
|
||||
@@ -19,6 +17,8 @@ import (
|
||||
"google.golang.org/protobuf/proto"
|
||||
"google.golang.org/protobuf/types/known/anypb"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
@@ -210,6 +210,7 @@ func (s *Source) parseSyslogMetadata(input []byte, remote string) (*source_metad
|
||||
}
|
||||
|
||||
func (s *Source) monitorConnection(ctx context.Context, conn net.Conn, chunksChan chan *sources.Chunk) {
|
||||
defer common.Recover(ctx)
|
||||
for {
|
||||
if common.IsDone(ctx) {
|
||||
return
|
||||
|
||||
Reference in New Issue
Block a user