diff --git a/main.go b/main.go index 2fc0fbef7..4b68ddc69 100644 --- a/main.go +++ b/main.go @@ -51,6 +51,7 @@ var ( printAvgDetectorTime = cli.Flag("print-avg-detector-time", "Print the average time spent on each detector.").Bool() noUpdate = cli.Flag("no-update", "Don't check for updates.").Bool() fail = cli.Flag("fail", "Exit with code 183 if results are found.").Bool() + verifiers = cli.Flag("verifier", "Set custom verification endpoints.").StringMap() archiveMaxSize = cli.Flag("archive-max-size", "Maximum size of archive to scan. (Byte units eg. 512B, 2KB, 4MB)").Bytes() archiveMaxDepth = cli.Flag("archive-max-depth", "Maximum depth of archive to scan.").Int() archiveTimeout = cli.Flag("archive-timeout", "Maximum time to spend extracting an archive.").Duration() @@ -219,6 +220,8 @@ func run(state overseer.State) { } } + urls := splitVerifierURLs(*verifiers) + if *archiveMaxSize != 0 { handlers.SetArchiveMaxSize(int(*archiveMaxSize)) } @@ -282,6 +285,7 @@ func run(state overseer.State) { engine.WithConcurrency(*concurrency), engine.WithDecoders(decoders.DefaultDecoders()...), engine.WithDetectors(!*noVerification, engine.DefaultDetectors()...), + engine.WithDetectors(!*noVerification, engine.CustomDetectors(ctx, urls)...), engine.WithDetectors(!*noVerification, conf.Detectors...), engine.WithFilterDetectors(includeFilter), engine.WithFilterDetectors(excludeFilter), @@ -492,6 +496,19 @@ func printAverageDetectorTime(e *engine.Engine) { } } +func splitVerifierURLs(verifierURLs map[string]string) map[string][]string { + verifiers := make(map[string][]string, len(verifierURLs)) + for k, v := range verifierURLs { + key := strings.ToLower(k) + sliceOfValues := strings.Split(v, ",") + for i, s := range sliceOfValues { + sliceOfValues[i] = strings.TrimSpace(s) + } + verifiers[key] = sliceOfValues + } + return verifiers +} + // logFatalFunc returns a log.Fatal style function. Calling the returned // function will terminate the program without cleanup. func logFatalFunc(logger logr.Logger) func(error, string, ...any) { diff --git a/pkg/detectors/github/github.go b/pkg/detectors/github/github.go index 54abf074f..7b08ab085 100644 --- a/pkg/detectors/github/github.go +++ b/pkg/detectors/github/github.go @@ -13,7 +13,34 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/common" ) -type Scanner struct{} +const defaultURL = "https://api.github.com" + +type Scanner struct { + verifierURLs []string +} + +// New creates a new Scanner with the given options. +func New(opts ...func(*Scanner)) *Scanner { + scanner := &Scanner{ + verifierURLs: make([]string, 0), + } + for _, opt := range opts { + opt(scanner) + } + + return scanner +} + +// WithVerifierURLs adds the given URLs to the list of URLs to check for +// verification of secrets. +func WithVerifierURLs(urls []string, includeDefault bool) func(*Scanner) { + return func(s *Scanner) { + if includeDefault { + urls = append(urls, defaultURL) + } + s.verifierURLs = append(s.verifierURLs, urls...) + } +} // Ensure the Scanner satisfies the interfaces at compile time. var _ detectors.Detector = (*Scanner)(nil) @@ -29,7 +56,7 @@ var ( // https://github.blog/changelog/2022-10-18-introducing-fine-grained-personal-access-tokens/ keyPat = regexp.MustCompile(`\b((?:ghp|gho|ghu|ghs|ghr|github_pat)_[a-zA-Z0-9_]{36,255})\b`) - //TODO: Oauth2 client_id and client_secret + // TODO: Oauth2 client_id and client_secret // https://developer.github.com/v3/#oauth2-keysecret ) @@ -71,27 +98,29 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result if verify { client := common.SaneHttpClient() // https://developer.github.com/v3/users/#get-the-authenticated-user - req, err := http.NewRequestWithContext(ctx, "GET", "https://api.github.com/user", nil) - if err != nil { - continue - } - req.Header.Add("Content-Type", "application/json; charset=utf-8") - req.Header.Add("Authorization", fmt.Sprintf("token %s", token)) - res, err := client.Do(req) - if err == nil { - if res.StatusCode >= 200 && res.StatusCode < 300 { - var userResponse userRes - err = json.NewDecoder(res.Body).Decode(&userResponse) - res.Body.Close() - if err == nil { - s1.Verified = true - s1.ExtraData = map[string]string{ - "username": userResponse.Login, - "url": userResponse.UserURL, - "account_type": userResponse.Type, - "site_admin": fmt.Sprintf("%t", userResponse.SiteAdmin), - "name": userResponse.Name, - "company": userResponse.Company, + for _, url := range s.verifierURLs { + req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("%s/user", url), nil) + if err != nil { + continue + } + req.Header.Add("Content-Type", "application/json; charset=utf-8") + req.Header.Add("Authorization", fmt.Sprintf("token %s", token)) + res, err := client.Do(req) + if err == nil { + if res.StatusCode >= 200 && res.StatusCode < 300 { + var userResponse userRes + err = json.NewDecoder(res.Body).Decode(&userResponse) + res.Body.Close() + if err == nil { + s1.Verified = true + s1.ExtraData = map[string]string{ + "username": userResponse.Login, + "url": userResponse.UserURL, + "account_type": userResponse.Type, + "site_admin": fmt.Sprintf("%t", userResponse.SiteAdmin), + "name": userResponse.Name, + "company": userResponse.Company, + } } } } diff --git a/pkg/detectors/github_old/github_old.go b/pkg/detectors/github_old/github_old.go index 2d468d931..f948aa548 100644 --- a/pkg/detectors/github_old/github_old.go +++ b/pkg/detectors/github_old/github_old.go @@ -12,7 +12,34 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +const defaultURL = "https://api.github.com" + +type Scanner struct { + verifierURLs []string +} + +// New creates a new Scanner with the given options. +func New(opts ...func(*Scanner)) *Scanner { + scanner := &Scanner{ + verifierURLs: make([]string, 0), + } + for _, opt := range opts { + opt(scanner) + } + + return scanner +} + +// WithVerifierURLs adds the given URLs to the list of URLs to check for +// verification of secrets. +func WithVerifierURLs(urls []string, includeDefault bool) func(*Scanner) { + return func(s *Scanner) { + if includeDefault { + urls = append(urls, defaultURL) + } + s.verifierURLs = append(s.verifierURLs, urls...) + } +} // Ensure the Scanner satisfies the interfaces at compile time. var _ detectors.Detector = (*Scanner)(nil) @@ -63,7 +90,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result continue } - s := detectors.Result{ + s1 := detectors.Result{ DetectorType: detectorspb.DetectorType_Github, Raw: []byte(token), } @@ -71,30 +98,32 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result if verify { client := common.SaneHttpClient() // https://developer.github.com/v3/users/#get-the-authenticated-user - req, err := http.NewRequestWithContext(ctx, "GET", "https://api.github.com/user", nil) - if err != nil { - continue - } - req.Header.Add("Content-Type", "application/json; charset=utf-8") - req.Header.Add("Authorization", fmt.Sprintf("token %s", token)) - res, err := client.Do(req) - if err == nil { - if res.StatusCode >= 200 && res.StatusCode < 300 { - var userResponse userRes - err = json.NewDecoder(res.Body).Decode(&userResponse) - res.Body.Close() - if err == nil { - s.Verified = true + for _, url := range s.verifierURLs { + req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("%s/user", url), nil) + if err != nil { + continue + } + req.Header.Add("Content-Type", "application/json; charset=utf-8") + req.Header.Add("Authorization", fmt.Sprintf("token %s", token)) + res, err := client.Do(req) + if err == nil { + if res.StatusCode >= 200 && res.StatusCode < 300 { + var userResponse userRes + err = json.NewDecoder(res.Body).Decode(&userResponse) + res.Body.Close() + if err == nil { + s1.Verified = true + } } } } } - if !s.Verified && detectors.IsKnownFalsePositive(token, detectors.DefaultFalsePositives, true) { + if !s1.Verified && detectors.IsKnownFalsePositive(token, detectors.DefaultFalsePositives, true) { continue } - results = append(results, s) + results = append(results, s1) } return results, nil diff --git a/pkg/detectors/gitlab/gitlab.go b/pkg/detectors/gitlab/gitlab.go index 4ed804ec1..bc7990d7a 100644 --- a/pkg/detectors/gitlab/gitlab.go +++ b/pkg/detectors/gitlab/gitlab.go @@ -12,7 +12,34 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +const defaultURL = "https://gitlab.com" + +type Scanner struct { + verifierURLs []string +} + +// New creates a new Scanner with the given options. +func New(opts ...func(*Scanner)) *Scanner { + scanner := &Scanner{ + verifierURLs: make([]string, 0), + } + for _, opt := range opts { + opt(scanner) + } + + return scanner +} + +// WithVerifierURLs adds the given URLs to the list of URLs to check for +// verification of secrets. +func WithVerifierURLs(urls []string, includeDefault bool) func(*Scanner) { + return func(s *Scanner) { + if includeDefault { + urls = append(urls, defaultURL) + } + s.verifierURLs = append(s.verifierURLs, urls...) + } +} // Ensure the Scanner satisfies the interfaces at compile time. var _ detectors.Detector = (*Scanner)(nil) @@ -56,24 +83,25 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result // they all grant access to different parts of the API. I couldn't find an endpoint that every // one of these scopes has access to, so we just check an example endpoint for each scope. If any // of them contain data, we know we have a valid key, but if they all fail, we don't - baseURL := "https://gitlab.com/api/v4" client := common.SaneHttpClient() - // test `read_user` scope - req, err := http.NewRequestWithContext(ctx, "GET", baseURL+"/user", nil) - if err != nil { - continue - } - req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) - res, err := client.Do(req) - if err == nil { - res.Body.Close() // The request body is unused. + for _, baseURL := range s.verifierURLs { + // test `read_user` scope + req, err := http.NewRequestWithContext(ctx, "GET", baseURL+"/api/v4/user", nil) + if err != nil { + continue + } + req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) + res, err := client.Do(req) + if err == nil { + res.Body.Close() // The request body is unused. - // 200 means good key and has `read_user` scope - // 403 means good key but not the right scope - // 401 is bad key - if res.StatusCode == http.StatusOK || res.StatusCode == http.StatusForbidden { - secret.Verified = true + // 200 means good key and has `read_user` scope + // 403 means good key but not the right scope + // 401 is bad key + if res.StatusCode == http.StatusOK || res.StatusCode == http.StatusForbidden { + secret.Verified = true + } } } } diff --git a/pkg/detectors/gitlabv2/gitlab.go b/pkg/detectors/gitlabv2/gitlab.go index 01ea72228..ffc0b7bf8 100644 --- a/pkg/detectors/gitlabv2/gitlab.go +++ b/pkg/detectors/gitlabv2/gitlab.go @@ -11,7 +11,34 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +const defaultURL = "https://gitlab.com" + +type Scanner struct { + verifierURLs []string +} + +// New creates a new Scanner with the given options. +func New(opts ...func(*Scanner)) *Scanner { + scanner := &Scanner{ + verifierURLs: make([]string, 0), + } + for _, opt := range opts { + opt(scanner) + } + + return scanner +} + +// WithVerifierURLs adds the given URLs to the list of URLs to check for +// verification of secrets. +func WithVerifierURLs(urls []string, includeDefault bool) func(*Scanner) { + return func(s *Scanner) { + if includeDefault { + urls = append(urls, defaultURL) + } + s.verifierURLs = append(s.verifierURLs, urls...) + } +} // Ensure the Scanner satisfies the interfaces at compile time. var _ detectors.Detector = (*Scanner)(nil) @@ -50,27 +77,27 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result // they all grant access to different parts of the API. I couldn't find an endpoint that every // one of these scopes has access to, so we just check an example endpoint for each scope. If any // of them contain data, we know we have a valid key, but if they all fail, we don't - baseURL := "https://gitlab.com/api/v4" - client := common.SaneHttpClient() + for _, baseURL := range s.verifierURLs { + // test `read_user` scope + req, err := http.NewRequestWithContext(ctx, "GET", baseURL+"/api/v4/user", nil) + if err != nil { + continue + } + req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", match[1])) + res, err := client.Do(req) + if err == nil { + res.Body.Close() // The request body is unused. - // test `read_user` scope - req, err := http.NewRequestWithContext(ctx, "GET", baseURL+"/user", nil) - if err != nil { - continue - } - req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", match[1])) - res, err := client.Do(req) - if err == nil { - res.Body.Close() // The request body is unused. - - // 200 means good key and has `read_user` scope - // 403 means good key but not the right scope - // 401 is bad key - if res.StatusCode == http.StatusOK || res.StatusCode == http.StatusForbidden { - secret.Verified = true + // 200 means good key and has `read_user` scope + // 403 means good key but not the right scope + // 401 is bad key + if res.StatusCode == http.StatusOK || res.StatusCode == http.StatusForbidden { + secret.Verified = true + } } } + } if !secret.Verified && detectors.IsKnownFalsePositive(string(secret.Raw), detectors.DefaultFalsePositives, true) { diff --git a/pkg/engine/defaults.go b/pkg/engine/defaults.go index 487a99276..6bb3acc2f 100644 --- a/pkg/engine/defaults.go +++ b/pkg/engine/defaults.go @@ -1,6 +1,7 @@ package engine import ( + "github.com/trufflesecurity/trufflehog/v3/pkg/context" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/abbysale" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/abuseipdb" @@ -731,6 +732,7 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/zipcodebase" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/zonkafeedback" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/zulipchat" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/etherscan" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/infura" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/alchemy" @@ -740,6 +742,65 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/coinmarketcap" ) +// CustomDetectors returns a list of detectors that are enabled by default, but +// can be overridden by the user. +func CustomDetectors(ctx context.Context, urls map[string][]string) []detectors.Detector { + defaultDetectors := DefaultDetectors() + if len(urls) == 0 { + return defaultDetectors + } + + for i, detector := range defaultDetectors { + + switch detector.Type() { + case detectorspb.DetectorType_Github: + githubUrls, ok := urls["github"] + if !ok { + ctx.Logger().V(2).Info("no GitHub urls to ignore") + continue + } + ctx.Logger().V(2).Info("ignoring GitHub urls: %v", githubUrls) + + versioner, ok := detector.(detectors.Versioner) + if !ok { + ctx.Logger().V(2).Info("Failed to get version for GitHub detector") + } + switch versioner.Version() { + case 1: + defaultDetectors[i] = github_old.New(github_old.WithVerifierURLs(githubUrls, true)) + default: + defaultDetectors[i] = github.New(github.WithVerifierURLs(githubUrls, true)) + } + + case detectorspb.DetectorType_Gitlab: + gitlabUrls, ok := urls["gitlab"] + if !ok { + ctx.Logger().V(2).Info("no GitLab urls to ignore") + continue + } + ctx.Logger().V(2).Info("ignoring GitLab urls: %v", gitlabUrls) + + versioner, ok := detector.(detectors.Versioner) + if !ok { + ctx.Logger().V(2).Info("Failed to get version for Gitlab detector") + } + switch versioner.Version() { + case 1: + defaultDetectors[i] = gitlab.New(gitlab.WithVerifierURLs(gitlabUrls, true)) + default: + defaultDetectors[i] = gitlabv2.New(gitlabv2.WithVerifierURLs(gitlabUrls, true)) + } + + case detectorspb.DetectorType_JiraToken: + // TODO(ahrav): Double check that we need to do this. + default: + ctx.Logger().V(5).Info("ignoring custom detector", "type", detector.Type()) + continue + } + } + return defaultDetectors +} + func DefaultDetectors() []detectors.Detector { return []detectors.Detector{ &heroku.Scanner{}, @@ -1505,4 +1566,5 @@ func DefaultDetectors() []detectors.Detector { bscscan.Scanner{}, coinmarketcap.Scanner{}, } + } diff --git a/pkg/engine/engine.go b/pkg/engine/engine.go index 2d4eaf9d9..65ff22617 100644 --- a/pkg/engine/engine.go +++ b/pkg/engine/engine.go @@ -156,7 +156,7 @@ func Start(ctx context.Context, options ...EngineOption) *Engine { "verification_disabled", len(e.detectors[false]), ) - // start the workers + // Start the workers. for i := 0; i < e.concurrency; i++ { e.workersWg.Add(1) go func() {