From fe5624c70923355128868cffd647b6e2cfe11443 Mon Sep 17 00:00:00 2001 From: Dustin Decker Date: Thu, 15 Aug 2024 11:34:28 -0700 Subject: [PATCH] Improve domain / url handling in detectors (#3221) * Strip path and params and use new client * update clients * additional client updates * revert client --- pkg/detectors/aha/aha.go | 3 +- pkg/detectors/apiflash/apiflash.go | 6 +- pkg/detectors/artifactory/artifactory.go | 3 +- .../auth0managementapitoken.go | 3 +- pkg/detectors/auth0oauth/auth0oauth.go | 8 +- pkg/detectors/azurebatch/azurebatch.go | 3 +- .../azurefunctionkey/azurefunctionkey.go | 3 +- .../azuresearchquerykey.go | 3 +- pkg/detectors/azurestorage/azurestorage.go | 2 +- pkg/detectors/caspio/caspio.go | 6 +- .../databrickstoken/databrickstoken.go | 6 +- pkg/detectors/deputy/deputy.go | 6 +- pkg/detectors/detectors.go | 10 ++ .../discordwebhook/discordwebhook.go | 3 +- pkg/detectors/fibery/fibery.go | 6 +- pkg/detectors/freshbooks/freshbooks.go | 8 +- .../grafanaserviceaccount.go | 6 +- pkg/detectors/http.go | 154 ++++++++++++++++++ pkg/detectors/http_test.go | 94 +++++++++++ pkg/detectors/invoiceocean/invoiceocean.go | 6 +- pkg/detectors/jiratoken/v1/jiratoken.go | 3 +- pkg/detectors/jiratoken/v2/jiratoken_v2.go | 3 +- pkg/detectors/kanban/kanban.go | 6 +- pkg/detectors/kanbantool/kanbantool.go | 6 +- pkg/detectors/liveagent/liveagent.go | 6 +- pkg/detectors/loggly/loggly.go | 6 +- pkg/detectors/metabase/metabase.go | 15 +- .../microsoftteamswebhook.go | 4 +- pkg/detectors/mite/mite.go | 6 +- pkg/detectors/okta/okta.go | 3 +- pkg/detectors/openvpn/openvpn.go | 3 +- .../planviewleankit/planviewleankit.go | 6 +- pkg/detectors/portainer/portainer.go | 17 +- .../portainertoken/portainertoken.go | 15 +- pkg/detectors/repairshopr/repairshopr.go | 6 +- pkg/detectors/ringcentral/ringcentral.go | 4 +- pkg/detectors/salesforce/salesforce.go | 5 +- pkg/detectors/salesmate/salesmate.go | 6 +- pkg/detectors/shopify/shopify.go | 3 +- pkg/detectors/signalwire/signalwire.go | 6 +- pkg/detectors/signalwire/signalwire_test.go | 2 +- pkg/detectors/slackwebhook/slackwebhook.go | 3 +- pkg/detectors/sugester/sugester.go | 6 +- pkg/detectors/thinkific/thinkific.go | 6 +- pkg/detectors/tineswebhook/tineswebhook.go | 6 +- pkg/detectors/uri/uri.go | 4 +- pkg/detectors/zapierwebhook/zapierwebhook.go | 6 +- pkg/detectors/zendeskapi/zendeskapi.go | 6 +- pkg/detectors/zulipchat/zulipchat.go | 2 +- 49 files changed, 385 insertions(+), 124 deletions(-) create mode 100644 pkg/detectors/http.go create mode 100644 pkg/detectors/http_test.go diff --git a/pkg/detectors/aha/aha.go b/pkg/detectors/aha/aha.go index 6788794aa..07d99671e 100644 --- a/pkg/detectors/aha/aha.go +++ b/pkg/detectors/aha/aha.go @@ -8,7 +8,6 @@ import ( regexp "github.com/wasilibs/go-re2" - "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) @@ -22,7 +21,7 @@ var ( // Ensure the Scanner satisfies the interface at compile time. _ detectors.Detector = (*Scanner)(nil) - defaultClient = common.SaneHttpClient() + defaultClient = detectors.DetectorHttpClientWithNoLocalAddresses // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"aha"}) + `\b([0-9a-f]{64})\b`) diff --git a/pkg/detectors/apiflash/apiflash.go b/pkg/detectors/apiflash/apiflash.go index 50dc97ccb..d335267c5 100644 --- a/pkg/detectors/apiflash/apiflash.go +++ b/pkg/detectors/apiflash/apiflash.go @@ -3,11 +3,11 @@ package apiflash import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" - "github.com/trufflesecurity/trufflehog/v3/pkg/common" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) @@ -20,7 +20,7 @@ type Scanner struct { var _ detectors.Detector = (*Scanner)(nil) var ( - client = common.SaneHttpClient() + client = detectors.DetectorHttpClientWithNoLocalAddresses // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"apiflash"}) + `\b([a-z0-9]{32})\b`) diff --git a/pkg/detectors/artifactory/artifactory.go b/pkg/detectors/artifactory/artifactory.go index afe9dd123..98841f984 100644 --- a/pkg/detectors/artifactory/artifactory.go +++ b/pkg/detectors/artifactory/artifactory.go @@ -8,7 +8,6 @@ import ( regexp "github.com/wasilibs/go-re2" - "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) @@ -22,7 +21,7 @@ var ( // Ensure the Scanner satisfies the interface at compile time. _ detectors.Detector = (*Scanner)(nil) - defaultClient = common.SaneHttpClient() + defaultClient = detectors.DetectorHttpClientWithNoLocalAddresses // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. keyPat = regexp.MustCompile(`\b([a-zA-Z0-9]{73}|\b[a-zA-Z0-9]{64})`) diff --git a/pkg/detectors/auth0managementapitoken/auth0managementapitoken.go b/pkg/detectors/auth0managementapitoken/auth0managementapitoken.go index 82b78697f..5d9a513ce 100644 --- a/pkg/detectors/auth0managementapitoken/auth0managementapitoken.go +++ b/pkg/detectors/auth0managementapitoken/auth0managementapitoken.go @@ -8,7 +8,6 @@ import ( regexp "github.com/wasilibs/go-re2" - "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) @@ -22,7 +21,7 @@ var _ detectors.Detector = (*Scanner)(nil) var _ detectors.MaxSecretSizeProvider = (*Scanner)(nil) var ( - client = common.SaneHttpClient() + client = detectors.DetectorHttpClientWithLocalAddresses // long jwt token but note this is default 8640000 seconds = 24 hours but could be set to maximum 2592000 seconds = 720 hours = 30 days // at https://manage.auth0.com/dashboard/us/dev-63memjo3/apis/management/explorer diff --git a/pkg/detectors/auth0oauth/auth0oauth.go b/pkg/detectors/auth0oauth/auth0oauth.go index c35e14e4e..93bf956b7 100644 --- a/pkg/detectors/auth0oauth/auth0oauth.go +++ b/pkg/detectors/auth0oauth/auth0oauth.go @@ -2,13 +2,13 @@ package auth0oauth import ( "context" - regexp "github.com/wasilibs/go-re2" "io" "net/http" "net/url" "strings" - "github.com/trufflesecurity/trufflehog/v3/pkg/common" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) @@ -21,7 +21,7 @@ type Scanner struct{ var _ detectors.Detector = (*Scanner)(nil) var ( - client = common.SaneHttpClient() + client = detectors.DetectorHttpClientWithLocalAddresses clientIdPat = regexp.MustCompile(detectors.PrefixRegex([]string{"auth0"}) + `\b([a-zA-Z0-9_-]{32,60})\b`) clientSecretPat = regexp.MustCompile(`\b([a-zA-Z0-9_-]{64,})\b`) @@ -102,7 +102,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result if !strings.Contains(body, "access_denied") { s1.Verified = true - } + } } } diff --git a/pkg/detectors/azurebatch/azurebatch.go b/pkg/detectors/azurebatch/azurebatch.go index da5fd9333..4bad3e0eb 100644 --- a/pkg/detectors/azurebatch/azurebatch.go +++ b/pkg/detectors/azurebatch/azurebatch.go @@ -12,7 +12,6 @@ import ( regexp "github.com/wasilibs/go-re2" - "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) @@ -27,7 +26,7 @@ var _ detectors.Detector = (*Scanner)(nil) var _ detectors.CustomFalsePositiveChecker = (*Scanner)(nil) var ( - defaultClient = common.SaneHttpClient() + defaultClient = detectors.DetectorHttpClientWithNoLocalAddresses // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. urlPat = regexp.MustCompile(`https://(.{1,50})\.(.{1,50})\.batch\.azure\.com`) secretPat = regexp.MustCompile(`[A-Za-z0-9+/=]{88}`) diff --git a/pkg/detectors/azurefunctionkey/azurefunctionkey.go b/pkg/detectors/azurefunctionkey/azurefunctionkey.go index 7af938ea2..3aca16851 100644 --- a/pkg/detectors/azurefunctionkey/azurefunctionkey.go +++ b/pkg/detectors/azurefunctionkey/azurefunctionkey.go @@ -8,7 +8,6 @@ import ( regexp "github.com/wasilibs/go-re2" - "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) @@ -22,7 +21,7 @@ type Scanner struct { var _ detectors.Detector = (*Scanner)(nil) var ( - defaultClient = common.SaneHttpClient() + defaultClient = detectors.DetectorHttpClientWithNoLocalAddresses // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"azure"}) + `\b([a-zA-Z0-9_-]{20,56})\b={0,2}`) azureUrlPat = regexp.MustCompile(`\bhttps:\/\/([a-zA-Z0-9-]{2,30})\.azurewebsites\.net\/api\/([a-zA-Z0-9-]{2,30})\b`) diff --git a/pkg/detectors/azuresearchquerykey/azuresearchquerykey.go b/pkg/detectors/azuresearchquerykey/azuresearchquerykey.go index 5e7b9497e..5d1fed88e 100644 --- a/pkg/detectors/azuresearchquerykey/azuresearchquerykey.go +++ b/pkg/detectors/azuresearchquerykey/azuresearchquerykey.go @@ -8,7 +8,6 @@ import ( regexp "github.com/wasilibs/go-re2" - "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) @@ -22,7 +21,7 @@ type Scanner struct { var _ detectors.Detector = (*Scanner)(nil) var ( - defaultClient = common.SaneHttpClient() + defaultClient = detectors.DetectorHttpClientWithNoLocalAddresses // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"azure"}) + `\b([0-9a-zA-Z]{52})\b`) urlPat = regexp.MustCompile(detectors.PrefixRegex([]string{"azure"}) + `https:\/\/([0-9a-z]{5,40})\.search\.windows\.net\/indexes\/([0-9a-z]{5,40})\b`) diff --git a/pkg/detectors/azurestorage/azurestorage.go b/pkg/detectors/azurestorage/azurestorage.go index 6aa0791ed..84afdd98b 100644 --- a/pkg/detectors/azurestorage/azurestorage.go +++ b/pkg/detectors/azurestorage/azurestorage.go @@ -25,7 +25,7 @@ type Scanner struct { var _ detectors.Detector = (*Scanner)(nil) var ( - defaultClient = http.DefaultClient + defaultClient = detectors.DetectorHttpClientWithNoLocalAddresses keyPat = regexp.MustCompile(`DefaultEndpointsProtocol=https;AccountName=(?P[^;]+);AccountKey=(?P[^;]+);EndpointSuffix=core\.windows\.net`) ) diff --git a/pkg/detectors/caspio/caspio.go b/pkg/detectors/caspio/caspio.go index b022e1123..195dc7237 100644 --- a/pkg/detectors/caspio/caspio.go +++ b/pkg/detectors/caspio/caspio.go @@ -3,11 +3,11 @@ package caspio import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" - "github.com/trufflesecurity/trufflehog/v3/pkg/common" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) @@ -20,7 +20,7 @@ type Scanner struct{ var _ detectors.Detector = (*Scanner)(nil) var ( - client = common.SaneHttpClient() + client = detectors.DetectorHttpClientWithNoLocalAddresses // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"caspio"}) + `\b([a-z0-9]{50})\b`) diff --git a/pkg/detectors/databrickstoken/databrickstoken.go b/pkg/detectors/databrickstoken/databrickstoken.go index 866fcbb99..43bfd7016 100644 --- a/pkg/detectors/databrickstoken/databrickstoken.go +++ b/pkg/detectors/databrickstoken/databrickstoken.go @@ -3,11 +3,11 @@ package databrickstoken import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" - "github.com/trufflesecurity/trufflehog/v3/pkg/common" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) @@ -21,7 +21,7 @@ type Scanner struct { var _ detectors.Detector = (*Scanner)(nil) var ( - defaultClient = common.SaneHttpClient() + defaultClient = detectors.DetectorHttpClientWithNoLocalAddresses // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. domain = regexp.MustCompile(`\b([a-z0-9-]+(?:\.[a-z0-9-]+)*\.(cloud\.databricks\.com|gcp\.databricks\.com|azurewebsites\.net))\b`) diff --git a/pkg/detectors/deputy/deputy.go b/pkg/detectors/deputy/deputy.go index ed0438fa5..dc1635151 100644 --- a/pkg/detectors/deputy/deputy.go +++ b/pkg/detectors/deputy/deputy.go @@ -3,11 +3,11 @@ package deputy import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" - "github.com/trufflesecurity/trufflehog/v3/pkg/common" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) @@ -20,7 +20,7 @@ type Scanner struct{ var _ detectors.Detector = (*Scanner)(nil) var ( - client = common.SaneHttpClient() + client = detectors.DetectorHttpClientWithNoLocalAddresses // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"deputy"}) + `\b([0-9a-z]{32})\b`) diff --git a/pkg/detectors/detectors.go b/pkg/detectors/detectors.go index ee4f1d86b..b3e975ff8 100644 --- a/pkg/detectors/detectors.go +++ b/pkg/detectors/detectors.go @@ -241,3 +241,13 @@ func RedactURL(u url.URL) string { u.User = url.UserPassword(u.User.Username(), "********") return strings.TrimSpace(strings.Replace(u.String(), "%2A", "*", -1)) } + +func ParseURLAndStripPathAndParams(u string) (*url.URL, error) { + parsedURL, err := url.Parse(u) + if err != nil { + return nil, err + } + parsedURL.Path = "" + parsedURL.RawQuery = "" + return parsedURL, nil +} diff --git a/pkg/detectors/discordwebhook/discordwebhook.go b/pkg/detectors/discordwebhook/discordwebhook.go index e05a0fba2..c01e5677b 100644 --- a/pkg/detectors/discordwebhook/discordwebhook.go +++ b/pkg/detectors/discordwebhook/discordwebhook.go @@ -7,7 +7,6 @@ import ( regexp "github.com/wasilibs/go-re2" - "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) @@ -18,7 +17,7 @@ type Scanner struct{} var _ detectors.Detector = (*Scanner)(nil) var ( - client = common.SaneHttpClient() + client = detectors.DetectorHttpClientWithNoLocalAddresses // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. keyPat = regexp.MustCompile(`(https:\/\/discord\.com\/api\/webhooks\/[0-9]{18}\/[0-9a-zA-Z-]{68})`) diff --git a/pkg/detectors/fibery/fibery.go b/pkg/detectors/fibery/fibery.go index dbccd03c1..477c0987a 100644 --- a/pkg/detectors/fibery/fibery.go +++ b/pkg/detectors/fibery/fibery.go @@ -3,12 +3,12 @@ package fibery import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" "time" - "github.com/trufflesecurity/trufflehog/v3/pkg/common" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) @@ -21,7 +21,7 @@ type Scanner struct { var _ detectors.Detector = (*Scanner)(nil) var ( - client = common.SaneHttpClient() + client = detectors.DetectorHttpClientWithNoLocalAddresses // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"fibery"}) + `\b([0-9a-f]{8}.[0-9a-f]{35})\b`) diff --git a/pkg/detectors/freshbooks/freshbooks.go b/pkg/detectors/freshbooks/freshbooks.go index 75e65ce4d..9702ba87d 100644 --- a/pkg/detectors/freshbooks/freshbooks.go +++ b/pkg/detectors/freshbooks/freshbooks.go @@ -3,12 +3,12 @@ package freshbooks import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "io" "net/http" "strings" - "github.com/trufflesecurity/trufflehog/v3/pkg/common" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) @@ -21,10 +21,10 @@ type Scanner struct{ var _ detectors.Detector = (*Scanner)(nil) var ( - client = common.SaneHttpClient() - + client = detectors.DetectorHttpClientWithNoLocalAddresses // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"freshbooks"}) + `\b([0-9a-z]{64})\b`) + // TODO: this domain pattern is too restrictive uriPat = regexp.MustCompile(detectors.PrefixRegex([]string{"freshbooks"}) + `\b(https://www.[0-9A-Za-z_-]{1,}.com)\b`) ) diff --git a/pkg/detectors/grafanaserviceaccount/grafanaserviceaccount.go b/pkg/detectors/grafanaserviceaccount/grafanaserviceaccount.go index 6df3aaf89..78f5a4105 100644 --- a/pkg/detectors/grafanaserviceaccount/grafanaserviceaccount.go +++ b/pkg/detectors/grafanaserviceaccount/grafanaserviceaccount.go @@ -3,11 +3,11 @@ package grafanaserviceaccount import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" - "github.com/trufflesecurity/trufflehog/v3/pkg/common" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) @@ -21,7 +21,7 @@ type Scanner struct { var _ detectors.Detector = (*Scanner)(nil) var ( - defaultClient = common.SaneHttpClient() + defaultClient = detectors.DetectorHttpClientWithNoLocalAddresses // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. keyPat = regexp.MustCompile(`\b(glsa_[0-9a-zA-Z_]{41})\b`) domainPat = regexp.MustCompile(`\b([a-zA-Z0-9-]+\.grafana\.net)\b`) diff --git a/pkg/detectors/http.go b/pkg/detectors/http.go new file mode 100644 index 000000000..a1780c1ce --- /dev/null +++ b/pkg/detectors/http.go @@ -0,0 +1,154 @@ +package detectors + +import ( + "context" + "errors" + "net" + "net/http" + "time" +) + +var DetectorHttpClientWithNoLocalAddresses *http.Client +var DetectorHttpClientWithLocalAddresses *http.Client + +const DefaultResponseTimeout = 5 * time.Second +const DefaultUserAgent = "TruffleHog" + +func init() { + DetectorHttpClientWithLocalAddresses = NewDetectorHttpClient( + WithTransport(NewDetectorTransport(DefaultUserAgent, nil)), + WithTimeout(DefaultResponseTimeout), + WithNoFollowRedirects(), + ) + DetectorHttpClientWithNoLocalAddresses = NewDetectorHttpClient( + WithTransport(NewDetectorTransport(DefaultUserAgent, nil)), + WithTimeout(DefaultResponseTimeout), + WithNoFollowRedirects(), + WithNoLocalIP(), + ) +} + +// ClientOption defines a function type that modifies an http.Client. +type ClientOption func(*http.Client) + +// WithNoFollowRedirects allows disabling automatic following of redirects. +func WithNoFollowRedirects() ClientOption { + return func(c *http.Client) { + c.CheckRedirect = func(req *http.Request, via []*http.Request) error { + return http.ErrUseLastResponse + } + } +} + +type detectorTransport struct { + T http.RoundTripper + userAgent string +} + +func (t *detectorTransport) RoundTrip(req *http.Request) (*http.Response, error) { + req.Header.Add("User-Agent", t.userAgent) + return t.T.RoundTrip(req) +} + +var defaultDialer = &net.Dialer{ + Timeout: 2 * time.Second, + KeepAlive: 5 * time.Second, +} + +func NewDetectorTransport(userAgent string, T http.RoundTripper) http.RoundTripper { + if T == nil { + T = &http.Transport{ + Proxy: http.ProxyFromEnvironment, + DialContext: defaultDialer.DialContext, + MaxIdleConns: 100, + MaxIdleConnsPerHost: 5, + IdleConnTimeout: 90 * time.Second, + TLSHandshakeTimeout: 3 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + } + } + return &detectorTransport{T: T, userAgent: userAgent} +} + +func isLocalIP(ip net.IP) bool { + if ip.IsLoopback() || ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() || ip.IsPrivate() { + return true + } + + return false +} + +var ErrNoLocalIP = errors.New("dialing local IP addresses is not allowed") + +func WithNoLocalIP() ClientOption { + return func(c *http.Client) { + if c.Transport == nil { + c.Transport = &http.Transport{} + } + + // Type assertion to get the underlying *http.Transport + transport, ok := c.Transport.(*http.Transport) + if !ok { + // If c.Transport is not *http.Transport, check if it is wrapped in a detectorTransport + dt, ok := c.Transport.(*detectorTransport) + if !ok { + panic("unsupported transport type") + } + transport, ok = dt.T.(*http.Transport) + if !ok { + panic("underlying transport is not *http.Transport") + } + } + + // If the original DialContext is nil, set it to the default dialer + if transport.DialContext == nil { + transport.DialContext = defaultDialer.DialContext + } + originalDialContext := transport.DialContext + transport.DialContext = func(ctx context.Context, network, addr string) (net.Conn, error) { + host, port, err := net.SplitHostPort(addr) + if err != nil { + return nil, err + } + + ips, err := net.LookupIP(host) + if err != nil { + return nil, err + } + + for _, ip := range ips { + if isLocalIP(ip) { + return nil, ErrNoLocalIP + } + } + + return originalDialContext(ctx, network, net.JoinHostPort(host, port)) + } + } +} + +// WithTransport sets a custom transport for the http.Client. +func WithTransport(transport http.RoundTripper) ClientOption { + return func(c *http.Client) { + c.Transport = transport + } +} + +// WithTimeout sets a timeout for the http.Client. +func WithTimeout(timeout time.Duration) ClientOption { + return func(c *http.Client) { + c.Timeout = timeout + } +} + +func NewDetectorHttpClient(opts ...ClientOption) *http.Client { + httpClient := &http.Client{ + Transport: NewDetectorTransport(DefaultUserAgent, nil), + Timeout: DefaultResponseTimeout, + } + + for _, opt := range opts { + opt(httpClient) + } + return httpClient +} diff --git a/pkg/detectors/http_test.go b/pkg/detectors/http_test.go new file mode 100644 index 000000000..bbe6b9dfe --- /dev/null +++ b/pkg/detectors/http_test.go @@ -0,0 +1,94 @@ +package detectors + +import ( + "context" + "net" + "net/http" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestWithNoLocalIP(t *testing.T) { + t.Run("Prevents dialing local IP", func(t *testing.T) { + client := &http.Client{} + WithNoLocalIP()(client) + + transport, ok := client.Transport.(*http.Transport) + assert.True(t, ok, "Expected transport to be *http.Transport") + + _, err := transport.DialContext(context.Background(), "tcp", "127.0.0.1:8080") + assert.Error(t, err) + assert.ErrorIs(t, err, ErrNoLocalIP) + }) + + t.Run("Allows dialing non-local host", func(t *testing.T) { + client := &http.Client{} + WithNoLocalIP()(client) + + transport, ok := client.Transport.(*http.Transport) + assert.True(t, ok, "Expected transport to be *http.Transport") + + conn, err := transport.DialContext(context.Background(), "tcp", "google.com:80") + assert.NoError(t, err) + assert.NotNil(t, conn) + conn.Close() + }) + + t.Run("Allows dialing non-local IP", func(t *testing.T) { + client := &http.Client{} + WithNoLocalIP()(client) + + transport, ok := client.Transport.(*http.Transport) + assert.True(t, ok, "Expected transport to be *http.Transport") + + conn, err := transport.DialContext(context.Background(), "tcp", "1.1.1.1:80") + assert.NoError(t, err) + assert.NotNil(t, conn) + conn.Close() + }) + + t.Run("Handles invalid address", func(t *testing.T) { + client := &http.Client{} + WithNoLocalIP()(client) + + transport, ok := client.Transport.(*http.Transport) + assert.True(t, ok, "Expected transport to be *http.Transport") + + _, err := transport.DialContext(context.Background(), "tcp", "invalid-address") + assert.Error(t, err) + }) + + t.Run("Handles non-existent hostname", func(t *testing.T) { + client := &http.Client{} + WithNoLocalIP()(client) + + transport, ok := client.Transport.(*http.Transport) + assert.True(t, ok, "Expected transport to be *http.Transport") + + _, err := transport.DialContext(context.Background(), "tcp", "non-existent-host.local:80") + assert.Error(t, err) + }) +} + +func TestIsLocalIP(t *testing.T) { + testCases := []struct { + name string + ip net.IP + expected bool + }{ + {"Loopback IPv4", net.ParseIP("127.0.0.1"), true}, + {"Loopback IPv6", net.ParseIP("::1"), true}, + {"Private IPv4", net.ParseIP("192.168.1.1"), true}, + {"Private IPv6", net.ParseIP("fd00::1"), true}, + {"Public IPv4", net.ParseIP("8.8.8.8"), false}, + {"Public IPv6", net.ParseIP("2001:4860:4860::8888"), false}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + result := isLocalIP(tc.ip) + assert.Equal(t, tc.expected, result) + }) + } +} diff --git a/pkg/detectors/invoiceocean/invoiceocean.go b/pkg/detectors/invoiceocean/invoiceocean.go index f95a4ec4c..2d8add465 100644 --- a/pkg/detectors/invoiceocean/invoiceocean.go +++ b/pkg/detectors/invoiceocean/invoiceocean.go @@ -3,11 +3,11 @@ package invoiceocean import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" - "github.com/trufflesecurity/trufflehog/v3/pkg/common" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) @@ -20,7 +20,7 @@ type Scanner struct{ var _ detectors.Detector = (*Scanner)(nil) var ( - client = common.SaneHttpClient() + client = detectors.DetectorHttpClientWithNoLocalAddresses // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"invoiceocean"}) + `\b([0-9A-Za-z]{20})\b`) diff --git a/pkg/detectors/jiratoken/v1/jiratoken.go b/pkg/detectors/jiratoken/v1/jiratoken.go index ae23754e5..988ac6700 100644 --- a/pkg/detectors/jiratoken/v1/jiratoken.go +++ b/pkg/detectors/jiratoken/v1/jiratoken.go @@ -9,7 +9,6 @@ import ( regexp "github.com/wasilibs/go-re2" - "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) @@ -26,7 +25,7 @@ var _ detectors.Versioner = (*Scanner)(nil) func (Scanner) Version() int { return 1 } var ( - defaultClient = common.SaneHttpClient() + defaultClient = detectors.DetectorHttpClientWithLocalAddresses // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. tokenPat = regexp.MustCompile(detectors.PrefixRegex([]string{"jira"}) + `\b([a-zA-Z-0-9]{24})\b`) diff --git a/pkg/detectors/jiratoken/v2/jiratoken_v2.go b/pkg/detectors/jiratoken/v2/jiratoken_v2.go index 3501bde9b..cac0a5d3a 100644 --- a/pkg/detectors/jiratoken/v2/jiratoken_v2.go +++ b/pkg/detectors/jiratoken/v2/jiratoken_v2.go @@ -9,7 +9,6 @@ import ( regexp "github.com/wasilibs/go-re2" - "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) @@ -26,7 +25,7 @@ var _ detectors.Versioner = (*Scanner)(nil) func (Scanner) Version() int { return 2 } var ( - defaultClient = common.SaneHttpClient() + defaultClient = detectors.DetectorHttpClientWithLocalAddresses // https://support.atlassian.com/atlassian-account/docs/manage-api-tokens-for-your-atlassian-account/ // Tokens created after Jan 18 2023 use a variable length diff --git a/pkg/detectors/kanban/kanban.go b/pkg/detectors/kanban/kanban.go index d3fb775ea..8cb4dfe50 100644 --- a/pkg/detectors/kanban/kanban.go +++ b/pkg/detectors/kanban/kanban.go @@ -3,11 +3,11 @@ package kanban import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" - "github.com/trufflesecurity/trufflehog/v3/pkg/common" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) @@ -20,7 +20,7 @@ type Scanner struct{ var _ detectors.Detector = (*Scanner)(nil) var ( - client = common.SaneHttpClient() + client = detectors.DetectorHttpClientWithNoLocalAddresses // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"kanban"}) + `\b([0-9A-Z]{12})\b`) diff --git a/pkg/detectors/kanbantool/kanbantool.go b/pkg/detectors/kanbantool/kanbantool.go index a6bb8e23e..3fad48c98 100644 --- a/pkg/detectors/kanbantool/kanbantool.go +++ b/pkg/detectors/kanbantool/kanbantool.go @@ -3,11 +3,11 @@ package kanbantool import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" - "github.com/trufflesecurity/trufflehog/v3/pkg/common" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) @@ -20,7 +20,7 @@ type Scanner struct { var _ detectors.Detector = (*Scanner)(nil) var ( - client = common.SaneHttpClient() + client = detectors.DetectorHttpClientWithNoLocalAddresses // Make sure that your group is surrounded in boundary characters such as below to reduce false positives keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"kanbantool"}) + `\b([0-9A-Z]{12})\b`) diff --git a/pkg/detectors/liveagent/liveagent.go b/pkg/detectors/liveagent/liveagent.go index a4ed7f394..603e9f1ca 100644 --- a/pkg/detectors/liveagent/liveagent.go +++ b/pkg/detectors/liveagent/liveagent.go @@ -3,11 +3,11 @@ package liveagent import ( "context" "encoding/json" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" - "github.com/trufflesecurity/trufflehog/v3/pkg/common" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) @@ -20,7 +20,7 @@ type Scanner struct{ var _ detectors.Detector = (*Scanner)(nil) var ( - client = common.SaneHttpClient() + client = detectors.DetectorHttpClientWithNoLocalAddresses // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. domainPat = regexp.MustCompile(`\b(https?://[A-Za-z0-9-]+\.ladesk\.com)\b`) diff --git a/pkg/detectors/loggly/loggly.go b/pkg/detectors/loggly/loggly.go index 4de7e41d6..b89f77715 100644 --- a/pkg/detectors/loggly/loggly.go +++ b/pkg/detectors/loggly/loggly.go @@ -3,11 +3,11 @@ package loggly import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" - "github.com/trufflesecurity/trufflehog/v3/pkg/common" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) @@ -21,7 +21,7 @@ type Scanner struct { var _ detectors.Detector = (*Scanner)(nil) var ( - defaultClient = common.SaneHttpClient() + defaultClient = detectors.DetectorHttpClientWithNoLocalAddresses // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. domainPat = regexp.MustCompile(`\b([a-zA-Z0-9-]+\.loggly\.com)\b`) keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"loggly"}) + `\b([a-z0-9]{8}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{12})\b`) diff --git a/pkg/detectors/metabase/metabase.go b/pkg/detectors/metabase/metabase.go index 66fd9dc9a..b2d8815db 100644 --- a/pkg/detectors/metabase/metabase.go +++ b/pkg/detectors/metabase/metabase.go @@ -3,12 +3,12 @@ package metabase import ( "context" "encoding/json" - regexp "github.com/wasilibs/go-re2" "io" "net/http" "strings" - "github.com/trufflesecurity/trufflehog/v3/pkg/common" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) @@ -21,7 +21,7 @@ type Scanner struct{ var _ detectors.Detector = (*Scanner)(nil) var ( - client = common.SaneHttpClient() + client = detectors.DetectorHttpClientWithLocalAddresses // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"metabase"}) + `\b([a-zA-Z0-9-]{36})\b`) @@ -54,6 +54,12 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } resURLMatch := strings.TrimSpace(urlMatch[1]) + u, err := detectors.ParseURLAndStripPathAndParams(resURLMatch) + if err != nil { + // if the URL is invalid just move onto the next one + continue + } + s1 := detectors.Result{ DetectorType: detectorspb.DetectorType_Metabase, Raw: []byte(resMatch), @@ -61,7 +67,8 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } if verify { - req, err := http.NewRequestWithContext(ctx, http.MethodGet, resURLMatch+"/api/user/current", nil) + u.Path = "/api/user/current" + req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil) if err != nil { continue } diff --git a/pkg/detectors/microsoftteamswebhook/microsoftteamswebhook.go b/pkg/detectors/microsoftteamswebhook/microsoftteamswebhook.go index 446da7a81..31816704e 100644 --- a/pkg/detectors/microsoftteamswebhook/microsoftteamswebhook.go +++ b/pkg/detectors/microsoftteamswebhook/microsoftteamswebhook.go @@ -6,11 +6,9 @@ import ( "io" "net/http" "strings" - "time" regexp "github.com/wasilibs/go-re2" - "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) @@ -24,7 +22,7 @@ type Scanner struct { var _ detectors.Detector = (*Scanner)(nil) var ( - defaultClient = common.SaneHttpClientTimeOut(5 * time.Second) + defaultClient = detectors.DetectorHttpClientWithNoLocalAddresses // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. keyPat = regexp.MustCompile(`(https:\/\/[a-zA-Z-0-9]+\.webhook\.office\.com\/webhookb2\/[a-zA-Z-0-9]{8}-[a-zA-Z-0-9]{4}-[a-zA-Z-0-9]{4}-[a-zA-Z-0-9]{4}-[a-zA-Z-0-9]{12}\@[a-zA-Z-0-9]{8}-[a-zA-Z-0-9]{4}-[a-zA-Z-0-9]{4}-[a-zA-Z-0-9]{4}-[a-zA-Z-0-9]{12}\/IncomingWebhook\/[a-zA-Z-0-9]{32}\/[a-zA-Z-0-9]{8}-[a-zA-Z-0-9]{4}-[a-zA-Z-0-9]{4}-[a-zA-Z-0-9]{4}-[a-zA-Z-0-9]{12})`) diff --git a/pkg/detectors/mite/mite.go b/pkg/detectors/mite/mite.go index 9d98c76a6..a068cb678 100644 --- a/pkg/detectors/mite/mite.go +++ b/pkg/detectors/mite/mite.go @@ -3,11 +3,11 @@ package mite import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" - "github.com/trufflesecurity/trufflehog/v3/pkg/common" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) @@ -20,7 +20,7 @@ type Scanner struct{ var _ detectors.Detector = (*Scanner)(nil) var ( - client = common.SaneHttpClient() + client = detectors.DetectorHttpClientWithNoLocalAddresses // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"mite"}) + `\b([0-9a-z]{16})\b`) diff --git a/pkg/detectors/okta/okta.go b/pkg/detectors/okta/okta.go index d3e8f2a2b..9642d2b3a 100644 --- a/pkg/detectors/okta/okta.go +++ b/pkg/detectors/okta/okta.go @@ -9,7 +9,6 @@ import ( regexp "github.com/wasilibs/go-re2" - "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) @@ -64,7 +63,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Set("Content-Type", "application/json") req.Header.Set("Authorization", fmt.Sprintf("SSWS %s", token)) - resp, err := common.SaneHttpClient().Do(req) + resp, err := detectors.DetectorHttpClientWithNoLocalAddresses.Do(req) if err != nil { continue } diff --git a/pkg/detectors/openvpn/openvpn.go b/pkg/detectors/openvpn/openvpn.go index 96fea975d..8fc24973b 100644 --- a/pkg/detectors/openvpn/openvpn.go +++ b/pkg/detectors/openvpn/openvpn.go @@ -9,7 +9,6 @@ import ( regexp "github.com/wasilibs/go-re2" - "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) @@ -23,7 +22,7 @@ type Scanner struct { var _ detectors.Detector = (*Scanner)(nil) var ( - defaultClient = common.SaneHttpClient() + defaultClient = detectors.DetectorHttpClientWithNoLocalAddresses clientIDPat = regexp.MustCompile(detectors.PrefixRegex([]string{"openvpn"}) + `\b([A-Za-z0-9-]{3,40}\.[A-Za-z0-9-]{3,40})\b`) clientSecretPat = regexp.MustCompile(`\b([a-zA-Z0-9_-]{64,})\b`) diff --git a/pkg/detectors/planviewleankit/planviewleankit.go b/pkg/detectors/planviewleankit/planviewleankit.go index f4547dc19..8c565352b 100644 --- a/pkg/detectors/planviewleankit/planviewleankit.go +++ b/pkg/detectors/planviewleankit/planviewleankit.go @@ -3,11 +3,11 @@ package planviewleankit import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" - "github.com/trufflesecurity/trufflehog/v3/pkg/common" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) @@ -20,7 +20,7 @@ type Scanner struct{ var _ detectors.Detector = (*Scanner)(nil) var ( - client = common.SaneHttpClient() + client = detectors.DetectorHttpClientWithNoLocalAddresses // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"planviewleankit", "planview"}) + `\b([0-9a-f]{128})\b`) diff --git a/pkg/detectors/portainer/portainer.go b/pkg/detectors/portainer/portainer.go index aee0be138..56af42e7e 100644 --- a/pkg/detectors/portainer/portainer.go +++ b/pkg/detectors/portainer/portainer.go @@ -3,11 +3,11 @@ package portainer import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" - "github.com/trufflesecurity/trufflehog/v3/pkg/common" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) @@ -21,7 +21,7 @@ type Scanner struct { var _ detectors.Detector = (*Scanner)(nil) var ( - defaultClient = common.SaneHttpClient() + defaultClient = detectors.DetectorHttpClientWithLocalAddresses // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. endpointPat = regexp.MustCompile(detectors.PrefixRegex([]string{"portainer"}) + `\b(https?:\/\/\S+(:[0-9]{4,5})?)\b`) tokenPat = regexp.MustCompile(detectors.PrefixRegex([]string{"portainer"}) + `\b(eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9\.[0-9A-Za-z]{50,310}\.[0-9A-Z-a-z\-_]{43})\b`) @@ -49,6 +49,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result for _, endpointMatch := range endpointMatches { resEndpointMatch := strings.TrimSpace(endpointMatch[1]) + u, err := detectors.ParseURLAndStripPathAndParams(resEndpointMatch) + if err != nil { + // if the URL is invalid just move onto the next one + continue + } + u.Path = "/api/endpoints" + s1 := detectors.Result{ DetectorType: detectorspb.DetectorType_Portainer, Raw: []byte(resMatch), @@ -60,7 +67,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result if client == nil { client = defaultClient } - req, err := http.NewRequestWithContext(ctx, "GET", resEndpointMatch+"/api/endpoints", nil) + req, err := http.NewRequestWithContext(ctx, "GET", u.String(), nil) if err != nil { continue } @@ -81,8 +88,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } } - - if len(endpointMatches) > 0 { results = append(results, s1) } diff --git a/pkg/detectors/portainertoken/portainertoken.go b/pkg/detectors/portainertoken/portainertoken.go index 54ad63c1d..53b88df1d 100644 --- a/pkg/detectors/portainertoken/portainertoken.go +++ b/pkg/detectors/portainertoken/portainertoken.go @@ -3,11 +3,11 @@ package portainertoken import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" - "github.com/trufflesecurity/trufflehog/v3/pkg/common" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) @@ -21,7 +21,7 @@ type Scanner struct { var _ detectors.Detector = (*Scanner)(nil) var ( - defaultClient = common.SaneHttpClient() + defaultClient = detectors.DetectorHttpClientWithLocalAddresses // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"portainertoken"}) + `\b(ptr_[A-Za-z0-9\/_\-+=]{20,60})`) endpointPat = regexp.MustCompile(detectors.PrefixRegex([]string{"portainer"}) + `\b(https?:\/\/\S+(:[0-9]{4,5})?)\b`) @@ -49,6 +49,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result for _, endpointMatch := range endpointMatches { resEndpointMatch := strings.TrimSpace(endpointMatch[1]) + u, err := detectors.ParseURLAndStripPathAndParams(resEndpointMatch) + if err != nil { + // if the URL is invalid just move onto the next one + continue + } + u.Path = "/api/stacks" + s1 := detectors.Result{ DetectorType: detectorspb.DetectorType_PortainerToken, Raw: []byte(resMatch), @@ -60,7 +67,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result if client == nil { client = defaultClient } - req, err := http.NewRequestWithContext(ctx, "GET", resEndpointMatch+"/api/stacks", nil) + req, err := http.NewRequestWithContext(ctx, "GET", u.String(), nil) if err != nil { continue } diff --git a/pkg/detectors/repairshopr/repairshopr.go b/pkg/detectors/repairshopr/repairshopr.go index 47976fa9d..58ee10fb6 100644 --- a/pkg/detectors/repairshopr/repairshopr.go +++ b/pkg/detectors/repairshopr/repairshopr.go @@ -3,11 +3,11 @@ package repairshopr import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" - "github.com/trufflesecurity/trufflehog/v3/pkg/common" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) @@ -20,7 +20,7 @@ type Scanner struct { var _ detectors.Detector = (*Scanner)(nil) var ( - client = common.SaneHttpClient() + client = detectors.DetectorHttpClientWithNoLocalAddresses // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"repairshopr"}) + `\b([a-zA-Z0-9-]{51})\b`) diff --git a/pkg/detectors/ringcentral/ringcentral.go b/pkg/detectors/ringcentral/ringcentral.go index a71075512..11dff8a5f 100644 --- a/pkg/detectors/ringcentral/ringcentral.go +++ b/pkg/detectors/ringcentral/ringcentral.go @@ -3,10 +3,11 @@ package ringcentral import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" @@ -24,6 +25,7 @@ var ( // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"ringcentral"}) + `\b([0-9A-Za-z_-]{22})\b`) + // TODO: this domain pattern is too restrictive uriPat = regexp.MustCompile(detectors.PrefixRegex([]string{"ringcentral"}) + `\b(https://www.[0-9A-Za-z_-]{1,}.com)\b`) ) diff --git a/pkg/detectors/salesforce/salesforce.go b/pkg/detectors/salesforce/salesforce.go index e220ad5ca..f74e24d92 100644 --- a/pkg/detectors/salesforce/salesforce.go +++ b/pkg/detectors/salesforce/salesforce.go @@ -3,10 +3,11 @@ package salesforce import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" @@ -25,7 +26,7 @@ type Scanner struct { var _ detectors.Detector = (*Scanner)(nil) var ( - defaultClient = common.SaneHttpClient() + defaultClient = detectors.DetectorHttpClientWithNoLocalAddresses // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. accessTokenPat = regexp.MustCompile(`\b00[a-zA-Z0-9]{13}![a-zA-Z0-9_.]{96}\b`) instancePat = regexp.MustCompile(`\bhttps://[0-9a-zA-Z-\.]{1,100}\.my\.salesforce\.com\b`) diff --git a/pkg/detectors/salesmate/salesmate.go b/pkg/detectors/salesmate/salesmate.go index 141502d72..27112a1c0 100644 --- a/pkg/detectors/salesmate/salesmate.go +++ b/pkg/detectors/salesmate/salesmate.go @@ -3,11 +3,11 @@ package salesmate import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" - "github.com/trufflesecurity/trufflehog/v3/pkg/common" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) @@ -20,7 +20,7 @@ type Scanner struct{ var _ detectors.Detector = (*Scanner)(nil) var ( - client = common.SaneHttpClient() + client = detectors.DetectorHttpClientWithNoLocalAddresses // Make sure that your group is surrounded in boundary characters such as below to reduce false positives keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"salesmate"}) + `\b([0-9Aa-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})\b`) diff --git a/pkg/detectors/shopify/shopify.go b/pkg/detectors/shopify/shopify.go index 00ac455bb..13b11ecf9 100644 --- a/pkg/detectors/shopify/shopify.go +++ b/pkg/detectors/shopify/shopify.go @@ -8,7 +8,6 @@ import ( regexp "github.com/wasilibs/go-re2" - "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) @@ -22,7 +21,7 @@ var _ detectors.Detector = (*Scanner)(nil) var _ detectors.CustomFalsePositiveChecker = (*Scanner)(nil) var ( - client = common.SaneHttpClient() + client = detectors.DetectorHttpClientWithNoLocalAddresses // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. keyPat = regexp.MustCompile(`\b(shppa_|shpat_)([0-9A-Fa-f]{32})\b`) diff --git a/pkg/detectors/signalwire/signalwire.go b/pkg/detectors/signalwire/signalwire.go index 743cb5587..570aeb3b7 100644 --- a/pkg/detectors/signalwire/signalwire.go +++ b/pkg/detectors/signalwire/signalwire.go @@ -4,11 +4,11 @@ import ( "context" b64 "encoding/base64" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" - "github.com/trufflesecurity/trufflehog/v3/pkg/common" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) @@ -21,7 +21,7 @@ type Scanner struct { var _ detectors.Detector = (*Scanner)(nil) var ( - client = common.SaneHttpClient() + client = detectors.DetectorHttpClientWithNoLocalAddresses // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"signalwire"}) + `\b([0-9A-Za-z]{50})\b`) diff --git a/pkg/detectors/signalwire/signalwire_test.go b/pkg/detectors/signalwire/signalwire_test.go index 50fc14142..a4c06f265 100644 --- a/pkg/detectors/signalwire/signalwire_test.go +++ b/pkg/detectors/signalwire/signalwire_test.go @@ -23,7 +23,7 @@ func TestSignalwire_FromChunk(t *testing.T) { if err != nil { t.Fatalf("could not get test secrets from GCP: %s", err) } - id := testSecrets.MustGetField("SGINALWIRE_ID") + id := testSecrets.MustGetField("SIGNALWIRE_ID") secret := testSecrets.MustGetField("SIGNALWIRE_TOKEN") url := testSecrets.MustGetField("SIGNALWIRE_URL") inactiveSecret := testSecrets.MustGetField("SIGNALWIRE_INACTIVE") diff --git a/pkg/detectors/slackwebhook/slackwebhook.go b/pkg/detectors/slackwebhook/slackwebhook.go index d8042114b..508501fe6 100644 --- a/pkg/detectors/slackwebhook/slackwebhook.go +++ b/pkg/detectors/slackwebhook/slackwebhook.go @@ -8,7 +8,6 @@ import ( "net/http" "strings" - "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" regexp "github.com/wasilibs/go-re2" @@ -23,7 +22,7 @@ type Scanner struct { var _ detectors.Detector = (*Scanner)(nil) var ( - defaultClient = common.SaneHttpClient() + defaultClient = detectors.DetectorHttpClientWithNoLocalAddresses // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. keyPats = map[string]*regexp.Regexp{ "Slack Service Web Hook": regexp.MustCompile(`(https://hooks\.slack\.com/services/T[A-Z0-9]+/B[A-Z0-9]+/[A-Za-z0-9]{23,25})`), diff --git a/pkg/detectors/sugester/sugester.go b/pkg/detectors/sugester/sugester.go index e22be1d9e..c604e61dd 100644 --- a/pkg/detectors/sugester/sugester.go +++ b/pkg/detectors/sugester/sugester.go @@ -2,11 +2,11 @@ package sugester import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" - "github.com/trufflesecurity/trufflehog/v3/pkg/common" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) @@ -19,7 +19,7 @@ type Scanner struct{ var _ detectors.Detector = (*Scanner)(nil) var ( - client = common.SaneHttpClient() + client = detectors.DetectorHttpClientWithNoLocalAddresses // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"sugester"}) + `\b([a-zA-Z0-9]{32})\b`) diff --git a/pkg/detectors/thinkific/thinkific.go b/pkg/detectors/thinkific/thinkific.go index d0cb6631d..fdfbcb638 100644 --- a/pkg/detectors/thinkific/thinkific.go +++ b/pkg/detectors/thinkific/thinkific.go @@ -3,12 +3,12 @@ package thinkific import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "io" "net/http" "strings" - "github.com/trufflesecurity/trufflehog/v3/pkg/common" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) @@ -21,7 +21,7 @@ type Scanner struct{ var _ detectors.Detector = (*Scanner)(nil) var ( - client = common.SaneHttpClient() + client = detectors.DetectorHttpClientWithNoLocalAddresses // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"thinkific"}) + `\b([0-9a-f]{32})\b`) diff --git a/pkg/detectors/tineswebhook/tineswebhook.go b/pkg/detectors/tineswebhook/tineswebhook.go index 7126e2d7c..09654eb79 100644 --- a/pkg/detectors/tineswebhook/tineswebhook.go +++ b/pkg/detectors/tineswebhook/tineswebhook.go @@ -7,7 +7,6 @@ import ( regexp "github.com/wasilibs/go-re2" - "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) @@ -18,7 +17,7 @@ type Scanner struct{} var _ detectors.Detector = (*Scanner)(nil) var ( - client = common.SaneHttpClient() + client = detectors.DetectorHttpClientWithNoLocalAddresses // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. keyPat = regexp.MustCompile(`(https://[\w-]+\.tines\.com/webhook/[a-z0-9]{32}/[a-z0-9]{32})`) @@ -56,9 +55,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result res, err := client.Do(req) if err == nil { defer res.Body.Close() - if err != nil { - continue - } if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } diff --git a/pkg/detectors/uri/uri.go b/pkg/detectors/uri/uri.go index cd8af3004..e383ac9b9 100644 --- a/pkg/detectors/uri/uri.go +++ b/pkg/detectors/uri/uri.go @@ -9,7 +9,6 @@ import ( regexp "github.com/wasilibs/go-re2" - "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) @@ -26,7 +25,8 @@ var _ detectors.CustomFalsePositiveChecker = (*Scanner)(nil) var ( keyPat = regexp.MustCompile(`\b(?:https?:)?\/\/[\S]{3,50}:([\S]{3,50})@[-.%\w\/:]+\b`) - defaultClient = common.SaneHttpClient() + // TODO: make local addr opt-out + defaultClient = detectors.DetectorHttpClientWithNoLocalAddresses ) // Keywords are used for efficiently pre-filtering chunks. diff --git a/pkg/detectors/zapierwebhook/zapierwebhook.go b/pkg/detectors/zapierwebhook/zapierwebhook.go index 9f1d45ffe..faaad087b 100644 --- a/pkg/detectors/zapierwebhook/zapierwebhook.go +++ b/pkg/detectors/zapierwebhook/zapierwebhook.go @@ -2,11 +2,11 @@ package zapierwebhook import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" - "github.com/trufflesecurity/trufflehog/v3/pkg/common" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) @@ -17,7 +17,7 @@ type Scanner struct{} var _ detectors.Detector = (*Scanner)(nil) var ( - client = common.SaneHttpClient() + client = detectors.DetectorHttpClientWithNoLocalAddresses // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. keyPat = regexp.MustCompile(`(https:\/\/hooks\.zapier\.com\/hooks\/catch\/[A-Za-z0-9\/]{16})`) diff --git a/pkg/detectors/zendeskapi/zendeskapi.go b/pkg/detectors/zendeskapi/zendeskapi.go index 913723a5d..3d40cca4c 100644 --- a/pkg/detectors/zendeskapi/zendeskapi.go +++ b/pkg/detectors/zendeskapi/zendeskapi.go @@ -4,11 +4,11 @@ import ( "context" b64 "encoding/base64" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" - "github.com/trufflesecurity/trufflehog/v3/pkg/common" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) @@ -21,7 +21,7 @@ type Scanner struct{ var _ detectors.Detector = (*Scanner)(nil) var ( - client = common.SaneHttpClient() + client = detectors.DetectorHttpClientWithNoLocalAddresses token = regexp.MustCompile(detectors.PrefixRegex([]string{"zendesk"}) + `([A-Za-z0-9_-]{40})`) email = regexp.MustCompile(`\b([a-zA-Z-0-9-]{5,16}\@[a-zA-Z-0-9]{4,16}\.[a-zA-Z-0-9]{3,6})\b`) diff --git a/pkg/detectors/zulipchat/zulipchat.go b/pkg/detectors/zulipchat/zulipchat.go index eb5e3d956..661d03f20 100644 --- a/pkg/detectors/zulipchat/zulipchat.go +++ b/pkg/detectors/zulipchat/zulipchat.go @@ -23,7 +23,7 @@ type Scanner struct { var _ detectors.Detector = (*Scanner)(nil) var ( - defaultClient = common.SaneHttpClient() + defaultClient = detectors.DetectorHttpClientWithNoLocalAddresses // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. keyPat = regexp.MustCompile(common.BuildRegex(common.AlphaNumPattern, "", 32))