trufflesecurity/trufflehog
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix(jiratoken): stop 202 reverification retry loop (#5200)

Browse Source
This commit is contained in:
Ahsan-Sarbaz
2026-08-12 18:50:11 +05:00
committed by GitHub
parent f9daad8e65
commit 96b593f460
6 changed files with 304 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/detectors/jiratoken/v1/jiratoken.go
+109 -13
View File
@@ -8,6 +8,9 @@ import (
"fmt"
"io"
"net/http"
"net/url"
"slices"
"sort"
"strings"
regexp "github.com/wasilibs/go-re2"
@@ -20,15 +23,25 @@ import (
type Scanner struct {
detectors.DefaultMultiPartCredentialProvider
detectors.EndpointSetter
client *http.Client
}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var _ detectors.Versioner = (*Scanner)(nil)
var _ detectors.EndpointCustomizer = (*Scanner)(nil)
var _ detectors.CloudProvider = (*Scanner)(nil)
func (Scanner) Version() int { return 1 }
// CloudHost is used whenever the scanned data carries no Atlassian host of its own.
// reason: https://community.atlassian.com/forums/Jira-Product-Discovery-questions/Authorization-issues-with-GRAPHQL/qaq-p/2640943
// The graphql API answers on this host as long as the credentials are valid.
const CloudHost = "api.atlassian.com"
func (Scanner) CloudEndpoint() string { return "https://" + CloudHost }
var (
defaultClient = detectors.DetectorHttpClientWithLocalAddresses
@@ -40,8 +53,79 @@ var (
invalidHosts = simple.NewCache[struct{}]()
errNoHost = errors.New("no such host")
// atlassianDomains are the only hosts that can answer the verification request.
// domainPat matches any domain-shaped string near an atlassian/jira/confluence keyword, so without
// this filter the detector would send the email and token to unrelated hosts found in the same chunk.
atlassianDomains = []string{"atlassian.net", "atlassian.com", "jira.com"}
)
// IsAtlassianHost reports whether host is owned by Atlassian and can therefore be used for verification.
func IsAtlassianHost(host string) bool {
host = strings.ToLower(strings.TrimSuffix(host, "."))
for _, domain := range atlassianDomains {
if host == domain || strings.HasSuffix(host, "."+domain) {
return true
}
}
return false
}
// EndpointHost reduces an endpoint to its bare host. Found domains arrive without a scheme while cloud and
// user-configured endpoints carry one, and the analyzer expects SecretParts["domain"] to be a bare host.
func EndpointHost(endpoint string) string {
endpoint = strings.TrimSuffix(strings.TrimSpace(endpoint), "/")
if u, err := url.Parse(endpoint); err == nil && u.Host != "" {
return u.Host
}
return endpoint
}
// FoundHosts returns the Atlassian hosts among the domains matched in the data, sorted so results are
// deterministic. domainPat matches any domain-shaped string, so everything else is dropped rather than
// being sent the email and token.
func FoundHosts(domains map[string]struct{}) []string {
hosts := make([]string, 0, len(domains))
for domain := range domains {
if IsAtlassianHost(domain) {
hosts = append(hosts, domain)
}
}
sort.Strings(hosts)
return hosts
}
// VerificationHosts reduces endpoints to the bare hosts to verify against, dropping duplicates. The
// analyzer expects SecretParts["domain"] to be a bare host, and a user-configured endpoint can repeat a
// found one.
//
// CloudHost is only a fallback for data that named no Atlassian host, and EndpointSetter places it ahead
// of the found hosts, so it is dropped when the data named hosts of its own. Data that named CloudHost
// itself keeps it, since then it is a found host rather than the fallback.
func VerificationHosts(endpoints, found []string) []string {
seen := make(map[string]struct{}, len(endpoints))
hosts := make([]string, 0, len(endpoints))
dropCloud := len(found) > 0 && !slices.Contains(found, CloudHost)
for _, endpoint := range endpoints {
host := EndpointHost(endpoint)
if host == "" || (dropCloud && host == CloudHost) {
continue
}
if _, ok := seen[host]; ok {
continue
}
seen[host] = struct{}{}
hosts = append(hosts, host)
}
return hosts
}
// verificationURL builds the graphql URL for a bare host such as "acme.atlassian.net".
func verificationURL(host string) string {
return "https://" + host + "/gateway/api/graphql"
}
type JIRAGraphQLResponse struct {
Data struct {
Me struct {
@@ -83,24 +167,20 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
uniqueEmails[strings.ToLower(email[1])] = struct{}{}
}
if len(uniqueDomains) == 0 {
// reason: https://community.atlassian.com/forums/Jira-Product-Discovery-questions/Authorization-issues-with-GRAPHQL/qaq-p/2640943
// In case we don't find any domain matches we can use this as the graphql API works with this domain if our authentication is valid
uniqueDomains["api.atlassian.com"] = struct{}{}
}
found := FoundHosts(uniqueDomains)
hosts := VerificationHosts(s.Endpoints(found...), found)
for email := range uniqueEmails {
for token := range uniqueTokens {
for domain := range uniqueDomains {
for _, domain := range hosts {
if invalidHosts.Exists(domain) {
delete(uniqueDomains, domain)
continue
}
s1 := detectors.Result{
DetectorType: detector_typepb.DetectorType_JiraToken,
Raw: []byte(token),
RawV2: []byte(fmt.Sprintf("%s:%s:%s", email, token, domain)),
RawV2: fmt.Appendf(nil, "%s:%s:%s", email, token, domain),
ExtraData: map[string]string{
"rotation_guide": "https://howtorotate.com/docs/tutorials/atlassian/",
"version": fmt.Sprintf("%d", s.Version()),
@@ -123,6 +203,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
s1.SetVerificationError(verificationErr, token)
}
// The credential is the same across endpoints, so once one of them verifies it there
// is nothing to learn from the rest.
if s1.Verified {
results = append(results, s1)
break
}
}
results = append(results, s1)
@@ -146,7 +233,7 @@ func VerifyJiraToken(ctx context.Context, client *http.Client, email, domain, to
}
// api docs: https://developer.atlassian.com/platform/atlassian-graphql-api/graphql/#authentication
req, err := http.NewRequestWithContext(ctx, http.MethodPost, "https://"+domain+"/gateway/api/graphql", bytes.NewBuffer(jsonBody))
req, err := http.NewRequestWithContext(ctx, http.MethodPost, verificationURL(domain), bytes.NewBuffer(jsonBody))
if err != nil {
return false, err
}
@@ -171,8 +258,8 @@ func VerifyJiraToken(ctx context.Context, client *http.Client, email, domain, to
}()
// the API returns 200 if the token is valid
switch resp.StatusCode {
case http.StatusOK:
switch {
case resp.StatusCode == http.StatusOK:
var jiraResp JIRAGraphQLResponse
if err := json.NewDecoder(resp.Body).Decode(&jiraResp); err != nil {
return false, nil // can't decode response in case of 200 OK = not valid JIRA domain
@@ -181,10 +268,19 @@ func VerifyJiraToken(ctx context.Context, client *http.Client, email, domain, to
// A 200 on its own isn't enough - unrelated hosts can also return JSON and get flagged as verified.
// Only trust it when the body actually carries the authenticated user's name.
return jiraResp.Data.Me.User.Name != "", nil
case http.StatusUnauthorized:
case resp.StatusCode == http.StatusUnauthorized, resp.StatusCode == http.StatusForbidden:
// The credentials were rejected. Nothing to retry.
return false, nil
default:
case resp.StatusCode == http.StatusTooManyRequests, resp.StatusCode >= 500:
// Rate limiting and server errors are genuinely transient, so keep them indeterminate
// and let the secret be retried.
return false, fmt.Errorf("unexpected status code: %d", resp.StatusCode)
default:
// Anything else (202, 3xx, 404, ...) is not an Atlassian authentication response at all.
// domainPat matches any domain-shaped string near a jira/atlassian/confluence keyword, so this
// request may well have gone to an unrelated host that happily accepts POSTs. Retrying will
// return the same status forever, so treat it as a terminal "not verified" instead of an error.
return false, nil
}
}
pkg/detectors/jiratoken/v1/jiratoken_integration_test.go
+8 -2
View File
@@ -118,7 +118,7 @@ func TestJiraToken_FromChunk(t *testing.T) {
wantVerificationErr: true,
},
{
name: "found, verified but unexpected api surface",
name: "found, unexpected api surface is terminal and not an error",
s: Scanner{client: common.ConstantResponseHttpClient(404, "")},
args: args{
ctx: context.Background(),
@@ -136,11 +136,14 @@ func TestJiraToken_FromChunk(t *testing.T) {
},
},
wantErr: false,
wantVerificationErr: true,
wantVerificationErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
tt.s.SetCloudEndpoint(tt.s.CloudEndpoint())
tt.s.UseCloudEndpoint(true)
tt.s.UseFoundEndpoints(true)
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
if (err != nil) != tt.wantErr {
t.Errorf("JiraToken.FromData() error = %v, wantErr %v", err, tt.wantErr)
@@ -165,6 +168,9 @@ func TestJiraToken_FromChunk(t *testing.T) {
func BenchmarkFromData(benchmark *testing.B) {
ctx := context.Background()
s := Scanner{}
s.SetCloudEndpoint(s.CloudEndpoint())
s.UseCloudEndpoint(true)
s.UseFoundEndpoints(true)
for name, data := range detectors.MustGetBenchmarkData() {
benchmark.Run(name, func(b *testing.B) {
b.ResetTimer()
pkg/detectors/jiratoken/v1/jiratoken_test.go
+161 -4
View File
@@ -15,7 +15,9 @@ import (
var (
validTokenPattern = "Z7VoIYJ0K4rFWLBfkhOsLAWX"
invalidTokenPattern = "Z7VoI?J0K4rF#LBfkhO&LAWX"
validDomainPattern = "hereisavalidsubdomain.heresalongdomain.com"
validDomainPattern = "hereisavalidsubdomain.atlassian.net"
foreignDomainPattern = "hereisavalidsubdomain.heresalongdomain.com"
cloudDomain = "api.atlassian.com"
invalidDomainPattern = "?y4r3fs1ewqec12v1e3tl.5Hcsrcehic89saXd.ro@"
validEmailPattern = "[email protected]"
invalidEmailPattern = "xfKF_BZq7/grum.com"
@@ -24,6 +26,9 @@ var (
func TestJiraToken_Pattern(t *testing.T) {
d := Scanner{}
d.SetCloudEndpoint(d.CloudEndpoint())
d.UseCloudEndpoint(true)
d.UseFoundEndpoints(true)
ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
name string
@@ -33,7 +38,16 @@ func TestJiraToken_Pattern(t *testing.T) {
{
name: "valid pattern - with keyword jira",
input: fmt.Sprintf("%s %s \n%s %s\n%s %s", keyword, validTokenPattern, keyword, validDomainPattern, keyword, validEmailPattern),
want: []string{validEmailPattern + ":" + validTokenPattern + ":" + validDomainPattern},
// The tenant host found in the data is used on its own. The cloud endpoint must not shadow it,
// or SecretParts["domain"] would stop being the host the analyzer needs.
want: []string{validEmailPattern + ":" + validTokenPattern + ":" + validDomainPattern},
},
{
// A domain that isn't Atlassian-owned can't answer the verification request, so it is dropped
// and only the cloud endpoint remains.
name: "non-atlassian domain falls back to the cloud endpoint",
input: fmt.Sprintf("%s %s \n%s %s\n%s %s", keyword, validTokenPattern, keyword, foreignDomainPattern, keyword, validEmailPattern),
want: []string{validEmailPattern + ":" + validTokenPattern + ":" + cloudDomain},
},
{
name: "valid pattern - key out of prefix range",
@@ -90,12 +104,121 @@ func TestJiraToken_Pattern(t *testing.T) {
}
}
func TestIsAtlassianHost(t *testing.T) {
tests := map[string]bool{
"example.atlassian.net": true,
"atlassian.net": true,
"api.atlassian.com": true,
"acme.jira.com": true,
"EXAMPLE.Atlassian.Net": true,
"example.atlassian.net.": true,
// Suffix lookalikes must not pass, or the filter would be trivially bypassed.
"notatlassian.net": false,
"atlassian.net.evil.com": false,
"example.com": false,
"hereisavalidsubdomain.co.uk": false,
}
for host, want := range tests {
t.Run(host, func(t *testing.T) {
if got := IsAtlassianHost(host); got != want {
t.Errorf("IsAtlassianHost(%q) = %v, want %v", host, got, want)
}
})
}
}
func TestFoundHosts(t *testing.T) {
tests := []struct {
name string
domains []string
want []string
}{
{
name: "non-atlassian domains are dropped",
domains: []string{"tenant.atlassian.net", "hooks.example.com"},
want: []string{"tenant.atlassian.net"},
},
{
name: "nothing atlassian found",
domains: []string{"hooks.example.com"},
want: []string{},
},
{
// The cloud host is a normal Atlassian host when it shows up in the data.
name: "cloud host in the data is kept",
domains: []string{CloudHost},
want: []string{CloudHost},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
domains := make(map[string]struct{}, len(test.domains))
for _, d := range test.domains {
domains[d] = struct{}{}
}
if diff := cmp.Diff(test.want, FoundHosts(domains)); diff != "" {
t.Errorf("FoundHosts() diff: (-want +got)\n%s", diff)
}
})
}
}
func TestVerificationHosts(t *testing.T) {
const cloudEndpoint = "https://" + CloudHost
tests := []struct {
name string
endpoints []string
found []string
want []string
}{
{
// EndpointSetter puts the cloud endpoint ahead of the found ones, so leaving it in would let it
// win the break-on-verified and report a domain the analyzer cannot use.
name: "cloud dropped when the data named a host",
endpoints: []string{cloudEndpoint, "tenant.atlassian.net"},
found: []string{"tenant.atlassian.net"},
want: []string{"tenant.atlassian.net"},
},
{
name: "cloud kept when the data named nothing",
endpoints: []string{cloudEndpoint},
found: nil,
want: []string{CloudHost},
},
{
// Dropping the cloud host here would leave no hosts at all and produce no results.
name: "cloud kept when the data named it",
endpoints: []string{cloudEndpoint, CloudHost},
found: []string{CloudHost},
want: []string{CloudHost},
},
{
name: "configured endpoints are reduced to hosts and deduped",
endpoints: []string{"https://jira.example.com:8443/", cloudEndpoint, "tenant.atlassian.net", "https://tenant.atlassian.net", ""},
found: []string{"tenant.atlassian.net"},
want: []string{"jira.example.com:8443", "tenant.atlassian.net"},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
if diff := cmp.Diff(test.want, VerificationHosts(test.endpoints, test.found)); diff != "" {
t.Errorf("VerificationHosts() diff: (-want +got)\n%s", diff)
}
})
}
}
func TestJiraToken_Verify(t *testing.T) {
tests := []struct {
name string
statusCode int
body string
wantVerified bool
wantErr bool
}{
{
name: "authenticated user in body is verified",
@@ -116,14 +239,48 @@ func TestJiraToken_Verify(t *testing.T) {
body: `{"code":401,"message":"Unauthorized"}`,
wantVerified: false,
},
{
name: "forbidden is not verified",
statusCode: 403,
body: `{"code":403,"message":"Forbidden"}`,
wantVerified: false,
},
{
// Atlassian never answers 202 here. It comes from unrelated hosts that accept the POST, and it
// would repeat on every retry, so it has to be terminal rather than an indeterminate error.
name: "accepted is not verified and not an error",
statusCode: 202,
body: `{"status":"accepted"}`,
wantVerified: false,
},
{
name: "not found is not verified and not an error",
statusCode: 404,
body: `{"code":404,"message":"Not Found"}`,
wantVerified: false,
},
{
name: "rate limited is indeterminate",
statusCode: 429,
body: `{"code":429,"message":"Too Many Requests"}`,
wantVerified: false,
wantErr: true,
},
{
name: "server error is indeterminate",
statusCode: 500,
body: `{"code":500,"message":"Internal Server Error"}`,
wantVerified: false,
wantErr: true,
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
client := common.ConstantResponseHttpClient(test.statusCode, test.body)
verified, err := VerifyJiraToken(context.Background(), client, "[email protected]", "example.atlassian.net", validTokenPattern)
if err != nil {
t.Fatalf("unexpected error: %v", err)
if (err != nil) != test.wantErr {
t.Fatalf("got error = %v, wantErr %v", err, test.wantErr)
}
if verified != test.wantVerified {
t.Errorf("got verified = %v, want %v", verified, test.wantVerified)
pkg/detectors/jiratoken/v2/jiratoken_v2.go
+17 -7
View File
@@ -17,14 +17,19 @@ import (
type Scanner struct {
client *http.Client
detectors.DefaultMultiPartCredentialProvider
detectors.EndpointSetter
}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var _ detectors.Versioner = (*Scanner)(nil)
var _ detectors.EndpointCustomizer = (*Scanner)(nil)
var _ detectors.CloudProvider = (*Scanner)(nil)
func (Scanner) Version() int { return 2 }
func (Scanner) CloudEndpoint() string { return "https://" + v1.CloudHost }
var (
defaultClient = detectors.DetectorHttpClientWithLocalAddresses
@@ -66,19 +71,17 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
uniqueEmails[strings.ToLower(email[1])] = struct{}{}
}
if len(uniqueDomains) == 0 {
// reason: https://community.atlassian.com/forums/Jira-Product-Discovery-questions/Authorization-issues-with-GRAPHQL/qaq-p/2640943
// In case we don't find any domain matches we can use this as the graphql API works with this domain if our authentication is valid
uniqueDomains["api.atlassian.com"] = struct{}{}
}
// domainPat here is not even keyword-gated, so it captures every domain in the chunk.
found := v1.FoundHosts(uniqueDomains)
hosts := v1.VerificationHosts(s.Endpoints(found...), found)
for email := range uniqueEmails {
for token := range uniqueTokens {
for domain := range uniqueDomains {
for _, domain := range hosts {
s1 := detectors.Result{
DetectorType: detector_typepb.DetectorType_JiraToken,
Raw: []byte(token),
RawV2: []byte(fmt.Sprintf("%s:%s:%s", email, token, domain)),
RawV2: fmt.Appendf(nil, "%s:%s:%s", email, token, domain),
ExtraData: map[string]string{
"rotation_guide": "https://howtorotate.com/docs/tutorials/atlassian/",
"version": fmt.Sprintf("%d", s.Version()),
@@ -95,6 +98,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
isVerified, verificationErr := v1.VerifyJiraToken(ctx, client, email, domain, token)
s1.Verified = isVerified
s1.SetVerificationError(verificationErr, token)
// The credential is the same across endpoints, so once one of them verifies it there
// is nothing to learn from the rest.
if s1.Verified {
results = append(results, s1)
break
}
}
results = append(results, s1)
pkg/detectors/jiratoken/v2/jiratoken_v2_integration_test.go
+6
View File
@@ -141,6 +141,9 @@ func TestJiraToken_FromChunk(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
tt.s.SetCloudEndpoint(tt.s.CloudEndpoint())
tt.s.UseCloudEndpoint(true)
tt.s.UseFoundEndpoints(true)
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
if (err != nil) != tt.wantErr {
t.Errorf("JiraToken.FromData() error = %v, wantErr %v", err, tt.wantErr)
@@ -165,6 +168,9 @@ func TestJiraToken_FromChunk(t *testing.T) {
func BenchmarkFromData(benchmark *testing.B) {
ctx := context.Background()
s := Scanner{}
s.SetCloudEndpoint(s.CloudEndpoint())
s.UseCloudEndpoint(true)
s.UseFoundEndpoints(true)
for name, data := range detectors.MustGetBenchmarkData() {
benchmark.Run(name, func(b *testing.B) {
b.ResetTimer()
pkg/detectors/jiratoken/v2/jiratoken_v2_test.go
+3
View File
@@ -12,6 +12,9 @@ import (
func TestJiraToken_Pattern(t *testing.T) {
d := Scanner{}
d.SetCloudEndpoint(d.CloudEndpoint())
d.UseCloudEndpoint(true)
d.UseFoundEndpoints(true)
ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
name string