[INS-455] Unify common logic in Atlassian Data Center detectors (#4907)
* unify common logic in atlassian data center detectors * initialize url pat once * engine_test fix * remove bitbucketdatacenter from defaults test exclude list
This commit is contained in:
+6
-42
@@ -3,13 +3,13 @@ package bitbucketdatacenter
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
regexp "github.com/wasilibs/go-re2"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/atlassiandatacenter"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb"
|
||||
)
|
||||
|
||||
@@ -30,8 +30,7 @@ var (
|
||||
// and are usually between the length of 40-50 character
|
||||
// consisting of both alphanumeric and some special character like +, _, @ and etc
|
||||
userPat = regexp.MustCompile(`\b(BBDC-[A-Za-z0-9+/@_-]{40,50})(?:[^A-Za-z0-9+/@_-]|$)`)
|
||||
|
||||
urlPat = regexp.MustCompile(detectors.PrefixRegex([]string{"atlassian", "bitbucket"}) + `(https?://[a-zA-Z0-9.-]+(?::\d+)?)`)
|
||||
urlPat = atlassiandatacenter.GetURLPat([]string{"atlassian", "bitbucket"})
|
||||
)
|
||||
|
||||
func (s Scanner) Keywords() []string {
|
||||
@@ -52,24 +51,11 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
return results, nil
|
||||
}
|
||||
|
||||
foundURLs := make(map[string]struct{})
|
||||
for _, match := range urlPat.FindAllStringSubmatch(dataStr, -1) {
|
||||
foundURLs[match[1]] = struct{}{}
|
||||
}
|
||||
|
||||
endpoints := make([]string, 0, len(foundURLs))
|
||||
for endpoint := range foundURLs {
|
||||
endpoints = append(endpoints, endpoint)
|
||||
}
|
||||
|
||||
var uniqueUrls = make(map[string]struct{})
|
||||
for _, endpoint := range s.Endpoints(endpoints...) {
|
||||
uniqueUrls[endpoint] = struct{}{}
|
||||
}
|
||||
endpoints := atlassiandatacenter.FindEndpoints(dataStr, urlPat, s.Endpoints)
|
||||
|
||||
// create combination results that can be verified
|
||||
for secret := range uniqueSecretPat {
|
||||
for bitBucketURL := range uniqueUrls {
|
||||
for _, bitBucketURL := range endpoints {
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detector_typepb.DetectorType_BitbucketDataCenter,
|
||||
Raw: []byte(secret),
|
||||
@@ -109,30 +95,8 @@ func verifyMatch(ctx context.Context, client *http.Client, secretPat, baseURL st
|
||||
q.Set("limit", "1")
|
||||
u.RawQuery = q.Encode()
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), http.NoBody)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
req.Header.Set("Accept", "application/json")
|
||||
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", secretPat))
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
defer func() {
|
||||
_, _ = io.Copy(io.Discard, resp.Body)
|
||||
_ = resp.Body.Close()
|
||||
}()
|
||||
|
||||
switch resp.StatusCode {
|
||||
case http.StatusOK:
|
||||
return true, nil
|
||||
case http.StatusUnauthorized:
|
||||
return false, nil
|
||||
default:
|
||||
return false, fmt.Errorf("unexpected HTTP response status %d", resp.StatusCode)
|
||||
}
|
||||
isVerified, _, err := atlassiandatacenter.MakeVerifyRequest(ctx, client, u.String(), secretPat)
|
||||
return isVerified, err
|
||||
}
|
||||
|
||||
func (s Scanner) Type() detector_typepb.DetectorType {
|
||||
@@ -0,0 +1,124 @@
|
||||
package atlassiandatacenter
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
regexp "github.com/wasilibs/go-re2"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
)
|
||||
|
||||
// GetDCTokenPat returns a compiled regex that matches Atlassian Data Center PATs
|
||||
// (Jira DC and Confluence DC style) scoped to the given keyword prefixes.
|
||||
//
|
||||
// PATs are 44-char base64 strings decoding to "<numeric-id>:<random-bytes>".
|
||||
// The first character is always M, N, or O because the numeric ID begins with
|
||||
// an ASCII digit (0x30–0x39). The trailing boundary prevents matching substrings
|
||||
// of longer base64 strings or base64-padded tokens.
|
||||
//
|
||||
// This does not apply to Bitbucket DC tokens, which use a BBDC- prefix format.
|
||||
func GetDCTokenPat(prefixes []string) *regexp.Regexp {
|
||||
return regexp.MustCompile(
|
||||
detectors.PrefixRegex(prefixes) + `\b([MNO][A-Za-z0-9+/]{43})(?:[^A-Za-z0-9+/=]|\z)`,
|
||||
)
|
||||
}
|
||||
|
||||
// GetURLPat returns a compiled regex that matches self-hosted Atlassian instance
|
||||
// URLs (scheme + alphanumeric-starting host + optional port up to 5 digits),
|
||||
// scoped to the given keyword prefixes. Callers should store the result in a
|
||||
// package-level var so the regex is compiled once at init time rather than per chunk.
|
||||
func GetURLPat(prefixes []string) *regexp.Regexp {
|
||||
return regexp.MustCompile(detectors.PrefixRegex(prefixes) + `(https?://[a-zA-Z0-9][a-zA-Z0-9.\-]*(?::\d{1,5})?)`)
|
||||
}
|
||||
|
||||
// FindEndpoints extracts all URLs matching urlPat from data, passes them through
|
||||
// the resolve function (typically s.Endpoints), deduplicates the results, and
|
||||
// returns them as a slice with trailing slashes stripped.
|
||||
func FindEndpoints(data string, urlPat *regexp.Regexp, resolve func(...string) []string) []string {
|
||||
seen := make(map[string]struct{})
|
||||
for _, m := range urlPat.FindAllStringSubmatch(data, -1) {
|
||||
seen[m[1]] = struct{}{}
|
||||
}
|
||||
|
||||
raw := make([]string, 0, len(seen))
|
||||
for u := range seen {
|
||||
raw = append(raw, u)
|
||||
}
|
||||
|
||||
resolved := make(map[string]struct{})
|
||||
for _, u := range resolve(raw...) {
|
||||
resolved[strings.TrimRight(u, "/")] = struct{}{}
|
||||
}
|
||||
|
||||
result := make([]string, 0, len(resolved))
|
||||
for u := range resolved {
|
||||
result = append(result, u)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// IsStructuralPAT decodes a candidate base64 string and checks that it matches
|
||||
// the "<numeric id>:<random bytes>" structure used by Jira and Confluence DC PATs:
|
||||
// one or more ASCII digits, a colon, then at least one more byte.
|
||||
func IsStructuralPAT(candidate string) bool {
|
||||
raw, err := base64.StdEncoding.DecodeString(candidate)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
colon := bytes.IndexByte(raw, ':')
|
||||
if colon <= 0 || colon == len(raw)-1 {
|
||||
return false
|
||||
}
|
||||
for _, b := range raw[:colon] {
|
||||
if b < '0' || b > '9' {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// MakeVerifyRequest sends a Bearer-authenticated GET request to fullURL and
|
||||
// interprets the response:
|
||||
// - 200: returns (true, decoded JSON body as map or nil if unparseable, nil)
|
||||
// - 401: returns (false, nil, nil)
|
||||
// - other: returns (false, nil, error describing the unexpected status)
|
||||
//
|
||||
// A non-nil error is also returned for network failures.
|
||||
// Callers that need fields from the response body (e.g. display name, email)
|
||||
// can read them from the returned map; callers that don't need the body can
|
||||
// ignore it.
|
||||
func MakeVerifyRequest(ctx context.Context, client *http.Client, fullURL, token string) (bool, map[string]any, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, fullURL, http.NoBody)
|
||||
if err != nil {
|
||||
return false, nil, err
|
||||
}
|
||||
req.Header.Set("Accept", "application/json")
|
||||
req.Header.Set("Authorization", "Bearer "+token)
|
||||
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return false, nil, err
|
||||
}
|
||||
defer func() {
|
||||
_, _ = io.Copy(io.Discard, resp.Body)
|
||||
_ = resp.Body.Close()
|
||||
}()
|
||||
|
||||
switch resp.StatusCode {
|
||||
case http.StatusOK:
|
||||
var body map[string]any
|
||||
_ = json.NewDecoder(resp.Body).Decode(&body)
|
||||
return true, body, nil
|
||||
case http.StatusUnauthorized:
|
||||
return false, nil, nil
|
||||
default:
|
||||
return false, nil, fmt.Errorf("unexpected HTTP response status %d", resp.StatusCode)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,305 @@
|
||||
package atlassiandatacenter
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"net/http"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"gopkg.in/h2non/gock.v1"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
)
|
||||
|
||||
// Real-format sample PATs that decode to "<numeric id>:<random bytes>".
|
||||
const (
|
||||
// Jira DC sample.
|
||||
jiraToken = "NTg4OTI1Mzk1OTA1OiBb9S4WPEoK6cmOe6pq6VO0lt6M"
|
||||
|
||||
// Confluence DC samples.
|
||||
confluenceToken1 = "NTk3MjQzOTIyNTAwOtFOuTsHRIp1E81GApKpC2xpEzfz"
|
||||
confluenceToken2 = "NDc4MjM3OTUxMzk2OopoSkTDTnBcWIw0Wa4bico9zOLK"
|
||||
|
||||
// 44-char base64 that starts with [MNO] (passes the regex) but decodes
|
||||
// to bytes with no colon — must be rejected by IsStructuralPAT.
|
||||
nonStructural = "MAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"
|
||||
)
|
||||
|
||||
func encode(b []byte) string { return base64.StdEncoding.EncodeToString(b) }
|
||||
|
||||
func TestIsStructuralPAT(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
candidate string
|
||||
want bool
|
||||
}{
|
||||
{
|
||||
name: "valid real Jira token",
|
||||
candidate: jiraToken,
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "valid real Confluence token 1",
|
||||
candidate: confluenceToken1,
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "valid real Confluence token 2",
|
||||
candidate: confluenceToken2,
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "valid - digits before colon",
|
||||
candidate: encode([]byte("123456789012:01234567890123456789")),
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "invalid base64",
|
||||
candidate: "!!!not-base64!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!",
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "no colon in decoded bytes",
|
||||
candidate: encode([]byte("588925395905012345678901234567890")),
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "colon at position 0",
|
||||
candidate: encode([]byte(":01234567890123456789012345678901")),
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "colon at last position",
|
||||
candidate: encode([]byte("58892539590501234567890123456789:")),
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
// decodes to "0a:xxx..." — 'a' is not a digit
|
||||
name: "non-digit before colon",
|
||||
candidate: "MGE6eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4",
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "non-structural: passes regex but no colon in decoded bytes",
|
||||
candidate: nonStructural,
|
||||
want: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
assert.Equal(t, tt.want, IsStructuralPAT(tt.candidate))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetDCTokenPat(t *testing.T) {
|
||||
pat := GetDCTokenPat([]string{"jira", "atlassian"})
|
||||
|
||||
// Known-valid token — must match and capture exactly the token (no trailing char).
|
||||
m := pat.FindStringSubmatch("jira token: " + jiraToken)
|
||||
require.NotNil(t, m, "expected a match")
|
||||
assert.Equal(t, jiraToken, m[1], "captured group should be exactly the token")
|
||||
|
||||
// Token followed by newline — must match and NOT capture the newline.
|
||||
m2 := pat.FindStringSubmatch("jira token: " + jiraToken + "\n")
|
||||
require.NotNil(t, m2)
|
||||
assert.Equal(t, jiraToken, m2[1], "captured group must not include the trailing newline")
|
||||
|
||||
// Token starts with 'A' — not M/N/O — must not match.
|
||||
assert.Nil(t, pat.FindStringSubmatch("jira token: ATg4OTI1Mzk1OTA1OiBb9S4WPEoK6cmOe6pq6VO0lt6M"))
|
||||
|
||||
// Token followed by base64 padding — trailing boundary must reject it.
|
||||
assert.Nil(t, pat.FindStringSubmatch("jira token: "+jiraToken+"="))
|
||||
|
||||
// Token followed by more base64 chars — must not match (longer string).
|
||||
assert.Nil(t, pat.FindStringSubmatch("jira token: "+jiraToken+"AAAA"))
|
||||
}
|
||||
|
||||
func TestFindEndpoints(t *testing.T) {
|
||||
urlPat := GetURLPat([]string{"jira", "atlassian"})
|
||||
|
||||
// identity resolver: returns exactly what it receives (simulates UseFoundEndpoints only)
|
||||
identity := func(urls ...string) []string { return urls }
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
data string
|
||||
resolve func(...string) []string
|
||||
want []string
|
||||
}{
|
||||
{
|
||||
name: "URL near keyword is returned",
|
||||
data: "jira url: https://jira.corp.com",
|
||||
resolve: identity,
|
||||
want: []string{"https://jira.corp.com"},
|
||||
},
|
||||
{
|
||||
name: "URL not near any keyword is ignored",
|
||||
data: "unrelated url: https://example.com",
|
||||
resolve: identity,
|
||||
want: []string{},
|
||||
},
|
||||
{
|
||||
name: "duplicate URLs in data are deduplicated",
|
||||
data: "jira: https://jira.corp.com\natlassian: https://jira.corp.com",
|
||||
resolve: identity,
|
||||
want: []string{"https://jira.corp.com"},
|
||||
},
|
||||
{
|
||||
name: "trailing slash is stripped",
|
||||
data: "jira url: https://jira.corp.com/",
|
||||
resolve: identity,
|
||||
want: []string{"https://jira.corp.com"},
|
||||
},
|
||||
{
|
||||
name: "URL with port is accepted",
|
||||
data: "jira url: https://jira.corp.com:8443",
|
||||
resolve: identity,
|
||||
want: []string{"https://jira.corp.com:8443"},
|
||||
},
|
||||
{
|
||||
name: "multiple distinct URLs are all returned",
|
||||
data: "jira prod: https://jira.prod.com\natlassian staging: https://jira.staging.com",
|
||||
resolve: identity,
|
||||
want: []string{"https://jira.prod.com", "https://jira.staging.com"},
|
||||
},
|
||||
{
|
||||
name: "resolve can inject configured endpoints not in data",
|
||||
data: "no urls here but jira keyword present",
|
||||
resolve: func(urls ...string) []string {
|
||||
return append(urls, "https://configured.jira.com")
|
||||
},
|
||||
want: []string{"https://configured.jira.com"},
|
||||
},
|
||||
{
|
||||
name: "resolve can filter out URLs",
|
||||
data: "jira url: https://jira.corp.com",
|
||||
resolve: func(urls ...string) []string {
|
||||
return []string{} // simulate UseFoundEndpoints(false) with no configured endpoint
|
||||
},
|
||||
want: []string{},
|
||||
},
|
||||
{
|
||||
name: "no data returns empty slice",
|
||||
data: "",
|
||||
resolve: identity,
|
||||
want: []string{},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := FindEndpoints(tt.data, urlPat, tt.resolve)
|
||||
assert.ElementsMatch(t, tt.want, got)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestMakeVerifyRequest(t *testing.T) {
|
||||
const testURL = "http://dc.example.com/rest/api/test"
|
||||
const testToken = "NTg4OTI1Mzk1OTA1OiBb9S4WPEoK6cmOe6pq6VO0lt6M"
|
||||
|
||||
t.Run("200: verified=true, body decoded", func(t *testing.T) {
|
||||
client := common.SaneHttpClient()
|
||||
defer gock.Off()
|
||||
defer gock.RestoreClient(client)
|
||||
gock.InterceptClient(client)
|
||||
|
||||
gock.New("http://dc.example.com").
|
||||
Get("/rest/api/test").
|
||||
MatchHeader("Authorization", "Bearer "+testToken).
|
||||
MatchHeader("Accept", "application/json").
|
||||
Reply(http.StatusOK).
|
||||
JSON(map[string]any{"displayName": "Alice", "emailAddress": "[email protected]"})
|
||||
|
||||
verified, body, err := MakeVerifyRequest(context.Background(), client, testURL, testToken)
|
||||
require.NoError(t, err)
|
||||
assert.True(t, verified)
|
||||
require.NotNil(t, body)
|
||||
assert.Equal(t, "Alice", body["displayName"])
|
||||
assert.Equal(t, "[email protected]", body["emailAddress"])
|
||||
})
|
||||
|
||||
t.Run("200: verified=true, body nil when response is not JSON", func(t *testing.T) {
|
||||
client := common.SaneHttpClient()
|
||||
defer gock.Off()
|
||||
defer gock.RestoreClient(client)
|
||||
gock.InterceptClient(client)
|
||||
|
||||
gock.New("http://dc.example.com").
|
||||
Get("/rest/api/test").
|
||||
Reply(http.StatusOK).
|
||||
BodyString("not json")
|
||||
|
||||
verified, body, err := MakeVerifyRequest(context.Background(), client, testURL, testToken)
|
||||
require.NoError(t, err)
|
||||
assert.True(t, verified)
|
||||
assert.Nil(t, body)
|
||||
})
|
||||
|
||||
t.Run("401: verified=false, no error", func(t *testing.T) {
|
||||
client := common.SaneHttpClient()
|
||||
defer gock.Off()
|
||||
defer gock.RestoreClient(client)
|
||||
gock.InterceptClient(client)
|
||||
|
||||
gock.New("http://dc.example.com").
|
||||
Get("/rest/api/test").
|
||||
Reply(http.StatusUnauthorized)
|
||||
|
||||
verified, body, err := MakeVerifyRequest(context.Background(), client, testURL, testToken)
|
||||
require.NoError(t, err)
|
||||
assert.False(t, verified)
|
||||
assert.Nil(t, body)
|
||||
})
|
||||
|
||||
t.Run("unexpected status: verified=false, error returned", func(t *testing.T) {
|
||||
client := common.SaneHttpClient()
|
||||
defer gock.Off()
|
||||
defer gock.RestoreClient(client)
|
||||
gock.InterceptClient(client)
|
||||
|
||||
gock.New("http://dc.example.com").
|
||||
Get("/rest/api/test").
|
||||
Reply(http.StatusInternalServerError)
|
||||
|
||||
verified, body, err := MakeVerifyRequest(context.Background(), client, testURL, testToken)
|
||||
require.Error(t, err)
|
||||
assert.False(t, verified)
|
||||
assert.Nil(t, body)
|
||||
})
|
||||
|
||||
t.Run("uses GET method", func(t *testing.T) {
|
||||
client := common.SaneHttpClient()
|
||||
defer gock.Off()
|
||||
defer gock.RestoreClient(client)
|
||||
gock.InterceptClient(client)
|
||||
|
||||
// gock.Get only matches GET; a POST would not match and would error.
|
||||
gock.New("http://dc.example.com").
|
||||
Get("/rest/api/test").
|
||||
Reply(http.StatusOK)
|
||||
|
||||
_, _, err := MakeVerifyRequest(context.Background(), client, testURL, testToken)
|
||||
require.NoError(t, err)
|
||||
assert.True(t, gock.IsDone(), "gock interceptor was not matched — request may not have been GET")
|
||||
})
|
||||
|
||||
t.Run("propagates network error", func(t *testing.T) {
|
||||
client := common.SaneHttpClient()
|
||||
defer gock.Off()
|
||||
defer gock.RestoreClient(client)
|
||||
gock.InterceptClient(client)
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel() // cancel immediately to force a network error
|
||||
|
||||
verified, body, err := MakeVerifyRequest(ctx, client, testURL, testToken)
|
||||
assert.Error(t, err)
|
||||
assert.False(t, verified)
|
||||
assert.Nil(t, body)
|
||||
})
|
||||
}
|
||||
+10
-66
@@ -1,19 +1,15 @@
|
||||
package confluencedatacenter
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
regexp "github.com/wasilibs/go-re2"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/cache/simple"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/atlassiandatacenter"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb"
|
||||
)
|
||||
|
||||
@@ -35,12 +31,9 @@ var (
|
||||
|
||||
keywords = []string{"confluence", "atlassian", "wiki"}
|
||||
|
||||
// 44-char base64 PAT; decoded form must match the structural check below.
|
||||
tokenPat = regexp.MustCompile(detectors.PrefixRegex(keywords) + `\b([MNO][A-Za-z0-9+/]{43})(?:[^A-Za-z0-9+/=]|\z)`)
|
||||
|
||||
// Self-hosted instance URL: scheme + host + optional port. Keyword-scoped
|
||||
// so unrelated URLs in the same chunk don't get paired with tokens.
|
||||
urlPat = regexp.MustCompile(detectors.PrefixRegex(keywords) + `\b(https?://[a-zA-Z0-9.\-]+(?::\d+)?)\b`)
|
||||
// 44-char base64 PAT; decoded form must match the structural check in atlassiandatacenter.IsStructuralPAT.
|
||||
tokenPat = atlassiandatacenter.GetDCTokenPat(keywords)
|
||||
urlPat = atlassiandatacenter.GetURLPat(keywords)
|
||||
|
||||
invalidHosts = simple.NewCache[struct{}]()
|
||||
errNoHost = errors.New("no such host")
|
||||
@@ -65,26 +58,6 @@ func (s Scanner) getClient() *http.Client {
|
||||
return defaultClient
|
||||
}
|
||||
|
||||
// isStructuralPAT decodes a candidate base64 string and checks that it matches
|
||||
// the "<numeric id>:<random bytes>" structure used by Confluence DC PATs:
|
||||
// one or more ASCII digits, a colon, then at least one more byte.
|
||||
func isStructuralPAT(candidate string) bool {
|
||||
raw, err := base64.StdEncoding.DecodeString(candidate)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
colon := bytes.IndexByte(raw, ':')
|
||||
if colon <= 0 || colon == len(raw)-1 {
|
||||
return false
|
||||
}
|
||||
for _, b := range raw[:colon] {
|
||||
if b < '0' || b > '9' {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
@@ -93,7 +66,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
if _, seen := uniqueTokens[m[1]]; seen {
|
||||
continue
|
||||
}
|
||||
if isStructuralPAT(m[1]) {
|
||||
if atlassiandatacenter.IsStructuralPAT(m[1]) {
|
||||
uniqueTokens[m[1]] = struct{}{}
|
||||
}
|
||||
}
|
||||
@@ -101,20 +74,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
foundURLs := make([]string, 0)
|
||||
for _, m := range urlPat.FindAllStringSubmatch(dataStr, -1) {
|
||||
foundURLs = append(foundURLs, m[1])
|
||||
}
|
||||
uniqueURLs := make(map[string]struct{})
|
||||
for _, endpoint := range s.Endpoints(foundURLs...) {
|
||||
uniqueURLs[strings.TrimRight(endpoint, "/")] = struct{}{}
|
||||
}
|
||||
allURLs := atlassiandatacenter.FindEndpoints(dataStr, urlPat, s.Endpoints)
|
||||
|
||||
// Filter hosts cached as unreachable from prior calls once up front.
|
||||
// invalidHosts may also grow during this call (see the verify branch
|
||||
// below); those are skipped lazily inside the inner loop.
|
||||
liveURLs := make([]string, 0, len(uniqueURLs))
|
||||
for u := range uniqueURLs {
|
||||
liveURLs := make([]string, 0, len(allURLs))
|
||||
for _, u := range allURLs {
|
||||
if !invalidHosts.Exists(u) {
|
||||
liveURLs = append(liveURLs, u)
|
||||
}
|
||||
@@ -175,35 +141,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
|
||||
func verifyPAT(ctx context.Context, client *http.Client, baseURL, token string) (bool, error) {
|
||||
endpoint := strings.TrimRight(baseURL, "/") + "/rest/api/user/current"
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, http.NoBody)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
req.Header.Set("Accept", "application/json")
|
||||
req.Header.Set("Authorization", "Bearer "+token)
|
||||
|
||||
resp, err := client.Do(req)
|
||||
isVerified, _, err := atlassiandatacenter.MakeVerifyRequest(ctx, client, endpoint, token)
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "no such host") {
|
||||
return false, errNoHost
|
||||
}
|
||||
return false, err
|
||||
}
|
||||
defer func() {
|
||||
_, _ = io.Copy(io.Discard, resp.Body)
|
||||
_ = resp.Body.Close()
|
||||
}()
|
||||
|
||||
switch resp.StatusCode {
|
||||
case http.StatusOK:
|
||||
return true, nil
|
||||
case http.StatusUnauthorized:
|
||||
// Auth header outright rejected — unambiguously an invalid credential.
|
||||
return false, nil
|
||||
default:
|
||||
// 403 included: /rest/api/user/current should always be readable by a
|
||||
// valid PAT, so a Forbidden here signals something unexpected rather
|
||||
// than a definitively invalid token.
|
||||
return false, fmt.Errorf("unexpected HTTP response status %d", resp.StatusCode)
|
||||
}
|
||||
return isVerified, nil
|
||||
}
|
||||
+18
-79
@@ -1,17 +1,11 @@
|
||||
package jiradatacenterpat
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
|
||||
regexp "github.com/wasilibs/go-re2"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/atlassiandatacenter"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb"
|
||||
)
|
||||
|
||||
@@ -31,12 +25,14 @@ var (
|
||||
var (
|
||||
defaultClient = detectors.DetectorHttpClientWithNoLocalAddresses
|
||||
|
||||
keywords = []string{"jira", "atlassian"}
|
||||
|
||||
// PATs are base64-encoded strings of the form <12-digit-id>:<20-random-bytes> (33 bytes, 44 chars, no padding).
|
||||
// Since the first byte is always an ASCII digit (0x30–0x39), the first base64 character is always M, N, or O.
|
||||
// This is also verified by generating 25+ tokens.
|
||||
// The trailing boundary (?:[^A-Za-z0-9+/=]|\z) is used instead of \b to correctly handle tokens ending in + or /.
|
||||
patPat = regexp.MustCompile(detectors.PrefixRegex([]string{"jira", "atlassian"}) + `\b([MNO][A-Za-z0-9+/]{43})(?:[^A-Za-z0-9+/=]|\z)`)
|
||||
urlPat = regexp.MustCompile(detectors.PrefixRegex([]string{"jira", "atlassian"}) + `(https?://[A-Za-z0-9][A-Za-z0-9.\-]*(?::\d{1,5})?)`)
|
||||
patPat = atlassiandatacenter.GetDCTokenPat(keywords)
|
||||
urlPat = atlassiandatacenter.GetURLPat(keywords)
|
||||
)
|
||||
|
||||
func (s Scanner) getClient() *http.Client {
|
||||
@@ -48,7 +44,7 @@ func (s Scanner) getClient() *http.Client {
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"jira", "atlassian"}
|
||||
return keywords
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify Jira Data Center PAT secrets in a given set of bytes.
|
||||
@@ -57,23 +53,12 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
|
||||
tokens := make(map[string]struct{})
|
||||
for _, match := range patPat.FindAllStringSubmatch(dataStr, -1) {
|
||||
if isStructuralPAT(match[1]) {
|
||||
if atlassiandatacenter.IsStructuralPAT(match[1]) {
|
||||
tokens[match[1]] = struct{}{}
|
||||
}
|
||||
}
|
||||
|
||||
uniqueURLs := make(map[string]struct{})
|
||||
for _, match := range urlPat.FindAllStringSubmatch(dataStr, -1) {
|
||||
uniqueURLs[match[1]] = struct{}{}
|
||||
}
|
||||
foundURLs := make([]string, 0, len(uniqueURLs))
|
||||
for url := range uniqueURLs {
|
||||
foundURLs = append(foundURLs, url)
|
||||
}
|
||||
endpoints := make(map[string]struct{})
|
||||
for _, endpoint := range s.Endpoints(foundURLs...) {
|
||||
endpoints[endpoint] = struct{}{}
|
||||
}
|
||||
endpoints := atlassiandatacenter.FindEndpoints(dataStr, urlPat, s.Endpoints)
|
||||
|
||||
for token := range tokens {
|
||||
if len(endpoints) == 0 {
|
||||
@@ -87,7 +72,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
continue
|
||||
}
|
||||
|
||||
for endpoint := range endpoints {
|
||||
for _, endpoint := range endpoints {
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detector_typepb.DetectorType_JiraDataCenterPAT,
|
||||
Raw: []byte(token),
|
||||
@@ -127,65 +112,19 @@ func verifyPAT(ctx context.Context, client *http.Client, baseURL, token string)
|
||||
}
|
||||
u.Path = "/rest/api/2/myself"
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), http.NoBody)
|
||||
if err != nil {
|
||||
return false, nil, err
|
||||
isVerified, body, err := atlassiandatacenter.MakeVerifyRequest(ctx, client, u.String(), token)
|
||||
if err != nil || !isVerified {
|
||||
return isVerified, nil, err
|
||||
}
|
||||
|
||||
req.Header.Set("Accept", "application/json")
|
||||
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token))
|
||||
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return false, nil, err
|
||||
extraData := map[string]string{"endpoint": baseURL}
|
||||
if name, ok := body["displayName"].(string); ok {
|
||||
extraData["display_name"] = name
|
||||
}
|
||||
defer func() {
|
||||
_, _ = io.Copy(io.Discard, resp.Body)
|
||||
_ = resp.Body.Close()
|
||||
}()
|
||||
|
||||
switch resp.StatusCode {
|
||||
case http.StatusOK:
|
||||
var result map[string]any
|
||||
extraData := map[string]string{
|
||||
"endpoint": baseURL,
|
||||
}
|
||||
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
|
||||
// 200 confirms the token is valid; failing to decode only means we can't extract extra data.
|
||||
return true, extraData, nil
|
||||
}
|
||||
if name, ok := result["displayName"].(string); ok {
|
||||
extraData["display_name"] = name
|
||||
}
|
||||
if email, ok := result["emailAddress"].(string); ok {
|
||||
extraData["email_address"] = email
|
||||
}
|
||||
return true, extraData, nil
|
||||
case http.StatusUnauthorized:
|
||||
return false, nil, nil
|
||||
default:
|
||||
return false, nil, fmt.Errorf("unexpected HTTP response status %d", resp.StatusCode)
|
||||
if email, ok := body["emailAddress"].(string); ok {
|
||||
extraData["email_address"] = email
|
||||
}
|
||||
}
|
||||
|
||||
// isStructuralPAT decodes a candidate base64 string and checks that it matches
|
||||
// the "<numeric id>:<random bytes>" structure used by Jira DC PATs:
|
||||
// one or more ASCII digits, a colon, then at least one more byte.
|
||||
func isStructuralPAT(candidate string) bool {
|
||||
raw, err := base64.StdEncoding.DecodeString(candidate)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
colon := bytes.IndexByte(raw, ':')
|
||||
if colon <= 0 || colon == len(raw)-1 {
|
||||
return false
|
||||
}
|
||||
for _, b := range raw[:colon] {
|
||||
if b < '0' || b > '9' {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
return true, extraData, nil
|
||||
}
|
||||
|
||||
func (s Scanner) Type() detector_typepb.DetectorType {
|
||||
-59
@@ -2,7 +2,6 @@ package jiradatacenterpat
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"testing"
|
||||
@@ -275,64 +274,6 @@ func TestJiraDataCenterPAT_FromData(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsStructuralPAT(t *testing.T) {
|
||||
encode := func(b []byte) string { return base64.StdEncoding.EncodeToString(b) }
|
||||
|
||||
// helper to build a 33-byte payload with a numeric id and random suffix
|
||||
numericIDPayload := func(id, suffix string) []byte {
|
||||
return []byte(id + ":" + suffix)
|
||||
}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
candidate string
|
||||
want bool
|
||||
}{
|
||||
{
|
||||
name: "valid real token",
|
||||
candidate: testToken,
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "valid - digits before colon",
|
||||
candidate: encode(numericIDPayload("123456789012", "01234567890123456789")),
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "invalid base64",
|
||||
candidate: "!!!not-base64!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!",
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "no colon",
|
||||
candidate: encode([]byte("588925395905012345678901234567890")),
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "colon at position 0",
|
||||
candidate: encode([]byte(":01234567890123456789012345678901")),
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "colon at last position",
|
||||
candidate: encode([]byte("58892539590501234567890123456789:")),
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "non-digit before colon",
|
||||
// decodes to "0a:xxx..." — 'a' is not a digit
|
||||
candidate: "MGE6eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4",
|
||||
want: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
assert.Equal(t, tt.want, isStructuralPAT(tt.candidate))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestJiraDataCenterPAT_NoURL(t *testing.T) {
|
||||
d := Scanner{client: common.SaneHttpClient()}
|
||||
|
||||
@@ -95,6 +95,7 @@ import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/billomat"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/bingsubscriptionkey"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/bitbar"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/atlassiandatacenter/bitbucketdatacenter"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/bitbucketapppassword"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/bitcoinaverage"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/bitfinex"
|
||||
@@ -186,7 +187,7 @@ import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/commercejs"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/commodities"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/companyhub"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/confluencedatacenter"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/atlassiandatacenter/confluencedatacenter"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/confluent"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/contentfulpersonalaccesstoken"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/conversiontools"
|
||||
@@ -399,7 +400,7 @@ import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/ipquality"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/ipstack"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/jdbc"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/jiradatacenterpat"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/atlassiandatacenter/jiradatacenterpat"
|
||||
jiratokenv1 "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/jiratoken/v1"
|
||||
jiratokenv2 "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/jiratoken/v2"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/jotform"
|
||||
@@ -968,6 +969,7 @@ func buildDetectorList() []detectors.Detector {
|
||||
&billomat.Scanner{},
|
||||
&bingsubscriptionkey.Scanner{},
|
||||
&bitbar.Scanner{},
|
||||
&bitbucketdatacenter.Scanner{},
|
||||
&bitbucketapppassword.Scanner{},
|
||||
&bitcoinaverage.Scanner{},
|
||||
&bitfinex.Scanner{},
|
||||
|
||||
@@ -116,16 +116,15 @@ var excludedFromDefaultList = map[detector_typepb.DetectorType]struct{}{
|
||||
// to buildDetectorList() — discovered by TestAllDetectorTypesAreInDefaultList.
|
||||
// They are not added immediately out of caution for the impact on customers/users.
|
||||
// Remove each entry once its detector has been carefully added.
|
||||
detector_typepb.DetectorType_BitbucketDataCenter: {},
|
||||
detector_typepb.DetectorType_DatadogApikey: {},
|
||||
detector_typepb.DetectorType_Guru: {},
|
||||
detector_typepb.DetectorType_IPInfo: {},
|
||||
detector_typepb.DetectorType_Lob: {},
|
||||
detector_typepb.DetectorType_Rev: {},
|
||||
detector_typepb.DetectorType_TLy: {},
|
||||
detector_typepb.DetectorType_Tru: {},
|
||||
detector_typepb.DetectorType_User: {},
|
||||
detector_typepb.DetectorType_Wit: {},
|
||||
detector_typepb.DetectorType_DatadogApikey: {},
|
||||
detector_typepb.DetectorType_Guru: {},
|
||||
detector_typepb.DetectorType_IPInfo: {},
|
||||
detector_typepb.DetectorType_Lob: {},
|
||||
detector_typepb.DetectorType_Rev: {},
|
||||
detector_typepb.DetectorType_TLy: {},
|
||||
detector_typepb.DetectorType_Tru: {},
|
||||
detector_typepb.DetectorType_User: {},
|
||||
detector_typepb.DetectorType_Wit: {},
|
||||
|
||||
// Reserved / special types.
|
||||
detector_typepb.DetectorType_CustomRegex: {}, // added dynamically via engine config, not via buildDetectorList()
|
||||
|
||||
@@ -1381,6 +1381,7 @@ func TestEngineInitializesCloudProviderDetectors(t *testing.T) {
|
||||
detector_typepb.DetectorType_HashiCorpVaultAuth: {},
|
||||
detector_typepb.DetectorType_JiraDataCenterPAT: {},
|
||||
detector_typepb.DetectorType_ConfluenceDataCenter: {},
|
||||
detector_typepb.DetectorType_BitbucketDataCenter: {},
|
||||
// these do not have any cloud endpoint
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user