trufflesecurity/trufflehog
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Enabled and Enhanced Artifactory Detector (#4207)

Browse Source 
* enhanced artifactory detector

* Enabled and enhanced artifactory detector

* fixed engine test

* removed noncapturing group
This commit is contained in:
Kashif Khan
2025-06-25 16:34:44 +05:00
committed by GitHub
parent 5eb357dd1c
commit 9d5669eae2
5 changed files with 138 additions and 78 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/detectors/artifactory/artifactory.go
+71 -23
View File
@@ -2,12 +2,15 @@ package artifactory
import (
"context"
"errors"
"fmt"
"io"
"net/http"
"strings"
regexp "github.com/wasilibs/go-re2"
"github.com/trufflesecurity/trufflehog/v3/pkg/cache/simple"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
@@ -26,14 +29,20 @@ var (
defaultClient = detectors.DetectorHttpClientWithNoLocalAddresses
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(`\b([a-zA-Z0-9]{73}|\b[a-zA-Z0-9]{64})`)
URLPat = regexp.MustCompile(`\b([A-Za-z0-9](?:[A-Za-z0-9\-]{0,61}[A-Za-z0-9])\.jfrog\.io)`)
keyPat = regexp.MustCompile(`\b([a-zA-Z0-9]{64,73})\b`)
URLPat = regexp.MustCompile(`\b([A-Za-z0-9][A-Za-z0-9\-]{0,61}[A-Za-z0-9]\.jfrog\.io)`)
invalidHosts = simple.NewCache[struct{}]()
errNoHost = errors.New("no such host")
)
func (Scanner) CloudEndpoint() string { return "" }
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"artifactory"}
return []string{"artifactory", "jfrog.io"}
}
func (s Scanner) getClient() *http.Client {
@@ -46,30 +55,50 @@ func (s Scanner) getClient() *http.Client {
// FromData will find and optionally verify Artifactory secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
URLmatches := URLPat.FindAllStringSubmatch(dataStr, -1)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
resURLMatch := ""
for _, URLmatch := range URLmatches {
resURLMatch = strings.TrimSpace(URLmatch[1])
var uniqueTokens, uniqueUrls = make(map[string]struct{}), make(map[string]struct{})
for _, match := range keyPat.FindAllStringSubmatch(dataStr, -1) {
uniqueTokens[match[1]] = struct{}{}
}
for _, match := range matches {
resMatch := strings.TrimSpace(match[1])
var foundUrls = make([]string, 0)
client := s.getClient()
for _, match := range URLPat.FindAllStringSubmatch(dataStr, -1) {
foundUrls = append(foundUrls, match[1])
}
// add found + configured endpoints to the list
for _, endpoint := range s.Endpoints(foundUrls...) {
// if any configured endpoint has `https://` remove it because we append that during verification
endpoint = strings.TrimPrefix(endpoint, "https://")
uniqueUrls[endpoint] = struct{}{}
}
for token := range uniqueTokens {
for url := range uniqueUrls {
if invalidHosts.Exists(url) {
delete(uniqueUrls, url)
continue
}
for _, URL := range s.Endpoints(resURLMatch) {
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_ArtifactoryAccessToken,
Raw: []byte(resMatch),
RawV2: []byte(resMatch + URL),
Raw: []byte(token),
RawV2: []byte(token + url),
}
if verify {
isVerified, verificationErr := verifyArtifactory(ctx, client, URL, resMatch)
isVerified, verificationErr := verifyArtifactory(ctx, s.getClient(), url, token)
s1.Verified = isVerified
s1.SetVerificationError(verificationErr, resMatch)
if verificationErr != nil {
if errors.Is(verificationErr, errNoHost) {
invalidHosts.Set(url, struct{}{})
continue
}
s1.SetVerificationError(verificationErr, token)
}
}
results = append(results, s1)
@@ -81,26 +110,45 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
}
func verifyArtifactory(ctx context.Context, client *http.Client, resURLMatch, resMatch string) (bool, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://"+resURLMatch+"/artifactory/api/storageinfo", nil)
req, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://"+resURLMatch+"/artifactory/api/system/ping", nil)
if err != nil {
return false, err
}
req.Header.Add("X-JFrog-Art-Api", resMatch)
res, err := client.Do(req)
resp, err := client.Do(req)
if err != nil {
// lookup foo.jfrog.io: no such host
if strings.Contains(err.Error(), "no such host") {
return false, errNoHost
}
return false, err
}
defer res.Body.Close()
switch res.StatusCode {
defer func() {
_, _ = io.Copy(io.Discard, resp.Body)
_ = resp.Body.Close()
}()
switch resp.StatusCode {
case http.StatusOK:
return true, nil
case http.StatusForbidden:
body, err := io.ReadAll(resp.Body)
if err != nil {
return false, err
}
if strings.Contains(string(body), "OK") {
return true, nil
}
return false, nil
case http.StatusUnauthorized, http.StatusForbidden, http.StatusFound: // 302 can occur if the url is incorrect
// https://jfrog.com/help/r/jfrog-rest-apis/error-responses
return false, nil
default:
return false, fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
return false, fmt.Errorf("unexpected HTTP response status %d", resp.StatusCode)
}
}
pkg/detectors/artifactory/artifactory_integration_test.go
+3 -37
View File
@@ -56,42 +56,6 @@ func TestArtifactory_FromChunk(t *testing.T) {
},
wantErr: false,
},
{
name: "found, real secrets, verification error due to timeout",
s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a artifactory secret %s and domain %s but not verified", secret, appURL)),
verify: true,
},
want: func() []detectors.Result {
r := detectors.Result{
DetectorType: detectorspb.DetectorType_ArtifactoryAccessToken,
Verified: false,
}
r.SetVerificationError(context.DeadlineExceeded)
return []detectors.Result{r}
}(),
wantErr: false,
},
{
name: "found, real secrets, verification error due to unexpected api surface",
s: Scanner{client: common.ConstantResponseHttpClient(500, "{}")},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a artifactory secret %s and domain %s but not verified", secret, appURL)),
verify: true,
},
want: func() []detectors.Result {
r := detectors.Result{
DetectorType: detectorspb.DetectorType_ArtifactoryAccessToken,
Verified: false,
}
r.SetVerificationError(fmt.Errorf("unexpected HTTP response status 500"))
return []detectors.Result{r}
}(),
wantErr: false,
},
{
name: "found, unverified",
s: Scanner{},
@@ -122,6 +86,8 @@ func TestArtifactory_FromChunk(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
tt.s.UseFoundEndpoints(true)
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
if (err != nil) != tt.wantErr {
t.Errorf("Artifactory.FromData() error = %v, wantErr %v", err, tt.wantErr)
@@ -143,7 +109,7 @@ func TestArtifactory_FromChunk(t *testing.T) {
t.Fatalf("wantVerificationError = %v, verification error = %v", tt.want[i].VerificationError(), got[i].VerificationError())
}
}
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError")
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError", "primarySecret")
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
t.Errorf("Artifactory.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
}
pkg/detectors/artifactory/artifactory_test.go
+61 -16
View File
@@ -2,7 +2,6 @@ package artifactory
import (
"context"
"fmt"
"testing"
"github.com/google/go-cmp/cmp"
@@ -11,13 +10,6 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
var (
validPattern = "5YcZhIKwxTdxwpZHf9c1Usu8xNtAklRsqWYXWf2qmjW0RSQQ0U4sVnrNgOwIJlTOqJf06T3dl / RVo8ytzB65L.jfrog.io"
// validPattern2 is for cloud endpoints so it does not have any JFrog endpoint
validPattern2 = "5YcZhIKwxTdxwpZHf9c1Usu8xNtAklRsqWYXWf2qmjW0RSQQ0U4sVnrNgOwIJlTOqJf06T3dl"
invalidPattern = "W0RSQQ0U4sVnrNgOwIJlTOqJf06T3dl^&5YcZhIKwxTdxwpZHf9c1Usu8xNtA / rtest#y$zB65L%.jfrog.io"
)
func TestArtifactory_Pattern(t *testing.T) {
d := Scanner{}
ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
@@ -31,23 +23,76 @@ func TestArtifactory_Pattern(t *testing.T) {
want []string
}{
{
name: "valid pattern",
input: fmt.Sprintf("artifactory credentials: %s", validPattern),
name: "valid pattern",
input: `
# artifactory credentials
Token: cmVmdGtuOjAxOjE3ODA1NTFAKEM6S2J2MGswemNzZzhaRnFlVUFAKEk3amlLcGZg
Url: rwxtOp.jfrog.io
`,
useCloudEndpoint: false,
useFoundEndpoint: true,
want: []string{"5YcZhIKwxTdxwpZHf9c1Usu8xNtAklRsqWYXWf2qmjW0RSQQ0U4sVnrNgOwIJlTOqJf06T3dlRVo8ytzB65L.jfrog.io"},
want: []string{"cmVmdGtuOjAxOjE3ODA1NTFAKEM6S2J2MGswemNzZzhaRnFlVUFAKEk3amlLcGZgrwxtOp.jfrog.io"},
},
{
name: "valid pattern - with cloud endpoints",
input: fmt.Sprintf("artifactory credentials: %s", validPattern2),
name: "valid pattern - with cloud endpoints",
input: `
# artifactory credentials
Token: cmVmdGtuOjAxOjE3ODA1NTFAKEM6S2J2MGswemNzZzhaRnFlVUFAKEk3amlLcGZg
`,
cloudEndpoint: "cloudendpoint.jfrog.io",
useCloudEndpoint: true,
useFoundEndpoint: false,
want: []string{"5YcZhIKwxTdxwpZHf9c1Usu8xNtAklRsqWYXWf2qmjW0RSQQ0U4sVnrNgOwIJlTOqJf06T3dlcloudendpoint.jfrog.io"},
want: []string{"cmVmdGtuOjAxOjE3ODA1NTFAKEM6S2J2MGswemNzZzhaRnFlVUFAKEk3amlLcGZgcloudendpoint.jfrog.io"},
},
{
name: "invalid pattern",
input: fmt.Sprintf("artifactory credentials: %s", invalidPattern),
name: "valid pattern - with cloud and found endpoints",
input: `
# artifactory credentials
Token: cmVmdGtuOjAxOjE3ODA1NTFAKEM6S2J2MGswemNzZzhaRnFlVUFAKEk3amlLcGZg
Url: rwxtOp.jfrog.io
`,
cloudEndpoint: "cloudendpoint.jfrog.io",
useCloudEndpoint: true,
useFoundEndpoint: true,
want: []string{
"cmVmdGtuOjAxOjE3ODA1NTFAKEM6S2J2MGswemNzZzhaRnFlVUFAKEk3amlLcGZgcloudendpoint.jfrog.io",
"cmVmdGtuOjAxOjE3ODA1NTFAKEM6S2J2MGswemNzZzhaRnFlVUFAKEk3amlLcGZgrwxtOp.jfrog.io",
},
},
{
name: "valid pattern - with disabled found endpoints",
input: `
# artifactory credentials
Token: cmVmdGtuOjAxOjE3ODA1NTFAKEM6S2J2MGswemNzZzhaRnFlVUFAKEk3amlLcGZg
Url: rwxtOp.jfrog.io
`,
cloudEndpoint: "cloudendpoint.jfrog.io",
useCloudEndpoint: true,
useFoundEndpoint: false,
want: []string{
"cmVmdGtuOjAxOjE3ODA1NTFAKEM6S2J2MGswemNzZzhaRnFlVUFAKEk3amlLcGZgcloudendpoint.jfrog.io",
},
},
{
name: "valid pattern - with https in configured endpoint",
input: `
# artifactory credentials
Token: cmVmdGtuOjAxOjE3ODA1NTFAKEM6S2J2MGswemNzZzhaRnFlVUFAKEk3amlLcGZg
`,
cloudEndpoint: "https://cloudendpoint.jfrog.io",
useCloudEndpoint: true,
useFoundEndpoint: false,
want: []string{
"cmVmdGtuOjAxOjE3ODA1NTFAKEM6S2J2MGswemNzZzhaRnFlVUFAKEk3amlLcGZgcloudendpoint.jfrog.io",
},
},
{
name: "invalid pattern",
input: `
# artifactory credentials
Token: cmVmdGtuOjAxOjE3ODA_NTFAKEM6S2J2MGswemNzZzhaRnFlVUFAKEk3amlLcGZg
Url: rwxtOp.jfroq.io
`,
useFoundEndpoint: true,
want: nil,
},
pkg/engine/defaults/defaults.go
+2 -1
View File
@@ -47,6 +47,7 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/appoptics"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/appsynergy"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/apptivo"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/artifactory"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/artsy"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/asanaoauth"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/asanapersonalaccesstoken"
@@ -897,7 +898,7 @@ func buildDetectorList() []detectors.Detector {
&appoptics.Scanner{},
&appsynergy.Scanner{},
&apptivo.Scanner{},
// &artifactory.Scanner{},
&artifactory.Scanner{},
&artsy.Scanner{},
&asanaoauth.Scanner{},
&asanapersonalaccesstoken.Scanner{},
pkg/engine/engine_test.go
+1 -1
View File
@@ -1228,7 +1228,7 @@ func TestEngineInitializesCloudProviderDetectors(t *testing.T) {
for _, det := range e.detectors {
if endpoints, ok := det.(interface{ Endpoints(...string) []string }); ok {
id := config.GetDetectorID(det)
if len(endpoints.Endpoints()) == 0 {
if len(endpoints.Endpoints()) == 0 && det.Type() != detectorspb.DetectorType_ArtifactoryAccessToken { // artifactory does not have any cloud endpoint
t.Fatalf("detector %q Endpoints() is empty", id.String())
}
count++