trufflesecurity/trufflehog
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Detectors Updates 1 for Tristate Verification (#2187)

Browse Source 
* updating alibaba

* updating agora

* updating aeroworkflow

* updating aha

* updating artifactory

* updating abbysale

* updating abstract

* updating abuseipdb

* updating accuweather

* updating adafruitio

* updating adzuna

* cleanup on abuseipdb

* cleanup on aha

* cleanup on abuseipdb

* cleanup on aeroworkflow

* cleanup on adzuna

* cleanup on accuweather

* cleanup/refactor

* update token pattern to be explicitly 73char (old) or 64char (new)

* comment to clarify 403 on Aha

* mocking out verified case for aha + adding inactive account test

* using contact response instead of gock

* update 403 to be determinate
This commit is contained in:
āh̳̕mͭͭͨͩ̐e̘ͬ́͋ͬ̊̓͂d
2024-01-30 12:20:56 -05:00
committed by GitHub
parent 232032410c
commit 7ece4c3e66
22 changed files with 1246 additions and 354 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/detectors/abbysale/abbysale.go
+50 -22
View File
@@ -2,22 +2,28 @@ package abbysale
import (
"context"
regexp "github.com/wasilibs/go-re2"
"fmt"
"net/http"
"strings"
regexp "github.com/wasilibs/go-re2"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
type Scanner struct{}
type Scanner struct {
client *http.Client
}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
const abbysaleURL = "https://api.abyssale.com"
var (
client = common.SaneHttpClient()
// Ensure the Scanner satisfies the interface at compile time.
_ detectors.Detector = (*Scanner)(nil)
defaultClient = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"abbysale"}) + `\b([a-z0-9A-Z]{40})\b`)
@@ -29,6 +35,13 @@ func (s Scanner) Keywords() []string {
return []string{"abbysale"}
}
func (s Scanner) getClient() *http.Client {
if s.client != nil {
return s.client
}
return defaultClient
}
// FromData will find and optionally verify Abbysale secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
@@ -47,23 +60,15 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
}
if verify {
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.abyssale.com/ready", nil)
if err != nil {
continue
}
req.Header.Add("x-api-key", resMatch)
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
}
}
client := s.getClient()
isVerified, verificationErr := verifyAbbysale(ctx, client, resMatch)
s1.Verified = isVerified
s1.SetVerificationError(verificationErr, resMatch)
}
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if !s1.Verified && detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
results = append(results, s1)
@@ -72,6 +77,29 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
return results, nil
}
func verifyAbbysale(ctx context.Context, client *http.Client, resMatch string) (bool, error) {
// https://developers.abyssale.com/rest-api/authentication
req, err := http.NewRequestWithContext(ctx, http.MethodGet, abbysaleURL+"/ready", nil)
if err != nil {
return false, err
}
req.Header.Add("x-api-key", resMatch)
res, err := client.Do(req)
if err != nil {
return false, err
}
defer res.Body.Close()
switch res.StatusCode {
case http.StatusOK:
return true, nil
case http.StatusForbidden:
return false, nil
default:
return false, fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
}
}
func (s Scanner) Type() detectorspb.DetectorType {
return detectorspb.DetectorType_Abbysale
}
pkg/detectors/abbysale/abbysale_test.go
+55 -7
View File
@@ -9,7 +9,8 @@ import (
"testing"
"time"
"github.com/kylelemons/godebug/pretty"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
@@ -43,7 +44,7 @@ func TestAbbysale_FromChunk(t *testing.T) {
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a abbysale secret %s within", secret)),
data: []byte(fmt.Sprintf("You can find a abbysale secret %s within but verified", secret)),
verify: true,
},
want: []detectors.Result{
@@ -54,12 +55,48 @@ func TestAbbysale_FromChunk(t *testing.T) {
},
wantErr: false,
},
{
name: "found, real secrets, verification error due to timeout",
s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a abbysale secret %s within but verified", secret)),
verify: true,
},
want: func() []detectors.Result {
r := detectors.Result{
DetectorType: detectorspb.DetectorType_Abbysale,
Verified: false,
}
r.SetVerificationError(context.DeadlineExceeded)
return []detectors.Result{r}
}(),
wantErr: false,
},
{
name: "found, real secrets, verification error due to unexpected api surface",
s: Scanner{client: common.ConstantResponseHttpClient(500, "{}")},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a abbysale secret %s within but verified", secret)),
verify: true,
},
want: func() []detectors.Result {
r := detectors.Result{
DetectorType: detectorspb.DetectorType_Abbysale,
Verified: false,
}
r.SetVerificationError(fmt.Errorf("unexpected HTTP response status 500"))
return []detectors.Result{r}
}(),
wantErr: false,
},
{
name: "found, unverified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a abbysale secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
data: []byte(fmt.Sprintf("You can find a abbysale secret %s within but verified", inactiveSecret)), // the secret would satisfy the regex but not pass validation
verify: true,
},
want: []detectors.Result{
@@ -84,8 +121,7 @@ func TestAbbysale_FromChunk(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
s := Scanner{}
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
if (err != nil) != tt.wantErr {
t.Errorf("Abbysale.FromData() error = %v, wantErr %v", err, tt.wantErr)
return
@@ -94,9 +130,21 @@ func TestAbbysale_FromChunk(t *testing.T) {
if len(got[i].Raw) == 0 {
t.Fatalf("no raw secret present: \n %+v", got[i])
}
got[i].Raw = nil
gotErr := ""
if got[i].VerificationError() != nil {
gotErr = got[i].VerificationError().Error()
}
wantErr := ""
if tt.want[i].VerificationError() != nil {
wantErr = tt.want[i].VerificationError().Error()
}
if gotErr != wantErr {
t.Fatalf("wantVerificationError = %v, verification error = %v", tt.want[i].VerificationError(), got[i].VerificationError())
}
}
if diff := pretty.Compare(got, tt.want); diff != "" {
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError")
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
t.Errorf("Abbysale.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
}
})
pkg/detectors/abstract/abstract.go
+48 -21
View File
@@ -12,13 +12,17 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
type Scanner struct{}
type Scanner struct {
client *http.Client
}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
const abstractURL = "https://exchange-rates.abstractapi.com"
var (
client = common.SaneHttpClient()
// Ensure the Scanner satisfies the interface at compile time.
_ detectors.Detector = (*Scanner)(nil)
defaultClient = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"abstract"}) + `\b([0-9a-z]{32})\b`)
@@ -30,6 +34,13 @@ func (s Scanner) Keywords() []string {
return []string{"abstract"}
}
func (s Scanner) getClient() *http.Client {
if s.client != nil {
return s.client
}
return defaultClient
}
// FromData will find and optionally verify Abstract secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
@@ -48,23 +59,15 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
}
if verify {
req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://exchange-rates.abstractapi.com/v1/live/?api_key=%s&base=USD", resMatch), nil)
if err != nil {
continue
}
req.Header.Add("Content-Type", "application/json")
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
}
}
client := s.getClient()
isVerified, verificationErr := verifyAbstract(ctx, client, resMatch)
s1.Verified = isVerified
s1.SetVerificationError(verificationErr, resMatch)
}
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if !s1.Verified && detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
results = append(results, s1)
@@ -73,6 +76,30 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
return results, nil
}
func verifyAbstract(ctx context.Context, client *http.Client, resMatch string) (bool, error) {
// https://docs.abstractapi.com/exchange-rates#response-and-error-codes
req, err := http.NewRequestWithContext(ctx, http.MethodGet, abstractURL+fmt.Sprintf("/v1/live/?api_key=%s&base=USD", resMatch), nil)
if err != nil {
return false, err
}
req.Header.Add("Content-Type", "application/json")
res, err := client.Do(req)
if err != nil {
return false, err
}
defer res.Body.Close()
// https://docs.abstractapi.com/exchange-rates#response-and-error-codes
switch res.StatusCode {
case http.StatusOK:
return true, nil
case http.StatusUnauthorized:
return false, nil
default:
return false, fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
}
}
func (s Scanner) Type() detectorspb.DetectorType {
return detectorspb.DetectorType_Abstract
}
pkg/detectors/abstract/abstract_test.go
+56 -8
View File
@@ -9,7 +9,8 @@ import (
"testing"
"time"
"github.com/kylelemons/godebug/pretty"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -19,7 +20,7 @@ import (
func TestAbstract_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
@@ -43,7 +44,7 @@ func TestAbstract_FromChunk(t *testing.T) {
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a abstract secret %s within", secret)),
data: []byte(fmt.Sprintf("You can find a abstract secret %s within but verified", secret)),
verify: true,
},
want: []detectors.Result{
@@ -54,12 +55,48 @@ func TestAbstract_FromChunk(t *testing.T) {
},
wantErr: false,
},
{
name: "found, real secrets, verification error due to timeout",
s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a abstract secret %s within but verified", secret)),
verify: true,
},
want: func() []detectors.Result {
r := detectors.Result{
DetectorType: detectorspb.DetectorType_Abstract,
Verified: false,
}
r.SetVerificationError(context.DeadlineExceeded)
return []detectors.Result{r}
}(),
wantErr: false,
},
{
name: "found, real secrets, verification error due to unexpected api surface",
s: Scanner{client: common.ConstantResponseHttpClient(500, "{}")},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a abstract secret %s within", secret)),
verify: true,
},
want: func() []detectors.Result {
r := detectors.Result{
DetectorType: detectorspb.DetectorType_Abstract,
Verified: false,
}
r.SetVerificationError(fmt.Errorf("unexpected HTTP response status 500"))
return []detectors.Result{r}
}(),
wantErr: false,
},
{
name: "found, unverified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a abstract secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
data: []byte(fmt.Sprintf("You can find a abstract secret %s within but not valid", inactiveSecret)),
verify: true,
},
want: []detectors.Result{
@@ -84,8 +121,8 @@ func TestAbstract_FromChunk(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
s := Scanner{}
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
time.Sleep(900 * time.Millisecond) // avoid rate limiting
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
if (err != nil) != tt.wantErr {
t.Errorf("Abstract.FromData() error = %v, wantErr %v", err, tt.wantErr)
return
@@ -94,9 +131,20 @@ func TestAbstract_FromChunk(t *testing.T) {
if len(got[i].Raw) == 0 {
t.Fatalf("no raw secret present: \n %+v", got[i])
}
got[i].Raw = nil
gotErr := ""
if got[i].VerificationError() != nil {
gotErr = got[i].VerificationError().Error()
}
wantErr := ""
if tt.want[i].VerificationError() != nil {
wantErr = tt.want[i].VerificationError().Error()
}
if gotErr != wantErr {
t.Fatalf("wantVerificationError = %v, verification error = %v", tt.want[i].VerificationError(), got[i].VerificationError())
}
}
if diff := pretty.Compare(got, tt.want); diff != "" {
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError")
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
t.Errorf("Abstract.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
}
})
pkg/detectors/abuseipdb/abuseipdb.go
+61 -34
View File
@@ -1,25 +1,31 @@
package abuseipdb
import (
"bytes"
"context"
"fmt"
"io"
"net/http"
// "log"
regexp "github.com/wasilibs/go-re2"
"strings"
regexp "github.com/wasilibs/go-re2"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
type Scanner struct{}
type Scanner struct {
client *http.Client
}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
const abuseipdbURL = "https://api.abuseipdb.com"
var (
client = common.SaneHttpClient()
// Ensure the Scanner satisfies the interface at compile time.
_ detectors.Detector = (*Scanner)(nil)
defaultClient = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"abuseipdb"}) + `\b([a-z0-9]{80})\b`)
@@ -31,6 +37,13 @@ func (s Scanner) Keywords() []string {
return []string{"abuseipdb"}
}
func (s Scanner) getClient() *http.Client {
if s.client != nil {
return s.client
}
return defaultClient
}
// FromData will find and optionally verify AbuseIPDB secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
@@ -49,41 +62,55 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
}
if verify {
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.abuseipdb.com/api/v2/check?ipAddress=118.25.6.39", nil)
if err != nil {
continue
}
req.Header.Add("Key", resMatch)
res, err := client.Do(req)
if err == nil {
bodyBytes, err := io.ReadAll(res.Body)
if err == nil {
bodyString := string(bodyBytes)
validResponse := strings.Contains(bodyString, `ipAddress`)
// errCode := strings.Contains(bodyString, `AbuseIPDB APIv2 Server.`)
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
if validResponse {
s1.Verified = true
} else {
s1.Verified = false
}
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
}
}
}
client := s.getClient()
isVerified, verificationErr := verifyAbuseIPDB(ctx, client, resMatch)
s1.Verified = isVerified
s1.SetVerificationError(verificationErr, resMatch)
}
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if !s1.Verified && detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
results = append(results, s1)
}
return results, nil
}
func verifyAbuseIPDB(ctx context.Context, client *http.Client, resMatch string) (bool, error) {
// https://docs.abuseipdb.com/#check-endpoint
req, err := http.NewRequestWithContext(ctx, http.MethodGet, abuseipdbURL+"/api/v2/check?ipAddress=8.8.8.8", nil)
if err != nil {
return false, err
}
req.Header.Add("Key", resMatch)
res, err := client.Do(req)
if err != nil {
return false, err
}
defer res.Body.Close()
switch res.StatusCode {
case http.StatusOK:
bodyBytes, err := io.ReadAll(res.Body)
if err != nil {
return false, err
}
validResponse := bytes.Contains(bodyBytes, []byte("ipAddress"))
if validResponse {
return true, nil
} else {
return false, nil
}
case http.StatusUnauthorized:
return false, nil
default:
return false, fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
}
}
func (s Scanner) Type() detectorspb.DetectorType {
return detectorspb.DetectorType_AbuseIPDB
}
pkg/detectors/abuseipdb/abuseipdb_test.go
+55 -8
View File
@@ -9,10 +9,11 @@ import (
"testing"
"time"
"github.com/kylelemons/godebug/pretty"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
@@ -43,7 +44,7 @@ func TestAbuseIPDB_FromChunk(t *testing.T) {
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a abuseipdb secret %s within", secret)),
data: []byte(fmt.Sprintf("You can find a abuseipdb secret %s within but verified", secret)),
verify: true,
},
want: []detectors.Result{
@@ -54,12 +55,48 @@ func TestAbuseIPDB_FromChunk(t *testing.T) {
},
wantErr: false,
},
{
name: "found, real secrets, verification error due to timeout",
s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a abuseipdb secret %s within", secret)),
verify: true,
},
want: func() []detectors.Result {
r := detectors.Result{
DetectorType: detectorspb.DetectorType_AbuseIPDB,
Verified: false,
}
r.SetVerificationError(context.DeadlineExceeded)
return []detectors.Result{r}
}(),
wantErr: false,
},
{
name: "found, real secrets, verification error due to unexpected api surface",
s: Scanner{client: common.ConstantResponseHttpClient(500, "{}")},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a abuseipdb secret %s within", secret)),
verify: true,
},
want: func() []detectors.Result {
r := detectors.Result{
DetectorType: detectorspb.DetectorType_AbuseIPDB,
Verified: false,
}
r.SetVerificationError(fmt.Errorf("unexpected HTTP response status 500"))
return []detectors.Result{r}
}(),
wantErr: false,
},
{
name: "found, unverified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a abuseipdb secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
data: []byte(fmt.Sprintf("You can find a abuseipdb secret %s within", inactiveSecret)), // the secret would satisfy the regex but not pass validation
verify: true,
},
want: []detectors.Result{
@@ -84,8 +121,7 @@ func TestAbuseIPDB_FromChunk(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
s := Scanner{}
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
if (err != nil) != tt.wantErr {
t.Errorf("AbuseIPDB.FromData() error = %v, wantErr %v", err, tt.wantErr)
return
@@ -94,9 +130,20 @@ func TestAbuseIPDB_FromChunk(t *testing.T) {
if len(got[i].Raw) == 0 {
t.Fatalf("no raw secret present: \n %+v", got[i])
}
got[i].Raw = nil
gotErr := ""
if got[i].VerificationError() != nil {
gotErr = got[i].VerificationError().Error()
}
wantErr := ""
if tt.want[i].VerificationError() != nil {
wantErr = tt.want[i].VerificationError().Error()
}
if gotErr != wantErr {
t.Fatalf("wantVerificationError = %v, verification error = %v", tt.want[i].VerificationError(), got[i].VerificationError())
}
}
if diff := pretty.Compare(got, tt.want); diff != "" {
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError")
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
t.Errorf("AbuseIPDB.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
}
})
pkg/detectors/accuweather/accuweather.go
+50 -21
View File
@@ -2,22 +2,28 @@ package accuweather
import (
"context"
regexp "github.com/wasilibs/go-re2"
"fmt"
"net/http"
"strings"
regexp "github.com/wasilibs/go-re2"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
type Scanner struct{}
type Scanner struct {
client *http.Client
}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
const accuweatherURL = "https://dataservice.accuweather.com"
var (
client = common.SaneHttpClient()
// Ensure the Scanner satisfies the interface at compile time.
_ detectors.Detector = (*Scanner)(nil)
defaultClient = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"accuweather"}) + `([a-z0-9A-Z\%]{35})\b`)
@@ -29,6 +35,13 @@ func (s Scanner) Keywords() []string {
return []string{"accuweather"}
}
func (s Scanner) getClient() *http.Client {
if s.client != nil {
return s.client
}
return defaultClient
}
// FromData will find and optionally verify Accuweather secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
@@ -47,22 +60,15 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
}
if verify {
req, err := http.NewRequestWithContext(ctx, "GET", "https://dataservice.accuweather.com/locations/v1/cities/autocomplete?apikey="+resMatch+"&q=----&language=en-us", nil)
if err != nil {
continue
}
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
}
}
client := s.getClient()
isVerified, verificationErr := verifyAccuweather(ctx, client, resMatch)
s1.Verified = isVerified
s1.SetVerificationError(verificationErr, resMatch)
}
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if !s1.Verified && detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
results = append(results, s1)
@@ -71,6 +77,29 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
return results, nil
}
func verifyAccuweather(ctx context.Context, client *http.Client, resMatch string) (bool, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, accuweatherURL+"/locations/v1/cities/autocomplete?apikey="+resMatch+"&q=----&language=en-us", nil)
if err != nil {
return false, err
}
res, err := client.Do(req)
if err != nil {
return false, err
}
defer res.Body.Close()
// https://developer.accuweather.com/accuweather-locations-api/apis/get/locations/v1/cities/autocomplete
switch res.StatusCode {
case http.StatusOK, http.StatusForbidden:
// 403 indicates lack of permission, but valid token
return true, nil
case http.StatusUnauthorized:
return false, nil
default:
return false, fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
}
}
func (s Scanner) Type() detectorspb.DetectorType {
return detectorspb.DetectorType_Accuweather
}
pkg/detectors/accuweather/accuweather_test.go
+55 -7
View File
@@ -9,10 +9,11 @@ import (
"testing"
"time"
"github.com/kylelemons/godebug/pretty"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
@@ -43,7 +44,7 @@ func TestAccuweather_FromChunk(t *testing.T) {
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a accuweather secret %s within", secret)),
data: []byte(fmt.Sprintf("You can find a accuweather secret %s within but verified", secret)),
verify: true,
},
want: []detectors.Result{
@@ -54,12 +55,48 @@ func TestAccuweather_FromChunk(t *testing.T) {
},
wantErr: false,
},
{
name: "found, real secrets, verification error due to timeout",
s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a accuweather secret %s within", secret)),
verify: true,
},
want: func() []detectors.Result {
r := detectors.Result{
DetectorType: detectorspb.DetectorType_Accuweather,
Verified: false,
}
r.SetVerificationError(context.DeadlineExceeded)
return []detectors.Result{r}
}(),
wantErr: false,
},
{
name: "found, real secrets, verification error due to unexpected api surface",
s: Scanner{client: common.ConstantResponseHttpClient(500, "{}")},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a accuweather secret %s within", secret)),
verify: true,
},
want: func() []detectors.Result {
r := detectors.Result{
DetectorType: detectorspb.DetectorType_Accuweather,
Verified: false,
}
r.SetVerificationError(fmt.Errorf("unexpected HTTP response status 500"))
return []detectors.Result{r}
}(),
wantErr: false,
},
{
name: "found, unverified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a accuweather secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
data: []byte(fmt.Sprintf("You can find a accuweather secret %s within but verified", inactiveSecret)),
verify: true,
},
want: []detectors.Result{
@@ -84,8 +121,7 @@ func TestAccuweather_FromChunk(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
s := Scanner{}
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
if (err != nil) != tt.wantErr {
t.Errorf("Accuweather.FromData() error = %v, wantErr %v", err, tt.wantErr)
return
@@ -94,9 +130,21 @@ func TestAccuweather_FromChunk(t *testing.T) {
if len(got[i].Raw) == 0 {
t.Fatalf("no raw secret present: \n %+v", got[i])
}
gotErr := ""
if got[i].VerificationError() != nil {
gotErr = got[i].VerificationError().Error()
}
wantErr := ""
if tt.want[i].VerificationError() != nil {
wantErr = tt.want[i].VerificationError().Error()
}
if gotErr != wantErr {
t.Fatalf("wantVerificationError = %v, verification error = %v", tt.want[i].VerificationError(), got[i].VerificationError())
}
got[i].Raw = nil
}
if diff := pretty.Compare(got, tt.want); diff != "" {
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError")
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
t.Errorf("Accuweather.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
}
})
pkg/detectors/adafruitio/adafruitio.go
+49 -21
View File
@@ -2,22 +2,28 @@ package adafruitio
import (
"context"
regexp "github.com/wasilibs/go-re2"
"fmt"
"net/http"
"strings"
regexp "github.com/wasilibs/go-re2"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
type Scanner struct{}
type Scanner struct {
client *http.Client
}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
const adafruitioURL = "https://io.adafruit.com"
var (
client = common.SaneHttpClient()
// Ensure the Scanner satisfies the interface at compile time.
_ detectors.Detector = (*Scanner)(nil)
defaultClient = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(`\b(aio\_[a-zA-Z0-9]{28})\b`)
@@ -29,6 +35,13 @@ func (s Scanner) Keywords() []string {
return []string{"aio_"}
}
func (s Scanner) getClient() *http.Client {
if s.client != nil {
return s.client
}
return defaultClient
}
// FromData will find and optionally verify AdafruitIO secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
@@ -47,22 +60,15 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
}
if verify {
req, err := http.NewRequestWithContext(ctx, "GET", "https://io.adafruit.com/api/v2/ladybugtest/feeds/?x-aio-key="+resMatch, nil)
if err != nil {
continue
}
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
}
}
client := s.getClient()
isVerified, verificationErr := verifyAdafruitIO(ctx, client, resMatch)
s1.Verified = isVerified
s1.SetVerificationError(verificationErr, resMatch)
}
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if !s1.Verified && detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
results = append(results, s1)
@@ -71,6 +77,28 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
return results, nil
}
func verifyAdafruitIO(ctx context.Context, client *http.Client, resMatch string) (bool, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, adafruitioURL+"/api/v2/ladybugtest/feeds/?x-aio-key="+resMatch, nil)
if err != nil {
return false, err
}
res, err := client.Do(req)
if err != nil {
return false, err
}
defer res.Body.Close()
// https://learn.adafruit.com/adafruit-io/http-status-codes
switch res.StatusCode {
case http.StatusOK:
return true, nil
case http.StatusUnauthorized:
return false, nil
default:
return false, fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
}
}
func (s Scanner) Type() detectorspb.DetectorType {
return detectorspb.DetectorType_AdafruitIO
}
pkg/detectors/adafruitio/adafruitio_test.go
+52 -5
View File
@@ -9,7 +9,8 @@ import (
"testing"
"time"
"github.com/kylelemons/godebug/pretty"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -54,6 +55,42 @@ func TestAdafruitIO_FromChunk(t *testing.T) {
},
wantErr: false,
},
{
name: "found, real secrets, verification error due to timeout",
s: Scanner{client: common.SaneHttpClientTimeOut(10 * time.Microsecond)},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a adafruitio secret %s within", secret)),
verify: true,
},
want: func() []detectors.Result {
r := detectors.Result{
DetectorType: detectorspb.DetectorType_AdafruitIO,
Verified: false,
}
r.SetVerificationError(context.DeadlineExceeded)
return []detectors.Result{r}
}(),
wantErr: false,
},
{
name: "found, real secrets, verification error due to unexpected api surface",
s: Scanner{client: common.ConstantResponseHttpClient(500, "{}")},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a adafruitio secret %s within", secret)),
verify: true,
},
want: func() []detectors.Result {
r := detectors.Result{
DetectorType: detectorspb.DetectorType_AdafruitIO,
Verified: false,
}
r.SetVerificationError(fmt.Errorf("unexpected HTTP response status 500"))
return []detectors.Result{r}
}(),
wantErr: false,
},
{
name: "found, unverified",
s: Scanner{},
@@ -84,8 +121,7 @@ func TestAdafruitIO_FromChunk(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
s := Scanner{}
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
if (err != nil) != tt.wantErr {
t.Errorf("AdafruitIO.FromData() error = %v, wantErr %v", err, tt.wantErr)
return
@@ -94,9 +130,20 @@ func TestAdafruitIO_FromChunk(t *testing.T) {
if len(got[i].Raw) == 0 {
t.Fatalf("no raw secret present: \n %+v", got[i])
}
got[i].Raw = nil
gotErr := ""
if got[i].VerificationError() != nil {
gotErr = got[i].VerificationError().Error()
}
wantErr := ""
if tt.want[i].VerificationError() != nil {
wantErr = tt.want[i].VerificationError().Error()
}
if gotErr != wantErr {
t.Fatalf("wantVerificationError = %v, verification error = %v", tt.want[i].VerificationError(), got[i].VerificationError())
}
}
if diff := pretty.Compare(got, tt.want); diff != "" {
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError")
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
t.Errorf("AdafruitIO.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
}
})
pkg/detectors/adzuna/adzuna.go
+48 -20
View File
@@ -12,13 +12,17 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
type Scanner struct{}
type Scanner struct {
client *http.Client
}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
const adzunaURL = "https://api.adzuna.com"
var (
client = common.SaneHttpClient()
// Ensure the Scanner satisfies the interface at compile time.
_ detectors.Detector = (*Scanner)(nil)
defaultClient = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"adzuna"}) + `\b([a-z0-9]{32})\b`)
@@ -31,6 +35,13 @@ func (s Scanner) Keywords() []string {
return []string{"adzuna"}
}
func (s Scanner) getClient() *http.Client {
if s.client != nil {
return s.client
}
return defaultClient
}
// FromData will find and optionally verify Adzuna secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
@@ -56,22 +67,15 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
}
if verify {
req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://api.adzuna.com/v1/api/jobs/gb/search/1?app_id=%s&app_key=%s", resIdMatch, resMatch), nil)
if err != nil {
continue
}
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
}
}
client := s.getClient()
isVerified, verificationErr := verifyAdzuna(ctx, client, resMatch, resIdMatch)
s1.Verified = isVerified
s1.SetVerificationError(verificationErr, resMatch)
}
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if !s1.Verified && detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
results = append(results, s1)
@@ -81,6 +85,30 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
return results, nil
}
func verifyAdzuna(ctx context.Context, client *http.Client, resMatch, resIdMatch string) (bool, error) {
// https://developer.adzuna.com/activedocs#!/adzuna/search
req, err := http.NewRequestWithContext(ctx, http.MethodGet, adzunaURL+fmt.Sprintf("/v1/api/jobs/us/search/1?app_id=%s&app_key=%s", resIdMatch, resMatch), nil)
if err != nil {
return false, err
}
res, err := client.Do(req)
if err != nil {
return false, err
}
defer res.Body.Close()
// https://developer.adzuna.com/overview
switch res.StatusCode {
case http.StatusOK:
return true, nil
case http.StatusUnauthorized:
return false, nil
default:
return false, fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
}
}
func (s Scanner) Type() detectorspb.DetectorType {
return detectorspb.DetectorType_Adzuna
}
pkg/detectors/adzuna/adzuna_test.go
+54 -7
View File
@@ -9,10 +9,11 @@ import (
"testing"
"time"
"github.com/kylelemons/godebug/pretty"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
@@ -55,12 +56,48 @@ func TestAdzuna_FromChunk(t *testing.T) {
},
wantErr: false,
},
{
name: "found, real secrets, verification error due to timeout",
s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a adzuna secret %s within adzuna %s", secret, id)),
verify: true,
},
want: func() []detectors.Result {
r := detectors.Result{
DetectorType: detectorspb.DetectorType_Adzuna,
Verified: false,
}
r.SetVerificationError(context.DeadlineExceeded)
return []detectors.Result{r}
}(),
wantErr: false,
},
{
name: "found, real secrets, verification error due to unexpected api surface",
s: Scanner{client: common.ConstantResponseHttpClient(500, "{}")},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a adzuna secret %s within adzuna %s", secret, id)),
verify: true,
},
want: func() []detectors.Result {
r := detectors.Result{
DetectorType: detectorspb.DetectorType_Adzuna,
Verified: false,
}
r.SetVerificationError(fmt.Errorf("unexpected HTTP response status 500"))
return []detectors.Result{r}
}(),
wantErr: false,
},
{
name: "found, unverified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a adzuna secret %s within adzuna %s but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation
data: []byte(fmt.Sprintf("You can find a adzuna secret %s within adzuna %s but not valid", inactiveSecret, id)),
verify: true,
},
want: []detectors.Result{
@@ -85,8 +122,7 @@ func TestAdzuna_FromChunk(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
s := Scanner{}
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
if (err != nil) != tt.wantErr {
t.Errorf("Adzuna.FromData() error = %v, wantErr %v", err, tt.wantErr)
return
@@ -95,9 +131,20 @@ func TestAdzuna_FromChunk(t *testing.T) {
if len(got[i].Raw) == 0 {
t.Fatalf("no raw secret present: \n %+v", got[i])
}
got[i].Raw = nil
gotErr := ""
if got[i].VerificationError() != nil {
gotErr = got[i].VerificationError().Error()
}
wantErr := ""
if tt.want[i].VerificationError() != nil {
wantErr = tt.want[i].VerificationError().Error()
}
if gotErr != wantErr {
t.Fatalf("wantVerificationError = %v, verification error = %v", tt.want[i].VerificationError(), got[i].VerificationError())
}
}
if diff := pretty.Compare(got, tt.want); diff != "" {
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError")
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
t.Errorf("Adzuna.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
}
})
pkg/detectors/aeroworkflow/aeroworkflow.go
+53 -24
View File
@@ -2,25 +2,31 @@ package aeroworkflow
import (
"context"
regexp "github.com/wasilibs/go-re2"
"fmt"
"net/http"
"strings"
regexp "github.com/wasilibs/go-re2"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
type Scanner struct{}
type Scanner struct {
client *http.Client
}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
const aeroworkflowURL = "https://api.aeroworkflow.com"
var (
client = common.SaneHttpClient()
// Ensure the Scanner satisfies the interface at compile time.
_ detectors.Detector = (*Scanner)(nil)
defaultClient = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"aeroworkflow"}) + `\b([a-zA-Z0-9^!]{20})\b`)
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"aeroworkflow"}) + `([a-zA-Z0-9^!?#:*;]{20})\b`)
idPat = regexp.MustCompile(detectors.PrefixRegex([]string{"aeroworkflow"}) + `\b([0-9]{1,})\b`)
)
@@ -30,6 +36,13 @@ func (s Scanner) Keywords() []string {
return []string{"aeroworkflow"}
}
func (s Scanner) getClient() *http.Client {
if s.client != nil {
return s.client
}
return defaultClient
}
// FromData will find and optionally verify Aeroworkflow secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
@@ -56,26 +69,16 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
}
if verify {
req, err := http.NewRequest("GET", "https://api.aeroworkflow.com/api/"+resIdMatch+"/v1/AeroAppointments", nil)
if err != nil {
continue
}
req.Header.Add("Accept", "application/json")
req.Header.Add("apikey", resMatch)
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
}
}
client := s.getClient()
isVerified, verificationErr := verifyAeroworkflow(ctx, client, resMatch, resIdMatch)
s1.Verified = isVerified
s1.SetVerificationError(verificationErr, resMatch)
}
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if !s1.Verified && detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
results = append(results, s1)
}
@@ -84,6 +87,32 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
return results, nil
}
func verifyAeroworkflow(ctx context.Context, client *http.Client, resMatch, resIdMatch string) (bool, error) {
req, err := http.NewRequest(http.MethodGet, aeroworkflowURL+"/api/"+resIdMatch+"/v1/AeroAppointments", nil)
if err != nil {
return false, err
}
req.Header.Add("Accept", "application/json")
req.Header.Add("apikey", resMatch)
res, err := client.Do(req)
if err != nil {
return false, err
}
defer res.Body.Close()
// https://api.aeroworkflow.com/swagger/index.html
switch res.StatusCode {
case http.StatusOK:
return true, nil
case http.StatusUnauthorized, http.StatusForbidden:
// 401 for invalid API key
// 403 for invalid Account ID
return false, nil
default:
return false, fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
}
}
func (s Scanner) Type() detectorspb.DetectorType {
return detectorspb.DetectorType_Aeroworkflow
}
pkg/detectors/aeroworkflow/aeroworkflow_test.go
+54 -9
View File
@@ -9,8 +9,8 @@ import (
"testing"
"time"
"github.com/kylelemons/godebug/pretty"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
@@ -19,13 +19,12 @@ import (
func TestAeroworkflow_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
secret := testSecrets.MustGetField("AEROWORKFLOW")
secret := testSecrets.MustGetField("AEROWORKFLOW_SECRET")
id := testSecrets.MustGetField("AEROWORKFLOW_ID")
inactiveSecret := testSecrets.MustGetField("AEROWORKFLOW_INACTIVE")
type args struct {
@@ -56,6 +55,42 @@ func TestAeroworkflow_FromChunk(t *testing.T) {
},
wantErr: false,
},
{
name: "found, real secrets, verification error due to timeout",
s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a aeroworkflow secret %s within aeroworkflow %s", secret, id)),
verify: true,
},
want: func() []detectors.Result {
r := detectors.Result{
DetectorType: detectorspb.DetectorType_Aeroworkflow,
Verified: false,
}
r.SetVerificationError(context.DeadlineExceeded)
return []detectors.Result{r}
}(),
wantErr: false,
},
{
name: "found, real secrets, verification error due to unexpected api surface",
s: Scanner{client: common.ConstantResponseHttpClient(500, "{}")},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a aeroworkflow secret %s within aeroworkflow %s", secret, id)),
verify: true,
},
want: func() []detectors.Result {
r := detectors.Result{
DetectorType: detectorspb.DetectorType_Aeroworkflow,
Verified: false,
}
r.SetVerificationError(fmt.Errorf("unexpected HTTP response status 500"))
return []detectors.Result{r}
}(),
wantErr: false,
},
{
name: "found, unverified",
s: Scanner{},
@@ -86,8 +121,7 @@ func TestAeroworkflow_FromChunk(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
s := Scanner{}
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
if (err != nil) != tt.wantErr {
t.Errorf("Aeroworkflow.FromData() error = %v, wantErr %v", err, tt.wantErr)
return
@@ -96,9 +130,20 @@ func TestAeroworkflow_FromChunk(t *testing.T) {
if len(got[i].Raw) == 0 {
t.Fatalf("no raw secret present: \n %+v", got[i])
}
got[i].Raw = nil
gotErr := ""
if got[i].VerificationError() != nil {
gotErr = got[i].VerificationError().Error()
}
wantErr := ""
if tt.want[i].VerificationError() != nil {
wantErr = tt.want[i].VerificationError().Error()
}
if gotErr != wantErr {
t.Fatalf("wantVerificationError = %v, verification error = %v", tt.want[i].VerificationError(), got[i].VerificationError())
}
}
if diff := pretty.Compare(got, tt.want); diff != "" {
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError")
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
t.Errorf("Aeroworkflow.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
}
})
pkg/detectors/agora/agora.go
+50 -22
View File
@@ -2,22 +2,28 @@ package agora
import (
"context"
regexp "github.com/wasilibs/go-re2"
"fmt"
"net/http"
"strings"
regexp "github.com/wasilibs/go-re2"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
type Scanner struct{}
type Scanner struct {
client *http.Client
}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
const agoraURL = "https://api.agora.io"
var (
client = common.SaneHttpClient()
// Ensure the Scanner satisfies the interface at compile time.
_ detectors.Detector = (*Scanner)(nil)
defaultClient = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"agora"}) + `\b([a-z0-9]{32})\b`)
@@ -30,6 +36,13 @@ func (s Scanner) Keywords() []string {
return []string{"agora"}
}
func (s Scanner) getClient() *http.Client {
if s.client != nil {
return s.client
}
return defaultClient
}
// FromData will find and optionally verify Agora secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
@@ -57,24 +70,15 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
}
if verify {
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.agora.io/dev/v1/projects", nil)
if err != nil {
continue
}
req.SetBasicAuth(resSecret, resMatch)
res, err := client.Do(req)
client := s.getClient()
isVerified, verificationErr := verifyAgora(ctx, client, resMatch, resSecret)
s1.Verified = isVerified
s1.SetVerificationError(verificationErr, resMatch)
}
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
}
}
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if !s1.Verified && detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
results = append(results, s1)
@@ -84,6 +88,30 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
return results, nil
}
func verifyAgora(ctx context.Context, client *http.Client, resMatch, resSecret string) (bool, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, agoraURL+"/dev/v1/projects", nil)
if err != nil {
return false, err
}
req.SetBasicAuth(resSecret, resMatch)
res, err := client.Do(req)
if err != nil {
return false, err
}
defer res.Body.Close()
// https://docs.agora.io/en/voice-calling/reference/agora-console-rest-api#get-all-projects
switch res.StatusCode {
case http.StatusOK, http.StatusCreated:
return true, nil
case http.StatusUnauthorized, http.StatusForbidden:
return false, nil
default:
return false, fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
}
}
func (s Scanner) Type() detectorspb.DetectorType {
return detectorspb.DetectorType_Agora
}
pkg/detectors/agora/agora_test.go
+76 -5
View File
@@ -9,7 +9,8 @@ import (
"testing"
"time"
"github.com/kylelemons/godebug/pretty"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
@@ -48,13 +49,61 @@ func TestAgora_FromChunk(t *testing.T) {
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_Agora,
Verified: false,
},
{
DetectorType: detectorspb.DetectorType_Agora,
Verified: true,
},
{
DetectorType: detectorspb.DetectorType_Agora,
Verified: false,
},
{
DetectorType: detectorspb.DetectorType_Agora,
Verified: false,
},
},
wantErr: false,
},
{
name: "found, real secrets, verification error due to timeout",
s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a agora secret %s within agora id %s but verified", secret, id)),
verify: true,
},
want: func() []detectors.Result {
r := detectors.Result{
DetectorType: detectorspb.DetectorType_Agora,
Verified: false,
}
r.SetVerificationError(context.DeadlineExceeded)
return []detectors.Result{r, r, r, r}
}(),
wantErr: false,
},
{
name: "found, real secrets, verification error due to unexpected api surface",
s: Scanner{client: common.ConstantResponseHttpClient(500, "{}")},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a agora secret %s within agora id %s but verified", secret, id)),
verify: true,
},
want: func() []detectors.Result {
r := detectors.Result{
DetectorType: detectorspb.DetectorType_Agora,
Verified: false,
}
r.SetVerificationError(fmt.Errorf("unexpected HTTP response status 500"))
return []detectors.Result{r, r, r, r}
}(),
wantErr: false,
},
{
name: "found, unverified",
s: Scanner{},
@@ -68,6 +117,18 @@ func TestAgora_FromChunk(t *testing.T) {
DetectorType: detectorspb.DetectorType_Agora,
Verified: false,
},
{
DetectorType: detectorspb.DetectorType_Agora,
Verified: false,
},
{
DetectorType: detectorspb.DetectorType_Agora,
Verified: false,
},
{
DetectorType: detectorspb.DetectorType_Agora,
Verified: false,
},
},
wantErr: false,
},
@@ -85,8 +146,7 @@ func TestAgora_FromChunk(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
s := Scanner{}
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
if (err != nil) != tt.wantErr {
t.Errorf("Agora.FromData() error = %v, wantErr %v", err, tt.wantErr)
return
@@ -95,9 +155,20 @@ func TestAgora_FromChunk(t *testing.T) {
if len(got[i].Raw) == 0 {
t.Fatal("no raw secret present")
}
got[i].Raw = nil
gotErr := ""
if got[i].VerificationError() != nil {
gotErr = got[i].VerificationError().Error()
}
wantErr := ""
if tt.want[i].VerificationError() != nil {
wantErr = tt.want[i].VerificationError().Error()
}
if gotErr != wantErr {
t.Fatalf("wantVerificationError = %v, verification error = %v", tt.want[i].VerificationError(), got[i].VerificationError())
}
}
if diff := pretty.Compare(got, tt.want); diff != "" {
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError")
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
t.Errorf("Agora.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
}
})
pkg/detectors/aha/aha.go
+61 -25
View File
@@ -3,25 +3,29 @@ package aha
import (
"context"
"fmt"
regexp "github.com/wasilibs/go-re2"
"net/http"
"strings"
regexp "github.com/wasilibs/go-re2"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
type Scanner struct{}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
type Scanner struct {
client *http.Client
}
var (
client = common.SaneHttpClient()
// Ensure the Scanner satisfies the interface at compile time.
_ detectors.Detector = (*Scanner)(nil)
defaultClient = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"aha"}) + `\b([0-9a-f]{64})\b`)
URLPat = regexp.MustCompile(`\b([A-Za-z0-9](?:[A-Za-z0-9\-]{0,61}[A-Za-z0-9])\.aha\.io)`)
)
// Keywords are used for efficiently pre-filtering chunks.
@@ -30,11 +34,26 @@ func (s Scanner) Keywords() []string {
return []string{"aha"}
}
func (s Scanner) getClient() *http.Client {
if s.client != nil {
return s.client
}
return defaultClient
}
// FromData will find and optionally verify Aha secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
URLmatches := URLPat.FindAllStringSubmatch(dataStr, -1)
resURLMatch := "aha.io"
for _, URLmatch := range URLmatches {
if len(URLmatch) != 2 {
continue
}
resURLMatch = strings.TrimSpace(URLmatch[1])
}
for _, match := range matches {
if len(match) != 2 {
@@ -48,24 +67,15 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
}
if verify {
req, err := http.NewRequestWithContext(ctx, "GET", "https://company.aha.io/api/v1/me", nil)
if err != nil {
continue
}
req.Header.Add("Accept", "application/vnd.aha+json; version=3")
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
}
}
client := s.getClient()
isVerified, verificationErr := verifyAha(ctx, client, resMatch, resURLMatch)
s1.Verified = isVerified
s1.SetVerificationError(verificationErr, resMatch)
}
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if !s1.Verified && detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
results = append(results, s1)
@@ -74,6 +84,32 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
return results, nil
}
func verifyAha(ctx context.Context, client *http.Client, resMatch, resURLMatch string) (bool, error) {
url := fmt.Sprintf("https://%s/api/v1/me", resURLMatch)
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return false, err
}
req.Header.Add("Accept", "application/vnd.aha+json; version=3")
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))
res, err := client.Do(req)
if err != nil {
return false, err
}
defer res.Body.Close()
// https://www.aha.io/api
switch res.StatusCode {
case http.StatusOK:
return true, nil
case http.StatusUnauthorized, http.StatusNotFound, http.StatusForbidden:
// 403 is a known case where an account is inactive bc of a trial ending or payment issue
return false, nil
default:
return false, fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
}
}
func (s Scanner) Type() detectorspb.DetectorType {
return detectorspb.DetectorType_Aha
}
pkg/detectors/aha/aha_test.go
+58 -11
View File
@@ -9,8 +9,8 @@ import (
"testing"
"time"
"github.com/kylelemons/godebug/pretty"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
@@ -20,11 +20,12 @@ import (
func TestAha_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
secret := testSecrets.MustGetField("AHA")
domain := testSecrets.MustGetField("AHA_DOMAIN")
secret := testSecrets.MustGetField("AHA_SECRET")
inactiveSecret := testSecrets.MustGetField("AHA_INACTIVE")
type args struct {
@@ -41,10 +42,10 @@ func TestAha_FromChunk(t *testing.T) {
}{
{
name: "found, verified",
s: Scanner{},
s: Scanner{client: common.ConstantResponseHttpClient(200, "{}")},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a aha secret %s within", secret)),
data: []byte(fmt.Sprintf("You can find a aha secret %s within %s but verified", secret, domain)),
verify: true,
},
want: []detectors.Result{
@@ -55,12 +56,48 @@ func TestAha_FromChunk(t *testing.T) {
},
wantErr: false,
},
{
name: "found, real secrets, verification error due to timeout",
s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a aha secret %s within %s but verified", secret, domain)),
verify: true,
},
want: func() []detectors.Result {
r := detectors.Result{
DetectorType: detectorspb.DetectorType_Aha,
Verified: false,
}
r.SetVerificationError(context.DeadlineExceeded)
return []detectors.Result{r}
}(),
wantErr: false,
},
{
name: "found, real secrets, verification error due to unexpected api surface",
s: Scanner{client: common.ConstantResponseHttpClient(500, "{}")},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a aha secret %s within %s but verified", secret, domain)),
verify: true,
},
want: func() []detectors.Result {
r := detectors.Result{
DetectorType: detectorspb.DetectorType_Aha,
Verified: false,
}
r.SetVerificationError(fmt.Errorf("unexpected HTTP response status 500"))
return []detectors.Result{r}
}(),
wantErr: false,
},
{
name: "found, unverified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a aha secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
data: []byte(fmt.Sprintf("You can find a aha secret %s within but not valid domain %s", inactiveSecret, domain)), // the secret would satisfy the regex but not pass validation
verify: true,
},
want: []detectors.Result{
@@ -85,8 +122,7 @@ func TestAha_FromChunk(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
s := Scanner{}
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
if (err != nil) != tt.wantErr {
t.Errorf("Aha.FromData() error = %v, wantErr %v", err, tt.wantErr)
return
@@ -95,9 +131,20 @@ func TestAha_FromChunk(t *testing.T) {
if len(got[i].Raw) == 0 {
t.Fatalf("no raw secret present: \n %+v", got[i])
}
got[i].Raw = nil
gotErr := ""
if got[i].VerificationError() != nil {
gotErr = got[i].VerificationError().Error()
}
wantErr := ""
if tt.want[i].VerificationError() != nil {
wantErr = tt.want[i].VerificationError().Error()
}
if gotErr != wantErr {
t.Fatalf("wantVerificationError = %v, verification error = %v", tt.want[i].VerificationError(), got[i].VerificationError())
}
}
if diff := pretty.Compare(got, tt.want); diff != "" {
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError")
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
t.Errorf("Aha.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
}
})
pkg/detectors/alibaba/alibaba.go
+88 -39
View File
@@ -6,25 +6,40 @@ import (
"crypto/rand"
"crypto/sha1"
"encoding/base64"
regexp "github.com/wasilibs/go-re2"
"encoding/json"
"fmt"
"net/http"
"net/url"
"strconv"
"strings"
"time"
regexp "github.com/wasilibs/go-re2"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
type Scanner struct{}
type Scanner struct {
client *http.Client
}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
type alibabaResp struct {
RequestId string `json:"RequestId"`
Message string `json:"Message"`
Recommend string `json:"Recommend"`
HostId string `json:"HostId"`
Code string `json:"Code"`
}
const alibabaURL = "https://ecs.aliyuncs.com"
var (
client = common.SaneHttpClient()
// Ensure the Scanner satisfies the interface at compile time.
_ detectors.Detector = (*Scanner)(nil)
defaultClient = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(`\b([a-zA-Z0-9]{30})\b`)
@@ -53,6 +68,7 @@ func GetSignature(input, key string) string {
h.Write([]byte(input))
return base64.StdEncoding.EncodeToString(h.Sum(nil))
}
func buildStringToSign(method, input string) string {
filter := strings.Replace(input, "+", "%20", -1)
filter = strings.Replace(filter, "%7E", "~", -1)
@@ -61,6 +77,13 @@ func buildStringToSign(method, input string) string {
return filter
}
func (s Scanner) getClient() *http.Client {
if s.client != nil {
return s.client
}
return defaultClient
}
// FromData will find and optionally verify Alibaba secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
@@ -87,42 +110,16 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
}
if verify {
req, err := http.NewRequestWithContext(ctx, "GET", "https://ecs.aliyuncs.com/?", nil)
if err != nil {
continue
}
dateISO := time.Now().UTC().Format("2006-01-02T15:04:05Z07:00")
params := req.URL.Query()
params.Add("AccessKeyId", resIdMatch)
params.Add("Action", "DescribeRegions")
params.Add("Format", "JSON")
params.Add("SignatureMethod", "HMAC-SHA1")
params.Add("SignatureNonce", randString(16))
params.Add("SignatureVersion", "1.0")
params.Add("Timestamp", dateISO)
params.Add("Version", "2014-05-26")
stringToSign := buildStringToSign(req.Method, params.Encode())
signature := GetSignature(stringToSign, resMatch+"&") // Get Signature HMAC SHA1
params.Add("Signature", signature)
req.URL.RawQuery = params.Encode()
req.Header.Add("Content-Type", "text/xml;charset=utf-8")
req.Header.Add("Content-Length", strconv.Itoa(len(params.Encode())))
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
}
}
client := s.getClient()
isVerified, verificationErr := verifyAlibaba(ctx, client, resIdMatch, resMatch)
s1.Verified = isVerified
s1.SetVerificationError(verificationErr, resMatch)
}
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if !s1.Verified && detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
results = append(results, s1)
}
}
@@ -130,6 +127,58 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
return results, nil
}
func verifyAlibaba(ctx context.Context, client *http.Client, resIdMatch, resMatch string) (bool, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, alibabaURL, nil)
if err != nil {
return false, err
}
dateISO := time.Now().UTC().Format("2006-01-02T15:04:05Z07:00")
params := req.URL.Query()
params.Add("AccessKeyId", resIdMatch)
params.Add("Action", "DescribeRegions")
params.Add("Format", "JSON")
params.Add("SignatureMethod", "HMAC-SHA1")
params.Add("SignatureNonce", randString(16))
params.Add("SignatureVersion", "1.0")
params.Add("Timestamp", dateISO)
params.Add("Version", "2014-05-26")
stringToSign := buildStringToSign(req.Method, params.Encode())
signature := GetSignature(stringToSign, resMatch+"&") // Get Signature HMAC SHA1
params.Add("Signature", signature)
req.URL.RawQuery = params.Encode()
req.Header.Add("Content-Type", "text/xml;charset=utf-8")
req.Header.Add("Content-Length", strconv.Itoa(len(params.Encode())))
res, err := client.Do(req)
if err != nil {
return false, err
}
defer res.Body.Close()
var alibabaResp alibabaResp
if err = json.NewDecoder(res.Body).Decode(&alibabaResp); err != nil {
return false, err
}
switch res.StatusCode {
case http.StatusOK:
return true, nil
case http.StatusNotFound, http.StatusBadRequest:
// 400 used for most of error cases
// 404 used if the AccessKeyId is not valid
return false, nil
default:
err := fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
if alibabaResp.Message != "" {
err = fmt.Errorf("%s: %s, %s", err, alibabaResp.Message, alibabaResp.Code)
}
return false, err
}
}
func (s Scanner) Type() detectorspb.DetectorType {
return detectorspb.DetectorType_Alibaba
}
pkg/detectors/alibaba/alibaba_test.go
+70 -6
View File
@@ -9,7 +9,8 @@ import (
"testing"
"time"
"github.com/kylelemons/godebug/pretty"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
@@ -55,6 +56,60 @@ func TestAlibaba_FromChunk(t *testing.T) {
},
wantErr: false,
},
{
name: "found, real secrets, verification error due to timeout",
s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a alibaba secret %s within alibaba %s", secret, id)),
verify: true,
},
want: func() []detectors.Result {
r := detectors.Result{
DetectorType: detectorspb.DetectorType_Alibaba,
Verified: false,
}
r.SetVerificationError(context.DeadlineExceeded)
return []detectors.Result{r}
}(),
wantErr: false,
},
{
name: "found, real secrets, verification error due to unexpected api surface",
s: Scanner{client: common.ConstantResponseHttpClient(500, "{}")},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a alibaba secret %s within alibaba %s", secret, id)),
verify: true,
},
want: func() []detectors.Result {
r := detectors.Result{
DetectorType: detectorspb.DetectorType_Alibaba,
Verified: false,
}
r.SetVerificationError(fmt.Errorf("unexpected HTTP response status 500"))
return []detectors.Result{r}
}(),
wantErr: false,
},
{
name: "found, real secrets, verification error due to broken json",
s: Scanner{client: common.ConstantResponseHttpClient(418, "{")},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a alibaba secret %s within alibaba %s", secret, id)),
verify: true,
},
want: func() []detectors.Result {
r := detectors.Result{
DetectorType: detectorspb.DetectorType_Alibaba,
Verified: false,
}
r.SetVerificationError(fmt.Errorf("unexpected EOF"))
return []detectors.Result{r}
}(),
wantErr: false,
},
{
name: "found, unverified",
s: Scanner{},
@@ -85,8 +140,7 @@ func TestAlibaba_FromChunk(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
s := Scanner{}
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
if (err != nil) != tt.wantErr {
t.Errorf("Alibaba.FromData() error = %v, wantErr %v", err, tt.wantErr)
return
@@ -95,10 +149,20 @@ func TestAlibaba_FromChunk(t *testing.T) {
if len(got[i].Raw) == 0 {
t.Fatalf("no raw secret present: \n %+v", got[i])
}
got[i].Raw = nil
got[i].RawV2 = nil
gotErr := ""
if got[i].VerificationError() != nil {
gotErr = got[i].VerificationError().Error()
}
wantErr := ""
if tt.want[i].VerificationError() != nil {
wantErr = tt.want[i].VerificationError().Error()
}
if gotErr != wantErr {
t.Fatalf("wantVerificationError = %v, verification error = %v", tt.want[i].VerificationError(), got[i].VerificationError())
}
}
if diff := pretty.Compare(got, tt.want); diff != "" {
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError")
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
t.Errorf("Alibaba.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
}
})
pkg/detectors/artifactory/artifactory.go
+50 -26
View File
@@ -2,25 +2,29 @@ package artifactory
import (
"context"
regexp "github.com/wasilibs/go-re2"
"fmt"
"net/http"
"strings"
regexp "github.com/wasilibs/go-re2"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
type Scanner struct{}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
type Scanner struct {
client *http.Client
}
var (
client = common.SaneHttpClient()
// Ensure the Scanner satisfies the interface at compile time.
_ detectors.Detector = (*Scanner)(nil)
defaultClient = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(`\b([a-zA-Z0-9]{73})`)
keyPat = regexp.MustCompile(`\b([a-zA-Z0-9]{73}|\b[a-zA-Z0-9]{64})`)
URLPat = regexp.MustCompile(`\b([A-Za-z0-9](?:[A-Za-z0-9\-]{0,61}[A-Za-z0-9])\.jfrog\.io)`)
)
@@ -30,6 +34,13 @@ func (s Scanner) Keywords() []string {
return []string{"artifactory"}
}
func (s Scanner) getClient() *http.Client {
if s.client != nil {
return s.client
}
return defaultClient
}
// FromData will find and optionally verify Artifactory secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
@@ -42,7 +53,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
continue
}
resURLMatch = strings.TrimSpace(URLmatch[1])
}
for _, match := range matches {
@@ -58,32 +68,46 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
}
if verify {
req, err := http.NewRequestWithContext(ctx, "GET", "https://"+resURLMatch+"/artifactory/api/storageinfo", nil)
if err != nil {
continue
}
req.Header.Add("X-JFrog-Art-Api", resMatch)
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
}
}
client := s.getClient()
isVerified, verificationErr := verifyArtifactory(ctx, client, resURLMatch, resMatch)
s1.Verified = isVerified
s1.SetVerificationError(verificationErr, resMatch)
}
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if !s1.Verified && detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
results = append(results, s1)
}
return results, nil
}
func verifyArtifactory(ctx context.Context, client *http.Client, resURLMatch, resMatch string) (bool, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://"+resURLMatch+"/artifactory/api/storageinfo", nil)
if err != nil {
return false, err
}
req.Header.Add("X-JFrog-Art-Api", resMatch)
res, err := client.Do(req)
if err != nil {
return false, err
}
defer res.Body.Close()
switch res.StatusCode {
case http.StatusOK:
return true, nil
case http.StatusForbidden:
// https://jfrog.com/help/r/jfrog-rest-apis/error-responses
return false, nil
default:
return false, fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
}
}
func (s Scanner) Type() detectorspb.DetectorType {
return detectorspb.DetectorType_ArtifactoryAccessToken
}
pkg/detectors/artifactory/artifactory_test.go
+53 -6
View File
@@ -9,7 +9,8 @@ import (
"testing"
"time"
"github.com/kylelemons/godebug/pretty"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
@@ -19,7 +20,7 @@ import (
func TestArtifactory_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
@@ -55,6 +56,42 @@ func TestArtifactory_FromChunk(t *testing.T) {
},
wantErr: false,
},
{
name: "found, real secrets, verification error due to timeout",
s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a artifactory secret %s and domain %s but not verified", secret, appURL)),
verify: true,
},
want: func() []detectors.Result {
r := detectors.Result{
DetectorType: detectorspb.DetectorType_ArtifactoryAccessToken,
Verified: false,
}
r.SetVerificationError(context.DeadlineExceeded)
return []detectors.Result{r}
}(),
wantErr: false,
},
{
name: "found, real secrets, verification error due to unexpected api surface",
s: Scanner{client: common.ConstantResponseHttpClient(500, "{}")},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a artifactory secret %s and domain %s but not verified", secret, appURL)),
verify: true,
},
want: func() []detectors.Result {
r := detectors.Result{
DetectorType: detectorspb.DetectorType_ArtifactoryAccessToken,
Verified: false,
}
r.SetVerificationError(fmt.Errorf("unexpected HTTP response status 500"))
return []detectors.Result{r}
}(),
wantErr: false,
},
{
name: "found, unverified",
s: Scanner{},
@@ -85,8 +122,7 @@ func TestArtifactory_FromChunk(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
s := Scanner{}
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
if (err != nil) != tt.wantErr {
t.Errorf("Artifactory.FromData() error = %v, wantErr %v", err, tt.wantErr)
return
@@ -95,9 +131,20 @@ func TestArtifactory_FromChunk(t *testing.T) {
if len(got[i].Raw) == 0 {
t.Fatalf("no raw secret present: \n %+v", got[i])
}
got[i].Raw = nil
gotErr := ""
if got[i].VerificationError() != nil {
gotErr = got[i].VerificationError().Error()
}
wantErr := ""
if tt.want[i].VerificationError() != nil {
wantErr = tt.want[i].VerificationError().Error()
}
if gotErr != wantErr {
t.Fatalf("wantVerificationError = %v, verification error = %v", tt.want[i].VerificationError(), got[i].VerificationError())
}
}
if diff := pretty.Compare(got, tt.want); diff != "" {
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError")
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
t.Errorf("Artifactory.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
}
})