trufflesecurity/trufflehog
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(mailgun): update detector (#2679)

Browse Source
This commit is contained in:
Richard Gomez
2024-11-07 08:47:07 -08:00
committed by GitHub
parent afe25cf34a
commit 034ca350ab
2 changed files with 207 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/detectors/mailgun/mailgun.go
+109 -41
View File
@@ -2,7 +2,9 @@ package mailgun
import (
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"strings"
@@ -15,16 +17,17 @@ import (
type Scanner struct {
detectors.DefaultMultiPartCredentialProvider
client *http.Client
}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var (
client = common.SaneHttpClient()
defaultClient = common.SaneHttpClient()
tokenPats = map[string]*regexp.Regexp{
"Original MailGun Token": regexp.MustCompile(detectors.PrefixRegex([]string{"mailgun"}) + `\b([a-zA-Z-0-9]{72})\b`),
"Original MailGun Token": regexp.MustCompile(detectors.PrefixRegex([]string{"mailgun"}) + `\b([a-zA-Z0-9-]{72})\b`),
"Key-MailGun Token": regexp.MustCompile(`\b(key-[a-z0-9]{32})\b`),
"Hex MailGun Token": regexp.MustCompile(`\b([a-f0-9]{32}-[a-f0-9]{8}-[a-f0-9]{8})\b`),
}
@@ -33,55 +36,120 @@ var (
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"mailgun"}
return []string{"mailgun", "key-"}
}
func (s Scanner) getClient() *http.Client {
if s.client != nil {
return s.client
}
return defaultClient
}
// FromData will find and optionally verify Mailgun secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
uniqueMatches := make(map[string]struct{})
for _, tokenPat := range tokenPats {
matches := tokenPat.FindAllStringSubmatch(dataStr, -1)
for _, match := range matches {
if len(match) != 2 {
continue
}
resMatch := strings.TrimSpace(match[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_Mailgun,
Raw: []byte(resMatch),
}
if verify {
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.mailgun.net/v3/domains", nil)
if err != nil {
continue
}
// If resMatch has "key" prefix, use it as the username for basic auth.
if strings.HasPrefix(resMatch, "key-") {
req.SetBasicAuth("api", resMatch)
} else {
req.Header.Add("Authorization", fmt.Sprintf("Basic %s", resMatch))
}
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
}
}
s1.AnalysisInfo = map[string]string{"key": resMatch}
}
results = append(results, s1)
for _, match := range tokenPat.FindAllStringSubmatch(dataStr, -1) {
uniqueMatches[match[1]] = struct{}{}
}
}
return results, nil
for match := range uniqueMatches {
s1 := detectors.Result{
DetectorType: s.Type(),
Raw: []byte(match),
AnalysisInfo: map[string]string{"key": match},
}
if verify {
client := s.getClient()
isVerified, extraData, verificationErr := verifyMatch(ctx, client, match)
s1.Verified = isVerified
s1.ExtraData = extraData
s1.SetVerificationError(verificationErr)
}
results = append(results, s1)
}
return
}
func verifyMatch(ctx context.Context, client *http.Client, token string) (bool, map[string]string, error) {
// https://documentation.mailgun.com/docs/mailgun/api-reference/openapi-final/tag/Domains/
req, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://api.mailgun.net/v3/domains", nil)
if err != nil {
return false, nil, err
}
if len(token) == 72 {
// This matches prior logic, but may not be correct.
req.Header.Add("Authorization", fmt.Sprintf("Basic %s", token))
} else {
// https://documentation.mailgun.com/docs/mailgun/api-reference/authentication/
req.SetBasicAuth("api", token)
}
req.Header.Add("Content-Type", "application/json")
res, err := client.Do(req)
if err != nil {
return false, nil, err
}
defer func() {
_, _ = io.Copy(io.Discard, res.Body)
_ = res.Body.Close()
}()
if res.StatusCode == http.StatusOK {
var domains domainResponse
if err := json.NewDecoder(res.Body).Decode(&domains); err != nil {
return false, nil, fmt.Errorf("error decoding response body: %w", err)
}
var extraData map[string]string
if len(domains.Items) > 0 {
sb := strings.Builder{}
for i, item := range domains.Items {
if i != 0 {
sb.WriteString(", ")
}
sb.WriteString(item.Name)
sb.WriteString(" (")
sb.WriteString(item.State)
sb.WriteString(",")
sb.WriteString(item.Type)
if item.IsDisabled {
sb.WriteString(",disabled")
}
sb.WriteString(")")
}
extraData = map[string]string{
"Domains": sb.String(),
}
}
return true, extraData, nil
} else if res.StatusCode == http.StatusUnauthorized {
return false, nil, nil
} else {
return false, nil, fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
}
}
type domainResponse struct {
TotalCount int `json:"total_count"`
Items []item `json:"items"`
}
type item struct {
ID string `json:"id"`
IsDisabled bool `json:"is_disabled"`
Name string `json:"name"`
State string `json:"state"`
Type string `json:"type"`
}
func (s Scanner) Type() detectorspb.DetectorType {
pkg/detectors/mailgun/mailgun_test.go
+98
View File
@@ -9,13 +9,111 @@ import (
"testing"
"time"
"github.com/google/go-cmp/cmp"
"github.com/kylelemons/godebug/pretty"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
func TestMailgun_Pattern(t *testing.T) {
d := Scanner{}
ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
name string
input string
want []string
}{
// TODO: Confirm that this is actually an "original token".
// It's just a hex token encoded as basic auth.
{
name: "original token",
input: `- request:
method: get
uri: https://api.mailgun.net/v3/integration-test.domain.invalid/templates/test.template
body:
encoding: US-ASCII
string: ''
headers:
Accept:
- "*/*"
User-Agent:
- rest-client/2.1.0 (darwin21.6.0 x86_64) ruby/2.5.1p57
Accept-Encoding:
- gzip;q=1.0,deflate;q=0.6,identity;q=0.3
Host:
- api.mailgun.net
Authorization:
- Basic YXBpOmFjZWM0YzA1YjFmMmZjZWJjZmE4ZGE2NDVkYTEwMjMxLTQxM2UzNzNjLTBhYWQzYzM3`,
want: []string{"YXBpOmFjZWM0YzA1YjFmMmZjZWJjZmE4ZGE2NDVkYTEwMjMxLTQxM2UzNzNjLTBhYWQzYzM3"},
},
{
name: "key- token",
input: `public static ClientResponse GetBounce() {
Client client = new Client();
client.addFilter(new HTTPBasicAuthFilter("api",
"key-3ax63njp29jz6fds4gc373sgvjxteol1"));
WebResource webResource =
client.resource("https://api.mailgun.net/v2/samples.mailgun.org/" +
"bounces/[email protected]");
return webResource.get(ClientResponse.class);
}`,
want: []string{"key-3ax63njp29jz6fds4gc373sgvjxteol1"},
},
{
name: "hex token",
input: `curl -X POST https://api.mailgun.net/v3/DOMAIN.TEST/messages -u "api:e915b5cdb9a582685d8f3fb1bea0f20f-07bc7b05-f14816a1"`,
want: []string{"e915b5cdb9a582685d8f3fb1bea0f20f-07bc7b05-f14816a1"},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
chunkSpecificDetectors := make(map[ahocorasick.DetectorKey]detectors.Detector, 2)
ahoCorasickCore.PopulateMatchingDetectors(test.input, chunkSpecificDetectors)
if len(chunkSpecificDetectors) == 0 {
t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
results, err := d.FromData(context.Background(), false, []byte(test.input))
if err != nil {
t.Errorf("error = %v", err)
return
}
if len(results) != len(test.want) {
if len(results) == 0 {
t.Errorf("did not receive result")
} else {
t.Errorf("expected %d results, only received %d", len(test.want), len(results))
}
return
}
actual := make(map[string]struct{}, len(results))
for _, r := range results {
if len(r.RawV2) > 0 {
actual[string(r.RawV2)] = struct{}{}
} else {
actual[string(r.Raw)] = struct{}{}
}
}
expected := make(map[string]struct{}, len(test.want))
for _, v := range test.want {
expected[v] = struct{}{}
}
if diff := cmp.Diff(expected, actual); diff != "" {
t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
}
})
}
}
func TestMailgun_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()