trufflesteeeveandahmed
114f4b6989
* Add Type() to detector interface The goal here is to allow the detector type information to be used without the need for reflection. This could possibly allow us to more easily inject information into detectors or filter them out if necessary. Co-authored-by: ahmed <[email protected]> * remove test detector --------- Co-authored-by: ahmed <[email protected]>
85 lines
2.4 KiB
Go
85 lines
2.4 KiB
Go
package scrapfly
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"net/http"
|
|
"regexp"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
|
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
|
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
|
)
|
|
|
|
type Scanner struct{}
|
|
|
|
// Ensure the Scanner satisfies the interface at compile time.
|
|
var _ detectors.Detector = (*Scanner)(nil)
|
|
|
|
var (
|
|
client = common.SaneHttpClient()
|
|
|
|
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
|
// examples of api valid keys:
|
|
// - scp-live-03b9e7d0d0024e4b8fccc1ffe923e899 (new format)
|
|
// - scp-test-03b9e7d0d0024e4b8fccc1ffe923e899 (new format)
|
|
// - 03b9e7d0d0024e4b8fccc1ffe923e899 (old format)
|
|
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"scrapfly"}) + `\b([a-z0-9]{32}|scp-(?:live|test)-[a-z0-9]{32})\b`)
|
|
)
|
|
|
|
// Keywords are used for efficiently pre-filtering chunks.
|
|
// Use identifiers in the secret preferably, or the provider name.
|
|
func (s Scanner) Keywords() []string {
|
|
return []string{"scrapfly"}
|
|
}
|
|
|
|
// FromData will find and optionally verify Scrapfly secrets in a given set of bytes.
|
|
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
|
dataStr := string(data)
|
|
|
|
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
|
|
|
for _, match := range matches {
|
|
if len(match) != 2 {
|
|
continue
|
|
}
|
|
resMatch := strings.TrimSpace(match[1])
|
|
|
|
s1 := detectors.Result{
|
|
DetectorType: detectorspb.DetectorType_Scrapfly,
|
|
Raw: []byte(resMatch),
|
|
}
|
|
|
|
if verify {
|
|
timeout := 10 * time.Second
|
|
client.Timeout = timeout
|
|
req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://api.scrapfly.io/scrape?key=%s&url=https://httpbin.org/status/200", resMatch), nil)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
res, err := client.Do(req)
|
|
if err == nil {
|
|
defer res.Body.Close()
|
|
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
|
s1.Verified = true
|
|
} else {
|
|
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
|
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
|
continue
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
results = append(results, s1)
|
|
}
|
|
|
|
return results, nil
|
|
}
|
|
|
|
func (s Scanner) Type() detectorspb.DetectorType {
|
|
return detectorspb.DetectorType_Scrapfly
|
|
}
|