Shahzad Haider
54ff78b9a8
* Add S3 object filtering fields to the sources proto * Skip S3 objects by key prefix or file extension before downloading them * Document the S3 source and how to narrow a scan to specific objects * Trim filter entries and report when a filter excludes every object * Document the S3 object filter edge cases and listing cost * Align S3 object filter logging with the logging RFC * Simplify S3 extension matching and always log a fully filtered bucket
81 lines
2.3 KiB
Go
81 lines
2.3 KiB
Go
package engine
|
|
|
|
import (
|
|
"fmt"
|
|
"runtime"
|
|
|
|
"google.golang.org/protobuf/proto"
|
|
"google.golang.org/protobuf/types/known/anypb"
|
|
|
|
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
|
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/credentialspb"
|
|
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
|
|
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
|
"github.com/trufflesecurity/trufflehog/v3/pkg/sources/s3"
|
|
)
|
|
|
|
// ScanS3 scans S3 buckets.
|
|
func (e *Engine) ScanS3(ctx context.Context, c sources.S3Config) (sources.JobProgressRef, error) {
|
|
connection := &sourcespb.S3{
|
|
Credential: &sourcespb.S3_Unauthenticated{},
|
|
}
|
|
if c.CloudCred {
|
|
if len(c.Key) > 0 || len(c.Secret) > 0 || len(c.SessionToken) > 0 {
|
|
return sources.JobProgressRef{}, fmt.Errorf("cannot use cloud environment and static credentials together")
|
|
}
|
|
connection.Credential = &sourcespb.S3_CloudEnvironment{}
|
|
}
|
|
if len(c.Key) > 0 && len(c.Secret) > 0 {
|
|
if len(c.SessionToken) > 0 {
|
|
connection.Credential = &sourcespb.S3_SessionToken{
|
|
SessionToken: &credentialspb.AWSSessionTokenSecret{
|
|
Key: c.Key,
|
|
Secret: c.Secret,
|
|
SessionToken: c.SessionToken,
|
|
},
|
|
}
|
|
} else {
|
|
connection.Credential = &sourcespb.S3_AccessKey{
|
|
AccessKey: &credentialspb.KeySecret{
|
|
Key: c.Key,
|
|
Secret: c.Secret,
|
|
},
|
|
}
|
|
}
|
|
}
|
|
if len(c.Buckets) > 0 {
|
|
connection.Buckets = c.Buckets
|
|
}
|
|
if len(c.IgnoreBuckets) > 0 {
|
|
connection.IgnoreBuckets = c.IgnoreBuckets
|
|
}
|
|
|
|
if len(c.Roles) > 0 {
|
|
connection.Roles = c.Roles
|
|
}
|
|
|
|
connection.IncludePrefixes = c.IncludePrefixes
|
|
connection.ExcludePrefixes = c.ExcludePrefixes
|
|
connection.IncludeExtensions = c.IncludeExtensions
|
|
connection.ExcludeExtensions = c.ExcludeExtensions
|
|
|
|
connection.Endpoint = c.Endpoint
|
|
connection.Region = c.Region
|
|
|
|
var conn anypb.Any
|
|
err := anypb.MarshalFrom(&conn, connection, proto.MarshalOptions{})
|
|
if err != nil {
|
|
ctx.Logger().Error(err, "failed to marshal S3 connection")
|
|
return sources.JobProgressRef{}, err
|
|
}
|
|
|
|
sourceName := "trufflehog - s3"
|
|
sourceID, jobID, _ := e.sourceManager.GetIDs(ctx, sourceName, s3.SourceType)
|
|
|
|
s3Source := &s3.Source{}
|
|
if err := s3Source.Init(ctx, sourceName, jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {
|
|
return sources.JobProgressRef{}, err
|
|
}
|
|
return e.sourceManager.EnumerateAndScan(ctx, sourceName, s3Source)
|
|
}
|