MustansirandAmaan Ullah
b6389e2419
* implemented Source unit for S3. Implemented. integration test * use bucket as source unit * remove code duplication, reuse from Chunks * remove unnecessary change * remove unused functions * revisit tests * revert unnecessary change * change SourceUnitKind to s3_bucket * handle nil objectCount inside scanBucket * handle nil objectCount outside loop * add bucket to resume log * add bucket and role to error log, remove enumerating log * implement sub unit resumption * add comment to checkpointer for unit scans * implement SourceUnitUnmarshaller on source with the new S3SourceUnit, add test to test resumption on multiple buckets with concurrent ChunkUnit processing * add role to SourceUnitID * Revert "add role to SourceUnitID" This reverts commit 549e6bede9fe4ed276f7a23f5a5b2bf037133264. * add role to source unit ID, keep track of resumption using source unit ID instead of just bucket name * rename bucket -> unitID in UnmarshalSourceUnit --------- Co-authored-by: Amaan Ullah <[email protected]>
52 lines
1.2 KiB
Go
52 lines
1.2 KiB
Go
package s3
|
|
|
|
import "testing"
|
|
|
|
func TestS3Unit(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
bucket string
|
|
role string
|
|
wantID string
|
|
wantDisplay string
|
|
}{
|
|
{
|
|
name: "Bucket with role",
|
|
bucket: "my-bucket",
|
|
role: "arn:aws:iam::123456789012:role/MyRole",
|
|
wantID: "arn:aws:iam::123456789012:role/MyRole|my-bucket",
|
|
wantDisplay: "Role=arn:aws:iam::123456789012:role/MyRole Bucket=my-bucket",
|
|
},
|
|
{
|
|
name: "Bucket without role",
|
|
bucket: "my-bucket",
|
|
wantID: "my-bucket",
|
|
wantDisplay: "my-bucket",
|
|
},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
unit := S3SourceUnit{
|
|
Bucket: tt.bucket,
|
|
Role: tt.role,
|
|
}
|
|
|
|
gotID, gotKind := unit.SourceUnitID()
|
|
gotDisplay := unit.Display()
|
|
|
|
if gotKind != SourceUnitKindBucket {
|
|
t.Errorf("SourceUnitID() got kind = %v, want %v", gotKind, SourceUnitKindBucket)
|
|
}
|
|
if gotID != tt.wantID {
|
|
t.Errorf("SourceUnitID() got id= %v, want %v", gotID, tt.wantID)
|
|
}
|
|
if gotDisplay != tt.wantDisplay {
|
|
t.Errorf("Display() = %v, want %v", gotDisplay, tt.wantDisplay)
|
|
}
|
|
})
|
|
}
|
|
}
|