wrap compReader with DiskbufferReader

This commit is contained in:
Ahrav Dutta
2024-04-18 18:32:23 -07:00
parent 330c65edb0
commit 6558bdad5d
3 changed files with 32 additions and 1 deletions
+7 -1
View File
@@ -156,7 +156,13 @@ func (h *defaultHandler) openArchive(ctx logContext.Context, depth int, reader i
h.metrics.incFilesProcessed()
return h.openArchive(ctx, depth+1, compReader, archiveChan)
reReader, err := diskbufferreader.New(compReader)
if err != nil {
return fmt.Errorf("error creating reusable reader: %w", err)
}
defer reReader.Close()
return h.openArchive(ctx, depth+1, reReader, archiveChan)
case archiver.Extractor:
err := archive.Extract(logContext.WithValue(ctx, depthKey, depth+1), arReader, nil, h.extractorHandler(archiveChan))
if err != nil {
+25
View File
@@ -117,6 +117,31 @@ func TestHandleNestedArchives(t *testing.T) {
assert.Equal(t, wantCount, count)
}
func TestHandleCompressedZip(t *testing.T) {
file, err := os.Open("testdata/example.zip.gz")
assert.Nil(t, err)
defer file.Close()
reader, err := diskbufferreader.New(file)
assert.NoError(t, err)
ctx := logContext.Background()
chunkCh := make(chan *sources.Chunk)
go func() {
defer close(chunkCh)
err := HandleFile(ctx, reader, &sources.Chunk{}, sources.ChanReporter{Ch: chunkCh})
assert.NoError(t, err)
}()
wantCount := 2
count := 0
for range chunkCh {
count++
}
assert.Equal(t, wantCount, count)
}
func TestHandleNestedCompressedArchive(t *testing.T) {
file, err := os.Open("testdata/nested-compressed-archive.tar.gz")
assert.Nil(t, err)
Binary file not shown.