trufflehog/pkg/sources/chunker.go

package sources

import (
	"bufio"
	"errors"
	"fmt"
	"io"

	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
)

const (
	// DefaultChunkSize used by the chunker.
	DefaultChunkSize = 10 * 1024
	// DefaultPeekSize is the size of the peek into the previous chunk.
	DefaultPeekSize = 3 * 1024
	// TotalChunkSize is the total size of a chunk with peek data.
	TotalChunkSize = DefaultChunkSize + DefaultPeekSize
)

type chunkReaderConfig struct {
	chunkSize int
	totalSize int
	peekSize  int
	fileSize  int
}

// ConfigOption is a function that configures a chunker.
type ConfigOption func(*chunkReaderConfig)

// WithChunkSize sets the chunk size.
func WithChunkSize(size int) ConfigOption {
	return func(c *chunkReaderConfig) { c.chunkSize = size }
}

// WithPeekSize sets the peek size.
func WithPeekSize(size int) ConfigOption {
	return func(c *chunkReaderConfig) { c.peekSize = size }
}

// WithFileSize sets the file size.
// Note: If WithChunkSize is also provided, WithChunkSize takes precedence.
func WithFileSize(size int) ConfigOption {
	return func(c *chunkReaderConfig) { c.fileSize = size }
}

// ChunkResult is the output unit of a ChunkReader,
// it contains the data and error of a chunk.
type ChunkResult struct {
	data []byte
	// contentSize is the size of actual content, excluding peek data
	contentSize int
	err         error
}

// Bytes for a ChunkResult.
func (cr ChunkResult) Bytes() []byte {
	return cr.data
}

// ContentSize returns the size of actual content, excluding peek data.
// Use this when you need to process only the unique content of each chunk
// without the overlapping peek portion.
func (cr ChunkResult) ContentSize() int {
	return cr.contentSize
}

// Error for a ChunkResult.
func (cr ChunkResult) Error() error {
	return cr.err
}

// NewChunkResult creates a ChunkResult with the given data and content size.
func NewChunkResult(data []byte, contentSize int) ChunkResult {
	return ChunkResult{data: data, contentSize: contentSize}
}

// NewChunkResultError creates a ChunkResult containing an error.
func NewChunkResultError(err error) ChunkResult {
	return ChunkResult{err: err}
}

const (
	// Size thresholds.
	xsmallFileSizeThreshold = 4 * 1024        // 4KB
	smallFileSizeThreshold  = 10 * 1024       // 10KB
	mediumFileSizeThreshold = 100 * 1024      // 100KB
	largeFileSizeThreshold  = 1 * 1024 * 1024 // 1MB

	// Chunk sizes.
	xsmallFileChunkSize = 1 << 12 // 4KB
	smallFileChunkSize  = 1 << 13 // 8KB
	mediumFileChunkSize = 1 << 14 // 16KB
	largeFileChunkSize  = 1 << 15 // 32KB
	xlargeFileChunkSize = 1 << 16 // 64KB
)

// ChunkReader reads chunks from a reader and returns a channel of chunks and a channel of errors.
// The channel of chunks is closed when the reader is closed.
// This should be used whenever a large amount of data is read from a reader.
// Ex: reading attachments, archives, etc.
type ChunkReader func(ctx context.Context, reader io.Reader) <-chan ChunkResult

// NewChunkReader returns a ChunkReader with the given options.
func NewChunkReader(opts ...ConfigOption) ChunkReader {
	config := applyOptions(opts)
	return createReaderFn(config)
}

func applyOptions(opts []ConfigOption) *chunkReaderConfig {
	// Set defaults.
	config := &chunkReaderConfig{
		chunkSize: DefaultChunkSize, // default
		peekSize:  DefaultPeekSize,  // default
	}

	for _, opt := range opts {
		opt(config)
	}

	// Prioritize chunkSize over fileSize if both are provided.
	if config.fileSize != 0 && config.chunkSize == DefaultChunkSize {
		config.chunkSize = calculateOptimalChunkSize(config.fileSize)
	}

	config.totalSize = config.chunkSize + config.peekSize

	return config
}

func calculateOptimalChunkSize(fileSize int) int {
	switch {
	case fileSize < xsmallFileSizeThreshold:
		return xsmallFileChunkSize
	case fileSize < smallFileSizeThreshold:
		return smallFileChunkSize
	case fileSize < mediumFileSizeThreshold:
		return mediumFileChunkSize
	case fileSize < largeFileSizeThreshold:
		return largeFileChunkSize
	default:
		return xlargeFileChunkSize
	}
}

func createReaderFn(config *chunkReaderConfig) ChunkReader {
	return func(ctx context.Context, reader io.Reader) <-chan ChunkResult {
		return readInChunks(ctx, reader, config)
	}
}

func readInChunks(ctx context.Context, reader io.Reader, config *chunkReaderConfig) <-chan ChunkResult {
	const channelSize = 64
	chunkReader := bufio.NewReaderSize(reader, config.chunkSize)
	chunkResultChan := make(chan ChunkResult, channelSize)

	go func() {
		defer close(chunkResultChan)

		// Defer a panic recovery to handle any panics that occur while reading, which can sometimes unavoidably happen
		// due to third-party library bugs.
		defer func() {
			if r := recover(); r != nil {
				var panicErr error
				if e, ok := r.(error); ok {
					panicErr = e
				} else {
					panicErr = fmt.Errorf("panic occurred: %v", r)
				}
				chunkResultChan <- NewChunkResultError(fmt.Errorf("panic error: %w", panicErr))
			}
		}()

		for {
			chunkBytes := make([]byte, config.totalSize)
			chunkBytes = chunkBytes[:config.chunkSize]
			n, err := io.ReadFull(chunkReader, chunkBytes)

			if n > 0 {
				peekData, _ := chunkReader.Peek(config.totalSize - n)
				chunkBytes = append(chunkBytes[:n], peekData...)
			}

			// If there is an error other than EOF, or if we have read some bytes, send the chunk.
			// io.ReadFull will only return io.EOF when n == 0.
			var chunkRes ChunkResult
			switch {
			case isErrAndNotEOF(err):
				ctx.Logger().Error(err, "error reading chunk")
				chunkRes = NewChunkResultError(err)
			case n > 0:
				chunkRes = NewChunkResult(chunkBytes, n)
			default:
				return
			}

			select {
			case <-ctx.Done():
				return
			case chunkResultChan <- chunkRes:
			}

			if err != nil {
				return
			}
		}
	}()
	return chunkResultChan
}

// reportableErr checks whether the error is one we are interested in flagging.
func isErrAndNotEOF(err error) bool {
	if err == nil {
		return false
	}
	if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) {
		return false
	}
	return true
}