diff --git a/pkg/sources/filesystem/filesystem.go b/pkg/sources/filesystem/filesystem.go index 0760fccee..5aced6666 100644 --- a/pkg/sources/filesystem/filesystem.go +++ b/pkg/sources/filesystem/filesystem.go @@ -125,26 +125,15 @@ func (s *Source) Chunks(ctx trContext.Context, chunksChan chan *sources.Chunk, _ } if fileInfo.Mode()&os.ModeSymlink != 0 { - if !s.canFollowSymlinks() { - // If the file or directory is a symlink but the followSymlinks is disable ignore the path - logger.Info("skipping, following symlinks is not allowed", "path", cleanPath) - continue - } // if the root path is a symlink we scan the symlink ctx.Logger().V(5).Info("Root path is a symlink", "path", cleanPath) - workerPool := new(errgroup.Group) - workerPool.SetLimit(s.concurrency) - initialDepth := 1 - err = s.scanSymlink(ctx, chunksChan, workerPool, rootPath, initialDepth, cleanPath) - _ = workerPool.Wait() + initialDepth := 0 + err = s.scanSymlink(ctx, chunksChan, rootPath, initialDepth, cleanPath) s.ClearEncodedResumeInfoFor(rootPath) } else if fileInfo.IsDir() { ctx.Logger().V(5).Info("Root path is a dir", "path", cleanPath) - workerPool := new(errgroup.Group) - workerPool.SetLimit(s.concurrency) - initialDepth := 1 - err = s.scanDir(ctx, chunksChan, workerPool, rootPath, initialDepth, cleanPath) - _ = workerPool.Wait() + initialDepth := 0 + err = s.scanDir(ctx, chunksChan, rootPath, initialDepth, cleanPath) s.ClearEncodedResumeInfoFor(rootPath) } else { if !fileInfo.Mode().IsRegular() { @@ -156,9 +145,7 @@ func (s *Source) Chunks(ctx trContext.Context, chunksChan chan *sources.Chunk, _ } if err != nil && !errors.Is(err, io.EOF) { - if !errors.Is(err, skipSymlinkErr) { - logger.Error(err, "error scanning filesystem") - } + logger.Error(err, "error scanning filesystem") } } @@ -168,14 +155,22 @@ func (s *Source) Chunks(ctx trContext.Context, chunksChan chan *sources.Chunk, _ func (s *Source) scanSymlink( ctx trContext.Context, chunksChan chan *sources.Chunk, - workerPool *errgroup.Group, rootPath string, depth int, path string, ) error { + if !s.canFollowSymlinks() { + // If the file or directory is a symlink but the followSymlinks is disable ignore the path + ctx.Logger().V(2).Info("skipping, following symlinks is not allowed", "path", path) + return nil + } + + depth++ + if depth > s.maxSymlinkDepth { return errors.New("max symlink depth reached") } + cleanPath := filepath.Clean(path) resolvedPath, err := os.Readlink(cleanPath) @@ -196,7 +191,7 @@ func (s *Source) scanSymlink( "resolvedPath", resolvedPath, "depth", depth, ) - return s.scanSymlink(ctx, chunksChan, workerPool, rootPath, depth+1, resolvedPath) + return s.scanSymlink(ctx, chunksChan, rootPath, depth, resolvedPath) } if fileInfo.IsDir() { @@ -207,7 +202,7 @@ func (s *Source) scanSymlink( "depth", depth, ) - return s.scanDir(ctx, chunksChan, workerPool, rootPath, depth+1, resolvedPath) + return s.scanDir(ctx, chunksChan, rootPath, depth, resolvedPath) } ctx.Logger().V(5).Info( "found symlink to file", @@ -223,25 +218,20 @@ func (s *Source) scanSymlink( // Resume checks are handled by the calling scanDir function. resumptionKey := rootPath - workerPool.Go(func() error { - if !fileInfo.Mode().Type().IsRegular() { - ctx.Logger().V(5).Info("skipping non-regular file", "path", resolvedPath) - return nil - } - if err := s.scanFile(ctx, chunksChan, resolvedPath); err != nil { - ctx.Logger().Error(err, "error scanning file", "path", resolvedPath) - } - s.SetEncodedResumeInfoFor(resumptionKey, cleanPath) + if !fileInfo.Mode().Type().IsRegular() { + ctx.Logger().V(5).Info("skipping non-regular file", "path", resolvedPath) return nil - }) - + } + if err := s.scanFile(ctx, chunksChan, resolvedPath); err != nil { + ctx.Logger().Error(err, "error scanning file", "path", resolvedPath) + } + s.SetEncodedResumeInfoFor(resumptionKey, cleanPath) return nil } func (s *Source) scanDir( ctx trContext.Context, chunksChan chan *sources.Chunk, - workerPool *errgroup.Group, rootPath string, depth int, path string, @@ -285,6 +275,9 @@ func (s *Source) scanDir( return fmt.Errorf("readdir error: %w", err) } + workerPool := new(errgroup.Group) + workerPool.SetLimit(s.concurrency) + for _, entry := range entries { entryPath := filepath.Join(path, entry.Name()) if s.filter != nil && !s.filter.Pass(entryPath) { @@ -308,7 +301,7 @@ func (s *Source) scanDir( // traverse into it to find where to resume. if entry.IsDir() && strings.HasPrefix(resumeAfter, entryPath+string(filepath.Separator)) { // Recurse into this directory to find the resume point. - if err := s.scanDir(ctx, chunksChan, workerPool, rootPath, depth, entryPath); err != nil { + if err := s.scanDir(ctx, chunksChan, rootPath, depth, entryPath); err != nil { ctx.Logger().Error(err, "error scanning directory", "path", entryPath) } // After recursing, clear local resumeAfter. The child scanDir will have @@ -323,17 +316,12 @@ func (s *Source) scanDir( if entry.Type()&os.ModeSymlink != 0 { ctx.Logger().V(5).Info("Entry found is a symlink", "path", entryPath) - if !s.canFollowSymlinks() { - // If the file or directory is a symlink but the followSymlinks is disable ignore the path - ctx.Logger().Info("skipping, following symlinks is not allowed", "path", entryPath) - continue - } - if err := s.scanSymlink(ctx, chunksChan, workerPool, rootPath, depth, entryPath); err != nil { + if err := s.scanSymlink(ctx, chunksChan, rootPath, depth, entryPath); err != nil { ctx.Logger().Error(err, "error scanning symlink", "path", entryPath) } } else if entry.IsDir() { ctx.Logger().V(5).Info("Entry found is a directory", "path", entryPath) - if err := s.scanDir(ctx, chunksChan, workerPool, rootPath, depth, entryPath); err != nil { + if err := s.scanDir(ctx, chunksChan, rootPath, depth, entryPath); err != nil { ctx.Logger().Error(err, "error scanning directory", "path", entryPath) } } else { @@ -351,20 +339,18 @@ func (s *Source) scanDir( } } + _ = workerPool.Wait() // [TODO] Handle errors + return nil } -var skipSymlinkErr = errors.New("skipping symlink") - func (s *Source) scanFile(ctx trContext.Context, chunksChan chan *sources.Chunk, path string) error { fileCtx := trContext.WithValues(ctx, "path", path) - fileStat, err := os.Lstat(path) + + _, err := os.Lstat(path) if err != nil { return fmt.Errorf("unable to stat file: %w", err) } - if fileStat.Mode()&os.ModeSymlink != 0 { - return skipSymlinkErr - } // Check if file is binary and should be skipped if (s.skipBinaries || feature.ForceSkipBinaries.Load()) && common.IsBinary(path) { @@ -435,28 +421,17 @@ func (s *Source) ChunkUnit(ctx trContext.Context, unit sources.SourceUnit, repor go func() { defer close(ch) if fileInfo.Mode()&os.ModeSymlink != 0 { - if !s.canFollowSymlinks() { - // If the file or directory is a symlink but the followSymlinks is disable ignore the path - logger.Info("skipping, following symlinks is not allowed", "path", cleanPath) - return - } // if the root path is a symlink we scan the symlink ctx.Logger().V(5).Info("Root path is a symlink", "path", cleanPath) - workerPool := new(errgroup.Group) - workerPool.SetLimit(s.concurrency) - initialDepth := 1 - scanErr = s.scanSymlink(ctx, ch, workerPool, rootPath, initialDepth, cleanPath) - _ = workerPool.Wait() + initialDepth := 0 + scanErr = s.scanSymlink(ctx, ch, rootPath, initialDepth, cleanPath) s.ClearEncodedResumeInfoFor(rootPath) } else if fileInfo.IsDir() { ctx.Logger().V(5).Info("Root path is a dir", "path", cleanPath) - workerPool := new(errgroup.Group) - workerPool.SetLimit(s.concurrency) - initialDepth := 1 + initialDepth := 0 // TODO: Finer grain error tracking of individual chunks. - scanErr = s.scanDir(ctx, ch, workerPool, rootPath, initialDepth, cleanPath) - _ = workerPool.Wait() + scanErr = s.scanDir(ctx, ch, rootPath, initialDepth, cleanPath) s.ClearEncodedResumeInfoFor(rootPath) } else { ctx.Logger().V(5).Info("Root path is a file", "path", cleanPath) @@ -480,9 +455,7 @@ func (s *Source) ChunkUnit(ctx trContext.Context, unit sources.SourceUnit, repor } if scanErr != nil && !errors.Is(scanErr, io.EOF) { - if !errors.Is(scanErr, skipSymlinkErr) { - logger.Error(scanErr, "error scanning filesystem") - } + logger.Error(scanErr, "error scanning filesystem") return reporter.ChunkErr(ctx, scanErr) } return nil diff --git a/pkg/sources/filesystem/filesystem_symlink_test.go b/pkg/sources/filesystem/filesystem_symlink_test.go index 6af5bbcba..f382fe4ba 100644 --- a/pkg/sources/filesystem/filesystem_symlink_test.go +++ b/pkg/sources/filesystem/filesystem_symlink_test.go @@ -8,7 +8,6 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "golang.org/x/sync/errgroup" "google.golang.org/protobuf/types/known/anypb" trContext "github.com/trufflesecurity/trufflehog/v3/pkg/context" @@ -506,10 +505,8 @@ func TestScanSymlink_NoError(t *testing.T) { } chunks := make(chan *sources.Chunk, 10) go func() { - workerPool := new(errgroup.Group) - workerPool.SetLimit(src.concurrency) - err := src.scanSymlink(ctx, chunks, workerPool, filepath.Join(baseDir, "A"), 1, filepath.Join(baseDir, "A")) - _ = workerPool.Wait() + path := filepath.Join(baseDir, "A") + err := src.scanSymlink(ctx, chunks, path, 0, path) require.NoError(t, err) close(chunks) }() @@ -555,19 +552,11 @@ func TestScanSymlink_MaxDepthExceeded(t *testing.T) { maxSymlinkDepth: 2, } chunks := make(chan *sources.Chunk, 10) - workerPool := new(errgroup.Group) - workerPool.SetLimit(src.concurrency) - err = src.scanSymlink( - ctx, - chunks, - workerPool, - filepath.Join(baseDir, "A"), - 1, - filepath.Join(baseDir, "A"), - ) - _ = workerPool.Wait() + path := filepath.Join(baseDir, "A") + err = src.scanSymlink(ctx, chunks, path, 0, path) close(chunks) + require.Error(t, err) require.EqualError(t, err, "max symlink depth reached") } @@ -597,18 +586,8 @@ func TestScanSymlink_FileTarget(t *testing.T) { } chunks := make(chan *sources.Chunk, 10) - workerPool := new(errgroup.Group) - workerPool.SetLimit(src.concurrency) - err = src.scanSymlink( - ctx, - chunks, - workerPool, - symlinkPath, - 1, - symlinkPath, - ) - _ = workerPool.Wait() + err = src.scanSymlink(ctx, chunks, symlinkPath, 0, symlinkPath) require.NoError(t, err) close(chunks) var chunkCount int @@ -639,18 +618,8 @@ func TestScanSymlink_SelfLoop(t *testing.T) { } chunks := make(chan *sources.Chunk, 10) - workerPool := new(errgroup.Group) - workerPool.SetLimit(src.concurrency) - err = src.scanSymlink( - ctx, - chunks, - workerPool, - symlinkPath, - 1, - symlinkPath, - ) - _ = workerPool.Wait() + err = src.scanSymlink(ctx, chunks, symlinkPath, 0, symlinkPath) close(chunks) require.Error(t, err) require.EqualError(t, err, "max symlink depth reached") @@ -675,18 +644,8 @@ func TestScanSymlink_BrokenSymlink(t *testing.T) { } chunks := make(chan *sources.Chunk, 10) - workerPool := new(errgroup.Group) - workerPool.SetLimit(src.concurrency) - err = src.scanSymlink( - ctx, - chunks, - workerPool, - symlinkPath, - 0, - symlinkPath, - ) - _ = workerPool.Wait() + err = src.scanSymlink(ctx, chunks, symlinkPath, 0, symlinkPath) close(chunks) require.Error(t, err) require.Contains(t, err.Error(), "lstat error") @@ -714,18 +673,8 @@ func TestScanSymlink_TwoFileLoop(t *testing.T) { } chunks := make(chan *sources.Chunk, 10) - workerPool := new(errgroup.Group) - workerPool.SetLimit(src.concurrency) - err = src.scanSymlink( - ctx, - chunks, - workerPool, - fileA, - 0, - fileA, - ) - _ = workerPool.Wait() + err = src.scanSymlink(ctx, chunks, fileA, 0, fileA) close(chunks) require.Error(t, err) require.EqualError(t, err, "max symlink depth reached")