trufflesecurity/trufflehog
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Confine symlink state handling to scanSymlink in Filesystem source (#4807)

Browse Source 
* Confine symlink state handling to scanSymlink in Filesystem source

* Fix s.canFollowSymlinks snafu

* Update symlink tests to use starting depth 0

* Missed one

* Remove symlink checking from scanFile; this is now always handled in scanSymlink

* Confine errgroup.Groups to scanDir in the Filesystem source (#4808)

* Move path parameter after rootPath parameter in the Filesystem source

* Move the depth parameter too

* Only create an errgroup.Group inside scanDir (where it's used) in the Filesystem source
This commit is contained in:
Charlie Gunyon
2026-03-17 11:13:10 +01:00
committed by GitHub
parent d17df48451
commit afd5336caa
2 changed files with 47 additions and 125 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/sources/filesystem/filesystem.go
+38 -65
View File
@@ -125,26 +125,15 @@ func (s *Source) Chunks(ctx trContext.Context, chunksChan chan *sources.Chunk, _
}
if fileInfo.Mode()&os.ModeSymlink != 0 {
if !s.canFollowSymlinks() {
// If the file or directory is a symlink but the followSymlinks is disable ignore the path
logger.Info("skipping, following symlinks is not allowed", "path", cleanPath)
continue
}
// if the root path is a symlink we scan the symlink
ctx.Logger().V(5).Info("Root path is a symlink", "path", cleanPath)
workerPool := new(errgroup.Group)
workerPool.SetLimit(s.concurrency)
initialDepth := 1
err = s.scanSymlink(ctx, chunksChan, workerPool, rootPath, initialDepth, cleanPath)
_ = workerPool.Wait()
initialDepth := 0
err = s.scanSymlink(ctx, chunksChan, rootPath, initialDepth, cleanPath)
s.ClearEncodedResumeInfoFor(rootPath)
} else if fileInfo.IsDir() {
ctx.Logger().V(5).Info("Root path is a dir", "path", cleanPath)
workerPool := new(errgroup.Group)
workerPool.SetLimit(s.concurrency)
initialDepth := 1
err = s.scanDir(ctx, chunksChan, workerPool, rootPath, initialDepth, cleanPath)
_ = workerPool.Wait()
initialDepth := 0
err = s.scanDir(ctx, chunksChan, rootPath, initialDepth, cleanPath)
s.ClearEncodedResumeInfoFor(rootPath)
} else {
if !fileInfo.Mode().IsRegular() {
@@ -156,9 +145,7 @@ func (s *Source) Chunks(ctx trContext.Context, chunksChan chan *sources.Chunk, _
}
if err != nil && !errors.Is(err, io.EOF) {
if !errors.Is(err, skipSymlinkErr) {
logger.Error(err, "error scanning filesystem")
}
logger.Error(err, "error scanning filesystem")
}
}
@@ -168,14 +155,22 @@ func (s *Source) Chunks(ctx trContext.Context, chunksChan chan *sources.Chunk, _
func (s *Source) scanSymlink(
ctx trContext.Context,
chunksChan chan *sources.Chunk,
workerPool *errgroup.Group,
rootPath string,
depth int,
path string,
) error {
if !s.canFollowSymlinks() {
// If the file or directory is a symlink but the followSymlinks is disable ignore the path
ctx.Logger().V(2).Info("skipping, following symlinks is not allowed", "path", path)
return nil
}
depth++
if depth > s.maxSymlinkDepth {
return errors.New("max symlink depth reached")
}
cleanPath := filepath.Clean(path)
resolvedPath, err := os.Readlink(cleanPath)
@@ -196,7 +191,7 @@ func (s *Source) scanSymlink(
"resolvedPath", resolvedPath,
"depth", depth,
)
return s.scanSymlink(ctx, chunksChan, workerPool, rootPath, depth+1, resolvedPath)
return s.scanSymlink(ctx, chunksChan, rootPath, depth, resolvedPath)
}
if fileInfo.IsDir() {
@@ -207,7 +202,7 @@ func (s *Source) scanSymlink(
"depth", depth,
)
return s.scanDir(ctx, chunksChan, workerPool, rootPath, depth+1, resolvedPath)
return s.scanDir(ctx, chunksChan, rootPath, depth, resolvedPath)
}
ctx.Logger().V(5).Info(
"found symlink to file",
@@ -223,25 +218,20 @@ func (s *Source) scanSymlink(
// Resume checks are handled by the calling scanDir function.
resumptionKey := rootPath
workerPool.Go(func() error {
if !fileInfo.Mode().Type().IsRegular() {
ctx.Logger().V(5).Info("skipping non-regular file", "path", resolvedPath)
return nil
}
if err := s.scanFile(ctx, chunksChan, resolvedPath); err != nil {
ctx.Logger().Error(err, "error scanning file", "path", resolvedPath)
}
s.SetEncodedResumeInfoFor(resumptionKey, cleanPath)
if !fileInfo.Mode().Type().IsRegular() {
ctx.Logger().V(5).Info("skipping non-regular file", "path", resolvedPath)
return nil
})
}
if err := s.scanFile(ctx, chunksChan, resolvedPath); err != nil {
ctx.Logger().Error(err, "error scanning file", "path", resolvedPath)
}
s.SetEncodedResumeInfoFor(resumptionKey, cleanPath)
return nil
}
func (s *Source) scanDir(
ctx trContext.Context,
chunksChan chan *sources.Chunk,
workerPool *errgroup.Group,
rootPath string,
depth int,
path string,
@@ -285,6 +275,9 @@ func (s *Source) scanDir(
return fmt.Errorf("readdir error: %w", err)
}
workerPool := new(errgroup.Group)
workerPool.SetLimit(s.concurrency)
for _, entry := range entries {
entryPath := filepath.Join(path, entry.Name())
if s.filter != nil && !s.filter.Pass(entryPath) {
@@ -308,7 +301,7 @@ func (s *Source) scanDir(
// traverse into it to find where to resume.
if entry.IsDir() && strings.HasPrefix(resumeAfter, entryPath+string(filepath.Separator)) {
// Recurse into this directory to find the resume point.
if err := s.scanDir(ctx, chunksChan, workerPool, rootPath, depth, entryPath); err != nil {
if err := s.scanDir(ctx, chunksChan, rootPath, depth, entryPath); err != nil {
ctx.Logger().Error(err, "error scanning directory", "path", entryPath)
}
// After recursing, clear local resumeAfter. The child scanDir will have
@@ -323,17 +316,12 @@ func (s *Source) scanDir(
if entry.Type()&os.ModeSymlink != 0 {
ctx.Logger().V(5).Info("Entry found is a symlink", "path", entryPath)
if !s.canFollowSymlinks() {
// If the file or directory is a symlink but the followSymlinks is disable ignore the path
ctx.Logger().Info("skipping, following symlinks is not allowed", "path", entryPath)
continue
}
if err := s.scanSymlink(ctx, chunksChan, workerPool, rootPath, depth, entryPath); err != nil {
if err := s.scanSymlink(ctx, chunksChan, rootPath, depth, entryPath); err != nil {
ctx.Logger().Error(err, "error scanning symlink", "path", entryPath)
}
} else if entry.IsDir() {
ctx.Logger().V(5).Info("Entry found is a directory", "path", entryPath)
if err := s.scanDir(ctx, chunksChan, workerPool, rootPath, depth, entryPath); err != nil {
if err := s.scanDir(ctx, chunksChan, rootPath, depth, entryPath); err != nil {
ctx.Logger().Error(err, "error scanning directory", "path", entryPath)
}
} else {
@@ -351,20 +339,18 @@ func (s *Source) scanDir(
}
}
_ = workerPool.Wait() // [TODO] Handle errors
return nil
}
var skipSymlinkErr = errors.New("skipping symlink")
func (s *Source) scanFile(ctx trContext.Context, chunksChan chan *sources.Chunk, path string) error {
fileCtx := trContext.WithValues(ctx, "path", path)
fileStat, err := os.Lstat(path)
_, err := os.Lstat(path)
if err != nil {
return fmt.Errorf("unable to stat file: %w", err)
}
if fileStat.Mode()&os.ModeSymlink != 0 {
return skipSymlinkErr
}
// Check if file is binary and should be skipped
if (s.skipBinaries || feature.ForceSkipBinaries.Load()) && common.IsBinary(path) {
@@ -435,28 +421,17 @@ func (s *Source) ChunkUnit(ctx trContext.Context, unit sources.SourceUnit, repor
go func() {
defer close(ch)
if fileInfo.Mode()&os.ModeSymlink != 0 {
if !s.canFollowSymlinks() {
// If the file or directory is a symlink but the followSymlinks is disable ignore the path
logger.Info("skipping, following symlinks is not allowed", "path", cleanPath)
return
}
// if the root path is a symlink we scan the symlink
ctx.Logger().V(5).Info("Root path is a symlink", "path", cleanPath)
workerPool := new(errgroup.Group)
workerPool.SetLimit(s.concurrency)
initialDepth := 1
scanErr = s.scanSymlink(ctx, ch, workerPool, rootPath, initialDepth, cleanPath)
_ = workerPool.Wait()
initialDepth := 0
scanErr = s.scanSymlink(ctx, ch, rootPath, initialDepth, cleanPath)
s.ClearEncodedResumeInfoFor(rootPath)
} else if fileInfo.IsDir() {
ctx.Logger().V(5).Info("Root path is a dir", "path", cleanPath)
workerPool := new(errgroup.Group)
workerPool.SetLimit(s.concurrency)
initialDepth := 1
initialDepth := 0
// TODO: Finer grain error tracking of individual chunks.
scanErr = s.scanDir(ctx, ch, workerPool, rootPath, initialDepth, cleanPath)
_ = workerPool.Wait()
scanErr = s.scanDir(ctx, ch, rootPath, initialDepth, cleanPath)
s.ClearEncodedResumeInfoFor(rootPath)
} else {
ctx.Logger().V(5).Info("Root path is a file", "path", cleanPath)
@@ -480,9 +455,7 @@ func (s *Source) ChunkUnit(ctx trContext.Context, unit sources.SourceUnit, repor
}
if scanErr != nil && !errors.Is(scanErr, io.EOF) {
if !errors.Is(scanErr, skipSymlinkErr) {
logger.Error(scanErr, "error scanning filesystem")
}
logger.Error(scanErr, "error scanning filesystem")
return reporter.ChunkErr(ctx, scanErr)
}
return nil
pkg/sources/filesystem/filesystem_symlink_test.go
+9 -60
View File
@@ -8,7 +8,6 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"golang.org/x/sync/errgroup"
"google.golang.org/protobuf/types/known/anypb"
trContext "github.com/trufflesecurity/trufflehog/v3/pkg/context"
@@ -506,10 +505,8 @@ func TestScanSymlink_NoError(t *testing.T) {
}
chunks := make(chan *sources.Chunk, 10)
go func() {
workerPool := new(errgroup.Group)
workerPool.SetLimit(src.concurrency)
err := src.scanSymlink(ctx, chunks, workerPool, filepath.Join(baseDir, "A"), 1, filepath.Join(baseDir, "A"))
_ = workerPool.Wait()
path := filepath.Join(baseDir, "A")
err := src.scanSymlink(ctx, chunks, path, 0, path)
require.NoError(t, err)
close(chunks)
}()
@@ -555,19 +552,11 @@ func TestScanSymlink_MaxDepthExceeded(t *testing.T) {
maxSymlinkDepth: 2,
}
chunks := make(chan *sources.Chunk, 10)
workerPool := new(errgroup.Group)
workerPool.SetLimit(src.concurrency)
err = src.scanSymlink(
ctx,
chunks,
workerPool,
filepath.Join(baseDir, "A"),
1,
filepath.Join(baseDir, "A"),
)
_ = workerPool.Wait()
path := filepath.Join(baseDir, "A")
err = src.scanSymlink(ctx, chunks, path, 0, path)
close(chunks)
require.Error(t, err)
require.EqualError(t, err, "max symlink depth reached")
}
@@ -597,18 +586,8 @@ func TestScanSymlink_FileTarget(t *testing.T) {
}
chunks := make(chan *sources.Chunk, 10)
workerPool := new(errgroup.Group)
workerPool.SetLimit(src.concurrency)
err = src.scanSymlink(
ctx,
chunks,
workerPool,
symlinkPath,
1,
symlinkPath,
)
_ = workerPool.Wait()
err = src.scanSymlink(ctx, chunks, symlinkPath, 0, symlinkPath)
require.NoError(t, err)
close(chunks)
var chunkCount int
@@ -639,18 +618,8 @@ func TestScanSymlink_SelfLoop(t *testing.T) {
}
chunks := make(chan *sources.Chunk, 10)
workerPool := new(errgroup.Group)
workerPool.SetLimit(src.concurrency)
err = src.scanSymlink(
ctx,
chunks,
workerPool,
symlinkPath,
1,
symlinkPath,
)
_ = workerPool.Wait()
err = src.scanSymlink(ctx, chunks, symlinkPath, 0, symlinkPath)
close(chunks)
require.Error(t, err)
require.EqualError(t, err, "max symlink depth reached")
@@ -675,18 +644,8 @@ func TestScanSymlink_BrokenSymlink(t *testing.T) {
}
chunks := make(chan *sources.Chunk, 10)
workerPool := new(errgroup.Group)
workerPool.SetLimit(src.concurrency)
err = src.scanSymlink(
ctx,
chunks,
workerPool,
symlinkPath,
0,
symlinkPath,
)
_ = workerPool.Wait()
err = src.scanSymlink(ctx, chunks, symlinkPath, 0, symlinkPath)
close(chunks)
require.Error(t, err)
require.Contains(t, err.Error(), "lstat error")
@@ -714,18 +673,8 @@ func TestScanSymlink_TwoFileLoop(t *testing.T) {
}
chunks := make(chan *sources.Chunk, 10)
workerPool := new(errgroup.Group)
workerPool.SetLimit(src.concurrency)
err = src.scanSymlink(
ctx,
chunks,
workerPool,
fileA,
0,
fileA,
)
_ = workerPool.Wait()
err = src.scanSymlink(ctx, chunks, fileA, 0, fileA)
close(chunks)
require.Error(t, err)
require.EqualError(t, err, "max symlink depth reached")