[INS-283] Support following symlinks in filesystem source (#4742)

* enabled symlinks with maximum depth support

* resolved concurrency bugs and added maxDepthOption to cli

* Removed visited path map and tracked symlink depth by maintaining a counter variable

* separated symlink scanning from scanDir

* resolved bugbot comments

* introduced hash as a separator to avoid collisions

main.go

@@ -14,6 +14,7 @@ import (
 	"strings"
 	"sync"
 	"syscall"
+
 	"github.com/alecthomas/kingpin/v2"
 	"github.com/fatih/color"
 	"github.com/felixge/fgprof"
@@ -159,8 +160,9 @@ var (
 	filesystemDirectories = filesystemScan.Flag("directory", "Path to directory to scan. You can repeat this flag.").Strings()
 	// TODO: Add more filesystem scan options. Currently only supports scanning a list of directories.
 	// filesystemScanRecursive = filesystemScan.Flag("recursive", "Scan recursively.").Short('r').Bool()
-	filesystemScanIncludePaths = filesystemScan.Flag("include-paths", "Path to file with newline separated regexes for files to include in scan.").Short('i').String()
+	filesystemScanIncludePaths    = filesystemScan.Flag("include-paths", "Path to file with newline separated regexes for files to include in scan.").Short('i').String()
-	filesystemScanExcludePaths = filesystemScan.Flag("exclude-paths", "Path to file with newline separated regexes for files to exclude in scan.").Short('x').String()
+	filesystemScanExcludePaths    = filesystemScan.Flag("exclude-paths", "Path to file with newline separated regexes for files to exclude in scan.").Short('x').String()
+	filesystemScanMaxSymlinkDepth = filesystemScan.Flag("max-symlink-depth", "Maximum depth to follow symlinks during filesystem scan.").Short('s').Int32()
 
 	s3Scan              = cli.Command("s3", "Find credentials in S3 buckets.")
 	s3ScanKey           = s3Scan.Flag("key", "S3 key used to authenticate. Can be provided with environment variable AWS_ACCESS_KEY_ID.").Envar("AWS_ACCESS_KEY_ID").String()
@@ -906,6 +908,7 @@ func runSingleScan(ctx context.Context, cmd string, cfg engine.Config) (metrics,
 			Paths:            paths,
 			IncludePathsFile: *filesystemScanIncludePaths,
 			ExcludePathsFile: *filesystemScanExcludePaths,
+			MaxSymlinkDepth:  *filesystemScanMaxSymlinkDepth,
 		}
 		if ref, err := eng.ScanFileSystem(ctx, cfg); err != nil {
 			return scanMetrics, fmt.Errorf("failed to scan filesystem: %v", err)

pkg/engine/filesystem.go

@@ -18,6 +18,7 @@ func (e *Engine) ScanFileSystem(ctx context.Context, c sources.FilesystemConfig)
 		Paths:            c.Paths,
 		IncludePathsFile: c.IncludePathsFile,
 		ExcludePathsFile: c.ExcludePathsFile,
+		MaxSymlinkDepth:  c.MaxSymlinkDepth,
 	}
 	var conn anypb.Any
 	err := anypb.MarshalFrom(&conn, connection, proto.MarshalOptions{})

pkg/pb/sourcespb/sources.pb.validate.go

@@ -1703,6 +1703,8 @@ func (m *Filesystem) validate(all bool) error {
 
 	// no validation rules for SkipBinaries
 
+	// no validation rules for MaxSymlinkDepth
+
 	if len(errors) > 0 {
 		return FilesystemMultiError(errors)
 	}

pkg/sources/filesystem/filesystem.go

@@ -3,7 +3,6 @@ package filesystem
 import (
 	"fmt"
 	"io"
-	"io/fs"
 	"os"
 	"path/filepath"
 
@@ -37,6 +36,7 @@ type Source struct {
 	skipBinaries bool
 	sources.Progress
 	sources.CommonSourceUnitUnmarshaller
+	maxSymlinkDepth int
 }
 
 // Ensure the Source satisfies the interfaces at compile time
@@ -44,6 +44,9 @@ var _ sources.Source = (*Source)(nil)
 var _ sources.SourceUnitUnmarshaller = (*Source)(nil)
 var _ sources.SourceUnitEnumChunker = (*Source)(nil)
 
+// max symlink depth allowed
+const defaultMaxSymlinkDepth = 40
+
 // Type returns the type of source.
 // It is used for matching source types in configuration and job input.
 func (s *Source) Type() sourcespb.SourceType {
@@ -80,10 +83,30 @@ func (s *Source) Init(aCtx context.Context, name string, jobId sources.JobID, so
 		return fmt.Errorf("unable to create filter: %w", err)
 	}
 	s.filter = filter
-
+	err = s.setMaxSymlinkDepth(&conn)
+	if err != nil {
+		return err
+	}
 	return nil
 }
 
+func (s *Source) setMaxSymlinkDepth(conn *sourcespb.Filesystem) error {
+	depth := int(conn.GetMaxSymlinkDepth())
+	if depth > defaultMaxSymlinkDepth {
+		return fmt.Errorf(
+			"specified symlink depth %d exceeds the allowed max of %d",
+			depth,
+			defaultMaxSymlinkDepth,
+		)
+	}
+	s.maxSymlinkDepth = depth
+	return nil
+}
+
+func (s *Source) canFollowSymlinks() bool {
+	return s.maxSymlinkDepth > 0
+}
+
 // Chunks emits chunks of bytes over a channel.
 func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ ...sources.ChunkingTarget) error {
 	for i, path := range s.paths {
@@ -101,13 +124,33 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ .
 		}
 
 		if fileInfo.Mode()&os.ModeSymlink != 0 {
-			logger.Info("skipping, not a regular file", "path", cleanPath)
+			if !s.canFollowSymlinks() {
-			continue
+				// If the file or directory is a symlink but the followSymlinks is disable ignore the path
-		}
+				logger.Info("skipping, following symlinks is not allowed", "path", cleanPath)
-
+				continue
-		if fileInfo.IsDir() {
+			}
-			err = s.scanDir(ctx, cleanPath, chunksChan)
+			// if the root path is a symlink we scan the symlink
+			ctx.Logger().V(5).Info("Root path is a symlink", "path", cleanPath)
+			workerPool := new(errgroup.Group)
+			workerPool.SetLimit(s.concurrency)
+			initialDepth := 1
+			err = s.scanSymlink(ctx, cleanPath, chunksChan, workerPool, initialDepth, path)
+			_ = workerPool.Wait()
+			s.ClearEncodedResumeContainingId(path + "#")
+		} else if fileInfo.IsDir() {
+			ctx.Logger().V(5).Info("Root path is a dir", "path", cleanPath)
+			workerPool := new(errgroup.Group)
+			workerPool.SetLimit(s.concurrency)
+			initialDepth := 1
+			err = s.scanDir(ctx, cleanPath, chunksChan, workerPool, initialDepth, path)
+			_ = workerPool.Wait()
+			s.ClearEncodedResumeContainingId(path + "#")
 		} else {
+			if !fileInfo.Mode().IsRegular() {
+				logger.Info("skipping non-regular file", "path", cleanPath)
+				continue
+			}
+			ctx.Logger().V(5).Info("Root path is a file", "path", cleanPath)
 			err = s.scanFile(ctx, cleanPath, chunksChan)
 		}
 
@@ -120,61 +163,149 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ .
 
 	return nil
 }
-func (s *Source) scanDir(ctx context.Context, path string, chunksChan chan *sources.Chunk) error {
+
-	workerPool := new(errgroup.Group)
+func (s *Source) scanSymlink(
-	workerPool.SetLimit(s.concurrency)
+	ctx context.Context,
-	defer func() {
+	path string,
-		_ = workerPool.Wait()
+	chunksChan chan *sources.Chunk,
-		s.ClearEncodedResumeInfoFor(path)
+	workerPool *errgroup.Group,
-	}()
+	depth int,
-	startState := s.GetEncodedResumeInfoFor(path)
+	rootPath string,
+) error {
+	if depth > s.maxSymlinkDepth {
+		return errors.New("max symlink depth reached")
+	}
+	path = filepath.Clean(path)
+
+	resolvedPath, err := os.Readlink(path)
+	if err != nil {
+		return fmt.Errorf("readlink error: %w", err)
+	}
+	if !filepath.IsAbs(resolvedPath) {
+		resolvedPath = filepath.Join(filepath.Dir(path), resolvedPath)
+	}
+	fileInfo, err := os.Lstat(resolvedPath)
+	if err != nil {
+		return fmt.Errorf("lstat error: %w", err)
+	}
+	if fileInfo.Mode()&os.ModeSymlink != 0 {
+		ctx.Logger().V(5).Info(
+			"found symlink to symlink",
+			"symlinkPath", path,
+			"resolvedPath", resolvedPath,
+			"depth", depth,
+		)
+		return s.scanSymlink(ctx, resolvedPath, chunksChan, workerPool, depth+1, rootPath)
+	}
+
+	if fileInfo.IsDir() {
+		ctx.Logger().V(5).Info(
+			"found symlink to dir",
+			"symlinkPath", path,
+			"resolvedPath", resolvedPath,
+			"depth", depth,
+		)
+
+		return s.scanDir(ctx, resolvedPath, chunksChan, workerPool, depth+1, rootPath)
+	}
+	ctx.Logger().V(5).Info(
+		"found symlink to file",
+		"symlinkPath", path,
+		"resolvedPath", resolvedPath,
+		"depth", depth,
+	)
+	if s.filter != nil && !s.filter.Pass(resolvedPath) {
+		return nil
+	}
+	resumptionKey := rootPath + "#" + path
+	startState := s.GetEncodedResumeInfoFor(resumptionKey)
+	resuming := startState != ""
+	if resuming && startState == resolvedPath {
+		ctx.Logger().V(5).Info("skipping symlink, already scanned", "path", resolvedPath)
+		return nil
+	}
+	workerPool.Go(func() error {
+		if !fileInfo.Mode().Type().IsRegular() {
+			ctx.Logger().V(5).Info("skipping non-regular file", "path", resolvedPath)
+			return nil
+		}
+		if err := s.scanFile(ctx, resolvedPath, chunksChan); err != nil {
+			ctx.Logger().Error(err, "error scanning file", "path", resolvedPath)
+		}
+		s.SetEncodedResumeInfoFor(resumptionKey, resolvedPath)
+		return nil
+	})
+
+	return nil
+}
+
+func (s *Source) scanDir(
+	ctx context.Context,
+	path string,
+	chunksChan chan *sources.Chunk,
+	workerPool *errgroup.Group,
+	depth int,
+	rootPath string,
+) error {
+	// check if the full path is not matching any pattern in include
+	// FilterRuleSet and matching any exclude FilterRuleSet.
+	resumptionKey := rootPath + "#" + path
+	if s.filter != nil && s.filter.ShouldExclude(path) {
+		return nil
+	}
+	startState := s.GetEncodedResumeInfoFor(resumptionKey)
 	resuming := startState != ""
 
-	return fs.WalkDir(os.DirFS(path), ".", func(relativePath string, d fs.DirEntry, err error) error {
+	ctx.Logger().V(5).Info("Full path found is", "fullPath", path)
-		if err != nil {
-			ctx.Logger().Error(err, "error walking directory")
-			return nil
-		}
 
-		fullPath := filepath.Join(path, relativePath)
+	entries, err := os.ReadDir(path)
+	if err != nil {
+		return fmt.Errorf("readdir error: %w", err)
+	}
 
-		// check if the full path is not matching any pattern in include FilterRuleSet and matching any exclude FilterRuleSet.
+	for _, entry := range entries {
-		if s.filter != nil && !s.filter.Pass(fullPath) {
+		entryPath := filepath.Join(path, entry.Name())
-			// skip excluded directories
+		if s.filter != nil && !s.filter.Pass(entryPath) {
-			if d.IsDir() && s.filter.ShouldExclude(fullPath) {
+			if !entry.IsDir() && entry.Type()&os.ModeSymlink == 0 {
-				return fs.SkipDir
+				continue
 			}
-
-			return nil // skip the file
-		}
-
-		// Skip over non-regular files. We do this check here to suppress noisy
-		// logs for trying to scan directories and other non-regular files in
-		// our traversal.
-		if !d.Type().IsRegular() {
-			return nil
 		}
 
 		if resuming {
-			// The start state holds the path that last completed
+			if entryPath == startState {
-			// scanning. When we find it, we can start scanning
-			// again on the next one.
-			if fullPath == startState {
 				resuming = false
 			}
-			return nil
+		} else if entry.Type()&os.ModeSymlink != 0 {
-		}
+			ctx.Logger().V(5).Info("Entry found is a symlink", "path", entryPath)
-
+			if !s.canFollowSymlinks() {
-		workerPool.Go(func() error {
+				// If the file or directory is a symlink but the followSymlinks is disable ignore the path
-			if err = s.scanFile(ctx, fullPath, chunksChan); err != nil {
+				ctx.Logger().Info("skipping, following symlinks is not allowed", "path", entryPath)
-				ctx.Logger().Error(err, "error scanning file", "path", fullPath, "error", err)
+				continue
 			}
-			s.SetEncodedResumeInfoFor(path, fullPath)
+			if err := s.scanSymlink(ctx, entryPath, chunksChan, workerPool, depth, rootPath); err != nil {
-			return nil
+				ctx.Logger().Error(err, "error scanning symlink", "path", entryPath)
-		})
+			}
+		} else if entry.IsDir() {
+			ctx.Logger().V(5).Info("Entry found is a directory", "path", entryPath)
+			if err := s.scanDir(ctx, entryPath, chunksChan, workerPool, depth, rootPath); err != nil {
+				ctx.Logger().Error(err, "error scanning directory", "path", entryPath)
+			}
+		} else {
+			if !entry.Type().IsRegular() {
+				continue
+			}
+			ctx.Logger().V(5).Info("Entry found is a file", "path", entryPath)
+			workerPool.Go(func() error {
+				if err := s.scanFile(ctx, entryPath, chunksChan); err != nil {
+					ctx.Logger().Error(err, "error scanning file", "path", entryPath)
+				}
+				s.SetEncodedResumeInfoFor(resumptionKey, entryPath)
+				return nil
+			})
+		}
+	}
 
-		return nil
+	return nil
-	})
 }
 
 var skipSymlinkErr = errors.New("skipping symlink")
@@ -251,17 +382,44 @@ func (s *Source) ChunkUnit(ctx context.Context, unit sources.SourceUnit, reporte
 	if err != nil {
 		return reporter.ChunkErr(ctx, fmt.Errorf("unable to get file info: %w", err))
 	}
+	// This will always be the FileInfo we use to decide dir vs file
 
 	ch := make(chan *sources.Chunk)
 	var scanErr error
 	go func() {
 		defer close(ch)
-		if fileInfo.IsDir() {
+		if fileInfo.Mode()&os.ModeSymlink != 0 {
+			if !s.canFollowSymlinks() {
+				// If the file or directory is a symlink but the followSymlinks is disable ignore the path
+				logger.Info("skipping, following symlinks is not allowed", "path", cleanPath)
+				return
+			}
+			// if the root path is a symlink we scan the symlink
+			ctx.Logger().V(5).Info("Root path is a symlink", "path", cleanPath)
+			workerPool := new(errgroup.Group)
+			workerPool.SetLimit(s.concurrency)
+			initialDepth := 1
+			scanErr = s.scanSymlink(ctx, cleanPath, ch, workerPool, initialDepth, path)
+			_ = workerPool.Wait()
+			s.ClearEncodedResumeContainingId(path + "#")
+
+		} else if fileInfo.IsDir() {
+			ctx.Logger().V(5).Info("Root path is a dir", "path", cleanPath)
+			workerPool := new(errgroup.Group)
+			workerPool.SetLimit(s.concurrency)
+			initialDepth := 1
 			// TODO: Finer grain error tracking of individual chunks.
-			scanErr = s.scanDir(ctx, cleanPath, ch)
+			scanErr = s.scanDir(ctx, cleanPath, ch, workerPool, initialDepth, path)
+			_ = workerPool.Wait()
+			s.ClearEncodedResumeContainingId(path + "#")
 		} else {
+			ctx.Logger().V(5).Info("Root path is a file", "path", cleanPath)
 			// TODO: Finer grain error tracking of individual
 			// chunks (in the case of archives).
+			if !fileInfo.Mode().IsRegular() {
+				logger.Info("skipping non-regular file", "path", cleanPath)
+				return
+			}
 			scanErr = s.scanFile(ctx, cleanPath, ch)
 		}
 	}()

pkg/sources/filesystem/filesystem_symlink_test.go

@@ -0,0 +1,732 @@
+package filesystem
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"golang.org/x/sync/errgroup"
+	"google.golang.org/protobuf/types/known/anypb"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/sourcestest"
+)
+
+func probeSymlinkSupport(t *testing.T, baseDir string) {
+	probe := filepath.Join(baseDir, "symlink-probe")
+	if err := os.Symlink("x", probe); err != nil {
+		t.Skip("symlinks not supported")
+	}
+	_ = os.Remove(probe)
+}
+
+func TestScanDir_VisitedPath_PreventInfiniteRecursion(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*3)
+	defer cancel()
+	baseDir, cleanup, err := createTempDir("")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer cleanup()
+	// Skip if symlinks unsupported
+	probeSymlinkSupport(t, baseDir)
+
+	dirA := filepath.Join(baseDir, "A")
+	dirB := filepath.Join(baseDir, "B")
+	err = os.Mkdir(dirA, 0755)
+	if err != nil {
+		t.Fatalf("Unable to create directory A %v", err)
+	}
+	err = os.Mkdir(dirB, 0755)
+	if err != nil {
+		t.Fatalf("Unable to create directory B %v", err)
+	}
+
+	// We create
+	// A/linkToB -> /B
+	// B/linkToA -> /A
+	err = os.Symlink(dirB, filepath.Join(dirA, "linkToB"))
+	if err != nil {
+		t.Fatalf("Unable to create symlink %v", err)
+	}
+	err = os.Symlink(dirA, filepath.Join(dirB, "linkToA"))
+	if err != nil {
+		t.Fatalf("Unable to create symlink %v", err)
+	}
+
+	src := &Source{
+		concurrency:     1,
+		maxSymlinkDepth: 20,
+	}
+	chunks := make(chan *sources.Chunk, 10)
+	go func() {
+		err := src.Chunks(ctx, chunks)
+		require.NoError(t, err)
+		close(chunks)
+	}()
+	var chunkCount int
+	for range chunks {
+		chunkCount++
+	}
+	// Assert no chunks were emitted due to the infinite symlink loop
+	require.Equal(t, 0, chunkCount, "No chunks should be processed due to infinite symlink loop")
+}
+
+func TestChunks_DirectorySymlinkLoop(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*3)
+	defer cancel()
+	baseDir, cleanup, err := createTempDir("")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer cleanup()
+
+	probeSymlinkSupport(t, baseDir)
+
+	// We Create
+	// /A->/B
+	// /B->/A
+	err = os.Symlink(filepath.Join(baseDir, "B"), filepath.Join(baseDir, "A"))
+	if err != nil {
+		t.Fatalf("Unable to create symlink %v", err)
+	}
+	err = os.Symlink(filepath.Join(baseDir, "A"), filepath.Join(baseDir, "B"))
+	if err != nil {
+		t.Fatalf("Unable to create symlink %v", err)
+	}
+
+	src := &Source{
+		maxSymlinkDepth: 20,
+		concurrency:     1,
+		paths:           []string{filepath.Join(baseDir, "B")},
+	}
+
+	chunks := make(chan *sources.Chunk, 10)
+	// Run the scan
+	go func() {
+		err := src.Chunks(ctx, chunks)
+		require.NoError(t, err)
+		close(chunks)
+	}()
+	var chunkCount int
+	for range chunks {
+		chunkCount++
+	}
+	// Assert no chunks were emitted due to the infinite symlink loop
+	require.Equal(t, 0, chunkCount, "No chunks should be processed due to infinite symlink loop")
+}
+
+func TestChunkUnit_DirectorySymlinkLoop(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*3)
+	defer cancel()
+	baseDir, cleanup, err := createTempDir("")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer cleanup()
+
+	probeSymlinkSupport(t, baseDir)
+
+	// We Create
+	// /A->/B
+	// /B->/A
+	err = os.Symlink(filepath.Join(baseDir, "B"), filepath.Join(baseDir, "A"))
+	if err != nil {
+		t.Fatalf("Unable to create symlink %v", err)
+	}
+	err = os.Symlink(filepath.Join(baseDir, "A"), filepath.Join(baseDir, "B"))
+	if err != nil {
+		t.Fatalf("Unable to create symlink %v", err)
+	}
+
+	conn, err := anypb.New(&sourcespb.Filesystem{
+		MaxSymlinkDepth: 20,
+	})
+	assert.NoError(t, err)
+
+	// Initialize the source.
+	s := Source{}
+	err = s.Init(ctx, "test chunk unit", 0, 0, true, conn, 1)
+	assert.NoError(t, err)
+
+	reporter := sourcestest.TestReporter{}
+	err = s.ChunkUnit(ctx, sources.CommonSourceUnit{
+		ID: filepath.Join(baseDir, "B"),
+	}, &reporter)
+	assert.NoError(t, err)
+	// Assert no chunks were emitted due to the infinite symlink loop
+	assert.Equal(t, 0, len(reporter.Chunks))
+}
+
+func TestChunks_FileSymlinkLoop(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*3)
+	defer cancel()
+	baseDir, cleanup, err := createTempDir("")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer cleanup()
+
+	probeSymlinkSupport(t, baseDir)
+
+	// We Create
+	// /fileA->/fileB
+	// /fileB->/fileA
+	fileA := filepath.Join(baseDir, "fileA.txt")
+	fileB := filepath.Join(baseDir, "fileB.txt")
+	err = os.Symlink(fileA, fileB)
+	if err != nil {
+		t.Fatalf("Unable to create symlink %v", err)
+	}
+	err = os.Symlink(fileB, fileA)
+	if err != nil {
+		t.Fatalf("Unable to create symlink %v", err)
+	}
+
+	src := &Source{
+		maxSymlinkDepth: 20,
+		concurrency:     1,
+		paths:           []string{fileA},
+	}
+
+	chunks := make(chan *sources.Chunk, 10)
+	// Run the scan
+	go func() {
+		err := src.Chunks(ctx, chunks)
+		require.NoError(t, err)
+		close(chunks)
+	}()
+	var chunkCount int
+	for range chunks {
+		chunkCount++
+	}
+	// Assert no chunks were emitted due to the infinite symlink loop
+	require.Equal(t, 0, chunkCount, "No chunks should be processed due to infinite symlink loop")
+}
+
+func TestChunkUnit_FileSymlinkLoop(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
+	defer cancel()
+
+	baseDir, cleanup, err := createTempDir("")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer cleanup()
+
+	// Probe symlink support
+	probeSymlinkSupport(t, baseDir)
+
+	// Create two files that are symlinks to each other
+	fileA := filepath.Join(baseDir, "fileA.txt")
+	fileB := filepath.Join(baseDir, "fileB.txt")
+
+	if err := os.Symlink(fileB, fileA); err != nil {
+		t.Fatalf("Unable to create symlink %v", err)
+	}
+	if err := os.Symlink(fileA, fileB); err != nil {
+		t.Fatalf("Unable to create symlink %v", err)
+	}
+
+	conn, err := anypb.New(&sourcespb.Filesystem{
+		MaxSymlinkDepth: 20,
+	})
+	require.NoError(t, err)
+
+	s := Source{}
+	err = s.Init(ctx, "test chunk unit", 0, 0, true, conn, 1)
+	require.NoError(t, err)
+
+	reporter := sourcestest.TestReporter{}
+	err = s.ChunkUnit(ctx, sources.CommonSourceUnit{
+		ID: fileA,
+	}, &reporter)
+	require.NoError(t, err)
+
+	// Assert no chunks were emitted due to the infinite symlink loop
+	assert.Equal(t, 0, len(reporter.Chunks), "No chunks should be processed due to infinite symlink loop")
+}
+
+func TestChunks_ValidDirectorySymlink(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
+	defer cancel()
+
+	// Create a temporary base directory
+	baseDir, cleanup, err := createTempDir("")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer cleanup()
+
+	// Probe symlink support
+	probeSymlinkSupport(t, baseDir)
+
+	// Create a real file
+	dirA := filepath.Join(baseDir, "A")
+	err = os.Mkdir(dirA, 0755)
+	if err != nil {
+		t.Fatalf("Unable to create directory A %v", err)
+	}
+	dirB := filepath.Join(baseDir, "B")
+	err = os.Mkdir(dirB, 0755)
+	if err != nil {
+		t.Fatalf("Unable to create directory B %v", err)
+	}
+
+	data := "Hello world!"
+	file, cleanupFile, err := createTempFile(dirA, data)
+	assert.NoError(t, err)
+	defer cleanupFile()
+
+	// we create
+	// /B/link.txt->/A/trufflehogtest*
+	linkFile := filepath.Join(dirB, "link.txt")
+	if err := os.Symlink(file.Name(), linkFile); err != nil {
+		t.Fatalf("failed to create symlink: %v", err)
+	}
+
+	src := &Source{
+		concurrency:     1,
+		paths:           []string{dirB},
+		maxSymlinkDepth: 20,
+	}
+
+	chunksCh := make(chan *sources.Chunk, 1)
+	go func() {
+		defer close(chunksCh)
+		err = src.Chunks(ctx, chunksCh)
+		require.NoError(t, err)
+	}()
+	if err != nil {
+		t.Fatalf("unexpected error scanning symlink: %v", err)
+	}
+
+	for chunk := range chunksCh {
+		if string(chunk.Data) != data {
+			t.Fatalf("expected chunk.Data: %v to be equal to %v", string(chunk.Data), data)
+		}
+	}
+}
+
+func TestChunkUnit_ValidDirectorySymlink(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
+	defer cancel()
+
+	// Create a temporary base directory
+	baseDir, cleanup, err := createTempDir("")
+	require.NoError(t, err)
+	defer cleanup()
+
+	// Probe symlink support
+	probeSymlinkSupport(t, baseDir)
+
+	// Create directories
+	dirA := filepath.Join(baseDir, "A")
+	dirB := filepath.Join(baseDir, "B")
+	require.NoError(t, os.Mkdir(dirA, 0755))
+	require.NoError(t, os.Mkdir(dirB, 0755))
+
+	// Create a file in dirA
+	data := "Hello world!"
+	file, cleanupFile, err := createTempFile(dirA, data)
+	require.NoError(t, err)
+	defer cleanupFile()
+
+	// Create symlink: /B/link.txt -> /A/trufflehogtest*
+	linkFile := filepath.Join(dirB, "link.txt")
+	require.NoError(t, os.Symlink(file.Name(), linkFile))
+
+	// Prepare Source
+	conn, err := anypb.New(&sourcespb.Filesystem{
+		MaxSymlinkDepth: 20,
+	})
+	require.NoError(t, err)
+
+	src := Source{}
+	require.NoError(t, src.Init(ctx, "test chunk unit", 0, 0, true, conn, 1))
+
+	reporter := sourcestest.TestReporter{}
+	err = src.ChunkUnit(ctx, sources.CommonSourceUnit{
+		ID: dirB,
+	}, &reporter)
+	require.NoError(t, err)
+
+	// Assert exactly 1 chunk is scanned and data matches
+	require.Len(t, reporter.Chunks, 1, "Expected exactly 1 chunk from symlinked file")
+	require.Equal(t, data, string(reporter.Chunks[0].Data))
+}
+
+func TestChunks_ValidFileSymlink(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
+	defer cancel()
+
+	// Create a temporary base directory
+	baseDir, cleanup, err := createTempDir("")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer cleanup()
+
+	// Probe symlink support
+	probeSymlinkSupport(t, baseDir)
+
+	// Create a real file
+	dirA := filepath.Join(baseDir, "A")
+	err = os.Mkdir(dirA, 0755)
+	if err != nil {
+		t.Fatalf("Unable to create directory A %v", err)
+	}
+	dirB := filepath.Join(baseDir, "B")
+	err = os.Mkdir(dirB, 0755)
+	if err != nil {
+		t.Fatalf("Unable to create directory B %v", err)
+	}
+
+	data := "Hello world!"
+	file, cleanupFile, err := createTempFile(dirA, data)
+	assert.NoError(t, err)
+	defer cleanupFile()
+
+	// we create
+	// /B/link.txt->/A/trufflehogtest*
+	linkFile := filepath.Join(dirB, "link.txt")
+	if err := os.Symlink(file.Name(), linkFile); err != nil {
+		t.Fatalf("failed to create symlink: %v", err)
+	}
+
+	src := &Source{
+		concurrency:     1,
+		maxSymlinkDepth: 20,
+		paths:           []string{linkFile},
+	}
+
+	chunksCh := make(chan *sources.Chunk, 1)
+
+	go func() {
+		defer close(chunksCh)
+		err = src.Chunks(ctx, chunksCh)
+		if err != nil {
+			t.Errorf("src.scanFile() error=%v", err)
+		}
+	}()
+
+	if err != nil {
+		t.Fatalf("unexpected error scanning symlink: %v", err)
+	}
+
+	for chunk := range chunksCh {
+		if string(chunk.Data) != data {
+			t.Fatalf("expected chunk.Data: %v to be equal to %v", string(chunk.Data), data)
+		}
+	}
+}
+
+func TestChunkUnit_ValidFileSymlink(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
+	defer cancel()
+
+	// Create a temporary base directory
+	baseDir, cleanup, err := createTempDir("")
+	require.NoError(t, err)
+	defer cleanup()
+
+	probeSymlinkSupport(t, baseDir)
+
+	dirA := filepath.Join(baseDir, "A")
+	dirB := filepath.Join(baseDir, "B")
+	require.NoError(t, os.Mkdir(dirA, 0755))
+	require.NoError(t, os.Mkdir(dirB, 0755))
+
+	data := "Hello world!"
+	file, cleanupFile, err := createTempFile(dirA, data)
+	require.NoError(t, err)
+	defer cleanupFile()
+
+	// Create symlink: /B/link.txt -> /A/trufflehogtest*
+	linkFile := filepath.Join(dirB, "link.txt")
+	require.NoError(t, os.Symlink(file.Name(), linkFile))
+
+	conn, err := anypb.New(&sourcespb.Filesystem{
+		MaxSymlinkDepth: 20,
+	})
+	require.NoError(t, err)
+
+	src := Source{}
+	require.NoError(t, src.Init(ctx, "test chunk unit", 0, 0, true, conn, 1))
+
+	reporter := sourcestest.TestReporter{}
+	err = src.ChunkUnit(ctx, sources.CommonSourceUnit{
+		ID: linkFile,
+	}, &reporter)
+	require.NoError(t, err)
+
+	// Assert exactly 1 chunk scanned and data matches
+	require.Len(t, reporter.Chunks, 1, "Expected exactly 1 chunk from symlinked file")
+	require.Equal(t, data, string(reporter.Chunks[0].Data))
+}
+
+func TestScanSymlink_NoError(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*3)
+	defer cancel()
+	baseDir, cleanup, err := createTempDir("")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer cleanup()
+
+	probeSymlinkSupport(t, baseDir)
+
+	dirD := filepath.Join(baseDir, "D")
+	err = os.Mkdir(dirD, 0755)
+	if err != nil {
+		t.Fatalf("Unable to create directory D %v", err)
+	}
+
+	err = os.Symlink(filepath.Join(baseDir, "B"), filepath.Join(baseDir, "A"))
+	if err != nil {
+		t.Fatalf("Unable to create symlink %v", err)
+	}
+	err = os.Symlink(filepath.Join(baseDir, "C"), filepath.Join(baseDir, "B"))
+	if err != nil {
+		t.Fatalf("Unable to create symlink %v", err)
+	}
+	err = os.Symlink(filepath.Join(baseDir, "D"), filepath.Join(baseDir, "C"))
+	if err != nil {
+		t.Fatalf("Unable to create symlink %v", err)
+	}
+
+	src := &Source{
+		concurrency:     1,
+		maxSymlinkDepth: 20,
+	}
+	chunks := make(chan *sources.Chunk, 10)
+	go func() {
+		workerPool := new(errgroup.Group)
+		workerPool.SetLimit(src.concurrency)
+		err := src.scanSymlink(ctx, filepath.Join(baseDir, "A"), chunks, workerPool, 1, filepath.Join(baseDir, "A"))
+		_ = workerPool.Wait()
+		require.NoError(t, err)
+		close(chunks)
+	}()
+	var chunkCount int
+	for range chunks {
+		chunkCount++
+	}
+	require.Equal(t, 0, chunkCount, "No chunks should be because dir D has no file")
+}
+
+func TestScanSymlink_MaxDepthExceeded(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*3)
+	defer cancel()
+	baseDir, cleanup, err := createTempDir("")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer cleanup()
+
+	probeSymlinkSupport(t, baseDir)
+
+	dirD := filepath.Join(baseDir, "D")
+	err = os.Mkdir(dirD, 0755)
+	if err != nil {
+		t.Fatalf("Unable to create directory D %v", err)
+	}
+
+	err = os.Symlink(filepath.Join(baseDir, "B"), filepath.Join(baseDir, "A"))
+	if err != nil {
+		t.Fatalf("Unable to create symlink %v", err)
+	}
+	err = os.Symlink(filepath.Join(baseDir, "C"), filepath.Join(baseDir, "B"))
+	if err != nil {
+		t.Fatalf("Unable to create symlink %v", err)
+	}
+	err = os.Symlink(filepath.Join(baseDir, "D"), filepath.Join(baseDir, "C"))
+	if err != nil {
+		t.Fatalf("Unable to create symlink %v", err)
+	}
+
+	src := &Source{
+		concurrency:     1,
+		maxSymlinkDepth: 2,
+	}
+	chunks := make(chan *sources.Chunk, 10)
+	workerPool := new(errgroup.Group)
+	workerPool.SetLimit(src.concurrency)
+
+	err = src.scanSymlink(
+		ctx,
+		filepath.Join(baseDir, "A"),
+		chunks,
+		workerPool,
+		1,
+		filepath.Join(baseDir, "A"),
+	)
+	_ = workerPool.Wait()
+	close(chunks)
+	require.Error(t, err)
+	require.EqualError(t, err, "max symlink depth reached")
+}
+
+func TestScanSymlink_FileTarget(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
+	defer cancel()
+
+	baseDir, cleanup, err := createTempDir("")
+	require.NoError(t, err)
+	defer cleanup()
+
+	probeSymlinkSupport(t, baseDir)
+	// Create a file
+	filePath := filepath.Join(baseDir, "file.txt")
+	err = os.WriteFile(filePath, []byte("data"), 0644)
+	require.NoError(t, err)
+
+	// Create a symlink pointing to the file
+	symlinkPath := filepath.Join(baseDir, "link.txt")
+	err = os.Symlink(filePath, symlinkPath)
+	require.NoError(t, err)
+
+	src := &Source{
+		maxSymlinkDepth: 5,
+		concurrency:     1,
+	}
+
+	chunks := make(chan *sources.Chunk, 10)
+	workerPool := new(errgroup.Group)
+	workerPool.SetLimit(src.concurrency)
+
+	err = src.scanSymlink(
+		ctx,
+		symlinkPath,
+		chunks,
+		workerPool,
+		1,
+		symlinkPath,
+	)
+	_ = workerPool.Wait()
+	require.NoError(t, err)
+	close(chunks)
+	var chunkCount int
+	for chunk := range chunks {
+		require.Equal(t, "data", string(chunk.Data))
+		chunkCount++
+	}
+	require.Equal(t, 1, chunkCount, "Expected 1 chunk")
+
+}
+
+func TestScanSymlink_SelfLoop(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
+	defer cancel()
+
+	baseDir, cleanup, err := createTempDir("")
+	require.NoError(t, err)
+	defer cleanup()
+
+	probeSymlinkSupport(t, baseDir)
+
+	symlinkPath := filepath.Join(baseDir, "loop.txt")
+	err = os.Symlink(symlinkPath, symlinkPath)
+	require.NoError(t, err)
+
+	src := &Source{
+		maxSymlinkDepth: 5,
+	}
+
+	chunks := make(chan *sources.Chunk, 10)
+	workerPool := new(errgroup.Group)
+	workerPool.SetLimit(src.concurrency)
+
+	err = src.scanSymlink(
+		ctx,
+		symlinkPath,
+		chunks,
+		workerPool,
+		1,
+		symlinkPath,
+	)
+	_ = workerPool.Wait()
+	close(chunks)
+	require.Error(t, err)
+	require.EqualError(t, err, "max symlink depth reached")
+}
+
+func TestScanSymlink_BrokenSymlink(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
+	defer cancel()
+
+	baseDir, cleanup, err := createTempDir("")
+	require.NoError(t, err)
+	defer cleanup()
+
+	probeSymlinkSupport(t, baseDir)
+
+	symlinkPath := filepath.Join(baseDir, "broken")
+	err = os.Symlink(filepath.Join(baseDir, "nonexistent.txt"), symlinkPath)
+	require.NoError(t, err)
+
+	src := &Source{
+		maxSymlinkDepth: 5,
+	}
+
+	chunks := make(chan *sources.Chunk, 10)
+	workerPool := new(errgroup.Group)
+	workerPool.SetLimit(src.concurrency)
+
+	err = src.scanSymlink(
+		ctx,
+		symlinkPath,
+		chunks,
+		workerPool,
+		0,
+		symlinkPath,
+	)
+	_ = workerPool.Wait()
+	close(chunks)
+	require.Error(t, err)
+	require.Contains(t, err.Error(), "lstat error")
+}
+
+func TestScanSymlink_TwoFileLoop(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
+	defer cancel()
+
+	baseDir, cleanup, err := createTempDir("")
+	require.NoError(t, err)
+	defer cleanup()
+
+	probeSymlinkSupport(t, baseDir)
+
+	fileA := filepath.Join(baseDir, "fileA.txt")
+	fileB := filepath.Join(baseDir, "fileB.txt")
+
+	// A -> B, B -> A
+	require.NoError(t, os.Symlink(fileB, fileA))
+	require.NoError(t, os.Symlink(fileA, fileB))
+
+	src := &Source{
+		maxSymlinkDepth: 5,
+	}
+
+	chunks := make(chan *sources.Chunk, 10)
+	workerPool := new(errgroup.Group)
+	workerPool.SetLimit(src.concurrency)
+
+	err = src.scanSymlink(
+		ctx,
+		fileA,
+		chunks,
+		workerPool,
+		0,
+		fileA,
+	)
+	_ = workerPool.Wait()
+	close(chunks)
+	require.Error(t, err)
+	require.EqualError(t, err, "max symlink depth reached")
+}

pkg/sources/filesystem/filesystem_test.go

@@ -86,12 +86,12 @@ func TestSource_Scan(t *testing.T) {
 			for chunk := range chunksCh {
 				if chunk.SourceMetadata.GetFilesystem().GetFile() == "filesystem.go" {
 					counter++
-					if diff := cmp.Diff(tt.wantSourceMetadata, chunk.SourceMetadata, protocmp.Transform()); diff != "" {
+					if diff := cmp.Diff(tt.wantSourceMetadata, chunk.SourceMetadata, protocmp.Transform()); diff != "" && counter == 1 { // First chunk should start at line 1
 						t.Errorf("Source.Chunks() %s metadata mismatch (-want +got):\n%s", tt.name, diff)
 					}
 				}
 			}
-			assert.Equal(t, 1, counter)
+			assert.Equal(t, 2, counter)
 		})
 	}
 }
@@ -375,41 +375,41 @@ func TestScanSubDirFile(t *testing.T) {
 	t.Parallel()
 	ctx := context.Background()
 
-	// create a temp directory with files
+	// Use a fixed directory for the test
-	parentDir, cleanupParentDir, err := createTempDir("", "file1")
+	testDir := filepath.Join(os.TempDir(), "trufflehog-test")
+	err := os.MkdirAll(testDir, 0755)
 	require.NoError(t, err)
-	defer cleanupParentDir()
+	defer os.RemoveAll(testDir)
 
-	childDir, cleanupChildDir, err := createTempDir(parentDir, "file2")
+	// Create a subdirectory and file
+	childDir := filepath.Join(testDir, "child")
+	err = os.MkdirAll(childDir, 0755)
 	require.NoError(t, err)
-	defer cleanupChildDir()
 
-	// create a file in child directory
+	filePath := filepath.Join(childDir, "testfile.txt")
-	file, cleanupFile, err := createTempFile(childDir, "should scan this file")
+	err = os.WriteFile(filePath, []byte("should scan this file"), 0644)
 	require.NoError(t, err)
-	defer cleanupFile()
 
-	// create an IncludePathsFile that contains the file path
+	// Create an IncludePathsFile with the absolute path of the file
-	includeFile, cleanupFile, err := createTempFile("", file.Name()+"\n")
+	includeFilePath := filepath.Join(testDir, "include.txt")
+	err = os.WriteFile(includeFilePath, []byte(filePath+"\n"), 0644)
 	require.NoError(t, err)
-	defer cleanupFile()
 
 	conn, err := anypb.New(&sourcespb.Filesystem{
-		IncludePathsFile: includeFile.Name(),
+		IncludePathsFile: includeFilePath,
 	})
 	require.NoError(t, err)
 
-	// initialize the source.
+	// Initialize the source
 	s := Source{}
 	err = s.Init(ctx, "include sub directory file", 0, 0, true, conn, 1)
 	require.NoError(t, err)
 
 	reporter := sourcestest.TestReporter{}
 	err = s.ChunkUnit(ctx, sources.CommonSourceUnit{
-		ID: parentDir,
+		ID: testDir,
 	}, &reporter)
 	require.NoError(t, err)
-
 	require.Equal(t, 1, len(reporter.Chunks), "Expected chunks from included file")
 	require.Equal(t, 0, len(reporter.ChunkErrs), "Expected no errors")
 }

pkg/sources/sources.go

@@ -4,6 +4,7 @@ import (
 	"encoding/json"
 	"errors"
 	"runtime"
+	"strings"
 	"sync"
 
 	"google.golang.org/protobuf/types/known/anypb"
@@ -389,6 +390,8 @@ type FilesystemConfig struct {
 	IncludePathsFile string
 	// ExcludePathsFile is the path to a file containing a list of regexps to exclude from the scan.
 	ExcludePathsFile string
+	// MaxSymlinkDepth enables following symlink upto the depth specified with max depth of 40
+	MaxSymlinkDepth int32
 }
 
 // S3Config defines the optional configuration for an S3 source.
@@ -593,6 +596,22 @@ func (p *Progress) ClearEncodedResumeInfoFor(id string) {
 	p.EncodedResumeInfo = marshalEncodedResumeInfo(p.encodedResumeInfoByID)
 }
 
+// ClearEncodedResumeContainingId removes the encoded resume information
+// entries that contain the id
+func (p *Progress) ClearEncodedResumeContainingId(id string) {
+	p.mut.Lock()
+	defer p.mut.Unlock()
+	p.ensureEncodedResumeInfoByID()
+
+	for key := range p.encodedResumeInfoByID {
+		if strings.Contains(key, id) {
+			delete(p.encodedResumeInfoByID, key)
+		}
+	}
+
+	p.EncodedResumeInfo = marshalEncodedResumeInfo(p.encodedResumeInfoByID)
+}
+
 // ensureEncodedResumeInfoByID ensures the encodedResumeInfoByID attribute is a
 // non-nil map. The mutex must be held when calling this function.
 func (p *Progress) ensureEncodedResumeInfoByID() {

proto/sources.proto

@@ -185,6 +185,7 @@ message Filesystem {
   string include_paths_file = 3; // path to file containing newline separated list of paths
   string exclude_paths_file = 4; // path to file containing newline separated list of paths
   bool skip_binaries = 5; // allows skipping binary files from the scan
+  int32 max_symlink_depth = 6; // allows following symlinks upto specified max depth
 }
 
 message GCS {