trufflesecurity/trufflehog
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Enable optional symlink following for fs source

Browse Source
This commit is contained in:
Dustin Decker
2025-12-12 08:28:21 -08:00
parent 0fab92f434
commit 48945e6dc7
5 changed files with 867 additions and 681 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/pb/sourcespb/sources.pb.go
+678 -667
View File
File diff suppressed because it is too large Load Diff
pkg/pb/sourcespb/sources.pb.validate.go
+2
View File
@@ -1703,6 +1703,8 @@ func (m *Filesystem) validate(all bool) error {
// no validation rules for SkipBinaries
// no validation rules for FollowSymlinks
if len(errors) > 0 {
return FilesystemMultiError(errors)
}
pkg/sources/filesystem/filesystem.go
+56 -14
View File
@@ -26,15 +26,16 @@ import (
const SourceType = sourcespb.SourceType_SOURCE_TYPE_FILESYSTEM
type Source struct {
name string
sourceId sources.SourceID
jobId sources.JobID
concurrency int
verify bool
paths []string
log logr.Logger
filter *common.Filter
skipBinaries bool
name string
sourceId sources.SourceID
jobId sources.JobID
concurrency int
verify bool
paths []string
log logr.Logger
filter *common.Filter
skipBinaries bool
followSymlinks bool
sources.Progress
sources.CommonSourceUnitUnmarshaller
}
@@ -74,6 +75,7 @@ func (s *Source) Init(aCtx context.Context, name string, jobId sources.JobID, so
}
s.paths = append(conn.Paths, conn.Directories...)
s.skipBinaries = conn.GetSkipBinaries()
s.followSymlinks = conn.GetFollowSymlinks()
filter, err := common.FilterFromFiles(conn.IncludePathsFile, conn.ExcludePathsFile)
if err != nil {
@@ -94,13 +96,19 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ .
s.SetProgressComplete(i, len(s.paths), fmt.Sprintf("Path: %s", path), "")
cleanPath := filepath.Clean(path)
fileInfo, err := os.Lstat(cleanPath)
var fileInfo fs.FileInfo
var err error
if s.followSymlinks {
fileInfo, err = os.Stat(cleanPath)
} else {
fileInfo, err = os.Lstat(cleanPath)
}
if err != nil {
logger.Error(err, "unable to get file info")
continue
}
if fileInfo.Mode()&os.ModeSymlink != 0 {
if !s.followSymlinks && fileInfo.Mode()&os.ModeSymlink != 0 {
logger.Info("skipping, not a regular file", "path", cleanPath)
continue
}
@@ -148,6 +156,28 @@ func (s *Source) scanDir(ctx context.Context, path string, chunksChan chan *sour
return nil // skip the file
}
// Handle symlinks when followSymlinks is enabled
if s.followSymlinks && d.Type()&fs.ModeSymlink != 0 {
// Follow the symlink to see what it points to
targetInfo, err := os.Stat(fullPath)
if err != nil {
// Broken symlink or permission issue, skip it
ctx.Logger().V(5).Info("unable to follow symlink", "path", fullPath, "error", err)
return nil
}
// If the symlink points to a regular file, process it
if targetInfo.Mode().IsRegular() {
workerPool.Go(func() error {
if err := s.scanFile(ctx, fullPath, chunksChan); err != nil {
ctx.Logger().Error(err, "error scanning file", "path", fullPath, "error", err)
}
s.SetEncodedResumeInfoFor(path, fullPath)
return nil
})
}
return nil
}
// Skip over non-regular files. We do this check here to suppress noisy
// logs for trying to scan directories and other non-regular files in
// our traversal.
@@ -181,11 +211,17 @@ var skipSymlinkErr = errors.New("skipping symlink")
func (s *Source) scanFile(ctx context.Context, path string, chunksChan chan *sources.Chunk) error {
fileCtx := context.WithValues(ctx, "path", path)
fileStat, err := os.Lstat(path)
var fileStat fs.FileInfo
var err error
if s.followSymlinks {
fileStat, err = os.Stat(path)
} else {
fileStat, err = os.Lstat(path)
}
if err != nil {
return fmt.Errorf("unable to stat file: %w", err)
}
if fileStat.Mode()&os.ModeSymlink != 0 {
if !s.followSymlinks && fileStat.Mode()&os.ModeSymlink != 0 {
return skipSymlinkErr
}
@@ -247,7 +283,13 @@ func (s *Source) ChunkUnit(ctx context.Context, unit sources.SourceUnit, reporte
logger := ctx.Logger().WithValues("path", path)
cleanPath := filepath.Clean(path)
fileInfo, err := os.Lstat(cleanPath)
var fileInfo fs.FileInfo
var err error
if s.followSymlinks {
fileInfo, err = os.Stat(cleanPath)
} else {
fileInfo, err = os.Lstat(cleanPath)
}
if err != nil {
return reporter.ChunkErr(ctx, fmt.Errorf("unable to get file info: %w", err))
}
pkg/sources/filesystem/filesystem_test.go
+130
View File
@@ -461,6 +461,136 @@ func TestSkipBinaries(t *testing.T) {
require.NotContains(t, processedFiles, binaryFile, "Binary file should be skipped")
}
func TestFollowSymlinks(t *testing.T) {
t.Parallel()
ctx := context.Background()
// Create a temporary directory with a file and a symlink
tempDir, err := os.MkdirTemp("", "trufflehog_symlink_test")
require.NoError(t, err)
defer os.RemoveAll(tempDir)
// Create a real file
realFile := filepath.Join(tempDir, "real_file.txt")
fileContents := "secret data in real file"
err = os.WriteFile(realFile, []byte(fileContents), 0644)
require.NoError(t, err)
// Create a symlink pointing to the real file
symlinkFile := filepath.Join(tempDir, "symlink_file.txt")
err = os.Symlink(realFile, symlinkFile)
require.NoError(t, err)
// Test 1: followSymlinks = false (default) - should skip symlink
t.Run("skip symlinks when followSymlinks is false", func(t *testing.T) {
conn, err := anypb.New(&sourcespb.Filesystem{
Paths: []string{symlinkFile},
FollowSymlinks: false,
})
require.NoError(t, err)
s := Source{}
err = s.Init(ctx, "test skip symlinks", 0, 0, true, conn, 1)
require.NoError(t, err)
reporter := sourcestest.TestReporter{}
err = s.ChunkUnit(ctx, sources.CommonSourceUnit{
ID: symlinkFile,
}, &reporter)
require.NoError(t, err)
// Should not have any chunks because symlink was skipped
assert.Equal(t, 0, len(reporter.Chunks), "Expected no chunks when symlinks are skipped")
// Should have one error for the skipped symlink
assert.Equal(t, 1, len(reporter.ChunkErrs), "Expected one error for skipped symlink")
})
// Test 2: followSymlinks = true - should follow symlink
t.Run("follow symlinks when followSymlinks is true", func(t *testing.T) {
conn, err := anypb.New(&sourcespb.Filesystem{
Paths: []string{symlinkFile},
FollowSymlinks: true,
})
require.NoError(t, err)
s := Source{}
err = s.Init(ctx, "test follow symlinks", 0, 0, true, conn, 1)
require.NoError(t, err)
reporter := sourcestest.TestReporter{}
err = s.ChunkUnit(ctx, sources.CommonSourceUnit{
ID: symlinkFile,
}, &reporter)
require.NoError(t, err)
// Should have chunks because symlink was followed
assert.Equal(t, 1, len(reporter.Chunks), "Expected chunks when symlinks are followed")
assert.Equal(t, 0, len(reporter.ChunkErrs), "Expected no errors when following symlinks")
// Verify the content is correct
if len(reporter.Chunks) > 0 {
assert.Contains(t, string(reporter.Chunks[0].Data), fileContents, "Chunk should contain file contents")
}
})
// Test 3: Scanning directory with symlink using followSymlinks = false
t.Run("skip symlinks in directory scan when followSymlinks is false", func(t *testing.T) {
conn, err := anypb.New(&sourcespb.Filesystem{
Paths: []string{tempDir},
FollowSymlinks: false,
})
require.NoError(t, err)
s := Source{}
err = s.Init(ctx, "test directory skip symlinks", 0, 0, true, conn, 1)
require.NoError(t, err)
reporter := sourcestest.TestReporter{}
err = s.ChunkUnit(ctx, sources.CommonSourceUnit{
ID: tempDir,
}, &reporter)
require.NoError(t, err)
// Should have exactly one chunk from the real file only
assert.Equal(t, 1, len(reporter.Chunks), "Expected one chunk from real file only")
// Verify it's the real file, not the symlink
if len(reporter.Chunks) > 0 {
metadata := reporter.Chunks[0].SourceMetadata.GetFilesystem()
assert.NotNil(t, metadata)
// The path should be the real file
assert.Contains(t, metadata.File, "real_file.txt")
}
})
// Test 4: Scanning directory with symlink using followSymlinks = true
t.Run("follow symlinks in directory scan when followSymlinks is true", func(t *testing.T) {
conn, err := anypb.New(&sourcespb.Filesystem{
Paths: []string{tempDir},
FollowSymlinks: true,
})
require.NoError(t, err)
s := Source{}
err = s.Init(ctx, "test directory follow symlinks", 0, 0, true, conn, 1)
require.NoError(t, err)
reporter := sourcestest.TestReporter{}
err = s.ChunkUnit(ctx, sources.CommonSourceUnit{
ID: tempDir,
}, &reporter)
require.NoError(t, err)
// Should have two chunks: one from real file and one from symlink
assert.Equal(t, 2, len(reporter.Chunks), "Expected two chunks when following symlinks in directory")
// Verify both contain the same content
for _, chunk := range reporter.Chunks {
assert.Contains(t, string(chunk.Data), fileContents, "Both chunks should contain file contents")
}
})
}
// createTempFile is a helper function to create a temporary file in the given
// directory with the provided contents. If dir is "", the operating system's
// temp directory is used.
proto/sources.proto
+1
View File
@@ -184,6 +184,7 @@ message Filesystem {
string include_paths_file = 3; // path to file containing newline separated list of paths
string exclude_paths_file = 4; // path to file containing newline separated list of paths
bool skip_binaries = 5; // allows skipping binary files from the scan
bool follow_symlinks = 6; // allows following symbolic links during scan
}
message GCS {