diff --git a/pkg/sources/gitlab/gitlab.go b/pkg/sources/gitlab/gitlab.go index 741e58115..a1358c214 100644 --- a/pkg/sources/gitlab/gitlab.go +++ b/pkg/sources/gitlab/gitlab.go @@ -695,6 +695,9 @@ func (s *Source) getAllProjectReposV2( return apiClient.Projects.ListProjects(projectQueryOptions, p, gitlab.WithContext(ctx)) }) + // Track seen projects by ID to avoid reporting duplicates. + // Projects can appear multiple times in the API response (shared across groups, in subgroups, etc.) + seenProjects := make(map[int]struct{}) totalCount := 0 // process each project @@ -715,6 +718,13 @@ func (s *Source) getAllProjectReposV2( "project_id", project.ID, "project_name", project.NameWithNamespace) + // Skip projects we've already seen. + if _, exists := seenProjects[project.ID]; exists { + projCtx.Logger().V(3).Info("skipping project", "reason", "already processed") + continue + } + seenProjects[project.ID] = struct{}{} + // skip projects configured to be ignored. if ignoreRepo(project.PathWithNamespace) { projCtx.Logger().V(3).Info("skipping project", "reason", "ignored in config") @@ -748,7 +758,7 @@ func (s *Source) getAllProjectReposV2( } } - ctx.Logger().Info("Enumerated GitLab projects", "count", totalCount) + ctx.Logger().Info("Enumerated GitLab projects", "count", totalCount, "unique", len(seenProjects)) return nil } diff --git a/pkg/sources/gitlab/gitlab_test.go b/pkg/sources/gitlab/gitlab_test.go index 5a0848fc4..97e33d062 100644 --- a/pkg/sources/gitlab/gitlab_test.go +++ b/pkg/sources/gitlab/gitlab_test.go @@ -2,12 +2,14 @@ package gitlab import ( "reflect" + "sync" "testing" "github.com/stretchr/testify/assert" "golang.org/x/sync/errgroup" "github.com/trufflesecurity/trufflehog/v3/pkg/context" + "github.com/trufflesecurity/trufflehog/v3/pkg/sources" "github.com/trufflesecurity/trufflehog/v3/pkg/sources/git" ) @@ -134,6 +136,71 @@ func Test_scanRepos_SetProgressComplete(t *testing.T) { } } +func TestGetAllProjectReposV2_Deduplication(t *testing.T) { + // Test that getAllProjectReposV2 correctly deduplicates projects + // that appear multiple times in the API response + ctx := context.Background() + + // Track reported units + reportedProjects := make(map[string]int) + var mu sync.Mutex + reporter := sources.VisitorReporter{ + VisitUnit: func(ctx context.Context, unit sources.SourceUnit) error { + mu.Lock() + defer mu.Unlock() + unitID, _ := unit.SourceUnitID() + reportedProjects[unitID]++ + return nil + }, + } + + // This is a simplified test that verifies the deduplication logic works + // We can't easily mock the gitlab.Scan2 iterator without extensive setup, + // but we can verify that the seenProjects map would prevent duplicates + // by checking the logic in a simpler way + + // Simulate what happens when the same project ID appears multiple times + // The actual API call is: gitlab.Scan2(...) which returns an iterator + // For testing, we verify the deduplication map logic works correctly + + // Verify that if we process projects with the same ID multiple times, + // only the first one gets reported + seenProjects := make(map[int]struct{}) + + projects := []struct { + id int + url string + }{ + {1, "https://gitlab.com/org/project1.git"}, + {2, "https://gitlab.com/org/project2.git"}, + {1, "https://gitlab.com/org/project1.git"}, // Duplicate! + {3, "https://gitlab.com/org/project3.git"}, + {2, "https://gitlab.com/org/project2.git"}, // Duplicate! + } + + uniqueReported := 0 + for _, proj := range projects { + // Simulate the deduplication logic from getAllProjectReposV2 + if _, exists := seenProjects[proj.id]; exists { + continue + } + seenProjects[proj.id] = struct{}{} + + unit := git.SourceUnit{Kind: git.UnitRepo, ID: proj.url} + if err := reporter.VisitUnit(ctx, unit); err != nil { + t.Fatal(err) + } + uniqueReported++ + } + + // Should report exactly 3 unique projects + assert.Equal(t, 3, uniqueReported, "should have reported 3 unique projects") + assert.Equal(t, 3, len(reportedProjects), "should have 3 unique projects in map") + assert.Equal(t, 1, reportedProjects["https://gitlab.com/org/project1.git"], "project1 should be reported once") + assert.Equal(t, 1, reportedProjects["https://gitlab.com/org/project2.git"], "project2 should be reported once") + assert.Equal(t, 1, reportedProjects["https://gitlab.com/org/project3.git"], "project3 should be reported once") +} + func Test_normalizeGitlabEndpoint(t *testing.T) { testCases := map[string]struct { inputEndpoint string