package github import ( "bytes" "crypto/rand" "crypto/rsa" "crypto/x509" "encoding/pem" "errors" "fmt" "net/http" "net/http/httptest" "net/url" "reflect" "slices" "strconv" "strings" "testing" "time" "github.com/google/go-cmp/cmp" "github.com/google/go-github/v67/github" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "golang.org/x/sync/errgroup" "google.golang.org/protobuf/types/known/anypb" "gopkg.in/h2non/gock.v1" "github.com/trufflesecurity/trufflehog/v3/pkg/cache/simple" "github.com/trufflesecurity/trufflehog/v3/pkg/context" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/credentialspb" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb" "github.com/trufflesecurity/trufflehog/v3/pkg/sources" ) func createPrivateKey() string { key, err := rsa.GenerateKey(rand.Reader, 2048) if err != nil { panic(err) } data := x509.MarshalPKCS1PrivateKey(key) var pemKey bytes.Buffer if err := pem.Encode(&pemKey, &pem.Block{ Type: "RSA PRIVATE KEY", Bytes: data, }); err != nil { panic(err) } return pemKey.String() } func createTestSource(src *sourcespb.GitHub) (*Source, *anypb.Any) { s := &Source{} conn, err := anypb.New(src) if err != nil { panic(err) } return s, conn } func initTestSource(src *sourcespb.GitHub) *Source { s, conn := createTestSource(src) if err := s.Init(context.Background(), "test - github", 0, 1337, false, conn, 1); err != nil { panic(err) } gock.InterceptClient(s.connector.APIClient().Client()) if appConnector, ok := s.connector.(*appConnector); ok { gock.InterceptClient(appConnector.InstallationClient().Client()) } return s } func TestInit(t *testing.T) { source, conn := createTestSource(&sourcespb.GitHub{ Repositories: []string{"https://github.com/dustin-decker/secretsandstuff.git"}, Credential: &sourcespb.GitHub_Token{ Token: "super secret token", }, }) err := source.Init(context.Background(), "test - github", 0, 1337, false, conn, 1) assert.Nil(t, err) // TODO: test error case } func TestAddReposByOrg(t *testing.T) { defer gock.Off() gock.New("https://api.github.com"). Get("/orgs/super-secret-org/repos"). Reply(200). JSON([]map[string]string{ {"clone_url": "https://github.com/super-secret-repo.git", "full_name": "super-secret-repo"}, {"clone_url": "https://github.com/super-secret-repo2.git", "full_name": "secret/super-secret-repo2"}, }) s := initTestSource(&sourcespb.GitHub{ Credential: &sourcespb.GitHub_Token{ Token: "super secret token", }, Repositories: nil, IgnoreRepos: []string{"secret/super-*-repo2"}, CommentsTimeframeDays: 10, }) err := s.getReposByOrg(context.Background(), "super-secret-org", noopReporter()) assert.Nil(t, err) assert.Equal(t, 1, s.filteredRepoCache.Count()) ok := s.filteredRepoCache.Exists("super-secret-repo") assert.True(t, ok) assert.False(t, gock.HasUnmatchedRequest()) assert.True(t, gock.IsDone()) } func TestAddReposByOrg_Repositories(t *testing.T) { defer gock.Off() gock.New("https://api.github.com"). Get("/orgs/super-secret-org/repos"). Reply(200). JSON(`[ {"full_name": "super-secret-org/super-secret-repo", "clone_url": "https://github.com/super-secret-org/super-secret-repo.git", "size": 1}, {"full_name": "super-secret-org/super-secret-repo2", "clone_url": "https://github.com/super-secret-org/super-secret-repo2.git", "size": 1}, {"full_name": "super-secret-org/not-super-secret-repo", "clone_url": "https://github.com/super-secret-org/not-super-secret-repo.git", "size": 1} ]`) s := initTestSource(&sourcespb.GitHub{ Credential: &sourcespb.GitHub_Token{ Token: "super secret token", }, Repositories: []string{"super-secret-org/super-secret-repo", "super-secret-org/super-secret-repo2"}, Organizations: []string{"super-secret-org"}, }) err := s.getReposByOrg(context.Background(), "super-secret-org", noopReporter()) assert.Nil(t, err) assert.Equal(t, 2, s.filteredRepoCache.Count()) ok := s.filteredRepoCache.Exists("super-secret-org/super-secret-repo") assert.True(t, ok) ok = s.filteredRepoCache.Exists("super-secret-org/super-secret-repo2") assert.True(t, ok) assert.False(t, gock.HasUnmatchedRequest()) assert.True(t, gock.IsDone()) } func TestAddReposByOrg_IncludeRepos(t *testing.T) { defer gock.Off() gock.New("https://api.github.com"). Get("/orgs/super-secret-org/repos"). Reply(200). JSON(`[ {"full_name": "super-secret-org/super-secret-repo", "clone_url": "https://github.com/super-secret-org/super-secret-repo.git", "size": 1}, {"full_name": "super-secret-org/super-secret-repo2", "clone_url": "https://github.com/super-secret-org/super-secret-repo2.git", "size": 1}, {"full_name": "super-secret-org/not-super-secret-repo", "clone_url": "https://github.com/super-secret-org/not-super-secret-repo.git", "size": 1} ]`) s := initTestSource(&sourcespb.GitHub{ Credential: &sourcespb.GitHub_Token{ Token: "super secret token", }, IncludeRepos: []string{"super-secret-org/super*"}, Organizations: []string{"super-secret-org"}, }) err := s.getReposByOrg(context.Background(), "super-secret-org", noopReporter()) assert.Nil(t, err) assert.Equal(t, 2, s.filteredRepoCache.Count()) ok := s.filteredRepoCache.Exists("super-secret-org/super-secret-repo") assert.True(t, ok) ok = s.filteredRepoCache.Exists("super-secret-org/super-secret-repo2") assert.True(t, ok) assert.False(t, gock.HasUnmatchedRequest()) assert.True(t, gock.IsDone()) } func TestAddReposByUser(t *testing.T) { defer gock.Off() gock.New("https://api.github.com"). Get("/users/super-secret-user/repos"). Reply(200). JSON([]map[string]string{ {"full_name": "super-secret-user/super-secret-repo", "clone_url": "https://github.com/super-secret-user/super-secret-repo.git"}, {"full_name": "super-secret-user/super-secret-repo2", "clone_url": "https://github.com/super-secret-user/super-secret-repo2.git"}, }) s := initTestSource(&sourcespb.GitHub{ Credential: &sourcespb.GitHub_Token{ Token: "super secret token", }, IgnoreRepos: []string{"super-secret-user/super-secret-repo2"}, }) err := s.getReposByUser(context.Background(), "super-secret-user", false, noopReporter()) assert.Nil(t, err) assert.Equal(t, 1, s.filteredRepoCache.Count()) ok := s.filteredRepoCache.Exists("super-secret-user/super-secret-repo") assert.True(t, ok) assert.False(t, gock.HasUnmatchedRequest()) assert.True(t, gock.IsDone()) } func TestAddGistsByUser(t *testing.T) { defer gock.Off() gock.New("https://api.github.com"). Get("/users/super-secret-user/gists"). Reply(200). JSON([]map[string]string{{"id": "aa5a315d61ae9438b18d", "git_pull_url": "https://gist.github.com/aa5a315d61ae9438b18d.git"}}) s := initTestSource(&sourcespb.GitHub{Credential: &sourcespb.GitHub_Unauthenticated{}}) err := s.addUserGistsToCache(context.Background(), "super-secret-user", noopReporter()) assert.Nil(t, err) assert.Equal(t, 1, s.filteredRepoCache.Count()) ok := s.filteredRepoCache.Exists("aa5a315d61ae9438b18d") assert.True(t, ok) assert.False(t, gock.HasUnmatchedRequest()) assert.True(t, gock.IsDone()) } func TestIgnoreGistsByUser(t *testing.T) { defer gock.Off() gock.New("https://api.github.com"). Get("/users/super-secret-user/gists"). Reply(200). JSON([]map[string]string{{"id": "aa5a315d61ae9438b18d", "git_pull_url": "https://gist.github.com/aa5a315d61ae9438b18d.git"}}) s := initTestSource(&sourcespb.GitHub{Credential: &sourcespb.GitHub_Unauthenticated{}}) s.ignoreGists = true err := s.addUserGistsToCache(context.Background(), "super-secret-user", noopReporter()) assert.Nil(t, err) assert.Equal(t, 0, s.filteredRepoCache.Count()) ok := s.filteredRepoCache.Exists("aa5a315d61ae9438b18d") assert.False(t, ok) assert.False(t, gock.HasUnmatchedRequest()) assert.False(t, gock.IsDone()) } func TestAddMembersByOrg(t *testing.T) { defer gock.Off() gock.New("https://api.github.com"). Get("/orgs/org1/members"). Reply(200). JSON([]map[string]string{ {"login": "testman1"}, {"login": "testman2"}, }) s := initTestSource(&sourcespb.GitHub{Credential: &sourcespb.GitHub_Unauthenticated{}}) err := s.addMembersByOrg(context.Background(), "org1", noopReporter()) assert.Nil(t, err) assert.Equal(t, 2, len(s.memberCache)) _, ok := s.memberCache["testman1"] assert.True(t, ok) _, ok = s.memberCache["testman2"] assert.True(t, ok) assert.False(t, gock.HasUnmatchedRequest()) assert.True(t, gock.IsDone()) } func TestAddMembersByOrg_AuthFailure(t *testing.T) { defer gock.Off() gock.New("https://api.github.com"). Get("/orgs/org1/members"). Reply(401). JSON([]map[string]string{{ "message": "Bad credentials", "documentation_url": "https://docs.github.com/rest", "status": "401", }}) s := initTestSource(&sourcespb.GitHub{Credential: &sourcespb.GitHub_Unauthenticated{}}) err := s.addMembersByOrg(context.Background(), "org1", noopReporter()) assert.True(t, strings.HasPrefix(err.Error(), "could not list organization")) assert.False(t, gock.HasUnmatchedRequest()) assert.True(t, gock.IsDone()) } func TestAddMembersByOrg_NoMembers(t *testing.T) { defer gock.Off() gock.New("https://api.github.com"). Get("/orgs/org1/members"). Reply(200). JSON([]map[string]string{}) s := initTestSource(&sourcespb.GitHub{Credential: &sourcespb.GitHub_Unauthenticated{}}) err := s.addMembersByOrg(context.Background(), "org1", noopReporter()) assert.Equal(t, fmt.Sprintf("organization (%q) had 0 members: account may not have access to list organization members", "org1"), err.Error()) assert.False(t, gock.HasUnmatchedRequest()) assert.True(t, gock.IsDone()) } func TestAddMembersByApp(t *testing.T) { defer gock.Off() privateKey := createPrivateKey() gock.New("https://api.github.com"). Get("/app/installations"). Reply(200). JSON([]map[string]any{ {"id": 1337, "account": map[string]string{"login": "super-secret-org", "type": "Organization"}}, }) gock.New("https://api.github.com"). Post("/app/installations/1337/access_tokens"). Reply(200). JSON(map[string]string{"token": "dontlook"}) gock.New("https://api.github.com"). Get("/orgs/super-secret-org/members"). Reply(200). JSON([]map[string]any{ {"login": "ssm1"}, {"login": "ssm2"}, {"login": "ssm3"}, }) s := initTestSource(&sourcespb.GitHub{ Endpoint: "https://api.github.com", Credential: &sourcespb.GitHub_GithubApp{ GithubApp: &credentialspb.GitHubApp{ PrivateKey: privateKey, InstallationId: "1337", AppId: "4141", }, }}) err := s.addMembersByApp(context.Background(), s.connector.(*appConnector), noopReporter()) assert.Nil(t, err) assert.Equal(t, 3, len(s.memberCache)) _, ok := s.memberCache["ssm1"] assert.True(t, ok) _, ok = s.memberCache["ssm2"] assert.True(t, ok) _, ok = s.memberCache["ssm3"] assert.True(t, ok) assert.False(t, gock.HasUnmatchedRequest()) assert.True(t, gock.IsDone()) } func TestAddReposByApp(t *testing.T) { defer gock.Off() gock.New("https://api.github.com"). Get("/installation/repositories"). Reply(200). JSON(map[string]any{ "repositories": []map[string]string{ {"clone_url": "https://github/ssr1.git", "full_name": "ssr1"}, {"clone_url": "https://github/ssr2.git", "full_name": "ssr2"}, }, }) s := initTestSource(&sourcespb.GitHub{Credential: &sourcespb.GitHub_Unauthenticated{}}) err := s.getReposByApp(context.Background(), noopReporter()) assert.Nil(t, err) assert.Equal(t, 2, s.filteredRepoCache.Count()) ok := s.filteredRepoCache.Exists("ssr1") assert.True(t, ok) ok = s.filteredRepoCache.Exists("ssr2") assert.True(t, ok) assert.False(t, gock.HasUnmatchedRequest()) assert.True(t, gock.IsDone()) } // TestAppConnector_EnterpriseBaseURL verifies that GitHub App authentication // correctly sets the BaseURL on the internal AppsTransport for token refresh. // This is critical for GitHub Enterprise Server and GHEC with Data Residency. // Without this fix, token refresh requests would go to api.github.com instead // of the configured enterprise endpoint. func TestAppConnector_EnterpriseBaseURL(t *testing.T) { privateKey := createPrivateKey() enterpriseEndpoint := "https://api.example.ghe.com" connector, err := NewAppConnector( context.Background(), enterpriseEndpoint, &credentialspb.GitHubApp{ PrivateKey: privateKey, InstallationId: "1337", AppId: "4141", }, false) require.NoError(t, err) appConn, ok := connector.(*appConnector) require.True(t, ok, "connector should be an appConnector") // Get the HTTP client's transport, which should be a ghinstallation.Transport transport := appConn.APIClient().Client().Transport // Use reflection to access the Transport's BaseURL field transportVal := reflect.ValueOf(transport).Elem() baseURLField := transportVal.FieldByName("BaseURL") require.True(t, baseURLField.IsValid(), "Transport should have a BaseURL field") assert.Equal(t, enterpriseEndpoint, baseURLField.String(), "Transport.BaseURL should be set to enterprise endpoint") // Use reflection to access the internal appsTransport and verify its BaseURL // This is the critical check - the internal AppsTransport is used for token refresh appsTransportField := transportVal.FieldByName("appsTransport") require.True(t, appsTransportField.IsValid(), "Transport should have an appsTransport field") // Access the BaseURL of the internal AppsTransport appsTransportVal := appsTransportField.Elem() appsBaseURLField := appsTransportVal.FieldByName("BaseURL") require.True(t, appsBaseURLField.IsValid(), "AppsTransport should have a BaseURL field") assert.Equal(t, enterpriseEndpoint, appsBaseURLField.String(), "AppsTransport.BaseURL should be set to enterprise endpoint") } // TestValidate_ScanAllInstallationsWithoutDefaultInstallation covers the // scanAllInstallations-without-githubApp.installationId case: there is no // default installation token to call RateLimit with (see // appConnector.HasDefaultInstallation), so Validate must fall back to // checking App credentials via the app-level (JWT) client instead of // erroring against a non-existent installation ID 0. func TestValidate_ScanAllInstallationsWithoutDefaultInstallation(t *testing.T) { defer gock.Off() gock.New("https://api.github.com"). Get("/app/installations"). Reply(200). JSON([]map[string]any{}) gock.New("https://api.github.com"). Get("^/app$"). Reply(200). JSON(map[string]any{"id": 4141}) s := initTestSource(&sourcespb.GitHub{ Endpoint: "https://api.github.com", ScanAllInstallations: true, Credential: &sourcespb.GitHub_GithubApp{ GithubApp: &credentialspb.GitHubApp{ PrivateKey: createPrivateKey(), AppId: "4141", }, }}) errs := s.Validate(context.Background()) assert.Empty(t, errs) assert.False(t, gock.HasUnmatchedRequest()) assert.True(t, gock.IsDone()) } func TestAddOrgsByUser(t *testing.T) { defer gock.Off() // NOTE: addOrgsByUser calls /user/orgs to get the orgs of the // authenticated user gock.New("https://api.github.com"). Get("/user/orgs"). Reply(200). JSON([]map[string]any{ {"login": "sso2"}, }) s := initTestSource(&sourcespb.GitHub{Credential: &sourcespb.GitHub_Unauthenticated{}}) s.addOrgsByUser(context.Background(), "super-secret-user", noopReporter()) assert.Equal(t, 1, s.orgsCache.Count()) ok := s.orgsCache.Exists("sso2") assert.True(t, ok) assert.False(t, gock.HasUnmatchedRequest()) assert.True(t, gock.IsDone()) } func TestNormalizeRepos(t *testing.T) { defer gock.Off() tests := []struct { name string setup func() repos []string expected map[string]struct{} wantErr bool }{ { name: "repo url", setup: func() {}, repos: []string{"https://github.com/super-secret-user/super-secret-repo"}, expected: map[string]struct{}{ "https://github.com/super-secret-user/super-secret-repo.git": {}, }, }, { name: "not found", setup: func() { gock.New("https://api.github.com"). Get("/users/not-found/gists"). Reply(404) gock.New("https://api.github.com"). Get("/users/not-found/repos"). Reply(404) }, repos: []string{"not-found"}, expected: map[string]struct{}{}, wantErr: true, }, { name: "unexpected format", setup: func() {}, repos: []string{"/foo/"}, expected: map[string]struct{}{}, wantErr: true, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { defer gock.Off() tt.setup() s := initTestSource(&sourcespb.GitHub{Credential: &sourcespb.GitHub_Unauthenticated{}}) got, err := s.normalizeRepo(tt.repos[0]) if (err != nil) != tt.wantErr { t.Errorf("normalizeRepo() error = %v, wantErr %v", err, tt.wantErr) return } if got != "" { for k := range tt.expected { assert.Equal(t, got, k) } } res := make(map[string]struct{}, s.filteredRepoCache.Count()) for _, v := range s.filteredRepoCache.Keys() { res[v] = struct{}{} } if got == "" && !cmp.Equal(res, tt.expected) { t.Errorf("normalizeRepo() got = %v, want %v", s.repos, tt.expected) } }) } } func TestNormalizeRepo(t *testing.T) { // Test that normalizeRepo correctly identifies URLs with protocols source := &Source{} // Test case 1: HTTP URL result, err := source.normalizeRepo("https://github.com/org/repo.git") assert.NoError(t, err) assert.Contains(t, result, "github.com/org/repo") // Test case 2: HTTP URL without .git result, err = source.normalizeRepo("http://github.com/org/repo") assert.NoError(t, err) assert.Contains(t, result, "github.com/org/repo") // Test case 3: Git protocol URL result, err = source.normalizeRepo("git://github.com/org/repo.git") assert.NoError(t, err) assert.Contains(t, result, "github.com/org/repo") // Test case 4: SSH URL result, err = source.normalizeRepo("ssh://git@github.com/org/repo.git") assert.NoError(t, err) assert.Contains(t, result, "github.com/org/repo") // Test case 5: SCP-style SSH URL result, err = source.normalizeRepo("git@github.com:org/repo.git") assert.NoError(t, err) assert.Equal(t, "https://github.com/org/repo.git", result) // Test case 6: Org/repo format (should convert to full URL) result, err = source.normalizeRepo("org/repo") assert.NoError(t, err) assert.Contains(t, result, "github.com/org/repo") // Test case 7: Invalid format (no protocol, no slash) _, err = source.normalizeRepo("invalid") assert.Error(t, err) assert.Contains(t, err.Error(), "no repositories found") } func TestNormalizeRepo_Enterprise(t *testing.T) { tests := []struct { name string endpoint string wantResult string }{ { name: "only host", endpoint: "https://example.com", wantResult: "https://example.com/org/repo.git", }, { name: "host with path", endpoint: "https://example.com/api/v3", wantResult: "https://example.com/org/repo.git", }, { name: "host with path without scheme", endpoint: "example.com/api/v3", wantResult: "https://example.com/org/repo.git", }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { source := Source{ conn: &sourcespb.GitHub{ Endpoint: tt.endpoint, }, } result, err := source.normalizeRepo("org/repo") assert.NoError(t, err) assert.Equal(t, tt.wantResult, result) }) } } func TestHandleRateLimit(t *testing.T) { resetRateLimitForTest(t) s := initTestSource(&sourcespb.GitHub{Credential: &sourcespb.GitHub_Unauthenticated{}}) ctx := context.Background() assert.False(t, s.handleRateLimit(ctx, nil)) // Request reqUrl, _ := url.Parse("https://github.com/trufflesecurity/trufflehog") res := &github.Response{ Response: &http.Response{ StatusCode: 429, Header: make(http.Header), Request: &http.Request{ Method: "GET", URL: reqUrl, }, }, } res.Header.Set("x-ratelimit-remaining", "0") res.Header.Set("x-ratelimit-reset", strconv.FormatInt(time.Now().Unix()+1, 10)) // Error resetTime := github.Timestamp{ Time: time.Now().Add(time.Millisecond), } err := &github.RateLimitError{ Rate: github.Rate{ Limit: 5000, Remaining: 0, Reset: resetTime, }, Response: res.Response, Message: "Too Many Requests", } assert.True(t, s.handleRateLimit(ctx, err)) } func TestHandleRateLimitStopsSleepingWhenContextCancelled(t *testing.T) { resetRateLimitForTest(t) s := &Source{name: "test - github"} retryAfter := time.Hour err := &github.AbuseRateLimitError{ RetryAfter: &retryAfter, Message: "secondary rate limit", } ctx, cancel := context.WithCancel(context.Background()) cancel() start := time.Now() assert.False(t, s.handleRateLimit(ctx, err)) assert.Less(t, time.Since(start), 200*time.Millisecond) } func resetRateLimitForTest(t *testing.T) { t.Helper() rateLimitMu.Lock() previous := rateLimitResumeTime rateLimitResumeTime = time.Time{} rateLimitMu.Unlock() t.Cleanup(func() { rateLimitMu.Lock() rateLimitResumeTime = previous rateLimitMu.Unlock() }) } func TestEnumerateUnauthenticated(t *testing.T) { defer gock.Off() apiEndpoint := "https://api.github.com" gock.New(apiEndpoint). Get("/orgs/super-secret-org/repos"). Reply(200). JSON([]map[string]string{{"full_name": "super-secret-org/super-secret-repo", "clone_url": "https://github.com/super-secret-org/super-secret-repo.git"}}) s := initTestSource(&sourcespb.GitHub{ Endpoint: apiEndpoint, Credential: &sourcespb.GitHub_Unauthenticated{}, }) s.orgsCache = simple.NewCache[string]() s.orgsCache.Set("super-secret-org", "super-secret-org") // s.enumerateUnauthenticated(context.Background(), apiEndpoint) s.enumerateUnauthenticated(context.Background(), noopReporter()) assert.Equal(t, 1, s.filteredRepoCache.Count()) ok := s.filteredRepoCache.Exists("super-secret-org/super-secret-repo") assert.True(t, ok) assert.False(t, gock.HasUnmatchedRequest()) assert.True(t, gock.IsDone()) } func TestEnumerateWithToken(t *testing.T) { defer gock.Off() gock.New("https://api.github.com"). Get("/user"). Reply(200). JSON(map[string]string{"login": "super-secret-user"}) gock.New("https://api.github.com"). Get("/user/repos"). MatchParam("per_page", "100"). Reply(200). JSON([]map[string]string{{"clone_url": "https://github.com/super-secret-user/super-secret-repo.git", "full_name": "super-secret-user/super-secret-repo"}}) gock.New("https://api.github.com"). Get("/user/orgs"). MatchParam("per_page", "100"). Reply(200). JSON(`[]`) gock.New("https://api.github.com"). Get("/users/super-secret-user/gists"). Reply(200). JSON([]map[string]string{{"id": "super-secret-gist", "git_pull_url": "https://gist.github.com/super-secret-gist.git"}}) s := initTestSource(&sourcespb.GitHub{ Endpoint: "https://api.github.com", Credential: &sourcespb.GitHub_Token{ Token: "token", }, }) err := s.enumerateWithToken(context.Background(), false, noopReporter()) assert.Nil(t, err) assert.Equal(t, 2, s.filteredRepoCache.Count()) ok := s.filteredRepoCache.Exists("super-secret-user/super-secret-repo") assert.True(t, ok) ok = s.filteredRepoCache.Exists("super-secret-gist") assert.True(t, ok) assert.False(t, gock.HasUnmatchedRequest()) assert.True(t, gock.IsDone()) } func BenchmarkEnumerateWithToken(b *testing.B) { defer gock.Off() gock.New("https://api.github.com"). Get("/user"). Reply(200). JSON(map[string]string{"login": "super-secret-user"}) gock.New("https://api.github.com"). Get("/users/super-secret-user/repos"). Reply(200). JSON([]map[string]string{{"clone_url": "https://github.com/super-secret-repo.git"}}) gock.New("https://api.github.com"). Get("/user/orgs"). MatchParam("per_page", "100"). Reply(200). JSON([]map[string]string{{"clone_url": "https://github.com/super-secret-repo.git"}}) gock.New("https://api.github.com"). Get("/users/super-secret-user/gists"). Reply(200). JSON([]map[string]string{{"git_pull_url": "https://github.com/super-secret-gist.git"}}) s := initTestSource(&sourcespb.GitHub{ Endpoint: "https://api.github.com", Credential: &sourcespb.GitHub_Token{ Token: "token", }, }) b.ResetTimer() for i := 0; i < b.N; i++ { _ = s.enumerateWithToken(context.Background(), false, noopReporter()) } } func TestEnumerate(t *testing.T) { defer gock.Off() // Arrange gock.New("https://api.github.com"). Get("/user"). Reply(200). JSON(map[string]string{"login": "super-secret-user"}) // gock.New("https://api.github.com"). Get("/user/repos"). Reply(200). JSON(`[{"name": "super-secret-repo", "full_name": "super-secret-user/super-secret-repo", "owner": {"login": "super-secret-user"}, "clone_url": "https://github.com/super-secret-user/super-secret-repo.git", "has_wiki": false, "size": 1}]`) gock.New("https://api.github.com"). Get("/user/orgs"). MatchParam("per_page", "100"). Reply(200). JSON([]map[string]string{{"clone_url": "https://github.com/super-secret-user/super-secret-repo.git", "full_name": "super-secret-user/super-secret-repo"}}) gock.New("https://api.github.com"). Get("/users/super-secret-user/gists"). Reply(200). JSON(`[{"git_pull_url": "https://gist.github.com/2801a2b0523099d0614a951579d99ba9.git", "id": "2801a2b0523099d0614a951579d99ba9"}]`) s := initTestSource(&sourcespb.GitHub{ Endpoint: "https://api.github.com", Credential: &sourcespb.GitHub_Token{ Token: "super secret token", }, }) // Manually cache a repository to ensure that enumerate // doesn't make duplicate API calls. // See https://github.com/trufflesecurity/trufflehog/pull/2625 repo := func() *github.Repository { var ( name = "cached-repo" fullName = "cached-user/cached-repo" login = "cached-user" cloneUrl = "https://github.com/cached-user/cached-repo.git" owner = &github.User{ Login: &login, } hasWiki = false size = 1234 ) return &github.Repository{ Name: &name, FullName: &fullName, Owner: owner, HasWiki: &hasWiki, Size: &size, CloneURL: &cloneUrl, } }() s.cacheRepoInfo(repo) s.filteredRepoCache.Set(repo.GetFullName(), repo.GetCloneURL()) var reportedRepos []string reporter := sources.VisitorReporter{ VisitUnit: func(ctx context.Context, su sources.SourceUnit) error { url, _ := su.SourceUnitID() reportedRepos = append(reportedRepos, url) return nil }, } // Act err := s.Enumerate(context.Background(), reporter) slices.Sort(reportedRepos) // Assert assert.Nil(t, err) // Enumeration found all repos. assert.Equal(t, 3, s.filteredRepoCache.Count()) assert.True(t, s.filteredRepoCache.Exists("super-secret-user/super-secret-repo")) assert.True(t, s.filteredRepoCache.Exists("cached-user/cached-repo")) assert.True(t, s.filteredRepoCache.Exists("2801a2b0523099d0614a951579d99ba9")) assert.Equal(t, 3, len(s.repos)) assert.Equal(t, s.repos, reportedRepos) // Enumeration cached all repos. assert.Equal(t, 3, len(s.repoInfoCache.cache)) _, ok := s.repoInfoCache.get("https://github.com/super-secret-user/super-secret-repo.git") assert.True(t, ok) _, ok = s.repoInfoCache.get("https://github.com/cached-user/cached-repo.git") assert.True(t, ok) _, ok = s.repoInfoCache.get("https://gist.github.com/2801a2b0523099d0614a951579d99ba9.git") assert.True(t, ok) assert.False(t, gock.HasUnmatchedRequest()) assert.True(t, gock.IsDone()) } func setupMocks(b *testing.B) { b.Helper() gock.New("https://api.github.com"). Get("/user"). Reply(200). JSON(map[string]string{"login": "super-secret-user"}) gock.New("https://api.github.com"). Get("/users/super-secret-user/repos"). Reply(200). JSON(mockRepos()) gock.New("https://api.github.com"). Get("/user/orgs"). MatchParam("per_page", "100"). Reply(200). JSON([]map[string]string{{"clone_url": "https://github.com/super-secret-repo.git"}}) gock.New("https://api.github.com"). Get("/users/super-secret-user/gists"). Reply(200). JSON(mockGists()) } func mockRepos() []map[string]string { res := make([]map[string]string, 0, 10000) for i := 0; i < 10000; i++ { res = append(res, map[string]string{"clone_url": fmt.Sprintf("https://githu/super-secret-repo-%d.git", i)}) } return res } func mockGists() []map[string]string { res := make([]map[string]string, 0, 100) for i := 0; i < 100; i++ { res = append(res, map[string]string{"git_pull_url": fmt.Sprintf("https://githu/super-secret-gist-%d.git", i)}) } return res } func BenchmarkEnumerate(b *testing.B) { for i := 0; i < b.N; i++ { s := initTestSource(&sourcespb.GitHub{ Endpoint: "https://api.github.com", Credential: &sourcespb.GitHub_Token{ Token: "super secret token", }, }) setupMocks(b) b.StartTimer() _ = s.Enumerate(context.Background(), noopReporter()) } } func TestEnumerateWithToken_Repositories(t *testing.T) { defer gock.Off() gock.New("https://api.github.com"). Get("/user"). Reply(200). JSON(map[string]string{"login": "super-secret-user"}) s := initTestSource(&sourcespb.GitHub{ Endpoint: "https://api.github.com", Credential: &sourcespb.GitHub_Token{ Token: "token", }, }) s.repos = []string{"some-special-repo"} err := s.enumerateWithToken(context.Background(), false, noopReporter()) assert.Nil(t, err) assert.Equal(t, 1, len(s.repos)) assert.Equal(t, []string{"some-special-repo"}, s.repos) assert.False(t, gock.HasUnmatchedRequest()) assert.True(t, gock.IsDone()) } func TestEnumerateWithToken_IncludeRepos(t *testing.T) { defer gock.Off() gock.New("https://api.github.com"). Get("/user"). Reply(200). JSON(map[string]string{"login": "super-secret-user"}) s := initTestSource(&sourcespb.GitHub{ Endpoint: "https://api.github.com", Credential: &sourcespb.GitHub_Token{ Token: "token", }, }) s.repos = []string{"some-special-repo"} err := s.enumerateWithToken(context.Background(), false, noopReporter()) assert.Nil(t, err) assert.Equal(t, 1, len(s.repos)) assert.Equal(t, []string{"some-special-repo"}, s.repos) assert.False(t, gock.HasUnmatchedRequest()) assert.True(t, gock.IsDone()) } func TestEnumerateWithApp(t *testing.T) { defer gock.Off() privateKey := createPrivateKey() gock.New("https://api.github.com"). Post("/app/installations/1337/access_tokens"). Reply(200). JSON(map[string]string{"token": "dontlook"}) gock.New("https://api.github.com"). Get("/installation/repositories"). Reply(200). JSON(map[string]string{}) s := initTestSource(&sourcespb.GitHub{ Endpoint: "https://api.github.com", Credential: &sourcespb.GitHub_GithubApp{ GithubApp: &credentialspb.GitHubApp{ PrivateKey: privateKey, InstallationId: "1337", AppId: "4141", }, }, }) err := s.enumerateWithApp(context.Background(), s.connector.(*appConnector), noopReporter()) assert.Nil(t, err) assert.Equal(t, 0, len(s.repos)) assert.False(t, gock.HasUnmatchedRequest()) assert.True(t, gock.IsDone()) } func TestEnumerateWithAppScanAllInstallationsIncludesConfiguredInstallation(t *testing.T) { privateKey := createPrivateKey() server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") switch { case r.Method == http.MethodPost && strings.Contains(r.URL.Path, "/app/installations/") && strings.HasSuffix(r.URL.Path, "/access_tokens"): parts := strings.Split(r.URL.Path, "/") installID := parts[len(parts)-2] _, _ = fmt.Fprintf(w, `{"token":"token-%s","expires_at":"2099-01-01T00:00:00Z"}`, installID) case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/app/installations"): _, _ = w.Write([]byte(`[ {"id":1337,"account":{"login":"default-org","type":"Organization"}}, {"id":2448,"account":{"login":"other-org","type":"Organization"}} ]`)) case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/installation/repositories"): auth := r.Header.Get("Authorization") switch { case strings.Contains(auth, "token-1337"): _, _ = w.Write([]byte(`{"total_count":1,"repositories":[ {"name":"repo","full_name":"default-org/repo","clone_url":"https://github.com/default-org/repo.git","owner":{"login":"default-org","type":"Organization"},"size":1} ]}`)) case strings.Contains(auth, "token-2448"): _, _ = w.Write([]byte(`{"total_count":1,"repositories":[ {"name":"repo","full_name":"other-org/repo","clone_url":"https://github.com/other-org/repo.git","owner":{"login":"other-org","type":"Organization"},"size":1} ]}`)) default: http.Error(w, "unexpected installation token", http.StatusUnauthorized) } default: http.NotFound(w, r) } })) defer server.Close() s, conn := createTestSource(&sourcespb.GitHub{ Endpoint: server.URL, ScanAllInstallations: true, Credential: &sourcespb.GitHub_GithubApp{ GithubApp: &credentialspb.GitHubApp{ PrivateKey: privateKey, InstallationId: "1337", AppId: "4141", }, }, }) require.NoError(t, s.Init(context.Background(), "test - github", 0, 1337, false, conn, 1)) connector := s.connector.(*appConnector) err := s.enumerateWithApp(context.Background(), connector, noopReporter()) require.NoError(t, err) assert.Equal(t, int64(1337), connector.repoInstallationMap["https://github.com/default-org/repo.git"]) assert.Equal(t, int64(2448), connector.repoInstallationMap["https://github.com/other-org/repo.git"]) assert.True(t, s.filteredRepoCache.Exists("default-org/repo")) assert.True(t, s.filteredRepoCache.Exists("other-org/repo")) } func TestEnumerateWithAppScanAllInstallationsMapsExplicitReposBeforeMetadataFetch(t *testing.T) { privateKey := createPrivateKey() server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") switch { case r.Method == http.MethodPost && strings.Contains(r.URL.Path, "/app/installations/") && strings.HasSuffix(r.URL.Path, "/access_tokens"): parts := strings.Split(r.URL.Path, "/") installID := parts[len(parts)-2] _, _ = fmt.Fprintf(w, `{"token":"token-%s","expires_at":"2099-01-01T00:00:00Z"}`, installID) case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/app/installations"): _, _ = w.Write([]byte(`[ {"id":1337,"account":{"login":"default-org","type":"Organization"}}, {"id":2448,"account":{"login":"other-org","type":"Organization"}} ]`)) case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/installation/repositories"): auth := r.Header.Get("Authorization") switch { case strings.Contains(auth, "token-1337"): _, _ = w.Write([]byte(`{"total_count":1,"repositories":[ {"name":"repo","full_name":"default-org/repo","clone_url":"https://github.com/default-org/repo.git","owner":{"login":"default-org","type":"Organization"},"size":1} ]}`)) case strings.Contains(auth, "token-2448"): _, _ = w.Write([]byte(`{"total_count":1,"repositories":[ {"name":"repo","full_name":"other-org/repo","clone_url":"https://github.com/other-org/repo.git","owner":{"login":"other-org","type":"Organization"},"size":1} ]}`)) default: http.Error(w, "unexpected installation token", http.StatusUnauthorized) } case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/repos/other-org/repo"): if !strings.Contains(r.Header.Get("Authorization"), "token-2448") { http.Error(w, "wrong installation token", http.StatusForbidden) return } _, _ = w.Write([]byte(`{"name":"repo","full_name":"other-org/repo","clone_url":"https://github.com/other-org/repo.git","owner":{"login":"other-org","type":"Organization"},"size":1}`)) default: http.NotFound(w, r) } })) defer server.Close() s, conn := createTestSource(&sourcespb.GitHub{ Endpoint: server.URL, Repositories: []string{"https://github.com/other-org/repo.git"}, ScanAllInstallations: true, Credential: &sourcespb.GitHub_GithubApp{ GithubApp: &credentialspb.GitHubApp{ PrivateKey: privateKey, InstallationId: "1337", AppId: "4141", }, }, }) require.NoError(t, s.Init(context.Background(), "test - github", 0, 1337, false, conn, 1)) err := s.Enumerate(context.Background(), noopReporter()) require.NoError(t, err) connector := s.connector.(*appConnector) assert.Equal(t, int64(2448), connector.repoInstallationMap["https://github.com/other-org/repo.git"]) info, ok := s.repoInfoCache.get("https://github.com/other-org/repo.git") require.True(t, ok) assert.Equal(t, "other-org/repo", info.fullName) } func TestMapExplicitReposToInstallationsErrorsForUnmatchedRepos(t *testing.T) { privateKey := createPrivateKey() server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") switch { case r.Method == http.MethodPost && strings.Contains(r.URL.Path, "/app/installations/") && strings.HasSuffix(r.URL.Path, "/access_tokens"): parts := strings.Split(r.URL.Path, "/") installID := parts[len(parts)-2] _, _ = fmt.Fprintf(w, `{"token":"token-%s","expires_at":"2099-01-01T00:00:00Z"}`, installID) case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/app/installations"): _, _ = w.Write([]byte(`[{"id":2448,"account":{"login":"other-org","type":"Organization"}}]`)) case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/installation/repositories"): _, _ = w.Write([]byte(`{"total_count":1,"repositories":[ {"name":"different","full_name":"other-org/different","clone_url":"https://github.com/other-org/different.git","owner":{"login":"other-org","type":"Organization"},"size":1} ]}`)) default: http.NotFound(w, r) } })) defer server.Close() s, conn := createTestSource(&sourcespb.GitHub{ Endpoint: server.URL, Repositories: []string{"https://github.com/other-org/missing.git"}, ScanAllInstallations: true, Credential: &sourcespb.GitHub_GithubApp{ GithubApp: &credentialspb.GitHubApp{ PrivateKey: privateKey, InstallationId: "1337", AppId: "4141", }, }, }) err := s.Init(context.Background(), "test - github", 0, 1337, false, conn, 1) require.Error(t, err) assert.Contains(t, err.Error(), "configured repos were not found in any GitHub App installation") assert.Contains(t, err.Error(), "https://github.com/other-org/missing.git") } func TestScanAllInstallationsInitErrorsWhenInstallationsCannotBeListed(t *testing.T) { privateKey := createPrivateKey() server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") switch { case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/app/installations"): http.Error(w, "installation list failed", http.StatusInternalServerError) default: http.NotFound(w, r) } })) defer server.Close() s, conn := createTestSource(&sourcespb.GitHub{ Endpoint: server.URL, ScanAllInstallations: true, Credential: &sourcespb.GitHub_GithubApp{ GithubApp: &credentialspb.GitHubApp{ PrivateKey: privateKey, InstallationId: "1337", AppId: "4141", }, }, }) err := s.Init(context.Background(), "test - github", 0, 1337, false, conn, 1) require.Error(t, err) assert.Contains(t, err.Error(), "could not list installations") } func TestMapExplicitReposToInstallationsStopsAfterReposFound(t *testing.T) { privateKey := createPrivateKey() var listInstallationsCalls, unrelatedRepoListCalls int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") switch { case r.Method == http.MethodPost && strings.Contains(r.URL.Path, "/app/installations/") && strings.HasSuffix(r.URL.Path, "/access_tokens"): parts := strings.Split(r.URL.Path, "/") installID := parts[len(parts)-2] _, _ = fmt.Fprintf(w, `{"token":"token-%s","expires_at":"2099-01-01T00:00:00Z"}`, installID) case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/app/installations"): listInstallationsCalls++ _, _ = w.Write([]byte(`[ {"id":1337,"account":{"login":"default-org","type":"Organization"}}, {"id":2448,"account":{"login":"other-org","type":"Organization"}}, {"id":3559,"account":{"login":"unrelated-org","type":"Organization"}} ]`)) case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/installation/repositories"): auth := r.Header.Get("Authorization") switch { case strings.Contains(auth, "token-1337"): _, _ = w.Write([]byte(`{"total_count":0,"repositories":[]}`)) case strings.Contains(auth, "token-2448"): _, _ = w.Write([]byte(`{"total_count":1,"repositories":[ {"name":"repo","full_name":"other-org/repo","clone_url":"https://github.com/other-org/repo.git","owner":{"login":"other-org","type":"Organization"},"size":1} ]}`)) case strings.Contains(auth, "token-3559"): unrelatedRepoListCalls++ http.Error(w, "unrelated installation failed", http.StatusInternalServerError) default: http.Error(w, "unexpected installation token", http.StatusUnauthorized) } default: http.NotFound(w, r) } })) defer server.Close() s, conn := createTestSource(&sourcespb.GitHub{ Endpoint: server.URL, Repositories: []string{"https://github.com/other-org/repo.git"}, ScanAllInstallations: true, Credential: &sourcespb.GitHub_GithubApp{ GithubApp: &credentialspb.GitHubApp{ PrivateKey: privateKey, InstallationId: "1337", AppId: "4141", }, }, }) require.NoError(t, s.Init(context.Background(), "test - github", 0, 1337, false, conn, 1)) connector := s.connector.(*appConnector) require.NoError(t, s.mapExplicitReposToInstallations(context.Background(), connector)) require.NoError(t, s.mapExplicitReposToInstallations(context.Background(), connector)) installationID, mapped := connector.installationIDForRepo("https://github.com/other-org/repo.git") assert.True(t, mapped) assert.Equal(t, int64(2448), installationID) assert.Equal(t, 1, listInstallationsCalls) assert.Equal(t, 0, unrelatedRepoListCalls) } func TestScanAllInstallationsReusesInstallationsLoadedDuringInit(t *testing.T) { privateKey := createPrivateKey() var listInstallationsCalls int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") switch { case r.Method == http.MethodPost && strings.Contains(r.URL.Path, "/app/installations/") && strings.HasSuffix(r.URL.Path, "/access_tokens"): parts := strings.Split(r.URL.Path, "/") installID := parts[len(parts)-2] _, _ = fmt.Fprintf(w, `{"token":"token-%s","expires_at":"2099-01-01T00:00:00Z"}`, installID) case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/app/installations"): listInstallationsCalls++ _, _ = w.Write([]byte(`[ {"id":1337,"account":{"login":"default-org","type":"Organization"}}, {"id":2448,"account":{"login":"other-org","type":"Organization"}} ]`)) case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/installation/repositories"): if strings.Contains(r.Header.Get("Authorization"), "token-2448") { _, _ = w.Write([]byte(`{"total_count":1,"repositories":[ {"name":"repo","full_name":"other-org/repo","clone_url":"https://github.com/other-org/repo.git","owner":{"login":"other-org","type":"Organization"},"size":1} ]}`)) return } _, _ = w.Write([]byte(`{"total_count":0,"repositories":[]}`)) case r.Method == http.MethodGet && strings.Contains(r.URL.Path, "/orgs/") && strings.HasSuffix(r.URL.Path, "/members"): _, _ = w.Write([]byte(`[]`)) default: http.NotFound(w, r) } })) defer server.Close() s, conn := createTestSource(&sourcespb.GitHub{ Endpoint: server.URL, Repositories: []string{"https://github.com/other-org/repo.git"}, ScanAllInstallations: true, Credential: &sourcespb.GitHub_GithubApp{ GithubApp: &credentialspb.GitHubApp{ PrivateKey: privateKey, InstallationId: "1337", AppId: "4141", }, }, }) require.NoError(t, s.Init(context.Background(), "test - github", 0, 1337, false, conn, 1)) connector := s.connector.(*appConnector) require.NoError(t, s.enumerateAllInstallationRepos(context.Background(), connector, noopReporter())) require.NoError(t, s.addMembersByApp(context.Background(), connector, noopReporter())) assert.Equal(t, 1, listInstallationsCalls) } func TestMapExplicitReposToInstallationsMatchesSSHRepoURL(t *testing.T) { privateKey := createPrivateKey() const sshRepoURL = "ssh://git@github.com/other-org/repo.git" server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") switch { case r.Method == http.MethodPost && strings.Contains(r.URL.Path, "/app/installations/") && strings.HasSuffix(r.URL.Path, "/access_tokens"): _, _ = w.Write([]byte(`{"token":"token-2448","expires_at":"2099-01-01T00:00:00Z"}`)) case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/app/installations"): _, _ = w.Write([]byte(`[{"id":2448,"account":{"login":"other-org","type":"Organization"}}]`)) case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/installation/repositories"): _, _ = w.Write([]byte(`{"total_count":1,"repositories":[ {"name":"repo","full_name":"other-org/repo","clone_url":"https://github.com/other-org/repo.git","owner":{"login":"other-org","type":"Organization"},"size":1} ]}`)) case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/repos/other-org/repo"): _, _ = w.Write([]byte(`{"name":"repo","full_name":"other-org/repo","clone_url":"https://github.com/other-org/repo.git","owner":{"login":"other-org","type":"Organization"},"size":1}`)) default: http.NotFound(w, r) } })) defer server.Close() s, conn := createTestSource(&sourcespb.GitHub{ Endpoint: server.URL, Repositories: []string{sshRepoURL}, ScanAllInstallations: true, Credential: &sourcespb.GitHub_GithubApp{ GithubApp: &credentialspb.GitHubApp{ PrivateKey: privateKey, InstallationId: "1337", AppId: "4141", }, }, }) require.NoError(t, s.Init(context.Background(), "test - github", 0, 1337, false, conn, 1)) connector := s.connector.(*appConnector) installationID, mapped := connector.installationIDForRepo(sshRepoURL) assert.True(t, mapped) assert.Equal(t, int64(2448), installationID) } func TestMapExplicitReposToInstallationsMatchesSCPStyleSSHRepoURL(t *testing.T) { privateKey := createPrivateKey() const sshRepoURL = "git@github.com:other-org/repo.git" server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") switch { case r.Method == http.MethodPost && strings.Contains(r.URL.Path, "/app/installations/") && strings.HasSuffix(r.URL.Path, "/access_tokens"): _, _ = w.Write([]byte(`{"token":"token-2448","expires_at":"2099-01-01T00:00:00Z"}`)) case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/app/installations"): _, _ = w.Write([]byte(`[{"id":2448,"account":{"login":"other-org","type":"Organization"}}]`)) case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/installation/repositories"): _, _ = w.Write([]byte(`{"total_count":1,"repositories":[ {"name":"repo","full_name":"other-org/repo","clone_url":"https://github.com/other-org/repo.git","owner":{"login":"other-org","type":"Organization"},"size":1} ]}`)) case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/repos/other-org/repo"): _, _ = w.Write([]byte(`{"name":"repo","full_name":"other-org/repo","clone_url":"https://github.com/other-org/repo.git","owner":{"login":"other-org","type":"Organization"},"size":1}`)) default: http.NotFound(w, r) } })) defer server.Close() s, conn := createTestSource(&sourcespb.GitHub{ Endpoint: server.URL, Repositories: []string{sshRepoURL}, ScanAllInstallations: true, Credential: &sourcespb.GitHub_GithubApp{ GithubApp: &credentialspb.GitHubApp{ PrivateKey: privateKey, InstallationId: "1337", AppId: "4141", }, }, }) require.NoError(t, s.Init(context.Background(), "test - github", 0, 1337, false, conn, 1)) connector := s.connector.(*appConnector) installationID, mapped := connector.installationIDForRepo(sshRepoURL) assert.True(t, mapped) assert.Equal(t, int64(2448), installationID) } func TestMapExplicitReposToInstallationsRejectsHostMismatch(t *testing.T) { privateKey := createPrivateKey() const attackerRepoURL = "https://attacker.example/other-org/repo.git" server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") switch { case r.Method == http.MethodPost && strings.Contains(r.URL.Path, "/app/installations/") && strings.HasSuffix(r.URL.Path, "/access_tokens"): _, _ = w.Write([]byte(`{"token":"token-2448","expires_at":"2099-01-01T00:00:00Z"}`)) case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/app/installations"): _, _ = w.Write([]byte(`[{"id":2448,"account":{"login":"other-org","type":"Organization"}}]`)) case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/installation/repositories"): _, _ = w.Write([]byte(`{"total_count":1,"repositories":[ {"name":"repo","full_name":"other-org/repo","clone_url":"https://github.com/other-org/repo.git","owner":{"login":"other-org","type":"Organization"},"size":1} ]}`)) case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/repos/other-org/repo"): _, _ = w.Write([]byte(`{"name":"repo","full_name":"other-org/repo","clone_url":"https://github.com/other-org/repo.git","owner":{"login":"other-org","type":"Organization"},"size":1}`)) default: http.NotFound(w, r) } })) defer server.Close() s, conn := createTestSource(&sourcespb.GitHub{ Endpoint: server.URL, Repositories: []string{attackerRepoURL}, ScanAllInstallations: true, Credential: &sourcespb.GitHub_GithubApp{ GithubApp: &credentialspb.GitHubApp{ PrivateKey: privateKey, InstallationId: "1337", AppId: "4141", }, }, }) err := s.Init(context.Background(), "test - github", 0, 1337, false, conn, 1) require.Error(t, err) assert.Contains(t, err.Error(), "configured repos were not found") } // A repo absent from every installation listing but readable with the // default installation token (e.g. a public repo, or one the listing missed) // maps to the default installation instead of failing the scan. Repos the // default token cannot read are still rejected (see the preceding test and // TestScanAllInstallationsInaccessibleRepoStillFailsMapping). func TestMapExplicitReposToInstallationsFallsBackToDefaultForReadableRepo(t *testing.T) { privateKey := createPrivateKey() const repoURL = "https://github.com/other-org/repo.git" server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") switch { case r.Method == http.MethodPost && strings.Contains(r.URL.Path, "/app/installations/") && strings.HasSuffix(r.URL.Path, "/access_tokens"): _, _ = w.Write([]byte(`{"token":"token-2448","expires_at":"2099-01-01T00:00:00Z"}`)) case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/app/installations"): _, _ = w.Write([]byte(`[{"id":2448,"account":{"login":"other-org","type":"Organization"}}]`)) case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/installation/repositories"): _, _ = w.Write([]byte(`{"total_count":0,"repositories":[]}`)) case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/repos/other-org/repo"): _, _ = w.Write([]byte(`{"name":"repo","full_name":"other-org/repo","clone_url":"https://github.com/other-org/repo.git","owner":{"login":"other-org","type":"Organization"},"size":1}`)) default: http.NotFound(w, r) } })) defer server.Close() s, conn := createTestSource(&sourcespb.GitHub{ Endpoint: server.URL, Repositories: []string{repoURL}, ScanAllInstallations: true, Credential: &sourcespb.GitHub_GithubApp{ GithubApp: &credentialspb.GitHubApp{ PrivateKey: privateKey, InstallationId: "1337", AppId: "4141", }, }, }) err := s.Init(context.Background(), "test - github", 0, 1337, false, conn, 1) require.NoError(t, err) connector := s.connector.(*appConnector) installationID, mapped := connector.installationIDForRepo(repoURL) require.True(t, mapped) assert.Equal(t, int64(1337), installationID) } func TestMapExplicitReposToInstallationsMapsWikiRepoURL(t *testing.T) { privateKey := createPrivateKey() const wikiRepoURL = "https://github.com/other-org/repo.wiki.git" server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") switch { case r.Method == http.MethodPost && strings.Contains(r.URL.Path, "/app/installations/") && strings.HasSuffix(r.URL.Path, "/access_tokens"): _, _ = w.Write([]byte(`{"token":"token-2448","expires_at":"2099-01-01T00:00:00Z"}`)) case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/app/installations"): _, _ = w.Write([]byte(`[{"id":2448,"account":{"login":"other-org","type":"Organization"}}]`)) case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/installation/repositories"): _, _ = w.Write([]byte(`{"total_count":1,"repositories":[ {"name":"repo","full_name":"other-org/repo","clone_url":"https://github.com/other-org/repo.git","owner":{"login":"other-org","type":"Organization"},"size":1} ]}`)) case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/repos/other-org/repo"): _, _ = w.Write([]byte(`{"name":"repo","full_name":"other-org/repo","clone_url":"https://github.com/other-org/repo.git","owner":{"login":"other-org","type":"Organization"},"size":1}`)) default: http.NotFound(w, r) } })) defer server.Close() s, conn := createTestSource(&sourcespb.GitHub{ Endpoint: server.URL, Repositories: []string{wikiRepoURL}, ScanAllInstallations: true, Credential: &sourcespb.GitHub_GithubApp{ GithubApp: &credentialspb.GitHubApp{ PrivateKey: privateKey, InstallationId: "1337", AppId: "4141", }, }, }) require.NoError(t, s.Init(context.Background(), "test - github", 0, 1337, false, conn, 1)) connector := s.connector.(*appConnector) installationID, mapped := connector.installationIDForRepo("https://github.com/other-org/repo.git") assert.True(t, mapped) assert.Equal(t, int64(2448), installationID) installationID, mapped = connector.installationIDForRepo(wikiRepoURL) assert.True(t, mapped) assert.Equal(t, int64(2448), installationID) } func TestMapExplicitReposToInstallationsPrefersRealWikiSuffixedRepo(t *testing.T) { privateKey := createPrivateKey() const parentRepoURL = "https://github.com/other-org/repo.git" const wikiSuffixedRepoURL = "https://github.com/other-org/repo.wiki.git" const wikiOfWikiSuffixedRepoURL = "https://github.com/other-org/repo.wiki.wiki.git" server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") switch { case r.Method == http.MethodPost && strings.Contains(r.URL.Path, "/app/installations/") && strings.HasSuffix(r.URL.Path, "/access_tokens"): _, _ = w.Write([]byte(`{"token":"token-2448","expires_at":"2099-01-01T00:00:00Z"}`)) case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/app/installations"): _, _ = w.Write([]byte(`[{"id":2448,"account":{"login":"other-org","type":"Organization"}}]`)) case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/installation/repositories"): _, _ = w.Write([]byte(`{"total_count":2,"repositories":[ {"name":"repo.wiki","full_name":"other-org/repo.wiki","clone_url":"https://github.com/other-org/repo.wiki.git","owner":{"login":"other-org","type":"Organization"},"size":1,"has_wiki":true}, {"name":"repo","full_name":"other-org/repo","clone_url":"https://github.com/other-org/repo.git","owner":{"login":"other-org","type":"Organization"},"size":1} ]}`)) default: http.NotFound(w, r) } })) defer server.Close() s, conn := createTestSource(&sourcespb.GitHub{ Endpoint: server.URL, Repositories: []string{parentRepoURL, wikiSuffixedRepoURL}, ScanAllInstallations: true, Credential: &sourcespb.GitHub_GithubApp{ GithubApp: &credentialspb.GitHubApp{ PrivateKey: privateKey, InstallationId: "1337", AppId: "4141", }, }, }) require.NoError(t, s.Init(context.Background(), "test - github", 0, 1337, false, conn, 1)) connector := s.connector.(*appConnector) installationID, mapped := connector.installationIDForRepo(parentRepoURL) assert.True(t, mapped) assert.Equal(t, int64(2448), installationID) installationID, mapped = connector.installationIDForRepo(wikiSuffixedRepoURL) assert.True(t, mapped) assert.Equal(t, int64(2448), installationID) installationID, mapped = connector.installationIDForRepo(wikiOfWikiSuffixedRepoURL) assert.True(t, mapped) assert.Equal(t, int64(2448), installationID) } func TestMapExplicitReposToInstallationsMapsRedirectedRepoFromMetadata(t *testing.T) { privateKey := createPrivateKey() const oldRepoURL = "https://github.com/old-org/repo.git" const canonicalRepoURL = "https://github.com/new-org/repo.git" server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") switch { case r.Method == http.MethodPost && strings.Contains(r.URL.Path, "/app/installations/") && strings.HasSuffix(r.URL.Path, "/access_tokens"): _, _ = w.Write([]byte(`{"token":"token-2448","expires_at":"2099-01-01T00:00:00Z"}`)) case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/app/installations"): _, _ = w.Write([]byte(`[{"id":2448,"account":{"login":"new-org","type":"Organization"}}]`)) case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/installation/repositories"): _, _ = w.Write([]byte(`{"total_count":1,"repositories":[ {"name":"repo","full_name":"new-org/repo","clone_url":"https://github.com/new-org/repo.git","owner":{"login":"new-org","type":"Organization"},"size":1} ]}`)) case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/repos/old-org/repo"): if !strings.Contains(r.Header.Get("Authorization"), "token-2448") { http.Error(w, "wrong installation token", http.StatusForbidden) return } _, _ = w.Write([]byte(`{"name":"repo","full_name":"new-org/repo","clone_url":"https://github.com/new-org/repo.git","owner":{"login":"new-org","type":"Organization"},"size":1}`)) default: http.NotFound(w, r) } })) defer server.Close() s, conn := createTestSource(&sourcespb.GitHub{ Endpoint: server.URL, Repositories: []string{oldRepoURL}, ScanAllInstallations: true, Credential: &sourcespb.GitHub_GithubApp{ GithubApp: &credentialspb.GitHubApp{ PrivateKey: privateKey, InstallationId: "1337", AppId: "4141", }, }, }) require.NoError(t, s.Init(context.Background(), "test - github", 0, 1337, false, conn, 1)) connector := s.connector.(*appConnector) installationID, mapped := connector.installationIDForRepo(oldRepoURL) assert.True(t, mapped) assert.Equal(t, int64(2448), installationID) installationID, mapped = connector.installationIDForRepo(canonicalRepoURL) assert.True(t, mapped) assert.Equal(t, int64(2448), installationID) info, ok := s.repoInfoCache.get(oldRepoURL) require.True(t, ok) assert.Equal(t, "new-org/repo", info.fullName) } func TestRepoURLFromTargetMetadataErrorsOnRepositoryLinkMismatch(t *testing.T) { s := &Source{} _, err := s.repoURLFromTargetMetadata(&source_metadatapb.Github{ Repository: "https://github.com/other-org/repo.git", Link: "https://github.com/different-org/repo/blob/abcd1234/path/to/file.go#L1", }) require.Error(t, err) assert.Contains(t, err.Error(), "does not match link repository") } func TestRepoURLFromTargetMetadataErrorsOnHostMismatch(t *testing.T) { s := &Source{} _, err := s.repoURLFromTargetMetadata(&source_metadatapb.Github{ Repository: "https://github.company/other-org/repo.git", Link: "https://github.com/other-org/repo/blob/abcd1234/path/to/file.go#L1", }) require.Error(t, err) assert.Contains(t, err.Error(), "does not match link repository") } func TestRepoURLFromTargetMetadataAllowsBareRepositoryName(t *testing.T) { s := &Source{} repoURL, err := s.repoURLFromTargetMetadata(&source_metadatapb.Github{ Repository: "repo", Link: "https://github.com/other-org/repo/blob/abcd1234/path/to/file.go#L1", }) require.NoError(t, err) assert.Equal(t, "https://github.com/other-org/repo.git", repoURL) } func TestRepoURLFromTargetMetadataAllowsWikiRepositoryWithWikiLink(t *testing.T) { s := &Source{} repoURL, err := s.repoURLFromTargetMetadata(&source_metadatapb.Github{ Repository: "https://github.com/other-org/repo.wiki.git", Link: "https://github.com/other-org/repo/wiki/path/to/file.go", }) require.NoError(t, err) assert.Equal(t, "https://github.com/other-org/repo.git", repoURL) } func TestScanAllInstallationsMapsUnitBeforeMetadataFetch(t *testing.T) { privateKey := createPrivateKey() server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") switch { case r.Method == http.MethodPost && strings.Contains(r.URL.Path, "/app/installations/") && strings.HasSuffix(r.URL.Path, "/access_tokens"): parts := strings.Split(r.URL.Path, "/") installID := parts[len(parts)-2] _, _ = fmt.Fprintf(w, `{"token":"token-%s","expires_at":"2099-01-01T00:00:00Z"}`, installID) case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/app/installations"): _, _ = w.Write([]byte(`[ {"id":1337,"account":{"login":"default-org","type":"Organization"}}, {"id":2448,"account":{"login":"other-org","type":"Organization"}} ]`)) case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/installation/repositories"): auth := r.Header.Get("Authorization") switch { case strings.Contains(auth, "token-1337"): _, _ = w.Write([]byte(`{"total_count":1,"repositories":[]}`)) case strings.Contains(auth, "token-2448"): _, _ = w.Write([]byte(`{"total_count":1,"repositories":[ {"name":"repo","full_name":"other-org/repo","clone_url":"https://github.com/other-org/repo.git","owner":{"login":"other-org","type":"Organization"},"size":1} ]}`)) default: http.Error(w, "unexpected installation token", http.StatusUnauthorized) } case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/repos/other-org/repo"): if !strings.Contains(r.Header.Get("Authorization"), "token-2448") { http.Error(w, "wrong installation token", http.StatusForbidden) return } _, _ = w.Write([]byte(`{"name":"repo","full_name":"other-org/repo","clone_url":"https://github.com/other-org/repo.git","owner":{"login":"other-org","type":"Organization"},"size":1}`)) default: http.NotFound(w, r) } })) defer server.Close() s, conn := createTestSource(&sourcespb.GitHub{ Endpoint: server.URL, ScanAllInstallations: true, Credential: &sourcespb.GitHub_GithubApp{ GithubApp: &credentialspb.GitHubApp{ PrivateKey: privateKey, InstallationId: "1337", AppId: "4141", }, }, }) require.NoError(t, s.Init(context.Background(), "test - github", 0, 1337, false, conn, 1)) connector := s.connector.(*appConnector) err := s.mapReposToInstallations(context.Background(), connector, []string{"https://github.com/other-org/repo.git"}) require.NoError(t, err) repoURL, err := s.ensureRepoInfoCache(context.Background(), "https://github.com/other-org/repo.git", &unitErrorReporter{noopReporter()}) require.NoError(t, err) require.Equal(t, "https://github.com/other-org/repo.git", repoURL) installationID, mapped := connector.installationIDForRepo("https://github.com/other-org/repo.git") assert.True(t, mapped) assert.Equal(t, int64(2448), installationID) } // Regression test for member personal repos under scan_all_installations // (INT-789): they belong to no app installation, so the mapping must fall // back to the default installation when the API confirms the repo is // readable, both at chunk time and at enumeration time. func TestScanAllInstallationsMemberPersonalRepoFallsBackToDefaultInstallation(t *testing.T) { privateKey := createPrivateKey() server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") switch { case r.Method == http.MethodPost && strings.Contains(r.URL.Path, "/app/installations/") && strings.HasSuffix(r.URL.Path, "/access_tokens"): parts := strings.Split(r.URL.Path, "/") installID := parts[len(parts)-2] _, _ = fmt.Fprintf(w, `{"token":"token-%s","expires_at":"2099-01-01T00:00:00Z"}`, installID) case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/app/installations"): _, _ = w.Write([]byte(`[ {"id":1337,"account":{"login":"test-org","type":"Organization"}}, {"id":2448,"account":{"login":"other-org","type":"Organization"}} ]`)) case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/installation/repositories"): // Installations only ever contain org-owned repos. auth := r.Header.Get("Authorization") switch { case strings.Contains(auth, "token-1337"): _, _ = w.Write([]byte(`{"total_count":1,"repositories":[ {"name":"backend","full_name":"test-org/backend","clone_url":"https://github.com/test-org/backend.git","owner":{"login":"test-org","type":"Organization"},"size":1} ]}`)) case strings.Contains(auth, "token-2448"): _, _ = w.Write([]byte(`{"total_count":0,"repositories":[]}`)) default: http.Error(w, "unexpected installation token", http.StatusUnauthorized) } case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/repos/alice/dns-data"): // The member's personal repo exists and is public: any valid // token can read it, mirroring real GitHub behavior. _, _ = w.Write([]byte(`{"name":"dns-data","full_name":"alice/dns-data","clone_url":"https://github.com/alice/dns-data.git","owner":{"login":"alice","type":"User"},"private":false,"size":1}`)) case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/users/alice/repos"): _, _ = w.Write([]byte(`[{"name":"dns-data","full_name":"alice/dns-data","clone_url":"https://github.com/alice/dns-data.git","owner":{"login":"alice","type":"User"},"private":false,"size":1}]`)) default: http.NotFound(w, r) } })) defer server.Close() s, conn := createTestSource(&sourcespb.GitHub{ Endpoint: server.URL, ScanAllInstallations: true, ScanUsers: true, Credential: &sourcespb.GitHub_GithubApp{ GithubApp: &credentialspb.GitHubApp{ PrivateKey: privateKey, InstallationId: "1337", AppId: "4141", }, }, }) require.NoError(t, s.Init(context.Background(), "test - github", 0, 1337, false, conn, 1)) // The org-owned repo maps to its owning installation. connector := s.connector.(*appConnector) err := s.mapReposToInstallations(context.Background(), connector, []string{"https://github.com/test-org/backend.git"}) require.NoError(t, err) // The member's public personal repo is in no installation, but the API // confirms it is accessible, so it maps to the default installation. err = s.mapReposToInstallations(context.Background(), connector, []string{"https://github.com/alice/dns-data.git"}) require.NoError(t, err) installationID, mapped := connector.installationIDForRepo("https://github.com/alice/dns-data.git") require.True(t, mapped) require.Equal(t, int64(1337), installationID) // Enumeration must map member repos up front so same-process ChunkUnit // short-circuits without any mapping API calls. connector.mu.Lock() delete(connector.repoInstallationMap, "https://github.com/alice/dns-data.git") connector.mu.Unlock() require.NoError(t, s.getReposByUser(context.Background(), "alice", false, noopReporter())) installationID, mapped = connector.installationIDForRepo("https://github.com/alice/dns-data.git") require.True(t, mapped) require.Equal(t, int64(1337), installationID) } // Repos that no installation owns AND the default installation token cannot // read must still fail the mapping: the accessibility fallback must not turn // the check fail-open. func TestScanAllInstallationsInaccessibleRepoStillFailsMapping(t *testing.T) { privateKey := createPrivateKey() server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") switch { case r.Method == http.MethodPost && strings.Contains(r.URL.Path, "/app/installations/") && strings.HasSuffix(r.URL.Path, "/access_tokens"): parts := strings.Split(r.URL.Path, "/") installID := parts[len(parts)-2] _, _ = fmt.Fprintf(w, `{"token":"token-%s","expires_at":"2099-01-01T00:00:00Z"}`, installID) case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/app/installations"): _, _ = w.Write([]byte(`[{"id":1337,"account":{"login":"test-org","type":"Organization"}}]`)) case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/installation/repositories"): _, _ = w.Write([]byte(`{"total_count":0,"repositories":[]}`)) default: // /repos/... lookups 404: repo is private/nonexistent for // every installation token. http.NotFound(w, r) } })) defer server.Close() s, conn := createTestSource(&sourcespb.GitHub{ Endpoint: server.URL, ScanAllInstallations: true, Credential: &sourcespb.GitHub_GithubApp{ GithubApp: &credentialspb.GitHubApp{ PrivateKey: privateKey, InstallationId: "1337", AppId: "4141", }, }, }) require.NoError(t, s.Init(context.Background(), "test - github", 0, 1337, false, conn, 1)) connector := s.connector.(*appConnector) err := s.mapReposToInstallations(context.Background(), connector, []string{"https://github.com/ghost/private-repo.git"}) require.Error(t, err) require.Contains(t, err.Error(), "configured repos were not found in any GitHub App installation") } func TestEnumerateAllInstallationReposReportsInstallationErrors(t *testing.T) { privateKey := createPrivateKey() server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") switch { case r.Method == http.MethodPost && strings.Contains(r.URL.Path, "/app/installations/") && strings.HasSuffix(r.URL.Path, "/access_tokens"): parts := strings.Split(r.URL.Path, "/") installID := parts[len(parts)-2] _, _ = fmt.Fprintf(w, `{"token":"token-%s","expires_at":"2099-01-01T00:00:00Z"}`, installID) case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/app/installations"): _, _ = w.Write([]byte(`[{"id":1337,"account":{"login":"default-org","type":"Organization"}},{"id":2448,"account":{"login":"other-org","type":"Organization"}}]`)) case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/installation/repositories"): // Fail the first installation's repo listing; succeed for the second. if strings.Contains(r.Header.Get("Authorization"), "token-1337") { http.Error(w, "repo list failed", http.StatusInternalServerError) return } _, _ = w.Write([]byte(`{"total_count":1,"repositories":[{"id":1,"name":"repo-b","full_name":"other-org/repo-b","clone_url":"https://github.com/other-org/repo-b.git","size":1}]}`)) default: http.NotFound(w, r) } })) defer server.Close() s, conn := createTestSource(&sourcespb.GitHub{ Endpoint: server.URL, ScanAllInstallations: true, Credential: &sourcespb.GitHub_GithubApp{ GithubApp: &credentialspb.GitHubApp{ PrivateKey: privateKey, InstallationId: "1337", AppId: "4141", }, }, }) require.NoError(t, s.Init(context.Background(), "test - github", 0, 1337, false, conn, 1)) var reportedErrs []error var reportedUnits []sources.SourceUnit reporter := sources.VisitorReporter{ VisitUnit: func(_ context.Context, unit sources.SourceUnit) error { reportedUnits = append(reportedUnits, unit) return nil }, VisitErr: func(_ context.Context, err error) error { reportedErrs = append(reportedErrs, err) return nil }, } err := s.enumerateAllInstallationRepos(context.Background(), s.connector.(*appConnector), reporter) // A single installation failure is reported but doesn't abort the scan; // the other installation's repos are still enumerated. require.NoError(t, err) require.Len(t, reportedErrs, 1) assert.Contains(t, reportedErrs[0].Error(), "error enumerating repos for installation 1337") require.Len(t, reportedUnits, 1) unitID, _ := reportedUnits[0].SourceUnitID() assert.Equal(t, "https://github.com/other-org/repo-b.git", unitID) } func TestChunksReturnsScanAllInstallationEnumerationErrors(t *testing.T) { privateKey := createPrivateKey() server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") switch { case r.Method == http.MethodPost && strings.Contains(r.URL.Path, "/app/installations/") && strings.HasSuffix(r.URL.Path, "/access_tokens"): parts := strings.Split(r.URL.Path, "/") installID := parts[len(parts)-2] _, _ = fmt.Fprintf(w, `{"token":"token-%s","expires_at":"2099-01-01T00:00:00Z"}`, installID) case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/app/installations"): _, _ = w.Write([]byte(`[{"id":1337,"account":{"login":"default-org","type":"Organization"}},{"id":2448,"account":{"login":"other-org","type":"Organization"}}]`)) case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/installation/repositories"): if strings.Contains(r.Header.Get("Authorization"), "token-1337") { http.Error(w, "repo list failed", http.StatusInternalServerError) return } _, _ = w.Write([]byte(`{"total_count":0,"repositories":[]}`)) default: http.NotFound(w, r) } })) defer server.Close() s, conn := createTestSource(&sourcespb.GitHub{ Endpoint: server.URL, ScanAllInstallations: true, Credential: &sourcespb.GitHub_GithubApp{ GithubApp: &credentialspb.GitHubApp{ PrivateKey: privateKey, InstallationId: "1337", AppId: "4141", }, }, }) require.NoError(t, s.Init(context.Background(), "test - github", 0, 1337, false, conn, 1)) chunks := make(chan *sources.Chunk, 1) err := s.Chunks(context.Background(), chunks) require.Error(t, err) assert.Contains(t, err.Error(), "error enumerating repos for installation 1337") } // This only tests the resume info slice portion of setProgressCompleteWithRepo. func Test_setProgressCompleteWithRepo_resumeInfo(t *testing.T) { tests := []struct { startingResumeInfoSlice []string repoURL string wantResumeInfoSlice []string }{ { startingResumeInfoSlice: []string{}, repoURL: "a", wantResumeInfoSlice: []string{"a"}, }, { startingResumeInfoSlice: []string{"b"}, repoURL: "a", wantResumeInfoSlice: []string{"a", "b"}, }, } s := &Source{ repos: []string{}, } for _, tt := range tests { s.resumeInfoSlice = tt.startingResumeInfoSlice s.setProgressCompleteWithRepo(0, 0, tt.repoURL) if !reflect.DeepEqual(s.resumeInfoSlice, tt.wantResumeInfoSlice) { t.Errorf("s.setProgressCompleteWithRepo() got: %v, want: %v", s.resumeInfoSlice, tt.wantResumeInfoSlice) } } } func Test_setProgressCompleteWithRepo_Progress(t *testing.T) { repos := []string{"a", "b", "c", "d", "e"} tests := map[string]struct { repos []string index int offset int wantPercentComplete int64 wantSectionsCompleted int32 wantSectionsRemaining int32 }{ "starting from the beginning, no offset": { repos: repos, index: 0, offset: 0, wantPercentComplete: 0, wantSectionsCompleted: 0, wantSectionsRemaining: 5, }, "resume from the third, offset 2": { repos: repos[2:], index: 0, offset: 2, wantPercentComplete: 40, wantSectionsCompleted: 2, wantSectionsRemaining: 5, }, "resume from the third, on last repo, offset 2": { repos: repos[2:], index: 2, offset: 2, wantPercentComplete: 80, wantSectionsCompleted: 4, wantSectionsRemaining: 5, }, } for _, tt := range tests { s := &Source{ repos: tt.repos, } s.setProgressCompleteWithRepo(tt.index, tt.offset, "") gotProgress := s.GetProgress() if gotProgress.PercentComplete != tt.wantPercentComplete { t.Errorf("s.setProgressCompleteWithRepo() PercentComplete got: %v want: %v", gotProgress.PercentComplete, tt.wantPercentComplete) } if gotProgress.SectionsCompleted != tt.wantSectionsCompleted { t.Errorf("s.setProgressCompleteWithRepo() PercentComplete got: %v want: %v", gotProgress.SectionsCompleted, tt.wantSectionsCompleted) } if gotProgress.SectionsRemaining != tt.wantSectionsRemaining { t.Errorf("s.setProgressCompleteWithRepo() PercentComplete got: %v want: %v", gotProgress.SectionsRemaining, tt.wantSectionsRemaining) } } } func Test_scan_SetProgressComplete(t *testing.T) { testCases := []struct { name string repos []string wantComplete bool wantErr bool }{ { name: "no repos", wantComplete: true, }, { name: "one valid repo", repos: []string{"https://github.com/super-secret-user/super-secret-repo.git"}, wantComplete: true, wantErr: true, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { src := initTestSource(&sourcespb.GitHub{ Repositories: tc.repos, Credential: &sourcespb.GitHub_Unauthenticated{}, }) src.jobPool = &errgroup.Group{} _ = src.scan(context.Background(), nil) if !tc.wantErr { assert.Equal(t, "", src.GetProgress().EncodedResumeInfo) } gotComplete := src.GetProgress().PercentComplete == 100 if gotComplete != tc.wantComplete { t.Errorf("got: %v, want: %v", gotComplete, tc.wantComplete) } }) } } func TestGetRepoURLParts(t *testing.T) { repoURLs := []string{ "https://github.com/trufflesecurity/trufflehog.git", "git+https://github.com/trufflesecurity/trufflehog.git", "ssh://github.com/trufflesecurity/trufflehog.git", "ssh://git@github.com/trufflesecurity/trufflehog.git", "git+ssh://git@github.com/trufflesecurity/trufflehog.git", "git://github.com/trufflesecurity/trufflehog.git", } expected := []string{"github.com", "trufflesecurity", "trufflehog"} for _, tt := range repoURLs { _, parts, err := getRepoURLParts(tt) if err != nil { t.Fatalf("failed: %v", err) } assert.Equal(t, expected, parts) } gistURLs := map[string][]string{ // Gists "ssh://github.com/6df198861306313246466d23aa4102aa.git": nil, "ssh://gist.github.com/6df198861306313246466d23aa4102aa.git": {"gist.github.com", "6df198861306313246466d23aa4102aa"}, "https://gist.github.com/6df198861306313246466d23aa4102aa.git": {"gist.github.com", "6df198861306313246466d23aa4102aa"}, "https://gist.github.com/john-smith/6df198861306313246466d23aa4102aa.git": {"gist.github.com", "john-smith", "6df198861306313246466d23aa4102aa"}, "ssh://github.contoso.com/gist/6df198861306313246466d23aa4102aa.git": {"github.contoso.com", "gist", "6df198861306313246466d23aa4102aa"}, "https://github.contoso.com/gist/6df198861306313246466d23aa4102aa.git": {"github.contoso.com", "gist", "6df198861306313246466d23aa4102aa"}, "https://github.contoso.com/gist/john-smith/6df198861306313246466d23aa4102aa.git": {"github.contoso.com", "gist", "john-smith", "6df198861306313246466d23aa4102aa"}, "https://github.com/gist/john-smith/6df198861306313246466d23aa4102aa.git": nil, } for tt, expected := range gistURLs { _, parts, err := getRepoURLParts(tt) if err != nil { if expected == nil { continue } t.Fatalf("failed: %v", err) } assert.Equal(t, expected, parts) } } func TestGetRepoURLPartsWithTrailingHyphen(t *testing.T) { // Test for https://github.com/trufflesecurity/trufflehog/issues/4679 // Repository names ending with a hyphen should be preserved correctly. testCases := []struct { name string url string expected []string }{ { name: "https with trailing hyphen", url: "https://github.com/MYORG/my-repo-name-.git", expected: []string{"github.com", "MYORG", "my-repo-name-"}, }, { name: "https with trailing hyphen no .git", url: "https://github.com/MYORG/my-repo-.git", expected: []string{"github.com", "MYORG", "my-repo-"}, }, { name: "ssh with trailing hyphen", url: "ssh://git@github.com/MYORG/test-repo-.git", expected: []string{"github.com", "MYORG", "test-repo-"}, }, { name: "multiple hyphens with trailing", url: "https://github.com/org-name/my-test-repo-.git", expected: []string{"github.com", "org-name", "my-test-repo-"}, }, { name: "single trailing hyphen repo", url: "https://github.com/Org/-.git", expected: []string{"github.com", "Org", "-"}, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { _, parts, err := getRepoURLParts(tc.url) if err != nil { t.Fatalf("unexpected error: %v", err) } assert.Equal(t, tc.expected, parts) }) } } func TestGithubWebHost(t *testing.T) { tests := []struct { name string endpoint string want string }{ { name: "empty endpoint defaults to github", endpoint: "", want: "github.com", }, { name: "cloud api endpoint", endpoint: "https://api.github.com", want: "github.com", }, { name: "cloud api endpoint without scheme", endpoint: "api.github.com", want: "github.com", }, { name: "enterprise endpoint", endpoint: "https://github.example.com/api/v3", want: "github.example.com", }, { name: "enterprise endpoint without scheme", endpoint: "github.example.com/api/v3", want: "github.example.com", }, { name: "host ending with github.com substring is enterprise", endpoint: "https://legithub.com/api/v3", want: "legithub.com", }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { assert.Equal(t, tt.want, githubWebHost(tt.endpoint)) }) } } func TestWikiIsReachableSkipsHeadForPrivateRepos(t *testing.T) { var called bool server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { called = true http.Redirect(w, r, "/login", http.StatusFound) })) defer server.Close() repoURL := "https://github.example.com/owner/repo.git" s := &Source{ conn: &sourcespb.GitHub{Endpoint: server.URL}, repoInfoCache: newRepoInfoCache(), } s.repoInfoCache.put(repoURL, repoInfo{ owner: "owner", name: "repo", visibility: source_metadatapb.Visibility_private, }) assert.True(t, s.wikiIsReachable(context.Background(), repoURL)) assert.False(t, called) } func TestGetGistID(t *testing.T) { tests := []struct { trimmedURL []string expected string }{ {[]string{"https://gist.github.com", "12345"}, "12345"}, {[]string{"https://gist.github.com", "owner", "12345"}, "12345"}, } for _, tt := range tests { got := extractGistID(tt.trimmedURL) assert.Equal(t, tt.expected, got) } } // This isn't really a GitHub test, but GitHub is the only source that supports scan targeting right now, so this is // where I've put this targeted scan test. func Test_ScanMultipleTargets_MultipleErrors(t *testing.T) { s := &Source{conn: &sourcespb.GitHub{}} // This test doesn't require initialization ctx := context.Background() chunksChan := make(chan *sources.Chunk) targets := []sources.ChunkingTarget{ {SecretID: 1}, {SecretID: 2}, } // The specific error text doesn't matter for the test, but it has to match what the source generates want := []*sources.TargetedScanError{ {SecretID: 1, Err: errors.New("unable to cast metadata type for targeted scan")}, {SecretID: 2, Err: errors.New("unable to cast metadata type for targeted scan")}, } err := s.Chunks(ctx, chunksChan, targets...) unwrappable, ok := err.(interface{ Unwrap() []error }) if assert.True(t, ok, "returned error was not unwrappable") { got := unwrappable.Unwrap() assert.ElementsMatch(t, got, want) } } func TestScanAllInstallationsTargetMappingErrorIsTargetedScanError(t *testing.T) { s := &Source{ conn: &sourcespb.GitHub{ScanAllInstallations: true}, connector: &appConnector{ clientsByInstallationID: make(map[int64]*appInstallationClients), repoInstallationMap: make(map[string]int64), }, } ctx := context.Background() chunksChan := make(chan *sources.Chunk) targets := []sources.ChunkingTarget{ { SecretID: 1, QueryCriteria: &source_metadatapb.MetaData{ Data: &source_metadatapb.MetaData_Github{ Github: &source_metadatapb.Github{Link: "https://github.com/only-owner"}, }, }, }, {SecretID: 2}, } err := s.Chunks(ctx, chunksChan, targets...) unwrappable, ok := err.(interface{ Unwrap() []error }) if assert.True(t, ok, "returned error was not unwrappable") { got := unwrappable.Unwrap() require.Len(t, got, 2) for _, targetErr := range got { assert.IsType(t, &sources.TargetedScanError{}, targetErr) } } } func TestScanTargetRejectsWikiLinkHostMismatch(t *testing.T) { s := &Source{ conn: &sourcespb.GitHub{Endpoint: cloudV3Endpoint}, } target := sources.ChunkingTarget{ QueryCriteria: &source_metadatapb.MetaData{ Data: &source_metadatapb.MetaData_Github{ Github: &source_metadatapb.Github{ Link: "https://attacker.example/owner/repo/wiki/path/to/file", Commit: "abc123", File: "path/to/file", }, }, }, } err := s.scanTarget(context.Background(), target, sources.ChanReporter{Ch: make(chan *sources.Chunk, 1)}) require.Error(t, err) assert.Contains(t, err.Error(), "does not match GitHub endpoint host") } func TestRepositoryFiltering(t *testing.T) { // Test that the filteredRepoCache correctly filters repositories source := &Source{} // Test case 1: No filters specified (should include everything) cache1 := source.newFilteredRepoCache(context.Background(), simple.NewCache[string](), []string{}, []string{}) assert.True(t, cache1.wantRepo("org/repo1")) assert.True(t, cache1.wantRepo("org/repo2")) assert.True(t, cache1.wantRepo("org/repo3")) // Test case 2: Include filter specified (should only include matching repos) cache2 := source.newFilteredRepoCache(context.Background(), simple.NewCache[string](), []string{"org/repo1", "org/repo2"}, []string{}) assert.True(t, cache2.wantRepo("org/repo1")) assert.True(t, cache2.wantRepo("org/repo2")) assert.False(t, cache2.wantRepo("org/repo3")) // Test case 3: Exclude filter specified (should exclude matching repos) cache3 := source.newFilteredRepoCache(context.Background(), simple.NewCache[string](), []string{}, []string{"org/repo1"}) assert.False(t, cache3.wantRepo("org/repo1")) assert.True(t, cache3.wantRepo("org/repo2")) assert.True(t, cache3.wantRepo("org/repo3")) // Test case 4: Both include and exclude filters (exclude takes precedence) cache4 := source.newFilteredRepoCache(context.Background(), simple.NewCache[string](), []string{"org/repo1"}, []string{"org/repo1"}) assert.False(t, cache4.wantRepo("org/repo1")) // Test case 5: Wildcard patterns cache5 := source.newFilteredRepoCache(context.Background(), simple.NewCache[string](), []string{"org/*"}, []string{}) assert.True(t, cache5.wantRepo("org/repo1")) assert.True(t, cache5.wantRepo("org/repo2")) assert.False(t, cache5.wantRepo("other/repo1")) } func TestExplicitRepositoryBypass(t *testing.T) { // Test that explicit repositories are included in enumeration ctx := context.Background() // Set up mocks for the API calls gock.New("https://api.github.com"). Get("/user"). Reply(200). JSON(map[string]string{"login": "test-user"}) gock.New("https://api.github.com"). Get("/repos/org/explicit-repo"). Reply(200). JSON(map[string]any{ "full_name": "org/explicit-repo", "clone_url": "https://github.com/org/explicit-repo.git", "size": 1, }) gock.New("https://api.github.com"). Get("/repos/org/another-explicit"). Reply(200). JSON(map[string]any{ "full_name": "org/another-explicit", "clone_url": "https://github.com/org/another-explicit.git", "size": 1, }) // Create a source with explicit repositories source := initTestSource(&sourcespb.GitHub{ Credential: &sourcespb.GitHub_Token{ Token: "super secret token", }, Repositories: []string{ "https://github.com/org/explicit-repo.git", "https://github.com/org/another-explicit.git", }, }) // Test the Enumerate method err := source.Enumerate(ctx, noopReporter()) require.NoError(t, err) // Verify that explicit repositories are included in the enumeration assert.Len(t, source.repos, 2, "Should have 2 explicit repositories") assert.Contains(t, source.repos, "https://github.com/org/explicit-repo.git") assert.Contains(t, source.repos, "https://github.com/org/another-explicit.git") } // TestEnsureRepoInfoCache_RepoRedirect verifies that when the GitHub API returns // a canonical CloneURL that differs from the requested URL (i.e. a repository // has been renamed/moved), ensureRepoInfoCache stores info under BOTH the // original URL and the canonical URL so that scanRepo and visibilityOf can // look up metadata using either. func TestEnsureRepoInfoCache_RepoRedirect(t *testing.T) { defer gock.Off() const ( originalURL = "https://github.com/old-org/myrepo.git" canonicalURL = "https://github.com/new-org/myrepo.git" ) gock.New("https://api.github.com"). Get("/repos/old-org/myrepo"). Reply(200). JSON(map[string]any{ "name": "myrepo", "full_name": "new-org/myrepo", "clone_url": canonicalURL, "private": false, "has_wiki": false, "size": 100, "owner": map[string]string{"login": "new-org"}, }) s := initTestSource(&sourcespb.GitHub{ Credential: &sourcespb.GitHub_Token{Token: "token"}, }) returnedURL, err := s.ensureRepoInfoCache(context.Background(), originalURL, &unitErrorReporter{noopReporter()}) require.NoError(t, err) // Original URL returned so callers can continue using it. assert.Equal(t, originalURL, returnedURL) // Both URLs must be in the cache so scanRepo / visibilityOf don't miss. _, okOriginal := s.repoInfoCache.get(originalURL) assert.True(t, okOriginal, "original (pre-redirect) URL must be cached") _, okCanonical := s.repoInfoCache.get(canonicalURL) assert.True(t, okCanonical, "canonical (post-redirect) URL must be cached") assert.False(t, gock.HasUnmatchedRequest()) assert.True(t, gock.IsDone()) } // TestEnumerate_ExplicitRepoRedirect verifies the end-to-end Enumerate flow // when a user provides an explicit repo URL that GitHub redirects to a new // location (e.g. org rename). scanRepo must be able to look up cache metadata // using the original URL that ends up in s.repos. func TestEnumerate_ExplicitRepoRedirect(t *testing.T) { defer gock.Off() const ( originalURL = "https://github.com/old-org/myrepo.git" canonicalURL = "https://github.com/new-org/myrepo.git" ) gock.New("https://api.github.com"). Get("/user"). Reply(200). JSON(map[string]string{"login": "test-user"}) gock.New("https://api.github.com"). Get("/repos/old-org/myrepo"). Reply(200). JSON(map[string]any{ "name": "myrepo", "full_name": "new-org/myrepo", "clone_url": canonicalURL, "private": true, "has_wiki": false, "size": 50, "owner": map[string]string{"login": "new-org"}, }) s := initTestSource(&sourcespb.GitHub{ Credential: &sourcespb.GitHub_Token{Token: "token"}, Repositories: []string{originalURL}, }) err := s.Enumerate(context.Background(), noopReporter()) require.NoError(t, err) // s.repos must use the original URL (what was configured). assert.Contains(t, s.repos, originalURL) // scanRepo looks up repoInfoCache by the URL in s.repos. // Without the redirect fix this lookup fails with "no repoInfo for URL". info, ok := s.repoInfoCache.get(originalURL) assert.True(t, ok, "scanRepo must be able to look up metadata by original URL") assert.Equal(t, "new-org", info.owner) assert.False(t, gock.HasUnmatchedRequest()) assert.True(t, gock.IsDone()) } func noopReporter() sources.UnitReporter { return sources.VisitorReporter{ VisitUnit: func(context.Context, sources.SourceUnit) error { return nil }, } } // This tests reproduces a bug where both VisitUnit and VisitErr were called // for the same repository when caching the repository info failed. func TestFixBothUnitErrAndUnitOKCalled(t *testing.T) { cache := simple.NewCache[string]() cache.Set("myorg/myrepo", "an invalid url that will cause an error") s := &Source{ filteredRepoCache: &filteredRepoCache{ Cache: cache, }, conn: &sourcespb.GitHub{ Repositories: []string{"myorg/myrepo"}, }, orgsCache: simple.NewCache[string](), } var okCalled, errCalled bool reporter := sources.VisitorReporter{ VisitUnit: func(ctx context.Context, su sources.SourceUnit) error { okCalled = true return nil }, VisitErr: func(ctx context.Context, err error) error { errCalled = true return nil }, } err := s.Enumerate(context.Background(), reporter) require.NoError(t, err) // expectation is that only VisitErr is called assert.True(t, errCalled) assert.False(t, okCalled) } func TestExtractRepoNameFromURL(t *testing.T) { tests := []struct { name string url string expected string }{ { name: "git URL", url: "https://github.com/org/repo.git", expected: "org/repo", }, { name: "git URL with trailing slash", url: "https://github.com/org/repo.git/", expected: "org/repo", }, { name: "git URL without .git", url: "https://github.com/org/repo", expected: "org/repo", }, { name: "git enterprise URL", url: "https://example-enterprise.com/org/repo.git", expected: "org/repo", }, { name: "just org/repo", url: "org/repo", expected: "org/repo", }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { result := extractRepoNameFromUrl(tt.url) assert.Equal(t, tt.expected, result) }) } } func TestSource_ExcludeArchivedRepositories(t *testing.T) { tests := []struct { name string excludeArchived bool reposJSON string wantRepoCount int wantRepos []string }{ { name: "exclude archived repos when flag is true", excludeArchived: true, reposJSON: `[ {"full_name": "test-org/active-repo", "clone_url": "https://github.com/test-org/active-repo.git", "size": 1, "archived": false}, {"full_name": "test-org/archived-repo", "clone_url": "https://github.com/test-org/archived-repo.git", "size": 1, "archived": true}, {"full_name": "test-org/another-active", "clone_url": "https://github.com/test-org/another-active.git", "size": 1, "archived": false} ]`, wantRepoCount: 2, // Only non-archived wantRepos: []string{"test-org/active-repo", "test-org/another-active"}, }, { name: "include archived repos when flag is false", excludeArchived: false, reposJSON: `[ {"full_name": "test-org/active-repo", "clone_url": "https://github.com/test-org/active-repo.git", "size": 1, "archived": false}, {"full_name": "test-org/archived-repo", "clone_url": "https://github.com/test-org/archived-repo.git", "size": 1, "archived": true}, {"full_name": "test-org/another-active", "clone_url": "https://github.com/test-org/another-active.git", "size": 1, "archived": false} ]`, wantRepoCount: 3, // All repos wantRepos: []string{"test-org/active-repo", "test-org/archived-repo", "test-org/another-active"}, }, { name: "handle all archived repos", excludeArchived: true, reposJSON: `[ {"full_name": "test-org/archived-1", "clone_url": "https://github.com/test-org/archived-1.git", "size": 1, "archived": true}, {"full_name": "test-org/archived-2", "clone_url": "https://github.com/test-org/archived-2.git", "size": 1, "archived": true} ]`, wantRepoCount: 0, // None included wantRepos: []string{}, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { defer gock.Off() gock.New("https://api.github.com"). Get("/orgs/test-org/repos"). Reply(200). JSON(tt.reposJSON) s := initTestSource(&sourcespb.GitHub{ Credential: &sourcespb.GitHub_Token{ Token: "test-token", }, ExcludeArchived: tt.excludeArchived, }) err := s.getReposByOrg(context.Background(), "test-org", noopReporter()) assert.Nil(t, err) assert.Equal(t, tt.wantRepoCount, s.filteredRepoCache.Count()) // Verify expected repos are in cache for _, repo := range tt.wantRepos { ok := s.filteredRepoCache.Exists(repo) assert.True(t, ok, "expected repo %s to be in cache", repo) } // Verify archived repos are NOT in cache when excluding if tt.excludeArchived { allRepos := []string{"test-org/archived-repo", "test-org/archived-1", "test-org/archived-2"} for _, repo := range allRepos { if !slices.Contains(tt.wantRepos, repo) { ok := s.filteredRepoCache.Exists(repo) assert.False(t, ok, "archived repo %s should not be in cache when excluding", repo) } } } assert.False(t, gock.HasUnmatchedRequest()) assert.True(t, gock.IsDone()) }) } } func TestSource_ExcludeArchivedForkInteraction(t *testing.T) { tests := []struct { name string includeForks bool excludeArchived bool reposJSON string wantRepoCount int wantRepos []string notWantRepos []string }{ { name: "fork+archived skipped when forks included and archived excluded", includeForks: true, excludeArchived: true, reposJSON: `[ {"full_name": "test-org/active", "clone_url": "https://github.com/test-org/active.git", "size": 1, "archived": false, "fork": false}, {"full_name": "test-org/fork-active", "clone_url": "https://github.com/test-org/fork-active.git", "size": 1, "archived": false, "fork": true}, {"full_name": "test-org/fork-archived", "clone_url": "https://github.com/test-org/fork-archived.git", "size": 1, "archived": true, "fork": true} ]`, wantRepoCount: 2, wantRepos: []string{"test-org/active", "test-org/fork-active"}, notWantRepos: []string{"test-org/fork-archived"}, }, { name: "fork+archived included when archived allowed", includeForks: true, excludeArchived: false, reposJSON: `[ {"full_name": "test-org/active", "clone_url": "https://github.com/test-org/active.git", "size": 1, "archived": false, "fork": false}, {"full_name": "test-org/fork-archived", "clone_url": "https://github.com/test-org/fork-archived.git", "size": 1, "archived": true, "fork": true} ]`, wantRepoCount: 2, wantRepos: []string{"test-org/active", "test-org/fork-archived"}, }, { name: "fork+archived dropped by fork filter when forks excluded", includeForks: false, excludeArchived: true, reposJSON: `[ {"full_name": "test-org/active", "clone_url": "https://github.com/test-org/active.git", "size": 1, "archived": false, "fork": false}, {"full_name": "test-org/fork-archived", "clone_url": "https://github.com/test-org/fork-archived.git", "size": 1, "archived": true, "fork": true} ]`, wantRepoCount: 1, wantRepos: []string{"test-org/active"}, notWantRepos: []string{"test-org/fork-archived"}, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { defer gock.Off() gock.New("https://api.github.com"). Get("/orgs/test-org/repos"). Reply(200). JSON(tt.reposJSON) s := initTestSource(&sourcespb.GitHub{ Credential: &sourcespb.GitHub_Token{ Token: "test-token", }, IncludeForks: tt.includeForks, ExcludeArchived: tt.excludeArchived, }) err := s.getReposByOrg(context.Background(), "test-org", noopReporter()) assert.Nil(t, err) assert.Equal(t, tt.wantRepoCount, s.filteredRepoCache.Count()) for _, repo := range tt.wantRepos { assert.True(t, s.filteredRepoCache.Exists(repo), "expected repo %s to be in cache", repo) } for _, repo := range tt.notWantRepos { assert.False(t, s.filteredRepoCache.Exists(repo), "repo %s should not be in cache", repo) } assert.False(t, gock.HasUnmatchedRequest()) assert.True(t, gock.IsDone()) }) } } func TestIsGHECloud(t *testing.T) { tests := []struct { name string endpoint string want bool }{ // GHE.com endpoints — should return true {"api subdomain", "https://api.company.ghe.com", true}, {"web subdomain", "https://company.ghe.com", true}, {"api subdomain trailing slash", "https://api.company.ghe.com/", true}, {"web subdomain trailing slash", "https://company.ghe.com/", true}, {"uppercase", "https://API.company.GHE.COM", true}, {"mixed case", "https://Api.company.Ghe.Com/", true}, {"deep subdomain", "https://api.team.company.ghe.com", true}, // GHES endpoints — should return false {"ghes custom domain", "https://github.mycompany.com", false}, {"ghes with path", "https://github.mycompany.com/api/v3", false}, {"ghes internal", "https://github.internal.corp.com", false}, // github.com — should return false {"cloud api", "https://api.github.com", false}, {"cloud web", "https://github.com", false}, // Edge cases — should return false {"empty string", "", false}, {"localhost", "http://localhost:8080", false}, {"malicious suffix", "https://notghe.com", false}, {"partial match", "https://api.fakeghe.com.evil.com", false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { got := isGHECloud(tt.endpoint) if got != tt.want { t.Errorf("isGHECloud(%q) = %v, want %v", tt.endpoint, got, tt.want) } }) } } func TestNormalizeGHECloudAPIEndpoint(t *testing.T) { tests := []struct { name string input string want string wantErr bool }{ { name: "web URL to API URL", input: "https://company.ghe.com", want: "https://api.company.ghe.com/", }, { name: "web URL with trailing slash", input: "https://company.ghe.com/", want: "https://api.company.ghe.com/", }, { name: "api URL already correct", input: "https://api.company.ghe.com", want: "https://api.company.ghe.com/", }, { name: "api URL with trailing slash", input: "https://api.company.ghe.com/", want: "https://api.company.ghe.com/", }, { name: "preserves port", input: "http://company.ghe.com:8080", want: "http://api.company.ghe.com:8080/", }, { name: "api with port", input: "http://api.company.ghe.com:8080", want: "http://api.company.ghe.com:8080/", }, { name: "strips path", input: "https://company.ghe.com/some/path", want: "https://api.company.ghe.com/", }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { got, err := normalizeGHECloudAPIEndpoint(tt.input) if (err != nil) != tt.wantErr { t.Errorf("normalizeGHECloudAPIEndpoint(%q) error = %v, wantErr %v", tt.input, err, tt.wantErr) return } if got != tt.want { t.Errorf("normalizeGHECloudAPIEndpoint(%q) = %q, want %q", tt.input, got, tt.want) } }) } } func TestCreateAPIClient_GHECloud_NoAPIV3(t *testing.T) { // The most critical test: GHE.com clients must NOT have /api/v3/ in their BaseURL. endpoints := []string{ "https://api.company.ghe.com", "https://company.ghe.com", "https://api.company.ghe.com/", "https://company.ghe.com/", } for _, endpoint := range endpoints { t.Run(endpoint, func(t *testing.T) { client, err := createGHECloudClient(http.DefaultClient, endpoint) if err != nil { t.Fatalf("createGHECloudClient(%q) unexpected error: %v", endpoint, err) } baseURL := client.BaseURL.String() // Must NOT contain /api/v3 if strings.Contains(baseURL, "/api/v3") { t.Errorf("GHE.com client BaseURL must not contain /api/v3/, got: %s", baseURL) } // Must point to api.company.ghe.com if client.BaseURL.Hostname() != "api.company.ghe.com" { t.Errorf("expected hostname api.company.ghe.com, got: %s", client.BaseURL.Hostname()) } // Must have trailing slash if !strings.HasSuffix(baseURL, "/") { t.Errorf("BaseURL must have trailing slash, got: %s", baseURL) } }) } } func TestCreateGraphqlClient_GHECloud(t *testing.T) { // GHE.com GraphQL should be at /graphql, NOT /api/graphql. // We can't easily inspect the URL from the githubv4.Client, // so we test the URL construction logic indirectly by checking // the normalized endpoint. apiURL, err := normalizeGHECloudAPIEndpoint("https://company.ghe.com") if err != nil { t.Fatal(err) } graphqlURL := strings.TrimRight(apiURL, "/") + "/graphql" expected := "https://api.company.ghe.com/graphql" if graphqlURL != expected { t.Errorf("GHE.com GraphQL URL = %q, want %q", graphqlURL, expected) } } func TestCreateAPIClient_GHES_HasAPIV3(t *testing.T) { // GHES clients SHOULD have /api/v3/ — make sure we didn't break that. // Note: WithEnterpriseURLs returns an error for invalid URLs, but for // valid ones it appends /api/v3/. client, err := createAPIClient(context.Background(), http.DefaultClient, "https://github.mycompany.com") if err != nil { t.Fatalf("unexpected error: %v", err) } if !strings.Contains(client.BaseURL.String(), "/api/v3") { t.Errorf("GHES client BaseURL should contain /api/v3/, got: %s", client.BaseURL.String()) } } func TestCreateAPIClient_CloudGitHub(t *testing.T) { // Regular github.com should use the default client (api.github.com). client, err := createAPIClient(context.Background(), http.DefaultClient, cloudV3Endpoint) if err != nil { t.Fatalf("unexpected error: %v", err) } if client.BaseURL.String() != "https://api.github.com/" { t.Errorf("expected https://api.github.com/, got: %s", client.BaseURL.String()) } }