trufflesecurity/trufflehog
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
Files
sql_server_extra_data_components
trufflehog/pkg/sources/github/github_test.go
T
Kashif Khan 166bf18f3a Make GitHub App installationId optional when scanning all installations (#5275) 
When scanAllInstallations is true, the scanner no longer requires
githubApp.installationId in the config. Repos, gists, and org
members outside any installation's listing now fail with a clear
error instead of the app hitting a nil client or a bad request.
2026-09-07 09:32:43 +05:00

2999 lines
105 KiB
Go
Raw Permalink Blame History

package github
import (
"bytes"
"crypto/rand"
"crypto/rsa"
"crypto/x509"
"encoding/pem"
"errors"
"fmt"
"net/http"
"net/http/httptest"
"net/url"
"reflect"
"slices"
"strconv"
"strings"
"testing"
"time"
"github.com/google/go-cmp/cmp"
"github.com/google/go-github/v67/github"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"golang.org/x/sync/errgroup"
"google.golang.org/protobuf/types/known/anypb"
"gopkg.in/h2non/gock.v1"
"github.com/trufflesecurity/trufflehog/v3/pkg/cache/simple"
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/credentialspb"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
)
func createPrivateKey() string {
key, err := rsa.GenerateKey(rand.Reader, 2048)
if err != nil {
panic(err)
}
data := x509.MarshalPKCS1PrivateKey(key)
var pemKey bytes.Buffer
if err := pem.Encode(&pemKey, &pem.Block{
Type: "RSA PRIVATE KEY",
Bytes: data,
}); err != nil {
panic(err)
}
return pemKey.String()
}
func createTestSource(src *sourcespb.GitHub) (*Source, *anypb.Any) {
s := &Source{}
conn, err := anypb.New(src)
if err != nil {
panic(err)
}
return s, conn
}
func initTestSource(src *sourcespb.GitHub) *Source {
s, conn := createTestSource(src)
if err := s.Init(context.Background(), "test - github", 0, 1337, false, conn, 1); err != nil {
panic(err)
}
gock.InterceptClient(s.connector.APIClient().Client())
if appConnector, ok := s.connector.(*appConnector); ok {
gock.InterceptClient(appConnector.InstallationClient().Client())
}
return s
}
func TestInit(t *testing.T) {
source, conn := createTestSource(&sourcespb.GitHub{
Repositories: []string{"https://github.com/dustin-decker/secretsandstuff.git"},
Credential: &sourcespb.GitHub_Token{
Token: "super secret token",
},
})
err := source.Init(context.Background(), "test - github", 0, 1337, false, conn, 1)
assert.Nil(t, err)
// TODO: test error case
}
func TestAddReposByOrg(t *testing.T) {
defer gock.Off()
gock.New("https://api.github.com").
Get("/orgs/super-secret-org/repos").
Reply(200).
JSON([]map[string]string{
{"clone_url": "https://github.com/super-secret-repo.git", "full_name": "super-secret-repo"},
{"clone_url": "https://github.com/super-secret-repo2.git", "full_name": "secret/super-secret-repo2"},
})
s := initTestSource(&sourcespb.GitHub{
Credential: &sourcespb.GitHub_Token{
Token: "super secret token",
},
Repositories: nil,
IgnoreRepos: []string{"secret/super-*-repo2"},
CommentsTimeframeDays: 10,
})
err := s.getReposByOrg(context.Background(), "super-secret-org", noopReporter())
assert.Nil(t, err)
assert.Equal(t, 1, s.filteredRepoCache.Count())
ok := s.filteredRepoCache.Exists("super-secret-repo")
assert.True(t, ok)
assert.False(t, gock.HasUnmatchedRequest())
assert.True(t, gock.IsDone())
}
func TestAddReposByOrg_Repositories(t *testing.T) {
defer gock.Off()
gock.New("https://api.github.com").
Get("/orgs/super-secret-org/repos").
Reply(200).
JSON(`[
{"full_name": "super-secret-org/super-secret-repo", "clone_url": "https://github.com/super-secret-org/super-secret-repo.git", "size": 1},
{"full_name": "super-secret-org/super-secret-repo2", "clone_url": "https://github.com/super-secret-org/super-secret-repo2.git", "size": 1},
{"full_name": "super-secret-org/not-super-secret-repo", "clone_url": "https://github.com/super-secret-org/not-super-secret-repo.git", "size": 1}
]`)
s := initTestSource(&sourcespb.GitHub{
Credential: &sourcespb.GitHub_Token{
Token: "super secret token",
},
Repositories: []string{"super-secret-org/super-secret-repo", "super-secret-org/super-secret-repo2"},
Organizations: []string{"super-secret-org"},
})
err := s.getReposByOrg(context.Background(), "super-secret-org", noopReporter())
assert.Nil(t, err)
assert.Equal(t, 2, s.filteredRepoCache.Count())
ok := s.filteredRepoCache.Exists("super-secret-org/super-secret-repo")
assert.True(t, ok)
ok = s.filteredRepoCache.Exists("super-secret-org/super-secret-repo2")
assert.True(t, ok)
assert.False(t, gock.HasUnmatchedRequest())
assert.True(t, gock.IsDone())
}
func TestAddReposByOrg_IncludeRepos(t *testing.T) {
defer gock.Off()
gock.New("https://api.github.com").
Get("/orgs/super-secret-org/repos").
Reply(200).
JSON(`[
{"full_name": "super-secret-org/super-secret-repo", "clone_url": "https://github.com/super-secret-org/super-secret-repo.git", "size": 1},
{"full_name": "super-secret-org/super-secret-repo2", "clone_url": "https://github.com/super-secret-org/super-secret-repo2.git", "size": 1},
{"full_name": "super-secret-org/not-super-secret-repo", "clone_url": "https://github.com/super-secret-org/not-super-secret-repo.git", "size": 1}
]`)
s := initTestSource(&sourcespb.GitHub{
Credential: &sourcespb.GitHub_Token{
Token: "super secret token",
},
IncludeRepos: []string{"super-secret-org/super*"},
Organizations: []string{"super-secret-org"},
})
err := s.getReposByOrg(context.Background(), "super-secret-org", noopReporter())
assert.Nil(t, err)
assert.Equal(t, 2, s.filteredRepoCache.Count())
ok := s.filteredRepoCache.Exists("super-secret-org/super-secret-repo")
assert.True(t, ok)
ok = s.filteredRepoCache.Exists("super-secret-org/super-secret-repo2")
assert.True(t, ok)
assert.False(t, gock.HasUnmatchedRequest())
assert.True(t, gock.IsDone())
}
func TestAddReposByUser(t *testing.T) {
defer gock.Off()
gock.New("https://api.github.com").
Get("/users/super-secret-user/repos").
Reply(200).
JSON([]map[string]string{
{"full_name": "super-secret-user/super-secret-repo", "clone_url": "https://github.com/super-secret-user/super-secret-repo.git"},
{"full_name": "super-secret-user/super-secret-repo2", "clone_url": "https://github.com/super-secret-user/super-secret-repo2.git"},
})
s := initTestSource(&sourcespb.GitHub{
Credential: &sourcespb.GitHub_Token{
Token: "super secret token",
},
IgnoreRepos: []string{"super-secret-user/super-secret-repo2"},
})
err := s.getReposByUser(context.Background(), "super-secret-user", false, noopReporter())
assert.Nil(t, err)
assert.Equal(t, 1, s.filteredRepoCache.Count())
ok := s.filteredRepoCache.Exists("super-secret-user/super-secret-repo")
assert.True(t, ok)
assert.False(t, gock.HasUnmatchedRequest())
assert.True(t, gock.IsDone())
}
func TestAddGistsByUser(t *testing.T) {
defer gock.Off()
gock.New("https://api.github.com").
Get("/users/super-secret-user/gists").
Reply(200).
JSON([]map[string]string{{"id": "aa5a315d61ae9438b18d", "git_pull_url": "https://gist.github.com/aa5a315d61ae9438b18d.git"}})
s := initTestSource(&sourcespb.GitHub{Credential: &sourcespb.GitHub_Unauthenticated{}})
err := s.addUserGistsToCache(context.Background(), "super-secret-user", noopReporter())
assert.Nil(t, err)
assert.Equal(t, 1, s.filteredRepoCache.Count())
ok := s.filteredRepoCache.Exists("aa5a315d61ae9438b18d")
assert.True(t, ok)
assert.False(t, gock.HasUnmatchedRequest())
assert.True(t, gock.IsDone())
}
func TestIgnoreGistsByUser(t *testing.T) {
defer gock.Off()
gock.New("https://api.github.com").
Get("/users/super-secret-user/gists").
Reply(200).
JSON([]map[string]string{{"id": "aa5a315d61ae9438b18d", "git_pull_url": "https://gist.github.com/aa5a315d61ae9438b18d.git"}})
s := initTestSource(&sourcespb.GitHub{Credential: &sourcespb.GitHub_Unauthenticated{}})
s.ignoreGists = true
err := s.addUserGistsToCache(context.Background(), "super-secret-user", noopReporter())
assert.Nil(t, err)
assert.Equal(t, 0, s.filteredRepoCache.Count())
ok := s.filteredRepoCache.Exists("aa5a315d61ae9438b18d")
assert.False(t, ok)
assert.False(t, gock.HasUnmatchedRequest())
assert.False(t, gock.IsDone())
}
func TestAddMembersByOrg(t *testing.T) {
defer gock.Off()
gock.New("https://api.github.com").
Get("/orgs/org1/members").
Reply(200).
JSON([]map[string]string{
{"login": "testman1"},
{"login": "testman2"},
})
s := initTestSource(&sourcespb.GitHub{Credential: &sourcespb.GitHub_Unauthenticated{}})
err := s.addMembersByOrg(context.Background(), "org1", noopReporter())
assert.Nil(t, err)
assert.Equal(t, 2, len(s.memberCache))
_, ok := s.memberCache["testman1"]
assert.True(t, ok)
_, ok = s.memberCache["testman2"]
assert.True(t, ok)
assert.False(t, gock.HasUnmatchedRequest())
assert.True(t, gock.IsDone())
}
func TestAddMembersByOrg_AuthFailure(t *testing.T) {
defer gock.Off()
gock.New("https://api.github.com").
Get("/orgs/org1/members").
Reply(401).
JSON([]map[string]string{{
"message": "Bad credentials",
"documentation_url": "https://docs.github.com/rest",
"status": "401",
}})
s := initTestSource(&sourcespb.GitHub{Credential: &sourcespb.GitHub_Unauthenticated{}})
err := s.addMembersByOrg(context.Background(), "org1", noopReporter())
assert.True(t, strings.HasPrefix(err.Error(), "could not list organization"))
assert.False(t, gock.HasUnmatchedRequest())
assert.True(t, gock.IsDone())
}
func TestAddMembersByOrg_NoMembers(t *testing.T) {
defer gock.Off()
gock.New("https://api.github.com").
Get("/orgs/org1/members").
Reply(200).
JSON([]map[string]string{})
s := initTestSource(&sourcespb.GitHub{Credential: &sourcespb.GitHub_Unauthenticated{}})
err := s.addMembersByOrg(context.Background(), "org1", noopReporter())
assert.Equal(t, fmt.Sprintf("organization (%q) had 0 members: account may not have access to list organization members", "org1"), err.Error())
assert.False(t, gock.HasUnmatchedRequest())
assert.True(t, gock.IsDone())
}
func TestAddMembersByApp(t *testing.T) {
defer gock.Off()
privateKey := createPrivateKey()
gock.New("https://api.github.com").
Get("/app/installations").
Reply(200).
JSON([]map[string]any{
{"id": 1337, "account": map[string]string{"login": "super-secret-org", "type": "Organization"}},
})
gock.New("https://api.github.com").
Post("/app/installations/1337/access_tokens").
Reply(200).
JSON(map[string]string{"token": "dontlook"})
gock.New("https://api.github.com").
Get("/orgs/super-secret-org/members").
Reply(200).
JSON([]map[string]any{
{"login": "ssm1"},
{"login": "ssm2"},
{"login": "ssm3"},
})
s := initTestSource(&sourcespb.GitHub{
Endpoint: "https://api.github.com",
Credential: &sourcespb.GitHub_GithubApp{
GithubApp: &credentialspb.GitHubApp{
PrivateKey: privateKey,
InstallationId: "1337",
AppId: "4141",
},
}})
err := s.addMembersByApp(context.Background(), s.connector.(*appConnector), noopReporter())
assert.Nil(t, err)
assert.Equal(t, 3, len(s.memberCache))
_, ok := s.memberCache["ssm1"]
assert.True(t, ok)
_, ok = s.memberCache["ssm2"]
assert.True(t, ok)
_, ok = s.memberCache["ssm3"]
assert.True(t, ok)
assert.False(t, gock.HasUnmatchedRequest())
assert.True(t, gock.IsDone())
}
func TestAddReposByApp(t *testing.T) {
defer gock.Off()
gock.New("https://api.github.com").
Get("/installation/repositories").
Reply(200).
JSON(map[string]any{
"repositories": []map[string]string{
{"clone_url": "https://github/ssr1.git", "full_name": "ssr1"},
{"clone_url": "https://github/ssr2.git", "full_name": "ssr2"},
},
})
s := initTestSource(&sourcespb.GitHub{Credential: &sourcespb.GitHub_Unauthenticated{}})
err := s.getReposByApp(context.Background(), noopReporter())
assert.Nil(t, err)
assert.Equal(t, 2, s.filteredRepoCache.Count())
ok := s.filteredRepoCache.Exists("ssr1")
assert.True(t, ok)
ok = s.filteredRepoCache.Exists("ssr2")
assert.True(t, ok)
assert.False(t, gock.HasUnmatchedRequest())
assert.True(t, gock.IsDone())
}
// TestAppConnector_EnterpriseBaseURL verifies that GitHub App authentication
// correctly sets the BaseURL on the internal AppsTransport for token refresh.
// This is critical for GitHub Enterprise Server and GHEC with Data Residency.
// Without this fix, token refresh requests would go to api.github.com instead
// of the configured enterprise endpoint.
func TestAppConnector_EnterpriseBaseURL(t *testing.T) {
privateKey := createPrivateKey()
enterpriseEndpoint := "https://api.example.ghe.com"
connector, err := NewAppConnector(
context.Background(),
enterpriseEndpoint,
&credentialspb.GitHubApp{
PrivateKey: privateKey,
InstallationId: "1337",
AppId: "4141",
},
false)
require.NoError(t, err)
appConn, ok := connector.(*appConnector)
require.True(t, ok, "connector should be an appConnector")
// Get the HTTP client's transport, which should be a ghinstallation.Transport
transport := appConn.APIClient().Client().Transport
// Use reflection to access the Transport's BaseURL field
transportVal := reflect.ValueOf(transport).Elem()
baseURLField := transportVal.FieldByName("BaseURL")
require.True(t, baseURLField.IsValid(), "Transport should have a BaseURL field")
assert.Equal(t, enterpriseEndpoint, baseURLField.String(),
"Transport.BaseURL should be set to enterprise endpoint")
// Use reflection to access the internal appsTransport and verify its BaseURL
// This is the critical check - the internal AppsTransport is used for token refresh
appsTransportField := transportVal.FieldByName("appsTransport")
require.True(t, appsTransportField.IsValid(), "Transport should have an appsTransport field")
// Access the BaseURL of the internal AppsTransport
appsTransportVal := appsTransportField.Elem()
appsBaseURLField := appsTransportVal.FieldByName("BaseURL")
require.True(t, appsBaseURLField.IsValid(), "AppsTransport should have a BaseURL field")
assert.Equal(t, enterpriseEndpoint, appsBaseURLField.String(),
"AppsTransport.BaseURL should be set to enterprise endpoint")
}
// TestValidate_ScanAllInstallationsWithoutDefaultInstallation covers the
// scanAllInstallations-without-githubApp.installationId case: there is no
// default installation token to call RateLimit with (see
// appConnector.HasDefaultInstallation), so Validate must fall back to
// checking App credentials via the app-level (JWT) client instead of
// erroring against a non-existent installation ID 0.
func TestValidate_ScanAllInstallationsWithoutDefaultInstallation(t *testing.T) {
defer gock.Off()
gock.New("https://api.github.com").
Get("/app/installations").
Reply(200).
JSON([]map[string]any{})
gock.New("https://api.github.com").
Get("^/app$").
Reply(200).
JSON(map[string]any{"id": 4141})
s := initTestSource(&sourcespb.GitHub{
Endpoint: "https://api.github.com",
ScanAllInstallations: true,
Credential: &sourcespb.GitHub_GithubApp{
GithubApp: &credentialspb.GitHubApp{
PrivateKey: createPrivateKey(),
AppId: "4141",
},
}})
errs := s.Validate(context.Background())
assert.Empty(t, errs)
assert.False(t, gock.HasUnmatchedRequest())
assert.True(t, gock.IsDone())
}
func TestAddOrgsByUser(t *testing.T) {
defer gock.Off()
// NOTE: addOrgsByUser calls /user/orgs to get the orgs of the
// authenticated user
gock.New("https://api.github.com").
Get("/user/orgs").
Reply(200).
JSON([]map[string]any{
{"login": "sso2"},
})
s := initTestSource(&sourcespb.GitHub{Credential: &sourcespb.GitHub_Unauthenticated{}})
s.addOrgsByUser(context.Background(), "super-secret-user", noopReporter())
assert.Equal(t, 1, s.orgsCache.Count())
ok := s.orgsCache.Exists("sso2")
assert.True(t, ok)
assert.False(t, gock.HasUnmatchedRequest())
assert.True(t, gock.IsDone())
}
func TestNormalizeRepos(t *testing.T) {
defer gock.Off()
tests := []struct {
name string
setup func()
repos []string
expected map[string]struct{}
wantErr bool
}{
{
name: "repo url",
setup: func() {},
repos: []string{"https://github.com/super-secret-user/super-secret-repo"},
expected: map[string]struct{}{
"https://github.com/super-secret-user/super-secret-repo.git": {},
},
},
{
name: "not found",
setup: func() {
gock.New("https://api.github.com").
Get("/users/not-found/gists").
Reply(404)
gock.New("https://api.github.com").
Get("/users/not-found/repos").
Reply(404)
},
repos: []string{"not-found"},
expected: map[string]struct{}{},
wantErr: true,
},
{
name: "unexpected format",
setup: func() {},
repos: []string{"/foo/"},
expected: map[string]struct{}{},
wantErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
defer gock.Off()
tt.setup()
s := initTestSource(&sourcespb.GitHub{Credential: &sourcespb.GitHub_Unauthenticated{}})
got, err := s.normalizeRepo(tt.repos[0])
if (err != nil) != tt.wantErr {
t.Errorf("normalizeRepo() error = %v, wantErr %v", err, tt.wantErr)
return
}
if got != "" {
for k := range tt.expected {
assert.Equal(t, got, k)
}
}
res := make(map[string]struct{}, s.filteredRepoCache.Count())
for _, v := range s.filteredRepoCache.Keys() {
res[v] = struct{}{}
}
if got == "" && !cmp.Equal(res, tt.expected) {
t.Errorf("normalizeRepo() got = %v, want %v", s.repos, tt.expected)
}
})
}
}
func TestNormalizeRepo(t *testing.T) {
// Test that normalizeRepo correctly identifies URLs with protocols
source := &Source{}
// Test case 1: HTTP URL
result, err := source.normalizeRepo("https://github.com/org/repo.git")
assert.NoError(t, err)
assert.Contains(t, result, "github.com/org/repo")
// Test case 2: HTTP URL without .git
result, err = source.normalizeRepo("http://github.com/org/repo")
assert.NoError(t, err)
assert.Contains(t, result, "github.com/org/repo")
// Test case 3: Git protocol URL
result, err = source.normalizeRepo("git://github.com/org/repo.git")
assert.NoError(t, err)
assert.Contains(t, result, "github.com/org/repo")
// Test case 4: SSH URL
result, err = source.normalizeRepo("ssh://[email protected]/org/repo.git")
assert.NoError(t, err)
assert.Contains(t, result, "github.com/org/repo")
// Test case 5: SCP-style SSH URL
result, err = source.normalizeRepo("[email protected]:org/repo.git")
assert.NoError(t, err)
assert.Equal(t, "https://github.com/org/repo.git", result)
// Test case 6: Org/repo format (should convert to full URL)
result, err = source.normalizeRepo("org/repo")
assert.NoError(t, err)
assert.Contains(t, result, "github.com/org/repo")
// Test case 7: Invalid format (no protocol, no slash)
_, err = source.normalizeRepo("invalid")
assert.Error(t, err)
assert.Contains(t, err.Error(), "no repositories found")
}
func TestNormalizeRepo_Enterprise(t *testing.T) {
tests := []struct {
name string
endpoint string
wantResult string
}{
{
name: "only host",
endpoint: "https://example.com",
wantResult: "https://example.com/org/repo.git",
},
{
name: "host with path",
endpoint: "https://example.com/api/v3",
wantResult: "https://example.com/org/repo.git",
},
{
name: "host with path without scheme",
endpoint: "example.com/api/v3",
wantResult: "https://example.com/org/repo.git",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
source := Source{
conn: &sourcespb.GitHub{
Endpoint: tt.endpoint,
},
}
result, err := source.normalizeRepo("org/repo")
assert.NoError(t, err)
assert.Equal(t, tt.wantResult, result)
})
}
}
func TestHandleRateLimit(t *testing.T) {
resetRateLimitForTest(t)
s := initTestSource(&sourcespb.GitHub{Credential: &sourcespb.GitHub_Unauthenticated{}})
ctx := context.Background()
assert.False(t, s.handleRateLimit(ctx, nil))
// Request
reqUrl, _ := url.Parse("https://github.com/trufflesecurity/trufflehog")
res := &github.Response{
Response: &http.Response{
StatusCode: 429,
Header: make(http.Header),
Request: &http.Request{
Method: "GET",
URL: reqUrl,
},
},
}
res.Header.Set("x-ratelimit-remaining", "0")
res.Header.Set("x-ratelimit-reset", strconv.FormatInt(time.Now().Unix()+1, 10))
// Error
resetTime := github.Timestamp{
Time: time.Now().Add(time.Millisecond),
}
err := &github.RateLimitError{
Rate: github.Rate{
Limit: 5000,
Remaining: 0,
Reset: resetTime,
},
Response: res.Response,
Message: "Too Many Requests",
}
assert.True(t, s.handleRateLimit(ctx, err))
}
func TestHandleRateLimitStopsSleepingWhenContextCancelled(t *testing.T) {
resetRateLimitForTest(t)
s := &Source{name: "test - github"}
retryAfter := time.Hour
err := &github.AbuseRateLimitError{
RetryAfter: &retryAfter,
Message: "secondary rate limit",
}
ctx, cancel := context.WithCancel(context.Background())
cancel()
start := time.Now()
assert.False(t, s.handleRateLimit(ctx, err))
assert.Less(t, time.Since(start), 200*time.Millisecond)
}
func resetRateLimitForTest(t *testing.T) {
t.Helper()
rateLimitMu.Lock()
previous := rateLimitResumeTime
rateLimitResumeTime = time.Time{}
rateLimitMu.Unlock()
t.Cleanup(func() {
rateLimitMu.Lock()
rateLimitResumeTime = previous
rateLimitMu.Unlock()
})
}
func TestEnumerateUnauthenticated(t *testing.T) {
defer gock.Off()
apiEndpoint := "https://api.github.com"
gock.New(apiEndpoint).
Get("/orgs/super-secret-org/repos").
Reply(200).
JSON([]map[string]string{{"full_name": "super-secret-org/super-secret-repo", "clone_url": "https://github.com/super-secret-org/super-secret-repo.git"}})
s := initTestSource(&sourcespb.GitHub{
Endpoint: apiEndpoint,
Credential: &sourcespb.GitHub_Unauthenticated{},
})
s.orgsCache = simple.NewCache[string]()
s.orgsCache.Set("super-secret-org", "super-secret-org")
// s.enumerateUnauthenticated(context.Background(), apiEndpoint)
s.enumerateUnauthenticated(context.Background(), noopReporter())
assert.Equal(t, 1, s.filteredRepoCache.Count())
ok := s.filteredRepoCache.Exists("super-secret-org/super-secret-repo")
assert.True(t, ok)
assert.False(t, gock.HasUnmatchedRequest())
assert.True(t, gock.IsDone())
}
func TestEnumerateWithToken(t *testing.T) {
defer gock.Off()
gock.New("https://api.github.com").
Get("/user").
Reply(200).
JSON(map[string]string{"login": "super-secret-user"})
gock.New("https://api.github.com").
Get("/user/repos").
MatchParam("per_page", "100").
Reply(200).
JSON([]map[string]string{{"clone_url": "https://github.com/super-secret-user/super-secret-repo.git", "full_name": "super-secret-user/super-secret-repo"}})
gock.New("https://api.github.com").
Get("/user/orgs").
MatchParam("per_page", "100").
Reply(200).
JSON(`[]`)
gock.New("https://api.github.com").
Get("/users/super-secret-user/gists").
Reply(200).
JSON([]map[string]string{{"id": "super-secret-gist", "git_pull_url": "https://gist.github.com/super-secret-gist.git"}})
s := initTestSource(&sourcespb.GitHub{
Endpoint: "https://api.github.com",
Credential: &sourcespb.GitHub_Token{
Token: "token",
},
})
err := s.enumerateWithToken(context.Background(), false, noopReporter())
assert.Nil(t, err)
assert.Equal(t, 2, s.filteredRepoCache.Count())
ok := s.filteredRepoCache.Exists("super-secret-user/super-secret-repo")
assert.True(t, ok)
ok = s.filteredRepoCache.Exists("super-secret-gist")
assert.True(t, ok)
assert.False(t, gock.HasUnmatchedRequest())
assert.True(t, gock.IsDone())
}
func BenchmarkEnumerateWithToken(b *testing.B) {
defer gock.Off()
gock.New("https://api.github.com").
Get("/user").
Reply(200).
JSON(map[string]string{"login": "super-secret-user"})
gock.New("https://api.github.com").
Get("/users/super-secret-user/repos").
Reply(200).
JSON([]map[string]string{{"clone_url": "https://github.com/super-secret-repo.git"}})
gock.New("https://api.github.com").
Get("/user/orgs").
MatchParam("per_page", "100").
Reply(200).
JSON([]map[string]string{{"clone_url": "https://github.com/super-secret-repo.git"}})
gock.New("https://api.github.com").
Get("/users/super-secret-user/gists").
Reply(200).
JSON([]map[string]string{{"git_pull_url": "https://github.com/super-secret-gist.git"}})
s := initTestSource(&sourcespb.GitHub{
Endpoint: "https://api.github.com",
Credential: &sourcespb.GitHub_Token{
Token: "token",
},
})
b.ResetTimer()
for i := 0; i < b.N; i++ {
_ = s.enumerateWithToken(context.Background(), false, noopReporter())
}
}
func TestEnumerate(t *testing.T) {
defer gock.Off()
// Arrange
gock.New("https://api.github.com").
Get("/user").
Reply(200).
JSON(map[string]string{"login": "super-secret-user"})
//
gock.New("https://api.github.com").
Get("/user/repos").
Reply(200).
JSON(`[{"name": "super-secret-repo", "full_name": "super-secret-user/super-secret-repo", "owner": {"login": "super-secret-user"}, "clone_url": "https://github.com/super-secret-user/super-secret-repo.git", "has_wiki": false, "size": 1}]`)
gock.New("https://api.github.com").
Get("/user/orgs").
MatchParam("per_page", "100").
Reply(200).
JSON([]map[string]string{{"clone_url": "https://github.com/super-secret-user/super-secret-repo.git", "full_name": "super-secret-user/super-secret-repo"}})
gock.New("https://api.github.com").
Get("/users/super-secret-user/gists").
Reply(200).
JSON(`[{"git_pull_url": "https://gist.github.com/2801a2b0523099d0614a951579d99ba9.git", "id": "2801a2b0523099d0614a951579d99ba9"}]`)
s := initTestSource(&sourcespb.GitHub{
Endpoint: "https://api.github.com",
Credential: &sourcespb.GitHub_Token{
Token: "super secret token",
},
})
// Manually cache a repository to ensure that enumerate
// doesn't make duplicate API calls.
// See https://github.com/trufflesecurity/trufflehog/pull/2625
repo := func() *github.Repository {
var (
name = "cached-repo"
fullName = "cached-user/cached-repo"
login = "cached-user"
cloneUrl = "https://github.com/cached-user/cached-repo.git"
owner = &github.User{
Login: &login,
}
hasWiki = false
size = 1234
)
return &github.Repository{
Name: &name,
FullName: &fullName,
Owner: owner,
HasWiki: &hasWiki,
Size: &size,
CloneURL: &cloneUrl,
}
}()
s.cacheRepoInfo(repo)
s.filteredRepoCache.Set(repo.GetFullName(), repo.GetCloneURL())
var reportedRepos []string
reporter := sources.VisitorReporter{
VisitUnit: func(ctx context.Context, su sources.SourceUnit) error {
url, _ := su.SourceUnitID()
reportedRepos = append(reportedRepos, url)
return nil
},
}
// Act
err := s.Enumerate(context.Background(), reporter)
slices.Sort(reportedRepos)
// Assert
assert.Nil(t, err)
// Enumeration found all repos.
assert.Equal(t, 3, s.filteredRepoCache.Count())
assert.True(t, s.filteredRepoCache.Exists("super-secret-user/super-secret-repo"))
assert.True(t, s.filteredRepoCache.Exists("cached-user/cached-repo"))
assert.True(t, s.filteredRepoCache.Exists("2801a2b0523099d0614a951579d99ba9"))
assert.Equal(t, 3, len(s.repos))
assert.Equal(t, s.repos, reportedRepos)
// Enumeration cached all repos.
assert.Equal(t, 3, len(s.repoInfoCache.cache))
_, ok := s.repoInfoCache.get("https://github.com/super-secret-user/super-secret-repo.git")
assert.True(t, ok)
_, ok = s.repoInfoCache.get("https://github.com/cached-user/cached-repo.git")
assert.True(t, ok)
_, ok = s.repoInfoCache.get("https://gist.github.com/2801a2b0523099d0614a951579d99ba9.git")
assert.True(t, ok)
assert.False(t, gock.HasUnmatchedRequest())
assert.True(t, gock.IsDone())
}
func setupMocks(b *testing.B) {
b.Helper()
gock.New("https://api.github.com").
Get("/user").
Reply(200).
JSON(map[string]string{"login": "super-secret-user"})
gock.New("https://api.github.com").
Get("/users/super-secret-user/repos").
Reply(200).
JSON(mockRepos())
gock.New("https://api.github.com").
Get("/user/orgs").
MatchParam("per_page", "100").
Reply(200).
JSON([]map[string]string{{"clone_url": "https://github.com/super-secret-repo.git"}})
gock.New("https://api.github.com").
Get("/users/super-secret-user/gists").
Reply(200).
JSON(mockGists())
}
func mockRepos() []map[string]string {
res := make([]map[string]string, 0, 10000)
for i := 0; i < 10000; i++ {
res = append(res, map[string]string{"clone_url": fmt.Sprintf("https://githu/super-secret-repo-%d.git", i)})
}
return res
}
func mockGists() []map[string]string {
res := make([]map[string]string, 0, 100)
for i := 0; i < 100; i++ {
res = append(res, map[string]string{"git_pull_url": fmt.Sprintf("https://githu/super-secret-gist-%d.git", i)})
}
return res
}
func BenchmarkEnumerate(b *testing.B) {
for i := 0; i < b.N; i++ {
s := initTestSource(&sourcespb.GitHub{
Endpoint: "https://api.github.com",
Credential: &sourcespb.GitHub_Token{
Token: "super secret token",
},
})
setupMocks(b)
b.StartTimer()
_ = s.Enumerate(context.Background(), noopReporter())
}
}
func TestEnumerateWithToken_Repositories(t *testing.T) {
defer gock.Off()
gock.New("https://api.github.com").
Get("/user").
Reply(200).
JSON(map[string]string{"login": "super-secret-user"})
s := initTestSource(&sourcespb.GitHub{
Endpoint: "https://api.github.com",
Credential: &sourcespb.GitHub_Token{
Token: "token",
},
})
s.repos = []string{"some-special-repo"}
err := s.enumerateWithToken(context.Background(), false, noopReporter())
assert.Nil(t, err)
assert.Equal(t, 1, len(s.repos))
assert.Equal(t, []string{"some-special-repo"}, s.repos)
assert.False(t, gock.HasUnmatchedRequest())
assert.True(t, gock.IsDone())
}
func TestEnumerateWithToken_IncludeRepos(t *testing.T) {
defer gock.Off()
gock.New("https://api.github.com").
Get("/user").
Reply(200).
JSON(map[string]string{"login": "super-secret-user"})
s := initTestSource(&sourcespb.GitHub{
Endpoint: "https://api.github.com",
Credential: &sourcespb.GitHub_Token{
Token: "token",
},
})
s.repos = []string{"some-special-repo"}
err := s.enumerateWithToken(context.Background(), false, noopReporter())
assert.Nil(t, err)
assert.Equal(t, 1, len(s.repos))
assert.Equal(t, []string{"some-special-repo"}, s.repos)
assert.False(t, gock.HasUnmatchedRequest())
assert.True(t, gock.IsDone())
}
func TestEnumerateWithApp(t *testing.T) {
defer gock.Off()
privateKey := createPrivateKey()
gock.New("https://api.github.com").
Post("/app/installations/1337/access_tokens").
Reply(200).
JSON(map[string]string{"token": "dontlook"})
gock.New("https://api.github.com").
Get("/installation/repositories").
Reply(200).
JSON(map[string]string{})
s := initTestSource(&sourcespb.GitHub{
Endpoint: "https://api.github.com",
Credential: &sourcespb.GitHub_GithubApp{
GithubApp: &credentialspb.GitHubApp{
PrivateKey: privateKey,
InstallationId: "1337",
AppId: "4141",
},
},
})
err := s.enumerateWithApp(context.Background(), s.connector.(*appConnector), noopReporter())
assert.Nil(t, err)
assert.Equal(t, 0, len(s.repos))
assert.False(t, gock.HasUnmatchedRequest())
assert.True(t, gock.IsDone())
}
func TestEnumerateWithAppScanAllInstallationsIncludesConfiguredInstallation(t *testing.T) {
privateKey := createPrivateKey()
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
switch {
case r.Method == http.MethodPost && strings.Contains(r.URL.Path, "/app/installations/") && strings.HasSuffix(r.URL.Path, "/access_tokens"):
parts := strings.Split(r.URL.Path, "/")
installID := parts[len(parts)-2]
_, _ = fmt.Fprintf(w, `{"token":"token-%s","expires_at":"2099-01-01T00:00:00Z"}`, installID)
case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/app/installations"):
_, _ = w.Write([]byte(`[
{"id":1337,"account":{"login":"default-org","type":"Organization"}},
{"id":2448,"account":{"login":"other-org","type":"Organization"}}
]`))
case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/installation/repositories"):
auth := r.Header.Get("Authorization")
switch {
case strings.Contains(auth, "token-1337"):
_, _ = w.Write([]byte(`{"total_count":1,"repositories":[
{"name":"repo","full_name":"default-org/repo","clone_url":"https://github.com/default-org/repo.git","owner":{"login":"default-org","type":"Organization"},"size":1}
]}`))
case strings.Contains(auth, "token-2448"):
_, _ = w.Write([]byte(`{"total_count":1,"repositories":[
{"name":"repo","full_name":"other-org/repo","clone_url":"https://github.com/other-org/repo.git","owner":{"login":"other-org","type":"Organization"},"size":1}
]}`))
default:
http.Error(w, "unexpected installation token", http.StatusUnauthorized)
}
default:
http.NotFound(w, r)
}
}))
defer server.Close()
s, conn := createTestSource(&sourcespb.GitHub{
Endpoint: server.URL,
ScanAllInstallations: true,
Credential: &sourcespb.GitHub_GithubApp{
GithubApp: &credentialspb.GitHubApp{
PrivateKey: privateKey,
InstallationId: "1337",
AppId: "4141",
},
},
})
require.NoError(t, s.Init(context.Background(), "test - github", 0, 1337, false, conn, 1))
connector := s.connector.(*appConnector)
err := s.enumerateWithApp(context.Background(), connector, noopReporter())
require.NoError(t, err)
assert.Equal(t, int64(1337), connector.repoInstallationMap["https://github.com/default-org/repo.git"])
assert.Equal(t, int64(2448), connector.repoInstallationMap["https://github.com/other-org/repo.git"])
assert.True(t, s.filteredRepoCache.Exists("default-org/repo"))
assert.True(t, s.filteredRepoCache.Exists("other-org/repo"))
}
func TestEnumerateWithAppScanAllInstallationsMapsExplicitReposBeforeMetadataFetch(t *testing.T) {
privateKey := createPrivateKey()
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
switch {
case r.Method == http.MethodPost && strings.Contains(r.URL.Path, "/app/installations/") && strings.HasSuffix(r.URL.Path, "/access_tokens"):
parts := strings.Split(r.URL.Path, "/")
installID := parts[len(parts)-2]
_, _ = fmt.Fprintf(w, `{"token":"token-%s","expires_at":"2099-01-01T00:00:00Z"}`, installID)
case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/app/installations"):
_, _ = w.Write([]byte(`[
{"id":1337,"account":{"login":"default-org","type":"Organization"}},
{"id":2448,"account":{"login":"other-org","type":"Organization"}}
]`))
case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/installation/repositories"):
auth := r.Header.Get("Authorization")
switch {
case strings.Contains(auth, "token-1337"):
_, _ = w.Write([]byte(`{"total_count":1,"repositories":[
{"name":"repo","full_name":"default-org/repo","clone_url":"https://github.com/default-org/repo.git","owner":{"login":"default-org","type":"Organization"},"size":1}
]}`))
case strings.Contains(auth, "token-2448"):
_, _ = w.Write([]byte(`{"total_count":1,"repositories":[
{"name":"repo","full_name":"other-org/repo","clone_url":"https://github.com/other-org/repo.git","owner":{"login":"other-org","type":"Organization"},"size":1}
]}`))
default:
http.Error(w, "unexpected installation token", http.StatusUnauthorized)
}
case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/repos/other-org/repo"):
if !strings.Contains(r.Header.Get("Authorization"), "token-2448") {
http.Error(w, "wrong installation token", http.StatusForbidden)
return
}
_, _ = w.Write([]byte(`{"name":"repo","full_name":"other-org/repo","clone_url":"https://github.com/other-org/repo.git","owner":{"login":"other-org","type":"Organization"},"size":1}`))
default:
http.NotFound(w, r)
}
}))
defer server.Close()
s, conn := createTestSource(&sourcespb.GitHub{
Endpoint: server.URL,
Repositories: []string{"https://github.com/other-org/repo.git"},
ScanAllInstallations: true,
Credential: &sourcespb.GitHub_GithubApp{
GithubApp: &credentialspb.GitHubApp{
PrivateKey: privateKey,
InstallationId: "1337",
AppId: "4141",
},
},
})
require.NoError(t, s.Init(context.Background(), "test - github", 0, 1337, false, conn, 1))
err := s.Enumerate(context.Background(), noopReporter())
require.NoError(t, err)
connector := s.connector.(*appConnector)
assert.Equal(t, int64(2448), connector.repoInstallationMap["https://github.com/other-org/repo.git"])
info, ok := s.repoInfoCache.get("https://github.com/other-org/repo.git")
require.True(t, ok)
assert.Equal(t, "other-org/repo", info.fullName)
}
func TestMapExplicitReposToInstallationsErrorsForUnmatchedRepos(t *testing.T) {
privateKey := createPrivateKey()
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
switch {
case r.Method == http.MethodPost && strings.Contains(r.URL.Path, "/app/installations/") && strings.HasSuffix(r.URL.Path, "/access_tokens"):
parts := strings.Split(r.URL.Path, "/")
installID := parts[len(parts)-2]
_, _ = fmt.Fprintf(w, `{"token":"token-%s","expires_at":"2099-01-01T00:00:00Z"}`, installID)
case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/app/installations"):
_, _ = w.Write([]byte(`[{"id":2448,"account":{"login":"other-org","type":"Organization"}}]`))
case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/installation/repositories"):
_, _ = w.Write([]byte(`{"total_count":1,"repositories":[
{"name":"different","full_name":"other-org/different","clone_url":"https://github.com/other-org/different.git","owner":{"login":"other-org","type":"Organization"},"size":1}
]}`))
default:
http.NotFound(w, r)
}
}))
defer server.Close()
s, conn := createTestSource(&sourcespb.GitHub{
Endpoint: server.URL,
Repositories: []string{"https://github.com/other-org/missing.git"},
ScanAllInstallations: true,
Credential: &sourcespb.GitHub_GithubApp{
GithubApp: &credentialspb.GitHubApp{
PrivateKey: privateKey,
InstallationId: "1337",
AppId: "4141",
},
},
})
err := s.Init(context.Background(), "test - github", 0, 1337, false, conn, 1)
require.Error(t, err)
assert.Contains(t, err.Error(), "configured repos were not found in any GitHub App installation")
assert.Contains(t, err.Error(), "https://github.com/other-org/missing.git")
}
func TestScanAllInstallationsInitErrorsWhenInstallationsCannotBeListed(t *testing.T) {
privateKey := createPrivateKey()
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
switch {
case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/app/installations"):
http.Error(w, "installation list failed", http.StatusInternalServerError)
default:
http.NotFound(w, r)
}
}))
defer server.Close()
s, conn := createTestSource(&sourcespb.GitHub{
Endpoint: server.URL,
ScanAllInstallations: true,
Credential: &sourcespb.GitHub_GithubApp{
GithubApp: &credentialspb.GitHubApp{
PrivateKey: privateKey,
InstallationId: "1337",
AppId: "4141",
},
},
})
err := s.Init(context.Background(), "test - github", 0, 1337, false, conn, 1)
require.Error(t, err)
assert.Contains(t, err.Error(), "could not list installations")
}
func TestMapExplicitReposToInstallationsStopsAfterReposFound(t *testing.T) {
privateKey := createPrivateKey()
var listInstallationsCalls, unrelatedRepoListCalls int
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
switch {
case r.Method == http.MethodPost && strings.Contains(r.URL.Path, "/app/installations/") && strings.HasSuffix(r.URL.Path, "/access_tokens"):
parts := strings.Split(r.URL.Path, "/")
installID := parts[len(parts)-2]
_, _ = fmt.Fprintf(w, `{"token":"token-%s","expires_at":"2099-01-01T00:00:00Z"}`, installID)
case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/app/installations"):
listInstallationsCalls++
_, _ = w.Write([]byte(`[
{"id":1337,"account":{"login":"default-org","type":"Organization"}},
{"id":2448,"account":{"login":"other-org","type":"Organization"}},
{"id":3559,"account":{"login":"unrelated-org","type":"Organization"}}
]`))
case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/installation/repositories"):
auth := r.Header.Get("Authorization")
switch {
case strings.Contains(auth, "token-1337"):
_, _ = w.Write([]byte(`{"total_count":0,"repositories":[]}`))
case strings.Contains(auth, "token-2448"):
_, _ = w.Write([]byte(`{"total_count":1,"repositories":[
{"name":"repo","full_name":"other-org/repo","clone_url":"https://github.com/other-org/repo.git","owner":{"login":"other-org","type":"Organization"},"size":1}
]}`))
case strings.Contains(auth, "token-3559"):
unrelatedRepoListCalls++
http.Error(w, "unrelated installation failed", http.StatusInternalServerError)
default:
http.Error(w, "unexpected installation token", http.StatusUnauthorized)
}
default:
http.NotFound(w, r)
}
}))
defer server.Close()
s, conn := createTestSource(&sourcespb.GitHub{
Endpoint: server.URL,
Repositories: []string{"https://github.com/other-org/repo.git"},
ScanAllInstallations: true,
Credential: &sourcespb.GitHub_GithubApp{
GithubApp: &credentialspb.GitHubApp{
PrivateKey: privateKey,
InstallationId: "1337",
AppId: "4141",
},
},
})
require.NoError(t, s.Init(context.Background(), "test - github", 0, 1337, false, conn, 1))
connector := s.connector.(*appConnector)
require.NoError(t, s.mapExplicitReposToInstallations(context.Background(), connector))
require.NoError(t, s.mapExplicitReposToInstallations(context.Background(), connector))
installationID, mapped := connector.installationIDForRepo("https://github.com/other-org/repo.git")
assert.True(t, mapped)
assert.Equal(t, int64(2448), installationID)
assert.Equal(t, 1, listInstallationsCalls)
assert.Equal(t, 0, unrelatedRepoListCalls)
}
func TestScanAllInstallationsReusesInstallationsLoadedDuringInit(t *testing.T) {
privateKey := createPrivateKey()
var listInstallationsCalls int
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
switch {
case r.Method == http.MethodPost && strings.Contains(r.URL.Path, "/app/installations/") && strings.HasSuffix(r.URL.Path, "/access_tokens"):
parts := strings.Split(r.URL.Path, "/")
installID := parts[len(parts)-2]
_, _ = fmt.Fprintf(w, `{"token":"token-%s","expires_at":"2099-01-01T00:00:00Z"}`, installID)
case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/app/installations"):
listInstallationsCalls++
_, _ = w.Write([]byte(`[
{"id":1337,"account":{"login":"default-org","type":"Organization"}},
{"id":2448,"account":{"login":"other-org","type":"Organization"}}
]`))
case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/installation/repositories"):
if strings.Contains(r.Header.Get("Authorization"), "token-2448") {
_, _ = w.Write([]byte(`{"total_count":1,"repositories":[
{"name":"repo","full_name":"other-org/repo","clone_url":"https://github.com/other-org/repo.git","owner":{"login":"other-org","type":"Organization"},"size":1}
]}`))
return
}
_, _ = w.Write([]byte(`{"total_count":0,"repositories":[]}`))
case r.Method == http.MethodGet && strings.Contains(r.URL.Path, "/orgs/") && strings.HasSuffix(r.URL.Path, "/members"):
_, _ = w.Write([]byte(`[]`))
default:
http.NotFound(w, r)
}
}))
defer server.Close()
s, conn := createTestSource(&sourcespb.GitHub{
Endpoint: server.URL,
Repositories: []string{"https://github.com/other-org/repo.git"},
ScanAllInstallations: true,
Credential: &sourcespb.GitHub_GithubApp{
GithubApp: &credentialspb.GitHubApp{
PrivateKey: privateKey,
InstallationId: "1337",
AppId: "4141",
},
},
})
require.NoError(t, s.Init(context.Background(), "test - github", 0, 1337, false, conn, 1))
connector := s.connector.(*appConnector)
require.NoError(t, s.enumerateAllInstallationRepos(context.Background(), connector, noopReporter()))
require.NoError(t, s.addMembersByApp(context.Background(), connector, noopReporter()))
assert.Equal(t, 1, listInstallationsCalls)
}
func TestMapExplicitReposToInstallationsMatchesSSHRepoURL(t *testing.T) {
privateKey := createPrivateKey()
const sshRepoURL = "ssh://[email protected]/other-org/repo.git"
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
switch {
case r.Method == http.MethodPost && strings.Contains(r.URL.Path, "/app/installations/") && strings.HasSuffix(r.URL.Path, "/access_tokens"):
_, _ = w.Write([]byte(`{"token":"token-2448","expires_at":"2099-01-01T00:00:00Z"}`))
case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/app/installations"):
_, _ = w.Write([]byte(`[{"id":2448,"account":{"login":"other-org","type":"Organization"}}]`))
case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/installation/repositories"):
_, _ = w.Write([]byte(`{"total_count":1,"repositories":[
{"name":"repo","full_name":"other-org/repo","clone_url":"https://github.com/other-org/repo.git","owner":{"login":"other-org","type":"Organization"},"size":1}
]}`))
case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/repos/other-org/repo"):
_, _ = w.Write([]byte(`{"name":"repo","full_name":"other-org/repo","clone_url":"https://github.com/other-org/repo.git","owner":{"login":"other-org","type":"Organization"},"size":1}`))
default:
http.NotFound(w, r)
}
}))
defer server.Close()
s, conn := createTestSource(&sourcespb.GitHub{
Endpoint: server.URL,
Repositories: []string{sshRepoURL},
ScanAllInstallations: true,
Credential: &sourcespb.GitHub_GithubApp{
GithubApp: &credentialspb.GitHubApp{
PrivateKey: privateKey,
InstallationId: "1337",
AppId: "4141",
},
},
})
require.NoError(t, s.Init(context.Background(), "test - github", 0, 1337, false, conn, 1))
connector := s.connector.(*appConnector)
installationID, mapped := connector.installationIDForRepo(sshRepoURL)
assert.True(t, mapped)
assert.Equal(t, int64(2448), installationID)
}
func TestMapExplicitReposToInstallationsMatchesSCPStyleSSHRepoURL(t *testing.T) {
privateKey := createPrivateKey()
const sshRepoURL = "[email protected]:other-org/repo.git"
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
switch {
case r.Method == http.MethodPost && strings.Contains(r.URL.Path, "/app/installations/") && strings.HasSuffix(r.URL.Path, "/access_tokens"):
_, _ = w.Write([]byte(`{"token":"token-2448","expires_at":"2099-01-01T00:00:00Z"}`))
case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/app/installations"):
_, _ = w.Write([]byte(`[{"id":2448,"account":{"login":"other-org","type":"Organization"}}]`))
case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/installation/repositories"):
_, _ = w.Write([]byte(`{"total_count":1,"repositories":[
{"name":"repo","full_name":"other-org/repo","clone_url":"https://github.com/other-org/repo.git","owner":{"login":"other-org","type":"Organization"},"size":1}
]}`))
case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/repos/other-org/repo"):
_, _ = w.Write([]byte(`{"name":"repo","full_name":"other-org/repo","clone_url":"https://github.com/other-org/repo.git","owner":{"login":"other-org","type":"Organization"},"size":1}`))
default:
http.NotFound(w, r)
}
}))
defer server.Close()
s, conn := createTestSource(&sourcespb.GitHub{
Endpoint: server.URL,
Repositories: []string{sshRepoURL},
ScanAllInstallations: true,
Credential: &sourcespb.GitHub_GithubApp{
GithubApp: &credentialspb.GitHubApp{
PrivateKey: privateKey,
InstallationId: "1337",
AppId: "4141",
},
},
})
require.NoError(t, s.Init(context.Background(), "test - github", 0, 1337, false, conn, 1))
connector := s.connector.(*appConnector)
installationID, mapped := connector.installationIDForRepo(sshRepoURL)
assert.True(t, mapped)
assert.Equal(t, int64(2448), installationID)
}
func TestMapExplicitReposToInstallationsRejectsHostMismatch(t *testing.T) {
privateKey := createPrivateKey()
const attackerRepoURL = "https://attacker.example/other-org/repo.git"
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
switch {
case r.Method == http.MethodPost && strings.Contains(r.URL.Path, "/app/installations/") && strings.HasSuffix(r.URL.Path, "/access_tokens"):
_, _ = w.Write([]byte(`{"token":"token-2448","expires_at":"2099-01-01T00:00:00Z"}`))
case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/app/installations"):
_, _ = w.Write([]byte(`[{"id":2448,"account":{"login":"other-org","type":"Organization"}}]`))
case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/installation/repositories"):
_, _ = w.Write([]byte(`{"total_count":1,"repositories":[
{"name":"repo","full_name":"other-org/repo","clone_url":"https://github.com/other-org/repo.git","owner":{"login":"other-org","type":"Organization"},"size":1}
]}`))
case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/repos/other-org/repo"):
_, _ = w.Write([]byte(`{"name":"repo","full_name":"other-org/repo","clone_url":"https://github.com/other-org/repo.git","owner":{"login":"other-org","type":"Organization"},"size":1}`))
default:
http.NotFound(w, r)
}
}))
defer server.Close()
s, conn := createTestSource(&sourcespb.GitHub{
Endpoint: server.URL,
Repositories: []string{attackerRepoURL},
ScanAllInstallations: true,
Credential: &sourcespb.GitHub_GithubApp{
GithubApp: &credentialspb.GitHubApp{
PrivateKey: privateKey,
InstallationId: "1337",
AppId: "4141",
},
},
})
err := s.Init(context.Background(), "test - github", 0, 1337, false, conn, 1)
require.Error(t, err)
assert.Contains(t, err.Error(), "configured repos were not found")
}
// A repo absent from every installation listing but readable with the
// default installation token (e.g. a public repo, or one the listing missed)
// maps to the default installation instead of failing the scan. Repos the
// default token cannot read are still rejected (see the preceding test and
// TestScanAllInstallationsInaccessibleRepoStillFailsMapping).
func TestMapExplicitReposToInstallationsFallsBackToDefaultForReadableRepo(t *testing.T) {
privateKey := createPrivateKey()
const repoURL = "https://github.com/other-org/repo.git"
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
switch {
case r.Method == http.MethodPost && strings.Contains(r.URL.Path, "/app/installations/") && strings.HasSuffix(r.URL.Path, "/access_tokens"):
_, _ = w.Write([]byte(`{"token":"token-2448","expires_at":"2099-01-01T00:00:00Z"}`))
case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/app/installations"):
_, _ = w.Write([]byte(`[{"id":2448,"account":{"login":"other-org","type":"Organization"}}]`))
case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/installation/repositories"):
_, _ = w.Write([]byte(`{"total_count":0,"repositories":[]}`))
case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/repos/other-org/repo"):
_, _ = w.Write([]byte(`{"name":"repo","full_name":"other-org/repo","clone_url":"https://github.com/other-org/repo.git","owner":{"login":"other-org","type":"Organization"},"size":1}`))
default:
http.NotFound(w, r)
}
}))
defer server.Close()
s, conn := createTestSource(&sourcespb.GitHub{
Endpoint: server.URL,
Repositories: []string{repoURL},
ScanAllInstallations: true,
Credential: &sourcespb.GitHub_GithubApp{
GithubApp: &credentialspb.GitHubApp{
PrivateKey: privateKey,
InstallationId: "1337",
AppId: "4141",
},
},
})
err := s.Init(context.Background(), "test - github", 0, 1337, false, conn, 1)
require.NoError(t, err)
connector := s.connector.(*appConnector)
installationID, mapped := connector.installationIDForRepo(repoURL)
require.True(t, mapped)
assert.Equal(t, int64(1337), installationID)
}
func TestMapExplicitReposToInstallationsMapsWikiRepoURL(t *testing.T) {
privateKey := createPrivateKey()
const wikiRepoURL = "https://github.com/other-org/repo.wiki.git"
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
switch {
case r.Method == http.MethodPost && strings.Contains(r.URL.Path, "/app/installations/") && strings.HasSuffix(r.URL.Path, "/access_tokens"):
_, _ = w.Write([]byte(`{"token":"token-2448","expires_at":"2099-01-01T00:00:00Z"}`))
case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/app/installations"):
_, _ = w.Write([]byte(`[{"id":2448,"account":{"login":"other-org","type":"Organization"}}]`))
case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/installation/repositories"):
_, _ = w.Write([]byte(`{"total_count":1,"repositories":[
{"name":"repo","full_name":"other-org/repo","clone_url":"https://github.com/other-org/repo.git","owner":{"login":"other-org","type":"Organization"},"size":1}
]}`))
case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/repos/other-org/repo"):
_, _ = w.Write([]byte(`{"name":"repo","full_name":"other-org/repo","clone_url":"https://github.com/other-org/repo.git","owner":{"login":"other-org","type":"Organization"},"size":1}`))
default:
http.NotFound(w, r)
}
}))
defer server.Close()
s, conn := createTestSource(&sourcespb.GitHub{
Endpoint: server.URL,
Repositories: []string{wikiRepoURL},
ScanAllInstallations: true,
Credential: &sourcespb.GitHub_GithubApp{
GithubApp: &credentialspb.GitHubApp{
PrivateKey: privateKey,
InstallationId: "1337",
AppId: "4141",
},
},
})
require.NoError(t, s.Init(context.Background(), "test - github", 0, 1337, false, conn, 1))
connector := s.connector.(*appConnector)
installationID, mapped := connector.installationIDForRepo("https://github.com/other-org/repo.git")
assert.True(t, mapped)
assert.Equal(t, int64(2448), installationID)
installationID, mapped = connector.installationIDForRepo(wikiRepoURL)
assert.True(t, mapped)
assert.Equal(t, int64(2448), installationID)
}
func TestMapExplicitReposToInstallationsPrefersRealWikiSuffixedRepo(t *testing.T) {
privateKey := createPrivateKey()
const parentRepoURL = "https://github.com/other-org/repo.git"
const wikiSuffixedRepoURL = "https://github.com/other-org/repo.wiki.git"
const wikiOfWikiSuffixedRepoURL = "https://github.com/other-org/repo.wiki.wiki.git"
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
switch {
case r.Method == http.MethodPost && strings.Contains(r.URL.Path, "/app/installations/") && strings.HasSuffix(r.URL.Path, "/access_tokens"):
_, _ = w.Write([]byte(`{"token":"token-2448","expires_at":"2099-01-01T00:00:00Z"}`))
case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/app/installations"):
_, _ = w.Write([]byte(`[{"id":2448,"account":{"login":"other-org","type":"Organization"}}]`))
case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/installation/repositories"):
_, _ = w.Write([]byte(`{"total_count":2,"repositories":[
{"name":"repo.wiki","full_name":"other-org/repo.wiki","clone_url":"https://github.com/other-org/repo.wiki.git","owner":{"login":"other-org","type":"Organization"},"size":1,"has_wiki":true},
{"name":"repo","full_name":"other-org/repo","clone_url":"https://github.com/other-org/repo.git","owner":{"login":"other-org","type":"Organization"},"size":1}
]}`))
default:
http.NotFound(w, r)
}
}))
defer server.Close()
s, conn := createTestSource(&sourcespb.GitHub{
Endpoint: server.URL,
Repositories: []string{parentRepoURL, wikiSuffixedRepoURL},
ScanAllInstallations: true,
Credential: &sourcespb.GitHub_GithubApp{
GithubApp: &credentialspb.GitHubApp{
PrivateKey: privateKey,
InstallationId: "1337",
AppId: "4141",
},
},
})
require.NoError(t, s.Init(context.Background(), "test - github", 0, 1337, false, conn, 1))
connector := s.connector.(*appConnector)
installationID, mapped := connector.installationIDForRepo(parentRepoURL)
assert.True(t, mapped)
assert.Equal(t, int64(2448), installationID)
installationID, mapped = connector.installationIDForRepo(wikiSuffixedRepoURL)
assert.True(t, mapped)
assert.Equal(t, int64(2448), installationID)
installationID, mapped = connector.installationIDForRepo(wikiOfWikiSuffixedRepoURL)
assert.True(t, mapped)
assert.Equal(t, int64(2448), installationID)
}
func TestMapExplicitReposToInstallationsMapsRedirectedRepoFromMetadata(t *testing.T) {
privateKey := createPrivateKey()
const oldRepoURL = "https://github.com/old-org/repo.git"
const canonicalRepoURL = "https://github.com/new-org/repo.git"
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
switch {
case r.Method == http.MethodPost && strings.Contains(r.URL.Path, "/app/installations/") && strings.HasSuffix(r.URL.Path, "/access_tokens"):
_, _ = w.Write([]byte(`{"token":"token-2448","expires_at":"2099-01-01T00:00:00Z"}`))
case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/app/installations"):
_, _ = w.Write([]byte(`[{"id":2448,"account":{"login":"new-org","type":"Organization"}}]`))
case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/installation/repositories"):
_, _ = w.Write([]byte(`{"total_count":1,"repositories":[
{"name":"repo","full_name":"new-org/repo","clone_url":"https://github.com/new-org/repo.git","owner":{"login":"new-org","type":"Organization"},"size":1}
]}`))
case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/repos/old-org/repo"):
if !strings.Contains(r.Header.Get("Authorization"), "token-2448") {
http.Error(w, "wrong installation token", http.StatusForbidden)
return
}
_, _ = w.Write([]byte(`{"name":"repo","full_name":"new-org/repo","clone_url":"https://github.com/new-org/repo.git","owner":{"login":"new-org","type":"Organization"},"size":1}`))
default:
http.NotFound(w, r)
}
}))
defer server.Close()
s, conn := createTestSource(&sourcespb.GitHub{
Endpoint: server.URL,
Repositories: []string{oldRepoURL},
ScanAllInstallations: true,
Credential: &sourcespb.GitHub_GithubApp{
GithubApp: &credentialspb.GitHubApp{
PrivateKey: privateKey,
InstallationId: "1337",
AppId: "4141",
},
},
})
require.NoError(t, s.Init(context.Background(), "test - github", 0, 1337, false, conn, 1))
connector := s.connector.(*appConnector)
installationID, mapped := connector.installationIDForRepo(oldRepoURL)
assert.True(t, mapped)
assert.Equal(t, int64(2448), installationID)
installationID, mapped = connector.installationIDForRepo(canonicalRepoURL)
assert.True(t, mapped)
assert.Equal(t, int64(2448), installationID)
info, ok := s.repoInfoCache.get(oldRepoURL)
require.True(t, ok)
assert.Equal(t, "new-org/repo", info.fullName)
}
func TestRepoURLFromTargetMetadataErrorsOnRepositoryLinkMismatch(t *testing.T) {
s := &Source{}
_, err := s.repoURLFromTargetMetadata(&source_metadatapb.Github{
Repository: "https://github.com/other-org/repo.git",
Link: "https://github.com/different-org/repo/blob/abcd1234/path/to/file.go#L1",
})
require.Error(t, err)
assert.Contains(t, err.Error(), "does not match link repository")
}
func TestRepoURLFromTargetMetadataErrorsOnHostMismatch(t *testing.T) {
s := &Source{}
_, err := s.repoURLFromTargetMetadata(&source_metadatapb.Github{
Repository: "https://github.company/other-org/repo.git",
Link: "https://github.com/other-org/repo/blob/abcd1234/path/to/file.go#L1",
})
require.Error(t, err)
assert.Contains(t, err.Error(), "does not match link repository")
}
func TestRepoURLFromTargetMetadataAllowsBareRepositoryName(t *testing.T) {
s := &Source{}
repoURL, err := s.repoURLFromTargetMetadata(&source_metadatapb.Github{
Repository: "repo",
Link: "https://github.com/other-org/repo/blob/abcd1234/path/to/file.go#L1",
})
require.NoError(t, err)
assert.Equal(t, "https://github.com/other-org/repo.git", repoURL)
}
func TestRepoURLFromTargetMetadataAllowsWikiRepositoryWithWikiLink(t *testing.T) {
s := &Source{}
repoURL, err := s.repoURLFromTargetMetadata(&source_metadatapb.Github{
Repository: "https://github.com/other-org/repo.wiki.git",
Link: "https://github.com/other-org/repo/wiki/path/to/file.go",
})
require.NoError(t, err)
assert.Equal(t, "https://github.com/other-org/repo.git", repoURL)
}
func TestScanAllInstallationsMapsUnitBeforeMetadataFetch(t *testing.T) {
privateKey := createPrivateKey()
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
switch {
case r.Method == http.MethodPost && strings.Contains(r.URL.Path, "/app/installations/") && strings.HasSuffix(r.URL.Path, "/access_tokens"):
parts := strings.Split(r.URL.Path, "/")
installID := parts[len(parts)-2]
_, _ = fmt.Fprintf(w, `{"token":"token-%s","expires_at":"2099-01-01T00:00:00Z"}`, installID)
case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/app/installations"):
_, _ = w.Write([]byte(`[
{"id":1337,"account":{"login":"default-org","type":"Organization"}},
{"id":2448,"account":{"login":"other-org","type":"Organization"}}
]`))
case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/installation/repositories"):
auth := r.Header.Get("Authorization")
switch {
case strings.Contains(auth, "token-1337"):
_, _ = w.Write([]byte(`{"total_count":1,"repositories":[]}`))
case strings.Contains(auth, "token-2448"):
_, _ = w.Write([]byte(`{"total_count":1,"repositories":[
{"name":"repo","full_name":"other-org/repo","clone_url":"https://github.com/other-org/repo.git","owner":{"login":"other-org","type":"Organization"},"size":1}
]}`))
default:
http.Error(w, "unexpected installation token", http.StatusUnauthorized)
}
case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/repos/other-org/repo"):
if !strings.Contains(r.Header.Get("Authorization"), "token-2448") {
http.Error(w, "wrong installation token", http.StatusForbidden)
return
}
_, _ = w.Write([]byte(`{"name":"repo","full_name":"other-org/repo","clone_url":"https://github.com/other-org/repo.git","owner":{"login":"other-org","type":"Organization"},"size":1}`))
default:
http.NotFound(w, r)
}
}))
defer server.Close()
s, conn := createTestSource(&sourcespb.GitHub{
Endpoint: server.URL,
ScanAllInstallations: true,
Credential: &sourcespb.GitHub_GithubApp{
GithubApp: &credentialspb.GitHubApp{
PrivateKey: privateKey,
InstallationId: "1337",
AppId: "4141",
},
},
})
require.NoError(t, s.Init(context.Background(), "test - github", 0, 1337, false, conn, 1))
connector := s.connector.(*appConnector)
err := s.mapReposToInstallations(context.Background(), connector, []string{"https://github.com/other-org/repo.git"})
require.NoError(t, err)
repoURL, err := s.ensureRepoInfoCache(context.Background(), "https://github.com/other-org/repo.git", &unitErrorReporter{noopReporter()})
require.NoError(t, err)
require.Equal(t, "https://github.com/other-org/repo.git", repoURL)
installationID, mapped := connector.installationIDForRepo("https://github.com/other-org/repo.git")
assert.True(t, mapped)
assert.Equal(t, int64(2448), installationID)
}
// Regression test for member personal repos under scan_all_installations
// (INT-789): they belong to no app installation, so the mapping must fall
// back to the default installation when the API confirms the repo is
// readable, both at chunk time and at enumeration time.
func TestScanAllInstallationsMemberPersonalRepoFallsBackToDefaultInstallation(t *testing.T) {
privateKey := createPrivateKey()
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
switch {
case r.Method == http.MethodPost && strings.Contains(r.URL.Path, "/app/installations/") && strings.HasSuffix(r.URL.Path, "/access_tokens"):
parts := strings.Split(r.URL.Path, "/")
installID := parts[len(parts)-2]
_, _ = fmt.Fprintf(w, `{"token":"token-%s","expires_at":"2099-01-01T00:00:00Z"}`, installID)
case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/app/installations"):
_, _ = w.Write([]byte(`[
{"id":1337,"account":{"login":"test-org","type":"Organization"}},
{"id":2448,"account":{"login":"other-org","type":"Organization"}}
]`))
case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/installation/repositories"):
// Installations only ever contain org-owned repos.
auth := r.Header.Get("Authorization")
switch {
case strings.Contains(auth, "token-1337"):
_, _ = w.Write([]byte(`{"total_count":1,"repositories":[
{"name":"backend","full_name":"test-org/backend","clone_url":"https://github.com/test-org/backend.git","owner":{"login":"test-org","type":"Organization"},"size":1}
]}`))
case strings.Contains(auth, "token-2448"):
_, _ = w.Write([]byte(`{"total_count":0,"repositories":[]}`))
default:
http.Error(w, "unexpected installation token", http.StatusUnauthorized)
}
case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/repos/alice/dns-data"):
// The member's personal repo exists and is public: any valid
// token can read it, mirroring real GitHub behavior.
_, _ = w.Write([]byte(`{"name":"dns-data","full_name":"alice/dns-data","clone_url":"https://github.com/alice/dns-data.git","owner":{"login":"alice","type":"User"},"private":false,"size":1}`))
case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/users/alice/repos"):
_, _ = w.Write([]byte(`[{"name":"dns-data","full_name":"alice/dns-data","clone_url":"https://github.com/alice/dns-data.git","owner":{"login":"alice","type":"User"},"private":false,"size":1}]`))
default:
http.NotFound(w, r)
}
}))
defer server.Close()
s, conn := createTestSource(&sourcespb.GitHub{
Endpoint: server.URL,
ScanAllInstallations: true,
ScanUsers: true,
Credential: &sourcespb.GitHub_GithubApp{
GithubApp: &credentialspb.GitHubApp{
PrivateKey: privateKey,
InstallationId: "1337",
AppId: "4141",
},
},
})
require.NoError(t, s.Init(context.Background(), "test - github", 0, 1337, false, conn, 1))
// The org-owned repo maps to its owning installation.
connector := s.connector.(*appConnector)
err := s.mapReposToInstallations(context.Background(), connector, []string{"https://github.com/test-org/backend.git"})
require.NoError(t, err)
// The member's public personal repo is in no installation, but the API
// confirms it is accessible, so it maps to the default installation.
err = s.mapReposToInstallations(context.Background(), connector, []string{"https://github.com/alice/dns-data.git"})
require.NoError(t, err)
installationID, mapped := connector.installationIDForRepo("https://github.com/alice/dns-data.git")
require.True(t, mapped)
require.Equal(t, int64(1337), installationID)
// Enumeration must map member repos up front so same-process ChunkUnit
// short-circuits without any mapping API calls.
connector.mu.Lock()
delete(connector.repoInstallationMap, "https://github.com/alice/dns-data.git")
connector.mu.Unlock()
require.NoError(t, s.getReposByUser(context.Background(), "alice", false, noopReporter()))
installationID, mapped = connector.installationIDForRepo("https://github.com/alice/dns-data.git")
require.True(t, mapped)
require.Equal(t, int64(1337), installationID)
}
// Repos that no installation owns AND the default installation token cannot
// read must still fail the mapping: the accessibility fallback must not turn
// the check fail-open.
func TestScanAllInstallationsInaccessibleRepoStillFailsMapping(t *testing.T) {
privateKey := createPrivateKey()
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
switch {
case r.Method == http.MethodPost && strings.Contains(r.URL.Path, "/app/installations/") && strings.HasSuffix(r.URL.Path, "/access_tokens"):
parts := strings.Split(r.URL.Path, "/")
installID := parts[len(parts)-2]
_, _ = fmt.Fprintf(w, `{"token":"token-%s","expires_at":"2099-01-01T00:00:00Z"}`, installID)
case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/app/installations"):
_, _ = w.Write([]byte(`[{"id":1337,"account":{"login":"test-org","type":"Organization"}}]`))
case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/installation/repositories"):
_, _ = w.Write([]byte(`{"total_count":0,"repositories":[]}`))
default:
// /repos/... lookups 404: repo is private/nonexistent for
// every installation token.
http.NotFound(w, r)
}
}))
defer server.Close()
s, conn := createTestSource(&sourcespb.GitHub{
Endpoint: server.URL,
ScanAllInstallations: true,
Credential: &sourcespb.GitHub_GithubApp{
GithubApp: &credentialspb.GitHubApp{
PrivateKey: privateKey,
InstallationId: "1337",
AppId: "4141",
},
},
})
require.NoError(t, s.Init(context.Background(), "test - github", 0, 1337, false, conn, 1))
connector := s.connector.(*appConnector)
err := s.mapReposToInstallations(context.Background(), connector, []string{"https://github.com/ghost/private-repo.git"})
require.Error(t, err)
require.Contains(t, err.Error(), "configured repos were not found in any GitHub App installation")
}
func TestEnumerateAllInstallationReposReportsInstallationErrors(t *testing.T) {
privateKey := createPrivateKey()
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
switch {
case r.Method == http.MethodPost && strings.Contains(r.URL.Path, "/app/installations/") && strings.HasSuffix(r.URL.Path, "/access_tokens"):
parts := strings.Split(r.URL.Path, "/")
installID := parts[len(parts)-2]
_, _ = fmt.Fprintf(w, `{"token":"token-%s","expires_at":"2099-01-01T00:00:00Z"}`, installID)
case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/app/installations"):
_, _ = w.Write([]byte(`[{"id":1337,"account":{"login":"default-org","type":"Organization"}},{"id":2448,"account":{"login":"other-org","type":"Organization"}}]`))
case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/installation/repositories"):
// Fail the first installation's repo listing; succeed for the second.
if strings.Contains(r.Header.Get("Authorization"), "token-1337") {
http.Error(w, "repo list failed", http.StatusInternalServerError)
return
}
_, _ = w.Write([]byte(`{"total_count":1,"repositories":[{"id":1,"name":"repo-b","full_name":"other-org/repo-b","clone_url":"https://github.com/other-org/repo-b.git","size":1}]}`))
default:
http.NotFound(w, r)
}
}))
defer server.Close()
s, conn := createTestSource(&sourcespb.GitHub{
Endpoint: server.URL,
ScanAllInstallations: true,
Credential: &sourcespb.GitHub_GithubApp{
GithubApp: &credentialspb.GitHubApp{
PrivateKey: privateKey,
InstallationId: "1337",
AppId: "4141",
},
},
})
require.NoError(t, s.Init(context.Background(), "test - github", 0, 1337, false, conn, 1))
var reportedErrs []error
var reportedUnits []sources.SourceUnit
reporter := sources.VisitorReporter{
VisitUnit: func(_ context.Context, unit sources.SourceUnit) error {
reportedUnits = append(reportedUnits, unit)
return nil
},
VisitErr: func(_ context.Context, err error) error {
reportedErrs = append(reportedErrs, err)
return nil
},
}
err := s.enumerateAllInstallationRepos(context.Background(), s.connector.(*appConnector), reporter)
// A single installation failure is reported but doesn't abort the scan;
// the other installation's repos are still enumerated.
require.NoError(t, err)
require.Len(t, reportedErrs, 1)
assert.Contains(t, reportedErrs[0].Error(), "error enumerating repos for installation 1337")
require.Len(t, reportedUnits, 1)
unitID, _ := reportedUnits[0].SourceUnitID()
assert.Equal(t, "https://github.com/other-org/repo-b.git", unitID)
}
func TestChunksReturnsScanAllInstallationEnumerationErrors(t *testing.T) {
privateKey := createPrivateKey()
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
switch {
case r.Method == http.MethodPost && strings.Contains(r.URL.Path, "/app/installations/") && strings.HasSuffix(r.URL.Path, "/access_tokens"):
parts := strings.Split(r.URL.Path, "/")
installID := parts[len(parts)-2]
_, _ = fmt.Fprintf(w, `{"token":"token-%s","expires_at":"2099-01-01T00:00:00Z"}`, installID)
case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/app/installations"):
_, _ = w.Write([]byte(`[{"id":1337,"account":{"login":"default-org","type":"Organization"}},{"id":2448,"account":{"login":"other-org","type":"Organization"}}]`))
case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/installation/repositories"):
if strings.Contains(r.Header.Get("Authorization"), "token-1337") {
http.Error(w, "repo list failed", http.StatusInternalServerError)
return
}
_, _ = w.Write([]byte(`{"total_count":0,"repositories":[]}`))
default:
http.NotFound(w, r)
}
}))
defer server.Close()
s, conn := createTestSource(&sourcespb.GitHub{
Endpoint: server.URL,
ScanAllInstallations: true,
Credential: &sourcespb.GitHub_GithubApp{
GithubApp: &credentialspb.GitHubApp{
PrivateKey: privateKey,
InstallationId: "1337",
AppId: "4141",
},
},
})
require.NoError(t, s.Init(context.Background(), "test - github", 0, 1337, false, conn, 1))
chunks := make(chan *sources.Chunk, 1)
err := s.Chunks(context.Background(), chunks)
require.Error(t, err)
assert.Contains(t, err.Error(), "error enumerating repos for installation 1337")
}
// This only tests the resume info slice portion of setProgressCompleteWithRepo.
func Test_setProgressCompleteWithRepo_resumeInfo(t *testing.T) {
tests := []struct {
startingResumeInfoSlice []string
repoURL string
wantResumeInfoSlice []string
}{
{
startingResumeInfoSlice: []string{},
repoURL: "a",
wantResumeInfoSlice: []string{"a"},
},
{
startingResumeInfoSlice: []string{"b"},
repoURL: "a",
wantResumeInfoSlice: []string{"a", "b"},
},
}
s := &Source{
repos: []string{},
}
for _, tt := range tests {
s.resumeInfoSlice = tt.startingResumeInfoSlice
s.setProgressCompleteWithRepo(0, 0, tt.repoURL)
if !reflect.DeepEqual(s.resumeInfoSlice, tt.wantResumeInfoSlice) {
t.Errorf("s.setProgressCompleteWithRepo() got: %v, want: %v", s.resumeInfoSlice, tt.wantResumeInfoSlice)
}
}
}
func Test_setProgressCompleteWithRepo_Progress(t *testing.T) {
repos := []string{"a", "b", "c", "d", "e"}
tests := map[string]struct {
repos []string
index int
offset int
wantPercentComplete int64
wantSectionsCompleted int32
wantSectionsRemaining int32
}{
"starting from the beginning, no offset": {
repos: repos,
index: 0,
offset: 0,
wantPercentComplete: 0,
wantSectionsCompleted: 0,
wantSectionsRemaining: 5,
},
"resume from the third, offset 2": {
repos: repos[2:],
index: 0,
offset: 2,
wantPercentComplete: 40,
wantSectionsCompleted: 2,
wantSectionsRemaining: 5,
},
"resume from the third, on last repo, offset 2": {
repos: repos[2:],
index: 2,
offset: 2,
wantPercentComplete: 80,
wantSectionsCompleted: 4,
wantSectionsRemaining: 5,
},
}
for _, tt := range tests {
s := &Source{
repos: tt.repos,
}
s.setProgressCompleteWithRepo(tt.index, tt.offset, "")
gotProgress := s.GetProgress()
if gotProgress.PercentComplete != tt.wantPercentComplete {
t.Errorf("s.setProgressCompleteWithRepo() PercentComplete got: %v want: %v", gotProgress.PercentComplete, tt.wantPercentComplete)
}
if gotProgress.SectionsCompleted != tt.wantSectionsCompleted {
t.Errorf("s.setProgressCompleteWithRepo() PercentComplete got: %v want: %v", gotProgress.SectionsCompleted, tt.wantSectionsCompleted)
}
if gotProgress.SectionsRemaining != tt.wantSectionsRemaining {
t.Errorf("s.setProgressCompleteWithRepo() PercentComplete got: %v want: %v", gotProgress.SectionsRemaining, tt.wantSectionsRemaining)
}
}
}
func Test_scan_SetProgressComplete(t *testing.T) {
testCases := []struct {
name string
repos []string
wantComplete bool
wantErr bool
}{
{
name: "no repos",
wantComplete: true,
},
{
name: "one valid repo",
repos: []string{"https://github.com/super-secret-user/super-secret-repo.git"},
wantComplete: true,
wantErr: true,
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
src := initTestSource(&sourcespb.GitHub{
Repositories: tc.repos,
Credential: &sourcespb.GitHub_Unauthenticated{},
})
src.jobPool = &errgroup.Group{}
_ = src.scan(context.Background(), nil)
if !tc.wantErr {
assert.Equal(t, "", src.GetProgress().EncodedResumeInfo)
}
gotComplete := src.GetProgress().PercentComplete == 100
if gotComplete != tc.wantComplete {
t.Errorf("got: %v, want: %v", gotComplete, tc.wantComplete)
}
})
}
}
func TestGetRepoURLParts(t *testing.T) {
repoURLs := []string{
"https://github.com/trufflesecurity/trufflehog.git",
"git+https://github.com/trufflesecurity/trufflehog.git",
"ssh://github.com/trufflesecurity/trufflehog.git",
"ssh://[email protected]/trufflesecurity/trufflehog.git",
"git+ssh://[email protected]/trufflesecurity/trufflehog.git",
"git://github.com/trufflesecurity/trufflehog.git",
}
expected := []string{"github.com", "trufflesecurity", "trufflehog"}
for _, tt := range repoURLs {
_, parts, err := getRepoURLParts(tt)
if err != nil {
t.Fatalf("failed: %v", err)
}
assert.Equal(t, expected, parts)
}
gistURLs := map[string][]string{
// Gists
"ssh://github.com/6df198861306313246466d23aa4102aa.git": nil,
"ssh://gist.github.com/6df198861306313246466d23aa4102aa.git": {"gist.github.com", "6df198861306313246466d23aa4102aa"},
"https://gist.github.com/6df198861306313246466d23aa4102aa.git": {"gist.github.com", "6df198861306313246466d23aa4102aa"},
"https://gist.github.com/john-smith/6df198861306313246466d23aa4102aa.git": {"gist.github.com", "john-smith", "6df198861306313246466d23aa4102aa"},
"ssh://github.contoso.com/gist/6df198861306313246466d23aa4102aa.git": {"github.contoso.com", "gist", "6df198861306313246466d23aa4102aa"},
"https://github.contoso.com/gist/6df198861306313246466d23aa4102aa.git": {"github.contoso.com", "gist", "6df198861306313246466d23aa4102aa"},
"https://github.contoso.com/gist/john-smith/6df198861306313246466d23aa4102aa.git": {"github.contoso.com", "gist", "john-smith", "6df198861306313246466d23aa4102aa"},
"https://github.com/gist/john-smith/6df198861306313246466d23aa4102aa.git": nil,
}
for tt, expected := range gistURLs {
_, parts, err := getRepoURLParts(tt)
if err != nil {
if expected == nil {
continue
}
t.Fatalf("failed: %v", err)
}
assert.Equal(t, expected, parts)
}
}
func TestGetRepoURLPartsWithTrailingHyphen(t *testing.T) {
// Test for https://github.com/trufflesecurity/trufflehog/issues/4679
// Repository names ending with a hyphen should be preserved correctly.
testCases := []struct {
name string
url string
expected []string
}{
{
name: "https with trailing hyphen",
url: "https://github.com/MYORG/my-repo-name-.git",
expected: []string{"github.com", "MYORG", "my-repo-name-"},
},
{
name: "https with trailing hyphen no .git",
url: "https://github.com/MYORG/my-repo-.git",
expected: []string{"github.com", "MYORG", "my-repo-"},
},
{
name: "ssh with trailing hyphen",
url: "ssh://[email protected]/MYORG/test-repo-.git",
expected: []string{"github.com", "MYORG", "test-repo-"},
},
{
name: "multiple hyphens with trailing",
url: "https://github.com/org-name/my-test-repo-.git",
expected: []string{"github.com", "org-name", "my-test-repo-"},
},
{
name: "single trailing hyphen repo",
url: "https://github.com/Org/-.git",
expected: []string{"github.com", "Org", "-"},
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
_, parts, err := getRepoURLParts(tc.url)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
assert.Equal(t, tc.expected, parts)
})
}
}
func TestGithubWebHost(t *testing.T) {
tests := []struct {
name string
endpoint string
want string
}{
{
name: "empty endpoint defaults to github",
endpoint: "",
want: "github.com",
},
{
name: "cloud api endpoint",
endpoint: "https://api.github.com",
want: "github.com",
},
{
name: "cloud api endpoint without scheme",
endpoint: "api.github.com",
want: "github.com",
},
{
name: "enterprise endpoint",
endpoint: "https://github.example.com/api/v3",
want: "github.example.com",
},
{
name: "enterprise endpoint without scheme",
endpoint: "github.example.com/api/v3",
want: "github.example.com",
},
{
name: "host ending with github.com substring is enterprise",
endpoint: "https://legithub.com/api/v3",
want: "legithub.com",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
assert.Equal(t, tt.want, githubWebHost(tt.endpoint))
})
}
}
func TestWikiIsReachableSkipsHeadForPrivateRepos(t *testing.T) {
var called bool
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
called = true
http.Redirect(w, r, "/login", http.StatusFound)
}))
defer server.Close()
repoURL := "https://github.example.com/owner/repo.git"
s := &Source{
conn: &sourcespb.GitHub{Endpoint: server.URL},
repoInfoCache: newRepoInfoCache(),
}
s.repoInfoCache.put(repoURL, repoInfo{
owner: "owner",
name: "repo",
visibility: source_metadatapb.Visibility_private,
})
assert.True(t, s.wikiIsReachable(context.Background(), repoURL))
assert.False(t, called)
}
func TestGetGistID(t *testing.T) {
tests := []struct {
trimmedURL []string
expected string
}{
{[]string{"https://gist.github.com", "12345"}, "12345"},
{[]string{"https://gist.github.com", "owner", "12345"}, "12345"},
}
for _, tt := range tests {
got := extractGistID(tt.trimmedURL)
assert.Equal(t, tt.expected, got)
}
}
// This isn't really a GitHub test, but GitHub is the only source that supports scan targeting right now, so this is
// where I've put this targeted scan test.
func Test_ScanMultipleTargets_MultipleErrors(t *testing.T) {
s := &Source{conn: &sourcespb.GitHub{}} // This test doesn't require initialization
ctx := context.Background()
chunksChan := make(chan *sources.Chunk)
targets := []sources.ChunkingTarget{
{SecretID: 1},
{SecretID: 2},
}
// The specific error text doesn't matter for the test, but it has to match what the source generates
want := []*sources.TargetedScanError{
{SecretID: 1, Err: errors.New("unable to cast metadata type for targeted scan")},
{SecretID: 2, Err: errors.New("unable to cast metadata type for targeted scan")},
}
err := s.Chunks(ctx, chunksChan, targets...)
unwrappable, ok := err.(interface{ Unwrap() []error })
if assert.True(t, ok, "returned error was not unwrappable") {
got := unwrappable.Unwrap()
assert.ElementsMatch(t, got, want)
}
}
func TestScanAllInstallationsTargetMappingErrorIsTargetedScanError(t *testing.T) {
s := &Source{
conn: &sourcespb.GitHub{ScanAllInstallations: true},
connector: &appConnector{
clientsByInstallationID: make(map[int64]*appInstallationClients),
repoInstallationMap: make(map[string]int64),
},
}
ctx := context.Background()
chunksChan := make(chan *sources.Chunk)
targets := []sources.ChunkingTarget{
{
SecretID: 1,
QueryCriteria: &source_metadatapb.MetaData{
Data: &source_metadatapb.MetaData_Github{
Github: &source_metadatapb.Github{Link: "https://github.com/only-owner"},
},
},
},
{SecretID: 2},
}
err := s.Chunks(ctx, chunksChan, targets...)
unwrappable, ok := err.(interface{ Unwrap() []error })
if assert.True(t, ok, "returned error was not unwrappable") {
got := unwrappable.Unwrap()
require.Len(t, got, 2)
for _, targetErr := range got {
assert.IsType(t, &sources.TargetedScanError{}, targetErr)
}
}
}
func TestScanTargetRejectsWikiLinkHostMismatch(t *testing.T) {
s := &Source{
conn: &sourcespb.GitHub{Endpoint: cloudV3Endpoint},
}
target := sources.ChunkingTarget{
QueryCriteria: &source_metadatapb.MetaData{
Data: &source_metadatapb.MetaData_Github{
Github: &source_metadatapb.Github{
Link: "https://attacker.example/owner/repo/wiki/path/to/file",
Commit: "abc123",
File: "path/to/file",
},
},
},
}
err := s.scanTarget(context.Background(), target, sources.ChanReporter{Ch: make(chan *sources.Chunk, 1)})
require.Error(t, err)
assert.Contains(t, err.Error(), "does not match GitHub endpoint host")
}
func TestRepositoryFiltering(t *testing.T) {
// Test that the filteredRepoCache correctly filters repositories
source := &Source{}
// Test case 1: No filters specified (should include everything)
cache1 := source.newFilteredRepoCache(context.Background(), simple.NewCache[string](), []string{}, []string{})
assert.True(t, cache1.wantRepo("org/repo1"))
assert.True(t, cache1.wantRepo("org/repo2"))
assert.True(t, cache1.wantRepo("org/repo3"))
// Test case 2: Include filter specified (should only include matching repos)
cache2 := source.newFilteredRepoCache(context.Background(), simple.NewCache[string](), []string{"org/repo1", "org/repo2"}, []string{})
assert.True(t, cache2.wantRepo("org/repo1"))
assert.True(t, cache2.wantRepo("org/repo2"))
assert.False(t, cache2.wantRepo("org/repo3"))
// Test case 3: Exclude filter specified (should exclude matching repos)
cache3 := source.newFilteredRepoCache(context.Background(), simple.NewCache[string](), []string{}, []string{"org/repo1"})
assert.False(t, cache3.wantRepo("org/repo1"))
assert.True(t, cache3.wantRepo("org/repo2"))
assert.True(t, cache3.wantRepo("org/repo3"))
// Test case 4: Both include and exclude filters (exclude takes precedence)
cache4 := source.newFilteredRepoCache(context.Background(), simple.NewCache[string](), []string{"org/repo1"}, []string{"org/repo1"})
assert.False(t, cache4.wantRepo("org/repo1"))
// Test case 5: Wildcard patterns
cache5 := source.newFilteredRepoCache(context.Background(), simple.NewCache[string](), []string{"org/*"}, []string{})
assert.True(t, cache5.wantRepo("org/repo1"))
assert.True(t, cache5.wantRepo("org/repo2"))
assert.False(t, cache5.wantRepo("other/repo1"))
}
func TestExplicitRepositoryBypass(t *testing.T) {
// Test that explicit repositories are included in enumeration
ctx := context.Background()
// Set up mocks for the API calls
gock.New("https://api.github.com").
Get("/user").
Reply(200).
JSON(map[string]string{"login": "test-user"})
gock.New("https://api.github.com").
Get("/repos/org/explicit-repo").
Reply(200).
JSON(map[string]any{
"full_name": "org/explicit-repo",
"clone_url": "https://github.com/org/explicit-repo.git",
"size": 1,
})
gock.New("https://api.github.com").
Get("/repos/org/another-explicit").
Reply(200).
JSON(map[string]any{
"full_name": "org/another-explicit",
"clone_url": "https://github.com/org/another-explicit.git",
"size": 1,
})
// Create a source with explicit repositories
source := initTestSource(&sourcespb.GitHub{
Credential: &sourcespb.GitHub_Token{
Token: "super secret token",
},
Repositories: []string{
"https://github.com/org/explicit-repo.git",
"https://github.com/org/another-explicit.git",
},
})
// Test the Enumerate method
err := source.Enumerate(ctx, noopReporter())
require.NoError(t, err)
// Verify that explicit repositories are included in the enumeration
assert.Len(t, source.repos, 2, "Should have 2 explicit repositories")
assert.Contains(t, source.repos, "https://github.com/org/explicit-repo.git")
assert.Contains(t, source.repos, "https://github.com/org/another-explicit.git")
}
// TestEnsureRepoInfoCache_RepoRedirect verifies that when the GitHub API returns
// a canonical CloneURL that differs from the requested URL (i.e. a repository
// has been renamed/moved), ensureRepoInfoCache stores info under BOTH the
// original URL and the canonical URL so that scanRepo and visibilityOf can
// look up metadata using either.
func TestEnsureRepoInfoCache_RepoRedirect(t *testing.T) {
defer gock.Off()
const (
originalURL = "https://github.com/old-org/myrepo.git"
canonicalURL = "https://github.com/new-org/myrepo.git"
)
gock.New("https://api.github.com").
Get("/repos/old-org/myrepo").
Reply(200).
JSON(map[string]any{
"name": "myrepo",
"full_name": "new-org/myrepo",
"clone_url": canonicalURL,
"private": false,
"has_wiki": false,
"size": 100,
"owner": map[string]string{"login": "new-org"},
})
s := initTestSource(&sourcespb.GitHub{
Credential: &sourcespb.GitHub_Token{Token: "token"},
})
returnedURL, err := s.ensureRepoInfoCache(context.Background(), originalURL, &unitErrorReporter{noopReporter()})
require.NoError(t, err)
// Original URL returned so callers can continue using it.
assert.Equal(t, originalURL, returnedURL)
// Both URLs must be in the cache so scanRepo / visibilityOf don't miss.
_, okOriginal := s.repoInfoCache.get(originalURL)
assert.True(t, okOriginal, "original (pre-redirect) URL must be cached")
_, okCanonical := s.repoInfoCache.get(canonicalURL)
assert.True(t, okCanonical, "canonical (post-redirect) URL must be cached")
assert.False(t, gock.HasUnmatchedRequest())
assert.True(t, gock.IsDone())
}
// TestEnumerate_ExplicitRepoRedirect verifies the end-to-end Enumerate flow
// when a user provides an explicit repo URL that GitHub redirects to a new
// location (e.g. org rename). scanRepo must be able to look up cache metadata
// using the original URL that ends up in s.repos.
func TestEnumerate_ExplicitRepoRedirect(t *testing.T) {
defer gock.Off()
const (
originalURL = "https://github.com/old-org/myrepo.git"
canonicalURL = "https://github.com/new-org/myrepo.git"
)
gock.New("https://api.github.com").
Get("/user").
Reply(200).
JSON(map[string]string{"login": "test-user"})
gock.New("https://api.github.com").
Get("/repos/old-org/myrepo").
Reply(200).
JSON(map[string]any{
"name": "myrepo",
"full_name": "new-org/myrepo",
"clone_url": canonicalURL,
"private": true,
"has_wiki": false,
"size": 50,
"owner": map[string]string{"login": "new-org"},
})
s := initTestSource(&sourcespb.GitHub{
Credential: &sourcespb.GitHub_Token{Token: "token"},
Repositories: []string{originalURL},
})
err := s.Enumerate(context.Background(), noopReporter())
require.NoError(t, err)
// s.repos must use the original URL (what was configured).
assert.Contains(t, s.repos, originalURL)
// scanRepo looks up repoInfoCache by the URL in s.repos.
// Without the redirect fix this lookup fails with "no repoInfo for URL".
info, ok := s.repoInfoCache.get(originalURL)
assert.True(t, ok, "scanRepo must be able to look up metadata by original URL")
assert.Equal(t, "new-org", info.owner)
assert.False(t, gock.HasUnmatchedRequest())
assert.True(t, gock.IsDone())
}
func noopReporter() sources.UnitReporter {
return sources.VisitorReporter{
VisitUnit: func(context.Context, sources.SourceUnit) error {
return nil
},
}
}
// This tests reproduces a bug where both VisitUnit and VisitErr were called
// for the same repository when caching the repository info failed.
func TestFixBothUnitErrAndUnitOKCalled(t *testing.T) {
cache := simple.NewCache[string]()
cache.Set("myorg/myrepo", "an invalid url that will cause an error")
s := &Source{
filteredRepoCache: &filteredRepoCache{
Cache: cache,
},
conn: &sourcespb.GitHub{
Repositories: []string{"myorg/myrepo"},
},
orgsCache: simple.NewCache[string](),
}
var okCalled, errCalled bool
reporter := sources.VisitorReporter{
VisitUnit: func(ctx context.Context, su sources.SourceUnit) error {
okCalled = true
return nil
},
VisitErr: func(ctx context.Context, err error) error {
errCalled = true
return nil
},
}
err := s.Enumerate(context.Background(), reporter)
require.NoError(t, err)
// expectation is that only VisitErr is called
assert.True(t, errCalled)
assert.False(t, okCalled)
}
func TestExtractRepoNameFromURL(t *testing.T) {
tests := []struct {
name string
url string
expected string
}{
{
name: "git URL",
url: "https://github.com/org/repo.git",
expected: "org/repo",
},
{
name: "git URL with trailing slash",
url: "https://github.com/org/repo.git/",
expected: "org/repo",
},
{
name: "git URL without .git",
url: "https://github.com/org/repo",
expected: "org/repo",
},
{
name: "git enterprise URL",
url: "https://example-enterprise.com/org/repo.git",
expected: "org/repo",
},
{
name: "just org/repo",
url: "org/repo",
expected: "org/repo",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
result := extractRepoNameFromUrl(tt.url)
assert.Equal(t, tt.expected, result)
})
}
}
func TestSource_ExcludeArchivedRepositories(t *testing.T) {
tests := []struct {
name string
excludeArchived bool
reposJSON string
wantRepoCount int
wantRepos []string
}{
{
name: "exclude archived repos when flag is true",
excludeArchived: true,
reposJSON: `[
{"full_name": "test-org/active-repo", "clone_url": "https://github.com/test-org/active-repo.git", "size": 1, "archived": false},
{"full_name": "test-org/archived-repo", "clone_url": "https://github.com/test-org/archived-repo.git", "size": 1, "archived": true},
{"full_name": "test-org/another-active", "clone_url": "https://github.com/test-org/another-active.git", "size": 1, "archived": false}
]`,
wantRepoCount: 2, // Only non-archived
wantRepos: []string{"test-org/active-repo", "test-org/another-active"},
},
{
name: "include archived repos when flag is false",
excludeArchived: false,
reposJSON: `[
{"full_name": "test-org/active-repo", "clone_url": "https://github.com/test-org/active-repo.git", "size": 1, "archived": false},
{"full_name": "test-org/archived-repo", "clone_url": "https://github.com/test-org/archived-repo.git", "size": 1, "archived": true},
{"full_name": "test-org/another-active", "clone_url": "https://github.com/test-org/another-active.git", "size": 1, "archived": false}
]`,
wantRepoCount: 3, // All repos
wantRepos: []string{"test-org/active-repo", "test-org/archived-repo", "test-org/another-active"},
},
{
name: "handle all archived repos",
excludeArchived: true,
reposJSON: `[
{"full_name": "test-org/archived-1", "clone_url": "https://github.com/test-org/archived-1.git", "size": 1, "archived": true},
{"full_name": "test-org/archived-2", "clone_url": "https://github.com/test-org/archived-2.git", "size": 1, "archived": true}
]`,
wantRepoCount: 0, // None included
wantRepos: []string{},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
defer gock.Off()
gock.New("https://api.github.com").
Get("/orgs/test-org/repos").
Reply(200).
JSON(tt.reposJSON)
s := initTestSource(&sourcespb.GitHub{
Credential: &sourcespb.GitHub_Token{
Token: "test-token",
},
ExcludeArchived: tt.excludeArchived,
})
err := s.getReposByOrg(context.Background(), "test-org", noopReporter())
assert.Nil(t, err)
assert.Equal(t, tt.wantRepoCount, s.filteredRepoCache.Count())
// Verify expected repos are in cache
for _, repo := range tt.wantRepos {
ok := s.filteredRepoCache.Exists(repo)
assert.True(t, ok, "expected repo %s to be in cache", repo)
}
// Verify archived repos are NOT in cache when excluding
if tt.excludeArchived {
allRepos := []string{"test-org/archived-repo", "test-org/archived-1", "test-org/archived-2"}
for _, repo := range allRepos {
if !slices.Contains(tt.wantRepos, repo) {
ok := s.filteredRepoCache.Exists(repo)
assert.False(t, ok, "archived repo %s should not be in cache when excluding", repo)
}
}
}
assert.False(t, gock.HasUnmatchedRequest())
assert.True(t, gock.IsDone())
})
}
}
func TestSource_ExcludeArchivedForkInteraction(t *testing.T) {
tests := []struct {
name string
includeForks bool
excludeArchived bool
reposJSON string
wantRepoCount int
wantRepos []string
notWantRepos []string
}{
{
name: "fork+archived skipped when forks included and archived excluded",
includeForks: true,
excludeArchived: true,
reposJSON: `[
{"full_name": "test-org/active", "clone_url": "https://github.com/test-org/active.git", "size": 1, "archived": false, "fork": false},
{"full_name": "test-org/fork-active", "clone_url": "https://github.com/test-org/fork-active.git", "size": 1, "archived": false, "fork": true},
{"full_name": "test-org/fork-archived", "clone_url": "https://github.com/test-org/fork-archived.git", "size": 1, "archived": true, "fork": true}
]`,
wantRepoCount: 2,
wantRepos: []string{"test-org/active", "test-org/fork-active"},
notWantRepos: []string{"test-org/fork-archived"},
},
{
name: "fork+archived included when archived allowed",
includeForks: true,
excludeArchived: false,
reposJSON: `[
{"full_name": "test-org/active", "clone_url": "https://github.com/test-org/active.git", "size": 1, "archived": false, "fork": false},
{"full_name": "test-org/fork-archived", "clone_url": "https://github.com/test-org/fork-archived.git", "size": 1, "archived": true, "fork": true}
]`,
wantRepoCount: 2,
wantRepos: []string{"test-org/active", "test-org/fork-archived"},
},
{
name: "fork+archived dropped by fork filter when forks excluded",
includeForks: false,
excludeArchived: true,
reposJSON: `[
{"full_name": "test-org/active", "clone_url": "https://github.com/test-org/active.git", "size": 1, "archived": false, "fork": false},
{"full_name": "test-org/fork-archived", "clone_url": "https://github.com/test-org/fork-archived.git", "size": 1, "archived": true, "fork": true}
]`,
wantRepoCount: 1,
wantRepos: []string{"test-org/active"},
notWantRepos: []string{"test-org/fork-archived"},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
defer gock.Off()
gock.New("https://api.github.com").
Get("/orgs/test-org/repos").
Reply(200).
JSON(tt.reposJSON)
s := initTestSource(&sourcespb.GitHub{
Credential: &sourcespb.GitHub_Token{
Token: "test-token",
},
IncludeForks: tt.includeForks,
ExcludeArchived: tt.excludeArchived,
})
err := s.getReposByOrg(context.Background(), "test-org", noopReporter())
assert.Nil(t, err)
assert.Equal(t, tt.wantRepoCount, s.filteredRepoCache.Count())
for _, repo := range tt.wantRepos {
assert.True(t, s.filteredRepoCache.Exists(repo), "expected repo %s to be in cache", repo)
}
for _, repo := range tt.notWantRepos {
assert.False(t, s.filteredRepoCache.Exists(repo), "repo %s should not be in cache", repo)
}
assert.False(t, gock.HasUnmatchedRequest())
assert.True(t, gock.IsDone())
})
}
}
func TestIsGHECloud(t *testing.T) {
tests := []struct {
name string
endpoint string
want bool
}{
// GHE.com endpoints — should return true
{"api subdomain", "https://api.company.ghe.com", true},
{"web subdomain", "https://company.ghe.com", true},
{"api subdomain trailing slash", "https://api.company.ghe.com/", true},
{"web subdomain trailing slash", "https://company.ghe.com/", true},
{"uppercase", "https://API.company.GHE.COM", true},
{"mixed case", "https://Api.company.Ghe.Com/", true},
{"deep subdomain", "https://api.team.company.ghe.com", true},
// GHES endpoints — should return false
{"ghes custom domain", "https://github.mycompany.com", false},
{"ghes with path", "https://github.mycompany.com/api/v3", false},
{"ghes internal", "https://github.internal.corp.com", false},
// github.com — should return false
{"cloud api", "https://api.github.com", false},
{"cloud web", "https://github.com", false},
// Edge cases — should return false
{"empty string", "", false},
{"localhost", "http://localhost:8080", false},
{"malicious suffix", "https://notghe.com", false},
{"partial match", "https://api.fakeghe.com.evil.com", false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := isGHECloud(tt.endpoint)
if got != tt.want {
t.Errorf("isGHECloud(%q) = %v, want %v", tt.endpoint, got, tt.want)
}
})
}
}
func TestNormalizeGHECloudAPIEndpoint(t *testing.T) {
tests := []struct {
name string
input string
want string
wantErr bool
}{
{
name: "web URL to API URL",
input: "https://company.ghe.com",
want: "https://api.company.ghe.com/",
},
{
name: "web URL with trailing slash",
input: "https://company.ghe.com/",
want: "https://api.company.ghe.com/",
},
{
name: "api URL already correct",
input: "https://api.company.ghe.com",
want: "https://api.company.ghe.com/",
},
{
name: "api URL with trailing slash",
input: "https://api.company.ghe.com/",
want: "https://api.company.ghe.com/",
},
{
name: "preserves port",
input: "http://company.ghe.com:8080",
want: "http://api.company.ghe.com:8080/",
},
{
name: "api with port",
input: "http://api.company.ghe.com:8080",
want: "http://api.company.ghe.com:8080/",
},
{
name: "strips path",
input: "https://company.ghe.com/some/path",
want: "https://api.company.ghe.com/",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := normalizeGHECloudAPIEndpoint(tt.input)
if (err != nil) != tt.wantErr {
t.Errorf("normalizeGHECloudAPIEndpoint(%q) error = %v, wantErr %v", tt.input, err, tt.wantErr)
return
}
if got != tt.want {
t.Errorf("normalizeGHECloudAPIEndpoint(%q) = %q, want %q", tt.input, got, tt.want)
}
})
}
}
func TestCreateAPIClient_GHECloud_NoAPIV3(t *testing.T) {
// The most critical test: GHE.com clients must NOT have /api/v3/ in their BaseURL.
endpoints := []string{
"https://api.company.ghe.com",
"https://company.ghe.com",
"https://api.company.ghe.com/",
"https://company.ghe.com/",
}
for _, endpoint := range endpoints {
t.Run(endpoint, func(t *testing.T) {
client, err := createGHECloudClient(http.DefaultClient, endpoint)
if err != nil {
t.Fatalf("createGHECloudClient(%q) unexpected error: %v", endpoint, err)
}
baseURL := client.BaseURL.String()
// Must NOT contain /api/v3
if strings.Contains(baseURL, "/api/v3") {
t.Errorf("GHE.com client BaseURL must not contain /api/v3/, got: %s", baseURL)
}
// Must point to api.company.ghe.com
if client.BaseURL.Hostname() != "api.company.ghe.com" {
t.Errorf("expected hostname api.company.ghe.com, got: %s", client.BaseURL.Hostname())
}
// Must have trailing slash
if !strings.HasSuffix(baseURL, "/") {
t.Errorf("BaseURL must have trailing slash, got: %s", baseURL)
}
})
}
}
func TestCreateGraphqlClient_GHECloud(t *testing.T) {
// GHE.com GraphQL should be at /graphql, NOT /api/graphql.
// We can't easily inspect the URL from the githubv4.Client,
// so we test the URL construction logic indirectly by checking
// the normalized endpoint.
apiURL, err := normalizeGHECloudAPIEndpoint("https://company.ghe.com")
if err != nil {
t.Fatal(err)
}
graphqlURL := strings.TrimRight(apiURL, "/") + "/graphql"
expected := "https://api.company.ghe.com/graphql"
if graphqlURL != expected {
t.Errorf("GHE.com GraphQL URL = %q, want %q", graphqlURL, expected)
}
}
func TestCreateAPIClient_GHES_HasAPIV3(t *testing.T) {
// GHES clients SHOULD have /api/v3/ — make sure we didn't break that.
// Note: WithEnterpriseURLs returns an error for invalid URLs, but for
// valid ones it appends /api/v3/.
client, err := createAPIClient(context.Background(), http.DefaultClient, "https://github.mycompany.com")
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if !strings.Contains(client.BaseURL.String(), "/api/v3") {
t.Errorf("GHES client BaseURL should contain /api/v3/, got: %s", client.BaseURL.String())
}
}
func TestCreateAPIClient_CloudGitHub(t *testing.T) {
// Regular github.com should use the default client (api.github.com).
client, err := createAPIClient(context.Background(), http.DefaultClient, cloudV3Endpoint)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if client.BaseURL.String() != "https://api.github.com/" {
t.Errorf("expected https://api.github.com/, got: %s", client.BaseURL.String())
}
}
Reference in New Issue View Git Blame Copy Permalink