mirror of
https://github.com/trufflesecurity/trufflehog.git
synced 2024-09-20 14:42:03 +00:00
[oc-313] - Add GitHub metrics (#1324)
* Normalize repos during enumeration. * fix test. * Add benchmark. * Add benchmark. * Add more realistic benchmark values. * add gist mocks. * Remove old normalize fxn. * abstract away the repo cache. * update test. * increase repo count. * increase page limnit to 100. * move callee fxns below caller for Chunks. * Add context to normalize. * remove extra logic in normalize repo. * Delete new.txt * Delete old.txt * Handle errors in a thread safe manner. * fix test.' * fix test. * handle repos that are included by users. * Abstract include ignore logic within repoCache. * Add better comment around repoCache. * Rename params. * remove commented out code. * use repos instead of items. * remove commented out code. * Use ++ instead of atomic increment. * update to use logger var. * use cache pkg. * Use separate file for repo logic. * Address comments. * fix test. * make less sucky test. * Update test. * Add logs for duration and repo size. * fix integration test. * address comment.
This commit is contained in:
ahrav
GitHub
parent
88b4a283c4
commit
31844b12e3
4 changed files with 284 additions and 229 deletions
|
|
@ -29,7 +29,6 @@ import (
|
|||
"github.com/trufflesecurity/trufflehog/v3/pkg/cache/memory"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/giturl"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/credentialspb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
|
||||
|
|
@ -56,6 +55,8 @@ type Source struct {
|
|||
orgsCache cache.Cache
|
||||
filteredRepoCache *filteredRepoCache
|
||||
memberCache map[string]struct{}
|
||||
repoSizes repoSize
|
||||
totalRepoSize int // total size in bytes of all repos
|
||||
git *git.Git
|
||||
scanOptions *git.ScanOptions
|
||||
httpClient *http.Client
|
||||
|
|
@ -92,44 +93,25 @@ func (s *Source) JobID() int64 {
|
|||
return s.jobID
|
||||
}
|
||||
|
||||
func (s *Source) UserAndToken(ctx context.Context, installationClient *github.Client) (string, string, error) {
|
||||
switch cred := s.conn.GetCredential().(type) {
|
||||
case *sourcespb.GitHub_BasicAuth:
|
||||
return cred.BasicAuth.Username, cred.BasicAuth.Password, nil
|
||||
case *sourcespb.GitHub_Unauthenticated:
|
||||
// do nothing
|
||||
case *sourcespb.GitHub_GithubApp:
|
||||
id, err := strconv.ParseInt(cred.GithubApp.InstallationId, 10, 64)
|
||||
if err != nil {
|
||||
return "", "", errors.New(err)
|
||||
}
|
||||
// TODO: Check rate limit for this call.
|
||||
token, _, err := installationClient.Apps.CreateInstallationToken(
|
||||
ctx, id, &github.InstallationTokenOptions{})
|
||||
if err != nil {
|
||||
return "", "", errors.WrapPrefix(err, "unable to create installation token", 0)
|
||||
}
|
||||
return "x-access-token", token.GetToken(), nil // TODO: multiple workers request this, track the TTL
|
||||
case *sourcespb.GitHub_Token:
|
||||
var (
|
||||
ghUser *github.User
|
||||
resp *github.Response
|
||||
err error
|
||||
)
|
||||
for {
|
||||
ghUser, resp, err = s.apiClient.Users.Get(context.TODO(), "")
|
||||
if handled := s.handleRateLimit(err, resp); handled {
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return "", "", errors.New(err)
|
||||
}
|
||||
break
|
||||
}
|
||||
return ghUser.GetLogin(), cred.Token, nil
|
||||
}
|
||||
type repoSize struct {
|
||||
mu sync.RWMutex
|
||||
repoSizes map[string]int // size in bytes of each repo
|
||||
}
|
||||
|
||||
return "", "", errors.New("unhandled credential type for token fetch")
|
||||
func (r *repoSize) addRepo(repo string, size int) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
r.repoSizes[repo] = size
|
||||
}
|
||||
|
||||
func (r *repoSize) getRepo(repo string) int {
|
||||
r.mu.RLock()
|
||||
defer r.mu.RUnlock()
|
||||
return r.repoSizes[repo]
|
||||
}
|
||||
|
||||
func newRepoSize() repoSize {
|
||||
return repoSize{repoSizes: make(map[string]int)}
|
||||
}
|
||||
|
||||
// filteredRepoCache is a wrapper around cache.Cache that filters out repos
|
||||
|
|
@ -217,6 +199,7 @@ func (s *Source) Init(aCtx context.Context, name string, jobID, sourceID int64,
|
|||
s.filteredRepoCache = s.newFilteredRepoCache(memory.New(), s.conn.IncludeRepos, s.conn.IgnoreRepos)
|
||||
s.memberCache = make(map[string]struct{})
|
||||
|
||||
s.repoSizes = newRepoSize()
|
||||
s.repos = s.conn.Repositories
|
||||
for _, repo := range s.repos {
|
||||
r, err := s.normalizeRepo(repo)
|
||||
|
|
@ -353,8 +336,10 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
|
|||
}
|
||||
|
||||
func (s *Source) enumerate(ctx context.Context, apiEndpoint string) (*github.Client, error) {
|
||||
var installationClient *github.Client
|
||||
var err error
|
||||
var (
|
||||
installationClient *github.Client
|
||||
err error
|
||||
)
|
||||
|
||||
switch cred := s.conn.GetCredential().(type) {
|
||||
case *sourcespb.GitHub_BasicAuth:
|
||||
|
|
@ -584,7 +569,7 @@ func (s *Source) enumerateWithApp(ctx context.Context, apiEndpoint string, app *
|
|||
|
||||
// If no repos were provided, enumerate them.
|
||||
if len(s.repos) == 0 {
|
||||
if err = s.addReposByApp(ctx); err != nil {
|
||||
if err = s.getReposByApp(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
|
|
@ -666,13 +651,18 @@ func (s *Source) scan(ctx context.Context, installationClient *github.Client, ch
|
|||
s.scanOptions.BaseHash = s.conn.Base
|
||||
s.scanOptions.HeadHash = s.conn.Head
|
||||
|
||||
logger.V(2).Info(fmt.Sprintf("scanning repo %d/%d", i, len(s.repos)))
|
||||
repoSize := s.repoSizes.getRepo(repoURL)
|
||||
logger.V(2).Info(fmt.Sprintf("scanning repo %d/%d", i, len(s.repos)), "repo_size", repoSize)
|
||||
|
||||
now := time.Now()
|
||||
defer func(start time.Time) {
|
||||
logger.V(2).Info(fmt.Sprintf("scanned %d/%d repos", scanned, len(s.repos)), "repo_size", repoSize, "duration_seconds", time.Since(start).Seconds())
|
||||
}(now)
|
||||
if err = s.git.ScanRepo(ctx, repo, path, s.scanOptions, chunksChan); err != nil {
|
||||
scanErrs.Add(fmt.Errorf("error scanning repo %s: %w", repoURL, err))
|
||||
return nil
|
||||
}
|
||||
atomic.AddUint64(&scanned, 1)
|
||||
logger.V(2).Info(fmt.Sprintf("scanned %d/%d repos", scanned, len(s.repos)))
|
||||
|
||||
return nil
|
||||
})
|
||||
|
|
@ -687,49 +677,6 @@ func (s *Source) scan(ctx context.Context, installationClient *github.Client, ch
|
|||
return nil
|
||||
}
|
||||
|
||||
func (s *Source) cloneRepo(ctx context.Context, repoURL string, installationClient *github.Client) (string, *gogit.Repository, error) {
|
||||
var path string
|
||||
var repo *gogit.Repository
|
||||
var err error
|
||||
|
||||
switch s.conn.GetCredential().(type) {
|
||||
case *sourcespb.GitHub_BasicAuth:
|
||||
path, repo, err = git.CloneRepoUsingToken(ctx, s.conn.GetBasicAuth().GetPassword(), repoURL, s.conn.GetBasicAuth().GetUsername())
|
||||
if err != nil {
|
||||
return "", nil, fmt.Errorf("error cloning repo %s: %w", repoURL, err)
|
||||
}
|
||||
case *sourcespb.GitHub_Unauthenticated:
|
||||
path, repo, err = git.CloneRepoUsingUnauthenticated(ctx, repoURL)
|
||||
if err != nil {
|
||||
return "", nil, fmt.Errorf("error cloning repo %s: %w", repoURL, err)
|
||||
}
|
||||
|
||||
case *sourcespb.GitHub_GithubApp:
|
||||
s.githubUser, s.githubToken, err = s.UserAndToken(ctx, installationClient)
|
||||
if err != nil {
|
||||
return "", nil, fmt.Errorf("error getting token for repo %s: %w", repoURL, err)
|
||||
}
|
||||
|
||||
path, repo, err = git.CloneRepoUsingToken(ctx, s.githubToken, repoURL, s.githubUser)
|
||||
if err != nil {
|
||||
return "", nil, fmt.Errorf("error cloning repo %s: %w", repoURL, err)
|
||||
}
|
||||
case *sourcespb.GitHub_Token:
|
||||
// We never refresh user provided tokens, so if we already have them, we never need to try and fetch them again.
|
||||
if s.githubUser == "" || s.githubToken == "" {
|
||||
s.githubUser, s.githubToken, err = s.UserAndToken(ctx, installationClient)
|
||||
if err != nil {
|
||||
return "", nil, fmt.Errorf("error getting token for repo %s: %w", repoURL, err)
|
||||
}
|
||||
}
|
||||
path, repo, err = git.CloneRepoUsingToken(ctx, s.githubToken, repoURL, s.githubUser)
|
||||
if err != nil {
|
||||
return "", nil, fmt.Errorf("error cloning repo %s: %w", repoURL, err)
|
||||
}
|
||||
}
|
||||
return path, repo, nil
|
||||
}
|
||||
|
||||
// handleRateLimit returns true if a rate limit was handled
|
||||
// Unauthenticated access to most github endpoints has a rate limit of 60 requests per hour.
|
||||
// This will likely only be exhausted if many users/orgs are scanned without auth
|
||||
|
|
@ -766,90 +713,16 @@ func (s *Source) handleRateLimit(errIn error, res *github.Response) bool {
|
|||
return true
|
||||
}
|
||||
|
||||
func (s *Source) getReposByOrg(ctx context.Context, org string) error {
|
||||
logger := s.log.WithValues("org", org)
|
||||
|
||||
opts := &github.RepositoryListByOrgOptions{
|
||||
ListOptions: github.ListOptions{
|
||||
PerPage: defaultPagination,
|
||||
},
|
||||
func (s *Source) addReposForMembers(ctx context.Context) {
|
||||
s.log.Info("Fetching repos from members", "members", len(s.members))
|
||||
for member := range s.memberCache {
|
||||
if err := s.addUserGistsToCache(ctx, member); err != nil {
|
||||
s.log.Info("Unable to fetch gists by user", "user", member, "error", err)
|
||||
}
|
||||
if err := s.getReposByUser(ctx, member); err != nil {
|
||||
s.log.Info("Unable to fetch repos by user", "user", member, "error", err)
|
||||
}
|
||||
}
|
||||
|
||||
var numRepos, numForks int
|
||||
for {
|
||||
someRepos, res, err := s.apiClient.Repositories.ListByOrg(ctx, org, opts)
|
||||
if err == nil {
|
||||
res.Body.Close()
|
||||
}
|
||||
if handled := s.handleRateLimit(err, res); handled {
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not list repos for org %s: %w", org, err)
|
||||
}
|
||||
if len(someRepos) == 0 || res == nil {
|
||||
break
|
||||
}
|
||||
|
||||
logger.V(2).Info("Listed repos", "page", opts.Page, "last_page", res.LastPage)
|
||||
for _, r := range someRepos {
|
||||
if r.GetFork() {
|
||||
if !s.conn.IncludeForks {
|
||||
continue
|
||||
}
|
||||
numForks++
|
||||
}
|
||||
s.filteredRepoCache.Set(r.GetFullName(), r.GetCloneURL())
|
||||
numRepos++
|
||||
}
|
||||
if res.NextPage == 0 {
|
||||
break
|
||||
}
|
||||
opts.Page = res.NextPage
|
||||
}
|
||||
|
||||
logger.V(2).Info("found repos", "total", numRepos, "forks", numForks)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Source) getReposByUser(ctx context.Context, user string) error {
|
||||
opts := &github.RepositoryListOptions{
|
||||
ListOptions: github.ListOptions{
|
||||
PerPage: defaultPagination,
|
||||
},
|
||||
}
|
||||
|
||||
logger := s.log.WithValues("user", user)
|
||||
for {
|
||||
someRepos, res, err := s.apiClient.Repositories.List(ctx, user, opts)
|
||||
if err == nil {
|
||||
res.Body.Close()
|
||||
}
|
||||
if handled := s.handleRateLimit(err, res); handled {
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not list repos for user %s: %w", user, err)
|
||||
}
|
||||
if res == nil {
|
||||
break
|
||||
}
|
||||
|
||||
logger.V(2).Info("Listed repos", "page", opts.Page, "last_page", res.LastPage)
|
||||
for _, r := range someRepos {
|
||||
if r.GetFork() && !s.conn.IncludeForks {
|
||||
continue
|
||||
}
|
||||
|
||||
s.filteredRepoCache.Set(r.GetFullName(), r.GetCloneURL())
|
||||
}
|
||||
if res.NextPage == 0 {
|
||||
break
|
||||
}
|
||||
opts.Page = res.NextPage
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// addUserGistsToCache collects all the gist urls for a given user,
|
||||
|
|
@ -903,43 +776,6 @@ func (s *Source) addMembersByApp(ctx context.Context, installationClient *github
|
|||
return nil
|
||||
}
|
||||
|
||||
func (s *Source) addReposByApp(ctx context.Context) error {
|
||||
// Authenticated enumeration of repos.
|
||||
opts := &github.ListOptions{
|
||||
PerPage: defaultPagination,
|
||||
}
|
||||
|
||||
for {
|
||||
someRepos, res, err := s.apiClient.Apps.ListRepos(ctx, opts)
|
||||
if err == nil {
|
||||
res.Body.Close()
|
||||
}
|
||||
if handled := s.handleRateLimit(err, res); handled {
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return errors.WrapPrefix(err, "unable to list repositories", 0)
|
||||
}
|
||||
if res == nil {
|
||||
break
|
||||
}
|
||||
s.log.V(2).Info("Listed repos for app", "page", opts.Page, "last_page", res.LastPage)
|
||||
for _, r := range someRepos.Repositories {
|
||||
if r.GetFork() && !s.conn.IncludeForks {
|
||||
continue
|
||||
}
|
||||
s.filteredRepoCache.Set(r.GetFullName(), r.GetCloneURL())
|
||||
s.log.V(2).Info("Enumerated repo", "repo", r.GetFullName())
|
||||
}
|
||||
|
||||
if res.NextPage == 0 {
|
||||
break
|
||||
}
|
||||
opts.Page = res.NextPage
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Source) addAllVisibleOrgs(ctx context.Context) {
|
||||
s.log.V(2).Info("enumerating all visible organizations on GHE")
|
||||
// Enumeration on this endpoint does not use pages it uses a since ID.
|
||||
|
|
@ -1061,27 +897,6 @@ func (s *Source) addMembersByOrg(ctx context.Context, org string) error {
|
|||
return nil
|
||||
}
|
||||
|
||||
func (s *Source) addReposForMembers(ctx context.Context) {
|
||||
s.log.Info("Fetching repos from members", "members", len(s.members))
|
||||
for member := range s.memberCache {
|
||||
if err := s.addUserGistsToCache(ctx, member); err != nil {
|
||||
s.log.Info("Unable to fetch gists by user", "user", member, "error", err)
|
||||
}
|
||||
if err := s.getReposByUser(ctx, member); err != nil {
|
||||
s.log.Info("Unable to fetch repos by user", "user", member, "error", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Source) normalizeRepo(repo string) (string, error) {
|
||||
// If there's a '/', assume it's a URL and try to normalize it.
|
||||
if strings.ContainsRune(repo, '/') {
|
||||
return giturl.NormalizeGithubRepo(repo)
|
||||
}
|
||||
|
||||
return "", fmt.Errorf("no repositories found for %s", repo)
|
||||
}
|
||||
|
||||
// setProgressCompleteWithRepo calls the s.SetProgressComplete after safely setting up the encoded resume info string.
|
||||
func (s *Source) setProgressCompleteWithRepo(index int, offset int, repoURL string) {
|
||||
s.resumeInfoMutex.Lock()
|
||||
|
|
|
|||
|
|
@ -57,13 +57,14 @@ func TestSource_Token(t *testing.T) {
|
|||
httpClient: common.SaneHttpClient(),
|
||||
log: logr.Discard(),
|
||||
memberCache: map[string]struct{}{},
|
||||
repoSizes: newRepoSize(),
|
||||
}
|
||||
s.filteredRepoCache = s.newFilteredRepoCache(memory.New(), nil, nil)
|
||||
|
||||
installationClient, err := s.enumerateWithApp(ctx, "https://api.github.com", conn.GetGithubApp())
|
||||
assert.NoError(t, err)
|
||||
|
||||
user, token, err := s.UserAndToken(ctx, installationClient)
|
||||
user, token, err := s.userAndToken(ctx, installationClient)
|
||||
assert.NotEmpty(t, token)
|
||||
assert.NoError(t, err)
|
||||
|
||||
|
|
|
|||
|
|
@ -226,7 +226,7 @@ func TestAddReposByApp(t *testing.T) {
|
|||
})
|
||||
|
||||
s := initTestSource(nil)
|
||||
err := s.addReposByApp(context.TODO())
|
||||
err := s.getReposByApp(context.Background())
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, 2, s.filteredRepoCache.Count())
|
||||
ok := s.filteredRepoCache.Exists("ssr1")
|
||||
|
|
|
|||
239
pkg/sources/github/repo.go
Normal file
239
pkg/sources/github/repo.go
Normal file
|
|
@ -0,0 +1,239 @@
|
|||
package github
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
gogit "github.com/go-git/go-git/v5"
|
||||
"github.com/google/go-github/v42/github"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/giturl"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources/git"
|
||||
)
|
||||
|
||||
func (s *Source) cloneRepo(
|
||||
ctx context.Context,
|
||||
repoURL string,
|
||||
installationClient *github.Client,
|
||||
) (string, *gogit.Repository, error) {
|
||||
var (
|
||||
path string
|
||||
repo *gogit.Repository
|
||||
err error
|
||||
)
|
||||
|
||||
switch s.conn.GetCredential().(type) {
|
||||
case *sourcespb.GitHub_BasicAuth:
|
||||
path, repo, err = git.CloneRepoUsingToken(ctx, s.conn.GetBasicAuth().GetPassword(), repoURL, s.conn.GetBasicAuth().GetUsername())
|
||||
if err != nil {
|
||||
return "", nil, fmt.Errorf("error cloning repo %s: %w", repoURL, err)
|
||||
}
|
||||
case *sourcespb.GitHub_Unauthenticated:
|
||||
path, repo, err = git.CloneRepoUsingUnauthenticated(ctx, repoURL)
|
||||
if err != nil {
|
||||
return "", nil, fmt.Errorf("error cloning repo %s: %w", repoURL, err)
|
||||
}
|
||||
|
||||
case *sourcespb.GitHub_GithubApp:
|
||||
s.githubUser, s.githubToken, err = s.userAndToken(ctx, installationClient)
|
||||
if err != nil {
|
||||
return "", nil, fmt.Errorf("error getting token for repo %s: %w", repoURL, err)
|
||||
}
|
||||
|
||||
path, repo, err = git.CloneRepoUsingToken(ctx, s.githubToken, repoURL, s.githubUser)
|
||||
if err != nil {
|
||||
return "", nil, fmt.Errorf("error cloning repo %s: %w", repoURL, err)
|
||||
}
|
||||
|
||||
case *sourcespb.GitHub_Token:
|
||||
// We never refresh user provided tokens, so if we already have them, we never need to try and fetch them again.
|
||||
if s.githubUser == "" || s.githubToken == "" {
|
||||
s.githubUser, s.githubToken, err = s.userAndToken(ctx, installationClient)
|
||||
if err != nil {
|
||||
return "", nil, fmt.Errorf("error getting token for repo %s: %w", repoURL, err)
|
||||
}
|
||||
}
|
||||
path, repo, err = git.CloneRepoUsingToken(ctx, s.githubToken, repoURL, s.githubUser)
|
||||
if err != nil {
|
||||
return "", nil, fmt.Errorf("error cloning repo %s: %w", repoURL, err)
|
||||
}
|
||||
default:
|
||||
return "", nil, fmt.Errorf("unhandled credential type for repo %s", repoURL)
|
||||
}
|
||||
return path, repo, nil
|
||||
}
|
||||
|
||||
func (s *Source) userAndToken(ctx context.Context, installationClient *github.Client) (string, string, error) {
|
||||
switch cred := s.conn.GetCredential().(type) {
|
||||
case *sourcespb.GitHub_BasicAuth:
|
||||
return cred.BasicAuth.Username, cred.BasicAuth.Password, nil
|
||||
case *sourcespb.GitHub_Unauthenticated:
|
||||
// do nothing
|
||||
case *sourcespb.GitHub_GithubApp:
|
||||
id, err := strconv.ParseInt(cred.GithubApp.InstallationId, 10, 64)
|
||||
if err != nil {
|
||||
return "", "", fmt.Errorf("unable to parse installation id: %w", err)
|
||||
}
|
||||
// TODO: Check rate limit for this call.
|
||||
token, _, err := installationClient.Apps.CreateInstallationToken(
|
||||
ctx, id, &github.InstallationTokenOptions{})
|
||||
if err != nil {
|
||||
return "", "", fmt.Errorf("unable to create installation token: %w", err)
|
||||
}
|
||||
return "x-access-token", token.GetToken(), nil // TODO: multiple workers request this, track the TTL
|
||||
case *sourcespb.GitHub_Token:
|
||||
var (
|
||||
ghUser *github.User
|
||||
resp *github.Response
|
||||
err error
|
||||
)
|
||||
for {
|
||||
ghUser, resp, err = s.apiClient.Users.Get(ctx, "")
|
||||
if handled := s.handleRateLimit(err, resp); handled {
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return "", "", fmt.Errorf("unable to get user: %w", err)
|
||||
}
|
||||
break
|
||||
}
|
||||
return ghUser.GetLogin(), cred.Token, nil
|
||||
default:
|
||||
return "", "", fmt.Errorf("unhandled credential type")
|
||||
}
|
||||
|
||||
return "", "", fmt.Errorf("unhandled credential type")
|
||||
}
|
||||
|
||||
type repoListOptions interface {
|
||||
getListOptions() *github.ListOptions
|
||||
}
|
||||
|
||||
type repoLister func(ctx context.Context, target string, opts repoListOptions) ([]*github.Repository, *github.Response, error)
|
||||
|
||||
type appListOptions struct {
|
||||
github.ListOptions
|
||||
}
|
||||
|
||||
func (a *appListOptions) getListOptions() *github.ListOptions {
|
||||
return &a.ListOptions
|
||||
}
|
||||
|
||||
func (s *Source) getReposByApp(ctx context.Context) error {
|
||||
return s.processRepos(ctx, "", s.appListReposWrapper, &appListOptions{
|
||||
ListOptions: github.ListOptions{
|
||||
PerPage: defaultPagination,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
func (s *Source) appListReposWrapper(ctx context.Context, _ string, opts repoListOptions) ([]*github.Repository, *github.Response, error) {
|
||||
someRepos, res, err := s.apiClient.Apps.ListRepos(ctx, opts.getListOptions())
|
||||
if someRepos != nil {
|
||||
return someRepos.Repositories, res, err
|
||||
}
|
||||
return nil, res, err
|
||||
}
|
||||
|
||||
type userListOptions struct {
|
||||
github.RepositoryListOptions
|
||||
}
|
||||
|
||||
func (u *userListOptions) getListOptions() *github.ListOptions {
|
||||
return &u.ListOptions
|
||||
}
|
||||
|
||||
func (s *Source) getReposByUser(ctx context.Context, user string) error {
|
||||
return s.processRepos(ctx, user, s.userListReposWrapper, &userListOptions{
|
||||
RepositoryListOptions: github.RepositoryListOptions{
|
||||
ListOptions: github.ListOptions{
|
||||
PerPage: defaultPagination,
|
||||
},
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
func (s *Source) userListReposWrapper(ctx context.Context, user string, opts repoListOptions) ([]*github.Repository, *github.Response, error) {
|
||||
return s.apiClient.Repositories.List(ctx, user, &opts.(*userListOptions).RepositoryListOptions)
|
||||
}
|
||||
|
||||
type orgListOptions struct {
|
||||
github.RepositoryListByOrgOptions
|
||||
}
|
||||
|
||||
func (o *orgListOptions) getListOptions() *github.ListOptions {
|
||||
return &o.ListOptions
|
||||
}
|
||||
|
||||
func (s *Source) getReposByOrg(ctx context.Context, org string) error {
|
||||
return s.processRepos(ctx, org, s.orgListReposWrapper, &orgListOptions{
|
||||
RepositoryListByOrgOptions: github.RepositoryListByOrgOptions{
|
||||
ListOptions: github.ListOptions{
|
||||
PerPage: defaultPagination,
|
||||
},
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
func (s *Source) orgListReposWrapper(ctx context.Context, org string, opts repoListOptions) ([]*github.Repository, *github.Response, error) {
|
||||
return s.apiClient.Repositories.ListByOrg(ctx, org, &opts.(*orgListOptions).RepositoryListByOrgOptions)
|
||||
}
|
||||
|
||||
func (s *Source) processRepos(ctx context.Context, target string, listRepos repoLister, listOpts repoListOptions) error {
|
||||
logger := s.log.WithValues("target", target)
|
||||
opts := listOpts.getListOptions()
|
||||
|
||||
var (
|
||||
numRepos, numForks int
|
||||
)
|
||||
|
||||
for {
|
||||
someRepos, res, err := listRepos(ctx, target, listOpts)
|
||||
if err == nil {
|
||||
res.Body.Close()
|
||||
}
|
||||
if handled := s.handleRateLimit(err, res); handled {
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if res == nil {
|
||||
break
|
||||
}
|
||||
|
||||
s.log.V(2).Info("Listed repos", "page", opts.Page, "last_page", res.LastPage)
|
||||
for _, r := range someRepos {
|
||||
if r.GetFork() && !s.conn.IncludeForks {
|
||||
continue
|
||||
}
|
||||
numForks++
|
||||
|
||||
repoName, repoURL := r.GetFullName(), r.GetCloneURL()
|
||||
s.repoSizes.addRepo(repoURL, r.GetSize())
|
||||
s.totalRepoSize += r.GetSize()
|
||||
s.filteredRepoCache.Set(repoName, repoURL)
|
||||
logger.V(3).Info("repo attributes", "name", repoName, "size", r.GetSize(), "repo_url", repoURL)
|
||||
}
|
||||
|
||||
if res.NextPage == 0 {
|
||||
break
|
||||
}
|
||||
opts.Page = res.NextPage
|
||||
}
|
||||
logger.V(2).Info("found repos", "total", numRepos, "num_forks", numForks)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Source) normalizeRepo(repo string) (string, error) {
|
||||
// If there's a '/', assume it's a URL and try to normalize it.
|
||||
if strings.ContainsRune(repo, '/') {
|
||||
return giturl.NormalizeGithubRepo(repo)
|
||||
}
|
||||
|
||||
return "", fmt.Errorf("no repositories found for %s", repo)
|
||||
}
|
||||
Loading…
Reference in a new issue