Mirrors/trufflehog
2
0
Fork 0
mirror of https://github.com/trufflesecurity/trufflehog.git synced 2024-11-10 07:04:24 +00:00
Code Issues Projects Releases Packages Wiki Activity

fix(github): scan user repos (#2814)

Browse source
This commit is contained in:
Richard Gomez 2024-05-23 10:40:40 -04:00 committed by GitHub
parent f7214cfee3
commit 1441289d41
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 107 additions and 47 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

74
pkg/sources/github/github.go
View file

@ -15,12 +15,12 @@ import (
"time"
"golang.org/x/exp/rand"
"golang.org/x/oauth2"
"github.com/bradleyfalzon/ghinstallation/v2"
"github.com/go-logr/logr"
"github.com/gobwas/glob"
"github.com/google/go-github/v62/github"
"golang.org/x/oauth2"
"golang.org/x/sync/errgroup"
"google.golang.org/protobuf/proto"
"google.golang.org/protobuf/types/known/anypb"
@ -480,8 +480,17 @@ func (s *Source) enumerateBasicAuth(ctx context.Context, apiEndpoint string, bas
s.apiClient = ghClient
for _, org := range s.orgsCache.Keys() {
if err := s.getReposByOrg(ctx, org); err != nil {
s.log.Error(err, "error fetching repos for org or user")
orgCtx := context.WithValue(ctx, "account", org)
userType, err := s.getReposByOrgOrUser(ctx, org)
if err != nil {
orgCtx.Logger().Error(err, "error fetching repos for org or user")
continue
}
if userType == organization && s.conn.ScanUsers {
if err := s.addMembersByOrg(ctx, org); err != nil {
orgCtx.Logger().Error(err, "Unable to add members by org")
}
}
}
@ -499,17 +508,15 @@ func (s *Source) enumerateUnauthenticated(ctx context.Context, apiEndpoint strin
}
for _, org := range s.orgsCache.Keys() {
if err := s.getReposByOrg(ctx, org); err != nil {
s.log.Error(err, "error fetching repos for org")
orgCtx := context.WithValue(ctx, "account", org)
userType, err := s.getReposByOrgOrUser(ctx, org)
if err != nil {
orgCtx.Logger().Error(err, "error fetching repos for org or user")
continue
}
// We probably don't need to do this, since getting repos by org makes more sense?
if err := s.getReposByUser(ctx, org); err != nil {
s.log.Error(err, "error fetching repos for user")
}
if s.conn.ScanUsers {
s.log.Info("Enumerating unauthenticated does not support scanning organization members")
if userType == organization && s.conn.ScanUsers {
orgCtx.Logger().Info("WARNING: Enumerating unauthenticated does not support scanning organization members (--include-members)")
}
}
}
@ -562,16 +569,16 @@ func (s *Source) enumerateWithToken(ctx context.Context, apiEndpoint, token stri
if s.orgsCache.Count() > 0 {
specificScope = true
for _, org := range s.orgsCache.Keys() {
logger := s.log.WithValues("org", org)
if err := s.getReposByOrg(ctx, org); err != nil {
logger.Error(err, "error fetching repos for org")
orgCtx := context.WithValue(ctx, "account", org)
userType, err := s.getReposByOrgOrUser(ctx, org)
if err != nil {
orgCtx.Logger().Error(err, "error fetching repos for org or user")
continue
}
if s.conn.ScanUsers {
err := s.addMembersByOrg(ctx, org)
if err != nil {
logger.Error(err, "Unable to add members by org")
continue
if userType == organization && s.conn.ScanUsers {
if err := s.addMembersByOrg(ctx, org); err != nil {
orgCtx.Logger().Error(err, "Unable to add members by org")
}
}
}
@ -593,27 +600,28 @@ func (s *Source) enumerateWithToken(ctx context.Context, apiEndpoint, token stri
}
for _, org := range s.orgsCache.Keys() {
logger := s.log.WithValues("org", org)
if err := s.getReposByOrg(ctx, org); err != nil {
logger.Error(err, "error fetching repos by org")
orgCtx := context.WithValue(ctx, "account", org)
userType, err := s.getReposByOrgOrUser(ctx, org)
if err != nil {
orgCtx.Logger().Error(err, "error fetching repos for org or user")
continue
}
if err := s.getReposByUser(ctx, ghUser.GetLogin()); err != nil {
logger.Error(err, "error fetching repos by user")
}
if s.conn.ScanUsers {
err := s.addMembersByOrg(ctx, org)
if err != nil {
logger.Error(err, "Unable to add members by org for org")
if userType == organization && s.conn.ScanUsers {
if err := s.addMembersByOrg(ctx, org); err != nil {
orgCtx.Logger().Error(err, "Unable to add members by org for org")
}
}
}
if err := s.getReposByUser(ctx, ghUser.GetLogin()); err != nil {
s.log.Error(err, "error fetching repos for the current user", "user", ghUser.GetLogin())
}
// If we enabled ScanUsers above, we've already added the gists for the current user and users from the orgs.
// So if we don't have ScanUsers enabled, add the user gists as normal.
if err := s.addUserGistsToCache(ctx, ghUser.GetLogin()); err != nil {
s.log.Error(err, "error fetching gists", "user", ghUser.GetLogin())
s.log.Error(err, "error fetching gists for the current user", "user", ghUser.GetLogin())
}
return nil
@ -693,7 +701,7 @@ func (s *Source) enumerateWithApp(ctx context.Context, apiEndpoint string, app *
s.log.Info("Scanning repos", "org_members", len(s.memberCache))
for member := range s.memberCache {
logger := s.log.WithValues("member", member)
if err := s.getReposByUser(ctx, member); err != nil {
if err := s.addUserGistsToCache(ctx, member); err != nil {
logger.Error(err, "error fetching gists by user")
}
if err := s.getReposByUser(ctx, member); err != nil {

80
pkg/sources/github/repo.go
View file

@ -1,6 +1,7 @@
package github
import (
"errors"
"fmt"
"io"
"net/http"
@ -169,14 +170,6 @@ func (a *appListOptions) getListOptions() *github.ListOptions {
return &a.ListOptions
}
func (s *Source) getReposByApp(ctx context.Context) error {
return s.processRepos(ctx, "", s.appListReposWrapper, &appListOptions{
ListOptions: github.ListOptions{
PerPage: defaultPagination,
},
})
}
func (s *Source) appListReposWrapper(ctx context.Context, _ string, opts repoListOptions) ([]*github.Repository, *github.Response, error) {
someRepos, res, err := s.apiClient.Apps.ListRepos(ctx, opts.getListOptions())
if someRepos != nil {
@ -185,6 +178,14 @@ func (s *Source) appListReposWrapper(ctx context.Context, _ string, opts repoLis
return nil, res, err
}
func (s *Source) getReposByApp(ctx context.Context) error {
return s.processRepos(ctx, "", s.appListReposWrapper, &appListOptions{
ListOptions: github.ListOptions{
PerPage: defaultPagination,
},
})
}
type userListOptions struct {
github.RepositoryListByUserOptions
}
@ -193,6 +194,10 @@ func (u *userListOptions) getListOptions() *github.ListOptions {
return &u.ListOptions
}
func (s *Source) userListReposWrapper(ctx context.Context, user string, opts repoListOptions) ([]*github.Repository, *github.Response, error) {
return s.apiClient.Repositories.ListByUser(ctx, user, &opts.(*userListOptions).RepositoryListByUserOptions)
}
func (s *Source) getReposByUser(ctx context.Context, user string) error {
return s.processRepos(ctx, user, s.userListReposWrapper, &userListOptions{
RepositoryListByUserOptions: github.RepositoryListByUserOptions{
@ -203,10 +208,6 @@ func (s *Source) getReposByUser(ctx context.Context, user string) error {
})
}
func (s *Source) userListReposWrapper(ctx context.Context, user string, opts repoListOptions) ([]*github.Repository, *github.Response, error) {
return s.apiClient.Repositories.ListByUser(ctx, user, &opts.(*userListOptions).RepositoryListByUserOptions)
}
type orgListOptions struct {
github.RepositoryListByOrgOptions
}
@ -215,6 +216,10 @@ func (o *orgListOptions) getListOptions() *github.ListOptions {
return &o.ListOptions
}
func (s *Source) orgListReposWrapper(ctx context.Context, org string, opts repoListOptions) ([]*github.Repository, *github.Response, error) {
return s.apiClient.Repositories.ListByOrg(ctx, org, &opts.(*orgListOptions).RepositoryListByOrgOptions)
}
func (s *Source) getReposByOrg(ctx context.Context, org string) error {
return s.processRepos(ctx, org, s.orgListReposWrapper, &orgListOptions{
RepositoryListByOrgOptions: github.RepositoryListByOrgOptions{
@ -225,8 +230,55 @@ func (s *Source) getReposByOrg(ctx context.Context, org string) error {
})
}
func (s *Source) orgListReposWrapper(ctx context.Context, org string, opts repoListOptions) ([]*github.Repository, *github.Response, error) {
return s.apiClient.Repositories.ListByOrg(ctx, org, &opts.(*orgListOptions).RepositoryListByOrgOptions)
// userType indicates whether an account belongs to a person or organization.
//
// See:
// - https://docs.github.com/en/get-started/learning-about-github/types-of-github-accounts
// - https://docs.github.com/en/rest/users/users?apiVersion=2022-11-28#get-a-user
type userType int
const (
// Default invalid state.
unknown userType = iota
// The account is a person (https://docs.github.com/en/rest/users/users).
user
// The account is an organization (https://docs.github.com/en/rest/orgs/orgs).
organization
)
func (s *Source) getReposByOrgOrUser(ctx context.Context, name string) (userType, error) {
var err error
// List repositories for the organization |name|.
err = s.getReposByOrg(ctx, name)
if err == nil {
return organization, nil
} else if !isGitHub404Error(err) {
return unknown, err
}
// List repositories for the user |name|.
err = s.getReposByUser(ctx, name)
if err == nil {
if err := s.addUserGistsToCache(ctx, name); err != nil {
ctx.Logger().Error(err, "Unable to add user to cache")
}
return user, nil
} else if !isGitHub404Error(err) {
return unknown, err
}
return unknown, fmt.Errorf("account '%s' not found", name)
}
// isGitHub404Error returns true if |err| is a `github.ErrorResponse` and has the status code `404`.
func isGitHub404Error(err error) bool {
var ghErr *github.ErrorResponse
if !errors.As(err, &ghErr) {
return false
}
return ghErr.Response.StatusCode == http.StatusNotFound
}
func (s *Source) processRepos(ctx context.Context, target string, listRepos repoLister, listOpts repoListOptions) error {