Mirrors/trufflehog
2
0
Fork 0
mirror of https://github.com/trufflesecurity/trufflehog.git synced 2024-11-10 07:04:24 +00:00
Code Issues Projects Releases Packages Wiki Activity

Auth GitHub in Init (#3131)

Browse source 
The GitHub source currently applies its authentication configuration as the first step of enumeration. This is incompatible with both targeted scans and scan job reports, and also means that authentication logic has to be duplicated into the validation flow. This PR moves it into Init so that it's available to targeted scans and, eventually, unit-specific scans. This also allows us to remove the copy of the old logic that was in Validate.

As part of the work I've also cleaned up the integration test suite. (Several of them were apparently disabled back when they ran on every push, but now that we're not doing that, we can re-enable them.)
This commit is contained in:
Cody Rose 2024-08-05 15:13:29 -04:00 committed by GitHub
parent c2e5506b95
commit f26b502c2e
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
9 changed files with 568 additions and 607 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

39
pkg/sources/github/connector.go Normal file
View file

@ -0,0 +1,39 @@
package github
import (
"fmt"
gogit "github.com/go-git/go-git/v5"
"github.com/google/go-github/v63/github"
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
)
const cloudEndpoint = "https://api.github.com"
type connector interface {
// APIClient returns a configured GitHub client that can be used for GitHub API operations.
APIClient() *github.Client
// Clone clones a repository using the configured authentication information.
Clone(ctx context.Context, repoURL string) (string, *gogit.Repository, error)
}
func newConnector(source *Source) (connector, error) {
apiEndpoint := source.conn.Endpoint
if apiEndpoint == "" || endsWithGithub.MatchString(apiEndpoint) {
apiEndpoint = cloudEndpoint
}
switch cred := source.conn.GetCredential().(type) {
case *sourcespb.GitHub_GithubApp:
return newAppConnector(apiEndpoint, cred.GithubApp)
case *sourcespb.GitHub_BasicAuth:
return newBasicAuthConnector(apiEndpoint, cred.BasicAuth)
case *sourcespb.GitHub_Token:
return newTokenConnector(apiEndpoint, cred.Token, source.handleRateLimit)
case *sourcespb.GitHub_Unauthenticated:
return newUnauthenticatedConnector(apiEndpoint)
default:
return nil, fmt.Errorf("unknown connection type")
}
}

96
pkg/sources/github/connector_app.go Normal file
View file

@ -0,0 +1,96 @@
package github
import (
"fmt"
"strconv"
"github.com/bradleyfalzon/ghinstallation/v2"
gogit "github.com/go-git/go-git/v5"
"github.com/google/go-github/v63/github"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/credentialspb"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources/git"
)
type appConnector struct {
apiClient *github.Client
installationClient *github.Client
installationID int64
}
var _ connector = (*appConnector)(nil)
func newAppConnector(apiEndpoint string, app *credentialspb.GitHubApp) (*appConnector, error) {
installationID, err := strconv.ParseInt(app.InstallationId, 10, 64)
if err != nil {
return nil, fmt.Errorf("could not parse app installation ID %q: %w", app.InstallationId, err)
}
appID, err := strconv.ParseInt(app.AppId, 10, 64)
if err != nil {
return nil, fmt.Errorf("could not parse app ID %q: %w", appID, err)
}
const httpTimeoutSeconds = 60
httpClient := common.RetryableHTTPClientTimeout(int64(httpTimeoutSeconds))
installationTransport, err := ghinstallation.NewAppsTransport(
httpClient.Transport,
appID,
[]byte(app.PrivateKey))
if err != nil {
return nil, fmt.Errorf("could not create installation client transport: %w", err)
}
installationTransport.BaseURL = apiEndpoint
installationHttpClient := common.RetryableHTTPClientTimeout(60)
installationHttpClient.Transport = installationTransport
installationClient, err := github.NewClient(installationHttpClient).WithEnterpriseURLs(apiEndpoint, apiEndpoint)
if err != nil {
return nil, fmt.Errorf("could not create installation client: %w", err)
}
apiTransport, err := ghinstallation.New(
httpClient.Transport,
appID,
installationID,
[]byte(app.PrivateKey))
if err != nil {
return nil, fmt.Errorf("could not create API client transport: %w", err)
}
apiTransport.BaseURL = apiEndpoint
httpClient.Transport = apiTransport
apiClient, err := github.NewClient(httpClient).WithEnterpriseURLs(apiEndpoint, apiEndpoint)
if err != nil {
return nil, fmt.Errorf("could not create API client: %w", err)
}
return &appConnector{
apiClient: apiClient,
installationClient: installationClient,
installationID: installationID,
}, nil
}
func (c *appConnector) APIClient() *github.Client {
return c.apiClient
}
func (c *appConnector) Clone(ctx context.Context, repoURL string) (string, *gogit.Repository, error) {
// TODO: Check rate limit for this call.
token, _, err := c.installationClient.Apps.CreateInstallationToken(
ctx,
c.installationID,
&github.InstallationTokenOptions{})
if err != nil {
return "", nil, fmt.Errorf("could not create installation token: %w", err)
}
return git.CloneRepoUsingToken(ctx, token.GetToken(), repoURL, "x-access-token")
}
func (c *appConnector) InstallationClient() *github.Client {
return c.installationClient
}

48
pkg/sources/github/connector_basicauth.go Normal file
View file

@ -0,0 +1,48 @@
package github
import (
"fmt"
gogit "github.com/go-git/go-git/v5"
"github.com/google/go-github/v63/github"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/credentialspb"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources/git"
)
type basicAuthConnector struct {
apiClient *github.Client
username string
password string
}
var _ connector = (*basicAuthConnector)(nil)
func newBasicAuthConnector(apiEndpoint string, cred *credentialspb.BasicAuth) (*basicAuthConnector, error) {
const httpTimeoutSeconds = 60
httpClient := common.RetryableHTTPClientTimeout(int64(httpTimeoutSeconds))
httpClient.Transport = &github.BasicAuthTransport{
Username: cred.Username,
Password: cred.Password,
}
apiClient, err := createGitHubClient(httpClient, apiEndpoint)
if err != nil {
return nil, fmt.Errorf("could not create API client: %w", err)
}
return &basicAuthConnector{
apiClient: apiClient,
username: cred.Username,
password: cred.Password,
}, nil
}
func (c *basicAuthConnector) APIClient() *github.Client {
return c.apiClient
}
func (c *basicAuthConnector) Clone(ctx context.Context, repoURL string) (string, *gogit.Repository, error) {
return git.CloneRepoUsingToken(ctx, c.password, repoURL, c.username)
}

97
pkg/sources/github/connector_token.go Normal file
View file

@ -0,0 +1,97 @@
package github
import (
"fmt"
"strings"
"sync"
gogit "github.com/go-git/go-git/v5"
"github.com/google/go-github/v63/github"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources/git"
"golang.org/x/oauth2"
)
type tokenConnector struct {
apiClient *github.Client
token string
isGitHubEnterprise bool
handleRateLimit func(error) bool
user string
userMu sync.Mutex
}
var _ connector = (*tokenConnector)(nil)
func newTokenConnector(apiEndpoint string, token string, handleRateLimit func(error) bool) (*tokenConnector, error) {
const httpTimeoutSeconds = 60
httpClient := common.RetryableHTTPClientTimeout(int64(httpTimeoutSeconds))
tokenSource := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: token})
httpClient.Transport = &oauth2.Transport{
Base: httpClient.Transport,
Source: oauth2.ReuseTokenSource(nil, tokenSource),
}
apiClient, err := createGitHubClient(httpClient, apiEndpoint)
if err != nil {
return nil, fmt.Errorf("could not create API client: %w", err)
}
return &tokenConnector{
apiClient: apiClient,
token: token,
isGitHubEnterprise: !strings.EqualFold(apiEndpoint, cloudEndpoint),
handleRateLimit: handleRateLimit,
}, nil
}
func (c *tokenConnector) APIClient() *github.Client {
return c.apiClient
}
func (c *tokenConnector) Clone(ctx context.Context, repoURL string) (string, *gogit.Repository, error) {
if err := c.setUserIfUnset(ctx); err != nil {
return "", nil, err
}
return git.CloneRepoUsingToken(ctx, c.token, repoURL, c.user)
}
func (c *tokenConnector) IsGithubEnterprise() bool {
return c.isGitHubEnterprise
}
func (c *tokenConnector) getUser(ctx context.Context) (string, error) {
var (
user *github.User
err error
)
for {
user, _, err = c.apiClient.Users.Get(ctx, "")
if c.handleRateLimit(err) {
continue
}
if err != nil {
return "", fmt.Errorf("could not get GitHub user: %w", err)
}
break
}
return user.GetLogin(), nil
}
func (c *tokenConnector) setUserIfUnset(ctx context.Context) error {
c.userMu.Lock()
defer c.userMu.Unlock()
if c.user != "" {
return nil
}
user, err := c.getUser(ctx)
if err != nil {
return err
}
c.user = user
return nil
}

37
pkg/sources/github/connector_unauthenticated.go Normal file
View file

@ -0,0 +1,37 @@
package github
import (
"fmt"
gogit "github.com/go-git/go-git/v5"
"github.com/google/go-github/v63/github"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources/git"
)
type unauthenticatedConnector struct {
apiClient *github.Client
}
var _ connector = (*unauthenticatedConnector)(nil)
func newUnauthenticatedConnector(apiEndpoint string) (*unauthenticatedConnector, error) {
const httpTimeoutSeconds = 60
httpClient := common.RetryableHTTPClientTimeout(int64(httpTimeoutSeconds))
apiClient, err := createGitHubClient(httpClient, apiEndpoint)
if err != nil {
return nil, fmt.Errorf("could not create API client: %w", err)
}
return &unauthenticatedConnector{
apiClient: apiClient,
}, nil
}
func (c *unauthenticatedConnector) APIClient() *github.Client {
return c.apiClient
}
func (c *unauthenticatedConnector) Clone(ctx context.Context, repoURL string) (string, *gogit.Repository, error) {
return git.CloneRepoUsingUnauthenticated(ctx, repoURL)
}

269
pkg/sources/github/github.go
View file

@ -8,18 +8,15 @@ import (
"os"
"regexp"
"sort"
"strconv"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/bradleyfalzon/ghinstallation/v2"
"github.com/go-logr/logr"
"github.com/gobwas/glob"
"github.com/google/go-github/v63/github"
"golang.org/x/exp/rand"
"golang.org/x/oauth2"
"golang.org/x/sync/errgroup"
"google.golang.org/protobuf/proto"
"google.golang.org/protobuf/types/known/anypb"
@ -29,7 +26,6 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
"github.com/trufflesecurity/trufflehog/v3/pkg/giturl"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/credentialspb"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
"github.com/trufflesecurity/trufflehog/v3/pkg/sanitizer"
@ -48,11 +44,6 @@ const (
type Source struct {
name string
// Protects the user and token.
userMu sync.Mutex
githubUser string
githubToken string
sourceID sources.SourceID
jobID sources.JobID
verify bool
@ -69,13 +60,12 @@ type Source struct {
scanOptMu sync.Mutex // protects the scanOptions
scanOptions *git.ScanOptions
httpClient *http.Client
log logr.Logger
conn *sourcespb.GitHub
jobPool *errgroup.Group
resumeInfoMutex sync.Mutex
resumeInfoSlice []string
apiClient *github.Client
connector connector
includePRComments bool
includeIssueComments bool
@ -198,9 +188,6 @@ func (s *Source) Init(aCtx context.Context, name string, jobID sources.JobID, so
s.jobPool = &errgroup.Group{}
s.jobPool.SetLimit(concurrency)
s.httpClient = common.RetryableHTTPClientTimeout(60)
s.apiClient = github.NewClient(s.httpClient)
var conn sourcespb.GitHub
err = anypb.UnmarshalTo(connection, &conn, proto.UnmarshalOptions{})
if err != nil {
@ -208,6 +195,12 @@ func (s *Source) Init(aCtx context.Context, name string, jobID sources.JobID, so
}
s.conn = &conn
connector, err := newConnector(s)
if err != nil {
return fmt.Errorf("could not create connector: %w", err)
}
s.connector = connector
s.orgsCache = memory.New[string]()
for _, org := range s.conn.Organizations {
s.orgsCache.Set(org, org)
@ -270,63 +263,13 @@ func (s *Source) Init(aCtx context.Context, name string, jobID sources.JobID, so
return nil
}
// Validate is used by enterprise CLI to validate the Github config file.
// Validate is used by enterprise CLI to validate the GitHub config file.
func (s *Source) Validate(ctx context.Context) []error {
var (
errs []error
ghClient *github.Client
err error
)
apiEndpoint := s.conn.Endpoint
switch cred := s.conn.GetCredential().(type) {
case *sourcespb.GitHub_BasicAuth:
s.httpClient.Transport = &github.BasicAuthTransport{
Username: cred.BasicAuth.Username,
Password: cred.BasicAuth.Password,
}
ghClient, err = createGitHubClient(s.httpClient, apiEndpoint)
if err != nil {
errs = append(errs, fmt.Errorf("error creating GitHub client: %+v", err))
}
case *sourcespb.GitHub_Unauthenticated:
ghClient, err = createGitHubClient(s.httpClient, apiEndpoint)
if err != nil {
errs = append(errs, fmt.Errorf("error creating GitHub client: %+v", err))
}
case *sourcespb.GitHub_Token:
s.githubToken = cred.Token
ts := oauth2.StaticTokenSource(
&oauth2.Token{AccessToken: s.githubToken},
)
s.httpClient.Transport = &oauth2.Transport{
Base: s.httpClient.Transport,
Source: oauth2.ReuseTokenSource(nil, ts),
}
ghClient, err = createGitHubClient(s.httpClient, apiEndpoint)
if err != nil {
errs = append(errs, fmt.Errorf("error creating GitHub client: %+v", err))
}
default:
errs = append(errs, fmt.Errorf("invalid configuration given for source. Name: %s, Type: %s", s.name, s.Type()))
if _, _, err := s.connector.APIClient().Users.Get(ctx, ""); err != nil {
return []error{err}
}
// Run a simple query to check if the client is actually valid
if ghClient != nil {
err = checkGitHubConnection(ctx, ghClient)
if err != nil {
errs = append(errs, err)
}
}
return errs
}
func checkGitHubConnection(ctx context.Context, client *github.Client) error {
_, _, err := client.Users.Get(ctx, "")
return err
return nil
}
func (s *Source) visibilityOf(ctx context.Context, repoURL string) source_metadatapb.Visibility {
@ -347,15 +290,8 @@ func (s *Source) visibilityOf(ctx context.Context, repoURL string) source_metada
return repoInfo.visibility
}
const cloudEndpoint = "https://api.github.com"
// Chunks emits chunks of bytes over a channel.
func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, targets ...sources.ChunkingTarget) error {
apiEndpoint := s.conn.Endpoint
if len(apiEndpoint) == 0 || endsWithGithub.MatchString(apiEndpoint) {
apiEndpoint = cloudEndpoint
}
// If targets are provided, we're only scanning the data in those targets.
// Otherwise, we're scanning all data.
// This allows us to only scan the commit where a vulnerability was found.
@ -369,38 +305,32 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, tar
githubSecondsSpentRateLimited.WithLabelValues(s.name).Set(0)
githubReposScanned.WithLabelValues(s.name).Set(0)
installationClient, err := s.enumerate(ctx, apiEndpoint)
err := s.enumerate(ctx)
if err != nil {
return err
return fmt.Errorf("error enumerating: %w", err)
}
return s.scan(ctx, installationClient, chunksChan)
return s.scan(ctx, chunksChan)
}
func (s *Source) enumerate(ctx context.Context, apiEndpoint string) (*github.Client, error) {
var (
installationClient *github.Client
err error
)
switch cred := s.conn.GetCredential().(type) {
case *sourcespb.GitHub_BasicAuth:
if err = s.enumerateBasicAuth(ctx, apiEndpoint, cred.BasicAuth); err != nil {
return nil, err
func (s *Source) enumerate(ctx context.Context) error {
// I'm not wild about switching on the connector type here (as opposed to dispatching to the connector itself) but
// this felt like a compromise that allowed me to isolate connection logic without rewriting the entire source.
switch c := s.connector.(type) {
case *appConnector:
if err := s.enumerateWithApp(ctx, c.InstallationClient()); err != nil {
return err
}
case *sourcespb.GitHub_Unauthenticated:
s.enumerateUnauthenticated(ctx, apiEndpoint)
case *sourcespb.GitHub_Token:
if err = s.enumerateWithToken(ctx, apiEndpoint, cred.Token); err != nil {
return nil, err
case *basicAuthConnector:
if err := s.enumerateBasicAuth(ctx); err != nil {
return err
}
case *sourcespb.GitHub_GithubApp:
if installationClient, err = s.enumerateWithApp(ctx, apiEndpoint, cred.GithubApp); err != nil {
return nil, err
case *tokenConnector:
if err := s.enumerateWithToken(ctx, c.IsGithubEnterprise()); err != nil {
return err
}
default:
// TODO: move this error to Init
return nil, fmt.Errorf("invalid configuration given for source. Name: %s, Type: %s", s.name, s.Type())
case *unauthenticatedConnector:
s.enumerateUnauthenticated(ctx)
}
s.repos = make([]string, 0, s.filteredRepoCache.Count())
@ -425,7 +355,7 @@ RepoLoop:
// Cache gist info.
for {
gistID := extractGistID(urlParts)
gist, _, err := s.apiClient.Gists.Get(repoCtx, gistID)
gist, _, err := s.connector.APIClient().Gists.Get(repoCtx, gistID)
// Normalize the URL to the Gist's pull URL.
// See https://github.com/trufflesecurity/trufflehog/pull/2625#issuecomment-2025507937
repo = gist.GetGitPullURL()
@ -442,7 +372,7 @@ RepoLoop:
} else {
// Cache repository info.
for {
ghRepo, _, err := s.apiClient.Repositories.Get(repoCtx, urlParts[1], urlParts[2])
ghRepo, _, err := s.connector.APIClient().Repositories.Get(repoCtx, urlParts[1], urlParts[2])
if s.handleRateLimit(err) {
continue
}
@ -462,20 +392,10 @@ RepoLoop:
// We must sort the repos so we can resume later if necessary.
sort.Strings(s.repos)
return installationClient, nil
return nil
}
func (s *Source) enumerateBasicAuth(ctx context.Context, apiEndpoint string, basicAuth *credentialspb.BasicAuth) error {
s.httpClient.Transport = &github.BasicAuthTransport{
Username: basicAuth.Username,
Password: basicAuth.Password,
}
ghClient, err := createGitHubClient(s.httpClient, apiEndpoint)
if err != nil {
s.log.Error(err, "error creating GitHub client")
}
s.apiClient = ghClient
func (s *Source) enumerateBasicAuth(ctx context.Context) error {
for _, org := range s.orgsCache.Keys() {
orgCtx := context.WithValue(ctx, "account", org)
userType, err := s.getReposByOrgOrUser(ctx, org)
@ -494,12 +414,7 @@ func (s *Source) enumerateBasicAuth(ctx context.Context, apiEndpoint string, bas
return nil
}
func (s *Source) enumerateUnauthenticated(ctx context.Context, apiEndpoint string) {
ghClient, err := createGitHubClient(s.httpClient, apiEndpoint)
if err != nil {
s.log.Error(err, "error creating GitHub client")
}
s.apiClient = ghClient
func (s *Source) enumerateUnauthenticated(ctx context.Context) {
if s.orgsCache.Count() > unauthGithubOrgRateLimt {
s.log.Info("You may experience rate limiting when using the unauthenticated GitHub api. Consider using an authenticated scan instead.")
}
@ -518,31 +433,13 @@ func (s *Source) enumerateUnauthenticated(ctx context.Context, apiEndpoint strin
}
}
func (s *Source) enumerateWithToken(ctx context.Context, apiEndpoint, token string) error {
// Needed for clones.
s.githubToken = token
func (s *Source) enumerateWithToken(ctx context.Context, isGithubEnterprise bool) error {
ctx.Logger().V(1).Info("Enumerating with token")
// Needed to list repos.
ts := oauth2.StaticTokenSource(
&oauth2.Token{AccessToken: token},
)
s.httpClient.Transport = &oauth2.Transport{
Base: s.httpClient.Transport,
Source: oauth2.ReuseTokenSource(nil, ts),
}
// If we're using public GitHub, make a regular client.
// Otherwise, make an enterprise client.
ghClient, err := createGitHubClient(s.httpClient, apiEndpoint)
if err != nil {
s.log.Error(err, "error creating GitHub client")
}
s.apiClient = ghClient
ctx.Logger().V(1).Info("Enumerating with token", "endpoint", apiEndpoint)
var ghUser *github.User
var err error
for {
ghUser, _, err = s.apiClient.Users.Get(ctx, "")
ghUser, _, err = s.connector.APIClient().Users.Get(ctx, "")
if s.handleRateLimit(err) {
continue
}
@ -562,8 +459,7 @@ func (s *Source) enumerateWithToken(ctx context.Context, apiEndpoint, token stri
s.log.Error(err, "Unable to fetch gists for the current user", "user", ghUser.GetLogin())
}
isGHE := !strings.EqualFold(apiEndpoint, cloudEndpoint)
if isGHE {
if isGithubEnterprise {
s.addAllVisibleOrgs(ctx)
} else {
// Scan for orgs is default with a token.
@ -597,67 +493,18 @@ func (s *Source) enumerateWithToken(ctx context.Context, apiEndpoint, token stri
return nil
}
func (s *Source) enumerateWithApp(ctx context.Context, apiEndpoint string, app *credentialspb.GitHubApp) (installationClient *github.Client, err error) {
installationID, err := strconv.ParseInt(app.InstallationId, 10, 64)
if err != nil {
return nil, err
}
appID, err := strconv.ParseInt(app.AppId, 10, 64)
if err != nil {
return nil, err
}
// This client is required to create installation tokens for cloning.
// Otherwise, the required JWT is not in the request for the token :/
// This client uses the source's original HTTP transport, so make sure
// to build it before modifying that transport (such as is done during
// the creation of the other API client below).
appItr, err := ghinstallation.NewAppsTransport(
s.httpClient.Transport,
appID,
[]byte(app.PrivateKey))
if err != nil {
return nil, err
}
appItr.BaseURL = apiEndpoint
// Does this need to be separate from |s.httpClient|?
instHTTPClient := common.RetryableHTTPClientTimeout(60)
instHTTPClient.Transport = appItr
installationClient, err = github.NewClient(instHTTPClient).WithEnterpriseURLs(apiEndpoint, apiEndpoint)
if err != nil {
return nil, err
}
// This client is used for most APIs.
itr, err := ghinstallation.New(
s.httpClient.Transport,
appID,
installationID,
[]byte(app.PrivateKey))
if err != nil {
return nil, err
}
itr.BaseURL = apiEndpoint
s.httpClient.Transport = itr
s.apiClient, err = github.NewClient(s.httpClient).WithEnterpriseURLs(apiEndpoint, apiEndpoint)
if err != nil {
return nil, err
}
func (s *Source) enumerateWithApp(ctx context.Context, installationClient *github.Client) error {
// If no repos were provided, enumerate them.
if len(s.repos) == 0 {
if err = s.getReposByApp(ctx); err != nil {
return nil, err
if err := s.getReposByApp(ctx); err != nil {
return err
}
// Check if we need to find user repos.
if s.conn.ScanUsers {
err := s.addMembersByApp(ctx, installationClient)
if err != nil {
return nil, err
return err
}
s.log.Info("Scanning repos", "org_members", len(s.memberCache))
for member := range s.memberCache {
@ -672,7 +519,7 @@ func (s *Source) enumerateWithApp(ctx context.Context, apiEndpoint string, app *
}
}
return installationClient, nil
return nil
}
func createGitHubClient(httpClient *http.Client, apiEndpoint string) (*github.Client, error) {
@ -685,7 +532,7 @@ func createGitHubClient(httpClient *http.Client, apiEndpoint string) (*github.Cl
return github.NewClient(httpClient).WithEnterpriseURLs(apiEndpoint, apiEndpoint)
}
func (s *Source) scan(ctx context.Context, installationClient *github.Client, chunksChan chan *sources.Chunk) error {
func (s *Source) scan(ctx context.Context, chunksChan chan *sources.Chunk) error {
var scannedCount uint64 = 1
s.log.V(2).Info("Found repos to scan", "count", len(s.repos))
@ -730,7 +577,7 @@ func (s *Source) scan(ctx context.Context, installationClient *github.Client, ch
return nil
}
repoCtx := context.WithValues(ctx, "repo", repoURL)
duration, err := s.cloneAndScanRepo(repoCtx, installationClient, repoURL, repoInfo, chunksChan)
duration, err := s.cloneAndScanRepo(repoCtx, repoURL, repoInfo, chunksChan)
if err != nil {
scanErrs.Add(err)
return nil
@ -741,7 +588,7 @@ func (s *Source) scan(ctx context.Context, installationClient *github.Client, ch
wikiURL := strings.TrimSuffix(repoURL, ".git") + ".wiki.git"
wikiCtx := context.WithValue(ctx, "repo", wikiURL)
_, err := s.cloneAndScanRepo(wikiCtx, installationClient, wikiURL, repoInfo, chunksChan)
_, err := s.cloneAndScanRepo(wikiCtx, wikiURL, repoInfo, chunksChan)
if err != nil {
// Ignore "Repository not found" errors.
// It's common for GitHub's API to say a repo has a wiki when it doesn't.
@ -777,11 +624,11 @@ func (s *Source) scan(ctx context.Context, installationClient *github.Client, ch
return nil
}
func (s *Source) cloneAndScanRepo(ctx context.Context, client *github.Client, repoURL string, repoInfo repoInfo, chunksChan chan *sources.Chunk) (time.Duration, error) {
func (s *Source) cloneAndScanRepo(ctx context.Context, repoURL string, repoInfo repoInfo, chunksChan chan *sources.Chunk) (time.Duration, error) {
var duration time.Duration
ctx.Logger().V(2).Info("attempting to clone repo")
path, repo, err := s.cloneRepo(ctx, repoURL, client)
path, repo, err := s.cloneRepo(ctx, repoURL)
if err != nil {
return duration, err
}
@ -894,7 +741,7 @@ func (s *Source) addUserGistsToCache(ctx context.Context, user string) error {
gistOpts := &github.GistListOptions{}
logger := s.log.WithValues("user", user)
for {
gists, res, err := s.apiClient.Gists.List(ctx, user, gistOpts)
gists, res, err := s.connector.APIClient().Gists.List(ctx, user, gistOpts)
if s.handleRateLimit(err) {
continue
}
@ -951,7 +798,7 @@ func (s *Source) addAllVisibleOrgs(ctx context.Context) {
},
}
for {
orgs, _, err := s.apiClient.Organizations.ListAll(ctx, orgOpts)
orgs, _, err := s.connector.APIClient().Organizations.ListAll(ctx, orgOpts)
if s.handleRateLimit(err) {
continue
}
@ -990,7 +837,7 @@ func (s *Source) addOrgsByUser(ctx context.Context, user string) {
}
logger := s.log.WithValues("user", user)
for {
orgs, resp, err := s.apiClient.Organizations.List(ctx, "", orgOpts)
orgs, resp, err := s.connector.APIClient().Organizations.List(ctx, "", orgOpts)
if s.handleRateLimit(err) {
continue
}
@ -1023,7 +870,7 @@ func (s *Source) addMembersByOrg(ctx context.Context, org string) error {
logger := s.log.WithValues("org", org)
for {
members, res, err := s.apiClient.Organizations.ListMembers(ctx, org, opts)
members, res, err := s.connector.APIClient().Organizations.ListMembers(ctx, org, opts)
if s.handleRateLimit(err) {
continue
}
@ -1145,7 +992,7 @@ func (s *Source) processGistComments(ctx context.Context, gistURL string, urlPar
Page: initialPage,
}
for {
comments, _, err := s.apiClient.Gists.ListComments(ctx, gistID, options)
comments, _, err := s.connector.APIClient().Gists.ListComments(ctx, gistID, options)
if s.handleRateLimit(err) {
continue
}
@ -1258,7 +1105,7 @@ func (s *Source) processIssues(ctx context.Context, repoInfo repoInfo, chunksCha
}
for {
issues, _, err := s.apiClient.Issues.ListByRepo(ctx, repoInfo.owner, repoInfo.name, bodyTextsOpts)
issues, _, err := s.connector.APIClient().Issues.ListByRepo(ctx, repoInfo.owner, repoInfo.name, bodyTextsOpts)
if s.handleRateLimit(err) {
continue
}
@ -1330,7 +1177,7 @@ func (s *Source) processIssueComments(ctx context.Context, repoInfo repoInfo, ch
}
for {
issueComments, _, err := s.apiClient.Issues.ListComments(ctx, repoInfo.owner, repoInfo.name, allComments, issueOpts)
issueComments, _, err := s.connector.APIClient().Issues.ListComments(ctx, repoInfo.owner, repoInfo.name, allComments, issueOpts)
if s.handleRateLimit(err) {
continue
}
@ -1395,7 +1242,7 @@ func (s *Source) processPRs(ctx context.Context, repoInfo repoInfo, chunksChan c
}
for {
prs, _, err := s.apiClient.PullRequests.List(ctx, repoInfo.owner, repoInfo.name, prOpts)
prs, _, err := s.connector.APIClient().PullRequests.List(ctx, repoInfo.owner, repoInfo.name, prOpts)
if s.handleRateLimit(err) {
continue
}
@ -1427,7 +1274,7 @@ func (s *Source) processPRComments(ctx context.Context, repoInfo repoInfo, chunk
}
for {
prComments, _, err := s.apiClient.PullRequests.ListComments(ctx, repoInfo.owner, repoInfo.name, allComments, prOpts)
prComments, _, err := s.connector.APIClient().PullRequests.ListComments(ctx, repoInfo.owner, repoInfo.name, allComments, prOpts)
if s.handleRateLimit(err) {
continue
}

336
pkg/sources/github/github_integration_test.go
View file

@ -21,7 +21,6 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources/git"
)
func TestSource_Token(t *testing.T) {
@ -42,7 +41,8 @@ func TestSource_Token(t *testing.T) {
githubInstallationIDNew := secret.MustGetField("GITHUB_INSTALLATION_ID_NEW")
githubAppIDNew := secret.MustGetField("GITHUB_APP_ID_NEW")
conn := &sourcespb.GitHub{
src := &sourcespb.GitHub{
Endpoint: "https://api.github.com",
Credential: &sourcespb.GitHub_GithubApp{
GithubApp: &credentialspb.GitHubApp{
PrivateKey: githubPrivateKeyNew,
@ -51,32 +51,24 @@ func TestSource_Token(t *testing.T) {
},
},
}
conn, err := anypb.New(src)
if err != nil {
panic(err)
}
s := Source{
conn: conn,
httpClient: common.SaneHttpClient(),
conn: src,
log: logr.Discard(),
memberCache: map[string]struct{}{},
repoInfoCache: newRepoInfoCache(),
}
s.Init(ctx, "github integration test source", 0, 0, false, conn, 1)
s.filteredRepoCache = s.newFilteredRepoCache(memory.New[string](), nil, nil)
installationClient, err := s.enumerateWithApp(ctx, "https://api.github.com", conn.GetGithubApp())
err = s.enumerateWithApp(ctx, s.connector.(*appConnector).InstallationClient())
assert.NoError(t, err)
user, token, err := s.userAndToken(ctx, installationClient)
assert.NotEmpty(t, token)
assert.NoError(t, err)
// user provided
_, _, err = git.CloneRepoUsingToken(ctx, token, "https://github.com/truffle-test-integration-org/another-test-repo.git", user)
assert.NoError(t, err)
// no user provided
_, _, err = git.CloneRepoUsingToken(ctx, token, "https://github.com/truffle-test-integration-org/another-test-repo.git", "")
assert.Error(t, err)
_, _, err = s.cloneRepo(ctx, "https://github.com/truffle-test-integration-org/another-test-repo.git", installationClient)
_, _, err = s.cloneRepo(ctx, "https://github.com/truffle-test-integration-org/another-test-repo.git")
assert.NoError(t, err)
}
@ -316,16 +308,6 @@ func TestSource_Scan(t *testing.T) {
githubInstallationIDNew := secret.MustGetField("GITHUB_INSTALLATION_ID_NEW")
githubAppIDNew := secret.MustGetField("GITHUB_APP_ID_NEW")
// OLD app for breaking app change tests
// githubPrivateKeyB64 := secret.MustGetField("GITHUB_PRIVATE_KEY")
// githubPrivateKeyBytes, err := base64.StdEncoding.DecodeString(githubPrivateKeyB64)
// if err != nil {
// t.Fatal(err)
// }
// githubPrivateKey := string(githubPrivateKeyBytes)
// githubInstallationID := secret.MustGetField("GITHUB_INSTALLATION_ID")
// githubAppID := secret.MustGetField("GITHUB_APP_ID")
type init struct {
name string
verify bool
@ -403,117 +385,78 @@ func TestSource_Scan(t *testing.T) {
wantChunk: nil,
wantErr: false,
minRepo: 1,
minOrg: 1,
},
{
name: "token authenticated, username in org",
init: init{
name: "test source",
connection: &sourcespb.GitHub{
Organizations: []string{"truffle-sandbox"},
Credential: &sourcespb.GitHub_Token{
Token: githubToken,
},
},
},
wantChunk: nil,
wantErr: false,
minRepo: 2,
minOrg: 1,
},
{
name: "token authenticated, no org or user (enum)",
// This configuration currently will only find gists from the user. No repos or orgs will be scanned.
init: init{
name: "test source",
connection: &sourcespb.GitHub{
Credential: &sourcespb.GitHub_Token{
Token: githubToken,
},
},
},
wantChunk: nil,
wantErr: false,
minRepo: 2,
minOrg: 0,
},
// {
// name: "token authenticated, username in org",
// init: init{
// name: "test source",
// connection: &sourcespb.GitHub{
// Organizations: []string{"truffle-sandbox"},
// Credential: &sourcespb.GitHub_Token{
// Token: githubToken,
// },
// },
// },
// wantChunk: nil,
// wantErr: false,
// minRepo: 0, // I think enumerating users with the org API does not work for newer users! Or maybe just newer users with a `-` in their name?
// // See also: https://github.com/trufflesecurity/trufflehog/issues/874
// minOrg: 0,
// },
// {
// name: "token authenticated, org in repo",
// // I do not think that this is a supported case, but adding the test to specify there is no requirement.
// init: init{
// name: "test source",
// connection: &sourcespb.GitHub{
// Repositories: []string{"truffle-test-integration-org"},
// Credential: &sourcespb.GitHub_Token{
// Token: githubToken,
// },
// },
// },
// wantChunk: nil,
// wantErr: false,
// minRepo: 0,
// minOrg: 0,
// },
/*
{
name: "token authenticated, no org or user (enum)",
// This configuration currently will only find gists from the user. No repos or orgs will be scanned.
init: init{
name: "test source",
connection: &sourcespb.GitHub{
Credential: &sourcespb.GitHub_Token{
Token: githubToken,
{
name: "unauthenticated, single org",
init: init{
name: "test source",
connection: &sourcespb.GitHub{
Organizations: []string{"trufflesecurity"},
Credential: &sourcespb.GitHub_Unauthenticated{},
IncludeForks: true,
},
},
wantChunk: nil,
wantErr: false,
minRepo: 40,
minOrg: 1,
},
{
name: "unauthenticated, single repo",
init: init{
name: "test source",
connection: &sourcespb.GitHub{
Repositories: []string{"https://github.com/trufflesecurity/driftwood.git"},
Credential: &sourcespb.GitHub_Unauthenticated{},
},
},
wantChunk: &sources.Chunk{
SourceType: sourcespb.SourceType_SOURCE_TYPE_GITHUB,
SourceName: "test source",
SourceMetadata: &source_metadatapb.MetaData{
Data: &source_metadatapb.MetaData_Github{
Github: &source_metadatapb.Github{
Repository: "https://github.com/trufflesecurity/driftwood.git",
},
},
},
wantChunk: nil,
wantErr: false,
minRepo: 0,
minOrg: 0,
Verify: false,
},
{
name: "app authenticated (old), no repo or org (enum)",
init: init{
name: "test source",
connection: &sourcespb.GitHub{
ScanUsers: false,
Credential: &sourcespb.GitHub_GithubApp{
GithubApp: &credentialspb.GitHubApp{
PrivateKey: githubPrivateKey,
InstallationId: githubInstallationID,
AppId: githubAppID,
},
},
},
},
wantChunk: nil,
wantErr: false,
minRepo: 3,
minOrg: 0,
},
*/
// {
// name: "unauthenticated, single org",
// init: init{
// name: "test source",
// connection: &sourcespb.GitHub{
// Organizations: []string{"trufflesecurity"},
// Credential: &sourcespb.GitHub_Unauthenticated{},
// },
// },
// wantChunk: nil,
// wantErr: false,
// minRepo: 3,
// minOrg: 1,
// },
// {
// name: "unauthenticated, single repo",
// init: init{
// name: "test source",
// connection: &sourcespb.GitHub{
// Repositories: []string{"https://github.com/trufflesecurity/driftwood.git"},
// Credential: &sourcespb.GitHub_Unauthenticated{},
// },
// },
// wantChunk: &sources.Chunk{
// SourceType: sourcespb.SourceType_SOURCE_TYPE_GITHUB,
// SourceName: "test source",
// SourceMetadata: &source_metadatapb.MetaData{
// Data: &source_metadatapb.MetaData_Github{
// Github: &source_metadatapb.Github{
// Repository: "https://github.com/trufflesecurity/driftwood.git",
// },
// },
// },
// Verify: false,
// },
// wantErr: false,
// },
wantErr: false,
},
{
name: "app authenticated, no repo or org",
init: init{
@ -531,7 +474,7 @@ func TestSource_Scan(t *testing.T) {
},
wantChunk: nil,
wantErr: false,
minRepo: 1,
minRepo: 32,
minOrg: 0,
},
{
@ -623,19 +566,6 @@ func TestSource_paginateGists(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*30)
defer cancel()
secret, err := common.GetTestSecret(ctx)
if err != nil {
t.Fatal(fmt.Errorf("failed to access secret: %v", err))
}
// For the NEW github app test (+Member enum)
githubPrivateKeyB64New := secret.MustGetField("GITHUB_PRIVATE_KEY_NEW")
githubPrivateKeyBytesNew, err := base64.StdEncoding.DecodeString(githubPrivateKeyB64New)
if err != nil {
t.Fatal(err)
}
githubPrivateKeyNew := string(githubPrivateKeyBytesNew)
githubInstallationIDNew := secret.MustGetField("GITHUB_INSTALLATION_ID_NEW")
githubAppIDNew := secret.MustGetField("GITHUB_APP_ID_NEW")
type init struct {
name string
verify bool
@ -654,13 +584,7 @@ func TestSource_paginateGists(t *testing.T) {
init: init{
name: "test source",
connection: &sourcespb.GitHub{
Credential: &sourcespb.GitHub_GithubApp{
GithubApp: &credentialspb.GitHubApp{
PrivateKey: githubPrivateKeyNew,
InstallationId: githubInstallationIDNew,
AppId: githubAppIDNew,
},
},
Credential: &sourcespb.GitHub_Unauthenticated{},
},
},
wantChunk: &sources.Chunk{
@ -678,54 +602,19 @@ func TestSource_paginateGists(t *testing.T) {
user: "truffle-sandbox",
minRepos: 1,
},
// {
// name: "get multiple pages of gists",
// init: init{
// name: "test source",
// connection: &sourcespb.GitHub{
// Credential: &sourcespb.GitHub_GithubApp{
// GithubApp: &credentialspb.GitHubApp{
// PrivateKey: githubPrivateKeyNew,
// InstallationId: githubInstallationIDNew,
// AppId: githubAppIDNew,
// },
// },
// },
// },
// wantChunk: nil,
// wantErr: false,
// user: "andrew",
// minRepos: 101,
// },
/* {
name: "get multiple pages of gists",
init: init{
name: "test source",
connection: &sourcespb.GitHub{
Credential: &sourcespb.GitHub_GithubApp{
GithubApp: &credentialspb.GitHubApp{
PrivateKey: githubPrivateKeyNew,
InstallationId: githubInstallationIDNew,
AppId: githubAppIDNew,
},
},
},
},
wantChunk: &sources.Chunk{
SourceName: "test source",
SourceMetadata: &source_metadatapb.MetaData{
Data: &source_metadatapb.MetaData_Github{
Github: &source_metadatapb.Github{
Repository: "https://gist.github.com/872df3b78b9ec3e7dbe597fb5a202121.git",
},
},
},
Verify: false,
},
wantErr: false,
user: "andrew",
{
name: "get multiple pages of gists",
init: init{
name: "test source",
connection: &sourcespb.GitHub{
Credential: &sourcespb.GitHub_Unauthenticated{},
},
*/
},
wantChunk: nil,
wantErr: false,
user: "andrew",
minRepos: 101,
},
}
for _, tt := range tests {
@ -744,7 +633,7 @@ func TestSource_paginateGists(t *testing.T) {
}
chunksCh := make(chan *sources.Chunk, 5)
go func() {
s.addUserGistsToCache(ctx, tt.user)
assert.NoError(t, s.addUserGistsToCache(ctx, tt.user))
chunksCh <- &sources.Chunk{}
}()
var wantedRepo string
@ -810,35 +699,6 @@ func githubCommentCheckFunc(gotChunk, wantChunk *sources.Chunk, i int, t *testin
}
}
// func TestSource_paginateRepos(t *testing.T) {
// type args struct {
// ctx context.Context
// apiClient *github.Client
// }
// tests := []struct {
// name string
// org string
// args args
// }{
// {
// org: "fakeNetflix",
// args: args{
// ctx: context.Background(),
// apiClient: github.NewClient(common.SaneHttpClient()),
// },
// },
// }
// for _, tt := range tests {
// t.Run(tt.name, func(t *testing.T) {
// s := &Source{httpClient: common.SaneHttpClient()}
// s.paginateRepos(tt.args.ctx, tt.args.apiClient, tt.org)
// if len(s.repos) < 101 {
// t.Errorf("expected > 100 repos, got %d", len(s.repos))
// }
// })
// }
// }
func TestSource_Chunks_TargetedScan(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*3000)
defer cancel()
@ -870,10 +730,10 @@ func TestSource_Chunks_TargetedScan(t *testing.T) {
queryCriteria: &source_metadatapb.MetaData{
Data: &source_metadatapb.MetaData_Github{
Github: &source_metadatapb.Github{
Repository: "test_keys",
Link: "https://github.com/trufflesecurity/test_keys/blob/fbc14303ffbf8fb1c2c1914e8dda7d0121633aca/keys#L4",
Commit: "fbc14303ffbf8fb1c2c1914e8dda7d0121633aca",
File: "keys",
Repository: "test-secrets",
Link: "https://github.com/truffle-sandbox/test-secrets/blob/0416560b1330d8ac42045813251d85c688717eaf/new_key#L2",
Commit: "0416560b1330d8ac42045813251d85c688717eaf",
File: "new_key",
},
},
},
@ -884,7 +744,7 @@ func TestSource_Chunks_TargetedScan(t *testing.T) {
name: "targeted scan, one file in med commit",
init: init{
name: "test source",
connection: &sourcespb.GitHub{Credential: &sourcespb.GitHub_Token{Token: githubToken}},
connection: &sourcespb.GitHub{Credential: &sourcespb.GitHub_Unauthenticated{}},
queryCriteria: &source_metadatapb.MetaData{
Data: &source_metadatapb.MetaData_Github{
Github: &source_metadatapb.Github{
@ -902,7 +762,7 @@ func TestSource_Chunks_TargetedScan(t *testing.T) {
name: "no file in commit",
init: init{
name: "test source",
connection: &sourcespb.GitHub{Credential: &sourcespb.GitHub_Token{Token: githubToken}},
connection: &sourcespb.GitHub{Credential: &sourcespb.GitHub_Unauthenticated{}},
queryCriteria: &source_metadatapb.MetaData{
Data: &source_metadatapb.MetaData_Github{
Github: &source_metadatapb.Github{
@ -921,7 +781,7 @@ func TestSource_Chunks_TargetedScan(t *testing.T) {
name: "invalid query criteria, malformed link",
init: init{
name: "test source",
connection: &sourcespb.GitHub{Credential: &sourcespb.GitHub_Token{Token: githubToken}},
connection: &sourcespb.GitHub{Credential: &sourcespb.GitHub_Unauthenticated{}},
queryCriteria: &source_metadatapb.MetaData{
Data: &source_metadatapb.MetaData_Github{
Github: &source_metadatapb.Github{

136
pkg/sources/github/github_test.go
View file

@ -30,6 +30,23 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
)
func createPrivateKey() string {
key, err := rsa.GenerateKey(rand.Reader, 2048)
if err != nil {
panic(err)
}
data := x509.MarshalPKCS1PrivateKey(key)
var pemKey bytes.Buffer
if err := pem.Encode(&pemKey, &pem.Block{
Type: "RSA PRIVATE KEY",
Bytes: data,
}); err != nil {
panic(err)
}
return pemKey.String()
}
func createTestSource(src *sourcespb.GitHub) (*Source, *anypb.Any) {
s := &Source{}
conn, err := anypb.New(src)
@ -44,8 +61,10 @@ func initTestSource(src *sourcespb.GitHub) *Source {
if err := s.Init(context.Background(), "test - github", 0, 1337, false, conn, 1); err != nil {
panic(err)
}
s.apiClient = github.NewClient(s.httpClient)
gock.InterceptClient(s.httpClient)
gock.InterceptClient(s.connector.APIClient().Client())
if appConnector, ok := s.connector.(*appConnector); ok {
gock.InterceptClient(appConnector.InstallationClient().Client())
}
return s
}
@ -81,7 +100,6 @@ func TestAddReposByOrg(t *testing.T) {
Repositories: nil,
IgnoreRepos: []string{"secret/super-*-repo2"},
})
// gock works here because github.NewClient is using the default HTTP Transport
err := s.getReposByOrg(context.Background(), "super-secret-org")
assert.Nil(t, err)
assert.Equal(t, 1, s.filteredRepoCache.Count())
@ -110,7 +128,6 @@ func TestAddReposByOrg_IncludeRepos(t *testing.T) {
IncludeRepos: []string{"super-secret-org/super*"},
Organizations: []string{"super-secret-org"},
})
// gock works here because github.NewClient is using the default HTTP Transport
err := s.getReposByOrg(context.Background(), "super-secret-org")
assert.Nil(t, err)
assert.Equal(t, 2, s.filteredRepoCache.Count())
@ -156,7 +173,7 @@ func TestAddGistsByUser(t *testing.T) {
Reply(200).
JSON([]map[string]string{{"id": "aa5a315d61ae9438b18d", "git_pull_url": "https://gist.github.com/aa5a315d61ae9438b18d.git"}})
s := initTestSource(nil)
s := initTestSource(&sourcespb.GitHub{Credential: &sourcespb.GitHub_Unauthenticated{}})
err := s.addUserGistsToCache(context.Background(), "super-secret-user")
assert.Nil(t, err)
assert.Equal(t, 1, s.filteredRepoCache.Count())
@ -177,7 +194,7 @@ func TestAddMembersByOrg(t *testing.T) {
{"login": "testman2"},
})
s := initTestSource(nil)
s := initTestSource(&sourcespb.GitHub{Credential: &sourcespb.GitHub_Unauthenticated{}})
err := s.addMembersByOrg(context.Background(), "org1")
assert.Nil(t, err)
assert.Equal(t, 2, len(s.memberCache))
@ -192,12 +209,18 @@ func TestAddMembersByOrg(t *testing.T) {
func TestAddMembersByApp(t *testing.T) {
defer gock.Off()
privateKey := createPrivateKey()
gock.New("https://api.github.com").
Get("/app/installations").
Reply(200).
JSON([]map[string]any{
{"account": map[string]string{"login": "super-secret-org", "type": "Organization"}},
})
gock.New("https://api.github.com").
Post("/app/installations/1337/access_tokens").
Reply(200).
JSON(map[string]string{"token": "dontlook"})
gock.New("https://api.github.com").
Get("/orgs/super-secret-org/members").
Reply(200).
@ -207,8 +230,16 @@ func TestAddMembersByApp(t *testing.T) {
{"login": "ssm3"},
})
s := initTestSource(nil)
err := s.addMembersByApp(context.Background(), github.NewClient(nil))
s := initTestSource(&sourcespb.GitHub{
Endpoint: "https://api.github.com",
Credential: &sourcespb.GitHub_GithubApp{
GithubApp: &credentialspb.GitHubApp{
PrivateKey: privateKey,
InstallationId: "1337",
AppId: "4141",
},
}})
err := s.addMembersByApp(context.Background(), s.connector.(*appConnector).InstallationClient())
assert.Nil(t, err)
assert.Equal(t, 3, len(s.memberCache))
_, ok := s.memberCache["ssm1"]
@ -234,7 +265,7 @@ func TestAddReposByApp(t *testing.T) {
},
})
s := initTestSource(nil)
s := initTestSource(&sourcespb.GitHub{Credential: &sourcespb.GitHub_Unauthenticated{}})
err := s.getReposByApp(context.Background())
assert.Nil(t, err)
assert.Equal(t, 2, s.filteredRepoCache.Count())
@ -258,7 +289,7 @@ func TestAddOrgsByUser(t *testing.T) {
{"login": "sso2"},
})
s := initTestSource(nil)
s := initTestSource(&sourcespb.GitHub{Credential: &sourcespb.GitHub_Unauthenticated{}})
s.addOrgsByUser(context.Background(), "super-secret-user")
assert.Equal(t, 1, s.orgsCache.Count())
ok := s.orgsCache.Exists("sso2")
@ -312,7 +343,7 @@ func TestNormalizeRepos(t *testing.T) {
t.Run(tt.name, func(t *testing.T) {
defer gock.Off()
tt.setup()
s := initTestSource(nil)
s := initTestSource(&sourcespb.GitHub{Credential: &sourcespb.GitHub_Unauthenticated{}})
got, err := s.normalizeRepo(tt.repos[0])
if (err != nil) != tt.wantErr {
@ -337,7 +368,7 @@ func TestNormalizeRepos(t *testing.T) {
}
func TestHandleRateLimit(t *testing.T) {
s := initTestSource(nil)
s := initTestSource(&sourcespb.GitHub{Credential: &sourcespb.GitHub_Unauthenticated{}})
assert.False(t, s.handleRateLimit(nil))
// Request
@ -381,10 +412,14 @@ func TestEnumerateUnauthenticated(t *testing.T) {
Reply(200).
JSON([]map[string]string{{"full_name": "super-secret-org/super-secret-repo", "clone_url": "https://github.com/super-secret-org/super-secret-repo.git"}})
s := initTestSource(nil)
s := initTestSource(&sourcespb.GitHub{
Endpoint: apiEndpoint,
Credential: &sourcespb.GitHub_Unauthenticated{},
})
s.orgsCache = memory.New[string]()
s.orgsCache.Set("super-secret-org", "super-secret-org")
s.enumerateUnauthenticated(context.Background(), apiEndpoint)
//s.enumerateUnauthenticated(context.Background(), apiEndpoint)
s.enumerateUnauthenticated(context.Background())
assert.Equal(t, 1, s.filteredRepoCache.Count())
ok := s.filteredRepoCache.Exists("super-secret-org/super-secret-repo")
assert.True(t, ok)
@ -417,8 +452,13 @@ func TestEnumerateWithToken(t *testing.T) {
Reply(200).
JSON([]map[string]string{{"id": "super-secret-gist", "git_pull_url": "https://gist.github.com/super-secret-gist.git"}})
s := initTestSource(nil)
err := s.enumerateWithToken(context.Background(), "https://api.github.com", "token")
s := initTestSource(&sourcespb.GitHub{
Endpoint: "https://api.github.com",
Credential: &sourcespb.GitHub_Token{
Token: "token",
},
})
err := s.enumerateWithToken(context.Background(), false)
assert.Nil(t, err)
assert.Equal(t, 2, s.filteredRepoCache.Count())
ok := s.filteredRepoCache.Exists("super-secret-user/super-secret-repo")
@ -453,11 +493,16 @@ func BenchmarkEnumerateWithToken(b *testing.B) {
Reply(200).
JSON([]map[string]string{{"git_pull_url": "https://github.com/super-secret-gist.git"}})
s := initTestSource(nil)
s := initTestSource(&sourcespb.GitHub{
Endpoint: "https://api.github.com",
Credential: &sourcespb.GitHub_Token{
Token: "token",
},
})
b.ResetTimer()
for i := 0; i < b.N; i++ {
_ = s.enumerateWithToken(context.Background(), "https://api.github.com", "token")
_ = s.enumerateWithToken(context.Background(), false)
}
}
@ -488,6 +533,7 @@ func TestEnumerate(t *testing.T) {
JSON(`[{"git_pull_url": "https://gist.github.com/2801a2b0523099d0614a951579d99ba9.git", "id": "2801a2b0523099d0614a951579d99ba9"}]`)
s := initTestSource(&sourcespb.GitHub{
Endpoint: "https://api.github.com",
Credential: &sourcespb.GitHub_Token{
Token: "super secret token",
},
@ -521,7 +567,7 @@ func TestEnumerate(t *testing.T) {
s.filteredRepoCache.Set(repo.GetFullName(), repo.GetCloneURL())
// Act
_, err := s.enumerate(context.Background(), "https://api.github.com")
err := s.enumerate(context.Background())
// Assert
assert.Nil(t, err)
@ -586,6 +632,7 @@ func mockGists() []map[string]string {
func BenchmarkEnumerate(b *testing.B) {
for i := 0; i < b.N; i++ {
s := initTestSource(&sourcespb.GitHub{
Endpoint: "https://api.github.com",
Credential: &sourcespb.GitHub_Token{
Token: "super secret token",
},
@ -593,7 +640,7 @@ func BenchmarkEnumerate(b *testing.B) {
setupMocks(b)
b.StartTimer()
_, _ = s.enumerate(context.Background(), "https://api.github.com")
_ = s.enumerate(context.Background())
}
}
@ -605,10 +652,15 @@ func TestEnumerateWithToken_IncludeRepos(t *testing.T) {
Reply(200).
JSON(map[string]string{"login": "super-secret-user"})
s := initTestSource(nil)
s := initTestSource(&sourcespb.GitHub{
Endpoint: "https://api.github.com",
Credential: &sourcespb.GitHub_Token{
Token: "token",
},
})
s.repos = []string{"some-special-repo"}
err := s.enumerateWithToken(context.Background(), "https://api.github.com", "token")
err := s.enumerateWithToken(context.Background(), false)
assert.Nil(t, err)
assert.Equal(t, 1, len(s.repos))
assert.Equal(t, []string{"some-special-repo"}, s.repos)
@ -619,23 +671,7 @@ func TestEnumerateWithToken_IncludeRepos(t *testing.T) {
func TestEnumerateWithApp(t *testing.T) {
defer gock.Off()
// generate a private key (it just needs to be in the right format)
privateKey := func() string {
key, err := rsa.GenerateKey(rand.Reader, 2048)
if err != nil {
panic(err)
}
data := x509.MarshalPKCS1PrivateKey(key)
var pemKey bytes.Buffer
if err := pem.Encode(&pemKey, &pem.Block{
Type: "RSA PRIVATE KEY",
Bytes: data,
}); err != nil {
panic(err)
}
return pemKey.String()
}()
privateKey := createPrivateKey()
gock.New("https://api.github.com").
Post("/app/installations/1337/access_tokens").
@ -647,16 +683,17 @@ func TestEnumerateWithApp(t *testing.T) {
Reply(200).
JSON(map[string]string{})
s := initTestSource(nil)
_, err := s.enumerateWithApp(
context.Background(),
"https://api.github.com",
&credentialspb.GitHubApp{
InstallationId: "1337",
AppId: "4141",
PrivateKey: privateKey,
s := initTestSource(&sourcespb.GitHub{
Endpoint: "https://api.github.com",
Credential: &sourcespb.GitHub_GithubApp{
GithubApp: &credentialspb.GitHubApp{
PrivateKey: privateKey,
InstallationId: "1337",
AppId: "4141",
},
},
)
})
err := s.enumerateWithApp(context.Background(), s.connector.(*appConnector).InstallationClient())
assert.Nil(t, err)
assert.Equal(t, 0, len(s.repos))
assert.False(t, gock.HasUnmatchedRequest())
@ -775,10 +812,11 @@ func Test_scan_SetProgressComplete(t *testing.T) {
t.Run(tc.name, func(t *testing.T) {
src := initTestSource(&sourcespb.GitHub{
Repositories: tc.repos,
Credential: &sourcespb.GitHub_Unauthenticated{},
})
src.jobPool = &errgroup.Group{}
_ = src.scan(context.Background(), nil, nil)
_ = src.scan(context.Background(), nil)
if !tc.wantErr {
assert.Equal(t, "", src.GetProgress().EncodedResumeInfo)
}

117
pkg/sources/github/repo.go
View file

@ -5,7 +5,6 @@ import (
"fmt"
"io"
"net/http"
"strconv"
"strings"
"sync"
@ -15,8 +14,6 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
"github.com/trufflesecurity/trufflehog/v3/pkg/giturl"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources/git"
)
type repoInfoCache struct {
@ -53,107 +50,8 @@ type repoInfo struct {
visibility source_metadatapb.Visibility
}
func (s *Source) cloneRepo(
ctx context.Context,
repoURL string,
installationClient *github.Client,
) (string, *gogit.Repository, error) {
var (
path string
repo *gogit.Repository
err error
)
switch s.conn.GetCredential().(type) {
case *sourcespb.GitHub_BasicAuth:
path, repo, err = git.CloneRepoUsingToken(ctx, s.conn.GetBasicAuth().GetPassword(), repoURL, s.conn.GetBasicAuth().GetUsername())
if err != nil {
return "", nil, err
}
case *sourcespb.GitHub_Unauthenticated:
path, repo, err = git.CloneRepoUsingUnauthenticated(ctx, repoURL)
if err != nil {
return "", nil, err
}
case *sourcespb.GitHub_GithubApp:
s.githubUser, s.githubToken, err = s.userAndToken(ctx, installationClient)
if err != nil {
return "", nil, fmt.Errorf("error getting token for repo %s: %w", repoURL, err)
}
path, repo, err = git.CloneRepoUsingToken(ctx, s.githubToken, repoURL, s.githubUser)
if err != nil {
return "", nil, err
}
case *sourcespb.GitHub_Token:
if err := s.getUserAndToken(ctx, repoURL, installationClient); err != nil {
return "", nil, fmt.Errorf("error getting token for repo %s: %w", repoURL, err)
}
path, repo, err = git.CloneRepoUsingToken(ctx, s.githubToken, repoURL, s.githubUser)
if err != nil {
return "", nil, err
}
default:
return "", nil, fmt.Errorf("unhandled credential type for repo %s: %T", repoURL, s.conn.GetCredential())
}
return path, repo, nil
}
func (s *Source) getUserAndToken(ctx context.Context, repoURL string, installationClient *github.Client) error {
// We never refresh user provided tokens, so if we already have them, we never need to try and fetch them again.
s.userMu.Lock()
defer s.userMu.Unlock()
if s.githubUser == "" || s.githubToken == "" {
var err error
s.githubUser, s.githubToken, err = s.userAndToken(ctx, installationClient)
if err != nil {
return fmt.Errorf("error getting token for repo %s: %w", repoURL, err)
}
}
return nil
}
func (s *Source) userAndToken(ctx context.Context, installationClient *github.Client) (string, string, error) {
switch cred := s.conn.GetCredential().(type) {
case *sourcespb.GitHub_BasicAuth:
return cred.BasicAuth.Username, cred.BasicAuth.Password, nil
case *sourcespb.GitHub_Unauthenticated:
// do nothing
case *sourcespb.GitHub_GithubApp:
id, err := strconv.ParseInt(cred.GithubApp.InstallationId, 10, 64)
if err != nil {
return "", "", fmt.Errorf("unable to parse installation id: %w", err)
}
// TODO: Check rate limit for this call.
token, _, err := installationClient.Apps.CreateInstallationToken(
ctx, id, &github.InstallationTokenOptions{})
if err != nil {
return "", "", fmt.Errorf("unable to create installation token: %w", err)
}
return "x-access-token", token.GetToken(), nil // TODO: multiple workers request this, track the TTL
case *sourcespb.GitHub_Token:
var (
ghUser *github.User
err error
)
for {
ghUser, _, err = s.apiClient.Users.Get(ctx, "")
if s.handleRateLimit(err) {
continue
}
if err != nil {
return "", "", fmt.Errorf("unable to get user: %w", err)
}
break
}
return ghUser.GetLogin(), cred.Token, nil
default:
return "", "", fmt.Errorf("unhandled credential type")
}
return "", "", fmt.Errorf("unhandled credential type")
func (s *Source) cloneRepo(ctx context.Context, repoURL string) (string, *gogit.Repository, error) {
return s.connector.Clone(ctx, repoURL)
}
type repoListOptions interface {
@ -171,7 +69,7 @@ func (a *appListOptions) getListOptions() *github.ListOptions {
}
func (s *Source) appListReposWrapper(ctx context.Context, _ string, opts repoListOptions) ([]*github.Repository, *github.Response, error) {
someRepos, res, err := s.apiClient.Apps.ListRepos(ctx, opts.getListOptions())
someRepos, res, err := s.connector.APIClient().Apps.ListRepos(ctx, opts.getListOptions())
if someRepos != nil {
return someRepos.Repositories, res, err
}
@ -195,7 +93,7 @@ func (u *userListOptions) getListOptions() *github.ListOptions {
}
func (s *Source) userListReposWrapper(ctx context.Context, user string, opts repoListOptions) ([]*github.Repository, *github.Response, error) {
return s.apiClient.Repositories.ListByUser(ctx, user, &opts.(*userListOptions).RepositoryListByUserOptions)
return s.connector.APIClient().Repositories.ListByUser(ctx, user, &opts.(*userListOptions).RepositoryListByUserOptions)
}
func (s *Source) getReposByUser(ctx context.Context, user string) error {
@ -217,7 +115,8 @@ func (o *orgListOptions) getListOptions() *github.ListOptions {
}
func (s *Source) orgListReposWrapper(ctx context.Context, org string, opts repoListOptions) ([]*github.Repository, *github.Response, error) {
return s.apiClient.Repositories.ListByOrg(ctx, org, &opts.(*orgListOptions).RepositoryListByOrgOptions)
// TODO: It's possible to exclude forks when making the API request rather than doing post-request filtering
return s.connector.APIClient().Repositories.ListByOrg(ctx, org, &opts.(*orgListOptions).RepositoryListByOrgOptions)
}
func (s *Source) getReposByOrg(ctx context.Context, org string) error {
@ -371,7 +270,7 @@ func (s *Source) wikiIsReachable(ctx context.Context, repoURL string) bool {
return false
}
res, err := s.httpClient.Do(req)
res, err := s.connector.APIClient().Client().Do(req)
if err != nil {
return false
}
@ -398,7 +297,7 @@ func (s *Source) getDiffForFileInCommit(ctx context.Context, query commitQuery)
err error
)
for {
commit, _, err = s.apiClient.Repositories.GetCommit(ctx, query.owner, query.repo, query.sha, nil)
commit, _, err = s.connector.APIClient().Repositories.GetCommit(ctx, query.owner, query.repo, query.sha, nil)
if s.handleRateLimit(err) {
continue
}