Mirrors/trufflehog
2
0
Fork 0
mirror of https://github.com/trufflesecurity/trufflehog.git synced 2024-11-10 07:04:24 +00:00
Code Issues Projects Releases Packages Wiki Activity

Move GitHub integration tests behind a build flag and add unit tests (#527)

Browse source 
* Add unit tests and refactor some logic

* Move integration tests to a separate file behind a build flag

* Fix bugs in normalizeRepos

* Address lint errors

* Sort slices before comparing because order doesn't matter
This commit is contained in:
Miccah 2022-05-09 10:31:00 -05:00 committed by GitHub
parent 01792585aa
commit edaf1e1fd3
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
5 changed files with 1004 additions and 593 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
go.mod
View file

@ -31,13 +31,13 @@ require (
github.com/joho/godotenv v1.4.0
github.com/jpillora/overseer v1.1.6
github.com/kylelemons/godebug v1.1.0
github.com/mattn/go-colorable v0.1.12
github.com/paulbellamy/ratecounter v0.2.0
github.com/pkg/errors v0.9.1
github.com/razorpay/razorpay-go v0.0.0-20210728161131-0341409a6ab2
github.com/rs/zerolog v1.26.1
github.com/sergi/go-diff v1.2.0
github.com/sirupsen/logrus v1.8.1
github.com/stretchr/testify v1.7.0
github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502
github.com/xanzy/go-gitlab v0.64.0
github.com/zricethezav/gitleaks/v8 v8.5.2
@ -48,6 +48,7 @@ require (
google.golang.org/genproto v0.0.0-20220405205423-9d709892a2bf
google.golang.org/protobuf v1.28.0
gopkg.in/alecthomas/kingpin.v2 v2.2.6
gopkg.in/h2non/gock.v1 v1.1.2
)
require (
@ -71,6 +72,7 @@ require (
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.4 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.4 // indirect
github.com/aws/smithy-go v1.11.2 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/dimchansky/utfbom v1.1.1 // indirect
github.com/emirpasic/gods v1.12.0 // indirect
github.com/go-git/gcfg v1.5.0 // indirect
@ -84,6 +86,7 @@ require (
github.com/google/go-querystring v1.1.0 // indirect
github.com/google/pprof v0.0.0-20211214055906-6f57359322fd // indirect
github.com/googleapis/gax-go/v2 v2.2.0 // indirect
github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 // indirect
github.com/hashicorp/go-cleanhttp v0.5.1 // indirect
github.com/imdario/mergo v0.3.12 // indirect
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
@ -91,11 +94,13 @@ require (
github.com/jpillora/s3 v1.1.4 // indirect
github.com/json-iterator/go v1.1.11 // indirect
github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351 // indirect
github.com/mattn/go-colorable v0.1.12 // indirect
github.com/mattn/go-isatty v0.0.14 // indirect
github.com/mitchellh/go-homedir v1.1.0 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.1 // indirect
github.com/pkg/diff v0.0.0-20200914180035-5b29258ca4f7 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/xanzy/ssh-agent v0.3.0 // indirect
github.com/yusufpapurcu/wmi v1.2.2 // indirect
go.opencensus.io v0.23.0 // indirect
@ -110,4 +115,5 @@ require (
google.golang.org/grpc v1.45.0 // indirect
gopkg.in/ini.v1 v1.66.2 // indirect
gopkg.in/warnings.v0 v0.1.2 // indirect
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
)

6
go.sum
View file

@ -296,6 +296,8 @@ github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/ad
github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
github.com/h2non/filetype v1.1.3 h1:FKkx9QbD7HR/zjK1Ia5XiBsq9zdLi5Kf3zGyFTAFkGg=
github.com/h2non/filetype v1.1.3/go.mod h1:319b3zT68BvV+WRj7cwy856M2ehB3HqNOt6sy1HndBY=
github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 h1:2VTzZjLZBgl62/EtslCrtky5vbi9dd7HrQPQIx6wqiw=
github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542/go.mod h1:Ow0tF8D4Kplbc8s8sSb3V2oUCygFHVp8gC3Dn6U4MNI=
github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q=
github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8=
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
@ -391,6 +393,8 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJ
github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32 h1:W6apQkHrMkS0Muv8G/TipAy/FJl/rCYT0+EuS8+Z0z4=
github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32/go.mod h1:9wM+0iRr9ahx58uYLpLIr5fm8diHn0JbqRycJi6w0Ms=
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk=
github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
@ -930,6 +934,8 @@ gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
gopkg.in/h2non/gock.v1 v1.1.2 h1:jBbHXgGBK/AoPVfJh5x4r/WxIrElvbLel8TCZkkZJoY=
gopkg.in/h2non/gock.v1 v1.1.2/go.mod h1:n7UGz/ckNChHiK05rDoiC4MYSunEC/lyaUm2WWaDva0=
gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
gopkg.in/ini.v1 v1.66.2 h1:XfR1dOYubytKy4Shzc2LHrrGhU0lDCfDGG1yLPmpgsI=
gopkg.in/ini.v1 v1.66.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=

155
pkg/sources/github/github.go
View file

@ -11,13 +11,13 @@ import (
"strconv"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/bradleyfalzon/ghinstallation/v2"
"github.com/go-errors/errors"
gogit "github.com/go-git/go-git/v5"
"github.com/google/go-github/v42/github"
"github.com/sirupsen/logrus"
log "github.com/sirupsen/logrus"
"golang.org/x/oauth2"
"golang.org/x/sync/semaphore"
@ -37,8 +37,8 @@ import (
type Source struct {
name string
sourceId int64
jobId int64
sourceID int64
jobID int64
verify bool
repos []string
orgs []string
@ -64,11 +64,11 @@ func (s *Source) Type() sourcespb.SourceType {
}
func (s *Source) SourceID() int64 {
return s.sourceId
return s.sourceID
}
func (s *Source) JobID() int64 {
return s.jobId
return s.jobID
}
func (s *Source) Token(ctx context.Context, installationClient *github.Client) (string, error) {
@ -94,13 +94,13 @@ func (s *Source) Token(ctx context.Context, installationClient *github.Client) (
}
// Init returns an initialized GitHub source.
func (s *Source) Init(aCtx context.Context, name string, jobId, sourceId int64, verify bool, connection *anypb.Any, concurrency int) error {
func (s *Source) Init(aCtx context.Context, name string, jobID, sourceID int64, verify bool, connection *anypb.Any, concurrency int) error {
s.log = log.WithField("source", s.Type()).WithField("name", name)
s.aCtx = aCtx
s.name = name
s.sourceId = sourceId
s.jobId = jobId
s.sourceID = sourceID
s.jobID = jobID
s.verify = verify
s.jobSem = semaphore.NewWeighted(int64(concurrency))
@ -333,16 +333,27 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
}
func (s *Source) scan(ctx context.Context, installationClient *github.Client, chunksChan chan *sources.Chunk) error {
scanned := 0
var scanned uint64
log.Debugf("Found %v total repos to scan", len(s.repos))
wg := sync.WaitGroup{}
errs := []error{}
var errsMut sync.Mutex
errs := make(chan error, 1)
reportErr := func(err error) {
// save the error if there's room, otherwise log and drop it
select {
case errs <- err:
default:
log.WithError(err).Warn("dropping error")
}
}
for i, repoURL := range s.repos {
if err := s.jobSem.Acquire(ctx, 1); err != nil {
// Acquire blocks until it can acquire the semaphore or returns an
// error if the context is finished
log.WithError(err).Debug("could not acquire semaphore")
continue
reportErr(err)
break
}
wg.Add(1)
go func(ctx context.Context, repoURL string, i int) {
@ -367,10 +378,7 @@ func (s *Source) scan(ctx context.Context, installationClient *github.Client, ch
var token string
token, err = s.Token(ctx, installationClient)
if err != nil {
// TODO: maybe we can use a channel here
errsMut.Lock()
errs = append(errs, err)
errsMut.Unlock()
reportErr(err)
return
}
path, repo, err = git.CloneRepoUsingToken(token, repoURL, "clone")
@ -391,20 +399,20 @@ func (s *Source) scan(ctx context.Context, installationClient *github.Client, ch
if err != nil {
log.WithError(err).Errorf("unable to scan repo, continuing")
}
// TODO: use atomic library
scanned++
logrus.Debugf("scanned %d/%d repos", scanned, len(s.repos))
atomic.AddUint64(&scanned, 1)
log.Debugf("scanned %d/%d repos", scanned, len(s.repos))
}(ctx, repoURL, i)
}
wg.Wait()
// This only returns first error which is what we did prior to concurrency
if len(errs) > 0 {
return errs[0]
select {
case err := <-errs:
return err
default:
return nil
}
return nil
}
// handleRateLimit returns true if a rate limit was handled
@ -443,7 +451,8 @@ func handleRateLimit(errIn error, res *github.Response) bool {
return true
}
func (s *Source) addReposByOrg(ctx context.Context, apiClient *github.Client, org string) error {
func (s *Source) getReposByOrg(ctx context.Context, apiClient *github.Client, org string) ([]string, error) {
repos := []string{}
opts := &github.RepositoryListByOrgOptions{
ListOptions: github.ListOptions{
PerPage: 100,
@ -459,7 +468,7 @@ func (s *Source) addReposByOrg(ctx context.Context, apiClient *github.Client, or
continue
}
if err != nil {
return fmt.Errorf("could not list repos for org %s: %w", org, err)
return nil, fmt.Errorf("could not list repos for org %s: %w", org, err)
}
if len(someRepos) == 0 {
break
@ -472,7 +481,7 @@ func (s *Source) addReposByOrg(ctx context.Context, apiClient *github.Client, or
continue
}
}
common.AddStringSliceItem(r.GetCloneURL(), &s.repos)
repos = append(repos, r.GetCloneURL())
}
if res.NextPage == 0 {
break
@ -480,10 +489,23 @@ func (s *Source) addReposByOrg(ctx context.Context, apiClient *github.Client, or
opts.Page = res.NextPage
}
log.WithField("org", org).Debugf("Found %d repos (%d forks)", numRepos, numForks)
return repos, nil
}
func (s *Source) addReposByOrg(ctx context.Context, apiClient *github.Client, org string) error {
repos, err := s.getReposByOrg(ctx, apiClient, org)
if err != nil {
return err
}
// add the repos to the set of repos
for _, repo := range repos {
common.AddStringSliceItem(repo, &s.repos)
}
return nil
}
func (s *Source) addReposByUser(ctx context.Context, apiClient *github.Client, user string) error {
func (s *Source) getReposByUser(ctx context.Context, apiClient *github.Client, user string) ([]string, error) {
repos := []string{}
opts := &github.RepositoryListOptions{
ListOptions: github.ListOptions{
PerPage: 50,
@ -498,23 +520,36 @@ func (s *Source) addReposByUser(ctx context.Context, apiClient *github.Client, u
continue
}
if err != nil {
return fmt.Errorf("could not list repos for user %s: %w", user, err)
return nil, fmt.Errorf("could not list repos for user %s: %w", user, err)
}
for _, r := range someRepos {
if r.GetFork() && !s.conn.IncludeForks {
continue
}
common.AddStringSliceItem(r.GetCloneURL(), &s.repos)
repos = append(repos, r.GetCloneURL())
}
if res.NextPage == 0 {
break
}
opts.Page = res.NextPage
}
return repos, nil
}
func (s *Source) addReposByUser(ctx context.Context, apiClient *github.Client, user string) error {
repos, err := s.getReposByUser(ctx, apiClient, user)
if err != nil {
return err
}
// add the repos to the set of repos
for _, repo := range repos {
common.AddStringSliceItem(repo, &s.repos)
}
return nil
}
func (s *Source) addGistsByUser(ctx context.Context, apiClient *github.Client, user string) {
func (s *Source) getGistsByUser(ctx context.Context, apiClient *github.Client, user string) ([]string, error) {
gistURLs := []string{}
gistOpts := &github.GistListOptions{}
for {
gists, resp, err := apiClient.Gists.List(ctx, user, gistOpts)
@ -525,21 +560,33 @@ func (s *Source) addGistsByUser(ctx context.Context, apiClient *github.Client, u
continue
}
if err != nil {
log.WithError(err).Warnf("Could not get gists for user %s", user)
log.WithError(err).Warnf("could not list repos for user %s", user)
return nil, fmt.Errorf("could not list repos for user %s: %w", user, err)
}
for _, gist := range gists {
common.AddStringSliceItem(gist.GetGitPullURL(), &s.repos)
gistURLs = append(gistURLs, gist.GetGitPullURL())
}
if resp == nil || resp.NextPage == 0 {
break
}
gistOpts.Page = resp.NextPage
}
return
return gistURLs, nil
}
func (s *Source) addGistsByUser(ctx context.Context, apiClient *github.Client, user string) error {
gists, err := s.getGistsByUser(ctx, apiClient, user)
if err != nil {
return err
}
// add the gists to the set of repos
for _, gist := range gists {
common.AddStringSliceItem(gist, &s.repos)
}
return nil
}
func (s *Source) addMembersByApp(ctx context.Context, installationClient *github.Client, apiClient *github.Client) error {
opts := &github.ListOptions{
PerPage: 500,
}
@ -650,24 +697,34 @@ func (s *Source) addOrgsByUser(ctx context.Context, apiClient *github.Client, us
func (s *Source) normalizeRepos(ctx context.Context, apiClient *github.Client) {
// TODO: Add check/fix for repos that are missing scheme
repoIter := make([]string, len(s.repos))
copy(repoIter, s.repos)
for _, repo := range repoIter {
if parts := strings.Split(repo, "/"); len(parts) == 1 {
origSources := len(s.repos)
s.addGistsByUser(ctx, apiClient, repo)
if err := s.addReposByUser(ctx, apiClient, repo); err != nil {
log.WithError(err).Error("error fetching repos by user")
}
if origSources != len(s.repos) {
common.RemoveStringSliceItem(repo, &s.repos)
normalizedRepos := map[string]struct{}{}
for _, repo := range s.repos {
// if there's a '/', assume it's a URL and try to normalize it
if strings.ContainsRune(repo, '/') {
repoNormalized, err := giturl.NormalizeGithubRepo(repo)
if err != nil {
log.WithError(err).Warnf("Repo not in expected format: %s", repo)
continue
}
normalizedRepos[repoNormalized] = struct{}{}
continue
}
repoNormalized, err := giturl.NormalizeGithubRepo(repo)
if err != nil {
log.WithError(err).Warnf("Repo not in expected format: %s", repo)
// otherwise, assume it's a user and enumerate repositories and gists
if repos, err := s.getReposByUser(ctx, apiClient, repo); err == nil {
for _, repo := range repos {
normalizedRepos[repo] = struct{}{}
}
}
common.AddStringSliceItem(repoNormalized, &s.repos)
if gists, err := s.getGistsByUser(ctx, apiClient, repo); err == nil {
for _, gist := range gists {
normalizedRepos[gist] = struct{}{}
}
}
}
// replace s.repos
s.repos = s.repos[:0]
for key := range normalizedRepos {
s.repos = append(s.repos, key)
}
}

587
pkg/sources/github/github_integration_test.go Normal file
View file

@ -0,0 +1,587 @@
//go:build integration
// +build integration
package github
import (
"context"
"encoding/base64"
"fmt"
"os"
"testing"
"time"
"github.com/google/go-github/v42/github"
"github.com/kylelemons/godebug/pretty"
"github.com/mattn/go-colorable"
log "github.com/sirupsen/logrus"
"google.golang.org/protobuf/types/known/anypb"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/credentialspb"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
)
func TestSource_Scan(t *testing.T) {
os.Setenv("DO_NOT_RANDOMIZE", "true")
ctx, cancel := context.WithTimeout(context.Background(), time.Second*300)
defer cancel()
secret, err := common.GetTestSecret(ctx)
if err != nil {
t.Fatal(fmt.Errorf("failed to access secret: %v", err))
}
// For the personal access token test
githubToken := secret.MustGetField("GITHUB_TOKEN")
//For the NEW github app test (+Member enum)
githubPrivateKeyB64New := secret.MustGetField("GITHUB_PRIVATE_KEY_NEW")
githubPrivateKeyBytesNew, err := base64.StdEncoding.DecodeString(githubPrivateKeyB64New)
if err != nil {
t.Fatal(err)
}
githubPrivateKeyNew := string(githubPrivateKeyBytesNew)
githubInstallationIDNew := secret.MustGetField("GITHUB_INSTALLATION_ID_NEW")
githubAppIDNew := secret.MustGetField("GITHUB_APP_ID_NEW")
//OLD app for breaking app change tests
// githubPrivateKeyB64 := secret.MustGetField("GITHUB_PRIVATE_KEY")
// githubPrivateKeyBytes, err := base64.StdEncoding.DecodeString(githubPrivateKeyB64)
// if err != nil {
// t.Fatal(err)
// }
// githubPrivateKey := string(githubPrivateKeyBytes)
// githubInstallationID := secret.MustGetField("GITHUB_INSTALLATION_ID")
// githubAppID := secret.MustGetField("GITHUB_APP_ID")
type init struct {
name string
verify bool
connection *sourcespb.GitHub
}
tests := []struct {
name string
init init
wantChunk *sources.Chunk
wantErr bool
minRepo int
minOrg int
}{
{
name: "token authenticated, single repo",
init: init{
name: "test source",
connection: &sourcespb.GitHub{
Repositories: []string{"https://github.com/dustin-decker/secretsandstuff.git"},
Credential: &sourcespb.GitHub_Token{
Token: githubToken,
},
},
},
wantChunk: &sources.Chunk{
SourceType: sourcespb.SourceType_SOURCE_TYPE_GITHUB,
SourceName: "test source",
SourceMetadata: &source_metadatapb.MetaData{
Data: &source_metadatapb.MetaData_Github{
Github: &source_metadatapb.Github{
Repository: "https://github.com/dustin-decker/secretsandstuff.git",
},
},
},
Verify: false,
},
wantErr: false,
},
{
name: "token authenticated, single repo, no .git",
init: init{
name: "test source",
connection: &sourcespb.GitHub{
Repositories: []string{"https://github.com/dustin-decker/secretsandstuff"},
Credential: &sourcespb.GitHub_Token{
Token: githubToken,
},
},
},
wantChunk: &sources.Chunk{
SourceType: sourcespb.SourceType_SOURCE_TYPE_GITHUB,
SourceName: "test source",
SourceMetadata: &source_metadatapb.MetaData{
Data: &source_metadatapb.MetaData_Github{
Github: &source_metadatapb.Github{
Repository: "https://github.com/dustin-decker/secretsandstuff.git",
},
},
},
Verify: false,
},
wantErr: false,
},
{
name: "token authenticated, single org",
init: init{
name: "test source",
connection: &sourcespb.GitHub{
Organizations: []string{"trufflesecurity"},
Credential: &sourcespb.GitHub_Token{
Token: githubToken,
},
},
},
wantChunk: nil,
wantErr: false,
minRepo: 3,
minOrg: 0,
},
{
name: "token authenticated, username in org",
init: init{
name: "test source",
connection: &sourcespb.GitHub{
Organizations: []string{"dustin-decker"},
Credential: &sourcespb.GitHub_Token{
Token: githubToken,
},
},
},
wantChunk: nil,
wantErr: false,
minRepo: 3,
minOrg: 0,
},
{
name: "token authenticated, username in repo",
init: init{
name: "test source",
connection: &sourcespb.GitHub{
Repositories: []string{"dustin-decker"},
Credential: &sourcespb.GitHub_Token{
Token: githubToken,
},
},
},
wantChunk: nil,
wantErr: false,
minRepo: 3,
minOrg: 0,
},
{
name: "token authenticated, org in repo",
// I do not think that this is a supported case, but adding the test to specify there is no requirement.
init: init{
name: "test source",
connection: &sourcespb.GitHub{
Repositories: []string{"trufflesecurity"},
Credential: &sourcespb.GitHub_Token{
Token: githubToken,
},
},
},
wantChunk: nil,
wantErr: false,
minRepo: 0,
minOrg: 0,
},
/*
{
name: "token authenticated, no org or user (enum)",
// This configuration currently will only find gists from the user. No repos or orgs will be scanned.
init: init{
name: "test source",
connection: &sourcespb.GitHub{
Credential: &sourcespb.GitHub_Token{
Token: githubToken,
},
},
},
wantChunk: nil,
wantErr: false,
minRepo: 0,
minOrg: 0,
},
{
name: "app authenticated (old), no repo or org (enum)",
init: init{
name: "test source",
connection: &sourcespb.GitHub{
ScanUsers: false,
Credential: &sourcespb.GitHub_GithubApp{
GithubApp: &credentialspb.GitHubApp{
PrivateKey: githubPrivateKey,
InstallationId: githubInstallationID,
AppId: githubAppID,
},
},
},
},
wantChunk: nil,
wantErr: false,
minRepo: 3,
minOrg: 0,
},
*/
{
name: "unauthenticated, single org",
init: init{
name: "test source",
connection: &sourcespb.GitHub{
Organizations: []string{"trufflesecurity"},
Credential: &sourcespb.GitHub_Unauthenticated{},
},
},
wantChunk: nil,
wantErr: false,
minRepo: 3,
minOrg: 1,
},
{
name: "unauthenticated, single repo",
init: init{
name: "test source",
connection: &sourcespb.GitHub{
Repositories: []string{"https://github.com/trufflesecurity/driftwood.git"},
Credential: &sourcespb.GitHub_Unauthenticated{},
},
},
wantChunk: &sources.Chunk{
SourceType: sourcespb.SourceType_SOURCE_TYPE_GITHUB,
SourceName: "test source",
SourceMetadata: &source_metadatapb.MetaData{
Data: &source_metadatapb.MetaData_Github{
Github: &source_metadatapb.Github{
Repository: "https://github.com/trufflesecurity/driftwood.git",
},
},
},
Verify: false,
},
wantErr: false,
},
/*
{
name: "app authenticated, no repo or org",
init: init{
name: "test source",
connection: &sourcespb.GitHub{
ScanUsers: true,
Credential: &sourcespb.GitHub_GithubApp{
GithubApp: &credentialspb.GitHubApp{
PrivateKey: githubPrivateKeyNew,
InstallationId: githubInstallationIDNew,
AppId: githubAppIDNew,
},
},
},
},
wantChunk: nil,
wantErr: false,
minRepo: 3,
minOrg: 0,
},
*/
{
name: "app authenticated, single repo",
init: init{
name: "test source",
connection: &sourcespb.GitHub{
Repositories: []string{"https://github.com/trufflesecurity/driftwood.git"},
Credential: &sourcespb.GitHub_GithubApp{
GithubApp: &credentialspb.GitHubApp{
PrivateKey: githubPrivateKeyNew,
InstallationId: githubInstallationIDNew,
AppId: githubAppIDNew,
},
},
},
},
wantChunk: &sources.Chunk{
SourceType: sourcespb.SourceType_SOURCE_TYPE_GITHUB,
SourceName: "test source",
SourceMetadata: &source_metadatapb.MetaData{
Data: &source_metadatapb.MetaData_Github{
Github: &source_metadatapb.Github{
Repository: "https://github.com/trufflesecurity/driftwood.git",
},
},
},
Verify: false,
},
wantErr: false,
minRepo: 1,
minOrg: 0,
},
{
name: "app authenticated, single org",
init: init{
name: "test source",
connection: &sourcespb.GitHub{
Organizations: []string{"trufflesecurity"},
Credential: &sourcespb.GitHub_GithubApp{
GithubApp: &credentialspb.GitHubApp{
PrivateKey: githubPrivateKeyNew,
InstallationId: githubInstallationIDNew,
AppId: githubAppIDNew,
},
},
},
},
wantChunk: nil,
wantErr: false,
minRepo: 3,
minOrg: 1,
},
}
for i, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
log.Debugf("Beginning test %d: %s", i, tt.name)
s := Source{}
log.SetLevel(log.DebugLevel)
//uncomment for windows Testing
log.SetFormatter(&log.TextFormatter{ForceColors: true})
log.SetOutput(colorable.NewColorableStdout())
conn, err := anypb.New(tt.init.connection)
if err != nil {
t.Fatal(err)
}
err = s.Init(ctx, tt.init.name, 0, 0, tt.init.verify, conn, 4)
if (err != nil) != tt.wantErr {
t.Errorf("Source.Init() error = %v, wantErr %v", err, tt.wantErr)
return
}
chunksCh := make(chan *sources.Chunk, 5)
go func() {
err = s.Chunks(ctx, chunksCh)
if (err != nil) != tt.wantErr {
if ctx.Err() != nil {
return
}
t.Errorf("Source.Chunks() error = %v, wantErr %v", err, tt.wantErr)
return
}
}()
if err = common.HandleTestChannel(chunksCh, basicCheckFunc(tt.minOrg, tt.minRepo, tt.wantChunk, &s)); err != nil {
t.Error(err)
}
})
}
}
func TestSource_paginateGists(t *testing.T) {
os.Setenv("DO_NOT_RANDOMIZE", "true")
ctx, cancel := context.WithTimeout(context.Background(), time.Second*30)
defer cancel()
secret, err := common.GetTestSecret(ctx)
if err != nil {
t.Fatal(fmt.Errorf("failed to access secret: %v", err))
}
//For the NEW github app test (+Member enum)
githubPrivateKeyB64New := secret.MustGetField("GITHUB_PRIVATE_KEY_NEW")
githubPrivateKeyBytesNew, err := base64.StdEncoding.DecodeString(githubPrivateKeyB64New)
if err != nil {
t.Fatal(err)
}
githubPrivateKeyNew := string(githubPrivateKeyBytesNew)
githubInstallationIDNew := secret.MustGetField("GITHUB_INSTALLATION_ID_NEW")
githubAppIDNew := secret.MustGetField("GITHUB_APP_ID_NEW")
type init struct {
name string
verify bool
connection *sourcespb.GitHub
}
tests := []struct {
name string
init init
wantChunk *sources.Chunk
wantErr bool
user string
minRepos int
}{
{
name: "get gist secret",
init: init{
name: "test source",
connection: &sourcespb.GitHub{
Credential: &sourcespb.GitHub_GithubApp{
GithubApp: &credentialspb.GitHubApp{
PrivateKey: githubPrivateKeyNew,
InstallationId: githubInstallationIDNew,
AppId: githubAppIDNew,
},
},
},
},
wantChunk: &sources.Chunk{
SourceName: "test source",
SourceMetadata: &source_metadatapb.MetaData{
Data: &source_metadatapb.MetaData_Github{
Github: &source_metadatapb.Github{
Repository: "https://gist.github.com/be45ad1ebabe98482d9c0bb80c07c619.git",
},
},
},
Verify: false,
},
wantErr: false,
user: "dustin-decker",
minRepos: 1,
},
{
name: "get multiple pages of gists",
init: init{
name: "test source",
connection: &sourcespb.GitHub{
Credential: &sourcespb.GitHub_GithubApp{
GithubApp: &credentialspb.GitHubApp{
PrivateKey: githubPrivateKeyNew,
InstallationId: githubInstallationIDNew,
AppId: githubAppIDNew,
},
},
},
},
wantChunk: nil,
wantErr: false,
user: "andrew",
minRepos: 101,
},
/* {
name: "get multiple pages of gists",
init: init{
name: "test source",
connection: &sourcespb.GitHub{
Credential: &sourcespb.GitHub_GithubApp{
GithubApp: &credentialspb.GitHubApp{
PrivateKey: githubPrivateKeyNew,
InstallationId: githubInstallationIDNew,
AppId: githubAppIDNew,
},
},
},
},
wantChunk: &sources.Chunk{
SourceName: "test source",
SourceMetadata: &source_metadatapb.MetaData{
Data: &source_metadatapb.MetaData_Github{
Github: &source_metadatapb.Github{
Repository: "https://gist.github.com/872df3b78b9ec3e7dbe597fb5a202121.git",
},
},
},
Verify: false,
},
wantErr: false,
user: "andrew",
},
*/
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
s := Source{}
log.SetLevel(log.DebugLevel)
//uncomment for windows Testing
log.SetFormatter(&log.TextFormatter{ForceColors: true})
log.SetOutput(colorable.NewColorableStdout())
conn, err := anypb.New(tt.init.connection)
if err != nil {
t.Fatal(err)
}
err = s.Init(ctx, tt.init.name, 0, 0, tt.init.verify, conn, 4)
if (err != nil) != tt.wantErr {
t.Errorf("Source.Init() error = %v, wantErr %v", err, tt.wantErr)
return
}
chunksCh := make(chan *sources.Chunk, 5)
go func() {
s.addGistsByUser(ctx, github.NewClient(s.httpClient), tt.user)
chunksCh <- &sources.Chunk{}
}()
var wantedRepo string
if tt.wantChunk != nil {
wantedRepo = tt.wantChunk.SourceMetadata.GetGithub().Repository
}
if err = common.HandleTestChannel(chunksCh, gistsCheckFunc(wantedRepo, tt.minRepos, &s)); err != nil {
t.Error(err)
}
})
}
}
func gistsCheckFunc(expected string, minRepos int, s *Source) common.ChunkFunc {
return func(chunk *sources.Chunk) error {
if minRepos != 0 && minRepos > len(s.repos) {
return fmt.Errorf("didn't find enough repos. expected: %d, got :%d", minRepos, len(s.repos))
}
if expected != "" {
for _, repo := range s.repos {
if repo == expected {
return nil
}
}
return fmt.Errorf("expected repo not included: %s", expected)
}
return nil
}
}
func basicCheckFunc(minOrg, minRepo int, wantChunk *sources.Chunk, s *Source) common.ChunkFunc {
return func(chunk *sources.Chunk) error {
if minOrg != 0 && minOrg > len(s.orgs) {
return fmt.Errorf("incorrect number of orgs. expected at least: %d, got %d", minOrg, len(s.orgs))
}
if minRepo != 0 && minRepo > len(s.repos) {
return fmt.Errorf("incorrect number of repos. expected at least: %d, got %d", minRepo, len(s.repos))
}
if wantChunk != nil {
if diff := pretty.Compare(chunk.SourceMetadata.GetGithub().Repository, wantChunk.SourceMetadata.GetGithub().Repository); diff == "" {
return nil
}
return common.MatchError
}
return nil
}
}
// func TestSource_paginateRepos(t *testing.T) {
// type args struct {
// ctx context.Context
// apiClient *github.Client
// }
// tests := []struct {
// name string
// org string
// args args
// }{
// {
// org: "fakeNetflix",
// args: args{
// ctx: context.Background(),
// apiClient: github.NewClient(common.SaneHttpClient()),
// },
// },
// }
// for _, tt := range tests {
// t.Run(tt.name, func(t *testing.T) {
// s := &Source{httpClient: common.SaneHttpClient()}
// s.paginateRepos(tt.args.ctx, tt.args.apiClient, tt.org)
// if len(s.repos) < 101 {
// t.Errorf("expected > 100 repos, got %d", len(s.repos))
// }
// })
// }
// }

841
pkg/sources/github/github_test.go
View file

@ -1,584 +1,339 @@
package github
import (
"bytes"
"context"
"encoding/base64"
"crypto/rand"
"crypto/rsa"
"crypto/x509"
"encoding/pem"
"fmt"
"os"
"net/http"
"sort"
"strconv"
"testing"
"time"
"github.com/google/go-github/v42/github"
"github.com/kylelemons/godebug/pretty"
"github.com/mattn/go-colorable"
log "github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert"
"google.golang.org/protobuf/types/known/anypb"
"gopkg.in/h2non/gock.v1"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/credentialspb"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
)
func TestSource_Scan(t *testing.T) {
os.Setenv("DO_NOT_RANDOMIZE", "true")
ctx, cancel := context.WithTimeout(context.Background(), time.Second*300)
defer cancel()
secret, err := common.GetTestSecret(ctx)
func createTestSource(src *sourcespb.GitHub) (*Source, *anypb.Any) {
s := &Source{}
conn, err := anypb.New(src)
if err != nil {
t.Fatal(fmt.Errorf("failed to access secret: %v", err))
}
// For the personal access token test
githubToken := secret.MustGetField("GITHUB_TOKEN")
//For the NEW github app test (+Member enum)
githubPrivateKeyB64New := secret.MustGetField("GITHUB_PRIVATE_KEY_NEW")
githubPrivateKeyBytesNew, err := base64.StdEncoding.DecodeString(githubPrivateKeyB64New)
if err != nil {
t.Fatal(err)
}
githubPrivateKeyNew := string(githubPrivateKeyBytesNew)
githubInstallationIDNew := secret.MustGetField("GITHUB_INSTALLATION_ID_NEW")
githubAppIDNew := secret.MustGetField("GITHUB_APP_ID_NEW")
//OLD app for breaking app change tests
// githubPrivateKeyB64 := secret.MustGetField("GITHUB_PRIVATE_KEY")
// githubPrivateKeyBytes, err := base64.StdEncoding.DecodeString(githubPrivateKeyB64)
// if err != nil {
// t.Fatal(err)
// }
// githubPrivateKey := string(githubPrivateKeyBytes)
// githubInstallationID := secret.MustGetField("GITHUB_INSTALLATION_ID")
// githubAppID := secret.MustGetField("GITHUB_APP_ID")
type init struct {
name string
verify bool
connection *sourcespb.GitHub
}
tests := []struct {
name string
init init
wantChunk *sources.Chunk
wantErr bool
minRepo int
minOrg int
}{
{
name: "token authenticated, single repo",
init: init{
name: "test source",
connection: &sourcespb.GitHub{
Repositories: []string{"https://github.com/dustin-decker/secretsandstuff.git"},
Credential: &sourcespb.GitHub_Token{
Token: githubToken,
},
},
},
wantChunk: &sources.Chunk{
SourceType: sourcespb.SourceType_SOURCE_TYPE_GITHUB,
SourceName: "test source",
SourceMetadata: &source_metadatapb.MetaData{
Data: &source_metadatapb.MetaData_Github{
Github: &source_metadatapb.Github{
Repository: "https://github.com/dustin-decker/secretsandstuff.git",
},
},
},
Verify: false,
},
wantErr: false,
},
{
name: "token authenticated, single repo, no .git",
init: init{
name: "test source",
connection: &sourcespb.GitHub{
Repositories: []string{"https://github.com/dustin-decker/secretsandstuff"},
Credential: &sourcespb.GitHub_Token{
Token: githubToken,
},
},
},
wantChunk: &sources.Chunk{
SourceType: sourcespb.SourceType_SOURCE_TYPE_GITHUB,
SourceName: "test source",
SourceMetadata: &source_metadatapb.MetaData{
Data: &source_metadatapb.MetaData_Github{
Github: &source_metadatapb.Github{
Repository: "https://github.com/dustin-decker/secretsandstuff.git",
},
},
},
Verify: false,
},
wantErr: false,
},
{
name: "token authenticated, single org",
init: init{
name: "test source",
connection: &sourcespb.GitHub{
Organizations: []string{"trufflesecurity"},
Credential: &sourcespb.GitHub_Token{
Token: githubToken,
},
},
},
wantChunk: nil,
wantErr: false,
minRepo: 3,
minOrg: 0,
},
{
name: "token authenticated, username in org",
init: init{
name: "test source",
connection: &sourcespb.GitHub{
Organizations: []string{"dustin-decker"},
Credential: &sourcespb.GitHub_Token{
Token: githubToken,
},
},
},
wantChunk: nil,
wantErr: false,
minRepo: 3,
minOrg: 0,
},
{
name: "token authenticated, username in repo",
init: init{
name: "test source",
connection: &sourcespb.GitHub{
Repositories: []string{"dustin-decker"},
Credential: &sourcespb.GitHub_Token{
Token: githubToken,
},
},
},
wantChunk: nil,
wantErr: false,
minRepo: 3,
minOrg: 0,
},
{
name: "token authenticated, org in repo",
// I do not think that this is a supported case, but adding the test to specify there is no requirement.
init: init{
name: "test source",
connection: &sourcespb.GitHub{
Repositories: []string{"trufflesecurity"},
Credential: &sourcespb.GitHub_Token{
Token: githubToken,
},
},
},
wantChunk: nil,
wantErr: false,
minRepo: 0,
minOrg: 0,
},
/*
{
name: "token authenticated, no org or user (enum)",
// This configuration currently will only find gists from the user. No repos or orgs will be scanned.
init: init{
name: "test source",
connection: &sourcespb.GitHub{
Credential: &sourcespb.GitHub_Token{
Token: githubToken,
},
},
},
wantChunk: nil,
wantErr: false,
minRepo: 0,
minOrg: 0,
},
{
name: "app authenticated (old), no repo or org (enum)",
init: init{
name: "test source",
connection: &sourcespb.GitHub{
ScanUsers: false,
Credential: &sourcespb.GitHub_GithubApp{
GithubApp: &credentialspb.GitHubApp{
PrivateKey: githubPrivateKey,
InstallationId: githubInstallationID,
AppId: githubAppID,
},
},
},
},
wantChunk: nil,
wantErr: false,
minRepo: 3,
minOrg: 0,
},
*/
{
name: "unauthenticated, single org",
init: init{
name: "test source",
connection: &sourcespb.GitHub{
Organizations: []string{"trufflesecurity"},
Credential: &sourcespb.GitHub_Unauthenticated{},
},
},
wantChunk: nil,
wantErr: false,
minRepo: 3,
minOrg: 1,
},
{
name: "unauthenticated, single repo",
init: init{
name: "test source",
connection: &sourcespb.GitHub{
Repositories: []string{"https://github.com/trufflesecurity/driftwood.git"},
Credential: &sourcespb.GitHub_Unauthenticated{},
},
},
wantChunk: &sources.Chunk{
SourceType: sourcespb.SourceType_SOURCE_TYPE_GITHUB,
SourceName: "test source",
SourceMetadata: &source_metadatapb.MetaData{
Data: &source_metadatapb.MetaData_Github{
Github: &source_metadatapb.Github{
Repository: "https://github.com/trufflesecurity/driftwood.git",
},
},
},
Verify: false,
},
wantErr: false,
},
/*
{
name: "app authenticated, no repo or org",
init: init{
name: "test source",
connection: &sourcespb.GitHub{
ScanUsers: true,
Credential: &sourcespb.GitHub_GithubApp{
GithubApp: &credentialspb.GitHubApp{
PrivateKey: githubPrivateKeyNew,
InstallationId: githubInstallationIDNew,
AppId: githubAppIDNew,
},
},
},
},
wantChunk: nil,
wantErr: false,
minRepo: 3,
minOrg: 0,
},
*/
{
name: "app authenticated, single repo",
init: init{
name: "test source",
connection: &sourcespb.GitHub{
Repositories: []string{"https://github.com/trufflesecurity/driftwood.git"},
Credential: &sourcespb.GitHub_GithubApp{
GithubApp: &credentialspb.GitHubApp{
PrivateKey: githubPrivateKeyNew,
InstallationId: githubInstallationIDNew,
AppId: githubAppIDNew,
},
},
},
},
wantChunk: &sources.Chunk{
SourceType: sourcespb.SourceType_SOURCE_TYPE_GITHUB,
SourceName: "test source",
SourceMetadata: &source_metadatapb.MetaData{
Data: &source_metadatapb.MetaData_Github{
Github: &source_metadatapb.Github{
Repository: "https://github.com/trufflesecurity/driftwood.git",
},
},
},
Verify: false,
},
wantErr: false,
minRepo: 1,
minOrg: 0,
},
{
name: "app authenticated, single org",
init: init{
name: "test source",
connection: &sourcespb.GitHub{
Organizations: []string{"trufflesecurity"},
Credential: &sourcespb.GitHub_GithubApp{
GithubApp: &credentialspb.GitHubApp{
PrivateKey: githubPrivateKeyNew,
InstallationId: githubInstallationIDNew,
AppId: githubAppIDNew,
},
},
},
},
wantChunk: nil,
wantErr: false,
minRepo: 3,
minOrg: 1,
},
}
for i, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
log.Debugf("Beginning test %d: %s", i, tt.name)
s := Source{}
log.SetLevel(log.DebugLevel)
//uncomment for windows Testing
log.SetFormatter(&log.TextFormatter{ForceColors: true})
log.SetOutput(colorable.NewColorableStdout())
conn, err := anypb.New(tt.init.connection)
if err != nil {
t.Fatal(err)
}
err = s.Init(ctx, tt.init.name, 0, 0, tt.init.verify, conn, 4)
if (err != nil) != tt.wantErr {
t.Errorf("Source.Init() error = %v, wantErr %v", err, tt.wantErr)
return
}
chunksCh := make(chan *sources.Chunk, 5)
go func() {
err = s.Chunks(ctx, chunksCh)
if (err != nil) != tt.wantErr {
if ctx.Err() != nil {
return
}
t.Errorf("Source.Chunks() error = %v, wantErr %v", err, tt.wantErr)
return
}
}()
if err = common.HandleTestChannel(chunksCh, basicCheckFunc(tt.minOrg, tt.minRepo, tt.wantChunk, &s)); err != nil {
t.Error(err)
}
})
panic(err)
}
return s, conn
}
func TestSource_paginateGists(t *testing.T) {
os.Setenv("DO_NOT_RANDOMIZE", "true")
ctx, cancel := context.WithTimeout(context.Background(), time.Second*30)
defer cancel()
secret, err := common.GetTestSecret(ctx)
if err != nil {
t.Fatal(fmt.Errorf("failed to access secret: %v", err))
}
//For the NEW github app test (+Member enum)
githubPrivateKeyB64New := secret.MustGetField("GITHUB_PRIVATE_KEY_NEW")
githubPrivateKeyBytesNew, err := base64.StdEncoding.DecodeString(githubPrivateKeyB64New)
if err != nil {
t.Fatal(err)
}
githubPrivateKeyNew := string(githubPrivateKeyBytesNew)
githubInstallationIDNew := secret.MustGetField("GITHUB_INSTALLATION_ID_NEW")
githubAppIDNew := secret.MustGetField("GITHUB_APP_ID_NEW")
type init struct {
name string
verify bool
connection *sourcespb.GitHub
func initTestSource(src *sourcespb.GitHub) *Source {
s, conn := createTestSource(src)
if err := s.Init(context.TODO(), "test - github", 0, 1337, false, conn, 1); err != nil {
panic(err)
}
return s
}
func TestInit(t *testing.T) {
source, conn := createTestSource(&sourcespb.GitHub{
Repositories: []string{"https://github.com/dustin-decker/secretsandstuff.git"},
Credential: &sourcespb.GitHub_Token{
Token: "super secret token",
},
})
err := source.Init(context.TODO(), "test - github", 0, 1337, false, conn, 1)
assert.Nil(t, err)
// TODO: test error case
}
func TestAddReposByOrg(t *testing.T) {
defer gock.Off()
gock.New("https://api.github.com").
Get("/orgs/super-secret-org/repos").
Reply(200).
JSON([]map[string]string{{"clone_url": "super-secret-repo"}})
s := initTestSource(nil)
// gock works here because github.NewClient is using the default HTTP Transport
err := s.addReposByOrg(context.TODO(), github.NewClient(nil), "super-secret-org")
assert.Nil(t, err)
assert.Equal(t, 1, len(s.repos))
assert.Equal(t, []string{"super-secret-repo"}, s.repos)
assert.True(t, gock.IsDone())
}
func TestAddReposByUser(t *testing.T) {
defer gock.Off()
gock.New("https://api.github.com").
Get("/users/super-secret-user/repos").
Reply(200).
JSON([]map[string]string{{"clone_url": "super-secret-repo"}})
s := initTestSource(nil)
err := s.addReposByUser(context.TODO(), github.NewClient(nil), "super-secret-user")
assert.Nil(t, err)
assert.Equal(t, 1, len(s.repos))
assert.Equal(t, []string{"super-secret-repo"}, s.repos)
assert.True(t, gock.IsDone())
}
func TestAddGistsByUser(t *testing.T) {
defer gock.Off()
gock.New("https://api.github.com").
Get("/users/super-secret-user/gists").
Reply(200).
JSON([]map[string]string{{"git_pull_url": "super-secret-gist"}})
s := initTestSource(nil)
err := s.addGistsByUser(context.TODO(), github.NewClient(nil), "super-secret-user")
assert.Nil(t, err)
assert.Equal(t, 1, len(s.repos))
assert.Equal(t, []string{"super-secret-gist"}, s.repos)
assert.True(t, gock.IsDone())
}
func TestAddMembersByApp(t *testing.T) {
defer gock.Off()
gock.New("https://api.github.com").
Get("/app/installations").
Reply(200).
JSON([]map[string]interface{}{
{"account": map[string]string{"login": "super-secret-org"}},
})
gock.New("https://api.github.com").
Get("/orgs/super-secret-org/members").
Reply(200).
JSON([]map[string]interface{}{
{"login": "ssm1"},
{"login": "ssm2"},
{"login": "ssm3"},
})
s := initTestSource(nil)
err := s.addMembersByApp(context.TODO(), github.NewClient(nil), github.NewClient(nil))
assert.Nil(t, err)
assert.Equal(t, 3, len(s.members))
assert.Equal(t, []string{"ssm1", "ssm2", "ssm3"}, s.members)
assert.True(t, gock.IsDone())
}
func TestAddReposByApp(t *testing.T) {
defer gock.Off()
gock.New("https://api.github.com").
Get("/installation/repositories").
Reply(200).
JSON(map[string]interface{}{
"repositories": []map[string]string{
{"clone_url": "ssr1"},
{"clone_url": "ssr2"},
},
})
s := initTestSource(nil)
err := s.addReposByApp(context.TODO(), github.NewClient(nil))
assert.Nil(t, err)
assert.Equal(t, 2, len(s.repos))
assert.Equal(t, []string{"ssr1", "ssr2"}, s.repos)
assert.True(t, gock.IsDone())
}
func TestAddOrgsByUser(t *testing.T) {
defer gock.Off()
// NOTE: addOrgsByUser calls /user/orgs to get the orgs of the
// authenticated user
gock.New("https://api.github.com").
Get("/user/orgs").
Reply(200).
JSON([]map[string]interface{}{
{"name": "sso1"},
{"login": "sso2"},
})
s := initTestSource(nil)
s.addOrgsByUser(context.TODO(), github.NewClient(nil), "super-secret-user")
assert.Equal(t, 2, len(s.orgs))
assert.Equal(t, []string{"sso1", "sso2"}, s.orgs)
assert.True(t, gock.IsDone())
}
func TestNormalizeRepos(t *testing.T) {
defer gock.Off()
tests := []struct {
name string
init init
wantChunk *sources.Chunk
wantErr bool
user string
minRepos int
name string
setup func()
repos []string
expected []string
}{
{
name: "get gist secret",
init: init{
name: "test source",
connection: &sourcespb.GitHub{
Credential: &sourcespb.GitHub_GithubApp{
GithubApp: &credentialspb.GitHubApp{
PrivateKey: githubPrivateKeyNew,
InstallationId: githubInstallationIDNew,
AppId: githubAppIDNew,
},
},
},
},
wantChunk: &sources.Chunk{
SourceName: "test source",
SourceMetadata: &source_metadatapb.MetaData{
Data: &source_metadatapb.MetaData_Github{
Github: &source_metadatapb.Github{
Repository: "https://gist.github.com/be45ad1ebabe98482d9c0bb80c07c619.git",
},
},
},
Verify: false,
},
wantErr: false,
user: "dustin-decker",
minRepos: 1,
name: "repo url",
setup: func() {},
repos: []string{"https://github.com/super-secret-user/super-secret-repo"},
expected: []string{"https://github.com/super-secret-user/super-secret-repo.git"},
},
{
name: "get multiple pages of gists",
init: init{
name: "test source",
connection: &sourcespb.GitHub{
Credential: &sourcespb.GitHub_GithubApp{
GithubApp: &credentialspb.GitHubApp{
PrivateKey: githubPrivateKeyNew,
InstallationId: githubInstallationIDNew,
AppId: githubAppIDNew,
},
},
},
name: "username with gists",
setup: func() {
gock.New("https://api.github.com").
Get("/users/super-secret-user/gists").
Reply(200).
JSON([]map[string]string{{"git_pull_url": "https://github.com/super-secret-user/super-secret-gist.git"}})
gock.New("https://api.github.com").
Get("/users/super-secret-user/repos").
Reply(200).
JSON([]map[string]string{{"clone_url": "https://github.com/super-secret-user/super-secret-repo.git"}})
},
repos: []string{"super-secret-user"},
expected: []string{
"https://github.com/super-secret-user/super-secret-repo.git",
"https://github.com/super-secret-user/super-secret-gist.git",
},
wantChunk: nil,
wantErr: false,
user: "andrew",
minRepos: 101,
},
/* {
name: "get multiple pages of gists",
init: init{
name: "test source",
connection: &sourcespb.GitHub{
Credential: &sourcespb.GitHub_GithubApp{
GithubApp: &credentialspb.GitHubApp{
PrivateKey: githubPrivateKeyNew,
InstallationId: githubInstallationIDNew,
AppId: githubAppIDNew,
},
},
},
},
wantChunk: &sources.Chunk{
SourceName: "test source",
SourceMetadata: &source_metadatapb.MetaData{
Data: &source_metadatapb.MetaData_Github{
Github: &source_metadatapb.Github{
Repository: "https://gist.github.com/872df3b78b9ec3e7dbe597fb5a202121.git",
},
},
},
Verify: false,
},
wantErr: false,
user: "andrew",
},
*/
{
name: "not found",
setup: func() {
gock.New("https://api.github.com").
Get("/users/not-found/gists").
Reply(404)
gock.New("https://api.github.com").
Get("/users/not-found/repos").
Reply(404)
},
repos: []string{"not-found"},
expected: []string{},
},
{
name: "unexpected format",
setup: func() {},
repos: []string{"/foo/"},
expected: []string{},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
s := Source{}
defer gock.Off()
tt.setup()
s := initTestSource(nil)
s.repos = tt.repos
log.SetLevel(log.DebugLevel)
//uncomment for windows Testing
log.SetFormatter(&log.TextFormatter{ForceColors: true})
log.SetOutput(colorable.NewColorableStdout())
s.normalizeRepos(context.TODO(), github.NewClient(nil))
assert.Equal(t, len(tt.expected), len(s.repos))
// sort and compare
sort.Slice(tt.expected, func(i, j int) bool { return tt.expected[i] < tt.expected[j] })
sort.Slice(s.repos, func(i, j int) bool { return s.repos[i] < s.repos[j] })
assert.Equal(t, tt.expected, s.repos)
conn, err := anypb.New(tt.init.connection)
if err != nil {
t.Fatal(err)
}
err = s.Init(ctx, tt.init.name, 0, 0, tt.init.verify, conn, 4)
if (err != nil) != tt.wantErr {
t.Errorf("Source.Init() error = %v, wantErr %v", err, tt.wantErr)
return
}
chunksCh := make(chan *sources.Chunk, 5)
go func() {
s.addGistsByUser(ctx, github.NewClient(s.httpClient), tt.user)
chunksCh <- &sources.Chunk{}
}()
var wantedRepo string
if tt.wantChunk != nil {
wantedRepo = tt.wantChunk.SourceMetadata.GetGithub().Repository
}
if err = common.HandleTestChannel(chunksCh, gistsCheckFunc(wantedRepo, tt.minRepos, &s)); err != nil {
t.Error(err)
}
assert.True(t, gock.IsDone())
})
}
}
func gistsCheckFunc(expected string, minRepos int, s *Source) common.ChunkFunc {
return func(chunk *sources.Chunk) error {
if minRepos != 0 && minRepos > len(s.repos) {
return fmt.Errorf("didn't find enough repos. expected: %d, got :%d", minRepos, len(s.repos))
}
if expected != "" {
for _, repo := range s.repos {
if repo == expected {
return nil
}
}
return fmt.Errorf("expected repo not included: %s", expected)
}
return nil
}
func TestHandleRateLimit(t *testing.T) {
assert.False(t, handleRateLimit(nil, nil))
err := &github.RateLimitError{}
res := &github.Response{Response: &http.Response{Header: make(http.Header)}}
res.Header.Set("x-ratelimit-remaining", "0")
res.Header.Set("x-ratelimit-reset", strconv.FormatInt(time.Now().Unix()+1, 10))
assert.True(t, handleRateLimit(err, res))
}
func basicCheckFunc(minOrg, minRepo int, wantChunk *sources.Chunk, s *Source) common.ChunkFunc {
return func(chunk *sources.Chunk) error {
if minOrg != 0 && minOrg > len(s.orgs) {
return fmt.Errorf("incorrect number of orgs. expected at least: %d, got %d", minOrg, len(s.orgs))
}
if minRepo != 0 && minRepo > len(s.repos) {
return fmt.Errorf("incorrect number of repos. expected at least: %d, got %d", minRepo, len(s.repos))
}
if wantChunk != nil {
if diff := pretty.Compare(chunk.SourceMetadata.GetGithub().Repository, wantChunk.SourceMetadata.GetGithub().Repository); diff == "" {
return nil
}
return common.MatchError
}
return nil
}
func TestEnumerateUnauthenticated(t *testing.T) {
defer gock.Off()
gock.New("https://api.github.com").
Get("/orgs/super-secret-org/repos").
Reply(200).
JSON([]map[string]string{{"clone_url": "super-secret-repo"}})
s := initTestSource(nil)
s.orgs = []string{"super-secret-org"}
_ = s.enumerateUnauthenticated(context.TODO())
assert.Equal(t, 1, len(s.repos))
assert.Equal(t, []string{"super-secret-repo"}, s.repos)
assert.True(t, gock.IsDone())
}
// func TestSource_paginateRepos(t *testing.T) {
// type args struct {
// ctx context.Context
// apiClient *github.Client
// }
// tests := []struct {
// name string
// org string
// args args
// }{
// {
// org: "fakeNetflix",
// args: args{
// ctx: context.Background(),
// apiClient: github.NewClient(common.SaneHttpClient()),
// },
// },
// }
// for _, tt := range tests {
// t.Run(tt.name, func(t *testing.T) {
// s := &Source{httpClient: common.SaneHttpClient()}
// s.paginateRepos(tt.args.ctx, tt.args.apiClient, tt.org)
// if len(s.repos) < 101 {
// t.Errorf("expected > 100 repos, got %d", len(s.repos))
// }
// })
// }
// }
func TestEnumerateWithToken(t *testing.T) {
defer gock.Off()
gock.New("https://api.github.com").
Get("/user").
Reply(200).
JSON(map[string]string{"login": "super-secret-user"})
gock.New("https://api.github.com").
Get("/users/super-secret-user/repos").
Reply(200).
JSON([]map[string]string{{"clone_url": "super-secret-repo"}})
s := initTestSource(nil)
_, err := s.enumerateWithToken(context.TODO(), "https://api.github.com", "token")
assert.Nil(t, err)
assert.Equal(t, 1, len(s.repos))
assert.Equal(t, []string{"super-secret-repo"}, s.repos)
assert.True(t, gock.IsDone())
}
func TestEnumerateWithApp(t *testing.T) {
defer gock.Off()
// generate a private key (it just needs to be in the right format)
privateKey := func() string {
key, err := rsa.GenerateKey(rand.Reader, 2048)
if err != nil {
panic(err)
}
data := x509.MarshalPKCS1PrivateKey(key)
var pemKey bytes.Buffer
if err := pem.Encode(&pemKey, &pem.Block{
Type: "RSA PRIVATE KEY",
Bytes: data,
}); err != nil {
panic(err)
}
return pemKey.String()
}()
gock.New("https://api.github.com").
Post("/app/installations/1337/access_tokens").
Reply(200).
JSON(map[string]string{"token": "dontlook"})
gock.New("https://api.github.com").
Get("/installation/repositories").
Reply(200).
JSON(map[string]string{})
s := initTestSource(nil)
_, _, err := s.enumerateWithApp(
context.TODO(),
"https://api.github.com",
&credentialspb.GitHubApp{
InstallationId: "1337",
AppId: "4141",
PrivateKey: privateKey,
},
)
fmt.Println(err)
assert.Nil(t, err)
assert.Equal(t, 0, len(s.repos))
assert.True(t, gock.IsDone())
}