mirror of
https://github.com/trufflesecurity/trufflehog.git
synced 2024-09-20 06:31:57 +00:00
Gitlab scan targets (#2470)
* add method to scan targets * Add logic to handle targetted scan * address comments * remove pagination opts * add kvp with scan type
This commit is contained in:
ahrav
GitHub
parent
4d231af19d
commit
9ef5151200
3 changed files with 510 additions and 311 deletions
|
|
@ -160,13 +160,20 @@ func (s *Source) Init(_ context.Context, name string, jobId sources.JobID, sourc
|
|||
}
|
||||
|
||||
// Chunks emits chunks of bytes over a channel.
|
||||
func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ ...sources.ChunkingTarget) error {
|
||||
func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, targets ...sources.ChunkingTarget) error {
|
||||
// Start client.
|
||||
apiClient, err := s.newClient()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// If targets are provided, we're only scanning the data in those targets.
|
||||
// Otherwise, we're scanning all data.
|
||||
// This allows us to only scan the commit where a vulnerability was found.
|
||||
if len(targets) > 0 {
|
||||
return s.scanTargets(ctx, apiClient, targets, chunksChan)
|
||||
}
|
||||
|
||||
gitlabReposScanned.WithLabelValues(s.name).Set(0)
|
||||
// Get repo within target.
|
||||
repos, errs := normalizeRepos(s.repos)
|
||||
|
|
@ -204,6 +211,62 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ .
|
|||
return s.scanRepos(ctx, chunksChan)
|
||||
}
|
||||
|
||||
func (s *Source) scanTargets(ctx context.Context, client *gitlab.Client, targets []sources.ChunkingTarget, chunksChan chan *sources.Chunk) error {
|
||||
ctx = context.WithValues(ctx, "scan_type", "targeted")
|
||||
for _, tgt := range targets {
|
||||
if err := s.scanTarget(ctx, client, tgt, chunksChan); err != nil {
|
||||
ctx.Logger().Error(err, "error scanning target")
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Source) scanTarget(ctx context.Context, client *gitlab.Client, target sources.ChunkingTarget, chunksChan chan *sources.Chunk) error {
|
||||
metaType, ok := target.QueryCriteria.GetData().(*source_metadatapb.MetaData_Gitlab)
|
||||
if !ok {
|
||||
return fmt.Errorf("unable to cast metadata type for targeted scan")
|
||||
}
|
||||
meta := metaType.Gitlab
|
||||
projID, sha := int(meta.GetProjectId()), meta.GetCommit()
|
||||
if projID == 0 || sha == "" {
|
||||
return fmt.Errorf("project ID and commit SHA must be provided for targeted scan")
|
||||
}
|
||||
|
||||
aCtx := context.WithValues(ctx, "project_id", projID, "commit", sha)
|
||||
|
||||
diffs, _, err := client.Commits.GetCommitDiff(projID, sha, new(gitlab.GetCommitDiffOptions), gitlab.WithContext(ctx))
|
||||
if err != nil {
|
||||
return fmt.Errorf("error fetching diffs for commit %s: %w", sha, err)
|
||||
}
|
||||
|
||||
for _, diff := range diffs {
|
||||
if diff.Diff == "" {
|
||||
aCtx.Logger().V(4).Info("skipping empty diff", "file", diff.NewPath)
|
||||
continue
|
||||
}
|
||||
|
||||
chunk := &sources.Chunk{
|
||||
SourceType: s.Type(),
|
||||
SourceName: s.name,
|
||||
SourceID: s.SourceID(),
|
||||
JobID: s.JobID(),
|
||||
SecretID: target.SecretID,
|
||||
Data: []byte(diff.Diff),
|
||||
SourceMetadata: &source_metadatapb.MetaData{
|
||||
Data: &source_metadatapb.MetaData_Gitlab{Gitlab: meta},
|
||||
},
|
||||
Verify: s.verify,
|
||||
}
|
||||
|
||||
if err := common.CancellableWrite(ctx, chunksChan, chunk); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Source) Validate(ctx context.Context) []error {
|
||||
// The client is only used to query Gitlab for a repo list - it's not used to actually clone anything. Thus, we
|
||||
// don't use it if there is a list of explicitly configured repos. However, constructing it validates that the
|
||||
|
|
|
|||
446
pkg/sources/gitlab/gitlab_integration_test.go
Normal file
446
pkg/sources/gitlab/gitlab_integration_test.go
Normal file
|
|
@ -0,0 +1,446 @@
|
|||
//go:build integration
|
||||
// +build integration
|
||||
|
||||
package gitlab
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kylelemons/godebug/pretty"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"google.golang.org/protobuf/types/known/anypb"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/credentialspb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
)
|
||||
|
||||
func TestSource_Scan(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
secret, err := common.GetTestSecret(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(fmt.Errorf("failed to access secret: %v", err))
|
||||
}
|
||||
token := secret.MustGetField("GITLAB_TOKEN")
|
||||
basicUser := secret.MustGetField("GITLAB_USER")
|
||||
basicPass := secret.MustGetField("GITLAB_PASS")
|
||||
|
||||
type init struct {
|
||||
name string
|
||||
verify bool
|
||||
connection *sourcespb.GitLab
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
init init
|
||||
wantChunk *sources.Chunk
|
||||
wantReposScanned int
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "token auth, enumerate repo, with explicit ignore",
|
||||
init: init{
|
||||
name: "test source",
|
||||
connection: &sourcespb.GitLab{
|
||||
Credential: &sourcespb.GitLab_Token{
|
||||
Token: token,
|
||||
},
|
||||
IgnoreRepos: []string{"tes1188/learn-gitlab"},
|
||||
},
|
||||
},
|
||||
wantChunk: &sources.Chunk{
|
||||
SourceType: sourcespb.SourceType_SOURCE_TYPE_GITLAB,
|
||||
SourceName: "test source",
|
||||
},
|
||||
wantReposScanned: 5,
|
||||
},
|
||||
{
|
||||
name: "token auth, enumerate repo, with glob ignore",
|
||||
init: init{
|
||||
name: "test source",
|
||||
connection: &sourcespb.GitLab{
|
||||
Credential: &sourcespb.GitLab_Token{
|
||||
Token: token,
|
||||
},
|
||||
IgnoreRepos: []string{"tes1188/*-gitlab"},
|
||||
},
|
||||
},
|
||||
wantChunk: &sources.Chunk{
|
||||
SourceType: sourcespb.SourceType_SOURCE_TYPE_GITLAB,
|
||||
SourceName: "test source",
|
||||
},
|
||||
wantReposScanned: 5,
|
||||
},
|
||||
{
|
||||
name: "token auth, scoped repo",
|
||||
init: init{
|
||||
name: "test source scoped",
|
||||
connection: &sourcespb.GitLab{
|
||||
Repositories: []string{"https://gitlab.com/testermctestface/testy.git"},
|
||||
Credential: &sourcespb.GitLab_Token{
|
||||
Token: token,
|
||||
},
|
||||
},
|
||||
},
|
||||
wantChunk: &sources.Chunk{
|
||||
SourceType: sourcespb.SourceType_SOURCE_TYPE_GITLAB,
|
||||
SourceName: "test source scoped",
|
||||
},
|
||||
wantReposScanned: 1,
|
||||
},
|
||||
{
|
||||
name: "basic auth, scoped repo",
|
||||
init: init{
|
||||
name: "test source basic auth scoped",
|
||||
connection: &sourcespb.GitLab{
|
||||
Repositories: []string{"https://gitlab.com/testermctestface/testy.git"},
|
||||
Credential: &sourcespb.GitLab_BasicAuth{
|
||||
BasicAuth: &credentialspb.BasicAuth{
|
||||
Username: basicUser,
|
||||
Password: basicPass,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
wantChunk: &sources.Chunk{
|
||||
SourceType: sourcespb.SourceType_SOURCE_TYPE_GITLAB,
|
||||
SourceName: "test source basic auth scoped",
|
||||
},
|
||||
wantReposScanned: 1,
|
||||
},
|
||||
{
|
||||
name: "basic auth access token, scoped repo",
|
||||
init: init{
|
||||
name: "test source basic auth access token scoped",
|
||||
connection: &sourcespb.GitLab{
|
||||
Repositories: []string{"https://gitlab.com/testermctestface/testy.git"},
|
||||
Credential: &sourcespb.GitLab_BasicAuth{
|
||||
BasicAuth: &credentialspb.BasicAuth{
|
||||
Username: basicUser,
|
||||
Password: token,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
wantChunk: &sources.Chunk{
|
||||
SourceType: sourcespb.SourceType_SOURCE_TYPE_GITLAB,
|
||||
SourceName: "test source basic auth access token scoped",
|
||||
},
|
||||
wantReposScanned: 1,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
s := Source{}
|
||||
|
||||
conn, err := anypb.New(tt.init.connection)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
err = s.Init(ctx, tt.init.name, 0, 0, tt.init.verify, conn, 10)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("Source.Init() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
chunksCh := make(chan *sources.Chunk, 1)
|
||||
go func() {
|
||||
defer close(chunksCh)
|
||||
err = s.Chunks(ctx, chunksCh)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("Source.Chunks() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
}()
|
||||
var chunkCnt int
|
||||
// Commits don't come in a deterministic order, so remove metadata comparison
|
||||
for gotChunk := range chunksCh {
|
||||
chunkCnt++
|
||||
gotChunk.Data = nil
|
||||
gotChunk.SourceMetadata = nil
|
||||
if diff := pretty.Compare(gotChunk, tt.wantChunk); diff != "" {
|
||||
t.Errorf("Source.Chunks() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
}
|
||||
}
|
||||
|
||||
assert.Equal(t, tt.wantReposScanned, len(s.repos))
|
||||
if chunkCnt < 1 {
|
||||
t.Errorf("0 chunks scanned.")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSource_Validate(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
secret, err := common.GetTestSecret(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(fmt.Errorf("failed to access secret: %v", err))
|
||||
}
|
||||
token := secret.MustGetField("GITLAB_TOKEN")
|
||||
tokenWrongScope := secret.MustGetField("GITLAB_TOKEN_WRONG_SCOPE")
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
connection *sourcespb.GitLab
|
||||
wantErrCount int
|
||||
wantErrs []string
|
||||
}{
|
||||
{
|
||||
name: "basic auth did not authenticate",
|
||||
connection: &sourcespb.GitLab{
|
||||
Credential: &sourcespb.GitLab_BasicAuth{
|
||||
BasicAuth: &credentialspb.BasicAuth{
|
||||
Username: "bad-user",
|
||||
Password: "bad-password",
|
||||
},
|
||||
},
|
||||
},
|
||||
wantErrCount: 1,
|
||||
},
|
||||
{
|
||||
name: "token did not authenticate",
|
||||
connection: &sourcespb.GitLab{
|
||||
Credential: &sourcespb.GitLab_Token{
|
||||
Token: "bad-token",
|
||||
},
|
||||
},
|
||||
wantErrCount: 1,
|
||||
},
|
||||
{
|
||||
name: "bad repo urls",
|
||||
connection: &sourcespb.GitLab{
|
||||
Credential: &sourcespb.GitLab_Token{
|
||||
Token: token,
|
||||
},
|
||||
Repositories: []string{
|
||||
"https://gitlab.com/testermctestface/testy", // valid
|
||||
"https://gitlab.com/testermctestface/testy/", // trailing slash
|
||||
"ssh:git@gitlab.com/testermctestface/testy", // bad protocol
|
||||
"https://gitlab.com", // no path
|
||||
"https://gitlab.com/", // no org name
|
||||
"https://gitlab.com//testy", // no org name
|
||||
"https://gitlab.com/testermctestface/", // no repo name
|
||||
},
|
||||
},
|
||||
wantErrCount: 6,
|
||||
},
|
||||
{
|
||||
name: "token does not have permission to list projects",
|
||||
connection: &sourcespb.GitLab{
|
||||
Credential: &sourcespb.GitLab_Token{
|
||||
Token: tokenWrongScope,
|
||||
},
|
||||
},
|
||||
wantErrCount: 1,
|
||||
},
|
||||
{
|
||||
name: "repositories and ignore globs both configured",
|
||||
connection: &sourcespb.GitLab{
|
||||
Credential: &sourcespb.GitLab_Token{
|
||||
Token: token,
|
||||
},
|
||||
Repositories: []string{
|
||||
"https://gitlab.com/testermctestface/testy", // valid
|
||||
},
|
||||
IgnoreRepos: []string{
|
||||
"tes1188/*-gitlab",
|
||||
"[", // glob doesn't compile, but this won't be checked
|
||||
},
|
||||
},
|
||||
wantErrCount: 1,
|
||||
},
|
||||
{
|
||||
name: "could not compile ignore glob(s)",
|
||||
connection: &sourcespb.GitLab{
|
||||
Credential: &sourcespb.GitLab_Token{
|
||||
Token: token,
|
||||
},
|
||||
IgnoreRepos: []string{
|
||||
"tes1188/*-gitlab",
|
||||
"[", // glob doesn't compile
|
||||
"[a-]", // glob doesn't compile
|
||||
},
|
||||
},
|
||||
wantErrCount: 2,
|
||||
},
|
||||
{
|
||||
name: "repositories do not exist or are not accessible",
|
||||
connection: &sourcespb.GitLab{
|
||||
Credential: &sourcespb.GitLab_Token{
|
||||
Token: token,
|
||||
},
|
||||
Repositories: []string{
|
||||
"https://gitlab.com/testermctestface/testy",
|
||||
"https://gitlab.com/testermctestface/doesn't-exist",
|
||||
"https://gitlab.com/testermctestface/also-doesn't-exist",
|
||||
},
|
||||
},
|
||||
wantErrCount: 2,
|
||||
},
|
||||
{
|
||||
name: "ignore globs exclude all repos",
|
||||
connection: &sourcespb.GitLab{
|
||||
Credential: &sourcespb.GitLab_Token{
|
||||
Token: token,
|
||||
},
|
||||
IgnoreRepos: []string{
|
||||
"*",
|
||||
},
|
||||
},
|
||||
wantErrCount: 1,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
s := Source{}
|
||||
|
||||
conn, err := anypb.New(tt.connection)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
err = s.Init(ctx, tt.name, 0, 0, false, conn, 1)
|
||||
if err != nil {
|
||||
t.Fatalf("Source.Init() error: %v", err)
|
||||
}
|
||||
|
||||
errs := s.Validate(ctx)
|
||||
|
||||
assert.Equal(t, tt.wantErrCount, len(errs))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSource_Chunks_TargetedScan(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
|
||||
defer cancel()
|
||||
|
||||
secret, err := common.GetTestSecret(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(fmt.Errorf("failed to access secret: %v", err))
|
||||
}
|
||||
|
||||
token := secret.MustGetField("GITLAB_TOKEN")
|
||||
|
||||
type init struct {
|
||||
name string
|
||||
verify bool
|
||||
connection *sourcespb.GitLab
|
||||
queryCriteria *source_metadatapb.MetaData
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
init init
|
||||
wantChunks int
|
||||
}{
|
||||
{
|
||||
name: "targeted scan; single diff",
|
||||
init: init{
|
||||
connection: &sourcespb.GitLab{Credential: &sourcespb.GitLab_Token{Token: token}},
|
||||
queryCriteria: &source_metadatapb.MetaData{
|
||||
Data: &source_metadatapb.MetaData_Gitlab{
|
||||
Gitlab: &source_metadatapb.Gitlab{
|
||||
Repository: "https://gitlab.com/testermctestface/testy.git",
|
||||
Link: "https://gitlab.com/testermctestface/testy/blob/30c407baee70d41d062114022a59ed8ee048880a/.gitlab-ci.yml#L1",
|
||||
Commit: "30c407baee70d41d062114022a59ed8ee048880a",
|
||||
ProjectId: 32561068,
|
||||
File: "keys",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
wantChunks: 1,
|
||||
},
|
||||
{
|
||||
name: "targeted scan; multiple diffs",
|
||||
init: init{
|
||||
connection: &sourcespb.GitLab{Credential: &sourcespb.GitLab_Token{Token: token}},
|
||||
queryCriteria: &source_metadatapb.MetaData{
|
||||
Data: &source_metadatapb.MetaData_Gitlab{
|
||||
Gitlab: &source_metadatapb.Gitlab{
|
||||
Commit: "b9a2fafeb0b978201e64f62efc9aa37c52a65045",
|
||||
ProjectId: 32561068,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
wantChunks: 2,
|
||||
},
|
||||
{
|
||||
name: "invalid query criteria, missing project ID",
|
||||
init: init{
|
||||
connection: &sourcespb.GitLab{Credential: &sourcespb.GitLab_Token{Token: token}},
|
||||
queryCriteria: &source_metadatapb.MetaData{
|
||||
Data: &source_metadatapb.MetaData_Gitlab{
|
||||
Gitlab: &source_metadatapb.Gitlab{
|
||||
Repository: "test_keys",
|
||||
Commit: "fbc14303ffbf8fb1c2c1914e8dda7d0121633aca",
|
||||
File: "not-the-file",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
wantChunks: 0,
|
||||
},
|
||||
{
|
||||
name: "invalid query criteria, missing commit",
|
||||
init: init{
|
||||
name: "test source",
|
||||
connection: &sourcespb.GitLab{Credential: &sourcespb.GitLab_Token{Token: token}},
|
||||
queryCriteria: &source_metadatapb.MetaData{
|
||||
Data: &source_metadatapb.MetaData_Gitlab{
|
||||
Gitlab: &source_metadatapb.Gitlab{
|
||||
Repository: "test_keys",
|
||||
ProjectId: 32561068,
|
||||
File: "not-the-file",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
wantChunks: 0,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
tt := tt
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
s := Source{}
|
||||
|
||||
conn, err := anypb.New(tt.init.connection)
|
||||
assert.NoError(t, err)
|
||||
|
||||
err = s.Init(ctx, tt.init.name, 0, 0, tt.init.verify, conn, 8)
|
||||
assert.NoError(t, err)
|
||||
|
||||
var wg sync.WaitGroup
|
||||
chunksCh := make(chan *sources.Chunk, 1)
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer close(chunksCh)
|
||||
defer wg.Done()
|
||||
err = s.Chunks(ctx, chunksCh, sources.ChunkingTarget{QueryCriteria: tt.init.queryCriteria})
|
||||
assert.NoError(t, err)
|
||||
}()
|
||||
|
||||
i := 0
|
||||
for range chunksCh {
|
||||
i++
|
||||
}
|
||||
wg.Wait()
|
||||
assert.Equal(t, tt.wantChunks, i)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
@ -1,326 +1,16 @@
|
|||
package gitlab
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"github.com/kylelemons/godebug/pretty"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"golang.org/x/sync/errgroup"
|
||||
"google.golang.org/protobuf/types/known/anypb"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/credentialspb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources/git"
|
||||
)
|
||||
|
||||
func TestSource_Scan(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
secret, err := common.GetTestSecret(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(fmt.Errorf("failed to access secret: %v", err))
|
||||
}
|
||||
token := secret.MustGetField("GITLAB_TOKEN")
|
||||
basicUser := secret.MustGetField("GITLAB_USER")
|
||||
basicPass := secret.MustGetField("GITLAB_PASS")
|
||||
|
||||
type init struct {
|
||||
name string
|
||||
verify bool
|
||||
connection *sourcespb.GitLab
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
init init
|
||||
wantChunk *sources.Chunk
|
||||
wantReposScanned int
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "token auth, enumerate repo, with explicit ignore",
|
||||
init: init{
|
||||
name: "test source",
|
||||
connection: &sourcespb.GitLab{
|
||||
Credential: &sourcespb.GitLab_Token{
|
||||
Token: token,
|
||||
},
|
||||
IgnoreRepos: []string{"tes1188/learn-gitlab"},
|
||||
},
|
||||
},
|
||||
wantChunk: &sources.Chunk{
|
||||
SourceType: sourcespb.SourceType_SOURCE_TYPE_GITLAB,
|
||||
SourceName: "test source",
|
||||
},
|
||||
wantReposScanned: 5,
|
||||
},
|
||||
{
|
||||
name: "token auth, enumerate repo, with glob ignore",
|
||||
init: init{
|
||||
name: "test source",
|
||||
connection: &sourcespb.GitLab{
|
||||
Credential: &sourcespb.GitLab_Token{
|
||||
Token: token,
|
||||
},
|
||||
IgnoreRepos: []string{"tes1188/*-gitlab"},
|
||||
},
|
||||
},
|
||||
wantChunk: &sources.Chunk{
|
||||
SourceType: sourcespb.SourceType_SOURCE_TYPE_GITLAB,
|
||||
SourceName: "test source",
|
||||
},
|
||||
wantReposScanned: 5,
|
||||
},
|
||||
{
|
||||
name: "token auth, scoped repo",
|
||||
init: init{
|
||||
name: "test source scoped",
|
||||
connection: &sourcespb.GitLab{
|
||||
Repositories: []string{"https://gitlab.com/testermctestface/testy.git"},
|
||||
Credential: &sourcespb.GitLab_Token{
|
||||
Token: token,
|
||||
},
|
||||
},
|
||||
},
|
||||
wantChunk: &sources.Chunk{
|
||||
SourceType: sourcespb.SourceType_SOURCE_TYPE_GITLAB,
|
||||
SourceName: "test source scoped",
|
||||
},
|
||||
wantReposScanned: 1,
|
||||
},
|
||||
{
|
||||
name: "basic auth, scoped repo",
|
||||
init: init{
|
||||
name: "test source basic auth scoped",
|
||||
connection: &sourcespb.GitLab{
|
||||
Repositories: []string{"https://gitlab.com/testermctestface/testy.git"},
|
||||
Credential: &sourcespb.GitLab_BasicAuth{
|
||||
BasicAuth: &credentialspb.BasicAuth{
|
||||
Username: basicUser,
|
||||
Password: basicPass,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
wantChunk: &sources.Chunk{
|
||||
SourceType: sourcespb.SourceType_SOURCE_TYPE_GITLAB,
|
||||
SourceName: "test source basic auth scoped",
|
||||
},
|
||||
wantReposScanned: 1,
|
||||
},
|
||||
{
|
||||
name: "basic auth access token, scoped repo",
|
||||
init: init{
|
||||
name: "test source basic auth access token scoped",
|
||||
connection: &sourcespb.GitLab{
|
||||
Repositories: []string{"https://gitlab.com/testermctestface/testy.git"},
|
||||
Credential: &sourcespb.GitLab_BasicAuth{
|
||||
BasicAuth: &credentialspb.BasicAuth{
|
||||
Username: basicUser,
|
||||
Password: token,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
wantChunk: &sources.Chunk{
|
||||
SourceType: sourcespb.SourceType_SOURCE_TYPE_GITLAB,
|
||||
SourceName: "test source basic auth access token scoped",
|
||||
},
|
||||
wantReposScanned: 1,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
s := Source{}
|
||||
|
||||
conn, err := anypb.New(tt.init.connection)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
err = s.Init(ctx, tt.init.name, 0, 0, tt.init.verify, conn, 10)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("Source.Init() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
chunksCh := make(chan *sources.Chunk, 1)
|
||||
go func() {
|
||||
defer close(chunksCh)
|
||||
err = s.Chunks(ctx, chunksCh)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("Source.Chunks() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
}()
|
||||
var chunkCnt int
|
||||
// Commits don't come in a deterministic order, so remove metadata comparison
|
||||
for gotChunk := range chunksCh {
|
||||
chunkCnt++
|
||||
gotChunk.Data = nil
|
||||
gotChunk.SourceMetadata = nil
|
||||
if diff := pretty.Compare(gotChunk, tt.wantChunk); diff != "" {
|
||||
t.Errorf("Source.Chunks() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
}
|
||||
}
|
||||
|
||||
assert.Equal(t, tt.wantReposScanned, len(s.repos))
|
||||
if chunkCnt < 1 {
|
||||
t.Errorf("0 chunks scanned.")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSource_Validate(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
secret, err := common.GetTestSecret(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(fmt.Errorf("failed to access secret: %v", err))
|
||||
}
|
||||
token := secret.MustGetField("GITLAB_TOKEN")
|
||||
tokenWrongScope := secret.MustGetField("GITLAB_TOKEN_WRONG_SCOPE")
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
connection *sourcespb.GitLab
|
||||
wantErrCount int
|
||||
wantErrs []string
|
||||
}{
|
||||
{
|
||||
name: "basic auth did not authenticate",
|
||||
connection: &sourcespb.GitLab{
|
||||
Credential: &sourcespb.GitLab_BasicAuth{
|
||||
BasicAuth: &credentialspb.BasicAuth{
|
||||
Username: "bad-user",
|
||||
Password: "bad-password",
|
||||
},
|
||||
},
|
||||
},
|
||||
wantErrCount: 1,
|
||||
},
|
||||
{
|
||||
name: "token did not authenticate",
|
||||
connection: &sourcespb.GitLab{
|
||||
Credential: &sourcespb.GitLab_Token{
|
||||
Token: "bad-token",
|
||||
},
|
||||
},
|
||||
wantErrCount: 1,
|
||||
},
|
||||
{
|
||||
name: "bad repo urls",
|
||||
connection: &sourcespb.GitLab{
|
||||
Credential: &sourcespb.GitLab_Token{
|
||||
Token: token,
|
||||
},
|
||||
Repositories: []string{
|
||||
"https://gitlab.com/testermctestface/testy", // valid
|
||||
"https://gitlab.com/testermctestface/testy/", // trailing slash
|
||||
"ssh:git@gitlab.com/testermctestface/testy", // bad protocol
|
||||
"https://gitlab.com", // no path
|
||||
"https://gitlab.com/", // no org name
|
||||
"https://gitlab.com//testy", // no org name
|
||||
"https://gitlab.com/testermctestface/", // no repo name
|
||||
},
|
||||
},
|
||||
wantErrCount: 6,
|
||||
},
|
||||
{
|
||||
name: "token does not have permission to list projects",
|
||||
connection: &sourcespb.GitLab{
|
||||
Credential: &sourcespb.GitLab_Token{
|
||||
Token: tokenWrongScope,
|
||||
},
|
||||
},
|
||||
wantErrCount: 1,
|
||||
},
|
||||
{
|
||||
name: "repositories and ignore globs both configured",
|
||||
connection: &sourcespb.GitLab{
|
||||
Credential: &sourcespb.GitLab_Token{
|
||||
Token: token,
|
||||
},
|
||||
Repositories: []string{
|
||||
"https://gitlab.com/testermctestface/testy", // valid
|
||||
},
|
||||
IgnoreRepos: []string{
|
||||
"tes1188/*-gitlab",
|
||||
"[", // glob doesn't compile, but this won't be checked
|
||||
},
|
||||
},
|
||||
wantErrCount: 1,
|
||||
},
|
||||
{
|
||||
name: "could not compile ignore glob(s)",
|
||||
connection: &sourcespb.GitLab{
|
||||
Credential: &sourcespb.GitLab_Token{
|
||||
Token: token,
|
||||
},
|
||||
IgnoreRepos: []string{
|
||||
"tes1188/*-gitlab",
|
||||
"[", // glob doesn't compile
|
||||
"[a-]", // glob doesn't compile
|
||||
},
|
||||
},
|
||||
wantErrCount: 2,
|
||||
},
|
||||
{
|
||||
name: "repositories do not exist or are not accessible",
|
||||
connection: &sourcespb.GitLab{
|
||||
Credential: &sourcespb.GitLab_Token{
|
||||
Token: token,
|
||||
},
|
||||
Repositories: []string{
|
||||
"https://gitlab.com/testermctestface/testy",
|
||||
"https://gitlab.com/testermctestface/doesn't-exist",
|
||||
"https://gitlab.com/testermctestface/also-doesn't-exist",
|
||||
},
|
||||
},
|
||||
wantErrCount: 2,
|
||||
},
|
||||
{
|
||||
name: "ignore globs exclude all repos",
|
||||
connection: &sourcespb.GitLab{
|
||||
Credential: &sourcespb.GitLab_Token{
|
||||
Token: token,
|
||||
},
|
||||
IgnoreRepos: []string{
|
||||
"*",
|
||||
},
|
||||
},
|
||||
wantErrCount: 1,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
s := Source{}
|
||||
|
||||
conn, err := anypb.New(tt.connection)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
err = s.Init(ctx, tt.name, 0, 0, false, conn, 1)
|
||||
if err != nil {
|
||||
t.Fatalf("Source.Init() error: %v", err)
|
||||
}
|
||||
|
||||
errs := s.Validate(ctx)
|
||||
|
||||
assert.Equal(t, tt.wantErrCount, len(errs))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func Test_setProgressCompleteWithRepo_resumeInfo(t *testing.T) {
|
||||
tests := []struct {
|
||||
startingResumeInfoSlice []string
|
||||
|
|
|
|||
Loading…
Reference in a new issue