[THOG-767] ignore gitlab repos (#853)

* Add ability to ignore repos.

* use std library slices.Contains.

* Add tests.

* Remove zero values from test.

go.mod

@@ -53,6 +53,7 @@ require (
 	go.mongodb.org/mongo-driver v1.10.2
 	go.uber.org/zap v1.23.0
 	golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d
+	golang.org/x/exp v0.0.0-20221018205818-5c77f4b2bbd7
 	golang.org/x/oauth2 v0.0.0-20220909003341-f21342109be1
 	golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4
 	google.golang.org/genproto v0.0.0-20220920201722-2b89144ce006

go.sum

@@ -520,6 +520,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0
 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
 golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
 golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
+golang.org/x/exp v0.0.0-20221018205818-5c77f4b2bbd7 h1:lhh9BuEmjIBOdOhMIGSQzdW5LbTtFbUV1k9O/Rlq3SE=
+golang.org/x/exp v0.0.0-20221018205818-5c77f4b2bbd7/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE=
 golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
 golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=

pkg/sources/github/github.go

@@ -19,6 +19,7 @@ import (
 	gogit "github.com/go-git/go-git/v5"
 	"github.com/google/go-github/v42/github"
 	log "github.com/sirupsen/logrus"
+	"golang.org/x/exp/slices"
 	"golang.org/x/oauth2"
 	"golang.org/x/sync/errgroup"
 	"google.golang.org/protobuf/proto"
@@ -698,22 +699,13 @@ func (s *Source) getReposByUser(ctx context.Context, user string) ([]string, err
 }
 
 func (s *Source) ignoreRepo(r string) bool {
-	if stringInSlice(r, s.ignoreRepos) {
+	if slices.Contains(s.ignoreRepos, r) {
 		s.log.Debugf("ignoring repo %s", r)
 		return true
 	}
 	return false
 }
 
-func stringInSlice(s string, l []string) bool {
-	for _, b := range l {
-		if b == s {
-			return true
-		}
-	}
-	return false
-}
-
 func (s *Source) getGistsByUser(ctx context.Context, user string) ([]string, error) {
 	var gistURLs []string
 	gistOpts := &github.GistListOptions{}

pkg/sources/gitlab/gitlab.go

@@ -22,6 +22,7 @@ import (
 	gogit "github.com/go-git/go-git/v5"
 	log "github.com/sirupsen/logrus"
 	"github.com/xanzy/go-gitlab"
+	"golang.org/x/exp/slices"
 	"golang.org/x/sync/semaphore"
 	"google.golang.org/protobuf/proto"
 	"google.golang.org/protobuf/types/known/anypb"
@@ -38,6 +39,7 @@ type Source struct {
 	token           string
 	url             string
 	repos           []string
+	ignoreRepos     []string
 	git             *git.Git
 	aCtx            context.Context
 	resumeInfoSlice []string
@@ -80,7 +82,9 @@ func (s *Source) Init(aCtx context.Context, name string, jobId, sourceId int64,
 	}
 
 	s.repos = conn.Repositories
+	s.ignoreRepos = conn.IgnoreRepos
 	s.url = conn.Endpoint
+
 	if conn.Endpoint != "" && !strings.HasSuffix(s.url, "/") {
 		s.url = s.url + "/"
 	}
@@ -365,6 +369,10 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
 		}
 		// Turn projects into URLs for Git cloner.
 		for _, prj := range projects {
+			if slices.Contains(s.ignoreRepos, prj.PathWithNamespace) {
+				log.Debugf("Ignoring repo %s", prj.PathWithNamespace)
+				continue
+			}
 			// Ensure the urls are valid before adding them to the repo list.
 			_, err := url.Parse(prj.HTTPURLToRepo)
 			if err != nil {
@@ -379,7 +387,7 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
 
 	s.repos = repos
 	// We must sort the repos so we can resume later if necessary.
-	sort.Strings(s.repos)
+	slices.Sort(s.repos)
 
 	errs = s.scanRepos(ctx, chunksChan)
 	for _, err := range errs {

pkg/sources/gitlab/gitlab_test.go

@@ -39,10 +39,11 @@ func TestSource_Scan(t *testing.T) {
 		connection *sourcespb.GitLab
 	}
 	tests := []struct {
-		name      string
-		init      init
-		wantChunk *sources.Chunk
-		wantErr   bool
+		name             string
+		init             init
+		wantChunk        *sources.Chunk
+		wantReposScanned int
+		wantErr          bool
 	}{
 		{
 			name: "token auth, enumerate repo",
@@ -52,14 +53,14 @@ func TestSource_Scan(t *testing.T) {
 					Credential: &sourcespb.GitLab_Token{
 						Token: token,
 					},
+					IgnoreRepos: []string{"tes1188/learn-gitlab"},
 				},
 			},
 			wantChunk: &sources.Chunk{
 				SourceType: sourcespb.SourceType_SOURCE_TYPE_GITLAB,
 				SourceName: "test source",
-				Verify:     false,
 			},
-			wantErr: false,
+			wantReposScanned: 2,
 		},
 		{
 			name: "token auth, scoped repo",
@@ -75,9 +76,8 @@ func TestSource_Scan(t *testing.T) {
 			wantChunk: &sources.Chunk{
 				SourceType: sourcespb.SourceType_SOURCE_TYPE_GITLAB,
 				SourceName: "test source scoped",
-				Verify:     false,
 			},
-			wantErr: false,
+			wantReposScanned: 1,
 		},
 		{
 			name: "basic auth, scoped repo",
@@ -96,9 +96,8 @@ func TestSource_Scan(t *testing.T) {
 			wantChunk: &sources.Chunk{
 				SourceType: sourcespb.SourceType_SOURCE_TYPE_GITLAB,
 				SourceName: "test source basic auth scoped",
-				Verify:     false,
 			},
-			wantErr: false,
+			wantReposScanned: 1,
 		},
 		{
 			name: "basic auth access token, scoped repo",
@@ -117,11 +116,11 @@ func TestSource_Scan(t *testing.T) {
 			wantChunk: &sources.Chunk{
 				SourceType: sourcespb.SourceType_SOURCE_TYPE_GITLAB,
 				SourceName: "test source basic auth access token scoped",
-				Verify:     false,
 			},
-			wantErr: false,
+			wantReposScanned: 1,
 		},
 	}
+
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			s := Source{}
@@ -156,6 +155,8 @@ func TestSource_Scan(t *testing.T) {
 					t.Errorf("Source.Chunks() %s diff: (-got +want)\n%s", tt.name, diff)
 				}
 			}
+
+			assert.Equal(t, tt.wantReposScanned, len(s.repos))
 			if chunkCnt < 1 {
 				t.Errorf("0 chunks scanned.")
 			}