From 1f0fd912054b6254bafa467d9b3788738e9e080b Mon Sep 17 00:00:00 2001
From: Miccah <m.castorina93@gmail.com>
Date: Wed, 8 Feb 2023 11:33:01 -0600
Subject: [PATCH] Skip repo and continue scanning when encountering an error
 (#1080)

---
 pkg/sources/git/git.go | 41 +++++++++++++++++++++++++----------------
 1 file changed, 25 insertions(+), 16 deletions(-)

diff --git a/pkg/sources/git/git.go b/pkg/sources/git/git.go
index a3577972c..943a29b82 100644
--- a/pkg/sources/git/git.go
+++ b/pkg/sources/git/git.go
@@ -125,13 +125,14 @@ func (s *Source) Init(aCtx context.Context, name string, jobId, sourceId int64,
 // Chunks emits chunks of bytes over a channel.
 func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {
 	// TODO: refactor to remove duplicate code
+	totalRepos := len(s.conn.Repositories) + len(s.conn.Directories)
 	switch cred := s.conn.GetCredential().(type) {
 	case *sourcespb.Git_BasicAuth:
 		user := cred.BasicAuth.Username
 		token := cred.BasicAuth.Password
 
 		for i, repoURI := range s.conn.Repositories {
-			s.SetProgressComplete(i, len(s.conn.Repositories), fmt.Sprintf("Repo: %s", repoURI), "")
+			s.SetProgressComplete(i, totalRepos, fmt.Sprintf("Repo: %s", repoURI), "")
 			if len(repoURI) == 0 {
 				continue
 			}
@@ -144,12 +145,13 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
 				return s.git.ScanRepo(ctx, repo, path, NewScanOptions(), chunksChan)
 			}(repoURI)
 			if err != nil {
-				return err
+				ctx.Logger().Info("error scanning repository", "repo", repoURI, "error", err)
+				continue
 			}
 		}
 	case *sourcespb.Git_Unauthenticated:
 		for i, repoURI := range s.conn.Repositories {
-			s.SetProgressComplete(i, len(s.conn.Repositories), fmt.Sprintf("Repo: %s", repoURI), "")
+			s.SetProgressComplete(i, totalRepos, fmt.Sprintf("Repo: %s", repoURI), "")
 			if len(repoURI) == 0 {
 				continue
 			}
@@ -162,12 +164,13 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
 				return s.git.ScanRepo(ctx, repo, path, NewScanOptions(), chunksChan)
 			}(repoURI)
 			if err != nil {
-				return err
+				ctx.Logger().Info("error scanning repository", "repo", repoURI, "error", err)
+				continue
 			}
 		}
 	case *sourcespb.Git_SshAuth:
 		for i, repoURI := range s.conn.Repositories {
-			s.SetProgressComplete(i, len(s.conn.Repositories), fmt.Sprintf("Repo: %s", repoURI), "")
+			s.SetProgressComplete(i, totalRepos, fmt.Sprintf("Repo: %s", repoURI), "")
 			if len(repoURI) == 0 {
 				continue
 			}
@@ -180,24 +183,26 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
 				return s.git.ScanRepo(ctx, repo, path, NewScanOptions(), chunksChan)
 			}(repoURI)
 			if err != nil {
-				return err
+				ctx.Logger().Info("error scanning repository", "repo", repoURI, "error", err)
+				continue
 			}
 		}
 	default:
 		return errors.New("invalid connection type for git source")
 	}
 
-	for i, u := range s.conn.Directories {
-		s.SetProgressComplete(i, len(s.conn.Repositories), fmt.Sprintf("Repo: %s", u), "")
+	for i, gitDir := range s.conn.Directories {
+		s.SetProgressComplete(len(s.conn.Repositories)+i, totalRepos, fmt.Sprintf("Repo: %s", gitDir), "")
 
-		if len(u) == 0 {
+		if len(gitDir) == 0 {
 			continue
 		}
-		if !strings.HasSuffix(u, "git") {
+		if !strings.HasSuffix(gitDir, "git") {
 			// try paths instead of url
-			repo, err := RepoFromPath(u)
+			repo, err := RepoFromPath(gitDir)
 			if err != nil {
-				return err
+				ctx.Logger().Info("error scanning repository", "repo", gitDir, "error", err)
+				continue
 			}
 
 			err = func(repoPath string) error {
@@ -206,16 +211,20 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
 				}
 
 				return s.git.ScanRepo(ctx, repo, repoPath, NewScanOptions(), chunksChan)
-			}(u)
+			}(gitDir)
 			if err != nil {
-				return err
+				ctx.Logger().Info("error scanning repository", "repo", gitDir, "error", err)
+				continue
 			}
 		}
 
 	}
 
-	ctx.Logger().V(1).Info("Git source finished scanning", "repo-count", len(s.conn.Repositories))
-	s.SetProgressComplete(len(s.conn.Repositories), len(s.conn.Repositories), fmt.Sprintf("Completed scanning source %s", s.name), "")
+	ctx.Logger().V(1).Info("Git source finished scanning", "repo-count", totalRepos)
+	s.SetProgressComplete(
+		totalRepos, totalRepos,
+		fmt.Sprintf("Completed scanning source %s", s.name), "",
+	)
 	return nil
 }