Mirrors/trufflehog
2
0
Fork 0
mirror of https://github.com/trufflesecurity/trufflehog.git synced 2024-11-10 15:14:38 +00:00
Code Issues Projects Releases Packages Wiki Activity

Skip repo and continue scanning when encountering an error (#1080)

Browse source
This commit is contained in:
Miccah 2023-02-08 11:33:01 -06:00 committed by GitHub
parent 0d73dbe638
commit 1f0fd91205
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
1 changed files with 25 additions and 16 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

41
pkg/sources/git/git.go
View file

@ -125,13 +125,14 @@ func (s *Source) Init(aCtx context.Context, name string, jobId, sourceId int64,
// Chunks emits chunks of bytes over a channel.
func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {
// TODO: refactor to remove duplicate code
totalRepos := len(s.conn.Repositories) + len(s.conn.Directories)
switch cred := s.conn.GetCredential().(type) {
case *sourcespb.Git_BasicAuth:
user := cred.BasicAuth.Username
token := cred.BasicAuth.Password
for i, repoURI := range s.conn.Repositories {
s.SetProgressComplete(i, len(s.conn.Repositories), fmt.Sprintf("Repo: %s", repoURI), "")
s.SetProgressComplete(i, totalRepos, fmt.Sprintf("Repo: %s", repoURI), "")
if len(repoURI) == 0 {
continue
}
@ -144,12 +145,13 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
return s.git.ScanRepo(ctx, repo, path, NewScanOptions(), chunksChan)
}(repoURI)
if err != nil {
return err
ctx.Logger().Info("error scanning repository", "repo", repoURI, "error", err)
continue
}
}
case *sourcespb.Git_Unauthenticated:
for i, repoURI := range s.conn.Repositories {
s.SetProgressComplete(i, len(s.conn.Repositories), fmt.Sprintf("Repo: %s", repoURI), "")
s.SetProgressComplete(i, totalRepos, fmt.Sprintf("Repo: %s", repoURI), "")
if len(repoURI) == 0 {
continue
}
@ -162,12 +164,13 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
return s.git.ScanRepo(ctx, repo, path, NewScanOptions(), chunksChan)
}(repoURI)
if err != nil {
return err
ctx.Logger().Info("error scanning repository", "repo", repoURI, "error", err)
continue
}
}
case *sourcespb.Git_SshAuth:
for i, repoURI := range s.conn.Repositories {
s.SetProgressComplete(i, len(s.conn.Repositories), fmt.Sprintf("Repo: %s", repoURI), "")
s.SetProgressComplete(i, totalRepos, fmt.Sprintf("Repo: %s", repoURI), "")
if len(repoURI) == 0 {
continue
}
@ -180,24 +183,26 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
return s.git.ScanRepo(ctx, repo, path, NewScanOptions(), chunksChan)
}(repoURI)
if err != nil {
return err
ctx.Logger().Info("error scanning repository", "repo", repoURI, "error", err)
continue
}
}
default:
return errors.New("invalid connection type for git source")
}
for i, u := range s.conn.Directories {
s.SetProgressComplete(i, len(s.conn.Repositories), fmt.Sprintf("Repo: %s", u), "")
for i, gitDir := range s.conn.Directories {
s.SetProgressComplete(len(s.conn.Repositories)+i, totalRepos, fmt.Sprintf("Repo: %s", gitDir), "")
if len(u) == 0 {
if len(gitDir) == 0 {
continue
}
if !strings.HasSuffix(u, "git") {
if !strings.HasSuffix(gitDir, "git") {
// try paths instead of url
repo, err := RepoFromPath(u)
repo, err := RepoFromPath(gitDir)
if err != nil {
return err
ctx.Logger().Info("error scanning repository", "repo", gitDir, "error", err)
continue
}
err = func(repoPath string) error {
@ -206,16 +211,20 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
}
return s.git.ScanRepo(ctx, repo, repoPath, NewScanOptions(), chunksChan)
}(u)
}(gitDir)
if err != nil {
return err
ctx.Logger().Info("error scanning repository", "repo", gitDir, "error", err)
continue
}
}
}
ctx.Logger().V(1).Info("Git source finished scanning", "repo-count", len(s.conn.Repositories))
s.SetProgressComplete(len(s.conn.Repositories), len(s.conn.Repositories), fmt.Sprintf("Completed scanning source %s", s.name), "")
ctx.Logger().V(1).Info("Git source finished scanning", "repo-count", totalRepos)
s.SetProgressComplete(
totalRepos, totalRepos,
fmt.Sprintf("Completed scanning source %s", s.name), "",
)
return nil
}