Github filter support for exclude and include (#1087)

* test

* Add missing head and base hash back.

---------

Co-authored-by: Ahrav Dutta <ahravdutta02@gmail.com>

main.go

@@ -62,15 +62,17 @@ var (
 	_                   = gitScan.Flag("entropy", "No-op flag for backwards compat.").Bool()
 	_                   = gitScan.Flag("regex", "No-op flag for backwards compat.").Bool()
 
-	githubScan           = cli.Command("github", "Find credentials in GitHub repositories.")
-	githubScanEndpoint   = githubScan.Flag("endpoint", "GitHub endpoint.").Default("https://api.github.com").String()
-	githubScanRepos      = githubScan.Flag("repo", `GitHub repository to scan. You can repeat this flag. Example: "https://github.com/dustin-decker/secretsandstuff"`).Strings()
-	githubScanOrgs       = githubScan.Flag("org", `GitHub organization to scan. You can repeat this flag. Example: "trufflesecurity"`).Strings()
-	githubScanToken      = githubScan.Flag("token", "GitHub token. Can be provided with environment variable GITHUB_TOKEN.").Envar("GITHUB_TOKEN").String()
-	githubIncludeForks   = githubScan.Flag("include-forks", "Include forks in scan.").Bool()
-	githubIncludeMembers = githubScan.Flag("include-members", "Include organization member repositories in scan.").Bool()
-	githubIncludeRepos   = githubScan.Flag("include-repos", `Repositories to include in an org scan. This can also be a glob pattern. You can repeat this flag. Must use Github repo full name. Example: "trufflesecurity/trufflehog", "trufflesecurity/t*"`).Strings()
-	githubExcludeRepos   = githubScan.Flag("exclude-repos", `Repositories to exclude in an org scan. This can also be a glob pattern. You can repeat this flag. Must use Github repo full name. Example: "trufflesecurity/driftwood", "trufflesecurity/d*"`).Strings()
+	githubScan             = cli.Command("github", "Find credentials in GitHub repositories.")
+	githubScanEndpoint     = githubScan.Flag("endpoint", "GitHub endpoint.").Default("https://api.github.com").String()
+	githubScanRepos        = githubScan.Flag("repo", `GitHub repository to scan. You can repeat this flag. Example: "https://github.com/dustin-decker/secretsandstuff"`).Strings()
+	githubScanOrgs         = githubScan.Flag("org", `GitHub organization to scan. You can repeat this flag. Example: "trufflesecurity"`).Strings()
+	githubScanToken        = githubScan.Flag("token", "GitHub token. Can be provided with environment variable GITHUB_TOKEN.").Envar("GITHUB_TOKEN").String()
+	githubIncludeForks     = githubScan.Flag("include-forks", "Include forks in scan.").Bool()
+	githubIncludeMembers   = githubScan.Flag("include-members", "Include organization member repositories in scan.").Bool()
+	githubIncludeRepos     = githubScan.Flag("include-repos", `Repositories to include in an org scan. This can also be a glob pattern. You can repeat this flag. Must use Github repo full name. Example: "trufflesecurity/trufflehog", "trufflesecurity/t*"`).Strings()
+	githubExcludeRepos     = githubScan.Flag("exclude-repos", `Repositories to exclude in an org scan. This can also be a glob pattern. You can repeat this flag. Must use Github repo full name. Example: "trufflesecurity/driftwood", "trufflesecurity/d*"`).Strings()
+	githubScanIncludePaths = githubScan.Flag("include-paths", "Path to file with newline separated regexes for files to include in scan.").Short('i').String()
+	githubScanExcludePaths = githubScan.Flag("exclude-paths", "Path to file with newline separated regexes for files to exclude in scan.").Short('x').String()
 
 	gitlabScan = cli.Command("gitlab", "Find credentials in GitLab repositories.")
 	// TODO: Add more GitLab options
@@ -246,6 +248,10 @@ func run(state overseer.State) {
 			logrus.WithError(err).Fatal("Failed to scan Git.")
 		}
 	case githubScan.FullCommand():
+		filter, err := common.FilterFromFiles(*githubScanIncludePaths, *githubScanExcludePaths)
+		if err != nil {
+			logrus.WithError(err).Fatal("could not create filter")
+		}
 		if len(*githubScanOrgs) == 0 && len(*githubScanRepos) == 0 {
 			logrus.Fatal("You must specify at least one organization or repository.")
 		}
@@ -260,6 +266,7 @@ func run(state overseer.State) {
 			IncludeRepos:   *githubIncludeRepos,
 			Repos:          *githubScanRepos,
 			Orgs:           *githubScanOrgs,
+			Filter:         filter,
 		}
 		if err := e.ScanGitHub(ctx, cfg); err != nil {
 			logrus.WithError(err).Fatal("Failed to scan Github.")

pkg/engine/github.go

@@ -1,6 +1,7 @@
 package engine
 
 import (
+	gogit "github.com/go-git/go-git/v5"
 	"google.golang.org/protobuf/proto"
 	"google.golang.org/protobuf/types/known/anypb"
 
@@ -8,6 +9,7 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/sources/git"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/sources/github"
 )
 
@@ -37,6 +39,7 @@ func (e *Engine) ScanGitHub(ctx context.Context, c sources.GithubConfig) error {
 		ctx.Logger().Error(err, "failed to marshal github connection")
 		return err
 	}
+
 	ctx = context.WithValues(ctx,
 		"source_type", source.Type().String(),
 		"source_name", "github",
@@ -47,6 +50,14 @@ func (e *Engine) ScanGitHub(ctx context.Context, c sources.GithubConfig) error {
 		return err
 	}
 
+	logOptions := &gogit.LogOptions{}
+	opts := []git.ScanOption{
+		git.ScanOptionFilter(c.Filter),
+		git.ScanOptionLogOptions(logOptions),
+	}
+	scanOptions := git.NewScanOptions(opts...)
+	source.WithScanOptions(scanOptions)
+
 	e.sourcesWg.Add(1)
 	go func() {
 		defer common.RecoverWithExit(ctx)

pkg/sources/github/github.go

@@ -55,6 +55,7 @@ type Source struct {
 	includeRepos,
 	ignoreRepos []string
 	git             *git.Git
+	scanOptions     *git.ScanOptions
 	httpClient      *http.Client
 	log             logr.Logger
 	conn            *sourcespb.GitHub
@@ -67,6 +68,10 @@ type Source struct {
 	sources.Progress
 }
 
+func (s *Source) WithScanOptions(scanOptions *git.ScanOptions) {
+	s.scanOptions = scanOptions
+}
+
 // Ensure the Source satisfies the interface at compile time
 var _ sources.Source = (*Source)(nil)
 var endsWithGithub = regexp.MustCompile(`github\.com/?$`)
@@ -551,13 +556,11 @@ func (s *Source) scan(ctx context.Context, installationClient *github.Client, ch
 			if err != nil {
 				return nil
 			}
-			// Base and head will only exist from incoming webhooks.
-			scanOptions := git.NewScanOptions(
-				git.ScanOptionBaseHash(s.conn.Base),
-				git.ScanOptionHeadCommit(s.conn.Head),
-			)
 
-			if err = s.git.ScanRepo(ctx, repo, path, scanOptions, chunksChan); err != nil {
+			s.scanOptions.BaseHash = s.conn.Base
+			s.scanOptions.HeadHash = s.conn.Head
+
+			if err = s.git.ScanRepo(ctx, repo, path, s.scanOptions, chunksChan); err != nil {
 				logger.Error(err, "unable to scan repo, continuing")
 				return nil
 			}

pkg/sources/sources.go

@@ -78,6 +78,8 @@ type GithubConfig struct {
 	ExcludeRepos,
 	// IncludeRepos is a list of repositories to include in the scan.
 	IncludeRepos []string
+	// Filter is the filter to use to scan the source.
+	Filter *common.Filter
 }
 
 // GitlabConfig defines the optional configuration for a gitlab source.