Allow for custom verifier (#1070)

* allow for custom verifier.

* Update engine.

* use custom detectors.

* set cap.

* Update verifiers.

* Remove nil check.

* resolved nit

* handle uppercase values

* updating missing url logs

* adding more descriptive variable names

* updating logs to use correct variables

* Removing toLower for urls

* if else nits

* Adding versioning for github and gitlab

---------

Co-authored-by: ahmed <ahmed.zahran@trufflesec.com>
Co-authored-by: ah̳̕mͭͭͨͩ̐e̘ͬ́͋ͬ̊̓͂d <13666360+0x1@users.noreply.github.com>

main.go

@@ -51,6 +51,7 @@ var (
 	printAvgDetectorTime = cli.Flag("print-avg-detector-time", "Print the average time spent on each detector.").Bool()
 	noUpdate             = cli.Flag("no-update", "Don't check for updates.").Bool()
 	fail                 = cli.Flag("fail", "Exit with code 183 if results are found.").Bool()
+	verifiers            = cli.Flag("verifier", "Set custom verification endpoints.").StringMap()
 	archiveMaxSize       = cli.Flag("archive-max-size", "Maximum size of archive to scan. (Byte units eg. 512B, 2KB, 4MB)").Bytes()
 	archiveMaxDepth      = cli.Flag("archive-max-depth", "Maximum depth of archive to scan.").Int()
 	archiveTimeout       = cli.Flag("archive-timeout", "Maximum time to spend extracting an archive.").Duration()
@@ -219,6 +220,8 @@ func run(state overseer.State) {
 		}
 	}
 
+	urls := splitVerifierURLs(*verifiers)
+
 	if *archiveMaxSize != 0 {
 		handlers.SetArchiveMaxSize(int(*archiveMaxSize))
 	}
@@ -282,6 +285,7 @@ func run(state overseer.State) {
 		engine.WithConcurrency(*concurrency),
 		engine.WithDecoders(decoders.DefaultDecoders()...),
 		engine.WithDetectors(!*noVerification, engine.DefaultDetectors()...),
+		engine.WithDetectors(!*noVerification, engine.CustomDetectors(ctx, urls)...),
 		engine.WithDetectors(!*noVerification, conf.Detectors...),
 		engine.WithFilterDetectors(includeFilter),
 		engine.WithFilterDetectors(excludeFilter),
@@ -492,6 +496,19 @@ func printAverageDetectorTime(e *engine.Engine) {
 	}
 }
 
+func splitVerifierURLs(verifierURLs map[string]string) map[string][]string {
+	verifiers := make(map[string][]string, len(verifierURLs))
+	for k, v := range verifierURLs {
+		key := strings.ToLower(k)
+		sliceOfValues := strings.Split(v, ",")
+		for i, s := range sliceOfValues {
+			sliceOfValues[i] = strings.TrimSpace(s)
+		}
+		verifiers[key] = sliceOfValues
+	}
+	return verifiers
+}
+
 // logFatalFunc returns a log.Fatal style function. Calling the returned
 // function will terminate the program without cleanup.
 func logFatalFunc(logger logr.Logger) func(error, string, ...any) {

pkg/detectors/github/github.go

@@ -13,7 +13,34 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 )
 
-type Scanner struct{}
+const defaultURL = "https://api.github.com"
+
+type Scanner struct {
+	verifierURLs []string
+}
+
+// New creates a new Scanner with the given options.
+func New(opts ...func(*Scanner)) *Scanner {
+	scanner := &Scanner{
+		verifierURLs: make([]string, 0),
+	}
+	for _, opt := range opts {
+		opt(scanner)
+	}
+
+	return scanner
+}
+
+// WithVerifierURLs adds the given URLs to the list of URLs to check for
+// verification of secrets.
+func WithVerifierURLs(urls []string, includeDefault bool) func(*Scanner) {
+	return func(s *Scanner) {
+		if includeDefault {
+			urls = append(urls, defaultURL)
+		}
+		s.verifierURLs = append(s.verifierURLs, urls...)
+	}
+}
 
 // Ensure the Scanner satisfies the interfaces at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
@@ -29,7 +56,7 @@ var (
 	// https://github.blog/changelog/2022-10-18-introducing-fine-grained-personal-access-tokens/
 	keyPat = regexp.MustCompile(`\b((?:ghp|gho|ghu|ghs|ghr|github_pat)_[a-zA-Z0-9_]{36,255})\b`)
 
-	//TODO: Oauth2 client_id and client_secret
+	// TODO: Oauth2 client_id and client_secret
 	// https://developer.github.com/v3/#oauth2-keysecret
 )
 
@@ -71,27 +98,29 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 		if verify {
 			client := common.SaneHttpClient()
 			// https://developer.github.com/v3/users/#get-the-authenticated-user
-			req, err := http.NewRequestWithContext(ctx, "GET", "https://api.github.com/user", nil)
-			if err != nil {
-				continue
-			}
-			req.Header.Add("Content-Type", "application/json; charset=utf-8")
-			req.Header.Add("Authorization", fmt.Sprintf("token %s", token))
-			res, err := client.Do(req)
-			if err == nil {
-				if res.StatusCode >= 200 && res.StatusCode < 300 {
-					var userResponse userRes
-					err = json.NewDecoder(res.Body).Decode(&userResponse)
-					res.Body.Close()
-					if err == nil {
-						s1.Verified = true
-						s1.ExtraData = map[string]string{
-							"username":     userResponse.Login,
-							"url":          userResponse.UserURL,
-							"account_type": userResponse.Type,
-							"site_admin":   fmt.Sprintf("%t", userResponse.SiteAdmin),
-							"name":         userResponse.Name,
-							"company":      userResponse.Company,
+			for _, url := range s.verifierURLs {
+				req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("%s/user", url), nil)
+				if err != nil {
+					continue
+				}
+				req.Header.Add("Content-Type", "application/json; charset=utf-8")
+				req.Header.Add("Authorization", fmt.Sprintf("token %s", token))
+				res, err := client.Do(req)
+				if err == nil {
+					if res.StatusCode >= 200 && res.StatusCode < 300 {
+						var userResponse userRes
+						err = json.NewDecoder(res.Body).Decode(&userResponse)
+						res.Body.Close()
+						if err == nil {
+							s1.Verified = true
+							s1.ExtraData = map[string]string{
+								"username":     userResponse.Login,
+								"url":          userResponse.UserURL,
+								"account_type": userResponse.Type,
+								"site_admin":   fmt.Sprintf("%t", userResponse.SiteAdmin),
+								"name":         userResponse.Name,
+								"company":      userResponse.Company,
+							}
 						}
 					}
 				}

pkg/detectors/github_old/github_old.go

@@ -12,7 +12,34 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 )
 
-type Scanner struct{}
+const defaultURL = "https://api.github.com"
+
+type Scanner struct {
+	verifierURLs []string
+}
+
+// New creates a new Scanner with the given options.
+func New(opts ...func(*Scanner)) *Scanner {
+	scanner := &Scanner{
+		verifierURLs: make([]string, 0),
+	}
+	for _, opt := range opts {
+		opt(scanner)
+	}
+
+	return scanner
+}
+
+// WithVerifierURLs adds the given URLs to the list of URLs to check for
+// verification of secrets.
+func WithVerifierURLs(urls []string, includeDefault bool) func(*Scanner) {
+	return func(s *Scanner) {
+		if includeDefault {
+			urls = append(urls, defaultURL)
+		}
+		s.verifierURLs = append(s.verifierURLs, urls...)
+	}
+}
 
 // Ensure the Scanner satisfies the interfaces at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
@@ -63,7 +90,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			continue
 		}
 
-		s := detectors.Result{
+		s1 := detectors.Result{
 			DetectorType: detectorspb.DetectorType_Github,
 			Raw:          []byte(token),
 		}
@@ -71,30 +98,32 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 		if verify {
 			client := common.SaneHttpClient()
 			// https://developer.github.com/v3/users/#get-the-authenticated-user
-			req, err := http.NewRequestWithContext(ctx, "GET", "https://api.github.com/user", nil)
-			if err != nil {
-				continue
-			}
-			req.Header.Add("Content-Type", "application/json; charset=utf-8")
-			req.Header.Add("Authorization", fmt.Sprintf("token %s", token))
-			res, err := client.Do(req)
-			if err == nil {
-				if res.StatusCode >= 200 && res.StatusCode < 300 {
-					var userResponse userRes
-					err = json.NewDecoder(res.Body).Decode(&userResponse)
-					res.Body.Close()
-					if err == nil {
-						s.Verified = true
+			for _, url := range s.verifierURLs {
+				req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("%s/user", url), nil)
+				if err != nil {
+					continue
+				}
+				req.Header.Add("Content-Type", "application/json; charset=utf-8")
+				req.Header.Add("Authorization", fmt.Sprintf("token %s", token))
+				res, err := client.Do(req)
+				if err == nil {
+					if res.StatusCode >= 200 && res.StatusCode < 300 {
+						var userResponse userRes
+						err = json.NewDecoder(res.Body).Decode(&userResponse)
+						res.Body.Close()
+						if err == nil {
+							s1.Verified = true
+						}
 					}
 				}
 			}
 		}
 
-		if !s.Verified && detectors.IsKnownFalsePositive(token, detectors.DefaultFalsePositives, true) {
+		if !s1.Verified && detectors.IsKnownFalsePositive(token, detectors.DefaultFalsePositives, true) {
 			continue
 		}
 
-		results = append(results, s)
+		results = append(results, s1)
 	}
 
 	return results, nil

pkg/detectors/gitlab/gitlab.go

@@ -12,7 +12,34 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 )
 
-type Scanner struct{}
+const defaultURL = "https://gitlab.com"
+
+type Scanner struct {
+	verifierURLs []string
+}
+
+// New creates a new Scanner with the given options.
+func New(opts ...func(*Scanner)) *Scanner {
+	scanner := &Scanner{
+		verifierURLs: make([]string, 0),
+	}
+	for _, opt := range opts {
+		opt(scanner)
+	}
+
+	return scanner
+}
+
+// WithVerifierURLs adds the given URLs to the list of URLs to check for
+// verification of secrets.
+func WithVerifierURLs(urls []string, includeDefault bool) func(*Scanner) {
+	return func(s *Scanner) {
+		if includeDefault {
+			urls = append(urls, defaultURL)
+		}
+		s.verifierURLs = append(s.verifierURLs, urls...)
+	}
+}
 
 // Ensure the Scanner satisfies the interfaces at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
@@ -56,24 +83,25 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			// they all grant access to different parts of the API. I couldn't find an endpoint that every
 			// one of these scopes has access to, so we just check an example endpoint for each scope. If any
 			// of them contain data, we know we have a valid key, but if they all fail, we don't
-			baseURL := "https://gitlab.com/api/v4"
 
 			client := common.SaneHttpClient()
-			// test `read_user` scope
-			req, err := http.NewRequestWithContext(ctx, "GET", baseURL+"/user", nil)
-			if err != nil {
-				continue
-			}
-			req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))
-			res, err := client.Do(req)
-			if err == nil {
-				res.Body.Close() // The request body is unused.
+			for _, baseURL := range s.verifierURLs {
+				// test `read_user` scope
+				req, err := http.NewRequestWithContext(ctx, "GET", baseURL+"/api/v4/user", nil)
+				if err != nil {
+					continue
+				}
+				req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))
+				res, err := client.Do(req)
+				if err == nil {
+					res.Body.Close() // The request body is unused.
 
-				// 200 means good key and has `read_user` scope
-				// 403 means good key but not the right scope
-				// 401 is bad key
-				if res.StatusCode == http.StatusOK || res.StatusCode == http.StatusForbidden {
-					secret.Verified = true
+					// 200 means good key and has `read_user` scope
+					// 403 means good key but not the right scope
+					// 401 is bad key
+					if res.StatusCode == http.StatusOK || res.StatusCode == http.StatusForbidden {
+						secret.Verified = true
+					}
 				}
 			}
 		}

pkg/detectors/gitlabv2/gitlab.go

@@ -11,7 +11,34 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 )
 
-type Scanner struct{}
+const defaultURL = "https://gitlab.com"
+
+type Scanner struct {
+	verifierURLs []string
+}
+
+// New creates a new Scanner with the given options.
+func New(opts ...func(*Scanner)) *Scanner {
+	scanner := &Scanner{
+		verifierURLs: make([]string, 0),
+	}
+	for _, opt := range opts {
+		opt(scanner)
+	}
+
+	return scanner
+}
+
+// WithVerifierURLs adds the given URLs to the list of URLs to check for
+// verification of secrets.
+func WithVerifierURLs(urls []string, includeDefault bool) func(*Scanner) {
+	return func(s *Scanner) {
+		if includeDefault {
+			urls = append(urls, defaultURL)
+		}
+		s.verifierURLs = append(s.verifierURLs, urls...)
+	}
+}
 
 // Ensure the Scanner satisfies the interfaces at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
@@ -50,27 +77,27 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			// they all grant access to different parts of the API. I couldn't find an endpoint that every
 			// one of these scopes has access to, so we just check an example endpoint for each scope. If any
 			// of them contain data, we know we have a valid key, but if they all fail, we don't
-			baseURL := "https://gitlab.com/api/v4"
-
 			client := common.SaneHttpClient()
+			for _, baseURL := range s.verifierURLs {
+				// test `read_user` scope
+				req, err := http.NewRequestWithContext(ctx, "GET", baseURL+"/api/v4/user", nil)
+				if err != nil {
+					continue
+				}
+				req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", match[1]))
+				res, err := client.Do(req)
+				if err == nil {
+					res.Body.Close() // The request body is unused.
 
-			// test `read_user` scope
-			req, err := http.NewRequestWithContext(ctx, "GET", baseURL+"/user", nil)
-			if err != nil {
-				continue
-			}
-			req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", match[1]))
-			res, err := client.Do(req)
-			if err == nil {
-				res.Body.Close() // The request body is unused.
-
-				// 200 means good key and has `read_user` scope
-				// 403 means good key but not the right scope
-				// 401 is bad key
-				if res.StatusCode == http.StatusOK || res.StatusCode == http.StatusForbidden {
-					secret.Verified = true
+					// 200 means good key and has `read_user` scope
+					// 403 means good key but not the right scope
+					// 401 is bad key
+					if res.StatusCode == http.StatusOK || res.StatusCode == http.StatusForbidden {
+						secret.Verified = true
+					}
 				}
 			}
+
 		}
 
 		if !secret.Verified && detectors.IsKnownFalsePositive(string(secret.Raw), detectors.DefaultFalsePositives, true) {

pkg/engine/defaults.go

@@ -1,6 +1,7 @@
 package engine
 
 import (
+	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/abbysale"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/abuseipdb"
@@ -731,6 +732,7 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/zipcodebase"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/zonkafeedback"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/zulipchat"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/etherscan"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/infura"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/alchemy"
@@ -740,6 +742,65 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/coinmarketcap"
 )
 
+// CustomDetectors returns a list of detectors that are enabled by default, but
+// can be overridden by the user.
+func CustomDetectors(ctx context.Context, urls map[string][]string) []detectors.Detector {
+	defaultDetectors := DefaultDetectors()
+	if len(urls) == 0 {
+		return defaultDetectors
+	}
+
+	for i, detector := range defaultDetectors {
+
+		switch detector.Type() {
+		case detectorspb.DetectorType_Github:
+			githubUrls, ok := urls["github"]
+			if !ok {
+				ctx.Logger().V(2).Info("no GitHub urls to ignore")
+				continue
+			}
+			ctx.Logger().V(2).Info("ignoring GitHub urls: %v", githubUrls)
+
+			versioner, ok := detector.(detectors.Versioner)
+			if !ok {
+				ctx.Logger().V(2).Info("Failed to get version for GitHub detector")
+			}
+			switch versioner.Version() {
+			case 1:
+				defaultDetectors[i] = github_old.New(github_old.WithVerifierURLs(githubUrls, true))
+			default:
+				defaultDetectors[i] = github.New(github.WithVerifierURLs(githubUrls, true))
+			}
+
+		case detectorspb.DetectorType_Gitlab:
+			gitlabUrls, ok := urls["gitlab"]
+			if !ok {
+				ctx.Logger().V(2).Info("no GitLab urls to ignore")
+				continue
+			}
+			ctx.Logger().V(2).Info("ignoring GitLab urls: %v", gitlabUrls)
+
+			versioner, ok := detector.(detectors.Versioner)
+			if !ok {
+				ctx.Logger().V(2).Info("Failed to get version for Gitlab detector")
+			}
+			switch versioner.Version() {
+			case 1:
+				defaultDetectors[i] = gitlab.New(gitlab.WithVerifierURLs(gitlabUrls, true))
+			default:
+				defaultDetectors[i] = gitlabv2.New(gitlabv2.WithVerifierURLs(gitlabUrls, true))
+			}
+
+		case detectorspb.DetectorType_JiraToken:
+			// TODO(ahrav): Double check that we need to do this.
+		default:
+			ctx.Logger().V(5).Info("ignoring custom detector", "type", detector.Type())
+			continue
+		}
+	}
+	return defaultDetectors
+}
+
 func DefaultDetectors() []detectors.Detector {
 	return []detectors.Detector{
 		&heroku.Scanner{},
@@ -1505,4 +1566,5 @@ func DefaultDetectors() []detectors.Detector {
 		bscscan.Scanner{},
 		coinmarketcap.Scanner{},
 	}
+
 }

pkg/engine/engine.go

@@ -156,7 +156,7 @@ func Start(ctx context.Context, options ...EngineOption) *Engine {
 		"verification_disabled", len(e.detectors[false]),
 	)
 
-	// start the workers
+	// Start the workers.
 	for i := 0; i < e.concurrency; i++ {
 		e.workersWg.Add(1)
 		go func() {