Add snifftest

.github/workflows/lint.yml

@@ -18,6 +18,8 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/setup-go@v2
+        with:
+          go-version: '1.17'
       - uses: actions/checkout@v3
       - name: golangci-lint
         uses: golangci/golangci-lint-action@v3

.github/workflows/release.yml

@@ -1,4 +1,4 @@
-name: release
+name: Release
 
 # on:
 #   push:

.github/workflows/snifftest.yml

@@ -0,0 +1,25 @@
+name: Snifftest
+
+on:
+  push:
+    tags:
+      - v*
+    branches:
+      - main
+  pull_request:
+
+permissions:
+  contents: read
+  pull-requests: read
+
+jobs:
+  Snifftest:
+    name: Run Snifftest
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/setup-go@v2
+        with:
+          go-version: '1.17'
+      - uses: actions/checkout@v3
+      - name: Run Snifftest
+        run: make snifftest

Makefile

@@ -46,3 +46,6 @@ protos:
 
 protos-windows:
 	docker run -v "$(shell cygpath -w $(shell pwd))":/pwd "${PROTOS_IMAGE}" bash -c "cd /pwd; ./scripts/gen_proto.sh"
+
+snifftest:
+	./hack/snifftest/snifftest.sh

go.mod

@@ -23,6 +23,7 @@ require (
 	github.com/joho/godotenv v1.4.0
 	github.com/kylelemons/godebug v1.1.0
 	github.com/mattn/go-colorable v0.1.12
+	github.com/paulbellamy/ratecounter v0.2.0
 	github.com/pkg/errors v0.9.1
 	github.com/razorpay/razorpay-go v0.0.0-20210728161131-0341409a6ab2
 	github.com/sergi/go-diff v1.2.0

go.sum

@@ -328,6 +328,8 @@ github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9
 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
 github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk=
+github.com/paulbellamy/ratecounter v0.2.0 h1:2L/RhJq+HA8gBQImDXtLPrDXK5qAj6ozWVK/zFXVJGs=
+github.com/paulbellamy/ratecounter v0.2.0/go.mod h1:Hfx1hDpSGoqxkVVpBi/IlYD7kChlfo5C6hzIHwPqfFE=
 github.com/pkg/diff v0.0.0-20200914180035-5b29258ca4f7 h1:+/+DxvQaYifJ+grD4klzrS5y+KJXldn/2YTl5JG+vZ8=
 github.com/pkg/diff v0.0.0-20200914180035-5b29258ca4f7/go.mod h1:zO8QMzTeZd5cpnIkz/Gn6iK0jDfGicM1nynOkkPIl28=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=

hack/snifftest/README.md

@@ -0,0 +1,33 @@
+# snifftest
+
+See the help pages with this command, or look further below to get started quickly.
+
+```
+go run snifftest/main.go 
+```
+
+## Show available secret scanners
+
+```
+go run snifftest/main.go show-scanners 
+```
+
+## Load a repo into a DB
+
+```
+go run snifftest/main.go load --db ~/sdb --repo https://github.com/Netflix/Hystrix.git  
+```
+
+## Scan
+
+All scanners
+
+```
+go run snifftest/main.go scan --db ~/sdb --scanner all --print 
+```
+
+Particular scanner
+
+```
+go run snifftest/main.go scan --db ~/sdb --scanner github --print --print-chunk --fail-threshold 5
+```

hack/snifftest/main.go

@@ -0,0 +1,227 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"reflect"
+	"runtime"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/paulbellamy/ratecounter"
+	log "github.com/sirupsen/logrus"
+	"golang.org/x/sync/semaphore"
+	"gopkg.in/alecthomas/kingpin.v2"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/decoders"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/engine"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/sources/git"
+)
+
+var (
+	// CLI flags and commands
+	app = kingpin.New("Snifftest", "Test secret detectors against data sets.")
+
+	showDetectorsCmd = app.Command("show-detectors", "Shows the available detectors.")
+
+	scanCmd           = app.Command("scan", "Scans data.")
+	scanCmdDetector   = scanCmd.Flag("detector", "Detector to scan with. 'all', or a specific name.").Default("all").String()
+	scanCmdExclude    = scanCmd.Flag("exclude", "Detector(s) to exclude").Strings()
+	scanCmdRepo       = scanCmd.Flag("repo", "URI to .git repo.").Required().String()
+	scanThreshold     = scanCmd.Flag("fail-threshold", "Result threshold that causes failure for a single scanner.").Int()
+	scanPrintRes      = scanCmd.Flag("print", "Print results.").Bool()
+	scanPrintChunkRes = scanCmd.Flag("print-chunk", "Print chunks that have results.").Bool()
+	scanVerify        = scanCmd.Flag("verify", "Verify found secrets.").Bool()
+)
+
+func main() {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Hour*2)
+	var cancelOnce sync.Once
+	defer cancelOnce.Do(cancel)
+
+	cmd := kingpin.MustParse(app.Parse(os.Args[1:]))
+
+	switch cmd {
+	case scanCmd.FullCommand():
+
+		chunksChan := make(chan *sources.Chunk, 10000)
+
+		var wgChunkers sync.WaitGroup
+
+		sem := semaphore.NewWeighted(int64(runtime.NumCPU()))
+
+		selectedScanners := map[string]detectors.Detector{}
+		allScanners := getAllScanners()
+
+		decoders := decoders.DefaultDecoders()
+
+		input := strings.ToLower(*scanCmdDetector)
+		if input == "all" {
+			selectedScanners = allScanners
+		} else {
+			_, ok := allScanners[input]
+			if !ok {
+				log.Fatal("could not find scanner by that name")
+			}
+			selectedScanners[input] = allScanners[input]
+		}
+		if len(selectedScanners) == 0 {
+			log.Fatal("no detectors selected")
+		}
+
+		for _, excluded := range *scanCmdExclude {
+			delete(selectedScanners, excluded)
+		}
+
+		log.Infof("loaded %d secret detectors", len(selectedScanners)+3)
+
+		var wgScanners sync.WaitGroup
+
+		var chunkCounter uint64
+		go func() {
+			counter := ratecounter.NewRateCounter(60 * time.Second)
+			var prev uint64
+			for {
+				time.Sleep(60 * time.Second)
+				counter.Incr(int64(chunkCounter - prev))
+				prev = chunkCounter
+				log.Infof("chunk scan rate: %d/sec", counter.Rate()/60)
+			}
+		}()
+
+		resCounter := make(map[string]*uint64)
+		failed := false
+
+		for i := 0; i < runtime.NumCPU(); i++ {
+			wgScanners.Add(1)
+
+			go func() {
+				defer wgScanners.Done()
+
+				for chunk := range chunksChan {
+					for name, scanner := range selectedScanners {
+						for _, dec := range decoders {
+							decoded := dec.FromChunk(&sources.Chunk{Data: chunk.Data})
+							if decoded != nil {
+								foundKeyword := false
+								for _, kw := range scanner.Keywords() {
+									if strings.Contains(strings.ToLower(string(decoded.Data)), strings.ToLower(kw)) {
+										foundKeyword = true
+									}
+								}
+								if !foundKeyword {
+									continue
+								}
+
+								res, err := scanner.FromData(ctx, *scanVerify, decoded.Data)
+								if err != nil {
+									log.Fatal(err)
+								}
+								if len(res) > 0 {
+									if resCounter[name] == nil {
+										zero := uint64(0)
+										resCounter[name] = &zero
+									}
+									atomic.AddUint64(resCounter[name], uint64(len(res)))
+									if *scanThreshold != 0 && int(*resCounter[name]) > *scanThreshold {
+										log.WithField("scanner", name).Errorf("exceeded result threshold of %d", *scanThreshold)
+										failed = true
+										os.Exit(1)
+									}
+
+									if *scanPrintRes {
+										for _, r := range res {
+											logger := log.WithField("secret", name).WithField("meta", chunk.SourceMetadata).WithField("result", string(r.Raw))
+											if *scanPrintChunkRes {
+												logger = logger.WithField("chunk", string(decoded.Data))
+											}
+											logger.Info("result")
+										}
+									}
+								}
+							}
+						}
+					}
+
+					atomic.AddUint64(&chunkCounter, uint64(1))
+				}
+			}()
+		}
+
+		for _, repo := range strings.Split(*scanCmdRepo, ",") {
+			sem.Acquire(ctx, 1)
+			wgChunkers.Add(1)
+			go func(r string) {
+				defer sem.Release(1)
+				defer wgChunkers.Done()
+				log.Infof("cloning %s", r)
+				path, repo, err := git.CloneRepoUsingUnauthenticated(r)
+				if err != nil {
+					log.Fatal(err)
+				}
+
+				log.Infof("cloned %s", r)
+
+				s := git.NewGit(sourcespb.SourceType_SOURCE_TYPE_GIT, 0, 0, "snifftest", false, runtime.NumCPU(),
+					func(file, email, commit, timestamp, repository string) *source_metadatapb.MetaData {
+						return &source_metadatapb.MetaData{
+							Data: &source_metadatapb.MetaData_Git{
+								Git: &source_metadatapb.Git{
+									Commit:     commit,
+									File:       file,
+									Email:      email,
+									Repository: repository,
+									Timestamp:  timestamp,
+								},
+							},
+						}
+					})
+
+				log.Infof("scanning %s", r)
+				err = s.ScanRepo(ctx, repo, git.NewScanOptions(), chunksChan)
+				if err != nil {
+					log.Fatal(err)
+				}
+				log.Infof("scanned %s", r)
+				defer os.RemoveAll(path)
+			}(repo)
+		}
+
+		go func() {
+			wgChunkers.Wait()
+			close(chunksChan)
+		}()
+
+		wgScanners.Wait()
+
+		log.WithField("chunks", chunkCounter).Info("completed")
+		for scanner, resultsCount := range resCounter {
+			log.WithField("results", *resultsCount).Info(scanner)
+		}
+
+		if failed {
+			os.Exit(1)
+		}
+	case showDetectorsCmd.FullCommand():
+		for s := range getAllScanners() {
+			fmt.Println(s)
+		}
+	}
+}
+
+func getAllScanners() map[string]detectors.Detector {
+	allScanners := map[string]detectors.Detector{}
+	for _, s := range engine.DefaultDetectors() {
+		secretType := reflect.Indirect(reflect.ValueOf(s)).Type().PkgPath()
+		path := strings.Split(secretType, "/")[len(strings.Split(secretType, "/"))-1]
+		allScanners[path] = s
+	}
+	return allScanners
+}

hack/snifftest/snifftest.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+REPO_ARRAY=(
+        "https://github.com/Netflix/Hystrix.git"
+        # "https://github.com/facebook/flow.git"
+        # "https://github.com/Netflix/vizceral.git"
+        # "https://github.com/Netflix/metaflow.git"
+        # "https://github.com/Netflix/dgs-framework.git"
+        # "https://github.com/Netflix/vector.git"
+        # "https://github.com/expressjs/express.git"
+)
+REPOS=$(printf "%s," "${REPO_ARRAY[@]}" | cut -d "," -f 1-${#REPO_ARRAY[@]})
+go run hack/snifftest/main.go scan --exclude privatekey --exclude uri --exclude github_old --repo "$REPOS" --detector all --print  --fail-threshold 99

pkg/engine/defaults.go

@@ -6,7 +6,6 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/abuseipdb"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/accuweather"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/adafruitio"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/adobeio"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/adzuna"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/agora"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/airbrakeprojectkey"
@@ -33,7 +32,6 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/apify"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/apimatic"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/apiscience"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/apollo"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/appcues"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/appfollow"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/appsynergy"
@@ -145,7 +143,6 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/customerguru"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/customerio"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/d7network"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/dailyco"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/dandelion"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/datadogtoken"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/datafire"
@@ -320,7 +317,6 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/mailmodo"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/mailsac"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/mandrill"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/manifest"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/mapbox"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/mapquest"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/marketstack"
@@ -337,7 +333,6 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/mindmeister"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/mite"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/mixmax"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/mixpanel"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/moderation"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/monday"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/moonclerck"
@@ -495,7 +490,6 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/smooch"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/snipcart"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/snykkey"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/sparkpost"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/splunkobservabilitytoken"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/spoonacular"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/sportsmonk"
@@ -713,7 +707,7 @@ func DefaultDetectors() []detectors.Detector {
 		&notion.Scanner{},
 		&droneci.Scanner{},
 		&ipstack.Scanner{},
-		&adobeio.Scanner{},
+		// &adobeio.Scanner{},
 		&sslmate.Scanner{},
 		&buildkite.Scanner{},
 		&shodankey.Scanner{},
@@ -819,7 +813,7 @@ func DefaultDetectors() []detectors.Detector {
 		&clockify.Scanner{},
 		&karmacrm.Scanner{},
 		&revampcrm.Scanner{},
-		&apollo.Scanner{},
+		// &apollo.Scanner{},
 		&artsy.Scanner{},
 		&vpnapi.Scanner{},
 		&dnscheck.Scanner{},
@@ -910,7 +904,7 @@ func DefaultDetectors() []detectors.Detector {
 		spoonacular.Scanner{},
 		finnhub.Scanner{},
 		checkout.Scanner{},
-		mixpanel.Scanner{},
+		// mixpanel.Scanner{},
 		ipgeolocation.Scanner{},
 		tmetric.Scanner{},
 		fullstory.Scanner{},
@@ -949,7 +943,7 @@ func DefaultDetectors() []detectors.Detector {
 		zipcodeapi.Scanner{},
 		gyazo.Scanner{},
 		fakejson.Scanner{},
-		sparkpost.Scanner{},
+		// sparkpost.Scanner{},
 		locationiq.Scanner{},
 		saucelabs.Scanner{},
 		enigma.Scanner{},
@@ -1079,7 +1073,7 @@ func DefaultDetectors() []detectors.Detector {
 		edenai.Scanner{},
 		urlscan.Scanner{},
 		zenscrape.Scanner{},
-		dailyco.Scanner{},
+		// dailyco.Scanner{},
 		nicereply.Scanner{},
 		hive.Scanner{},
 		clustdoc.Scanner{},
@@ -1178,7 +1172,7 @@ func DefaultDetectors() []detectors.Detector {
 		gocanvas.Scanner{},
 		formio.Scanner{},
 		livestorm.Scanner{},
-		manifest.Scanner{},
+		// manifest.Scanner{},
 		formbucket.Scanner{},
 		apiscience.Scanner{},
 		dronahq.Scanner{},