From b4ac103d11f556a8ff7647ed2a639185f49c076c Mon Sep 17 00:00:00 2001 From: Dustin Decker Date: Thu, 10 Mar 2022 21:41:35 -0800 Subject: [PATCH] Add snifftest --- .github/workflows/lint.yml | 2 + .github/workflows/release.yml | 2 +- .github/workflows/snifftest.yml | 25 ++++ Makefile | 3 + go.mod | 1 + go.sum | 2 + hack/snifftest/README.md | 33 +++++ hack/snifftest/main.go | 227 ++++++++++++++++++++++++++++++++ hack/snifftest/snifftest.sh | 13 ++ pkg/engine/defaults.go | 18 +-- 10 files changed, 313 insertions(+), 13 deletions(-) create mode 100644 .github/workflows/snifftest.yml create mode 100644 hack/snifftest/README.md create mode 100644 hack/snifftest/main.go create mode 100755 hack/snifftest/snifftest.sh diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 2d5a17108..548b2e077 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -18,6 +18,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/setup-go@v2 + with: + go-version: '1.17' - uses: actions/checkout@v3 - name: golangci-lint uses: golangci/golangci-lint-action@v3 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a6c68949d..849b64ccd 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,4 +1,4 @@ -name: release +name: Release # on: # push: diff --git a/.github/workflows/snifftest.yml b/.github/workflows/snifftest.yml new file mode 100644 index 000000000..b70bfca64 --- /dev/null +++ b/.github/workflows/snifftest.yml @@ -0,0 +1,25 @@ +name: Snifftest + +on: + push: + tags: + - v* + branches: + - main + pull_request: + +permissions: + contents: read + pull-requests: read + +jobs: + Snifftest: + name: Run Snifftest + runs-on: ubuntu-latest + steps: + - uses: actions/setup-go@v2 + with: + go-version: '1.17' + - uses: actions/checkout@v3 + - name: Run Snifftest + run: make snifftest \ No newline at end of file diff --git a/Makefile b/Makefile index c5860346c..1550e8f51 100644 --- a/Makefile +++ b/Makefile @@ -46,3 +46,6 @@ protos: protos-windows: docker run -v "$(shell cygpath -w $(shell pwd))":/pwd "${PROTOS_IMAGE}" bash -c "cd /pwd; ./scripts/gen_proto.sh" + +snifftest: + ./hack/snifftest/snifftest.sh diff --git a/go.mod b/go.mod index eba188813..5fa61d3e0 100644 --- a/go.mod +++ b/go.mod @@ -23,6 +23,7 @@ require ( github.com/joho/godotenv v1.4.0 github.com/kylelemons/godebug v1.1.0 github.com/mattn/go-colorable v0.1.12 + github.com/paulbellamy/ratecounter v0.2.0 github.com/pkg/errors v0.9.1 github.com/razorpay/razorpay-go v0.0.0-20210728161131-0341409a6ab2 github.com/sergi/go-diff v1.2.0 diff --git a/go.sum b/go.sum index b477ef5d9..b0fcd60c2 100644 --- a/go.sum +++ b/go.sum @@ -328,6 +328,8 @@ github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= +github.com/paulbellamy/ratecounter v0.2.0 h1:2L/RhJq+HA8gBQImDXtLPrDXK5qAj6ozWVK/zFXVJGs= +github.com/paulbellamy/ratecounter v0.2.0/go.mod h1:Hfx1hDpSGoqxkVVpBi/IlYD7kChlfo5C6hzIHwPqfFE= github.com/pkg/diff v0.0.0-20200914180035-5b29258ca4f7 h1:+/+DxvQaYifJ+grD4klzrS5y+KJXldn/2YTl5JG+vZ8= github.com/pkg/diff v0.0.0-20200914180035-5b29258ca4f7/go.mod h1:zO8QMzTeZd5cpnIkz/Gn6iK0jDfGicM1nynOkkPIl28= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= diff --git a/hack/snifftest/README.md b/hack/snifftest/README.md new file mode 100644 index 000000000..6bfa66179 --- /dev/null +++ b/hack/snifftest/README.md @@ -0,0 +1,33 @@ +# snifftest + +See the help pages with this command, or look further below to get started quickly. + +``` +go run snifftest/main.go +``` + +## Show available secret scanners + +``` +go run snifftest/main.go show-scanners +``` + +## Load a repo into a DB + +``` +go run snifftest/main.go load --db ~/sdb --repo https://github.com/Netflix/Hystrix.git +``` + +## Scan + +All scanners + +``` +go run snifftest/main.go scan --db ~/sdb --scanner all --print +``` + +Particular scanner + +``` +go run snifftest/main.go scan --db ~/sdb --scanner github --print --print-chunk --fail-threshold 5 +``` \ No newline at end of file diff --git a/hack/snifftest/main.go b/hack/snifftest/main.go new file mode 100644 index 000000000..40fbba3c7 --- /dev/null +++ b/hack/snifftest/main.go @@ -0,0 +1,227 @@ +package main + +import ( + "context" + "fmt" + "os" + "reflect" + "runtime" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/paulbellamy/ratecounter" + log "github.com/sirupsen/logrus" + "golang.org/x/sync/semaphore" + "gopkg.in/alecthomas/kingpin.v2" + + "github.com/trufflesecurity/trufflehog/v3/pkg/decoders" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/engine" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb" + "github.com/trufflesecurity/trufflehog/v3/pkg/sources" + "github.com/trufflesecurity/trufflehog/v3/pkg/sources/git" +) + +var ( + // CLI flags and commands + app = kingpin.New("Snifftest", "Test secret detectors against data sets.") + + showDetectorsCmd = app.Command("show-detectors", "Shows the available detectors.") + + scanCmd = app.Command("scan", "Scans data.") + scanCmdDetector = scanCmd.Flag("detector", "Detector to scan with. 'all', or a specific name.").Default("all").String() + scanCmdExclude = scanCmd.Flag("exclude", "Detector(s) to exclude").Strings() + scanCmdRepo = scanCmd.Flag("repo", "URI to .git repo.").Required().String() + scanThreshold = scanCmd.Flag("fail-threshold", "Result threshold that causes failure for a single scanner.").Int() + scanPrintRes = scanCmd.Flag("print", "Print results.").Bool() + scanPrintChunkRes = scanCmd.Flag("print-chunk", "Print chunks that have results.").Bool() + scanVerify = scanCmd.Flag("verify", "Verify found secrets.").Bool() +) + +func main() { + ctx, cancel := context.WithTimeout(context.Background(), time.Hour*2) + var cancelOnce sync.Once + defer cancelOnce.Do(cancel) + + cmd := kingpin.MustParse(app.Parse(os.Args[1:])) + + switch cmd { + case scanCmd.FullCommand(): + + chunksChan := make(chan *sources.Chunk, 10000) + + var wgChunkers sync.WaitGroup + + sem := semaphore.NewWeighted(int64(runtime.NumCPU())) + + selectedScanners := map[string]detectors.Detector{} + allScanners := getAllScanners() + + decoders := decoders.DefaultDecoders() + + input := strings.ToLower(*scanCmdDetector) + if input == "all" { + selectedScanners = allScanners + } else { + _, ok := allScanners[input] + if !ok { + log.Fatal("could not find scanner by that name") + } + selectedScanners[input] = allScanners[input] + } + if len(selectedScanners) == 0 { + log.Fatal("no detectors selected") + } + + for _, excluded := range *scanCmdExclude { + delete(selectedScanners, excluded) + } + + log.Infof("loaded %d secret detectors", len(selectedScanners)+3) + + var wgScanners sync.WaitGroup + + var chunkCounter uint64 + go func() { + counter := ratecounter.NewRateCounter(60 * time.Second) + var prev uint64 + for { + time.Sleep(60 * time.Second) + counter.Incr(int64(chunkCounter - prev)) + prev = chunkCounter + log.Infof("chunk scan rate: %d/sec", counter.Rate()/60) + } + }() + + resCounter := make(map[string]*uint64) + failed := false + + for i := 0; i < runtime.NumCPU(); i++ { + wgScanners.Add(1) + + go func() { + defer wgScanners.Done() + + for chunk := range chunksChan { + for name, scanner := range selectedScanners { + for _, dec := range decoders { + decoded := dec.FromChunk(&sources.Chunk{Data: chunk.Data}) + if decoded != nil { + foundKeyword := false + for _, kw := range scanner.Keywords() { + if strings.Contains(strings.ToLower(string(decoded.Data)), strings.ToLower(kw)) { + foundKeyword = true + } + } + if !foundKeyword { + continue + } + + res, err := scanner.FromData(ctx, *scanVerify, decoded.Data) + if err != nil { + log.Fatal(err) + } + if len(res) > 0 { + if resCounter[name] == nil { + zero := uint64(0) + resCounter[name] = &zero + } + atomic.AddUint64(resCounter[name], uint64(len(res))) + if *scanThreshold != 0 && int(*resCounter[name]) > *scanThreshold { + log.WithField("scanner", name).Errorf("exceeded result threshold of %d", *scanThreshold) + failed = true + os.Exit(1) + } + + if *scanPrintRes { + for _, r := range res { + logger := log.WithField("secret", name).WithField("meta", chunk.SourceMetadata).WithField("result", string(r.Raw)) + if *scanPrintChunkRes { + logger = logger.WithField("chunk", string(decoded.Data)) + } + logger.Info("result") + } + } + } + } + } + } + + atomic.AddUint64(&chunkCounter, uint64(1)) + } + }() + } + + for _, repo := range strings.Split(*scanCmdRepo, ",") { + sem.Acquire(ctx, 1) + wgChunkers.Add(1) + go func(r string) { + defer sem.Release(1) + defer wgChunkers.Done() + log.Infof("cloning %s", r) + path, repo, err := git.CloneRepoUsingUnauthenticated(r) + if err != nil { + log.Fatal(err) + } + + log.Infof("cloned %s", r) + + s := git.NewGit(sourcespb.SourceType_SOURCE_TYPE_GIT, 0, 0, "snifftest", false, runtime.NumCPU(), + func(file, email, commit, timestamp, repository string) *source_metadatapb.MetaData { + return &source_metadatapb.MetaData{ + Data: &source_metadatapb.MetaData_Git{ + Git: &source_metadatapb.Git{ + Commit: commit, + File: file, + Email: email, + Repository: repository, + Timestamp: timestamp, + }, + }, + } + }) + + log.Infof("scanning %s", r) + err = s.ScanRepo(ctx, repo, git.NewScanOptions(), chunksChan) + if err != nil { + log.Fatal(err) + } + log.Infof("scanned %s", r) + defer os.RemoveAll(path) + }(repo) + } + + go func() { + wgChunkers.Wait() + close(chunksChan) + }() + + wgScanners.Wait() + + log.WithField("chunks", chunkCounter).Info("completed") + for scanner, resultsCount := range resCounter { + log.WithField("results", *resultsCount).Info(scanner) + } + + if failed { + os.Exit(1) + } + case showDetectorsCmd.FullCommand(): + for s := range getAllScanners() { + fmt.Println(s) + } + } +} + +func getAllScanners() map[string]detectors.Detector { + allScanners := map[string]detectors.Detector{} + for _, s := range engine.DefaultDetectors() { + secretType := reflect.Indirect(reflect.ValueOf(s)).Type().PkgPath() + path := strings.Split(secretType, "/")[len(strings.Split(secretType, "/"))-1] + allScanners[path] = s + } + return allScanners +} diff --git a/hack/snifftest/snifftest.sh b/hack/snifftest/snifftest.sh new file mode 100755 index 000000000..452dcd43e --- /dev/null +++ b/hack/snifftest/snifftest.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +REPO_ARRAY=( + "https://github.com/Netflix/Hystrix.git" + # "https://github.com/facebook/flow.git" + # "https://github.com/Netflix/vizceral.git" + # "https://github.com/Netflix/metaflow.git" + # "https://github.com/Netflix/dgs-framework.git" + # "https://github.com/Netflix/vector.git" + # "https://github.com/expressjs/express.git" +) +REPOS=$(printf "%s," "${REPO_ARRAY[@]}" | cut -d "," -f 1-${#REPO_ARRAY[@]}) +go run hack/snifftest/main.go scan --exclude privatekey --exclude uri --exclude github_old --repo "$REPOS" --detector all --print --fail-threshold 99 \ No newline at end of file diff --git a/pkg/engine/defaults.go b/pkg/engine/defaults.go index 0b7161052..71f9a2b0e 100644 --- a/pkg/engine/defaults.go +++ b/pkg/engine/defaults.go @@ -6,7 +6,6 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/abuseipdb" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/accuweather" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/adafruitio" - "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/adobeio" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/adzuna" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/agora" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/airbrakeprojectkey" @@ -33,7 +32,6 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/apify" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/apimatic" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/apiscience" - "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/apollo" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/appcues" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/appfollow" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/appsynergy" @@ -145,7 +143,6 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/customerguru" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/customerio" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/d7network" - "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/dailyco" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/dandelion" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/datadogtoken" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/datafire" @@ -320,7 +317,6 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/mailmodo" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/mailsac" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/mandrill" - "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/manifest" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/mapbox" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/mapquest" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/marketstack" @@ -337,7 +333,6 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/mindmeister" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/mite" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/mixmax" - "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/mixpanel" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/moderation" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/monday" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/moonclerck" @@ -495,7 +490,6 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/smooch" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/snipcart" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/snykkey" - "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/sparkpost" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/splunkobservabilitytoken" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/spoonacular" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/sportsmonk" @@ -713,7 +707,7 @@ func DefaultDetectors() []detectors.Detector { ¬ion.Scanner{}, &droneci.Scanner{}, &ipstack.Scanner{}, - &adobeio.Scanner{}, + // &adobeio.Scanner{}, &sslmate.Scanner{}, &buildkite.Scanner{}, &shodankey.Scanner{}, @@ -819,7 +813,7 @@ func DefaultDetectors() []detectors.Detector { &clockify.Scanner{}, &karmacrm.Scanner{}, &revampcrm.Scanner{}, - &apollo.Scanner{}, + // &apollo.Scanner{}, &artsy.Scanner{}, &vpnapi.Scanner{}, &dnscheck.Scanner{}, @@ -910,7 +904,7 @@ func DefaultDetectors() []detectors.Detector { spoonacular.Scanner{}, finnhub.Scanner{}, checkout.Scanner{}, - mixpanel.Scanner{}, + // mixpanel.Scanner{}, ipgeolocation.Scanner{}, tmetric.Scanner{}, fullstory.Scanner{}, @@ -949,7 +943,7 @@ func DefaultDetectors() []detectors.Detector { zipcodeapi.Scanner{}, gyazo.Scanner{}, fakejson.Scanner{}, - sparkpost.Scanner{}, + // sparkpost.Scanner{}, locationiq.Scanner{}, saucelabs.Scanner{}, enigma.Scanner{}, @@ -1079,7 +1073,7 @@ func DefaultDetectors() []detectors.Detector { edenai.Scanner{}, urlscan.Scanner{}, zenscrape.Scanner{}, - dailyco.Scanner{}, + // dailyco.Scanner{}, nicereply.Scanner{}, hive.Scanner{}, clustdoc.Scanner{}, @@ -1178,7 +1172,7 @@ func DefaultDetectors() []detectors.Detector { gocanvas.Scanner{}, formio.Scanner{}, livestorm.Scanner{}, - manifest.Scanner{}, + // manifest.Scanner{}, formbucket.Scanner{}, apiscience.Scanner{}, dronahq.Scanner{},