Add snifftest

This commit is contained in:
Dustin Decker 2022-03-10 21:41:35 -08:00 committed by Dustin Decker
parent b0e0c26137
commit b4ac103d11
10 changed files with 313 additions and 13 deletions

View file

@ -18,6 +18,8 @@ jobs:
runs-on: ubuntu-latest runs-on: ubuntu-latest
steps: steps:
- uses: actions/setup-go@v2 - uses: actions/setup-go@v2
with:
go-version: '1.17'
- uses: actions/checkout@v3 - uses: actions/checkout@v3
- name: golangci-lint - name: golangci-lint
uses: golangci/golangci-lint-action@v3 uses: golangci/golangci-lint-action@v3

View file

@ -1,4 +1,4 @@
name: release name: Release
# on: # on:
# push: # push:

25
.github/workflows/snifftest.yml vendored Normal file
View file

@ -0,0 +1,25 @@
name: Snifftest
on:
push:
tags:
- v*
branches:
- main
pull_request:
permissions:
contents: read
pull-requests: read
jobs:
Snifftest:
name: Run Snifftest
runs-on: ubuntu-latest
steps:
- uses: actions/setup-go@v2
with:
go-version: '1.17'
- uses: actions/checkout@v3
- name: Run Snifftest
run: make snifftest

View file

@ -46,3 +46,6 @@ protos:
protos-windows: protos-windows:
docker run -v "$(shell cygpath -w $(shell pwd))":/pwd "${PROTOS_IMAGE}" bash -c "cd /pwd; ./scripts/gen_proto.sh" docker run -v "$(shell cygpath -w $(shell pwd))":/pwd "${PROTOS_IMAGE}" bash -c "cd /pwd; ./scripts/gen_proto.sh"
snifftest:
./hack/snifftest/snifftest.sh

1
go.mod
View file

@ -23,6 +23,7 @@ require (
github.com/joho/godotenv v1.4.0 github.com/joho/godotenv v1.4.0
github.com/kylelemons/godebug v1.1.0 github.com/kylelemons/godebug v1.1.0
github.com/mattn/go-colorable v0.1.12 github.com/mattn/go-colorable v0.1.12
github.com/paulbellamy/ratecounter v0.2.0
github.com/pkg/errors v0.9.1 github.com/pkg/errors v0.9.1
github.com/razorpay/razorpay-go v0.0.0-20210728161131-0341409a6ab2 github.com/razorpay/razorpay-go v0.0.0-20210728161131-0341409a6ab2
github.com/sergi/go-diff v1.2.0 github.com/sergi/go-diff v1.2.0

2
go.sum
View file

@ -328,6 +328,8 @@ github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9
github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk=
github.com/paulbellamy/ratecounter v0.2.0 h1:2L/RhJq+HA8gBQImDXtLPrDXK5qAj6ozWVK/zFXVJGs=
github.com/paulbellamy/ratecounter v0.2.0/go.mod h1:Hfx1hDpSGoqxkVVpBi/IlYD7kChlfo5C6hzIHwPqfFE=
github.com/pkg/diff v0.0.0-20200914180035-5b29258ca4f7 h1:+/+DxvQaYifJ+grD4klzrS5y+KJXldn/2YTl5JG+vZ8= github.com/pkg/diff v0.0.0-20200914180035-5b29258ca4f7 h1:+/+DxvQaYifJ+grD4klzrS5y+KJXldn/2YTl5JG+vZ8=
github.com/pkg/diff v0.0.0-20200914180035-5b29258ca4f7/go.mod h1:zO8QMzTeZd5cpnIkz/Gn6iK0jDfGicM1nynOkkPIl28= github.com/pkg/diff v0.0.0-20200914180035-5b29258ca4f7/go.mod h1:zO8QMzTeZd5cpnIkz/Gn6iK0jDfGicM1nynOkkPIl28=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=

33
hack/snifftest/README.md Normal file
View file

@ -0,0 +1,33 @@
# snifftest
See the help pages with this command, or look further below to get started quickly.
```
go run snifftest/main.go
```
## Show available secret scanners
```
go run snifftest/main.go show-scanners
```
## Load a repo into a DB
```
go run snifftest/main.go load --db ~/sdb --repo https://github.com/Netflix/Hystrix.git
```
## Scan
All scanners
```
go run snifftest/main.go scan --db ~/sdb --scanner all --print
```
Particular scanner
```
go run snifftest/main.go scan --db ~/sdb --scanner github --print --print-chunk --fail-threshold 5
```

227
hack/snifftest/main.go Normal file
View file

@ -0,0 +1,227 @@
package main
import (
"context"
"fmt"
"os"
"reflect"
"runtime"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/paulbellamy/ratecounter"
log "github.com/sirupsen/logrus"
"golang.org/x/sync/semaphore"
"gopkg.in/alecthomas/kingpin.v2"
"github.com/trufflesecurity/trufflehog/v3/pkg/decoders"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/engine"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources/git"
)
var (
// CLI flags and commands
app = kingpin.New("Snifftest", "Test secret detectors against data sets.")
showDetectorsCmd = app.Command("show-detectors", "Shows the available detectors.")
scanCmd = app.Command("scan", "Scans data.")
scanCmdDetector = scanCmd.Flag("detector", "Detector to scan with. 'all', or a specific name.").Default("all").String()
scanCmdExclude = scanCmd.Flag("exclude", "Detector(s) to exclude").Strings()
scanCmdRepo = scanCmd.Flag("repo", "URI to .git repo.").Required().String()
scanThreshold = scanCmd.Flag("fail-threshold", "Result threshold that causes failure for a single scanner.").Int()
scanPrintRes = scanCmd.Flag("print", "Print results.").Bool()
scanPrintChunkRes = scanCmd.Flag("print-chunk", "Print chunks that have results.").Bool()
scanVerify = scanCmd.Flag("verify", "Verify found secrets.").Bool()
)
func main() {
ctx, cancel := context.WithTimeout(context.Background(), time.Hour*2)
var cancelOnce sync.Once
defer cancelOnce.Do(cancel)
cmd := kingpin.MustParse(app.Parse(os.Args[1:]))
switch cmd {
case scanCmd.FullCommand():
chunksChan := make(chan *sources.Chunk, 10000)
var wgChunkers sync.WaitGroup
sem := semaphore.NewWeighted(int64(runtime.NumCPU()))
selectedScanners := map[string]detectors.Detector{}
allScanners := getAllScanners()
decoders := decoders.DefaultDecoders()
input := strings.ToLower(*scanCmdDetector)
if input == "all" {
selectedScanners = allScanners
} else {
_, ok := allScanners[input]
if !ok {
log.Fatal("could not find scanner by that name")
}
selectedScanners[input] = allScanners[input]
}
if len(selectedScanners) == 0 {
log.Fatal("no detectors selected")
}
for _, excluded := range *scanCmdExclude {
delete(selectedScanners, excluded)
}
log.Infof("loaded %d secret detectors", len(selectedScanners)+3)
var wgScanners sync.WaitGroup
var chunkCounter uint64
go func() {
counter := ratecounter.NewRateCounter(60 * time.Second)
var prev uint64
for {
time.Sleep(60 * time.Second)
counter.Incr(int64(chunkCounter - prev))
prev = chunkCounter
log.Infof("chunk scan rate: %d/sec", counter.Rate()/60)
}
}()
resCounter := make(map[string]*uint64)
failed := false
for i := 0; i < runtime.NumCPU(); i++ {
wgScanners.Add(1)
go func() {
defer wgScanners.Done()
for chunk := range chunksChan {
for name, scanner := range selectedScanners {
for _, dec := range decoders {
decoded := dec.FromChunk(&sources.Chunk{Data: chunk.Data})
if decoded != nil {
foundKeyword := false
for _, kw := range scanner.Keywords() {
if strings.Contains(strings.ToLower(string(decoded.Data)), strings.ToLower(kw)) {
foundKeyword = true
}
}
if !foundKeyword {
continue
}
res, err := scanner.FromData(ctx, *scanVerify, decoded.Data)
if err != nil {
log.Fatal(err)
}
if len(res) > 0 {
if resCounter[name] == nil {
zero := uint64(0)
resCounter[name] = &zero
}
atomic.AddUint64(resCounter[name], uint64(len(res)))
if *scanThreshold != 0 && int(*resCounter[name]) > *scanThreshold {
log.WithField("scanner", name).Errorf("exceeded result threshold of %d", *scanThreshold)
failed = true
os.Exit(1)
}
if *scanPrintRes {
for _, r := range res {
logger := log.WithField("secret", name).WithField("meta", chunk.SourceMetadata).WithField("result", string(r.Raw))
if *scanPrintChunkRes {
logger = logger.WithField("chunk", string(decoded.Data))
}
logger.Info("result")
}
}
}
}
}
}
atomic.AddUint64(&chunkCounter, uint64(1))
}
}()
}
for _, repo := range strings.Split(*scanCmdRepo, ",") {
sem.Acquire(ctx, 1)
wgChunkers.Add(1)
go func(r string) {
defer sem.Release(1)
defer wgChunkers.Done()
log.Infof("cloning %s", r)
path, repo, err := git.CloneRepoUsingUnauthenticated(r)
if err != nil {
log.Fatal(err)
}
log.Infof("cloned %s", r)
s := git.NewGit(sourcespb.SourceType_SOURCE_TYPE_GIT, 0, 0, "snifftest", false, runtime.NumCPU(),
func(file, email, commit, timestamp, repository string) *source_metadatapb.MetaData {
return &source_metadatapb.MetaData{
Data: &source_metadatapb.MetaData_Git{
Git: &source_metadatapb.Git{
Commit: commit,
File: file,
Email: email,
Repository: repository,
Timestamp: timestamp,
},
},
}
})
log.Infof("scanning %s", r)
err = s.ScanRepo(ctx, repo, git.NewScanOptions(), chunksChan)
if err != nil {
log.Fatal(err)
}
log.Infof("scanned %s", r)
defer os.RemoveAll(path)
}(repo)
}
go func() {
wgChunkers.Wait()
close(chunksChan)
}()
wgScanners.Wait()
log.WithField("chunks", chunkCounter).Info("completed")
for scanner, resultsCount := range resCounter {
log.WithField("results", *resultsCount).Info(scanner)
}
if failed {
os.Exit(1)
}
case showDetectorsCmd.FullCommand():
for s := range getAllScanners() {
fmt.Println(s)
}
}
}
func getAllScanners() map[string]detectors.Detector {
allScanners := map[string]detectors.Detector{}
for _, s := range engine.DefaultDetectors() {
secretType := reflect.Indirect(reflect.ValueOf(s)).Type().PkgPath()
path := strings.Split(secretType, "/")[len(strings.Split(secretType, "/"))-1]
allScanners[path] = s
}
return allScanners
}

13
hack/snifftest/snifftest.sh Executable file
View file

@ -0,0 +1,13 @@
#!/usr/bin/env bash
REPO_ARRAY=(
"https://github.com/Netflix/Hystrix.git"
# "https://github.com/facebook/flow.git"
# "https://github.com/Netflix/vizceral.git"
# "https://github.com/Netflix/metaflow.git"
# "https://github.com/Netflix/dgs-framework.git"
# "https://github.com/Netflix/vector.git"
# "https://github.com/expressjs/express.git"
)
REPOS=$(printf "%s," "${REPO_ARRAY[@]}" | cut -d "," -f 1-${#REPO_ARRAY[@]})
go run hack/snifftest/main.go scan --exclude privatekey --exclude uri --exclude github_old --repo "$REPOS" --detector all --print --fail-threshold 99

View file

@ -6,7 +6,6 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/abuseipdb" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/abuseipdb"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/accuweather" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/accuweather"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/adafruitio" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/adafruitio"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/adobeio"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/adzuna" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/adzuna"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/agora" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/agora"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/airbrakeprojectkey" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/airbrakeprojectkey"
@ -33,7 +32,6 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/apify" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/apify"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/apimatic" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/apimatic"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/apiscience" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/apiscience"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/apollo"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/appcues" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/appcues"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/appfollow" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/appfollow"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/appsynergy" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/appsynergy"
@ -145,7 +143,6 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/customerguru" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/customerguru"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/customerio" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/customerio"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/d7network" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/d7network"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/dailyco"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/dandelion" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/dandelion"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/datadogtoken" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/datadogtoken"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/datafire" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/datafire"
@ -320,7 +317,6 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/mailmodo" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/mailmodo"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/mailsac" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/mailsac"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/mandrill" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/mandrill"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/manifest"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/mapbox" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/mapbox"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/mapquest" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/mapquest"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/marketstack" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/marketstack"
@ -337,7 +333,6 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/mindmeister" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/mindmeister"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/mite" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/mite"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/mixmax" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/mixmax"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/mixpanel"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/moderation" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/moderation"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/monday" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/monday"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/moonclerck" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/moonclerck"
@ -495,7 +490,6 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/smooch" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/smooch"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/snipcart" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/snipcart"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/snykkey" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/snykkey"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/sparkpost"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/splunkobservabilitytoken" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/splunkobservabilitytoken"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/spoonacular" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/spoonacular"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/sportsmonk" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/sportsmonk"
@ -713,7 +707,7 @@ func DefaultDetectors() []detectors.Detector {
&notion.Scanner{}, &notion.Scanner{},
&droneci.Scanner{}, &droneci.Scanner{},
&ipstack.Scanner{}, &ipstack.Scanner{},
&adobeio.Scanner{}, // &adobeio.Scanner{},
&sslmate.Scanner{}, &sslmate.Scanner{},
&buildkite.Scanner{}, &buildkite.Scanner{},
&shodankey.Scanner{}, &shodankey.Scanner{},
@ -819,7 +813,7 @@ func DefaultDetectors() []detectors.Detector {
&clockify.Scanner{}, &clockify.Scanner{},
&karmacrm.Scanner{}, &karmacrm.Scanner{},
&revampcrm.Scanner{}, &revampcrm.Scanner{},
&apollo.Scanner{}, // &apollo.Scanner{},
&artsy.Scanner{}, &artsy.Scanner{},
&vpnapi.Scanner{}, &vpnapi.Scanner{},
&dnscheck.Scanner{}, &dnscheck.Scanner{},
@ -910,7 +904,7 @@ func DefaultDetectors() []detectors.Detector {
spoonacular.Scanner{}, spoonacular.Scanner{},
finnhub.Scanner{}, finnhub.Scanner{},
checkout.Scanner{}, checkout.Scanner{},
mixpanel.Scanner{}, // mixpanel.Scanner{},
ipgeolocation.Scanner{}, ipgeolocation.Scanner{},
tmetric.Scanner{}, tmetric.Scanner{},
fullstory.Scanner{}, fullstory.Scanner{},
@ -949,7 +943,7 @@ func DefaultDetectors() []detectors.Detector {
zipcodeapi.Scanner{}, zipcodeapi.Scanner{},
gyazo.Scanner{}, gyazo.Scanner{},
fakejson.Scanner{}, fakejson.Scanner{},
sparkpost.Scanner{}, // sparkpost.Scanner{},
locationiq.Scanner{}, locationiq.Scanner{},
saucelabs.Scanner{}, saucelabs.Scanner{},
enigma.Scanner{}, enigma.Scanner{},
@ -1079,7 +1073,7 @@ func DefaultDetectors() []detectors.Detector {
edenai.Scanner{}, edenai.Scanner{},
urlscan.Scanner{}, urlscan.Scanner{},
zenscrape.Scanner{}, zenscrape.Scanner{},
dailyco.Scanner{}, // dailyco.Scanner{},
nicereply.Scanner{}, nicereply.Scanner{},
hive.Scanner{}, hive.Scanner{},
clustdoc.Scanner{}, clustdoc.Scanner{},
@ -1178,7 +1172,7 @@ func DefaultDetectors() []detectors.Detector {
gocanvas.Scanner{}, gocanvas.Scanner{},
formio.Scanner{}, formio.Scanner{},
livestorm.Scanner{}, livestorm.Scanner{},
manifest.Scanner{}, // manifest.Scanner{},
formbucket.Scanner{}, formbucket.Scanner{},
apiscience.Scanner{}, apiscience.Scanner{},
dronahq.Scanner{}, dronahq.Scanner{},