mirror of
https://github.com/trufflesecurity/trufflehog.git
synced 2024-09-20 06:31:57 +00:00
Add snifftest
This commit is contained in:
parent
b0e0c26137
commit
b4ac103d11
10 changed files with 313 additions and 13 deletions
2
.github/workflows/lint.yml
vendored
2
.github/workflows/lint.yml
vendored
|
@ -18,6 +18,8 @@ jobs:
|
|||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/setup-go@v2
|
||||
with:
|
||||
go-version: '1.17'
|
||||
- uses: actions/checkout@v3
|
||||
- name: golangci-lint
|
||||
uses: golangci/golangci-lint-action@v3
|
||||
|
|
2
.github/workflows/release.yml
vendored
2
.github/workflows/release.yml
vendored
|
@ -1,4 +1,4 @@
|
|||
name: release
|
||||
name: Release
|
||||
|
||||
# on:
|
||||
# push:
|
||||
|
|
25
.github/workflows/snifftest.yml
vendored
Normal file
25
.github/workflows/snifftest.yml
vendored
Normal file
|
@ -0,0 +1,25 @@
|
|||
name: Snifftest
|
||||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- v*
|
||||
branches:
|
||||
- main
|
||||
pull_request:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: read
|
||||
|
||||
jobs:
|
||||
Snifftest:
|
||||
name: Run Snifftest
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/setup-go@v2
|
||||
with:
|
||||
go-version: '1.17'
|
||||
- uses: actions/checkout@v3
|
||||
- name: Run Snifftest
|
||||
run: make snifftest
|
3
Makefile
3
Makefile
|
@ -46,3 +46,6 @@ protos:
|
|||
|
||||
protos-windows:
|
||||
docker run -v "$(shell cygpath -w $(shell pwd))":/pwd "${PROTOS_IMAGE}" bash -c "cd /pwd; ./scripts/gen_proto.sh"
|
||||
|
||||
snifftest:
|
||||
./hack/snifftest/snifftest.sh
|
||||
|
|
1
go.mod
1
go.mod
|
@ -23,6 +23,7 @@ require (
|
|||
github.com/joho/godotenv v1.4.0
|
||||
github.com/kylelemons/godebug v1.1.0
|
||||
github.com/mattn/go-colorable v0.1.12
|
||||
github.com/paulbellamy/ratecounter v0.2.0
|
||||
github.com/pkg/errors v0.9.1
|
||||
github.com/razorpay/razorpay-go v0.0.0-20210728161131-0341409a6ab2
|
||||
github.com/sergi/go-diff v1.2.0
|
||||
|
|
2
go.sum
2
go.sum
|
@ -328,6 +328,8 @@ github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9
|
|||
github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
|
||||
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
|
||||
github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk=
|
||||
github.com/paulbellamy/ratecounter v0.2.0 h1:2L/RhJq+HA8gBQImDXtLPrDXK5qAj6ozWVK/zFXVJGs=
|
||||
github.com/paulbellamy/ratecounter v0.2.0/go.mod h1:Hfx1hDpSGoqxkVVpBi/IlYD7kChlfo5C6hzIHwPqfFE=
|
||||
github.com/pkg/diff v0.0.0-20200914180035-5b29258ca4f7 h1:+/+DxvQaYifJ+grD4klzrS5y+KJXldn/2YTl5JG+vZ8=
|
||||
github.com/pkg/diff v0.0.0-20200914180035-5b29258ca4f7/go.mod h1:zO8QMzTeZd5cpnIkz/Gn6iK0jDfGicM1nynOkkPIl28=
|
||||
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
|
|
33
hack/snifftest/README.md
Normal file
33
hack/snifftest/README.md
Normal file
|
@ -0,0 +1,33 @@
|
|||
# snifftest
|
||||
|
||||
See the help pages with this command, or look further below to get started quickly.
|
||||
|
||||
```
|
||||
go run snifftest/main.go
|
||||
```
|
||||
|
||||
## Show available secret scanners
|
||||
|
||||
```
|
||||
go run snifftest/main.go show-scanners
|
||||
```
|
||||
|
||||
## Load a repo into a DB
|
||||
|
||||
```
|
||||
go run snifftest/main.go load --db ~/sdb --repo https://github.com/Netflix/Hystrix.git
|
||||
```
|
||||
|
||||
## Scan
|
||||
|
||||
All scanners
|
||||
|
||||
```
|
||||
go run snifftest/main.go scan --db ~/sdb --scanner all --print
|
||||
```
|
||||
|
||||
Particular scanner
|
||||
|
||||
```
|
||||
go run snifftest/main.go scan --db ~/sdb --scanner github --print --print-chunk --fail-threshold 5
|
||||
```
|
227
hack/snifftest/main.go
Normal file
227
hack/snifftest/main.go
Normal file
|
@ -0,0 +1,227 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"reflect"
|
||||
"runtime"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/paulbellamy/ratecounter"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"golang.org/x/sync/semaphore"
|
||||
"gopkg.in/alecthomas/kingpin.v2"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/decoders"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/engine"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources/git"
|
||||
)
|
||||
|
||||
var (
|
||||
// CLI flags and commands
|
||||
app = kingpin.New("Snifftest", "Test secret detectors against data sets.")
|
||||
|
||||
showDetectorsCmd = app.Command("show-detectors", "Shows the available detectors.")
|
||||
|
||||
scanCmd = app.Command("scan", "Scans data.")
|
||||
scanCmdDetector = scanCmd.Flag("detector", "Detector to scan with. 'all', or a specific name.").Default("all").String()
|
||||
scanCmdExclude = scanCmd.Flag("exclude", "Detector(s) to exclude").Strings()
|
||||
scanCmdRepo = scanCmd.Flag("repo", "URI to .git repo.").Required().String()
|
||||
scanThreshold = scanCmd.Flag("fail-threshold", "Result threshold that causes failure for a single scanner.").Int()
|
||||
scanPrintRes = scanCmd.Flag("print", "Print results.").Bool()
|
||||
scanPrintChunkRes = scanCmd.Flag("print-chunk", "Print chunks that have results.").Bool()
|
||||
scanVerify = scanCmd.Flag("verify", "Verify found secrets.").Bool()
|
||||
)
|
||||
|
||||
func main() {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Hour*2)
|
||||
var cancelOnce sync.Once
|
||||
defer cancelOnce.Do(cancel)
|
||||
|
||||
cmd := kingpin.MustParse(app.Parse(os.Args[1:]))
|
||||
|
||||
switch cmd {
|
||||
case scanCmd.FullCommand():
|
||||
|
||||
chunksChan := make(chan *sources.Chunk, 10000)
|
||||
|
||||
var wgChunkers sync.WaitGroup
|
||||
|
||||
sem := semaphore.NewWeighted(int64(runtime.NumCPU()))
|
||||
|
||||
selectedScanners := map[string]detectors.Detector{}
|
||||
allScanners := getAllScanners()
|
||||
|
||||
decoders := decoders.DefaultDecoders()
|
||||
|
||||
input := strings.ToLower(*scanCmdDetector)
|
||||
if input == "all" {
|
||||
selectedScanners = allScanners
|
||||
} else {
|
||||
_, ok := allScanners[input]
|
||||
if !ok {
|
||||
log.Fatal("could not find scanner by that name")
|
||||
}
|
||||
selectedScanners[input] = allScanners[input]
|
||||
}
|
||||
if len(selectedScanners) == 0 {
|
||||
log.Fatal("no detectors selected")
|
||||
}
|
||||
|
||||
for _, excluded := range *scanCmdExclude {
|
||||
delete(selectedScanners, excluded)
|
||||
}
|
||||
|
||||
log.Infof("loaded %d secret detectors", len(selectedScanners)+3)
|
||||
|
||||
var wgScanners sync.WaitGroup
|
||||
|
||||
var chunkCounter uint64
|
||||
go func() {
|
||||
counter := ratecounter.NewRateCounter(60 * time.Second)
|
||||
var prev uint64
|
||||
for {
|
||||
time.Sleep(60 * time.Second)
|
||||
counter.Incr(int64(chunkCounter - prev))
|
||||
prev = chunkCounter
|
||||
log.Infof("chunk scan rate: %d/sec", counter.Rate()/60)
|
||||
}
|
||||
}()
|
||||
|
||||
resCounter := make(map[string]*uint64)
|
||||
failed := false
|
||||
|
||||
for i := 0; i < runtime.NumCPU(); i++ {
|
||||
wgScanners.Add(1)
|
||||
|
||||
go func() {
|
||||
defer wgScanners.Done()
|
||||
|
||||
for chunk := range chunksChan {
|
||||
for name, scanner := range selectedScanners {
|
||||
for _, dec := range decoders {
|
||||
decoded := dec.FromChunk(&sources.Chunk{Data: chunk.Data})
|
||||
if decoded != nil {
|
||||
foundKeyword := false
|
||||
for _, kw := range scanner.Keywords() {
|
||||
if strings.Contains(strings.ToLower(string(decoded.Data)), strings.ToLower(kw)) {
|
||||
foundKeyword = true
|
||||
}
|
||||
}
|
||||
if !foundKeyword {
|
||||
continue
|
||||
}
|
||||
|
||||
res, err := scanner.FromData(ctx, *scanVerify, decoded.Data)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
if len(res) > 0 {
|
||||
if resCounter[name] == nil {
|
||||
zero := uint64(0)
|
||||
resCounter[name] = &zero
|
||||
}
|
||||
atomic.AddUint64(resCounter[name], uint64(len(res)))
|
||||
if *scanThreshold != 0 && int(*resCounter[name]) > *scanThreshold {
|
||||
log.WithField("scanner", name).Errorf("exceeded result threshold of %d", *scanThreshold)
|
||||
failed = true
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
if *scanPrintRes {
|
||||
for _, r := range res {
|
||||
logger := log.WithField("secret", name).WithField("meta", chunk.SourceMetadata).WithField("result", string(r.Raw))
|
||||
if *scanPrintChunkRes {
|
||||
logger = logger.WithField("chunk", string(decoded.Data))
|
||||
}
|
||||
logger.Info("result")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
atomic.AddUint64(&chunkCounter, uint64(1))
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
for _, repo := range strings.Split(*scanCmdRepo, ",") {
|
||||
sem.Acquire(ctx, 1)
|
||||
wgChunkers.Add(1)
|
||||
go func(r string) {
|
||||
defer sem.Release(1)
|
||||
defer wgChunkers.Done()
|
||||
log.Infof("cloning %s", r)
|
||||
path, repo, err := git.CloneRepoUsingUnauthenticated(r)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
log.Infof("cloned %s", r)
|
||||
|
||||
s := git.NewGit(sourcespb.SourceType_SOURCE_TYPE_GIT, 0, 0, "snifftest", false, runtime.NumCPU(),
|
||||
func(file, email, commit, timestamp, repository string) *source_metadatapb.MetaData {
|
||||
return &source_metadatapb.MetaData{
|
||||
Data: &source_metadatapb.MetaData_Git{
|
||||
Git: &source_metadatapb.Git{
|
||||
Commit: commit,
|
||||
File: file,
|
||||
Email: email,
|
||||
Repository: repository,
|
||||
Timestamp: timestamp,
|
||||
},
|
||||
},
|
||||
}
|
||||
})
|
||||
|
||||
log.Infof("scanning %s", r)
|
||||
err = s.ScanRepo(ctx, repo, git.NewScanOptions(), chunksChan)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
log.Infof("scanned %s", r)
|
||||
defer os.RemoveAll(path)
|
||||
}(repo)
|
||||
}
|
||||
|
||||
go func() {
|
||||
wgChunkers.Wait()
|
||||
close(chunksChan)
|
||||
}()
|
||||
|
||||
wgScanners.Wait()
|
||||
|
||||
log.WithField("chunks", chunkCounter).Info("completed")
|
||||
for scanner, resultsCount := range resCounter {
|
||||
log.WithField("results", *resultsCount).Info(scanner)
|
||||
}
|
||||
|
||||
if failed {
|
||||
os.Exit(1)
|
||||
}
|
||||
case showDetectorsCmd.FullCommand():
|
||||
for s := range getAllScanners() {
|
||||
fmt.Println(s)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func getAllScanners() map[string]detectors.Detector {
|
||||
allScanners := map[string]detectors.Detector{}
|
||||
for _, s := range engine.DefaultDetectors() {
|
||||
secretType := reflect.Indirect(reflect.ValueOf(s)).Type().PkgPath()
|
||||
path := strings.Split(secretType, "/")[len(strings.Split(secretType, "/"))-1]
|
||||
allScanners[path] = s
|
||||
}
|
||||
return allScanners
|
||||
}
|
13
hack/snifftest/snifftest.sh
Executable file
13
hack/snifftest/snifftest.sh
Executable file
|
@ -0,0 +1,13 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
REPO_ARRAY=(
|
||||
"https://github.com/Netflix/Hystrix.git"
|
||||
# "https://github.com/facebook/flow.git"
|
||||
# "https://github.com/Netflix/vizceral.git"
|
||||
# "https://github.com/Netflix/metaflow.git"
|
||||
# "https://github.com/Netflix/dgs-framework.git"
|
||||
# "https://github.com/Netflix/vector.git"
|
||||
# "https://github.com/expressjs/express.git"
|
||||
)
|
||||
REPOS=$(printf "%s," "${REPO_ARRAY[@]}" | cut -d "," -f 1-${#REPO_ARRAY[@]})
|
||||
go run hack/snifftest/main.go scan --exclude privatekey --exclude uri --exclude github_old --repo "$REPOS" --detector all --print --fail-threshold 99
|
|
@ -6,7 +6,6 @@ import (
|
|||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/abuseipdb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/accuweather"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/adafruitio"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/adobeio"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/adzuna"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/agora"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/airbrakeprojectkey"
|
||||
|
@ -33,7 +32,6 @@ import (
|
|||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/apify"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/apimatic"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/apiscience"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/apollo"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/appcues"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/appfollow"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/appsynergy"
|
||||
|
@ -145,7 +143,6 @@ import (
|
|||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/customerguru"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/customerio"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/d7network"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/dailyco"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/dandelion"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/datadogtoken"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/datafire"
|
||||
|
@ -320,7 +317,6 @@ import (
|
|||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/mailmodo"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/mailsac"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/mandrill"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/manifest"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/mapbox"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/mapquest"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/marketstack"
|
||||
|
@ -337,7 +333,6 @@ import (
|
|||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/mindmeister"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/mite"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/mixmax"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/mixpanel"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/moderation"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/monday"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/moonclerck"
|
||||
|
@ -495,7 +490,6 @@ import (
|
|||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/smooch"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/snipcart"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/snykkey"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/sparkpost"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/splunkobservabilitytoken"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/spoonacular"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/sportsmonk"
|
||||
|
@ -713,7 +707,7 @@ func DefaultDetectors() []detectors.Detector {
|
|||
¬ion.Scanner{},
|
||||
&droneci.Scanner{},
|
||||
&ipstack.Scanner{},
|
||||
&adobeio.Scanner{},
|
||||
// &adobeio.Scanner{},
|
||||
&sslmate.Scanner{},
|
||||
&buildkite.Scanner{},
|
||||
&shodankey.Scanner{},
|
||||
|
@ -819,7 +813,7 @@ func DefaultDetectors() []detectors.Detector {
|
|||
&clockify.Scanner{},
|
||||
&karmacrm.Scanner{},
|
||||
&revampcrm.Scanner{},
|
||||
&apollo.Scanner{},
|
||||
// &apollo.Scanner{},
|
||||
&artsy.Scanner{},
|
||||
&vpnapi.Scanner{},
|
||||
&dnscheck.Scanner{},
|
||||
|
@ -910,7 +904,7 @@ func DefaultDetectors() []detectors.Detector {
|
|||
spoonacular.Scanner{},
|
||||
finnhub.Scanner{},
|
||||
checkout.Scanner{},
|
||||
mixpanel.Scanner{},
|
||||
// mixpanel.Scanner{},
|
||||
ipgeolocation.Scanner{},
|
||||
tmetric.Scanner{},
|
||||
fullstory.Scanner{},
|
||||
|
@ -949,7 +943,7 @@ func DefaultDetectors() []detectors.Detector {
|
|||
zipcodeapi.Scanner{},
|
||||
gyazo.Scanner{},
|
||||
fakejson.Scanner{},
|
||||
sparkpost.Scanner{},
|
||||
// sparkpost.Scanner{},
|
||||
locationiq.Scanner{},
|
||||
saucelabs.Scanner{},
|
||||
enigma.Scanner{},
|
||||
|
@ -1079,7 +1073,7 @@ func DefaultDetectors() []detectors.Detector {
|
|||
edenai.Scanner{},
|
||||
urlscan.Scanner{},
|
||||
zenscrape.Scanner{},
|
||||
dailyco.Scanner{},
|
||||
// dailyco.Scanner{},
|
||||
nicereply.Scanner{},
|
||||
hive.Scanner{},
|
||||
clustdoc.Scanner{},
|
||||
|
@ -1178,7 +1172,7 @@ func DefaultDetectors() []detectors.Detector {
|
|||
gocanvas.Scanner{},
|
||||
formio.Scanner{},
|
||||
livestorm.Scanner{},
|
||||
manifest.Scanner{},
|
||||
// manifest.Scanner{},
|
||||
formbucket.Scanner{},
|
||||
apiscience.Scanner{},
|
||||
dronahq.Scanner{},
|
||||
|
|
Loading…
Reference in a new issue