mirror of
https://github.com/trufflesecurity/trufflehog.git
synced 2024-09-20 06:31:57 +00:00
Record avg detector time
This commit is contained in:
Dustin Decker
parent
270d83c3d7
commit
ee99f0c9a2
4 changed files with 61 additions and 25 deletions
6
go.mod
6
go.mod
|
|
@ -13,10 +13,11 @@ require (
|
|||
github.com/dgrijalva/jwt-go v3.2.0+incompatible
|
||||
github.com/envoyproxy/protoc-gen-validate v0.6.3
|
||||
github.com/fatih/color v1.13.0
|
||||
github.com/go-echarts/statsview v0.3.4
|
||||
github.com/felixge/fgprof v0.9.2
|
||||
github.com/go-errors/errors v1.4.2
|
||||
github.com/go-git/go-git/v5 v5.4.2
|
||||
github.com/google/go-github/v42 v42.0.0
|
||||
github.com/gorilla/mux v1.8.0
|
||||
github.com/h2non/filetype v1.1.3
|
||||
github.com/hashicorp/go-retryablehttp v0.7.0
|
||||
github.com/joho/godotenv v1.4.0
|
||||
|
|
@ -59,7 +60,6 @@ require (
|
|||
github.com/aws/smithy-go v1.10.0 // indirect
|
||||
github.com/dimchansky/utfbom v1.1.1 // indirect
|
||||
github.com/emirpasic/gods v1.12.0 // indirect
|
||||
github.com/go-echarts/go-echarts/v2 v2.2.3 // indirect
|
||||
github.com/go-git/gcfg v1.5.0 // indirect
|
||||
github.com/go-git/go-billy/v5 v5.3.1 // indirect
|
||||
github.com/golang-jwt/jwt/v4 v4.2.0 // indirect
|
||||
|
|
@ -68,6 +68,7 @@ require (
|
|||
github.com/google/go-cmp v0.5.6 // indirect
|
||||
github.com/google/go-github/v29 v29.0.2 // indirect
|
||||
github.com/google/go-querystring v1.1.0 // indirect
|
||||
github.com/google/pprof v0.0.0-20211214055906-6f57359322fd // indirect
|
||||
github.com/googleapis/gax-go/v2 v2.1.1 // indirect
|
||||
github.com/hashicorp/go-cleanhttp v0.5.1 // indirect
|
||||
github.com/imdario/mergo v0.3.12 // indirect
|
||||
|
|
@ -80,7 +81,6 @@ require (
|
|||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
|
||||
github.com/modern-go/reflect2 v1.0.1 // indirect
|
||||
github.com/pkg/diff v0.0.0-20200914180035-5b29258ca4f7 // indirect
|
||||
github.com/rs/cors v1.7.0 // indirect
|
||||
github.com/xanzy/ssh-agent v0.3.0 // indirect
|
||||
go.opencensus.io v0.23.0 // indirect
|
||||
golang.org/x/mod v0.5.0 // indirect
|
||||
|
|
|
|||
15
go.sum
15
go.sum
|
|
@ -145,14 +145,12 @@ github.com/envoyproxy/protoc-gen-validate v0.6.3 h1:HkntewfZJ9RofA/FX38zBCeIAqlL
|
|||
github.com/envoyproxy/protoc-gen-validate v0.6.3/go.mod h1:dyJXwwfPK2VSqiB9Klm1J6romD608Ba7Hij42vrOBCo=
|
||||
github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w=
|
||||
github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk=
|
||||
github.com/felixge/fgprof v0.9.2 h1:tAMHtWMyl6E0BimjVbFt7fieU6FpjttsZN7j0wT5blc=
|
||||
github.com/felixge/fgprof v0.9.2/go.mod h1:+VNi+ZXtHIQ6wIw6bUT8nXQRefQflWECoFyRealT5sg=
|
||||
github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
|
||||
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
|
||||
github.com/gliderlabs/ssh v0.2.2 h1:6zsha5zo/TWhRhwqCD3+EarCAgZ2yN28ipRnGPnwkI0=
|
||||
github.com/gliderlabs/ssh v0.2.2/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0=
|
||||
github.com/go-echarts/go-echarts/v2 v2.2.3 h1:H8oPdUpzuiV2K8S4xYZa1JRNjP3U0h7HVqvhPrmCk1A=
|
||||
github.com/go-echarts/go-echarts/v2 v2.2.3/go.mod h1:6TOomEztzGDVDkOSCFBq3ed7xOYfbOqhaBzD0YV771A=
|
||||
github.com/go-echarts/statsview v0.3.4 h1:CCuytRAutdnF901NrR4BzSjHXjUp8OyA3/iopgG/1/Y=
|
||||
github.com/go-echarts/statsview v0.3.4/go.mod h1:AehKjL9cTFMeIo5QdV8sQO43vFmfY65X5GMWa3XMciY=
|
||||
github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA=
|
||||
github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og=
|
||||
github.com/go-git/gcfg v1.5.0 h1:Q5ViNfGF8zFgyJWPqYwA7qGFoMTEiBmdlkcfRmpIMa4=
|
||||
|
|
@ -248,6 +246,8 @@ github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLe
|
|||
github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
|
||||
github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
|
||||
github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
|
||||
github.com/google/pprof v0.0.0-20211214055906-6f57359322fd h1:1FjCyPC+syAzJ5/2S8fqdZK1R22vvA0J7JZKcuOIQ7Y=
|
||||
github.com/google/pprof v0.0.0-20211214055906-6f57359322fd/go.mod h1:KgnwoLYCZ8IQu3XUZ8Nc/bM9CCZFOyjUNOSygVozoDg=
|
||||
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
|
||||
github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
|
||||
|
|
@ -255,6 +255,8 @@ github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5m
|
|||
github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0=
|
||||
github.com/googleapis/gax-go/v2 v2.1.1 h1:dp3bWCh+PPO1zjRRiCSczJav13sBvG4UhNyVTa1KqdU=
|
||||
github.com/googleapis/gax-go/v2 v2.1.1/go.mod h1:hddJymUZASv3XPyGkUpKj8pPO47Rmb0eJc8R6ouapiM=
|
||||
github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
|
||||
github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
|
||||
github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
|
||||
github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
|
||||
github.com/h2non/filetype v1.1.3 h1:FKkx9QbD7HR/zjK1Ia5XiBsq9zdLi5Kf3zGyFTAFkGg=
|
||||
|
|
@ -271,6 +273,7 @@ github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ
|
|||
github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho=
|
||||
github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
|
||||
github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
|
||||
github.com/ianlancetaylor/demangle v0.0.0-20210905161508-09a460cdf81d/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w=
|
||||
github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU=
|
||||
github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
|
||||
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A=
|
||||
|
|
@ -331,8 +334,6 @@ github.com/razorpay/razorpay-go v0.0.0-20210728161131-0341409a6ab2 h1:8XGvK6qfvE
|
|||
github.com/razorpay/razorpay-go v0.0.0-20210728161131-0341409a6ab2/go.mod h1:VcljkUylUJAUEvFfGVv/d5ht1to1dUgF4H1+3nv7i+Q=
|
||||
github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
|
||||
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
|
||||
github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik=
|
||||
github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU=
|
||||
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
|
||||
github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
|
||||
github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ=
|
||||
|
|
@ -348,7 +349,6 @@ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
|
|||
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
|
||||
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
|
||||
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
|
||||
github.com/stretchr/testify v1.6.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
|
||||
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
|
|
@ -545,6 +545,7 @@ golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912/go.mod h1:oPkhp1MJrh7nUepCBc
|
|||
golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220114195835-da31bd327af9 h1:XfKQ4OlFl8okEOr5UvAqFRVj8pY/4yfcXrddB8qAbU0=
|
||||
|
|
|
|||
37
main.go
37
main.go
|
|
@ -5,13 +5,17 @@ import (
|
|||
"encoding/json"
|
||||
"fmt"
|
||||
"log"
|
||||
"net/http"
|
||||
_ "net/http/pprof"
|
||||
"os"
|
||||
"runtime"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/felixge/fgprof"
|
||||
|
||||
"github.com/go-echarts/statsview"
|
||||
"github.com/go-echarts/statsview/viewer"
|
||||
"github.com/go-git/go-git/v5/plumbing"
|
||||
"github.com/gorilla/mux"
|
||||
"github.com/sirupsen/logrus"
|
||||
kingpin "gopkg.in/alecthomas/kingpin.v2"
|
||||
|
||||
|
|
@ -31,6 +35,7 @@ func main() {
|
|||
noVerification := cli.Flag("no-verification", "Don't verify the results.").Bool()
|
||||
onlyVerified := cli.Flag("only-verified", "Only output verified results.").Bool()
|
||||
// rules := cli.Flag("rules", "Path to file with custom rules.").String()
|
||||
printAvgDetectorTime := cli.Flag("print-avg-detector-time", "Print the average time spent on each detector.").Bool()
|
||||
|
||||
gitScan := cli.Command("git", "Find credentials in git repositories.")
|
||||
gitScanURI := gitScan.Arg("uri", "Git repository URL. https:// or file:// schema expected.").Required().String()
|
||||
|
|
@ -79,11 +84,13 @@ func main() {
|
|||
|
||||
if *debug {
|
||||
go func() {
|
||||
viewer.SetConfiguration(viewer.WithAddr(":18066"))
|
||||
viewer.SetConfiguration(viewer.WithLinkAddr("localhost:18066"))
|
||||
mgr := statsview.New()
|
||||
logrus.Info("starting pprof and metrics server on http://localhost:18066/debug/pprof and http://localhost:18066/debug/statsview")
|
||||
mgr.Start()
|
||||
router := mux.NewRouter()
|
||||
router.PathPrefix("/debug/pprof").Handler(http.DefaultServeMux)
|
||||
router.PathPrefix("/debug/fgprof").Handler(fgprof.Handler())
|
||||
if err := http.ListenAndServe(":18066", router); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
logrus.Info("starting pprof and fgprof server on :18066 /debug/pprof and /debug/fgprof")
|
||||
}()
|
||||
}
|
||||
|
||||
|
|
@ -161,4 +168,20 @@ func main() {
|
|||
}
|
||||
}
|
||||
logrus.Debugf("scanned %d chunks", e.ChunksScanned())
|
||||
|
||||
if *printAvgDetectorTime {
|
||||
printAverageDetectorTime(e)
|
||||
}
|
||||
}
|
||||
|
||||
func printAverageDetectorTime(e *engine.Engine) {
|
||||
fmt.Fprintln(os.Stderr, "Average detector time is the measurement of average time spent on each detector when results are returned.")
|
||||
for detectorName, durations := range e.DetectorAvgTime() {
|
||||
var total time.Duration
|
||||
for _, d := range durations {
|
||||
total += d
|
||||
}
|
||||
avgDuration := total / time.Duration(len(durations))
|
||||
fmt.Fprintf(os.Stderr, "%s: %s\n", detectorName, avgDuration)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -16,12 +16,13 @@ import (
|
|||
)
|
||||
|
||||
type Engine struct {
|
||||
concurrency int
|
||||
chunks chan *sources.Chunk
|
||||
results chan detectors.ResultWithMetadata
|
||||
decoders []decoders.Decoder
|
||||
detectors map[bool][]detectors.Detector
|
||||
chunksScanned uint64
|
||||
concurrency int
|
||||
chunks chan *sources.Chunk
|
||||
results chan detectors.ResultWithMetadata
|
||||
decoders []decoders.Decoder
|
||||
detectors map[bool][]detectors.Detector
|
||||
chunksScanned uint64
|
||||
detectorAvgTime map[string][]time.Duration
|
||||
}
|
||||
|
||||
type EngineOption func(*Engine)
|
||||
|
|
@ -53,8 +54,9 @@ func WithDecoders(decoders ...decoders.Decoder) EngineOption {
|
|||
|
||||
func Start(ctx context.Context, options ...EngineOption) *Engine {
|
||||
e := &Engine{
|
||||
chunks: make(chan *sources.Chunk),
|
||||
results: make(chan detectors.ResultWithMetadata),
|
||||
chunks: make(chan *sources.Chunk),
|
||||
results: make(chan detectors.ResultWithMetadata),
|
||||
detectorAvgTime: map[string][]time.Duration{},
|
||||
}
|
||||
|
||||
for _, option := range options {
|
||||
|
|
@ -120,6 +122,10 @@ func (e *Engine) ChunksScanned() uint64 {
|
|||
return e.chunksScanned
|
||||
}
|
||||
|
||||
func (e *Engine) DetectorAvgTime() map[string][]time.Duration {
|
||||
return e.detectorAvgTime
|
||||
}
|
||||
|
||||
func (e *Engine) detectorWorker(ctx context.Context) {
|
||||
for chunk := range e.chunks {
|
||||
for _, decoder := range e.decoders {
|
||||
|
|
@ -130,6 +136,7 @@ func (e *Engine) detectorWorker(ctx context.Context) {
|
|||
dataLower := strings.ToLower(string(decoded.Data))
|
||||
for verify, detectorsSet := range e.detectors {
|
||||
for _, detector := range detectorsSet {
|
||||
start := time.Now()
|
||||
foundKeyword := false
|
||||
for _, kw := range detector.Keywords() {
|
||||
if strings.Contains(dataLower, strings.ToLower(kw)) {
|
||||
|
|
@ -148,6 +155,11 @@ func (e *Engine) detectorWorker(ctx context.Context) {
|
|||
for _, result := range results {
|
||||
e.results <- detectors.CopyMetadata(chunk, result)
|
||||
}
|
||||
if len(results) > 0 {
|
||||
elasped := time.Since(start)
|
||||
detectorName := results[0].DetectorType.String()
|
||||
e.detectorAvgTime[detectorName] = append(e.detectorAvgTime[detectorName], elasped)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue