From a9817a3292d0f65e9357afda2c1078b7d84a553c Mon Sep 17 00:00:00 2001 From: Dustin Decker Date: Wed, 14 Feb 2024 15:27:03 -0800 Subject: [PATCH] Remove some noisy / less useful detectors (#2467) --- pkg/detectors/coinmarketcap/coinmarketcap.go | 78 ------------ .../coinmarketcap/coinmarketcap_test.go | 120 ------------------ pkg/detectors/etsyapikey/etsyapikey.go | 76 ----------- pkg/detectors/etsyapikey/etsyapikey_test.go | 120 ------------------ pkg/detectors/macaddress/macaddress.go | 75 ----------- pkg/detectors/macaddress/macaddress_test.go | 120 ------------------ pkg/detectors/nytimes/nytimes.go | 76 ----------- pkg/detectors/nytimes/nytimes_test.go | 120 ------------------ pkg/engine/defaults.go | 8 -- proto/detectors.proto | 8 +- 10 files changed, 4 insertions(+), 797 deletions(-) delete mode 100644 pkg/detectors/coinmarketcap/coinmarketcap.go delete mode 100644 pkg/detectors/coinmarketcap/coinmarketcap_test.go delete mode 100644 pkg/detectors/etsyapikey/etsyapikey.go delete mode 100644 pkg/detectors/etsyapikey/etsyapikey_test.go delete mode 100644 pkg/detectors/macaddress/macaddress.go delete mode 100644 pkg/detectors/macaddress/macaddress_test.go delete mode 100644 pkg/detectors/nytimes/nytimes.go delete mode 100644 pkg/detectors/nytimes/nytimes_test.go diff --git a/pkg/detectors/coinmarketcap/coinmarketcap.go b/pkg/detectors/coinmarketcap/coinmarketcap.go deleted file mode 100644 index b79744852..000000000 --- a/pkg/detectors/coinmarketcap/coinmarketcap.go +++ /dev/null @@ -1,78 +0,0 @@ -package coinmarketcap - -import ( - "context" - regexp "github.com/wasilibs/go-re2" - "net/http" - "strings" - - "github.com/trufflesecurity/trufflehog/v3/pkg/common" - "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" - "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" -) - -type Scanner struct{} - -// Ensure the Scanner satisfies the interface at compile time. -var _ detectors.Detector = (*Scanner)(nil) - -var ( - client = common.SaneHttpClient() - - // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. - keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"coinmarketcap", "CMC"}) + `\b([0-9Aa-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})\b`) -) - -// Keywords are used for efficiently pre-filtering chunks. -// Use identifiers in the secret preferably, or the provider name. -func (s Scanner) Keywords() []string { - return []string{"coinmarketcap", "CMC"} -} - -// FromData will find and optionally verify Coinmarketcap secrets in a given set of bytes. -func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) { - dataStr := string(data) - - matches := keyPat.FindAllStringSubmatch(dataStr, -1) - - for _, match := range matches { - if len(match) != 2 { - continue - } - resMatch := strings.TrimSpace(match[1]) - - s1 := detectors.Result{ - DetectorType: detectorspb.DetectorType_CoinMarketCap, - Raw: []byte(resMatch), - } - - if verify { - req, err := http.NewRequestWithContext(ctx, "GET", "https://pro-api.coinmarketcap.com/v1/cryptocurrency/listings/latest?start=1&limit=5000&convert=USD", nil) - if err != nil { - continue - } - req.Header.Add("Accept", "application/json") - req.Header.Add("X-CMC_PRO_API_KEY", resMatch) - res, err := client.Do(req) - if err == nil { - defer res.Body.Close() - if res.StatusCode >= 200 && res.StatusCode < 300 { - s1.Verified = true - } else { - // This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key. - if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) { - continue - } - } - } - } - - results = append(results, s1) - } - - return results, nil -} - -func (s Scanner) Type() detectorspb.DetectorType { - return detectorspb.DetectorType_CoinMarketCap -} diff --git a/pkg/detectors/coinmarketcap/coinmarketcap_test.go b/pkg/detectors/coinmarketcap/coinmarketcap_test.go deleted file mode 100644 index a456ed9bb..000000000 --- a/pkg/detectors/coinmarketcap/coinmarketcap_test.go +++ /dev/null @@ -1,120 +0,0 @@ -//go:build detectors -// +build detectors - -package coinmarketcap - -import ( - "context" - "fmt" - "testing" - "time" - - "github.com/kylelemons/godebug/pretty" - "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" - - "github.com/trufflesecurity/trufflehog/v3/pkg/common" - "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" -) - -func TestCoinmarketcap_FromChunk(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) - defer cancel() - testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4") - if err != nil { - t.Fatalf("could not get test secrets from GCP: %s", err) - } - secret := testSecrets.MustGetField("COINMARKETCAP") - inactiveSecret := testSecrets.MustGetField("COINMARKETCAP_INACTIVE") - - type args struct { - ctx context.Context - data []byte - verify bool - } - tests := []struct { - name string - s Scanner - args args - want []detectors.Result - wantErr bool - }{ - { - name: "found, verified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a coinmarketcap secret %s within", secret)), - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_CoinMarketCap, - Verified: true, - }, - }, - wantErr: false, - }, - { - name: "found, unverified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a coinmarketcap secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_CoinMarketCap, - Verified: false, - }, - }, - wantErr: false, - }, - { - name: "not found", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte("You cannot find the secret within"), - verify: true, - }, - want: nil, - wantErr: false, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - s := Scanner{} - got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) - if (err != nil) != tt.wantErr { - t.Errorf("Coinmarketcap.FromData() error = %v, wantErr %v", err, tt.wantErr) - return - } - for i := range got { - if len(got[i].Raw) == 0 { - t.Fatalf("no raw secret present: \n %+v", got[i]) - } - got[i].Raw = nil - } - if diff := pretty.Compare(got, tt.want); diff != "" { - t.Errorf("Coinmarketcap.FromData() %s diff: (-got +want)\n%s", tt.name, diff) - } - }) - } -} - -func BenchmarkFromData(benchmark *testing.B) { - ctx := context.Background() - s := Scanner{} - for name, data := range detectors.MustGetBenchmarkData() { - benchmark.Run(name, func(b *testing.B) { - b.ResetTimer() - for n := 0; n < b.N; n++ { - _, err := s.FromData(ctx, false, data) - if err != nil { - b.Fatal(err) - } - } - }) - } -} diff --git a/pkg/detectors/etsyapikey/etsyapikey.go b/pkg/detectors/etsyapikey/etsyapikey.go deleted file mode 100644 index e71d82b50..000000000 --- a/pkg/detectors/etsyapikey/etsyapikey.go +++ /dev/null @@ -1,76 +0,0 @@ -package etsyapikey - -import ( - "context" - regexp "github.com/wasilibs/go-re2" - "net/http" - "strings" - - "github.com/trufflesecurity/trufflehog/v3/pkg/common" - "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" - "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" -) - -type Scanner struct{} - -// Ensure the Scanner satisfies the interface at compile time. -var _ detectors.Detector = (*Scanner)(nil) - -var ( - client = common.SaneHttpClient() - - // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. - keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"etsy"}) + `\b([a-zA-Z-0-9]{24})\b`) -) - -// Keywords are used for efficiently pre-filtering chunks. -// Use identifiers in the secret preferably, or the provider name. -func (s Scanner) Keywords() []string { - return []string{"etsy"} -} - -// FromData will find and optionally verify EtsyApiKey secrets in a given set of bytes. -func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) { - dataStr := string(data) - - matches := keyPat.FindAllStringSubmatch(dataStr, -1) - - for _, match := range matches { - if len(match) != 2 { - continue - } - resMatch := strings.TrimSpace(match[1]) - - s1 := detectors.Result{ - DetectorType: detectorspb.DetectorType_EtsyApiKey, - Raw: []byte(resMatch), - } - - if verify { - req, err := http.NewRequestWithContext(ctx, "GET", "https://openapi.etsy.com/v2/listings/active?api_key="+resMatch, nil) - if err != nil { - continue - } - res, err := client.Do(req) - if err == nil { - defer res.Body.Close() - if res.StatusCode >= 200 && res.StatusCode < 300 { - s1.Verified = true - } else { - // This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key. - if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) { - continue - } - } - } - } - - results = append(results, s1) - } - - return results, nil -} - -func (s Scanner) Type() detectorspb.DetectorType { - return detectorspb.DetectorType_EtsyApiKey -} diff --git a/pkg/detectors/etsyapikey/etsyapikey_test.go b/pkg/detectors/etsyapikey/etsyapikey_test.go deleted file mode 100644 index 215149284..000000000 --- a/pkg/detectors/etsyapikey/etsyapikey_test.go +++ /dev/null @@ -1,120 +0,0 @@ -//go:build detectors -// +build detectors - -package etsyapikey - -import ( - "context" - "fmt" - "testing" - "time" - - "github.com/kylelemons/godebug/pretty" - "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" - - "github.com/trufflesecurity/trufflehog/v3/pkg/common" - "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" -) - -func TestEtsyApiKey_FromChunk(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) - defer cancel() - testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3") - if err != nil { - t.Fatalf("could not get test secrets from GCP: %s", err) - } - secret := testSecrets.MustGetField("ETSYAPIKEY_TOKEN") - inactiveSecret := testSecrets.MustGetField("ETSYAPIKEY_INACTIVE") - - type args struct { - ctx context.Context - data []byte - verify bool - } - tests := []struct { - name string - s Scanner - args args - want []detectors.Result - wantErr bool - }{ - { - name: "found, verified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a etsy secret %s within", secret)), - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_EtsyApiKey, - Verified: true, - }, - }, - wantErr: false, - }, - { - name: "found, unverified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a etsy secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_EtsyApiKey, - Verified: false, - }, - }, - wantErr: false, - }, - { - name: "not found", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte("You cannot find the secret within"), - verify: true, - }, - want: nil, - wantErr: false, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - s := Scanner{} - got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) - if (err != nil) != tt.wantErr { - t.Errorf("EtsyApiKey.FromData() error = %v, wantErr %v", err, tt.wantErr) - return - } - for i := range got { - if len(got[i].Raw) == 0 { - t.Fatalf("no raw secret present: \n %+v", got[i]) - } - got[i].Raw = nil - } - if diff := pretty.Compare(got, tt.want); diff != "" { - t.Errorf("EtsyApiKey.FromData() %s diff: (-got +want)\n%s", tt.name, diff) - } - }) - } -} - -func BenchmarkFromData(benchmark *testing.B) { - ctx := context.Background() - s := Scanner{} - for name, data := range detectors.MustGetBenchmarkData() { - benchmark.Run(name, func(b *testing.B) { - b.ResetTimer() - for n := 0; n < b.N; n++ { - _, err := s.FromData(ctx, false, data) - if err != nil { - b.Fatal(err) - } - } - }) - } -} diff --git a/pkg/detectors/macaddress/macaddress.go b/pkg/detectors/macaddress/macaddress.go deleted file mode 100644 index ba1b9f60b..000000000 --- a/pkg/detectors/macaddress/macaddress.go +++ /dev/null @@ -1,75 +0,0 @@ -package macaddress - -import ( - "context" - "fmt" - regexp "github.com/wasilibs/go-re2" - "net/http" - "strings" - - "github.com/trufflesecurity/trufflehog/v3/pkg/common" - "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" - "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" -) - -type Scanner struct{} - -// Ensure the Scanner satisfies the interface at compile time. -var _ detectors.Detector = (*Scanner)(nil) - -var ( - client = common.SaneHttpClient() - - // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. - keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"macaddress"}) + `\b([a-zA-Z0-9_]{32})\b`) -) - -// Keywords are used for efficiently pre-filtering chunks. -// Use identifiers in the secret preferably, or the provider name. -func (s Scanner) Keywords() []string { - return []string{"macaddress"} -} - -// FromData will find and optionally verify Macaddress secrets in a given set of bytes. -func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) { - dataStr := string(data) - - matches := keyPat.FindAllStringSubmatch(dataStr, -1) - for _, match := range matches { - if len(match) != 2 { - continue - } - resMatch := strings.TrimSpace(match[1]) - - s1 := detectors.Result{ - DetectorType: detectorspb.DetectorType_Macaddress, - Raw: []byte(resMatch), - } - - if verify { - req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://api.macaddress.io/v1?apiKey=%s&output=json&search=44:38:39:ff:ef:51", resMatch), nil) - if err != nil { - continue - } - res, err := client.Do(req) - if err == nil { - defer res.Body.Close() - if res.StatusCode >= 200 && res.StatusCode < 300 { - s1.Verified = true - } else { - // This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key. - if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) { - continue - } - } - } - } - - results = append(results, s1) - } - return results, nil -} - -func (s Scanner) Type() detectorspb.DetectorType { - return detectorspb.DetectorType_Macaddress -} diff --git a/pkg/detectors/macaddress/macaddress_test.go b/pkg/detectors/macaddress/macaddress_test.go deleted file mode 100644 index e1ee91a23..000000000 --- a/pkg/detectors/macaddress/macaddress_test.go +++ /dev/null @@ -1,120 +0,0 @@ -//go:build detectors -// +build detectors - -package macaddress - -import ( - "context" - "fmt" - "testing" - "time" - - "github.com/kylelemons/godebug/pretty" - "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" - - "github.com/trufflesecurity/trufflehog/v3/pkg/common" - "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" -) - -func TestMacaddress_FromChunk(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) - defer cancel() - testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1") - if err != nil { - t.Fatalf("could not get test secrets from GCP: %s", err) - } - secret := testSecrets.MustGetField("MACADDRESS") - inactiveSecret := testSecrets.MustGetField("MACADDRESS_INACTIVE") - - type args struct { - ctx context.Context - data []byte - verify bool - } - tests := []struct { - name string - s Scanner - args args - want []detectors.Result - wantErr bool - }{ - { - name: "found, verified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a macaddress secret %s within", secret)), - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_Macaddress, - Verified: true, - }, - }, - wantErr: false, - }, - { - name: "found, unverified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a macaddress secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_Macaddress, - Verified: false, - }, - }, - wantErr: false, - }, - { - name: "not found", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte("You cannot find the secret within"), - verify: true, - }, - want: nil, - wantErr: false, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - s := Scanner{} - got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) - if (err != nil) != tt.wantErr { - t.Errorf("Macaddress.FromData() error = %v, wantErr %v", err, tt.wantErr) - return - } - for i := range got { - if len(got[i].Raw) == 0 { - t.Fatalf("no raw secret present: \n %+v", got[i]) - } - got[i].Raw = nil - } - if diff := pretty.Compare(got, tt.want); diff != "" { - t.Errorf("Macaddress.FromData() %s diff: (-got +want)\n%s", tt.name, diff) - } - }) - } -} - -func BenchmarkFromData(benchmark *testing.B) { - ctx := context.Background() - s := Scanner{} - for name, data := range detectors.MustGetBenchmarkData() { - benchmark.Run(name, func(b *testing.B) { - b.ResetTimer() - for n := 0; n < b.N; n++ { - _, err := s.FromData(ctx, false, data) - if err != nil { - b.Fatal(err) - } - } - }) - } -} diff --git a/pkg/detectors/nytimes/nytimes.go b/pkg/detectors/nytimes/nytimes.go deleted file mode 100644 index b0398bc94..000000000 --- a/pkg/detectors/nytimes/nytimes.go +++ /dev/null @@ -1,76 +0,0 @@ -package nytimes - -import ( - "context" - regexp "github.com/wasilibs/go-re2" - "net/http" - "strings" - - "github.com/trufflesecurity/trufflehog/v3/pkg/common" - "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" - "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" -) - -type Scanner struct{} - -// Ensure the Scanner satisfies the interface at compile time. -var _ detectors.Detector = (*Scanner)(nil) - -var ( - client = common.SaneHttpClient() - - // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. - keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"nytimes"}) + `\b([a-z0-9A-Z-]{32})\b`) -) - -// Keywords are used for efficiently pre-filtering chunks. -// Use identifiers in the secret preferably, or the provider name. -func (s Scanner) Keywords() []string { - return []string{"nytimes"} -} - -// FromData will find and optionally verify Nytimes secrets in a given set of bytes. -func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) { - dataStr := string(data) - - matches := keyPat.FindAllStringSubmatch(dataStr, -1) - - for _, match := range matches { - if len(match) != 2 { - continue - } - resMatch := strings.TrimSpace(match[1]) - - s1 := detectors.Result{ - DetectorType: detectorspb.DetectorType_Nytimes, - Raw: []byte(resMatch), - } - - if verify { - req, err := http.NewRequestWithContext(ctx, "GET", "https://api.nytimes.com/svc/archive/v1/2019/1.json?api-key="+resMatch, nil) - if err != nil { - continue - } - res, err := client.Do(req) - if err == nil { - defer res.Body.Close() - if res.StatusCode >= 200 && res.StatusCode < 300 { - s1.Verified = true - } else { - // This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key. - if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) { - continue - } - } - } - } - - results = append(results, s1) - } - - return results, nil -} - -func (s Scanner) Type() detectorspb.DetectorType { - return detectorspb.DetectorType_Nytimes -} diff --git a/pkg/detectors/nytimes/nytimes_test.go b/pkg/detectors/nytimes/nytimes_test.go deleted file mode 100644 index 29c9f5c9b..000000000 --- a/pkg/detectors/nytimes/nytimes_test.go +++ /dev/null @@ -1,120 +0,0 @@ -//go:build detectors -// +build detectors - -package nytimes - -import ( - "context" - "fmt" - "testing" - "time" - - "github.com/kylelemons/godebug/pretty" - "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" - - "github.com/trufflesecurity/trufflehog/v3/pkg/common" - "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" -) - -func TestNytimes_FromChunk(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) - defer cancel() - testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3") - if err != nil { - t.Fatalf("could not get test secrets from GCP: %s", err) - } - secret := testSecrets.MustGetField("NYTIMES_TOKEN") - inactiveSecret := testSecrets.MustGetField("NYTIMES_INACTIVE") - - type args struct { - ctx context.Context - data []byte - verify bool - } - tests := []struct { - name string - s Scanner - args args - want []detectors.Result - wantErr bool - }{ - { - name: "found, verified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a nytimes secret %s within", secret)), - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_Nytimes, - Verified: true, - }, - }, - wantErr: false, - }, - { - name: "found, unverified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a nytimes secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_Nytimes, - Verified: false, - }, - }, - wantErr: false, - }, - { - name: "not found", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte("You cannot find the secret within"), - verify: true, - }, - want: nil, - wantErr: false, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - s := Scanner{} - got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) - if (err != nil) != tt.wantErr { - t.Errorf("Nytimes.FromData() error = %v, wantErr %v", err, tt.wantErr) - return - } - for i := range got { - if len(got[i].Raw) == 0 { - t.Fatalf("no raw secret present: \n %+v", got[i]) - } - got[i].Raw = nil - } - if diff := pretty.Compare(got, tt.want); diff != "" { - t.Errorf("Nytimes.FromData() %s diff: (-got +want)\n%s", tt.name, diff) - } - }) - } -} - -func BenchmarkFromData(benchmark *testing.B) { - ctx := context.Background() - s := Scanner{} - for name, data := range detectors.MustGetBenchmarkData() { - benchmark.Run(name, func(b *testing.B) { - b.ResetTimer() - for n := 0; n < b.N; n++ { - _, err := s.FromData(ctx, false, data) - if err != nil { - b.Fatal(err) - } - } - }) - } -} diff --git a/pkg/engine/defaults.go b/pkg/engine/defaults.go index 48b02f46f..bcb7b848c 100644 --- a/pkg/engine/defaults.go +++ b/pkg/engine/defaults.go @@ -157,7 +157,6 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/coinbase_waas" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/coinlayer" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/coinlib" - "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/coinmarketcap" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/collect2" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/column" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/commercejs" @@ -236,7 +235,6 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/envoyapikey" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/etherscan" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/ethplorer" - "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/etsyapikey" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/eventbrite" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/everhour" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/exchangerateapi" @@ -401,7 +399,6 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/loyverse" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/lunchmoney" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/luno" - "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/macaddress" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/madkudu" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/magicbell" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/magnetic" @@ -469,7 +466,6 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/numverify" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/nutritionix" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/nylas" - "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/nytimes" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/oanda" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/okta" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/omnisend" @@ -883,7 +879,6 @@ func DefaultDetectors() []detectors.Detector { &coinbase.Scanner{}, &confluent.Scanner{}, &zendeskapi.Scanner{}, - &etsyapikey.Scanner{}, &facebookoauth.Scanner{}, &litudeapikey.Scanner{}, &pubnubpublishkey.Scanner{}, @@ -963,7 +958,6 @@ func DefaultDetectors() []detectors.Detector { &delighted.Scanner{}, &abbysale.Scanner{}, &feedier.Scanner{}, - &nytimes.Scanner{}, &powrbot.Scanner{}, &magnetic.Scanner{}, &polygon.Scanner{}, @@ -1171,7 +1165,6 @@ func DefaultDetectors() []detectors.Detector { pixabay.Scanner{}, ipify.Scanner{}, youneedabudget.Scanner{}, - macaddress.Scanner{}, languagelayer.Scanner{}, gengo.Scanner{}, aylien.Scanner{}, @@ -1546,7 +1539,6 @@ func DefaultDetectors() []detectors.Detector { blocknative.Scanner{}, moralis.Scanner{}, bscscan.Scanner{}, - coinmarketcap.Scanner{}, percy.Scanner{}, tineswebhook.Scanner{}, pulumi.Scanner{}, diff --git a/proto/detectors.proto b/proto/detectors.proto index 57e366162..b8b6d2c3c 100644 --- a/proto/detectors.proto +++ b/proto/detectors.proto @@ -77,7 +77,7 @@ enum DetectorType { DigitalOceanToken = 64; DiscordBotToken = 65; DiscordWebhook = 66; - EtsyApiKey = 67; + EtsyApiKey = 67 [deprecated = true]; FastlyPersonalToken = 68; GoogleOauth2 = 69; ReCAPTCHA = 70; // Didn't do @@ -250,7 +250,7 @@ enum DetectorType { Feedier = 238; Abbysale = 239; Magnetic = 240; - Nytimes = 241; + Nytimes = 241 [deprecated = true]; Polygon = 242; Powrbot = 243; ProspectIO = 244 [deprecated = true]; @@ -417,7 +417,7 @@ enum DetectorType { FinancialModelingPrep = 406; Geocodio = 407; HereAPI = 408; - Macaddress = 409; + Macaddress = 409 [deprecated = true]; OOPSpam = 410; ProtocolsIO = 411; ScraperAPI = 412; @@ -919,7 +919,7 @@ enum DetectorType { BlockNative = 908; Moralis = 909; BscScan = 910; - CoinMarketCap = 911; + CoinMarketCap = 911 [deprecated = true]; Percy = 912; TinesWebhook = 913; Pulumi = 914;