Add Robinhood Crypto detector (#3254)

* Add Robinhood Crypto detector

* Address comment - use single keyword

pkg/detectors/robinhoodcrypto/robinhoodcrypto.go

@@ -0,0 +1,195 @@
+package robinhoodcrypto
+
+import (
+	"context"
+	"crypto/ed25519"
+	"encoding/base64"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"strings"
+	"time"
+
+	regexp "github.com/wasilibs/go-re2"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+type Scanner struct {
+	client *http.Client
+}
+
+// Ensure the Scanner satisfies the interface at compile time.
+var _ detectors.Detector = (*Scanner)(nil)
+
+var (
+	defaultClient = common.SaneHttpClient()
+
+	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
+	// Reference: https://docs.robinhood.com/crypto/trading/#section/Authentication
+	keyPat = regexp.MustCompile(`\b(rh-api-[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12})\b`)
+
+	// Matches base64 strings. Taken from https://stackoverflow.com/a/475217.
+	privKeyBase64Pat = regexp.MustCompile(`(?:[A-Za-z0-9+\/]{4})*(?:[A-Za-z0-9+\/]{2}==|[A-Za-z0-9+\/]{3}=)`)
+)
+
+// Keywords are used for efficiently pre-filtering chunks.
+// Use identifiers in the secret preferably, or the provider name.
+func (s Scanner) Keywords() []string {
+	return []string{"rh-api-"}
+}
+
+// FromData will find and optionally verify RobinhoodCrypto secrets in a given set of bytes.
+func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
+	dataStr := string(data)
+
+	apiKeyMatches := make(map[string]struct{})
+	for _, match := range keyPat.FindAllStringSubmatch(dataStr, -1) {
+		apiKeyMatches[match[1]] = struct{}{}
+	}
+
+	base64PrivateKeyMatches := make(map[string]struct{})
+	for _, match := range privKeyBase64Pat.FindAllString(dataStr, -1) {
+		base64PrivateKeyMatches[match] = struct{}{}
+	}
+
+	for apiKey := range apiKeyMatches {
+		for base64PrivateKey := range base64PrivateKeyMatches {
+			s1 := detectors.Result{
+				DetectorType: detectorspb.DetectorType_RobinhoodCrypto,
+				Raw:          []byte(apiKey),
+				RawV2:        []byte(apiKey + base64PrivateKey),
+			}
+
+			if verify {
+				client := s.client
+				if client == nil {
+					client = defaultClient
+				}
+
+				isVerified, extraData, verificationErr := verifyMatch(ctx, client, apiKey, base64PrivateKey)
+				s1.Verified = isVerified
+				s1.ExtraData = extraData
+				s1.SetVerificationError(verificationErr, apiKey, base64PrivateKey)
+			}
+
+			results = append(results, s1)
+		}
+	}
+
+	return
+}
+
+func verifyMatch(ctx context.Context, client *http.Client, apiKey, base64PrivateKey string) (
+	bool, map[string]string, error,
+) {
+	// Decode the base64 private key.
+	privateBytes, err := base64.StdEncoding.DecodeString(base64PrivateKey)
+	if err != nil {
+		return false, nil, fmt.Errorf("failed to decode base64 private key: %w", err)
+	}
+
+	// Sanity check the private key length.
+	if len(privateBytes) < 32 {
+		return false, nil, fmt.Errorf("private key is too short, expected at least 32 bytes, got %d", len(privateBytes))
+	}
+
+	// Create the private key from the seed.
+	privateKey := ed25519.NewKeyFromSeed(privateBytes[:32])
+
+	// Draft the message to be signed.
+	// Reference: https://docs.robinhood.com/crypto/trading/#section/Authentication/Headers-and-Signature
+	var (
+		timestamp = fmt.Sprint(time.Now().UTC().Unix())
+		path      = "/api/v1/crypto/trading/accounts/"
+		method    = http.MethodGet
+		body      = ""
+	)
+
+	message := apiKey + timestamp + path + method + body
+	signature := ed25519.Sign(privateKey, []byte(message))
+
+	req, err := http.NewRequestWithContext(ctx, method, "https://trading.robinhood.com/"+path, strings.NewReader(body))
+	if err != nil {
+		return false, nil, nil
+	}
+
+	// Set the required headers.
+	headers := map[string]string{
+		"x-api-key":   apiKey,
+		"x-signature": base64.StdEncoding.EncodeToString(signature),
+		"x-timestamp": timestamp,
+	}
+	for key, value := range headers {
+		req.Header.Add(key, value)
+	}
+
+	res, err := client.Do(req)
+	if err != nil {
+		return false, nil, err
+	}
+	defer func() {
+		_, _ = io.Copy(io.Discard, res.Body)
+		_ = res.Body.Close()
+	}()
+
+	switch res.StatusCode {
+	// StatusOK: The secret is verified.
+	case http.StatusOK:
+		// Include the additional information returned by the endpoint.
+		if len(res.Header) > 0 && res.Header.Get("Content-Type") == "application/json" {
+			response := struct {
+				AccountNumber       string `json:"account_number"`
+				Status              string `json:"status"`
+				BuyingPower         string `json:"buying_power"`
+				BuyingPowerCurrency string `json:"buying_power_currency"`
+			}{}
+
+			if err = json.NewDecoder(res.Body).Decode(&response); err != nil {
+				return true, nil, fmt.Errorf("failed to obtain additional information: %w", err)
+			}
+
+			return true, map[string]string{"Robinhood Crypto Account Number": response.AccountNumber}, nil
+		}
+
+		// The secret is verified, but there is no additional information.
+		return true, nil, nil
+
+	// StatusForbidden: The secret is valid, but the credentials do not have access to the endpoint.
+	case http.StatusForbidden:
+		return true, map[string]string{"Explanation": "Valid credentials without access to Get Crypto Trading Account Details API"}, nil
+
+	// StatusUnauthorized:
+	// Two scenarios can happen,
+	// 		1. The secret is verified, but is currently inactive.
+	// 		2. The secret is determinately not verified.
+	case http.StatusUnauthorized:
+		// Check if the secret is verified but currently inactive.
+		// We want to handle this case because an inactive secret can be activated in the future, at which point it
+		// becomes a security risk.
+		if len(res.Header) > 0 && res.Header.Get("Content-Type") == "text/plain" {
+			body, err := io.ReadAll(res.Body)
+			if err != nil {
+				// The secret is considered verified but inactive only if the body suggests so. Since the body is not
+				// readable, we cannot determine if the secret is verified but inactive.
+				return false, nil, fmt.Errorf("failed to read response body: %w", err)
+			}
+
+			if strings.TrimSpace(string(body)) == "API credential is not active." {
+				return true, map[string]string{"Explanation": "Valid credentials in inactive state"}, nil
+			}
+		}
+
+		// The secret is determinately not verified (nothing to do)
+		return false, nil, nil
+	default:
+		return false, nil, fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
+	}
+}
+
+func (s Scanner) Type() detectorspb.DetectorType {
+	return detectorspb.DetectorType_RobinhoodCrypto
+}

pkg/detectors/robinhoodcrypto/robinhoodcrypto_test.go

@@ -0,0 +1,274 @@
+//go:build detectors
+// +build detectors
+
+package robinhoodcrypto
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestRobinhoodCrypto_Pattern(t *testing.T) {
+	d := Scanner{}
+	ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
+	tests := []struct {
+		name  string
+		input string
+		want  []string
+	}{
+		{
+			name: "typical pattern",
+			input: `
+				api_key = "rh-api-e3bb245e-a45c-4729-8a9b-10201756f8cc"
+				private_key_base64 = "aVhXn8ghC9YqSz5RyFuKc6SsDC6SuPIqSW3IXH76ZlMCjOxkazBQjQFucJLk3uNorpBt6TbYpo/D1lHA7s4+hQ=="
+			`,
+			want: []string{
+				"rh-api-e3bb245e-a45c-4729-8a9b-10201756f8cc" +
+					"aVhXn8ghC9YqSz5RyFuKc6SsDC6SuPIqSW3IXH76ZlMCjOxkazBQjQFucJLk3uNorpBt6TbYpo/D1lHA7s4+hQ==",
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(
+			test.name, func(t *testing.T) {
+				matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+				if len(matchedDetectors) == 0 {
+					t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
+					return
+				}
+
+				results, err := d.FromData(context.Background(), false, []byte(test.input))
+				if err != nil {
+					t.Errorf("error = %v", err)
+					return
+				}
+
+				if len(results) != len(test.want) {
+					if len(results) == 0 {
+						t.Errorf("did not receive result")
+					} else {
+						t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+					}
+					return
+				}
+
+				actual := make(map[string]struct{}, len(results))
+				for _, r := range results {
+					if len(r.RawV2) > 0 {
+						actual[string(r.RawV2)] = struct{}{}
+					} else {
+						actual[string(r.Raw)] = struct{}{}
+					}
+				}
+				expected := make(map[string]struct{}, len(test.want))
+				for _, v := range test.want {
+					expected[v] = struct{}{}
+				}
+
+				if diff := cmp.Diff(expected, actual); diff != "" {
+					t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+				}
+			},
+		)
+	}
+}
+
+func TestRobinhoodcrypto_FromChunk(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	defer cancel()
+	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
+	if err != nil {
+		t.Fatalf("could not get test secrets from GCP: %s", err)
+	}
+
+	// Valid and active credentials.
+	apiKey := testSecrets.MustGetField("ROBINHOODCRYPTO_APIKEY")
+	privateKey := testSecrets.MustGetField("ROBINHOODCRYPTO_PRIVATEKEY")
+
+	// Valid but inactive credentials.
+	inactiveApiKey := testSecrets.MustGetField("ROBINHOODCRYPTO_APIKEY_INACTIVE")
+	inactivePrivateKey := testSecrets.MustGetField("ROBINHOODCRYPTO_PRIVATEKEY_INACTIVE")
+
+	// Invalid credentials.
+	deletedApiKey := testSecrets.MustGetField("ROBINHOODCRYPTO_APIKEY_DELETED")
+	deletedPrivateKey := testSecrets.MustGetField("ROBINHOODCRYPTO_PRIVATEKEY_DELETED")
+
+	type args struct {
+		ctx    context.Context
+		data   []byte
+		verify bool
+	}
+	tests := []struct {
+		name                string
+		s                   Scanner
+		args                args
+		want                []detectors.Result
+		wantErr             bool
+		wantVerificationErr bool
+	}{
+		{
+			name: "found, verified",
+			s:    Scanner{},
+			args: args{
+				ctx: context.Background(),
+				data: []byte(fmt.Sprintf(
+					"You can find a robinhoodcrypto api key %s and a private key %s within", apiKey, privateKey,
+				)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_RobinhoodCrypto,
+					Verified:     true,
+				},
+			},
+			wantErr:             false,
+			wantVerificationErr: false,
+		},
+		{
+			name: "found, verified, but inactive",
+			s:    Scanner{},
+			args: args{
+				ctx: context.Background(),
+				data: []byte(fmt.Sprintf(
+					"You can find a robinhoodcrypto api key %s and a private key %s within", inactiveApiKey,
+					inactivePrivateKey,
+				)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_RobinhoodCrypto,
+					Verified:     true,
+				},
+			},
+			wantErr:             false,
+			wantVerificationErr: false,
+		},
+		{
+			name: "found, unverified",
+			s:    Scanner{},
+			args: args{
+				ctx: context.Background(),
+				data: []byte(fmt.Sprintf(
+					"You can find a robinhoodcrypto api key %s and a private key %s within", deletedApiKey,
+					deletedPrivateKey,
+				)), // the secret would satisfy the regex but not pass validation
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_RobinhoodCrypto,
+					Verified:     false,
+				},
+			},
+			wantErr:             false,
+			wantVerificationErr: false,
+		},
+		{
+			name: "not found",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte("You cannot find the secret within"),
+				verify: true,
+			},
+			want:                nil,
+			wantErr:             false,
+			wantVerificationErr: false,
+		},
+		{
+			name: "found, would be verified if not for timeout",
+			s:    Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
+			args: args{
+				ctx: context.Background(),
+				data: []byte(fmt.Sprintf(
+					"You can find a robinhoodcrypto api key %s and a private key %s within", apiKey, privateKey,
+				)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_RobinhoodCrypto,
+					Verified:     false,
+				},
+			},
+			wantErr:             false,
+			wantVerificationErr: true,
+		},
+		{
+			name: "found, verified but unexpected api surface",
+			s:    Scanner{client: common.ConstantResponseHttpClient(404, "")},
+			args: args{
+				ctx: context.Background(),
+				data: []byte(fmt.Sprintf(
+					"You can find a robinhoodcrypto api key %s and a private key %s within", apiKey, privateKey,
+				)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_RobinhoodCrypto,
+					Verified:     false,
+				},
+			},
+			wantErr:             false,
+			wantVerificationErr: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(
+			tt.name, func(t *testing.T) {
+				got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+				if (err != nil) != tt.wantErr {
+					t.Errorf("Robinhoodcrypto.FromData() error = %v, wantErr %v", err, tt.wantErr)
+					return
+				}
+				for i := range got {
+					if len(got[i].Raw) == 0 {
+						t.Fatalf("no raw secret present: \n %+v", got[i])
+					}
+					if (got[i].VerificationError() != nil) != tt.wantVerificationErr {
+						t.Fatalf(
+							"wantVerificationError = %v, verification error = %v", tt.wantVerificationErr,
+							got[i].VerificationError(),
+						)
+					}
+				}
+				ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "ExtraData", "Raw", "RawV2", "verificationError")
+				if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
+					t.Errorf("Robinhoodcrypto.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+				}
+			},
+		)
+	}
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+	ctx := context.Background()
+	s := Scanner{}
+	for name, data := range detectors.MustGetBenchmarkData() {
+		benchmark.Run(
+			name, func(b *testing.B) {
+				b.ResetTimer()
+				for n := 0; n < b.N; n++ {
+					_, err := s.FromData(ctx, false, data)
+					if err != nil {
+						b.Fatal(err)
+					}
+				}
+			},
+		)
+	}
+}

pkg/engine/defaults.go

@@ -577,6 +577,7 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/ringcentral"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/ritekit"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/roaring"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/robinhoodcrypto"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/rocketreach"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/rockset"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/roninapp"
@@ -1626,6 +1627,7 @@ func DefaultDetectors() []detectors.Detector {
 		atlassianv1.Scanner{},
 		atlassianv2.Scanner{},
 		netsuite.Scanner{},
+		robinhoodcrypto.Scanner{},
 	}
 }
 

pkg/pb/detectorspb/detectors.pb.go

@@ -1097,6 +1097,7 @@ const (
 	DetectorType_EndorLabs                               DetectorType = 993
 	DetectorType_ElevenLabs                              DetectorType = 994
 	DetectorType_Netsuite                                DetectorType = 995
+	DetectorType_RobinhoodCrypto                         DetectorType = 996
 )
 
 // Enum value maps for DetectorType.
@@ -2094,6 +2095,7 @@ var (
 		993: "EndorLabs",
 		994: "ElevenLabs",
 		995: "Netsuite",
+		996: "RobinhoodCrypto",
 	}
 	DetectorType_value = map[string]int32{
 		"Alibaba":                               0,
@@ -3088,6 +3090,7 @@ var (
 		"EndorLabs":                        993,
 		"ElevenLabs":                       994,
 		"Netsuite":                         995,
+		"RobinhoodCrypto":                  996,
 	}
 )
 
@@ -3541,7 +3544,7 @@ var file_detectors_proto_rawDesc = []byte{
 	0x4c, 0x41, 0x49, 0x4e, 0x10, 0x01, 0x12, 0x0a, 0x0a, 0x06, 0x42, 0x41, 0x53, 0x45, 0x36, 0x34,
 	0x10, 0x02, 0x12, 0x09, 0x0a, 0x05, 0x55, 0x54, 0x46, 0x31, 0x36, 0x10, 0x03, 0x12, 0x13, 0x0a,
 	0x0f, 0x45, 0x53, 0x43, 0x41, 0x50, 0x45, 0x44, 0x5f, 0x55, 0x4e, 0x49, 0x43, 0x4f, 0x44, 0x45,
-	0x10, 0x04, 0x2a, 0x98, 0x7f, 0x0a, 0x0c, 0x44, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x54,
+	0x10, 0x04, 0x2a, 0xae, 0x7f, 0x0a, 0x0c, 0x44, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x54,
 	0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x41, 0x6c, 0x69, 0x62, 0x61, 0x62, 0x61, 0x10, 0x00,
 	0x12, 0x08, 0x0a, 0x04, 0x41, 0x4d, 0x51, 0x50, 0x10, 0x01, 0x12, 0x07, 0x0a, 0x03, 0x41, 0x57,
 	0x53, 0x10, 0x02, 0x12, 0x09, 0x0a, 0x05, 0x41, 0x7a, 0x75, 0x72, 0x65, 0x10, 0x03, 0x12, 0x0a,
@@ -4558,12 +4561,13 @@ var file_detectors_proto_rawDesc = []byte{
 	0x75, 0x69, 0x74, 0x65, 0x41, 0x70, 0x69, 0x4b, 0x65, 0x79, 0x10, 0xe0, 0x07, 0x12, 0x0e, 0x0a,
 	0x09, 0x45, 0x6e, 0x64, 0x6f, 0x72, 0x4c, 0x61, 0x62, 0x73, 0x10, 0xe1, 0x07, 0x12, 0x0f, 0x0a,
 	0x0a, 0x45, 0x6c, 0x65, 0x76, 0x65, 0x6e, 0x4c, 0x61, 0x62, 0x73, 0x10, 0xe2, 0x07, 0x12, 0x0d,
-	0x0a, 0x08, 0x4e, 0x65, 0x74, 0x73, 0x75, 0x69, 0x74, 0x65, 0x10, 0xe3, 0x07, 0x42, 0x3d, 0x5a,
-	0x3b, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66,
-	0x66, 0x6c, 0x65, 0x73, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x2f, 0x74, 0x72, 0x75, 0x66,
-	0x66, 0x6c, 0x65, 0x68, 0x6f, 0x67, 0x2f, 0x76, 0x33, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62,
-	0x2f, 0x64, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x73, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72,
-	0x6f, 0x74, 0x6f, 0x33,
+	0x0a, 0x08, 0x4e, 0x65, 0x74, 0x73, 0x75, 0x69, 0x74, 0x65, 0x10, 0xe3, 0x07, 0x12, 0x14, 0x0a,
+	0x0f, 0x52, 0x6f, 0x62, 0x69, 0x6e, 0x68, 0x6f, 0x6f, 0x64, 0x43, 0x72, 0x79, 0x70, 0x74, 0x6f,
+	0x10, 0xe4, 0x07, 0x42, 0x3d, 0x5a, 0x3b, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f,
+	0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x73, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74,
+	0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x68, 0x6f, 0x67, 0x2f, 0x76, 0x33, 0x2f,
+	0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x64, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x73,
+	0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
 }
 
 var (

proto/detectors.proto

@@ -1005,6 +1005,7 @@ enum DetectorType {
   EndorLabs = 993;
   ElevenLabs = 994;
   Netsuite = 995;
+  RobinhoodCrypto = 996;
 }
 
 message Result {