Mirrors/trufflehog
2
0
Fork 0
mirror of https://github.com/trufflesecurity/trufflehog.git synced 2024-11-14 08:57:40 +00:00
Code Issues Projects Releases Packages Wiki Activity

Merge branch 'main' into add-flexport-detector

Browse source
This commit is contained in:
ahmed 2024-08-30 14:55:51 -04:00
commit 2744a5ffc9
9 changed files with 529 additions and 88 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

195
pkg/detectors/robinhoodcrypto/robinhoodcrypto.go Normal file
View file

@ -0,0 +1,195 @@
package robinhoodcrypto
import (
"context"
"crypto/ed25519"
"encoding/base64"
"encoding/json"
"fmt"
"io"
"net/http"
"strings"
"time"
regexp "github.com/wasilibs/go-re2"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
type Scanner struct {
client *http.Client
}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var (
defaultClient = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
// Reference: https://docs.robinhood.com/crypto/trading/#section/Authentication
keyPat = regexp.MustCompile(`\b(rh-api-[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12})\b`)
// Matches base64 strings. Taken from https://stackoverflow.com/a/475217.
privKeyBase64Pat = regexp.MustCompile(`(?:[A-Za-z0-9+\/]{4})*(?:[A-Za-z0-9+\/]{2}==|[A-Za-z0-9+\/]{3}=)`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"rh-api-"}
}
// FromData will find and optionally verify RobinhoodCrypto secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
apiKeyMatches := make(map[string]struct{})
for _, match := range keyPat.FindAllStringSubmatch(dataStr, -1) {
apiKeyMatches[match[1]] = struct{}{}
}
base64PrivateKeyMatches := make(map[string]struct{})
for _, match := range privKeyBase64Pat.FindAllString(dataStr, -1) {
base64PrivateKeyMatches[match] = struct{}{}
}
for apiKey := range apiKeyMatches {
for base64PrivateKey := range base64PrivateKeyMatches {
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_RobinhoodCrypto,
Raw: []byte(apiKey),
RawV2: []byte(apiKey + base64PrivateKey),
}
if verify {
client := s.client
if client == nil {
client = defaultClient
}
isVerified, extraData, verificationErr := verifyMatch(ctx, client, apiKey, base64PrivateKey)
s1.Verified = isVerified
s1.ExtraData = extraData
s1.SetVerificationError(verificationErr, apiKey, base64PrivateKey)
}
results = append(results, s1)
}
}
return
}
func verifyMatch(ctx context.Context, client *http.Client, apiKey, base64PrivateKey string) (
bool, map[string]string, error,
) {
// Decode the base64 private key.
privateBytes, err := base64.StdEncoding.DecodeString(base64PrivateKey)
if err != nil {
return false, nil, fmt.Errorf("failed to decode base64 private key: %w", err)
}
// Sanity check the private key length.
if len(privateBytes) < 32 {
return false, nil, fmt.Errorf("private key is too short, expected at least 32 bytes, got %d", len(privateBytes))
}
// Create the private key from the seed.
privateKey := ed25519.NewKeyFromSeed(privateBytes[:32])
// Draft the message to be signed.
// Reference: https://docs.robinhood.com/crypto/trading/#section/Authentication/Headers-and-Signature
var (
timestamp = fmt.Sprint(time.Now().UTC().Unix())
path = "/api/v1/crypto/trading/accounts/"
method = http.MethodGet
body = ""
)
message := apiKey + timestamp + path + method + body
signature := ed25519.Sign(privateKey, []byte(message))
req, err := http.NewRequestWithContext(ctx, method, "https://trading.robinhood.com/"+path, strings.NewReader(body))
if err != nil {
return false, nil, nil
}
// Set the required headers.
headers := map[string]string{
"x-api-key": apiKey,
"x-signature": base64.StdEncoding.EncodeToString(signature),
"x-timestamp": timestamp,
}
for key, value := range headers {
req.Header.Add(key, value)
}
res, err := client.Do(req)
if err != nil {
return false, nil, err
}
defer func() {
_, _ = io.Copy(io.Discard, res.Body)
_ = res.Body.Close()
}()
switch res.StatusCode {
// StatusOK: The secret is verified.
case http.StatusOK:
// Include the additional information returned by the endpoint.
if len(res.Header) > 0 && res.Header.Get("Content-Type") == "application/json" {
response := struct {
AccountNumber string `json:"account_number"`
Status string `json:"status"`
BuyingPower string `json:"buying_power"`
BuyingPowerCurrency string `json:"buying_power_currency"`
}{}
if err = json.NewDecoder(res.Body).Decode(&response); err != nil {
return true, nil, fmt.Errorf("failed to obtain additional information: %w", err)
}
return true, map[string]string{"Robinhood Crypto Account Number": response.AccountNumber}, nil
}
// The secret is verified, but there is no additional information.
return true, nil, nil
// StatusForbidden: The secret is valid, but the credentials do not have access to the endpoint.
case http.StatusForbidden:
return true, map[string]string{"Explanation": "Valid credentials without access to Get Crypto Trading Account Details API"}, nil
// StatusUnauthorized:
// Two scenarios can happen,
// 1. The secret is verified, but is currently inactive.
// 2. The secret is determinately not verified.
case http.StatusUnauthorized:
// Check if the secret is verified but currently inactive.
// We want to handle this case because an inactive secret can be activated in the future, at which point it
// becomes a security risk.
if len(res.Header) > 0 && res.Header.Get("Content-Type") == "text/plain" {
body, err := io.ReadAll(res.Body)
if err != nil {
// The secret is considered verified but inactive only if the body suggests so. Since the body is not
// readable, we cannot determine if the secret is verified but inactive.
return false, nil, fmt.Errorf("failed to read response body: %w", err)
}
if strings.TrimSpace(string(body)) == "API credential is not active." {
return true, map[string]string{"Explanation": "Valid credentials in inactive state"}, nil
}
}
// The secret is determinately not verified (nothing to do)
return false, nil, nil
default:
return false, nil, fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
}
}
func (s Scanner) Type() detectorspb.DetectorType {
return detectorspb.DetectorType_RobinhoodCrypto
}

274
pkg/detectors/robinhoodcrypto/robinhoodcrypto_test.go Normal file
View file

@ -0,0 +1,274 @@
//go:build detectors
// +build detectors
package robinhoodcrypto
import (
"context"
"fmt"
"testing"
"time"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
func TestRobinhoodCrypto_Pattern(t *testing.T) {
d := Scanner{}
ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
name string
input string
want []string
}{
{
name: "typical pattern",
input: `
api_key = "rh-api-e3bb245e-a45c-4729-8a9b-10201756f8cc"
private_key_base64 = "aVhXn8ghC9YqSz5RyFuKc6SsDC6SuPIqSW3IXH76ZlMCjOxkazBQjQFucJLk3uNorpBt6TbYpo/D1lHA7s4+hQ=="
`,
want: []string{
"rh-api-e3bb245e-a45c-4729-8a9b-10201756f8cc" +
"aVhXn8ghC9YqSz5RyFuKc6SsDC6SuPIqSW3IXH76ZlMCjOxkazBQjQFucJLk3uNorpBt6TbYpo/D1lHA7s4+hQ==",
},
},
}
for _, test := range tests {
t.Run(
test.name, func(t *testing.T) {
matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
if len(matchedDetectors) == 0 {
t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
results, err := d.FromData(context.Background(), false, []byte(test.input))
if err != nil {
t.Errorf("error = %v", err)
return
}
if len(results) != len(test.want) {
if len(results) == 0 {
t.Errorf("did not receive result")
} else {
t.Errorf("expected %d results, only received %d", len(test.want), len(results))
}
return
}
actual := make(map[string]struct{}, len(results))
for _, r := range results {
if len(r.RawV2) > 0 {
actual[string(r.RawV2)] = struct{}{}
} else {
actual[string(r.Raw)] = struct{}{}
}
}
expected := make(map[string]struct{}, len(test.want))
for _, v := range test.want {
expected[v] = struct{}{}
}
if diff := cmp.Diff(expected, actual); diff != "" {
t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
}
},
)
}
}
func TestRobinhoodcrypto_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
// Valid and active credentials.
apiKey := testSecrets.MustGetField("ROBINHOODCRYPTO_APIKEY")
privateKey := testSecrets.MustGetField("ROBINHOODCRYPTO_PRIVATEKEY")
// Valid but inactive credentials.
inactiveApiKey := testSecrets.MustGetField("ROBINHOODCRYPTO_APIKEY_INACTIVE")
inactivePrivateKey := testSecrets.MustGetField("ROBINHOODCRYPTO_PRIVATEKEY_INACTIVE")
// Invalid credentials.
deletedApiKey := testSecrets.MustGetField("ROBINHOODCRYPTO_APIKEY_DELETED")
deletedPrivateKey := testSecrets.MustGetField("ROBINHOODCRYPTO_PRIVATEKEY_DELETED")
type args struct {
ctx context.Context
data []byte
verify bool
}
tests := []struct {
name string
s Scanner
args args
want []detectors.Result
wantErr bool
wantVerificationErr bool
}{
{
name: "found, verified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf(
"You can find a robinhoodcrypto api key %s and a private key %s within", apiKey, privateKey,
)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_RobinhoodCrypto,
Verified: true,
},
},
wantErr: false,
wantVerificationErr: false,
},
{
name: "found, verified, but inactive",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf(
"You can find a robinhoodcrypto api key %s and a private key %s within", inactiveApiKey,
inactivePrivateKey,
)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_RobinhoodCrypto,
Verified: true,
},
},
wantErr: false,
wantVerificationErr: false,
},
{
name: "found, unverified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf(
"You can find a robinhoodcrypto api key %s and a private key %s within", deletedApiKey,
deletedPrivateKey,
)), // the secret would satisfy the regex but not pass validation
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_RobinhoodCrypto,
Verified: false,
},
},
wantErr: false,
wantVerificationErr: false,
},
{
name: "not found",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte("You cannot find the secret within"),
verify: true,
},
want: nil,
wantErr: false,
wantVerificationErr: false,
},
{
name: "found, would be verified if not for timeout",
s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf(
"You can find a robinhoodcrypto api key %s and a private key %s within", apiKey, privateKey,
)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_RobinhoodCrypto,
Verified: false,
},
},
wantErr: false,
wantVerificationErr: true,
},
{
name: "found, verified but unexpected api surface",
s: Scanner{client: common.ConstantResponseHttpClient(404, "")},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf(
"You can find a robinhoodcrypto api key %s and a private key %s within", apiKey, privateKey,
)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_RobinhoodCrypto,
Verified: false,
},
},
wantErr: false,
wantVerificationErr: true,
},
}
for _, tt := range tests {
t.Run(
tt.name, func(t *testing.T) {
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
if (err != nil) != tt.wantErr {
t.Errorf("Robinhoodcrypto.FromData() error = %v, wantErr %v", err, tt.wantErr)
return
}
for i := range got {
if len(got[i].Raw) == 0 {
t.Fatalf("no raw secret present: \n %+v", got[i])
}
if (got[i].VerificationError() != nil) != tt.wantVerificationErr {
t.Fatalf(
"wantVerificationError = %v, verification error = %v", tt.wantVerificationErr,
got[i].VerificationError(),
)
}
}
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "ExtraData", "Raw", "RawV2", "verificationError")
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
t.Errorf("Robinhoodcrypto.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
}
},
)
}
}
func BenchmarkFromData(benchmark *testing.B) {
ctx := context.Background()
s := Scanner{}
for name, data := range detectors.MustGetBenchmarkData() {
benchmark.Run(
name, func(b *testing.B) {
b.ResetTimer()
for n := 0; n < b.N; n++ {
_, err := s.FromData(ctx, false, data)
if err != nil {
b.Fatal(err)
}
}
},
)
}
}

2
pkg/engine/defaults.go
View file

@ -577,6 +577,7 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/ringcentral"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/ritekit"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/roaring"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/robinhoodcrypto"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/rocketreach"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/rockset"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/roninapp"
@ -1626,6 +1627,7 @@ func DefaultDetectors() []detectors.Detector {
atlassianv1.Scanner{},
atlassianv2.Scanner{},
netsuite.Scanner{},
robinhoodcrypto.Scanner{},
}
}

5
pkg/engine/engine.go
View file

@ -457,6 +457,7 @@ func (e *Engine) initialize(ctx context.Context) error {
// This reflects the anticipated lower volume of data that needs re-verification.
// The buffer size is a trade-off between memory usage and the need to prevent blocking.
verificationOverlapChunksChanMultiplier = 25
resultsChanMultiplier = detectableChunksChanMultiplier
)
// Channels are used for communication between different parts of the engine,
@ -467,7 +468,7 @@ func (e *Engine) initialize(ctx context.Context) error {
e.verificationOverlapChunksChan = make(
chan verificationOverlapChunk, defaultChannelBuffer*verificationOverlapChunksChanMultiplier,
)
e.results = make(chan detectors.ResultWithMetadata, defaultChannelBuffer)
e.results = make(chan detectors.ResultWithMetadata, defaultChannelBuffer*resultsChanMultiplier)
e.dedupeCache = cache
ctx.Logger().V(4).Info("engine initialized")
@ -651,7 +652,7 @@ func (e *Engine) startVerificationOverlapWorkers(ctx context.Context) {
}
func (e *Engine) startNotifierWorkers(ctx context.Context) {
const notifierWorkerRatio = 4
const notifierWorkerRatio = 2
maxNotifierWorkers := 1
if numWorkers := e.concurrency / notifierWorkerRatio; numWorkers > 0 {
maxNotifierWorkers = numWorkers

26
pkg/pb/detectorspb/detectors.pb.go
View file

@ -1097,7 +1097,8 @@ const (
DetectorType_EndorLabs DetectorType = 993
DetectorType_ElevenLabs DetectorType = 994
DetectorType_Netsuite DetectorType = 995
DetectorType_Flexport DetectorType = 996
DetectorType_RobinhoodCrypto DetectorType = 996
DetectorType_Flexport DetectorType = 997
)
// Enum value maps for DetectorType.
@ -2095,7 +2096,8 @@ var (
993: "EndorLabs",
994: "ElevenLabs",
995: "Netsuite",
996: "Flexport",
996: "RobinhoodCrypto",
997: "Flexport",
}
DetectorType_value = map[string]int32{
"Alibaba": 0,
@ -3090,7 +3092,8 @@ var (
"EndorLabs": 993,
"ElevenLabs": 994,
"Netsuite": 995,
"Flexport": 996,
"RobinhoodCrypto": 996,
"Flexport": 997,
}
)
@ -3544,7 +3547,7 @@ var file_detectors_proto_rawDesc = []byte{
0x4c, 0x41, 0x49, 0x4e, 0x10, 0x01, 0x12, 0x0a, 0x0a, 0x06, 0x42, 0x41, 0x53, 0x45, 0x36, 0x34,
0x10, 0x02, 0x12, 0x09, 0x0a, 0x05, 0x55, 0x54, 0x46, 0x31, 0x36, 0x10, 0x03, 0x12, 0x13, 0x0a,
0x0f, 0x45, 0x53, 0x43, 0x41, 0x50, 0x45, 0x44, 0x5f, 0x55, 0x4e, 0x49, 0x43, 0x4f, 0x44, 0x45,
0x10, 0x04, 0x2a, 0xa7, 0x7f, 0x0a, 0x0c, 0x44, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x54,
0x10, 0x04, 0x2a, 0xbd, 0x7f, 0x0a, 0x0c, 0x44, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x54,
0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x41, 0x6c, 0x69, 0x62, 0x61, 0x62, 0x61, 0x10, 0x00,
0x12, 0x08, 0x0a, 0x04, 0x41, 0x4d, 0x51, 0x50, 0x10, 0x01, 0x12, 0x07, 0x0a, 0x03, 0x41, 0x57,
0x53, 0x10, 0x02, 0x12, 0x09, 0x0a, 0x05, 0x41, 0x7a, 0x75, 0x72, 0x65, 0x10, 0x03, 0x12, 0x0a,
@ -4561,13 +4564,14 @@ var file_detectors_proto_rawDesc = []byte{
0x75, 0x69, 0x74, 0x65, 0x41, 0x70, 0x69, 0x4b, 0x65, 0x79, 0x10, 0xe0, 0x07, 0x12, 0x0e, 0x0a,
0x09, 0x45, 0x6e, 0x64, 0x6f, 0x72, 0x4c, 0x61, 0x62, 0x73, 0x10, 0xe1, 0x07, 0x12, 0x0f, 0x0a,
0x0a, 0x45, 0x6c, 0x65, 0x76, 0x65, 0x6e, 0x4c, 0x61, 0x62, 0x73, 0x10, 0xe2, 0x07, 0x12, 0x0d,
0x0a, 0x08, 0x4e, 0x65, 0x74, 0x73, 0x75, 0x69, 0x74, 0x65, 0x10, 0xe3, 0x07, 0x12, 0x0d, 0x0a,
0x08, 0x46, 0x6c, 0x65, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x10, 0xe4, 0x07, 0x42, 0x3d, 0x5a, 0x3b,
0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66,
0x6c, 0x65, 0x73, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66,
0x6c, 0x65, 0x68, 0x6f, 0x67, 0x2f, 0x76, 0x33, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f,
0x64, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x73, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f,
0x74, 0x6f, 0x33,
0x0a, 0x08, 0x4e, 0x65, 0x74, 0x73, 0x75, 0x69, 0x74, 0x65, 0x10, 0xe3, 0x07, 0x12, 0x14, 0x0a,
0x0f, 0x52, 0x6f, 0x62, 0x69, 0x6e, 0x68, 0x6f, 0x6f, 0x64, 0x43, 0x72, 0x79, 0x70, 0x74, 0x6f,
0x10, 0xe4, 0x07, 0x12, 0x0d, 0x0a, 0x08, 0x46, 0x6c, 0x65, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x10,
0xe5, 0x07, 0x42, 0x3d, 0x5a, 0x3b, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d,
0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x73, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79,
0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x68, 0x6f, 0x67, 0x2f, 0x76, 0x33, 0x2f, 0x70,
0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x64, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x73, 0x70,
0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
}
var (

40
pkg/sources/github/github.go
View file

@ -16,6 +16,7 @@ import (
"github.com/go-logr/logr"
"github.com/gobwas/glob"
"github.com/google/go-github/v63/github"
"github.com/trufflesecurity/trufflehog/v3/pkg/handlers"
"golang.org/x/exp/rand"
"golang.org/x/sync/errgroup"
"google.golang.org/protobuf/proto"
@ -1392,35 +1393,34 @@ func (s *Source) scanTarget(ctx context.Context, target sources.ChunkingTarget,
return fmt.Errorf("invalid GitHub URL")
}
qry := commitQuery{
repo: segments[2],
owner: segments[1],
sha: meta.GetCommit(),
filename: meta.GetFile(),
readCloser, resp, err := s.connector.APIClient().Repositories.DownloadContents(
ctx,
segments[1],
segments[2],
meta.GetFile(),
&github.RepositoryContentGetOptions{Ref: meta.GetCommit()})
// As of this writing, if the returned readCloser is not nil, it's just the Body of the returned github.Response, so
// there's no need to independently close it.
if resp != nil && resp.Body != nil {
defer resp.Body.Close()
}
res, err := s.getDiffForFileInCommit(ctx, qry)
if err != nil {
return err
return fmt.Errorf("could not download file for scan: %w", err)
}
chunk := &sources.Chunk{
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("unexpected HTTP response status when trying to download file for scan: %v", resp.Status)
}
reporter := sources.ChanReporter{Ch: chunksChan}
chunkSkel := sources.Chunk{
SourceType: s.Type(),
SourceName: s.name,
SourceID: s.SourceID(),
JobID: s.JobID(),
SecretID: target.SecretID,
Data: []byte(stripLeadingPlusMinus(res)),
SourceMetadata: &source_metadatapb.MetaData{
Data: &source_metadatapb.MetaData_Github{Github: meta},
},
Verify: s.verify,
}
return common.CancellableWrite(ctx, chunksChan, chunk)
}
// stripLeadingPlusMinus removes leading + and - characters from lines in a diff string. These characters exist in the
// diffs returned when performing a targeted scan and need to be removed so that detectors are operating on the correct
// text.
func stripLeadingPlusMinus(diff string) string {
return strings.ReplaceAll(strings.ReplaceAll(diff, "\n+", "\n"), "\n-", "\n")
Verify: s.verify}
return handlers.HandleFile(ctx, readCloser, &chunkSkel, reporter)
}

18
pkg/sources/github/github_integration_test.go
View file

@ -758,6 +758,24 @@ func TestSource_Chunks_TargetedScan(t *testing.T) {
},
wantChunks: 1,
},
{
name: "targeted scan, binary file",
init: init{
name: "test source",
connection: &sourcespb.GitHub{Credential: &sourcespb.GitHub_Token{Token: githubToken}},
queryCriteria: &source_metadatapb.MetaData{
Data: &source_metadatapb.MetaData_Github{
Github: &source_metadatapb.Github{
Repository: "https://github.com/truffle-sandbox/test-secrets.git",
Link: "https://github.com/truffle-sandbox/test-secrets/blob/70bef8590f87257c0992eecc7db529827a12b801/null_text_w_ptp.ipynb",
Commit: "70bef8590f87257c0992eecc7db529827a12b801",
File: "null_text_w_ptp.ipynb",
},
},
},
},
wantChunks: 607,
},
{
name: "no file in commit",
init: init{

54
pkg/sources/github/repo.go
View file

@ -281,60 +281,6 @@ func (s *Source) wikiIsReachable(ctx context.Context, repoURL string) bool {
return wikiURL == res.Request.URL.String()
}
// commitQuery represents the details required to fetch a commit.
type commitQuery struct {
repo string
owner string
sha string
filename string
}
// getDiffForFileInCommit retrieves the diff for a specified file in a commit.
// If the file or its diff is not found, it returns an error.
func (s *Source) getDiffForFileInCommit(ctx context.Context, query commitQuery) (string, error) {
var (
commit *github.RepositoryCommit
err error
)
for {
commit, _, err = s.connector.APIClient().Repositories.GetCommit(ctx, query.owner, query.repo, query.sha, nil)
if s.handleRateLimit(err) {
continue
}
if err != nil {
return "", fmt.Errorf("error fetching commit %s: %w", query.sha, err)
}
break
}
if len(commit.Files) == 0 {
return "", fmt.Errorf("commit %s does not contain any files", query.sha)
}
res := new(strings.Builder)
// Only return the diff if the file is in the commit.
for _, file := range commit.Files {
if *file.Filename != query.filename {
continue
}
if file.Patch == nil {
return "", fmt.Errorf("commit %s file %s does not have a diff", query.sha, query.filename)
}
if _, err := res.WriteString(*file.Patch); err != nil {
return "", fmt.Errorf("buffer write error for commit %s file %s: %w", query.sha, query.filename, err)
}
res.WriteString("\n")
}
if res.Len() == 0 {
return "", fmt.Errorf("commit %s does not contain patch for file %s", query.sha, query.filename)
}
return res.String(), nil
}
func (s *Source) normalizeRepo(repo string) (string, error) {
// If there's a '/', assume it's a URL and try to normalize it.
if strings.ContainsRune(repo, '/') {

3
proto/detectors.proto
View file

@ -1005,7 +1005,8 @@ enum DetectorType {
EndorLabs = 993;
ElevenLabs = 994;
Netsuite = 995;
Flexport = 996;
RobinhoodCrypto = 996;
Flexport = 997;
}
message Result {