From 78bbb89a307a8f8947c0983fd12f2659570eb13d Mon Sep 17 00:00:00 2001 From: Zubair Khan Date: Thu, 31 Aug 2023 12:06:18 -0400 Subject: [PATCH] add tri-state verification for twilio detector (#1729) * add tri state for twilio * save progress * fix twilio tristate test * resolve lint issue --- pkg/detectors/twilio/twilio.go | 17 +++++++- pkg/detectors/twilio/twilio_test.go | 68 +++++++++++++++++++++++------ 2 files changed, 71 insertions(+), 14 deletions(-) diff --git a/pkg/detectors/twilio/twilio.go b/pkg/detectors/twilio/twilio.go index 393033f14..062364935 100644 --- a/pkg/detectors/twilio/twilio.go +++ b/pkg/detectors/twilio/twilio.go @@ -2,6 +2,7 @@ package twilio import ( "context" + "fmt" "net/http" "regexp" @@ -10,12 +11,15 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct { + client *http.Client +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) var ( + defaultClient = common.SaneHttpClient() identifierPat = regexp.MustCompile(`(?i)sid.{0,20}AC[0-9a-f]{32}`) // Should we have this? Seems restrictive. sidPat = regexp.MustCompile(`\bAC[0-9a-f]{32}\b`) keyPat = regexp.MustCompile(`\b[0-9a-f]{32}\b`) @@ -51,6 +55,11 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } if verify { + client = s.client + if client == nil { + client = defaultClient + } + req, err := http.NewRequestWithContext( ctx, "GET", "https://verify.twilio.com/v2/Services", nil) if err != nil { @@ -65,7 +74,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true + } else if res.StatusCode == 401 || res.StatusCode == 403 { + // The secret is determinately not verified (nothing to do) + } else { + s1.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode) } + } else { + s1.VerificationError = err } } diff --git a/pkg/detectors/twilio/twilio_test.go b/pkg/detectors/twilio/twilio_test.go index 8d6b4c304..a6f53eab4 100644 --- a/pkg/detectors/twilio/twilio_test.go +++ b/pkg/detectors/twilio/twilio_test.go @@ -6,11 +6,11 @@ package twilio import ( "context" "fmt" + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" "testing" "time" - "github.com/kylelemons/godebug/pretty" - "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" @@ -33,11 +33,12 @@ func TestTwilio_FromChunk(t *testing.T) { verify bool } tests := []struct { - name string - s Scanner - args args - want []detectors.Result - wantErr bool + name string + s Scanner + args args + want []detectors.Result + wantErr bool + wantVerificationErr bool }{ { name: "found, verified", @@ -52,6 +53,7 @@ func TestTwilio_FromChunk(t *testing.T) { DetectorType: detectorspb.DetectorType_Twilio, Verified: true, Redacted: id, + RawV2: []byte(id + secret), }, }, wantErr: false, @@ -69,6 +71,7 @@ func TestTwilio_FromChunk(t *testing.T) { DetectorType: detectorspb.DetectorType_Twilio, Verified: false, Redacted: id, + RawV2: []byte(id + secretInactive), }, }, wantErr: false, @@ -84,23 +87,62 @@ func TestTwilio_FromChunk(t *testing.T) { want: nil, wantErr: false, }, + { + name: "found, would be verified if not for timeout", + s: Scanner{client: common.SaneHttpClientTimeOut(100 * time.Microsecond)}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a twillio secret %s within awsId %s", secret, id)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_Twilio, + Verified: false, + Redacted: id, + RawV2: []byte(id + secret), + }, + }, + wantErr: false, + wantVerificationErr: true, + }, + { + name: "found, verified but unexpected api surface", + s: Scanner{client: common.ConstantResponseHttpClient(404, "")}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a twillio secret %s within awsId %s", secret, id)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_Twilio, + Verified: false, + Redacted: id, + RawV2: []byte(id + secret), + }, + }, + wantErr: false, + wantVerificationErr: true, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - s := Scanner{} - got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) if (err != nil) != tt.wantErr { t.Errorf("Twilio.FromData() error = %v, wantErr %v", err, tt.wantErr) return } for i := range got { if len(got[i].Raw) == 0 { - t.Fatal("no raw secret present") + t.Fatalf("no raw secret present: \n %+v", got[i]) + } + if (got[i].VerificationError != nil) != tt.wantVerificationErr { + t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError) } - got[i].Raw = nil - got[i].RawV2 = nil } - if diff := pretty.Compare(got, tt.want); diff != "" { + ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "VerificationError") + if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" { t.Errorf("Twilio.FromData() %s diff: (-got +want)\n%s", tt.name, diff) } })