Ignore canary IDs in notifications (#2526)

* Update aws.go

* Update aws.go

* Update tests

---------

Co-authored-by: Dustin Decker <dustin@trufflesec.com>

pkg/detectors/aws/aws.go

@@ -21,6 +21,7 @@ import (
 type scanner struct {
 	verificationClient *http.Client
 	skipIDs            map[string]struct{}
+	verifyCanaries     bool
 }
 
 // resourceTypes derived from: https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_identifiers.html#identifiers-unique-ids
@@ -95,6 +96,12 @@ func WithSkipIDs(skipIDs []string) func(*scanner) {
 	}
 }
 
+func WithVerifyCanaries() func(*scanner) {
+	return func(s *scanner) {
+		s.verifyCanaries = true
+	}
+}
+
 // Ensure the scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*scanner)(nil)
 
@@ -173,15 +180,17 @@ func (s scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			if err == nil {
 				s1.ExtraData["account"] = account
 			}
-			if _, ok := thinkstCanaryList[account]; ok {
+			if !s.verifyCanaries && !strings.Contains(dataStr, "Redacted: "+resIDMatch) {
-				s1.ExtraData["is_canary"] = "true"
+				if _, ok := thinkstCanaryList[account]; ok {
-				s1.ExtraData["message"] = thinkstMessage
+					s1.ExtraData["is_canary"] = "true"
-				s1.Verified = true
+					s1.ExtraData["message"] = thinkstMessage
-			}
+					s1.Verified = true
-			if _, ok := thinkstKnockoffsCanaryList[account]; ok {
+				}
-				s1.ExtraData["is_canary"] = "true"
+				if _, ok := thinkstKnockoffsCanaryList[account]; ok {
-				s1.ExtraData["message"] = thinkstKnockoffsMessage
+					s1.ExtraData["is_canary"] = "true"
-				s1.Verified = true
+					s1.ExtraData["message"] = thinkstKnockoffsMessage
+					s1.Verified = true
+				}
 			}
 
 			if verify && !s1.Verified {

pkg/detectors/aws/aws_test.go

@@ -49,7 +49,7 @@ func TestAWS_FromChunk(t *testing.T) {
 	}{
 		{
 			name: "found, verified",
-			s:    scanner{},
+			s:    scanner{verifyCanaries: true},
 			args: args{
 				ctx:    context.Background(),
 				data:   []byte(fmt.Sprintf("You can find a aws secret %s within aws %s", secret, id)),
@@ -73,7 +73,7 @@ func TestAWS_FromChunk(t *testing.T) {
 		},
 		{
 			name: "found, unverified",
-			s:    scanner{verificationClient: unverifiedSecretClient},
+			s:    scanner{verificationClient: unverifiedSecretClient, verifyCanaries: true},
 			args: args{
 				ctx:    context.Background(),
 				data:   []byte(fmt.Sprintf("You can find a aws secret %s within aws %s but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation
@@ -105,7 +105,7 @@ func TestAWS_FromChunk(t *testing.T) {
 		},
 		{
 			name: "found two, one included for every ID found",
-			s:    scanner{},
+			s:    scanner{verifyCanaries: true},
 			args: args{
 				ctx:    context.Background(),
 				data:   []byte(fmt.Sprintf("The verified ID is %s with a secret of %s, but the unverified ID is %s and this is the secret %s", id, secret, inactiveID, inactiveSecret)),
@@ -149,7 +149,9 @@ func TestAWS_FromChunk(t *testing.T) {
 		},
 		{
 			name: "found two, returned both because the active secret for one paired with the inactive ID, despite the hash",
-			s:    scanner{},
+			s: scanner{
+				verifyCanaries: true,
+			},
 			args: args{
 				ctx:    context.Background(),
 				data:   []byte(fmt.Sprintf("The verified ID is %s with a secret of %s, but the unverified ID is %s and the secret is this hash %s", id, secret, inactiveID, hash)),
@@ -172,13 +174,17 @@ func TestAWS_FromChunk(t *testing.T) {
 					DetectorType: detectorspb.DetectorType_AWS,
 					Verified:     false,
 					Redacted:     inactiveID,
+					ExtraData:    map[string]string{"account": "171436882533", "resource_type": "Access key"},
 				},
 			},
 			wantErr: false,
 		},
 		{
 			name: "found, unverified, with leading +",
-			s:    scanner{verificationClient: unverifiedSecretClient},
+			s: scanner{
+				verificationClient: unverifiedSecretClient,
+				verifyCanaries:     true,
+			},
 			args: args{
 				ctx:    context.Background(),
 				data:   []byte(fmt.Sprintf("You can find a aws secret %s within aws %s but not valid", "+HaNv9cTwheDKGJaws/+BMF2GgybQgBWdhcOOdfF", id)), // the secret would satisfy the regex but not pass validation
@@ -213,7 +219,10 @@ func TestAWS_FromChunk(t *testing.T) {
 		},
 		{
 			name: "found, would be verified if not for http timeout",
-			s:    scanner{verificationClient: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
+			s: scanner{
+				verificationClient: common.SaneHttpClientTimeOut(1 * time.Microsecond),
+				verifyCanaries:     true,
+			},
 			args: args{
 				ctx:    context.Background(),
 				data:   []byte(fmt.Sprintf("You can find a aws secret %s within aws %s", secret, id)),
@@ -235,7 +244,10 @@ func TestAWS_FromChunk(t *testing.T) {
 		},
 		{
 			name: "found, unverified due to unexpected http response status",
-			s:    scanner{verificationClient: common.ConstantResponseHttpClient(500, "internal server error")},
+			s: scanner{
+				verificationClient: common.ConstantResponseHttpClient(500, "internal server error"),
+				verifyCanaries:     true,
+			},
 			args: args{
 				ctx:    context.Background(),
 				data:   []byte(fmt.Sprintf("You can find a aws secret %s within aws %s", secret, id)),
@@ -257,7 +269,10 @@ func TestAWS_FromChunk(t *testing.T) {
 		},
 		{
 			name: "found, unverified due to unexpected 403 response reason",
-			s:    scanner{verificationClient: common.ConstantResponseHttpClient(403, `{"Error": {"Code": "SignatureDoesNotMatch"} }`)},
+			s: scanner{
+				verificationClient: common.ConstantResponseHttpClient(403, `{"Error": {"Code": "SignatureDoesNotMatch"} }`),
+				verifyCanaries:     true,
+			},
 			args: args{
 				ctx:    context.Background(),
 				data:   []byte(fmt.Sprintf("You can find a aws secret %s within aws %s", secret, id)),
@@ -279,7 +294,7 @@ func TestAWS_FromChunk(t *testing.T) {
 		},
 		{
 			name: "verified secret checked directly after unverified secret with same key id",
-			s:    scanner{},
+			s:    scanner{verifyCanaries: true},
 			args: args{
 				ctx:    context.Background(),
 				data:   []byte(fmt.Sprintf("%s\n%s\n%s", inactiveSecret, id, secret)),