fix(signable): ignore common false positives (#2230)

2024-11-10 07:04:24 +00:00 · 2024-01-09 20:53:25 -05:00 · 2024-01-09 20:53:25 -05:00 · 04bf244f38
commit 04bf244f38
parent 38f36475de
2 changed files with 209 additions and 25 deletions
--- a/pkg/detectors/signable/signable.go
+++ b/pkg/detectors/signable/signable.go
@ -13,16 +13,19 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 )

-type Scanner struct{}
+type Scanner struct {
+	client *http.Client
+}

 // Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)

 var (
-	client = common.SaneHttpClient()
+	defaultClient = common.SaneHttpClient()

 	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
-	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"signable"}) + `\b([a-zA-Z-0-9]{32})\b`)
+	tokenPat   = regexp.MustCompile(detectors.PrefixRegex([]string{".{0,2}signable"}) + `\b([a-zA-Z-0-9]{32})\b`)
+	keywordPat = regexp.MustCompile(`(?i)([a-z]{2})signable`)
 )

 // Keywords are used for efficiently pre-filtering chunks.
@ -35,41 +38,34 @@ func (s Scanner) Keywords() []string {
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
 	dataStr := string(data)

-	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
-
+	matches := tokenPat.FindAllStringSubmatch(dataStr, -1)
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue
 		}
-		resMatch := strings.TrimSpace(match[1])

+		if isCommonFalsePositive(match[0]) {
+			continue
+		}
+
+		resMatch := strings.TrimSpace(match[1])
 		s1 := detectors.Result{
 			DetectorType: detectorspb.DetectorType_Signable,
 			Raw:          []byte(resMatch),
 		}

 		if verify {
-			data := fmt.Sprintf("%s:", resMatch)
-			sEnc := b64.StdEncoding.EncodeToString([]byte(data))
-
-			req, err := http.NewRequestWithContext(ctx, "GET", "https://api.signable.co.uk/v1/templates?offset=0&limit=5", nil)
-			if err != nil {
-				continue
+			if s.client == nil {
+				s.client = defaultClient
 			}
+			isVerified, verificationErr := verifyResult(ctx, s.client, resMatch)
+			s1.Verified = isVerified
+			s1.SetVerificationError(verificationErr, resMatch)
+		}

-			req.Header.Add("Authorization", fmt.Sprintf("Basic %s", sEnc))
-			res, err := client.Do(req)
-			if err == nil {
-				defer res.Body.Close()
-				if res.StatusCode >= 200 && res.StatusCode < 300 {
-					s1.Verified = true
-				} else {
-					// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
-					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
-						continue
-					}
-				}
-			}
+		// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
+		if !s1.Verified && detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
+			continue
 		}

 		results = append(results, s1)
@ -78,6 +74,44 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 	return results, nil
 }

+// Eliminate the most common false positive.
+func isCommonFalsePositive(line string) bool {
+	// TODO: Skip lock files altogether. (https://github.com/trufflesecurity/trufflehog/issues/1517)
+	if strings.Contains(line, "helper-explode-assignable-expression") {
+		return true
+	}
+
+	// Eliminate false positives from `assignable` and `designable`.
+	for _, m := range keywordPat.FindAllStringSubmatch(line, -1) {
+		if strings.EqualFold(m[1], "as") || strings.EqualFold(m[1], "de") {
+			return true
+		}
+	}
+	return false
+}
+
+func verifyResult(ctx context.Context, client *http.Client, token string) (bool, error) {
+	data := fmt.Sprintf("%s:", token)
+	sEnc := b64.StdEncoding.EncodeToString([]byte(data))
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://api.signable.co.uk/v1/templates?offset=0&limit=5", nil)
+	if err != nil {
+		return false, err
+	}
+
+	req.Header.Add("Authorization", fmt.Sprintf("Basic %s", sEnc))
+	res, err := client.Do(req)
+	if err != nil {
+		return false, err
+	}
+
+	defer res.Body.Close()
+	if res.StatusCode >= 200 && res.StatusCode < 300 {
+		return true, nil
+	}
+	return false, nil
+}
+
 func (s Scanner) Type() detectorspb.DetectorType {
 	return detectorspb.DetectorType_Signable
 }
--- a/pkg/detectors/signable/signable_test.go
+++ b/pkg/detectors/signable/signable_test.go
@ -10,12 +10,162 @@ import (
 	"time"

 	"github.com/kylelemons/godebug/pretty"
+
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"

 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 )

+func TestSignable_Pattern(t *testing.T) {
+	tests := []struct {
+		name        string
+		data        string
+		shouldMatch bool
+		match       string
+	}{
+		// True positives
+		{
+			name:        "valid",
+			data:        `const signableToken = '40a1cd917bff1288f699a94a75b37a1a'`,
+			shouldMatch: true,
+			match:       "40a1cd917bff1288f699a94a75b37a1a",
+		},
+
+		// False positives
+		{
+			name: `invalid_assignable_yarn`,
+			data: `"  babel-helper-explode-assignable-expression@^6.24.1:
+    version "6.24.1"
+    resolved "https://registry.npmjs.org/babel-helper-explode-assignable-expression/-/babel-helper-explode-assignable-expression-6.24.1.tgz#f25b82cf7dc10433c55f70592d5746400ac22caa"
+    dependencies:
+      babel-runtime "^6.22.0"
+      babel-traverse "^6.24.1"
+      babel-types "^6.24.1"`,
+			shouldMatch: false,
+		},
+		{
+			name: `invalid_assignable_yarn`,
+			data: `"@babel/helper-explode-assignable-expression@^7.16.7":
+  version "7.16.7"
+  resolved "https://registry.yarnpkg.com/@babel/helper-explode-assignable-expression/-/helper-explode-assignable-expression-7.16.7.tgz#12a6d8522fdd834f194e868af6354e8650242b7a"
+  integrity sha512-KyUenhWMC8VrxzkGP0Jizjo4/Zx+1nNZhgocs+gLzyZyB8SHidhoq9KK/8Ato4anhwsivfkBLftky7gvzbZMtQ==
+  dependencies:
+    "@babel/types" "^7.16.7"`,
+			shouldMatch: false,
+		},
+		// https://github.com/tbenst/purescript-nix-example/blob/558c8d6cb605742218cfa14a3fa93c062324b885/yarn.nix
+		{
+			name: `invalid_assignable_nix`,
+			data: `    {
+      name = "_babel_helper_explode_assignable_expression___helper_explode_assignable_expression_7.8.3.tgz";
+      path = fetchurl {
+        name = "_babel_helper_explode_assignable_expression___helper_explode_assignable_expression_7.8.3.tgz";
+        url  = "https://registry.yarnpkg.com/@babel/helper-explode-assignable-expression/-/helper-explode-assignable-expression-7.8.3.tgz";
+        sha1 = "a728dc5b4e89e30fc2dfc7d04fa28a930653f982";
+      };
+    }`,
+			shouldMatch: false,
+		},
+		{
+			name: `invalid_assignable`,
+			data: `<tr><td colspan="2"><br><h2>Public Member Functions</h2></td></tr>
+<tr><td class="memItemLeft" nowrap align="right" valign="top"><a class="anchor" name="b0a0dbf6ca9028bbbb2240cad5882537"></a><!-- doxytag: member="boost::gil::Assignable::constraints" ref="b0a0dbf6ca9028bbbb2240cad5882537" args="()" -->
+void&nbsp;</td><td class="memItemRight" valign="bottom"><b>constraints</b> ()</td></tr>
+`,
+			shouldMatch: false,
+		},
+		{
+			name: `invalid_assignable`,
+			data: `File: enumIsAssignableToBuiltInEnum.kt - 6396cf8549625bfce8b8ca2511d7f347
+  NL("\n")
+  packageHeader`,
+			shouldMatch: false,
+		},
+		{
+			name: `invalid_assignable_php`,
+			data: `'./include/SugarObjects/forms/PersonFormBase.php' => '2c1846ef127d60a40ecbab2c0b312ff5',
+  './include/SugarObjects/implements/assignable/language/en_us.lang.php' => '90f14b03e22e1eed2a1b93e10b975ef5',
+  './include/SugarObjects/implements/assignable/vardefs.php' => '358e0c47f753c5577fbdc0de08553c02',
+  './include/SugarObjects/implements/security_groups/language/en_us.lang.php' => 'ac1fd4817cb4662e3bdf973836558bdb',`,
+			shouldMatch: false,
+		},
+		// https://github.com/past-due/warzone2100/blob/3e5637a7ed3d67ab92e439b94bf93f89f7bbea51/ChangeLog#L318
+		{
+			name: `invalid_designable`,
+			data: `   * Fix: Prevent map selection button list from going off the form (commit:aea66eb1aa557c73d97b8019e5e66fccbb79f66e, #1347)
+   * Fix: Fix odd EMP mortar pathway; Add EMP mortar to designable weapons (commit:bcf93b7fe640c09e8b1239fabfd901fde9760259, #1535)`,
+			shouldMatch: false,
+		},
+		// https://github.com/exis-io/Exis/blob/5383174f7b52112a97aadd09e6b9ea837c2fa07b/CardsAgainstHumanityDemo/swiftCardsAgainst/Pods/Pods.xcodeproj/project.pbxproj
+		{
+			name: `invalid_designable`,
+			data: `		570767CBD99941F484DED46232044DC3 /* DesignableView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 93111B182BD71051B9ED0B14A9EF6EB6 /* DesignableView.swift */; };
+		57140D31D50A2FDE1E26729DDE7CB762 /* M13ProgressHUD.h in Headers */ = {isa = PBXBuildFile; fileRef = ECF777CB8B4C090E8D271E62729F7DD3 /* M13ProgressHUD.h */; settings = {ATTRIBUTES = (Public, ); }; };`,
+			shouldMatch: false,
+		},
+		{
+			name:        `invalid_designable`,
+			data:        `{"nick":"hamster88","message":"this is my gist > https://gist.github.com/thedesignable/a05f628c649a81aae757945c352a8392","date":"2016-06-19T11:05:13.776Z","type":"message"}`,
+			shouldMatch: false,
+		},
+		{
+			name: `invalid_designable`,
+			data: `返回到故事板文件，选择视图(我将假设从现在起视图被选中)并打开 Identity Inspector。你会注意到一个*可设计的*状态指示器已经出现在自定义类部分。
+
+![Designables status](img/84bc9afc942815899347a31a425af7c6.png)`,
+			shouldMatch: false,
+		},
+		{
+			name:        `invalid_designable`,
+			data:        `&lt;h3 id=&#34;ibdesignable-x-paintcode:0b699a3cd6d609650a3fca90a5cd32cc&#34;&gt;IBDesignable x PaintCode&lt;/h3&gt;`,
+			shouldMatch: false,
+		},
+		{
+			name: `invalid_designable`,
+			data: `    </designables>
+    <resources>
+        <image name="a932cb605eb09bff88b88fdf1c3ef8aa" width="736" height="895"/>
+        <image name="plus" catalog="system" width="128" height="113"/>`,
+			shouldMatch: false,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			s := Scanner{}
+
+			results, err := s.FromData(context.Background(), false, []byte(test.data))
+			if err != nil {
+				t.Errorf("Signable.FromData() error = %v", err)
+				return
+			}
+
+			if test.shouldMatch {
+				if len(results) == 0 {
+					t.Errorf("%s: did not receive a match for '%v' when one was expected", test.name, test.data)
+					return
+				}
+				expected := test.data
+				if test.match != "" {
+					expected = test.match
+				}
+				result := results[0]
+				resultData := string(result.Raw)
+				if resultData != expected {
+					t.Errorf("%s: did not receive expected match.\n\texpected: '%s'\n\t  actual: '%s'", test.name, expected, resultData)
+					return
+				}
+			} else {
+				if len(results) > 0 {
+					t.Errorf("%s: received a match for '%v' when one wasn't wanted", test.name, test.data)
+					return
+				}
+			}
+		})
+	}
+}
+
 func TestSignable_FromChunk(t *testing.T) {
 	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
 	defer cancel()