additional similarity check for base64 and plain (#1462)

* additional similarity check for base64 and plain

* use bytes equal

* move logic into util function

pkg/common/utils.go

@@ -2,6 +2,7 @@ package common
 
 import (
 	"bufio"
+	"bytes"
 	"io"
 	"strings"
 )
@@ -24,6 +25,18 @@ func RemoveStringSliceItem(item string, slice *[]string) {
 	}
 }
 
+func MinInt(a, b int) int {
+	if a < b {
+		return a
+	}
+	return b
+}
+
+func BytesEqual(a, b []byte, numBytes int) bool {
+	limit := MinInt(numBytes, MinInt(len(a), len(b))-1)
+	return bytes.Equal(a[:limit], b[:limit])
+}
+
 func ResponseContainsSubstring(reader io.ReadCloser, target string) (bool, error) {
 	scanner := bufio.NewScanner(reader)
 	for scanner.Scan() {

pkg/engine/engine.go

@@ -272,11 +272,22 @@ func (e *Engine) detectorWorker(ctx context.Context) {
 					ctx.Logger().Info("unknown decoder type", "type", reflect.TypeOf(decoder).String())
 					decoderType = detectorspb.DecoderType_UNKNOWN
 				}
+
+				original := chunk.Data
 				decoded := decoder.FromChunk(chunk)
+
 				if decoded == nil {
 					continue
 				}
 
+				if decoded == nil ||
+					// check if the decoded data is similar "enough" to the original data. If it is, then we can skip scanning the decoded data as
+					// it's likely already picked up by the PLAIN decoder. See related issue: https://github.com/trufflesecurity/trufflehog/issues/1450
+					(decoded != nil &&
+						decoderType == detectorspb.DecoderType_BASE64 && common.BytesEqual(original, decoded.Data, 40)) {
+					continue
+				}
+
 				// build a map of all keywords that were matched in the chunk
 				for _, m := range e.prefilter.FindAll(string(decoded.Data)) {
 					matchedKeywords[strings.ToLower(string(decoded.Data[m.Start():m.End()]))] = struct{}{}