mirror of
https://github.com/trufflesecurity/trufflehog.git
synced 2024-11-10 07:04:24 +00:00
Implement CustomRegex detector (#950)
* Remove verifying successRanges because it is unused in webhook * Move custom_detectors validation code into its own file * Initial implementation of custom regex detector Secret verification is done via webhook. * Add CustomRegex detector type * Add upper bound to permutation * Return early if the context is canceled * Add headers from configuration * Add detector name as a key in the JSON body * Implement faster algorithm for productIndices
This commit is contained in:
Miccah
GitHub
parent
36ca2601e0
commit
861ad057c7
6 changed files with 584 additions and 318 deletions
|
|
@ -1,151 +1,226 @@
|
|||
package custom_detectors
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/custom_detectorspb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
// customRegex is a CustomRegex that is guaranteed to be valid.
|
||||
type customRegex *custom_detectorspb.CustomRegex
|
||||
// The maximum number of matches from one chunk. This const is used when
|
||||
// permutating each regex match to protect the scanner from doing too much work
|
||||
// for poorly defined regexps.
|
||||
const maxTotalMatches = 100
|
||||
|
||||
func ValidateKeywords(keywords []string) error {
|
||||
if len(keywords) == 0 {
|
||||
return fmt.Errorf("no keywords")
|
||||
}
|
||||
|
||||
for _, keyword := range keywords {
|
||||
if len(keyword) == 0 {
|
||||
return fmt.Errorf("empty keyword")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
// customRegexWebhook is a CustomRegex with webhook validation that is
|
||||
// guaranteed to be valid (assuming the data is not changed after
|
||||
// initialization).
|
||||
type customRegexWebhook struct {
|
||||
*custom_detectorspb.CustomRegex
|
||||
}
|
||||
|
||||
func ValidateRegex(regex map[string]string) error {
|
||||
if len(regex) == 0 {
|
||||
return fmt.Errorf("no regex")
|
||||
}
|
||||
// Ensure the Scanner satisfies the interface at compile time.
|
||||
var _ detectors.Detector = (*customRegexWebhook)(nil)
|
||||
|
||||
for _, r := range regex {
|
||||
if _, err := regexp.Compile(r); err != nil {
|
||||
return fmt.Errorf("invalid regex %q", r)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func ValidateVerifyEndpoint(endpoint string, unsafe bool) error {
|
||||
if len(endpoint) == 0 {
|
||||
return fmt.Errorf("no endpoint")
|
||||
}
|
||||
|
||||
if strings.HasPrefix(endpoint, "http://") && !unsafe {
|
||||
return fmt.Errorf("http endpoint must have unsafe=true")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func ValidateVerifyHeaders(headers []string) error {
|
||||
for _, header := range headers {
|
||||
if !strings.Contains(header, ":") {
|
||||
return fmt.Errorf("header %q must contain a colon", header)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func ValidateVerifyRanges(ranges []string) error {
|
||||
const httpLowerBound = 100
|
||||
const httpUpperBound = 599
|
||||
|
||||
for _, successRange := range ranges {
|
||||
if !strings.Contains(successRange, "-") {
|
||||
httpCode, err := strconv.Atoi(successRange)
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to convert http code to int %q", successRange)
|
||||
}
|
||||
|
||||
if httpCode < httpLowerBound || httpCode > httpUpperBound {
|
||||
return fmt.Errorf("invalid http status code %q", successRange)
|
||||
}
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
httpRange := strings.Split(successRange, "-")
|
||||
if len(httpRange) != 2 {
|
||||
return fmt.Errorf("invalid range format %q", successRange)
|
||||
}
|
||||
|
||||
lowerBound, err := strconv.Atoi(httpRange[0])
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to convert lower bound to int %q", successRange)
|
||||
}
|
||||
|
||||
upperBound, err := strconv.Atoi(httpRange[1])
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to convert upper bound to int %q", successRange)
|
||||
}
|
||||
|
||||
if lowerBound > upperBound {
|
||||
return fmt.Errorf("lower bound greater than upper bound on range %q", successRange)
|
||||
}
|
||||
|
||||
if lowerBound < httpLowerBound || upperBound > httpUpperBound {
|
||||
return fmt.Errorf("invalid http status code range %q", successRange)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func ValidateRegexVars(regex map[string]string, body ...string) error {
|
||||
for _, b := range body {
|
||||
matches := NewRegexVarString(b).variables
|
||||
|
||||
for match := range matches {
|
||||
if _, ok := regex[match]; !ok {
|
||||
return fmt.Errorf("body %q contains an unknown variable", b)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func NewCustomRegex(pb *custom_detectorspb.CustomRegex) (customRegex, error) {
|
||||
// NewWebhookCustomRegex initializes and validates a customRegexWebhook. An
|
||||
// unexported type is intentionally returned here to ensure the values have
|
||||
// been validated.
|
||||
func NewWebhookCustomRegex(pb *custom_detectorspb.CustomRegex) (*customRegexWebhook, error) {
|
||||
// TODO: Return all validation errors.
|
||||
if err := ValidateKeywords(pb.Keywords); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if err := ValidateRegex(pb.Regex); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for _, verify := range pb.Verify {
|
||||
|
||||
if err := ValidateVerifyEndpoint(verify.Endpoint, verify.Unsafe); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if err := ValidateVerifyHeaders(verify.Headers); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if err := ValidateVerifyRanges(verify.SuccessRanges); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if err := ValidateRegexVars(pb.Regex, append(verify.Headers, verify.Endpoint)...); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
return pb, nil
|
||||
// TODO: Copy only necessary data out of pb.
|
||||
return &customRegexWebhook{pb}, nil
|
||||
}
|
||||
|
||||
var httpClient = common.SaneHttpClient()
|
||||
|
||||
func (c *customRegexWebhook) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
regexMatches := make(map[string][][]string, len(c.GetRegex()))
|
||||
|
||||
// Find all submatches for each regex.
|
||||
for name, regex := range c.GetRegex() {
|
||||
regex, err := regexp.Compile(regex)
|
||||
if err != nil {
|
||||
// TODO: Log error.
|
||||
// This should never happen due to validation.
|
||||
continue
|
||||
}
|
||||
regexMatches[name] = regex.FindAllStringSubmatch(dataStr, -1)
|
||||
}
|
||||
|
||||
// Permutate each individual match.
|
||||
// {
|
||||
// "foo": [["match1"]]
|
||||
// "bar": [["match2"], ["match3"]]
|
||||
// }
|
||||
// becomes
|
||||
// [
|
||||
// {"foo": ["match1"], "bar": ["match2"]},
|
||||
// {"foo": ["match1"], "bar": ["match3"]},
|
||||
// ]
|
||||
matches := permutateMatches(regexMatches)
|
||||
|
||||
// Create result object and test for verification.
|
||||
for _, match := range matches {
|
||||
if common.IsDone(ctx) {
|
||||
// TODO: Log we're possibly leaving out results.
|
||||
return results, nil
|
||||
}
|
||||
var raw string
|
||||
for _, values := range match {
|
||||
// values[0] contains the entire regex match.
|
||||
raw += values[0]
|
||||
}
|
||||
result := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_CustomRegex,
|
||||
Raw: []byte(raw),
|
||||
}
|
||||
|
||||
if isKnownFalsePositive(match) {
|
||||
continue
|
||||
}
|
||||
if !verify {
|
||||
results = append(results, result)
|
||||
continue
|
||||
}
|
||||
// Verify via webhook.
|
||||
jsonBody, err := json.Marshal(map[string]map[string][]string{
|
||||
c.GetName(): match,
|
||||
})
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
// Try each config until we successfully verify.
|
||||
for _, verifyConfig := range c.GetVerify() {
|
||||
if common.IsDone(ctx) {
|
||||
// TODO: Log we're possibly leaving out results.
|
||||
return results, nil
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, "POST", verifyConfig.GetEndpoint(), bytes.NewReader(jsonBody))
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
for _, header := range verifyConfig.GetHeaders() {
|
||||
key, value, found := strings.Cut(header, ":")
|
||||
if !found {
|
||||
// Should be unreachable due to validation.
|
||||
continue
|
||||
}
|
||||
req.Header.Add(key, strings.TrimLeft(value, "\t\n\v\f\r "))
|
||||
}
|
||||
res, err := httpClient.Do(req)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
// TODO: Read response body.
|
||||
res.Body.Close()
|
||||
if res.StatusCode == http.StatusOK {
|
||||
result.Verified = true
|
||||
break
|
||||
}
|
||||
}
|
||||
results = append(results, result)
|
||||
}
|
||||
|
||||
return results, nil
|
||||
}
|
||||
|
||||
func (c *customRegexWebhook) Keywords() []string {
|
||||
return c.GetKeywords()
|
||||
}
|
||||
|
||||
// productIndices produces a permutation of indices for each length. Example:
|
||||
// productIndices(3, 2) -> [[0 0] [1 0] [2 0] [0 1] [1 1] [2 1]]. It returns
|
||||
// a slice of length no larger than maxTotalMatches.
|
||||
func productIndices(lengths ...int) [][]int {
|
||||
count := 1
|
||||
for _, l := range lengths {
|
||||
count *= l
|
||||
}
|
||||
if count == 0 {
|
||||
return nil
|
||||
}
|
||||
if count > maxTotalMatches {
|
||||
count = maxTotalMatches
|
||||
}
|
||||
|
||||
results := make([][]int, count)
|
||||
for i := 0; i < count; i++ {
|
||||
j := 1
|
||||
result := make([]int, 0, len(lengths))
|
||||
for _, l := range lengths {
|
||||
result = append(result, (i/j)%l)
|
||||
j *= l
|
||||
}
|
||||
results[i] = result
|
||||
}
|
||||
return results
|
||||
}
|
||||
|
||||
// permutateMatches converts the list of all regex matches into all possible
|
||||
// permutations selecting one from each named entry in the map. For example:
|
||||
// {"foo": [matchA, matchB], "bar": [matchC]} becomes
|
||||
// [{"foo": matchA, "bar": matchC}, {"foo": matchB, "bar": matchC}]
|
||||
func permutateMatches(regexMatches map[string][][]string) []map[string][]string {
|
||||
// Get a consistent order for names and their matching lengths.
|
||||
// The lengths are used in calculating the permutation so order matters.
|
||||
names := make([]string, 0, len(regexMatches))
|
||||
lengths := make([]int, 0, len(regexMatches))
|
||||
for key, value := range regexMatches {
|
||||
names = append(names, key)
|
||||
lengths = append(lengths, len(value))
|
||||
}
|
||||
|
||||
// Permutate all the indices for each match. For example, if "foo" has
|
||||
// [matchA, matchB] and "bar" has [matchC], we will get indices [0 0] [1 0].
|
||||
permutationIndices := productIndices(lengths...)
|
||||
|
||||
// Build {"foo": matchA, "bar": matchC} and {"foo": matchB, "bar": matchC}
|
||||
// from the indices.
|
||||
var matches []map[string][]string
|
||||
for _, permutation := range permutationIndices {
|
||||
candidate := make(map[string][]string, len(permutationIndices))
|
||||
for i, name := range names {
|
||||
candidate[name] = regexMatches[name][permutation[i]]
|
||||
}
|
||||
matches = append(matches, candidate)
|
||||
}
|
||||
|
||||
return matches
|
||||
}
|
||||
|
||||
// This function will check false positives for common test words, but also it
|
||||
// will make sure the key appears 'random' enough to be a real key.
|
||||
func isKnownFalsePositive(match map[string][]string) bool {
|
||||
for _, values := range match {
|
||||
for _, value := range values {
|
||||
if detectors.IsKnownFalsePositive(value, detectors.DefaultFalsePositives, true) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
|
|
|||
|
|
@ -102,226 +102,82 @@ func TestCustomDetectorsParsing(t *testing.T) {
|
|||
assert.Equal(t, []string{"Authorization: Bearer token"}, got.Verify[0].Headers)
|
||||
}
|
||||
|
||||
func TestCustomDetectorsKeywordValidation(t *testing.T) {
|
||||
func TestProductIndices(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
input []string
|
||||
wantErr bool
|
||||
name string
|
||||
input []int
|
||||
want [][]int
|
||||
}{
|
||||
{
|
||||
name: "Test empty list of keywords",
|
||||
input: []string{},
|
||||
wantErr: true,
|
||||
name: "zero",
|
||||
input: []int{3, 0},
|
||||
want: nil,
|
||||
},
|
||||
{
|
||||
name: "Test empty keyword",
|
||||
input: []string{""},
|
||||
wantErr: true,
|
||||
name: "one input",
|
||||
input: []int{3},
|
||||
want: [][]int{{0}, {1}, {2}},
|
||||
},
|
||||
{
|
||||
name: "Test valid keywords",
|
||||
input: []string{"hello", "world"},
|
||||
wantErr: false,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := ValidateKeywords(tt.input)
|
||||
|
||||
if (got != nil && !tt.wantErr) || (got == nil && tt.wantErr) {
|
||||
t.Errorf("ValidateKeywords() error = %v, wantErr %v", got, tt.wantErr)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCustomDetectorsRegexValidation(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
input map[string]string
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "Test list of keywords",
|
||||
input: map[string]string{
|
||||
"id_pat_example": "([a-zA-Z0-9]{32})",
|
||||
name: "two inputs",
|
||||
input: []int{3, 2},
|
||||
want: [][]int{
|
||||
{0, 0}, {1, 0}, {2, 0},
|
||||
{0, 1}, {1, 1}, {2, 1},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "Test empty list of keywords",
|
||||
input: map[string]string{},
|
||||
wantErr: true,
|
||||
name: "three inputs",
|
||||
input: []int{3, 2, 3},
|
||||
want: [][]int{
|
||||
{0, 0, 0}, {1, 0, 0}, {2, 0, 0},
|
||||
{0, 1, 0}, {1, 1, 0}, {2, 1, 0},
|
||||
{0, 0, 1}, {1, 0, 1}, {2, 0, 1},
|
||||
{0, 1, 1}, {1, 1, 1}, {2, 1, 1},
|
||||
{0, 0, 2}, {1, 0, 2}, {2, 0, 2},
|
||||
{0, 1, 2}, {1, 1, 2}, {2, 1, 2},
|
||||
},
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := ValidateRegex(tt.input)
|
||||
|
||||
if (got != nil && !tt.wantErr) || (got == nil && tt.wantErr) {
|
||||
t.Errorf("ValidateRegex() error = %v, wantErr %v", got, tt.wantErr)
|
||||
}
|
||||
got := productIndices(tt.input...)
|
||||
assert.Equal(t, tt.want, got)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCustomDetectorsVerifyEndpointValidation(t *testing.T) {
|
||||
func TestProductIndicesMax(t *testing.T) {
|
||||
got := productIndices(2, 3, 4, 5, 6)
|
||||
assert.GreaterOrEqual(t, 2*3*4*5*6, maxTotalMatches)
|
||||
assert.Equal(t, maxTotalMatches, len(got))
|
||||
}
|
||||
|
||||
func TestPermutateMatches(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
endpoint string
|
||||
unsafe bool
|
||||
wantErr bool
|
||||
name string
|
||||
input map[string][][]string
|
||||
want []map[string][]string
|
||||
}{
|
||||
{
|
||||
name: "Test http endpoint with unsafe flag",
|
||||
endpoint: "http://localhost:8000/{id_pat_example}",
|
||||
unsafe: true,
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "Test http endpoint without unsafe flag",
|
||||
endpoint: "http://localhost:8000/{id_pat_example}",
|
||||
unsafe: false,
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "Test https endpoint with unsafe flag",
|
||||
endpoint: "https://localhost:8000/{id_pat_example}",
|
||||
unsafe: true,
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "Test https endpoint without unsafe flag",
|
||||
endpoint: "https://localhost:8000/{id_pat_example}",
|
||||
unsafe: false,
|
||||
wantErr: false,
|
||||
name: "two matches",
|
||||
input: map[string][][]string{"foo": {{"matchA"}, {"matchB"}}, "bar": {{"matchC"}}},
|
||||
want: []map[string][]string{
|
||||
{"foo": {"matchA"}, "bar": {"matchC"}},
|
||||
{"foo": {"matchB"}, "bar": {"matchC"}},
|
||||
},
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := ValidateVerifyEndpoint(tt.endpoint, tt.unsafe)
|
||||
|
||||
if (got != nil && !tt.wantErr) || (got == nil && tt.wantErr) {
|
||||
t.Errorf("ValidateVerifyEndpoint() error = %v, wantErr %v", got, tt.wantErr)
|
||||
}
|
||||
got := permutateMatches(tt.input)
|
||||
assert.Equal(t, tt.want, got)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCustomDetectorsVerifyHeadersValidation(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
headers []string
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "Test single header",
|
||||
headers: []string{"Authorization: Bearer {secret_pat_example.0}"},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "Test invalid header",
|
||||
headers: []string{"Hello world"},
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "Test ugly header",
|
||||
headers: []string{"Hello:::::::world::hi:"},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "Test empty header",
|
||||
headers: []string{},
|
||||
wantErr: false,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := ValidateVerifyHeaders(tt.headers)
|
||||
|
||||
if (got != nil && !tt.wantErr) || (got == nil && tt.wantErr) {
|
||||
t.Errorf("ValidateVerifyHeaders() error = %v, wantErr %v", got, tt.wantErr)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCustomDetectorsVerifyRangeValidation(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
ranges []string
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "Test multiple mixed ranges",
|
||||
ranges: []string{"200", "300-350"},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "Test invalid non-number range",
|
||||
ranges: []string{"hi"},
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "Test invalid lower to upper range",
|
||||
ranges: []string{"200-100"},
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "Test invalid http range",
|
||||
ranges: []string{"400-1000"},
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "Test multiple ranges with invalid inputs",
|
||||
ranges: []string{"322", "hello-world", "100-200"},
|
||||
wantErr: true,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := ValidateVerifyRanges(tt.ranges)
|
||||
|
||||
if (got != nil && !tt.wantErr) || (got == nil && tt.wantErr) {
|
||||
t.Errorf("ValidateVerifyRanges() error = %v, wantErr %v", got, tt.wantErr)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCustomDetectorsVerifyRegexVarsValidation(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
regex map[string]string
|
||||
body string
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "Regex defined but not used in body",
|
||||
regex: map[string]string{"id": "[0-9]{1,10}", "id_pat_example": "([a-zA-Z0-9]{32})"},
|
||||
body: "hello world",
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "Regex defined and is used in body",
|
||||
regex: map[string]string{"id": "[0-9]{1,10}", "id_pat_example": "([a-zA-Z0-9]{32})"},
|
||||
body: "hello world {id}",
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "Regex var in body but not defined",
|
||||
regex: map[string]string{"id": "[0-9]{1,10}", "id_pat_example": "([a-zA-Z0-9]{32})"},
|
||||
body: "hello world {hello}",
|
||||
wantErr: true,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := ValidateRegexVars(tt.regex, tt.body)
|
||||
|
||||
if (got != nil && !tt.wantErr) || (got == nil && tt.wantErr) {
|
||||
t.Errorf("ValidateRegexVars() error = %v, wantErr %v", got, tt.wantErr)
|
||||
}
|
||||
})
|
||||
func BenchmarkProductIndices(b *testing.B) {
|
||||
for i := 0; i < b.N; i++ {
|
||||
_ = productIndices(3, 2, 6)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
103
pkg/custom_detectors/validation.go
Normal file
103
pkg/custom_detectors/validation.go
Normal file
|
|
@ -0,0 +1,103 @@
|
|||
package custom_detectors
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
func ValidateKeywords(keywords []string) error {
|
||||
if len(keywords) == 0 {
|
||||
return fmt.Errorf("no keywords")
|
||||
}
|
||||
|
||||
for _, keyword := range keywords {
|
||||
if len(keyword) == 0 {
|
||||
return fmt.Errorf("empty keyword")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func ValidateRegex(regex map[string]string) error {
|
||||
if len(regex) == 0 {
|
||||
return fmt.Errorf("no regex")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func ValidateVerifyEndpoint(endpoint string, unsafe bool) error {
|
||||
if len(endpoint) == 0 {
|
||||
return fmt.Errorf("no endpoint")
|
||||
}
|
||||
|
||||
if strings.HasPrefix(endpoint, "http://") && !unsafe {
|
||||
return fmt.Errorf("http endpoint must have unsafe=true")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func ValidateVerifyHeaders(headers []string) error {
|
||||
for _, header := range headers {
|
||||
if !strings.Contains(header, ":") {
|
||||
return fmt.Errorf("header %q must contain a colon", header)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func ValidateVerifyRanges(ranges []string) error {
|
||||
const httpLowerRange = 100
|
||||
const httpUpperRange = 599
|
||||
|
||||
for _, successRange := range ranges {
|
||||
if !strings.Contains(successRange, "-") {
|
||||
httpCode, err := strconv.Atoi(successRange)
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to convert http code to int %q", successRange)
|
||||
}
|
||||
|
||||
if httpCode < httpLowerRange || httpCode > httpUpperRange {
|
||||
return fmt.Errorf("invalid http status code %q", successRange)
|
||||
}
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
httpRange := strings.Split(successRange, "-")
|
||||
if len(httpRange) != 2 {
|
||||
return fmt.Errorf("invalid range format %q", successRange)
|
||||
}
|
||||
|
||||
lowerBound, err := strconv.Atoi(httpRange[0])
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to convert lower bound to int %q", successRange)
|
||||
}
|
||||
|
||||
upperBound, err := strconv.Atoi(httpRange[1])
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to convert upper bound to int %q", successRange)
|
||||
}
|
||||
|
||||
if lowerBound > upperBound {
|
||||
return fmt.Errorf("lower bound greater than upper bound on range %q", successRange)
|
||||
}
|
||||
|
||||
if lowerBound < httpLowerRange || upperBound > httpUpperRange {
|
||||
return fmt.Errorf("invalid http status code range %q", successRange)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func ValidateRegexVars(regex map[string]string, body ...string) error {
|
||||
for _, b := range body {
|
||||
matches := NewRegexVarString(b).variables
|
||||
for match := range matches {
|
||||
if _, ok := regex[match]; !ok {
|
||||
return fmt.Errorf("body %q contains an unknown variable", b)
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
227
pkg/custom_detectors/validation_test.go
Normal file
227
pkg/custom_detectors/validation_test.go
Normal file
|
|
@ -0,0 +1,227 @@
|
|||
package custom_detectors
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestCustomDetectorsKeywordValidation(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
input []string
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "Test empty list of keywords",
|
||||
input: []string{},
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "Test empty keyword",
|
||||
input: []string{""},
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "Test valid keywords",
|
||||
input: []string{"hello", "world"},
|
||||
wantErr: false,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := ValidateKeywords(tt.input)
|
||||
|
||||
if (got != nil && !tt.wantErr) || (got == nil && tt.wantErr) {
|
||||
t.Errorf("ValidateKeywords() error = %v, wantErr %v", got, tt.wantErr)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCustomDetectorsRegexValidation(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
input map[string]string
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "Test list of keywords",
|
||||
input: map[string]string{
|
||||
"id_pat_example": "([a-zA-Z0-9]{32})",
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "Test empty list of keywords",
|
||||
input: map[string]string{},
|
||||
wantErr: true,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := ValidateRegex(tt.input)
|
||||
|
||||
if (got != nil && !tt.wantErr) || (got == nil && tt.wantErr) {
|
||||
t.Errorf("ValidateRegex() error = %v, wantErr %v", got, tt.wantErr)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCustomDetectorsVerifyEndpointValidation(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
endpoint string
|
||||
unsafe bool
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "Test http endpoint with unsafe flag",
|
||||
endpoint: "http://localhost:8000/{id_pat_example}",
|
||||
unsafe: true,
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "Test http endpoint without unsafe flag",
|
||||
endpoint: "http://localhost:8000/{id_pat_example}",
|
||||
unsafe: false,
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "Test https endpoint with unsafe flag",
|
||||
endpoint: "https://localhost:8000/{id_pat_example}",
|
||||
unsafe: true,
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "Test https endpoint without unsafe flag",
|
||||
endpoint: "https://localhost:8000/{id_pat_example}",
|
||||
unsafe: false,
|
||||
wantErr: false,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := ValidateVerifyEndpoint(tt.endpoint, tt.unsafe)
|
||||
|
||||
if (got != nil && !tt.wantErr) || (got == nil && tt.wantErr) {
|
||||
t.Errorf("ValidateVerifyEndpoint() error = %v, wantErr %v", got, tt.wantErr)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCustomDetectorsVerifyHeadersValidation(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
headers []string
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "Test single header",
|
||||
headers: []string{"Authorization: Bearer {secret_pat_example.0}"},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "Test invalid header",
|
||||
headers: []string{"Hello world"},
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "Test ugly header",
|
||||
headers: []string{"Hello:::::::world::hi:"},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "Test empty header",
|
||||
headers: []string{},
|
||||
wantErr: false,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := ValidateVerifyHeaders(tt.headers)
|
||||
|
||||
if (got != nil && !tt.wantErr) || (got == nil && tt.wantErr) {
|
||||
t.Errorf("ValidateVerifyHeaders() error = %v, wantErr %v", got, tt.wantErr)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCustomDetectorsVerifyRangeValidation(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
ranges []string
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "Test multiple mixed ranges",
|
||||
ranges: []string{"200", "300-350"},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "Test invalid non-number range",
|
||||
ranges: []string{"hi"},
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "Test invalid lower to upper range",
|
||||
ranges: []string{"200-100"},
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "Test invalid http range",
|
||||
ranges: []string{"400-1000"},
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "Test multiple ranges with invalid inputs",
|
||||
ranges: []string{"322", "hello-world", "100-200"},
|
||||
wantErr: true,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := ValidateVerifyRanges(tt.ranges)
|
||||
|
||||
if (got != nil && !tt.wantErr) || (got == nil && tt.wantErr) {
|
||||
t.Errorf("ValidateVerifyRanges() error = %v, wantErr %v", got, tt.wantErr)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCustomDetectorsVerifyRegexVarsValidation(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
regex map[string]string
|
||||
body string
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "Regex defined but not used in body",
|
||||
regex: map[string]string{"id": "[0-9]{1,10}", "id_pat_example": "([a-zA-Z0-9]{32})"},
|
||||
body: "hello world",
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "Regex defined and is used in body",
|
||||
regex: map[string]string{"id": "[0-9]{1,10}", "id_pat_example": "([a-zA-Z0-9]{32})"},
|
||||
body: "hello world {id}",
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "Regex var in body but not defined",
|
||||
regex: map[string]string{"id": "[0-9]{1,10}", "id_pat_example": "([a-zA-Z0-9]{32})"},
|
||||
body: "hello world {hello}",
|
||||
wantErr: true,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := ValidateRegexVars(tt.regex, tt.body)
|
||||
|
||||
if (got != nil && !tt.wantErr) || (got == nil && tt.wantErr) {
|
||||
t.Errorf("ValidateRegexVars() error = %v, wantErr %v", got, tt.wantErr)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
@ -972,6 +972,7 @@ const (
|
|||
DetectorType_LDAP DetectorType = 901
|
||||
DetectorType_Shopify DetectorType = 902
|
||||
DetectorType_RabbitMQ DetectorType = 903
|
||||
DetectorType_CustomRegex DetectorType = 904
|
||||
)
|
||||
|
||||
// Enum value maps for DetectorType.
|
||||
|
|
@ -1877,6 +1878,7 @@ var (
|
|||
901: "LDAP",
|
||||
902: "Shopify",
|
||||
903: "RabbitMQ",
|
||||
904: "CustomRegex",
|
||||
}
|
||||
DetectorType_value = map[string]int32{
|
||||
"Alibaba": 0,
|
||||
|
|
@ -2779,6 +2781,7 @@ var (
|
|||
"LDAP": 901,
|
||||
"Shopify": 902,
|
||||
"RabbitMQ": 903,
|
||||
"CustomRegex": 904,
|
||||
}
|
||||
)
|
||||
|
||||
|
|
@ -3143,7 +3146,7 @@ var file_detectors_proto_rawDesc = []byte{
|
|||
0x0a, 0x0b, 0x44, 0x65, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a,
|
||||
0x07, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x09, 0x0a, 0x05, 0x50, 0x4c,
|
||||
0x41, 0x49, 0x4e, 0x10, 0x01, 0x12, 0x0a, 0x0a, 0x06, 0x42, 0x41, 0x53, 0x45, 0x36, 0x34, 0x10,
|
||||
0x02, 0x2a, 0xff, 0x70, 0x0a, 0x0c, 0x44, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x54, 0x79,
|
||||
0x02, 0x2a, 0x91, 0x71, 0x0a, 0x0c, 0x44, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x54, 0x79,
|
||||
0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x41, 0x6c, 0x69, 0x62, 0x61, 0x62, 0x61, 0x10, 0x00, 0x12,
|
||||
0x08, 0x0a, 0x04, 0x41, 0x4d, 0x51, 0x50, 0x10, 0x01, 0x12, 0x07, 0x0a, 0x03, 0x41, 0x57, 0x53,
|
||||
0x10, 0x02, 0x12, 0x09, 0x0a, 0x05, 0x41, 0x7a, 0x75, 0x72, 0x65, 0x10, 0x03, 0x12, 0x0a, 0x0a,
|
||||
|
|
@ -4047,11 +4050,12 @@ var file_detectors_proto_rawDesc = []byte{
|
|||
0x0a, 0x0a, 0x05, 0x52, 0x65, 0x64, 0x69, 0x73, 0x10, 0x84, 0x07, 0x12, 0x09, 0x0a, 0x04, 0x4c,
|
||||
0x44, 0x41, 0x50, 0x10, 0x85, 0x07, 0x12, 0x0c, 0x0a, 0x07, 0x53, 0x68, 0x6f, 0x70, 0x69, 0x66,
|
||||
0x79, 0x10, 0x86, 0x07, 0x12, 0x0d, 0x0a, 0x08, 0x52, 0x61, 0x62, 0x62, 0x69, 0x74, 0x4d, 0x51,
|
||||
0x10, 0x87, 0x07, 0x42, 0x3d, 0x5a, 0x3b, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f,
|
||||
0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x73, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74,
|
||||
0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x68, 0x6f, 0x67, 0x2f, 0x76, 0x33, 0x2f,
|
||||
0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x64, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x73,
|
||||
0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
|
||||
0x10, 0x87, 0x07, 0x12, 0x10, 0x0a, 0x0b, 0x43, 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x52, 0x65, 0x67,
|
||||
0x65, 0x78, 0x10, 0x88, 0x07, 0x42, 0x3d, 0x5a, 0x3b, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e,
|
||||
0x63, 0x6f, 0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x73, 0x65, 0x63, 0x75, 0x72,
|
||||
0x69, 0x74, 0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x68, 0x6f, 0x67, 0x2f, 0x76,
|
||||
0x33, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x64, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f,
|
||||
0x72, 0x73, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
|
||||
}
|
||||
|
||||
var (
|
||||
|
|
|
|||
|
|
@ -911,6 +911,7 @@ enum DetectorType {
|
|||
LDAP = 901;
|
||||
Shopify = 902;
|
||||
RabbitMQ = 903;
|
||||
CustomRegex = 904;
|
||||
}
|
||||
|
||||
message Result {
|
||||
|
|
|
|||
Loading…
Reference in a new issue