mirror of
https://github.com/trufflesecurity/trufflehog.git
synced 2024-09-20 06:31:57 +00:00
feat(mongodb): improve conn string matching
This commit is contained in:
parent
dc9c9a30b3
commit
4a2d1344a1
2 changed files with 224 additions and 24 deletions
|
@ -2,6 +2,7 @@ package mongodb
|
|||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
|
@ -12,7 +13,6 @@ import (
|
|||
"go.mongodb.org/mongo-driver/mongo/options"
|
||||
"go.mongodb.org/mongo-driver/mongo/readpref"
|
||||
"go.mongodb.org/mongo-driver/x/mongo/driver/auth"
|
||||
"go.mongodb.org/mongo-driver/x/mongo/driver/topology"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
|
@ -29,8 +29,9 @@ var _ detectors.CustomFalsePositiveChecker = (*Scanner)(nil)
|
|||
var (
|
||||
defaultTimeout = 2 * time.Second
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(`\b(mongodb(\+srv)?://[\S]{3,50}:([\S]{3,88})@[-.%\w\/:]+)\b`)
|
||||
connStrPat = regexp.MustCompile(`\b(mongodb(?:\+srv)?://(?P<username>\S{3,50}):(?P<password>\S{3,88})@(?P<host>[-.%\w]+(?::\d{1,5})?(?:,[-.%\w]+(?::\d{1,5})?)*)(?:/(?P<authdb>[\w-]+)?(?P<options>\?\w+=[\w@/.$-]+(?:&(?:amp;)?\w+=[\w@/.$-]+)*)?)?)(?:\b|$)`)
|
||||
// TODO: Add support for sharded cluster, replica set and Atlas Deployment.
|
||||
placeholderPasswordPat = regexp.MustCompile(`^[xX]+|\*+$`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
|
@ -43,11 +44,17 @@ func (s Scanner) Keywords() []string {
|
|||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
matches := connStrPat.FindAllStringSubmatch(dataStr, -1)
|
||||
|
||||
for _, match := range matches {
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
// Filter out common placeholder passwords.
|
||||
password := match[3]
|
||||
if password == "" || placeholderPasswordPat.MatchString(password) {
|
||||
continue
|
||||
}
|
||||
|
||||
// If the query string contains `&` the options will not be parsed.
|
||||
resMatch := strings.Replace(strings.TrimSpace(match[1]), "&", "&", -1)
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_MongoDB,
|
||||
Raw: []byte(resMatch),
|
||||
|
@ -61,9 +68,9 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
|||
if timeout == 0 {
|
||||
timeout = defaultTimeout
|
||||
}
|
||||
err := verifyUri(resMatch, timeout)
|
||||
s1.Verified = err == nil
|
||||
if !isErrDeterminate(err) {
|
||||
isVerified, verificationErr := verifyUri(ctx, resMatch, timeout)
|
||||
s1.Verified = isVerified
|
||||
if !isErrDeterminate(verificationErr) {
|
||||
s1.SetVerificationError(err, resMatch)
|
||||
}
|
||||
}
|
||||
|
@ -78,23 +85,14 @@ func (s Scanner) IsFalsePositive(_ detectors.Result) (bool, string) {
|
|||
}
|
||||
|
||||
func isErrDeterminate(err error) bool {
|
||||
switch e := err.(type) {
|
||||
case topology.ConnectionError:
|
||||
switch e.Unwrap().(type) {
|
||||
case *auth.Error:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
default:
|
||||
return false
|
||||
}
|
||||
var authErr *auth.Error
|
||||
return errors.As(err, &authErr)
|
||||
}
|
||||
|
||||
func verifyUri(uri string, timeout time.Duration) error {
|
||||
func verifyUri(ctx context.Context, uri string, timeout time.Duration) (bool, error) {
|
||||
parsed, err := url.Parse(uri)
|
||||
if err != nil {
|
||||
return err
|
||||
return false, err
|
||||
}
|
||||
|
||||
params := url.Values{}
|
||||
|
@ -114,16 +112,23 @@ func verifyUri(uri string, timeout time.Duration) error {
|
|||
parsed.Path = "/"
|
||||
uri = parsed.String()
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), timeout)
|
||||
ctx, cancel := context.WithTimeout(ctx, timeout)
|
||||
defer cancel()
|
||||
client, err := mongo.Connect(ctx, options.Client().SetTimeout(timeout).ApplyURI(uri))
|
||||
|
||||
clientOptions := options.Client().SetTimeout(timeout).ApplyURI(uri)
|
||||
if err = clientOptions.Validate(); err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
client, err := mongo.Connect(ctx, clientOptions)
|
||||
if err != nil {
|
||||
return err
|
||||
return false, err
|
||||
}
|
||||
defer func() {
|
||||
_ = client.Disconnect(ctx)
|
||||
}()
|
||||
return client.Ping(ctx, readpref.Primary())
|
||||
err = client.Ping(ctx, readpref.Primary())
|
||||
return err == nil, err
|
||||
}
|
||||
|
||||
func (s Scanner) Type() detectorspb.DetectorType {
|
||||
|
|
|
@ -19,6 +19,201 @@ import (
|
|||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
func TestMongoDB_Pattern(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
data string
|
||||
shouldMatch bool
|
||||
match string
|
||||
}{
|
||||
// True positives
|
||||
{
|
||||
name: "long_password",
|
||||
data: `mongodb://agenda-live:m21w7PFfRXQwfHZU1Fgx0rTX29ZBQaWMODLeAjsmyslVcMmcmy6CnLyu3byVDtdLYcCokze8lIE4KyAgSCGZxQ==@agenda-live.mongo.cosmos.azure.com:10255/?retryWrites=false&ssl=true&replicaSet=globaldb&maxIdleTimeMS=120000&appName=@agenda-live@`,
|
||||
shouldMatch: true,
|
||||
},
|
||||
{
|
||||
name: "long_password2",
|
||||
data: `mongodb://csb0230eada-2354-4c73-b3e4-8a1aaa996894:AiNtEyASbdXR5neJmTStMzKGItX2xvKuyEkcy65rviKD0ggZR19E1iVFIJ5ZAIY1xvvAiS5tOXsmACDbKDJIhQ==@csb0230eada-2354-4c73-b3e4-8a1aaa996894.mongo.cosmos.cloud-hostname.com:10255/csb-db0230eada-2354-4c73-b3e4-8a1aaa996894?ssl=true&replicaSet=globaldb&retrywrites=false&maxIdleTimeMS=120000&appName=@csb0230eada-2354-4c73-b3e4-8a1aaa996894@`,
|
||||
shouldMatch: true,
|
||||
},
|
||||
{
|
||||
name: "long_password3",
|
||||
data: `mongodb://amsdfasfsadfdfdfpshot:6xNRRsdfsdfafd9NodO8vAFFBEHidfdfdfa87QDKXdCMubACDbhfQH1g==@amssdfafdafdadbsnapshot.mongo.cosmos.azure.com:10255/?ssl=true&replicaSet=globaldb&retrywrites=false&maxIdleTimeMS=120000&appName=@amssadfasdfdbsnsdfadfapshot@`,
|
||||
shouldMatch: true,
|
||||
},
|
||||
{
|
||||
name: "single_host",
|
||||
data: `mongodb://myDBReader:D1fficultP%40ssw0rd@mongodb0.example.com`,
|
||||
shouldMatch: true,
|
||||
},
|
||||
{
|
||||
name: "single_host+port",
|
||||
data: `mongodb://myDBReader:D1fficultP%40ssw0rd@mongodb0.example.com:27017`,
|
||||
shouldMatch: true,
|
||||
},
|
||||
{
|
||||
name: "single_host+port+authdb",
|
||||
data: `mongodb://myDBReader:D1fficultP%40ssw0rd@mongodb0.example.com:27017/?authSource=admin`,
|
||||
shouldMatch: true,
|
||||
},
|
||||
{
|
||||
name: "single_host_ip",
|
||||
data: `mongodb://myDBReader:D1fficultP%40ssw0rd@192.168.74.143`,
|
||||
shouldMatch: true,
|
||||
},
|
||||
{
|
||||
name: "single_host_ip+port",
|
||||
data: `mongodb://myDBReader:D1fficultP%40ssw0rd@192.168.74.143:27017`,
|
||||
shouldMatch: true,
|
||||
},
|
||||
{
|
||||
name: "multiple_hosts_ip",
|
||||
data: `mongodb://root:root@192.168.74.143:27018,192.168.74.143:27019`,
|
||||
shouldMatch: true,
|
||||
},
|
||||
{
|
||||
name: "multiple_hosts_ip+slash",
|
||||
data: `mongodb://root:root@192.168.74.143:27018,192.168.74.143:27019/`,
|
||||
shouldMatch: true,
|
||||
},
|
||||
{
|
||||
name: "multiple_hosts+port+authdb",
|
||||
data: `mongodb://myDBReader:D1fficultP%40ssw0rd@mongodb0.example.com:27017,mongodb0.example.com:27017,mongodb0.example.com:27017/?authSource=admin`,
|
||||
shouldMatch: true,
|
||||
},
|
||||
{
|
||||
name: "multiple_hosts+options",
|
||||
data: `mongodb://username:password@mongodb1.example.com:27317,mongodb2.example.com,mongodb2.example.com:270/?connectTimeoutMS=300000&replicaSet=mySet&authSource=aDifferentAuthDB`,
|
||||
shouldMatch: true,
|
||||
},
|
||||
{
|
||||
name: "multiple_hosts2",
|
||||
data: `mongodb://prisma:risima@srv1.bu2lt.mongodb.net:27017,srv2.bu2lt.mongodb.net:27017,srv3.bu2lt.mongodb.net:27017/test?retryWrites=true&w=majority`,
|
||||
shouldMatch: true,
|
||||
},
|
||||
// TODO: These fail because the Go driver doesn't explicitly support `authMechanism=DEFAULT`[1].
|
||||
// However, this seems like a valid option[2] and I'm going to try to get that behaviour changed.
|
||||
//
|
||||
// [1] https://github.com/mongodb/mongo-go-driver/blob/master/x/mongo/driver/connstring/connstring.go#L450-L451
|
||||
// [2] https://www.mongodb.com/docs/drivers/node/current/fundamentals/authentication/mechanisms/
|
||||
{
|
||||
name: "encoded_options1",
|
||||
data: `mongodb://dave:password@localhost:27017/?authMechanism=DEFAULT&authSource=db&ssl=true"`,
|
||||
shouldMatch: true,
|
||||
match: "mongodb://dave:password@localhost:27017/?authMechanism=DEFAULT&authSource=db&ssl=true",
|
||||
},
|
||||
{
|
||||
name: "encoded_options2",
|
||||
data: `mongodb://cefapp:MdTc8Kc8DzlTE1RUl1JVDGS4zw1U1t6145sPWqeStWA50xEUKPfUCGlnk3ACkfqH6qLAwpnm9awpY1m8dg0YlQ==@cefapp.documents.azure.com:10250/?ssl=true&sslverifycertificate=false`,
|
||||
shouldMatch: true,
|
||||
match: "mongodb://cefapp:MdTc8Kc8DzlTE1RUl1JVDGS4zw1U1t6145sPWqeStWA50xEUKPfUCGlnk3ACkfqH6qLAwpnm9awpY1m8dg0YlQ==@cefapp.documents.azure.com:10250/?ssl=true&sslverifycertificate=false",
|
||||
},
|
||||
{
|
||||
name: "unix_socket",
|
||||
data: `mongodb://u%24ername:pa%24%24w%7B%7Drd@%2Ftmp%2Fmongodb-27017.sock/test`,
|
||||
shouldMatch: true,
|
||||
},
|
||||
{
|
||||
name: "dashes",
|
||||
data: `mongodb://db-user:db-password@mongodb-instance:27017/db-name`,
|
||||
shouldMatch: true,
|
||||
},
|
||||
{
|
||||
name: "protocol+srv",
|
||||
// TODO: Figure out how to handle `mongodb+srv`. It performs a DNS lookup, which fails if the host doesn't exist.
|
||||
//data: `mongodb+srv://root:randompassword@cluster0.ab1cd.mongodb.net/mydb?retryWrites=true&w=majority`,
|
||||
data: `mongodb://root:randompassword@cluster0.ab1cd.mongodb.net/mydb?retryWrites=true&w=majority`,
|
||||
shouldMatch: true,
|
||||
},
|
||||
{
|
||||
name: "0.0.0.0_host",
|
||||
data: `mongodb://username:password@0.0.0.0:27017/?authSource=admin`,
|
||||
shouldMatch: true,
|
||||
},
|
||||
{
|
||||
name: "localhost_host",
|
||||
data: `mongodb://username:password@localhost:27017/?authSource=admin`,
|
||||
shouldMatch: true,
|
||||
},
|
||||
{
|
||||
name: "127.0.0.1_host",
|
||||
data: `mongodb://username:password@127.0.0.1:27017/?authSource=admin`,
|
||||
shouldMatch: true,
|
||||
},
|
||||
{
|
||||
name: "docker_internal_host",
|
||||
data: `mongodb://username:password@host.docker.internal:27018/?authMechanism=PLAIN&tls=true&tlsCertificateKeyFile=/etc/certs/client.pem&tlsCaFile=/etc/certs/rootCA-cert.pem`,
|
||||
shouldMatch: true,
|
||||
},
|
||||
{
|
||||
name: "options_authsource_external",
|
||||
data: `mongodb://AKIAAAAAAAAAAAA:t9t2mawssecretkey@localhost:27017/?authMechanism=MONGODB-AWS&authsource=$external`,
|
||||
shouldMatch: true,
|
||||
},
|
||||
{
|
||||
name: "generic1",
|
||||
data: `mongodb://root:8b6zfr4b@fastgpt-mongo-mongodb.ns-hti44k5d.svc:27017/`,
|
||||
shouldMatch: true,
|
||||
},
|
||||
|
||||
// False positives
|
||||
{
|
||||
name: "no_password",
|
||||
data: `mongodb://mongodb0.example.com:27017/?replicaSet=myRepl`,
|
||||
shouldMatch: false,
|
||||
},
|
||||
{
|
||||
name: "empty",
|
||||
data: `mongodb://username:@mongodb0.example.com:27017/?replicaSet=myRepl`,
|
||||
shouldMatch: false,
|
||||
},
|
||||
{
|
||||
name: "placeholders_x+single_host",
|
||||
data: `mongodb://xxxx:xxxxx@xxxxxxx:3717/zkquant?replicaSet=mgset-3017917`,
|
||||
shouldMatch: false,
|
||||
},
|
||||
{
|
||||
name: "placeholders_x+multiple_hosts",
|
||||
data: `mongodb://xxxx:xxxxx@xxxxxxx:3717,xxxxxxx:3717/zkquant?replicaSet=mgset-3017917`,
|
||||
shouldMatch: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
s := Scanner{}
|
||||
|
||||
results, err := s.FromData(context.Background(), false, []byte(test.data))
|
||||
if err != nil {
|
||||
t.Errorf("MongoDB.FromData() error = %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
if test.shouldMatch {
|
||||
if len(results) == 0 {
|
||||
t.Errorf("%s: did not receive a match for '%v' when one was expected", test.name, test.data)
|
||||
return
|
||||
}
|
||||
expected := test.data
|
||||
if test.match != "" {
|
||||
expected = test.match
|
||||
}
|
||||
result := string(results[0].Raw)
|
||||
if result != expected {
|
||||
t.Errorf("%s: did not receive expected match.\n\texpected: '%s'\n\t actual: '%s'", test.name, expected, result)
|
||||
return
|
||||
}
|
||||
} else {
|
||||
if len(results) > 0 {
|
||||
t.Errorf("%s: received a match for '%v' when one wasn't wanted", test.name, test.data)
|
||||
return
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestMongoDB_FromChunk(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
|
||||
defer cancel()
|
||||
|
|
Loading…
Reference in a new issue