adding postgres detector (#2108)

* adding postgres detector

---------

Co-authored-by: Chair <chair@Chairs-MacBook-Pro.local>
Co-authored-by: ahmed <ahmed.zahran@trufflesec.com>
This commit is contained in:
dylanTruffle 2024-01-10 13:19:45 -08:00 committed by GitHub
parent fb927e011b
commit 3b4518cbab
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
5 changed files with 607 additions and 6 deletions

View file

@ -0,0 +1,254 @@
package postgres
import (
"context"
"database/sql"
"fmt"
"net/url"
"regexp"
"strings"
"time"
_ "github.com/lib/pq" // PostgreSQL driver
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
const (
defaultPort = "5432"
defaultHost = "localhost"
)
var (
_ detectors.Detector = (*Scanner)(nil)
uriPattern = regexp.MustCompile(`\b(?i)postgresql://[\S]+\b`)
hostnamePattern = regexp.MustCompile(`(?i)(?:host|server|address).{0,40}?(\b[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*\b)`)
portPattern = regexp.MustCompile(`(?i)(?:port|p).{0,40}?(\b[0-9]{1,5}\b)`)
usernamePattern = regexp.MustCompile(`(?im)(?:user|usr)\S{0,40}?[:=\s]{1,3}[ '"=]{0,1}([^:'"\s]{4,40})`)
passwordPattern = regexp.MustCompile(`(?im)(?:pass)\S{0,40}?[:=\s]{1,3}[ '"=]{0,1}([^:'"\s]{4,40})`)
)
type Scanner struct{}
func (s Scanner) Keywords() []string {
return []string{"postgres", "psql", "pghost"}
}
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) ([]detectors.Result, error) {
var results []detectors.Result
var pgURLs []url.URL
pgURLs = append(pgURLs, findUriMatches(string(data)))
pgURLs = append(pgURLs, findComponentMatches(verify, string(data))...)
for _, pgURL := range pgURLs {
if pgURL.User == nil {
continue
}
username := pgURL.User.Username()
password, _ := pgURL.User.Password()
hostport := pgURL.Host
result := detectors.Result{
DetectorType: detectorspb.DetectorType_Postgres,
Raw: []byte(hostport + username + password),
RawV2: []byte(hostport + username + password),
}
if verify {
timeoutInSeconds := getDeadlineInSeconds(ctx)
isVerified, verificationErr := verifyPostgres(&pgURL, timeoutInSeconds)
result.Verified = isVerified
result.SetVerificationError(verificationErr, password)
}
if !result.Verified && detectors.IsKnownFalsePositive(password, detectors.DefaultFalsePositives, true) {
continue
}
results = append(results, result)
}
return results, nil
}
func getDeadlineInSeconds(ctx context.Context) int {
deadline, ok := ctx.Deadline()
if !ok {
// Context does not have a deadline
return 0
}
duration := time.Until(deadline)
return int(duration.Seconds())
}
func findUriMatches(dataStr string) url.URL {
var pgURL url.URL
for _, uri := range uriPattern.FindAllString(dataStr, -1) {
pgURL, err := url.Parse(uri)
if err != nil {
continue
}
if pgURL.User != nil {
return *pgURL
}
}
return pgURL
}
// check if postgres is running
func postgresRunning(hostname, port string) bool {
connStr := fmt.Sprintf("host=%s port=%s sslmode=disable", hostname, port)
db, err := sql.Open("postgres", connStr)
if err != nil {
return false
}
defer db.Close()
return true
}
func findComponentMatches(verify bool, dataStr string) []url.URL {
usernameMatches := usernamePattern.FindAllStringSubmatch(dataStr, -1)
passwordMatches := passwordPattern.FindAllStringSubmatch(dataStr, -1)
hostnameMatches := hostnamePattern.FindAllStringSubmatch(dataStr, -1)
portMatches := portPattern.FindAllStringSubmatch(dataStr, -1)
var pgURLs []url.URL
hosts := findHosts(verify, hostnameMatches, portMatches)
for _, username := range dedupMatches(usernameMatches) {
for _, password := range dedupMatches(passwordMatches) {
for _, host := range hosts {
hostname, port := strings.Split(host, ":")[0], strings.Split(host, ":")[1]
if combinedLength := len(username) + len(password) + len(hostname); combinedLength > 255 {
continue
}
postgresURL := url.URL{
Scheme: "postgresql",
User: url.UserPassword(username, password),
Host: fmt.Sprintf("%s:%s", hostname, port),
}
pgURLs = append(pgURLs, postgresURL)
}
}
}
return pgURLs
}
// if verification is turned on, and we can confirm that postgres is running on at least one host,
// return only hosts where it's running. otherwise return all hosts.
func findHosts(verify bool, hostnameMatches, portMatches [][]string) []string {
hostnames := dedupMatches(hostnameMatches)
ports := dedupMatches(portMatches)
var hosts []string
if len(hostnames) < 1 {
hostnames = append(hostnames, defaultHost)
}
if len(ports) < 1 {
ports = append(ports, defaultPort)
}
for _, hostname := range hostnames {
for _, port := range ports {
hosts = append(hosts, fmt.Sprintf("%s:%s", hostname, port))
}
}
if verify {
var verifiedHosts []string
for _, host := range hosts {
parts := strings.Split(host, ":")
hostname, port := parts[0], parts[1]
if postgresRunning(hostname, port) {
verifiedHosts = append(verifiedHosts, host)
}
}
if len(verifiedHosts) > 0 {
return verifiedHosts
}
}
return hosts
}
// deduplicate matches in order to reduce the number of verification requests
func dedupMatches(matches [][]string) []string {
setOfMatches := make(map[string]struct{})
for _, match := range matches {
if len(match) > 1 {
setOfMatches[match[1]] = struct{}{}
}
}
var results []string
for match := range setOfMatches {
results = append(results, match)
}
return results
}
func verifyPostgres(pgURL *url.URL, timeoutInSeconds int) (bool, error) {
if pgURL.User == nil {
return false, nil
}
username := pgURL.User.Username()
password, _ := pgURL.User.Password()
hostname, port := pgURL.Hostname(), pgURL.Port()
if hostname == "" {
hostname = defaultHost
}
if port == "" {
port = defaultPort
}
sslmode := determineSSLMode(pgURL)
connStr := fmt.Sprintf("user=%s password=%s host=%s port=%s sslmode=%s", username, password, hostname, port, sslmode)
if timeoutInSeconds > 0 {
connStr = fmt.Sprintf("%s connect_timeout=%d", connStr, timeoutInSeconds)
}
db, err := sql.Open("postgres", connStr)
if err != nil {
if strings.Contains(err.Error(), "connection refused") {
// inactive host
return false, nil
}
return false, err
}
defer db.Close()
ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second)
defer cancel()
err = db.PingContext(ctx)
if err == nil {
return true, nil
} else if strings.Contains(err.Error(), "password authentication failed") || // incorrect username or password
strings.Contains(err.Error(), "connection refused") { // inactive host
return false, nil
}
// if ssl is not enabled, manually fall-back to sslmode=disable
if strings.Contains(err.Error(), "SSL is not enabled on the server") {
pgURL.RawQuery = fmt.Sprintf("sslmode=%s", "disable")
return verifyPostgres(pgURL, timeoutInSeconds)
}
return false, err
}
func determineSSLMode(pgURL *url.URL) string {
// default ssl mode is "prefer" per https://www.postgresql.org/docs/current/libpq-ssl.html
// but is currently not implemented in the driver per https://github.com/lib/pq/issues/1006
// default for the driver is "require". ideally we would use "allow" but that is also not supported by the driver.
sslmode := "require"
if sslQuery, ok := pgURL.Query()["sslmode"]; ok && len(sslQuery) > 0 {
sslmode = sslQuery[0]
}
return sslmode
}
func (s Scanner) Type() detectorspb.DetectorType {
return detectorspb.DetectorType_Postgres
}

View file

@ -0,0 +1,340 @@
//go:build detectors
// +build detectors
package postgres
import (
"bytes"
"context"
"errors"
"fmt"
"os/exec"
"strings"
"testing"
"time"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
var postgresDockerHash string
const (
postgresUser = "postgres"
postgresPass = "23201dabb56ca236f3dc6736c0f9afad"
postgresHost = "localhost"
postgresPort = "5433"
inactiveUser = "inactive"
inactivePass = "inactive"
inactivePort = "61000"
inactiveHost = "192.0.2.0"
)
func TestPostgres_FromChunk(t *testing.T) {
startPostgres()
defer stopPostgres()
type args struct {
ctx context.Context
data []byte
verify bool
}
tests := []struct {
name string
s Scanner
args args
want []detectors.Result
wantErr bool
}{
{
name: "not found",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte("You cannot find the secret within"),
verify: true,
},
want: nil,
wantErr: false,
},
{
name: "found with seperated credentials, verified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf(`
POSTGRES_USER=%s
POSTGRES_PASSWORD=%s
POSTGRES_ADDRESS=%s
POSTGRES_PORT=%s
`, postgresUser, postgresPass, postgresHost, postgresPort)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_Postgres,
Verified: true,
},
},
wantErr: false,
},
{
name: "found with single line credentials, verified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf(`postgresql://%s:%s@%s:%s/postgres`, postgresUser, postgresPass, postgresHost, postgresPort)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_Postgres,
Verified: true,
},
},
wantErr: false,
},
{
name: "found with json credentials, verified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf(
`DB_CONFIG={"user": "%s", "password": "%s", "host": "%s", "port": "%s", "database": "postgres"}`, postgresUser, postgresPass, postgresHost, postgresPort)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_Postgres,
Verified: true,
},
},
wantErr: false,
},
{
name: "found with seperated credentials, unverified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf(`
POSTGRES_USER=%s
POSTGRES_PASSWORD=%s
POSTGRES_ADDRESS=%s
POSTGRES_PORT=%s
`, postgresUser, inactivePass, postgresHost, postgresPort)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_Postgres,
Verified: false,
},
},
wantErr: false,
},
{
name: "found with seperated credentials - no port, unverified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf(`
POSTGRES_USER=%s
POSTGRES_PASSWORD=%s
POSTGRES_ADDRESS=%s
`, postgresUser, inactivePass, postgresHost)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_Postgres,
Verified: false,
},
},
wantErr: false,
},
{
name: "found with single line credentials, unverified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf(`postgresql://%s:%s@%s:%s/postgres`, postgresUser, inactivePass, postgresHost, postgresPort)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_Postgres,
Verified: false,
},
},
wantErr: false,
},
{
name: "found with json credentials, unverified - inactive password",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf(
`DB_CONFIG={"user": "%s", "password": "%s", "host": "%s", "port": "%s", "database": "postgres"}`, postgresUser, inactivePass, postgresHost, postgresPort)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_Postgres,
Verified: false,
},
},
wantErr: false,
},
{
name: "found with json credentials, unverified - inactive user",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf(
`DB_CONFIG={"user": "%s", "password": "%s", "host": "%s", "port": "%s", "database": "postgres"}`, inactiveUser, postgresPass, postgresHost, postgresPort)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_Postgres,
Verified: false,
},
},
wantErr: false,
},
{
name: "found, unverified due to error - inactive port",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf(`postgresql://%s:%s@%s:%s/postgres`, postgresUser, postgresPass, postgresHost, inactivePort)),
verify: true,
},
want: func() []detectors.Result {
r := detectors.Result{
DetectorType: detectorspb.DetectorType_Postgres,
Verified: false,
}
return []detectors.Result{r}
}(),
wantErr: false,
},
// This test seems take a long time to run (70s+) even with the timeout set to 1s. It's not clear why.
{
name: "found, unverified due to error - inactive host",
s: Scanner{},
args: func() args {
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
defer cancel()
return args{
ctx: ctx,
data: []byte(fmt.Sprintf(`postgresql://%s:%s@%s:%s/postgres`, postgresUser, postgresPass, inactiveHost, postgresPort)),
verify: true,
}
}(),
want: func() []detectors.Result {
r := detectors.Result{
DetectorType: detectorspb.DetectorType_Postgres,
Verified: false,
}
r.SetVerificationError(errors.New("i/o timeout"))
return []detectors.Result{r}
}(),
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
s := Scanner{}
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
if (err != nil) != tt.wantErr {
t.Errorf("postgres.FromData() error = %v, wantErr %v", err, tt.wantErr)
return
}
for i := range got {
if len(got[i].Raw) == 0 {
t.Fatalf("no raw secret present: \n %+v", got[i])
}
gotErr := ""
if got[i].VerificationError() != nil {
gotErr = got[i].VerificationError().Error()
}
wantErr := ""
if tt.want[i].VerificationError() != nil {
wantErr = tt.want[i].VerificationError().Error()
}
if gotErr != wantErr {
t.Fatalf("wantVerificationError = %v, verification error = %v", tt.want[i].VerificationError(), got[i].VerificationError())
}
}
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError")
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
t.Errorf("Postgres.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
}
})
}
}
func dockerLogLine(hash string, needle string) chan struct{} {
ch := make(chan struct{}, 1)
go func() {
for {
out, err := exec.Command("docker", "logs", hash).CombinedOutput()
if err != nil {
panic(err)
}
if strings.Contains(string(out), needle) {
ch <- struct{}{}
return
}
time.Sleep(1 * time.Second)
}
}()
return ch
}
func startPostgres() error {
cmd := exec.Command(
"docker", "run", "--rm", "-p", postgresPort+":"+defaultPort,
"-e", "POSTGRES_PASSWORD="+postgresPass,
"-e", "POSTGRES_USER="+postgresUser,
"-d", "postgres",
)
fmt.Println(cmd.String())
out, err := cmd.Output()
if err != nil {
return err
}
postgresDockerHash = string(bytes.TrimSpace(out))
select {
case <-dockerLogLine(postgresDockerHash, "PostgreSQL init process complete; ready for start up."):
return nil
case <-time.After(30 * time.Second):
stopPostgres()
return errors.New("timeout waiting for postgres database to be ready")
}
}
func stopPostgres() {
exec.Command("docker", "kill", postgresDockerHash).Run()
}
func BenchmarkFromData(benchmark *testing.B) {
ctx := context.Background()
s := Scanner{}
for name, data := range detectors.MustGetBenchmarkData() {
benchmark.Run(name, func(b *testing.B) {
b.ResetTimer()
for n := 0; n < b.N; n++ {
_, err := s.FromData(ctx, false, data)
if err != nil {
b.Fatal(err)
}
}
})
}
}

View file

@ -524,6 +524,7 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/positionstack"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/postageapp"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/postbacks"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/postgres"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/posthog"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/postman"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/postmark"
@ -1496,6 +1497,7 @@ func DefaultDetectors() []detectors.Detector {
speechtextai.Scanner{},
databox.Scanner{},
postbacks.Scanner{},
postgres.Scanner{},
collect2.Scanner{},
uclassify.Scanner{},
holistic.Scanner{},

View file

@ -1053,6 +1053,7 @@ const (
DetectorType_Overloop DetectorType = 965
DetectorType_Ngrok DetectorType = 966
DetectorType_Replicate DetectorType = 967
DetectorType_Postgres DetectorType = 968
)
// Enum value maps for DetectorType.
@ -2022,6 +2023,7 @@ var (
965: "Overloop",
966: "Ngrok",
967: "Replicate",
968: "Postgres",
}
DetectorType_value = map[string]int32{
"Alibaba": 0,
@ -2988,6 +2990,7 @@ var (
"Overloop": 965,
"Ngrok": 966,
"Replicate": 967,
"Postgres": 968,
}
)
@ -3366,7 +3369,7 @@ var file_detectors_proto_rawDesc = []byte{
0x44, 0x65, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x55,
0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x09, 0x0a, 0x05, 0x50, 0x4c, 0x41, 0x49,
0x4e, 0x10, 0x01, 0x12, 0x0a, 0x0a, 0x06, 0x42, 0x41, 0x53, 0x45, 0x36, 0x34, 0x10, 0x02, 0x12,
0x09, 0x0a, 0x05, 0x55, 0x54, 0x46, 0x31, 0x36, 0x10, 0x03, 0x2a, 0xf3, 0x79, 0x0a, 0x0c, 0x44,
0x09, 0x0a, 0x05, 0x55, 0x54, 0x46, 0x31, 0x36, 0x10, 0x03, 0x2a, 0x82, 0x7a, 0x0a, 0x0c, 0x44,
0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x41,
0x6c, 0x69, 0x62, 0x61, 0x62, 0x61, 0x10, 0x00, 0x12, 0x08, 0x0a, 0x04, 0x41, 0x4d, 0x51, 0x50,
0x10, 0x01, 0x12, 0x07, 0x0a, 0x03, 0x41, 0x57, 0x53, 0x10, 0x02, 0x12, 0x09, 0x0a, 0x05, 0x41,
@ -4342,11 +4345,12 @@ var file_detectors_proto_rawDesc = []byte{
0x63, 0x65, 0x10, 0xc4, 0x07, 0x12, 0x0d, 0x0a, 0x08, 0x4f, 0x76, 0x65, 0x72, 0x6c, 0x6f, 0x6f,
0x70, 0x10, 0xc5, 0x07, 0x12, 0x0a, 0x0a, 0x05, 0x4e, 0x67, 0x72, 0x6f, 0x6b, 0x10, 0xc6, 0x07,
0x12, 0x0e, 0x0a, 0x09, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x65, 0x10, 0xc7, 0x07,
0x42, 0x3d, 0x5a, 0x3b, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x74,
0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x73, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x2f, 0x74,
0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x68, 0x6f, 0x67, 0x2f, 0x76, 0x33, 0x2f, 0x70, 0x6b, 0x67,
0x2f, 0x70, 0x62, 0x2f, 0x64, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x73, 0x70, 0x62, 0x62,
0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
0x12, 0x0d, 0x0a, 0x08, 0x50, 0x6f, 0x73, 0x74, 0x67, 0x72, 0x65, 0x73, 0x10, 0xc8, 0x07, 0x42,
0x3d, 0x5a, 0x3b, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x74, 0x72,
0x75, 0x66, 0x66, 0x6c, 0x65, 0x73, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x2f, 0x74, 0x72,
0x75, 0x66, 0x66, 0x6c, 0x65, 0x68, 0x6f, 0x67, 0x2f, 0x76, 0x33, 0x2f, 0x70, 0x6b, 0x67, 0x2f,
0x70, 0x62, 0x2f, 0x64, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x73, 0x70, 0x62, 0x62, 0x06,
0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
}
var (

View file

@ -976,6 +976,7 @@ enum DetectorType {
Overloop = 965;
Ngrok = 966;
Replicate = 967;
Postgres = 968;
}
message Result {