Mirrors/trufflehog
2
0
Fork 0
mirror of https://github.com/trufflesecurity/trufflehog.git synced 2024-11-13 00:17:18 +00:00
Code Issues Projects Releases Packages Wiki Activity

merge main

Browse source
This commit is contained in:
Ahrav Dutta 2024-11-04 08:59:08 -08:00
commit 7b3f28c55d
161 changed files with 6946 additions and 4178 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

24
.github/workflows/snifftest.yml vendored
View file

@ -1,24 +0,0 @@
name: Snifftest
on:
push:
tags:
- v*
branches:
- main
permissions:
contents: read
pull-requests: read
jobs:
Snifftest:
name: Run Snifftest
runs-on: ubuntu-latest
steps:
- uses: actions/setup-go@v5
with:
go-version: "1.23"
- uses: actions/checkout@v4
- name: Run Snifftest
run: make snifftest

1
.gitignore vendored
View file

@ -9,3 +9,4 @@ tmp/go-test.json
.captain/detectors/timings.yaml
.captain/detectors/quarantines.yaml
.captain/detectors/flakes.yaml
.vscode

2
.goreleaser.yml
View file

@ -2,7 +2,7 @@ version: 2
builds:
- binary: trufflehog
ldflags:
- -X 'github.com/trufflesecurity/trufflehog/v3/pkg/version.BuildVersion={{ .Version }}'
- -s -w -X 'github.com/trufflesecurity/trufflehog/v3/pkg/version.BuildVersion={{ .Version }}'
env: [CGO_ENABLED=0]
goos:
- linux

14
CODEOWNERS
View file

@ -1,9 +1,17 @@
# catch-all
* @trufflesecurity/product-eng
# teams
pkg/sources/ @trufflesecurity/backend
pkg/detectors/ @trufflesecurity/detection
# Scanning
pkg/decoders/ @trufflesecurity/Scanning
pkg/engine/ @trufflesecurity/Scanning
pkg/gitparse/ @trufflesecurity/Scanning
pkg/giturl/ @trufflesecurity/Scanning
pkg/handlers/ @trufflesecurity/Scanning
pkg/iobuf/ @trufflesecurity/Scanning
pkg/sanitizer/ @trufflesecurity/Scanning
pkg/sources/ @trufflesecurity/Scanning
pkg/writers/ @trufflesecurity/Scanning
proto/ @trufflesecurity/Scanning
# critical detectors
pkg/detectors/aws/ @trufflesecurity/backend

23
CONTRIBUTING.md
View file

@ -11,3 +11,26 @@ Contributors need to [sign our CLA](https://cla-assistant.io/trufflesecurity/tru
## Adding new secret detectors
We have published some [documentation and tooling to get started on adding new secret detectors](hack/docs/Adding_Detectors_external.md). Let's improve detection together!
## Logging in TruffleHog
**Use fields over format strings**. For structured logging, fields allow us to better filter and search through logs than embedding data in the message.
**Differentiate logs coming from dependencies**. This can be done with a `"dep"` field that gets passed to the library. Sometimes it’s not possible to do this.
Limit log levels to _**info**_ (indicate normal or expected operation) and _**error**_ (functionality is impeded and should be checked by an engineer)
**Choose an appropriate verbosity level**
```
0. — logs we always want to see
1. — logs we could possibly want to turn off
2. — logs that are useful for debugging
3. — frequently called logs that may produce a lot of output
4. — extremely verbose logs or logs containing sensitive information
5. — ultimate verbosity
```
Example: `Logger().V(2).Info("skipping file: extension is ignored", "ext", mimeExt)`
**Either log an error or return it**. Doing one or the other will help defer logging for when there is more context for it and prevent duplicate “bubbling up” logs.
**Log contextual information**. Every log emitted should contain this context via fields to easily filter and search.

3
Makefile
View file

@ -61,8 +61,5 @@ release-protos-image:
docker buildx build --push --platform=linux/amd64,linux/arm64 \
-t ${PROTOS_IMAGE} -f hack/Dockerfile.protos .
snifftest:
./hack/snifftest/snifftest.sh
test-release:
goreleaser release --clean --skip-publish --snapshot

20
README.md
View file

@ -36,6 +36,26 @@ We take the revenue from the enterprise product to fund more awesome open source
</div>
# What is TruffleHog 🐽
TruffleHog is the most powerful secrets **Discovery, Classification, Validation,** and **Analysis** tool. In this context secret refers to a credential a machine uses to authenticate itself to another machine. This includes API keys, database passwords, private encryption keys, and more...
## Discovery 🔍
TruffleHog can look for secrets in many places including Git, chats, wikis, logs, API testing platforms, object stores, filesystems and more
## Classification 📁
TruffleHog classifies over 800 secret types, mapping them back to the specific identity they belong to. Is it an AWS secret? Stripe secret? Cloudflare secret? Postgres password? SSL Private key? Sometimes its hard to tell looking at it, so TruffleHog classifies everything it finds.
## Validation ✅
For every secret TruffleHog can classify, it can also log in to confirm if that secret is live or not. This step is critical to know if there’s an active present danger or not.
## Analysis 🔬
For the 20 some of the most commonly leaked out credential types, instead of sending one request to check if the secret can log in, TruffleHog can send many requests to learn everything there is to know about the secret. Who created it? What resources can it access? What permissions does it have on those resources?
# :loudspeaker: Join Our Community
Have questions? Feedback? Jump in slack or discord and hang out with us

58
go.mod
View file

@ -14,10 +14,10 @@ replace github.com/coinbase/waas-client-library-go => github.com/trufflesecurity
replace github.com/STARRY-S/zip => github.com/STARRY-S/zip v0.1.0
require (
cloud.google.com/go/secretmanager v1.14.1
cloud.google.com/go/storage v1.44.0
cloud.google.com/go/secretmanager v1.14.2
cloud.google.com/go/storage v1.46.0
github.com/Azure/go-autorest/autorest/azure/auth v0.5.13
github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2
github.com/AzureAD/microsoft-authentication-library-for-go v1.2.3
github.com/BobuSumisu/aho-corasick v1.0.3
github.com/TheZeroSlave/zapsentry v1.23.0
github.com/adrg/strutil v0.3.1
@ -27,7 +27,7 @@ require (
github.com/bill-rich/go-syslog v0.0.0-20220413021637-49edb52a574c
github.com/bitfinexcom/bitfinex-api-go v0.0.0-20210608095005-9e0b26f200fb
github.com/bradleyfalzon/ghinstallation/v2 v2.11.0
github.com/brianvoe/gofakeit/v7 v7.0.4
github.com/brianvoe/gofakeit/v7 v7.1.2
github.com/charmbracelet/bubbles v0.18.0
github.com/charmbracelet/bubbletea v0.27.0
github.com/charmbracelet/glamour v0.7.0
@ -38,10 +38,10 @@ require (
github.com/dustin/go-humanize v1.0.1
github.com/elastic/go-elasticsearch/v8 v8.15.0
github.com/envoyproxy/protoc-gen-validate v1.1.0
github.com/fatih/color v1.17.0
github.com/fatih/color v1.18.0
github.com/felixge/fgprof v0.9.5
github.com/gabriel-vasile/mimetype v1.4.5
github.com/getsentry/sentry-go v0.29.0
github.com/gabriel-vasile/mimetype v1.4.6
github.com/getsentry/sentry-go v0.29.1
github.com/go-errors/errors v1.5.1
github.com/go-git/go-git/v5 v5.12.0
github.com/go-ldap/ldap/v3 v3.4.8
@ -51,7 +51,7 @@ require (
github.com/go-sql-driver/mysql v1.8.1
github.com/gobwas/glob v0.2.3
github.com/golang-jwt/jwt v3.2.2+incompatible
github.com/golang-jwt/jwt/v4 v4.5.0
github.com/golang-jwt/jwt/v4 v4.5.1
github.com/google/go-cmp v0.6.0
github.com/google/go-containerregistry v0.20.2
github.com/google/go-github/v66 v66.0.0
@ -60,7 +60,7 @@ require (
github.com/hashicorp/go-retryablehttp v0.7.7
github.com/hashicorp/golang-lru/v2 v2.0.7
github.com/jedib0t/go-pretty v4.3.0+incompatible
github.com/jedib0t/go-pretty/v6 v6.6.0
github.com/jedib0t/go-pretty/v6 v6.6.1
github.com/jlaffaye/ftp v0.2.0
github.com/joho/godotenv v1.5.1
github.com/jpillora/overseer v1.1.6
@ -78,29 +78,29 @@ require (
github.com/patrickmn/go-cache v2.1.0+incompatible
github.com/paulbellamy/ratecounter v0.2.0
github.com/pkg/errors v0.9.1
github.com/prometheus/client_golang v1.20.4
github.com/prometheus/client_golang v1.20.5
github.com/rabbitmq/amqp091-go v1.10.0
github.com/sassoftware/go-rpmutils v0.4.0
github.com/schollz/progressbar/v3 v3.16.1
github.com/schollz/progressbar/v3 v3.17.0
github.com/sendgrid/sendgrid-go v3.16.0+incompatible
github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3
github.com/shuheiktgw/go-travis v0.3.1
github.com/snowflakedb/gosnowflake v1.11.2
github.com/snowflakedb/gosnowflake v1.12.0
github.com/stretchr/testify v1.9.0
github.com/tailscale/depaware v0.0.0-20240804103531-585336c3e1b3
github.com/testcontainers/testcontainers-go v0.33.0
github.com/testcontainers/testcontainers-go/modules/elasticsearch v0.33.0
github.com/testcontainers/testcontainers-go/modules/mongodb v0.33.0
github.com/testcontainers/testcontainers-go/modules/mssql v0.33.0
github.com/testcontainers/testcontainers-go/modules/mysql v0.33.0
github.com/testcontainers/testcontainers-go/modules/postgres v0.33.0
github.com/tailscale/depaware v0.0.0-20241028160002-3d7f3b30ed0e
github.com/testcontainers/testcontainers-go v0.34.0
github.com/testcontainers/testcontainers-go/modules/elasticsearch v0.34.0
github.com/testcontainers/testcontainers-go/modules/mongodb v0.34.0
github.com/testcontainers/testcontainers-go/modules/mssql v0.34.0
github.com/testcontainers/testcontainers-go/modules/mysql v0.34.0
github.com/testcontainers/testcontainers-go/modules/postgres v0.34.0
github.com/trufflesecurity/disk-buffer-reader v0.2.1
github.com/wasilibs/go-re2 v1.7.0
github.com/xanzy/go-gitlab v0.111.0
github.com/xanzy/go-gitlab v0.112.0
github.com/xo/dburl v0.23.2
go.mongodb.org/mongo-driver v1.17.1
go.uber.org/automaxprocs v1.6.0
go.uber.org/mock v0.4.0
go.uber.org/mock v0.5.0
go.uber.org/zap v1.27.0
golang.org/x/crypto v0.28.0
golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c
@ -108,7 +108,7 @@ require (
golang.org/x/oauth2 v0.23.0
golang.org/x/sync v0.8.0
golang.org/x/text v0.19.0
google.golang.org/api v0.200.0
google.golang.org/api v0.204.0
google.golang.org/protobuf v1.35.1
gopkg.in/h2non/gock.v1 v1.1.2
gopkg.in/yaml.v2 v2.4.0
@ -120,9 +120,9 @@ require (
require (
cel.dev/expr v0.16.1 // indirect
cloud.google.com/go v0.115.1 // indirect
cloud.google.com/go/auth v0.9.8 // indirect
cloud.google.com/go/auth/oauth2adapt v0.2.4 // indirect
cloud.google.com/go v0.116.0 // indirect
cloud.google.com/go/auth v0.10.0 // indirect
cloud.google.com/go/auth/oauth2adapt v0.2.5 // indirect
cloud.google.com/go/compute/metadata v0.5.2 // indirect
cloud.google.com/go/iam v1.2.1 // indirect
cloud.google.com/go/longrunning v0.6.1 // indirect
@ -178,7 +178,7 @@ require (
github.com/couchbase/gocbcoreps v0.1.3 // indirect
github.com/couchbase/goprotostellar v1.0.2 // indirect
github.com/couchbaselabs/gocbconnstr/v2 v2.0.0-20240607131231-fb385523de28 // indirect
github.com/cpuguy83/dockercfg v0.3.1 // indirect
github.com/cpuguy83/dockercfg v0.3.2 // indirect
github.com/cyphar/filepath-securejoin v0.2.4 // indirect
github.com/danieljoos/wincred v1.1.2 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
@ -319,9 +319,9 @@ require (
golang.org/x/time v0.7.0 // indirect
golang.org/x/tools v0.26.0 // indirect
golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect
google.golang.org/genproto v0.0.0-20241007155032-5fefd90f89a9 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20240930140551-af27646dc61f // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20241007155032-5fefd90f89a9 // indirect
google.golang.org/genproto v0.0.0-20241021214115-324edc3d5d38 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20241015192408-796eee8c2d53 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20241021214115-324edc3d5d38 // indirect
google.golang.org/grpc v1.67.1 // indirect
google.golang.org/grpc/stats/opentelemetry v0.0.0-20240907200651-3ffb98b2c93a // indirect
gopkg.in/warnings.v0 v0.1.2 // indirect

74
go.sum
View file

@ -11,12 +11,20 @@ cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6T
cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M=
cloud.google.com/go v0.115.1 h1:Jo0SM9cQnSkYfp44+v+NQXHpcHqlnRJk2qxh6yvxxxQ=
cloud.google.com/go v0.115.1/go.mod h1:DuujITeaufu3gL68/lOFIirVNJwQeyf5UXyi+Wbgknc=
cloud.google.com/go v0.116.0 h1:B3fRrSDkLRt5qSHWe40ERJvhvnQwdZiHu0bJOpldweE=
cloud.google.com/go v0.116.0/go.mod h1:cEPSRWPzZEswwdr9BxE6ChEn01dWlTaF05LiC2Xs70U=
cloud.google.com/go/auth v0.9.5 h1:4CTn43Eynw40aFVr3GpPqsQponx2jv0BQpjvajsbbzw=
cloud.google.com/go/auth v0.9.5/go.mod h1:Xo0n7n66eHyOWWCnitop6870Ilwo3PiZyodVkkH1xWM=
cloud.google.com/go/auth v0.9.8 h1:+CSJ0Gw9iVeSENVCKJoLHhdUykDgXSc4Qn+gu2BRtR8=
cloud.google.com/go/auth v0.9.8/go.mod h1:xxA5AqpDrvS+Gkmo9RqrGGRh6WSNKKOXhY3zNOr38tI=
cloud.google.com/go/auth v0.9.9 h1:BmtbpNQozo8ZwW2t7QJjnrQtdganSdmqeIBxHxNkEZQ=
cloud.google.com/go/auth v0.9.9/go.mod h1:xxA5AqpDrvS+Gkmo9RqrGGRh6WSNKKOXhY3zNOr38tI=
cloud.google.com/go/auth v0.10.0 h1:tWlkvFAh+wwTOzXIjrwM64karR1iTBZ/GRr0S/DULYo=
cloud.google.com/go/auth v0.10.0/go.mod h1:xxA5AqpDrvS+Gkmo9RqrGGRh6WSNKKOXhY3zNOr38tI=
cloud.google.com/go/auth/oauth2adapt v0.2.4 h1:0GWE/FUsXhf6C+jAkWgYm7X9tK8cuEIfy19DBn6B6bY=
cloud.google.com/go/auth/oauth2adapt v0.2.4/go.mod h1:jC/jOpwFP6JBxhB3P5Rr0a9HLMC/Pe3eaL4NmdvqPtc=
cloud.google.com/go/auth/oauth2adapt v0.2.5 h1:2p29+dePqsCHPP1bqDJcKj4qxRyYCcbzKpFyKGt3MTk=
cloud.google.com/go/auth/oauth2adapt v0.2.5/go.mod h1:AlmsELtlEBnaNTL7jCj8VQFLy6mbZv0s4Q7NGBeQ5E8=
cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=
cloud.google.com/go/compute/metadata v0.5.2 h1:UxK4uu/Tn+I3p2dYWTfiX4wva7aYlKixAHn3fyqngqo=
@ -36,10 +44,16 @@ cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2k
cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw=
cloud.google.com/go/secretmanager v1.14.1 h1:xlWSIg8rtBn5qCr2f3XtQP19+5COyf/ll49SEvi/0vM=
cloud.google.com/go/secretmanager v1.14.1/go.mod h1:L+gO+u2JA9CCyXpSR8gDH0o8EV7i/f0jdBOrUXcIV0U=
cloud.google.com/go/secretmanager v1.14.2 h1:2XscWCfy//l/qF96YE18/oUaNJynAx749Jg3u0CjQr8=
cloud.google.com/go/secretmanager v1.14.2/go.mod h1:Q18wAPMM6RXLC/zVpWTlqq2IBSbbm7pKBlM3lCKsmjw=
cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos=
cloud.google.com/go/storage v1.44.0 h1:abBzXf4UJKMmQ04xxJf9dYM/fNl24KHoTuBjyJDX2AI=
cloud.google.com/go/storage v1.44.0/go.mod h1:wpPblkIuMP5jCB/E48Pz9zIo2S/zD8g+ITmxKkPCITE=
cloud.google.com/go/storage v1.45.0 h1:5av0QcIVj77t+44mV4gffFC/LscFRUhto6UBMB5SimM=
cloud.google.com/go/storage v1.45.0/go.mod h1:wpPblkIuMP5jCB/E48Pz9zIo2S/zD8g+ITmxKkPCITE=
cloud.google.com/go/storage v1.46.0 h1:OTXISBpFd8KaA2ClT3K3oRk8UGOcTHtrZ1bW88xKiic=
cloud.google.com/go/storage v1.46.0/go.mod h1:lM+gMAW91EfXIeMTBmixRsKL/XCxysytoAgduVikjMk=
cloud.google.com/go/trace v1.11.0 h1:UHX6cOJm45Zw/KIbqHe4kII8PupLt/V5tscZUkeiJVI=
cloud.google.com/go/trace v1.11.0/go.mod h1:Aiemdi52635dBR7o3zuc9lLjXo3BwGaChEjCa3tJNmM=
dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk=
@ -89,6 +103,8 @@ github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 h1:mFRzDkZVAjdal+
github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU=
github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 h1:XHOnouVk1mxXfQidrMEnLlPk9UMeRtyBTnEFtxkV0kU=
github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI=
github.com/AzureAD/microsoft-authentication-library-for-go v1.2.3 h1:6LyjnnaLpcOKK0fbYisI+mb8CE7iNe7i89nMNQxFxs8=
github.com/AzureAD/microsoft-authentication-library-for-go v1.2.3/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI=
github.com/BobuSumisu/aho-corasick v1.0.3 h1:uuf+JHwU9CHP2Vx+wAy6jcksJThhJS9ehR8a+4nPE9g=
github.com/BobuSumisu/aho-corasick v1.0.3/go.mod h1:hm4jLcvZKI2vRF2WDU1N4p/jpWtpOzp3nLmi9AzX/XE=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
@ -168,6 +184,10 @@ github.com/bradleyfalzon/ghinstallation/v2 v2.11.0 h1:R9d0v+iobRHSaE4wKUnXFiZp53
github.com/bradleyfalzon/ghinstallation/v2 v2.11.0/go.mod h1:0LWKQwOHewXO/1acI6TtyE0Xc4ObDb2rFN7eHBAG71M=
github.com/brianvoe/gofakeit/v7 v7.0.4 h1:Mkxwz9jYg8Ad8NvT9HA27pCMZGFQo08MK6jD0QTKEww=
github.com/brianvoe/gofakeit/v7 v7.0.4/go.mod h1:QXuPeBw164PJCzCUZVmgpgHJ3Llj49jSLVkKPMtxtxA=
github.com/brianvoe/gofakeit/v7 v7.1.1 h1:/DEG+f/mFtqqNjhZ0AXA0aDzrnfE85AcAKVE+mMdxAQ=
github.com/brianvoe/gofakeit/v7 v7.1.1/go.mod h1:QXuPeBw164PJCzCUZVmgpgHJ3Llj49jSLVkKPMtxtxA=
github.com/brianvoe/gofakeit/v7 v7.1.2 h1:vSKaVScNhWVpf1rlyEKSvO8zKZfuDtGqoIHT//iNNb8=
github.com/brianvoe/gofakeit/v7 v7.1.2/go.mod h1:QXuPeBw164PJCzCUZVmgpgHJ3Llj49jSLVkKPMtxtxA=
github.com/bwesterb/go-ristretto v1.2.3/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0=
github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM=
github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
@ -232,6 +252,8 @@ github.com/couchbaselabs/gocbconnstr/v2 v2.0.0-20240607131231-fb385523de28 h1:lh
github.com/couchbaselabs/gocbconnstr/v2 v2.0.0-20240607131231-fb385523de28/go.mod h1:o7T431UOfFVHDNvMBUmUxpHnhivwv7BziUao/nMl81E=
github.com/cpuguy83/dockercfg v0.3.1 h1:/FpZ+JaygUR/lZP2NlFI2DVfrOEMAIKP5wWEJdoYe9E=
github.com/cpuguy83/dockercfg v0.3.1/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc=
github.com/cpuguy83/dockercfg v0.3.2 h1:DlJTyZGBDlXqUZ2Dk2Q3xHs/FtnooJJVaad2S9GKorA=
github.com/cpuguy83/dockercfg v0.3.2/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc=
github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY=
github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4=
@ -289,6 +311,8 @@ github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f h1:Y/CXytFA4m6
github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f/go.mod h1:vw97MGsxSvLiUE2X8qFplwetxpGLQrlU1Q9AUEIzCaM=
github.com/fatih/color v1.17.0 h1:GlRw1BRJxkpqUCBKzKOw098ed57fEsKeNjpTe3cSjK4=
github.com/fatih/color v1.17.0/go.mod h1:YZ7TlrGPkiz6ku9fK3TLD/pl3CpsiFyu8N92HLgmosI=
github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM=
github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU=
github.com/felixge/fgprof v0.9.5 h1:8+vR6yu2vvSKn08urWyEuxx75NWPEvybbkBirEpsbVY=
github.com/felixge/fgprof v0.9.5/go.mod h1:yKl+ERSa++RYOs32d8K6WEXCB4uXdLls4ZaZPpayhMM=
github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
@ -301,8 +325,12 @@ github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4
github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw=
github.com/gabriel-vasile/mimetype v1.4.5 h1:J7wGKdGu33ocBOhGy0z653k/lFKLFDPJMG8Gql0kxn4=
github.com/gabriel-vasile/mimetype v1.4.5/go.mod h1:ibHel+/kbxn9x2407k1izTA1S81ku1z/DlgOW2QE0M4=
github.com/gabriel-vasile/mimetype v1.4.6 h1:3+PzJTKLkvgjeTbts6msPJt4DixhT4YtFNf1gtGe3zc=
github.com/gabriel-vasile/mimetype v1.4.6/go.mod h1:JX1qVKqZd40hUPpAfiNTe0Sne7hdfKSbOqqmkq8GCXc=
github.com/getsentry/sentry-go v0.29.0 h1:YtWluuCFg9OfcqnaujpY918N/AhCCwarIDWOYSBAjCA=
github.com/getsentry/sentry-go v0.29.0/go.mod h1:jhPesDAL0Q0W2+2YEuVOvdWmVtdsr1+jtBrlDEVWwLY=
github.com/getsentry/sentry-go v0.29.1 h1:DyZuChN8Hz3ARxGVV8ePaNXh1dQ7d76AiB117xcREwA=
github.com/getsentry/sentry-go v0.29.1/go.mod h1:x3AtIzN01d6SiWkderzaH28Tm0lgkafpJ5Bm3li39O0=
github.com/gliderlabs/ssh v0.3.7 h1:iV3Bqi942d9huXnzEF2Mt+CY9gLu8DNM4Obd+8bODRE=
github.com/gliderlabs/ssh v0.3.7/go.mod h1:zpHEXBstFnQYtGnB8k8kQLol82umzn/2/snG7alWVD8=
github.com/go-asn1-ber/asn1-ber v1.5.5 h1:MNHlNMBDgEKD4TcKr36vQN68BA00aDfjIt3/bD50WnA=
@ -363,6 +391,8 @@ github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzw
github.com/golang-jwt/jwt/v4 v4.2.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg=
github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg=
github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
github.com/golang-jwt/jwt/v4 v4.5.1 h1:JdqV9zKUdtaa9gdPlywC3aeoEsR681PlKC+4F5gQgeo=
github.com/golang-jwt/jwt/v4 v4.5.1/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk=
github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9 h1:au07oEsX2xN0ktxqI+Sida1w446QrXBRJ0nee3SNZlA=
@ -506,6 +536,8 @@ github.com/jedib0t/go-pretty v4.3.0+incompatible h1:CGs8AVhEKg/n9YbUenWmNStRW2PH
github.com/jedib0t/go-pretty v4.3.0+incompatible/go.mod h1:XemHduiw8R651AF9Pt4FwCTKeG3oo7hrHJAoznj9nag=
github.com/jedib0t/go-pretty/v6 v6.6.0 h1:wmZVuAcEkZRT+Aq1xXpE8IGat4vE5WXOMmBpbQqERXw=
github.com/jedib0t/go-pretty/v6 v6.6.0/go.mod h1:zbn98qrYlh95FIhwwsbIip0LYpwSG8SUOScs+v9/t0E=
github.com/jedib0t/go-pretty/v6 v6.6.1 h1:iJ65Xjb680rHcikRj6DSIbzCex2huitmc7bDtxYVWyc=
github.com/jedib0t/go-pretty/v6 v6.6.1/go.mod h1:zbn98qrYlh95FIhwwsbIip0LYpwSG8SUOScs+v9/t0E=
github.com/jlaffaye/ftp v0.2.0 h1:lXNvW7cBu7R/68bknOX3MrRIIqZ61zELs1P2RAiA3lg=
github.com/jlaffaye/ftp v0.2.0/go.mod h1:is2Ds5qkhceAPy2xD6RLI6hmp/qysSoymZ+Z2uTnspI=
github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
@ -667,6 +699,8 @@ github.com/prashantv/gostub v1.1.0 h1:BTyx3RfQjRHnUWaGF9oQos79AlQ5k8WNktv7VGvVH4
github.com/prashantv/gostub v1.1.0/go.mod h1:A5zLQHz7ieHGG7is6LLXLz7I8+3LZzsrV0P1IAHhP5U=
github.com/prometheus/client_golang v1.20.4 h1:Tgh3Yr67PaOv/uTqloMsCEdeuFTatm5zIq5+qNN23vI=
github.com/prometheus/client_golang v1.20.4/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE=
github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y=
github.com/prometheus/client_golang v1.20.5/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE=
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E=
github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY=
@ -691,6 +725,8 @@ github.com/sassoftware/go-rpmutils v0.4.0 h1:ojND82NYBxgwrV+mX1CWsd5QJvvEZTKddtC
github.com/sassoftware/go-rpmutils v0.4.0/go.mod h1:3goNWi7PGAT3/dlql2lv3+MSN5jNYPjT5mVcQcIsYzI=
github.com/schollz/progressbar/v3 v3.16.1 h1:RnF1neWZFzLCoGx8yp1yF7SDl4AzNDI5y4I0aUJRrZQ=
github.com/schollz/progressbar/v3 v3.16.1/go.mod h1:I2ILR76gz5VXqYMIY/LdLecvMHDPVcQm3W/MSKi1TME=
github.com/schollz/progressbar/v3 v3.17.0 h1:Fv+vG6O6jnJwdjCelvfyYO7sF2jaUGQVmdH4CxcZdsQ=
github.com/schollz/progressbar/v3 v3.17.0/go.mod h1:5H4fLgifX+KeQCsEJnZTOepgZLe1jFF1lpPXb68IJTA=
github.com/sendgrid/rest v2.6.9+incompatible h1:1EyIcsNdn9KIisLW50MKwmSRSK+ekueiEMJ7NEoxJo0=
github.com/sendgrid/rest v2.6.9+incompatible/go.mod h1:kXX7q3jZtJXK5c5qK83bSGMdV6tsOE70KbHoqJls4lE=
github.com/sendgrid/sendgrid-go v3.16.0+incompatible h1:i8eE6IMkiCy7vusSdacHHSBUpXyTcTXy/Rl9N9aZ/Qw=
@ -740,18 +776,32 @@ github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsT
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
github.com/tailscale/depaware v0.0.0-20240804103531-585336c3e1b3 h1:1wRadBXZ4ddC71FYpzDTS3weuj0IOgmeptZtgTDGU7g=
github.com/tailscale/depaware v0.0.0-20240804103531-585336c3e1b3/go.mod h1:p9lPsd+cx33L3H9nNoecRRxPssFKUwwI50I3pZ0yT+8=
github.com/tailscale/depaware v0.0.0-20241028160002-3d7f3b30ed0e h1:Hb50wYyy5VblH5zpKkoy49TrJy3pxVWOaRSOEdzTWKA=
github.com/tailscale/depaware v0.0.0-20241028160002-3d7f3b30ed0e/go.mod h1:p9lPsd+cx33L3H9nNoecRRxPssFKUwwI50I3pZ0yT+8=
github.com/testcontainers/testcontainers-go v0.33.0 h1:zJS9PfXYT5O0ZFXM2xxXfk4J5UMw/kRiISng037Gxdw=
github.com/testcontainers/testcontainers-go v0.33.0/go.mod h1:W80YpTa8D5C3Yy16icheD01UTDu+LmXIA2Keo+jWtT8=
github.com/testcontainers/testcontainers-go v0.34.0 h1:5fbgF0vIN5u+nD3IWabQwRybuB4GY8G2HHgCkbMzMHo=
github.com/testcontainers/testcontainers-go v0.34.0/go.mod h1:6P/kMkQe8yqPHfPWNulFGdFHTD8HB2vLq/231xY2iPQ=
github.com/testcontainers/testcontainers-go/modules/elasticsearch v0.33.0 h1:tVsooNzk7SgYDO1OnqeIgihDYiD/vSBNBqwqCfauIJY=
github.com/testcontainers/testcontainers-go/modules/elasticsearch v0.33.0/go.mod h1:qmspvRf+Hx0iyqKQUmTg1jTNiO7HHGNrx98t/HksVfg=
github.com/testcontainers/testcontainers-go/modules/elasticsearch v0.34.0 h1:BBwJUs9xBpt1uOfO+yAr2pYW75MsyzuO/o70HTPnhe4=
github.com/testcontainers/testcontainers-go/modules/elasticsearch v0.34.0/go.mod h1:OqhRGYR+5VG0Dw506F6Ho9I4YG1kB+o9uPTKC0uPUA8=
github.com/testcontainers/testcontainers-go/modules/mongodb v0.33.0 h1:iXVA84s5hKMS5gn01GWOYHE3ymy/2b+0YkpFeTxB2XY=
github.com/testcontainers/testcontainers-go/modules/mongodb v0.33.0/go.mod h1:R6tMjTojRiaoo89fh/hf7tOmfzohdqSU17R9DwSVSog=
github.com/testcontainers/testcontainers-go/modules/mongodb v0.34.0 h1:o3bgcECyBFfMwqexCH/6vIJ8XzbCffCP/Euesu33rgY=
github.com/testcontainers/testcontainers-go/modules/mongodb v0.34.0/go.mod h1:ljLR42dN7k40CX0dp30R8BRIB3OOdvr7rBANEpfmMs4=
github.com/testcontainers/testcontainers-go/modules/mssql v0.33.0 h1:gD4pHUPnEm5Bwup8KFdVmwXJLpyVy1hsp6bOXHAUlTA=
github.com/testcontainers/testcontainers-go/modules/mssql v0.33.0/go.mod h1:HdgR2Q9SsGqohT6nhtU3tnG56iNGUV1Tr5If0QypZl0=
github.com/testcontainers/testcontainers-go/modules/mssql v0.34.0 h1:4Pf7ILuLnxhpeTgQfKzEMPuMQhasU3VaYer9l5HrQ3s=
github.com/testcontainers/testcontainers-go/modules/mssql v0.34.0/go.mod h1:L2eMWZ49X0XjewabzJ6TXSY5z4SAWM/2WOBqlIxYFD8=
github.com/testcontainers/testcontainers-go/modules/mysql v0.33.0 h1:1JN7YEEepTMJmGI2hW678IiiYoLM5HDp3vbCPmUokJ8=
github.com/testcontainers/testcontainers-go/modules/mysql v0.33.0/go.mod h1:9tZZwRW5s3RaI5X0Wnc+GXNJFXqbkKmob2nBHbfA/5E=
github.com/testcontainers/testcontainers-go/modules/mysql v0.34.0 h1:Tqz17mGXjPORHFS/oBUGdeJyIsZXLsVVHRhaBqhewGI=
github.com/testcontainers/testcontainers-go/modules/mysql v0.34.0/go.mod h1:hDpm3DLfjo7rd6232wWflEBDGr6Ow9ys43mJTiJwWx8=
github.com/testcontainers/testcontainers-go/modules/postgres v0.33.0 h1:c+Gt+XLJjqFAejgX4hSpnHIpC9eAhvgI/TFWL/PbrFI=
github.com/testcontainers/testcontainers-go/modules/postgres v0.33.0/go.mod h1:I4DazHBoWDyf69ByOIyt3OdNjefiUx372459txOpQ3o=
github.com/testcontainers/testcontainers-go/modules/postgres v0.34.0 h1:c51aBXT3v2HEBVarmaBnsKzvgZjC5amn0qsj8Naqi50=
github.com/testcontainers/testcontainers-go/modules/postgres v0.34.0/go.mod h1:EWP75ogLQU4M4L8U+20mFipjV4WIR9WtlMXSB6/wiuc=
github.com/tetratelabs/wazero v1.8.0 h1:iEKu0d4c2Pd+QSRieYbnQC9yiFlMS9D+Jr0LsRmcF4g=
github.com/tetratelabs/wazero v1.8.0/go.mod h1:yAI0XTsMBhREkM/YDAK/zNou3GoiAce1P6+rp/wQhjs=
github.com/therootcompany/xz v1.0.1 h1:CmOtsn1CbtmyYiusbfmhmkpAAETj0wBIH6kCYaX+xzw=
@ -784,6 +834,8 @@ github.com/xanzy/go-gitlab v0.110.0 h1:hsFIFp01v/0D0sdUXoZfRk6CROzZbHQplk6NzKSFK
github.com/xanzy/go-gitlab v0.110.0/go.mod h1:wKNKh3GkYDMOsGmnfuX+ITCmDuSDWFO0G+C4AygL9RY=
github.com/xanzy/go-gitlab v0.111.0 h1:4zT52QdDVxGYAGxN2VY8upSvZIiuiI+Z4d+c+7D/lII=
github.com/xanzy/go-gitlab v0.111.0/go.mod h1:wKNKh3GkYDMOsGmnfuX+ITCmDuSDWFO0G+C4AygL9RY=
github.com/xanzy/go-gitlab v0.112.0 h1:6Z0cqEooCvBMfBIHw+CgO4AKGRV8na/9781xOb0+DKw=
github.com/xanzy/go-gitlab v0.112.0/go.mod h1:wKNKh3GkYDMOsGmnfuX+ITCmDuSDWFO0G+C4AygL9RY=
github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM=
github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw=
github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
@ -857,6 +909,8 @@ go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
go.uber.org/mock v0.4.0 h1:VcM4ZOtdbR4f6VXfiOpwpVJDL6lCReaZ6mw31wqh7KU=
go.uber.org/mock v0.4.0/go.mod h1:a6FSlNadKUHUa9IP5Vyt1zh4fC7uAwxMutEAscFbkZc=
go.uber.org/mock v0.5.0 h1:KAMbZvZPyBPWgD14IrIQ38QCyjwpvVVV6K/bHl1IwQU=
go.uber.org/mock v0.5.0/go.mod h1:ge71pBPLYDk7QIi1LupWxdAykm7KIEFchiOqd6z7qMM=
go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
@ -1099,6 +1153,14 @@ google.golang.org/api v0.199.0 h1:aWUXClp+VFJmqE0JPvpZOK3LDQMyFKYIow4etYd9qxs=
google.golang.org/api v0.199.0/go.mod h1:ohG4qSztDJmZdjK/Ar6MhbAmb/Rpi4JHOqagsh90K28=
google.golang.org/api v0.200.0 h1:0ytfNWn101is6e9VBoct2wrGDjOi5vn7jw5KtaQgDrU=
google.golang.org/api v0.200.0/go.mod h1:Tc5u9kcbjO7A8SwGlYj4IiVifJU01UqXtEgDMYmBmV8=
google.golang.org/api v0.201.0 h1:+7AD9JNM3tREtawRMu8sOjSbb8VYcYXJG/2eEOmfDu0=
google.golang.org/api v0.201.0/go.mod h1:HVY0FCHVs89xIW9fzf/pBvOEm+OolHa86G/txFezyq4=
google.golang.org/api v0.202.0 h1:y1iuVHMqokQbimW79ZqPZWo4CiyFu6HcCYHwSNyzlfo=
google.golang.org/api v0.202.0/go.mod h1:3Jjeq7M/SFblTNCp7ES2xhq+WvGL0KeXI0joHQBfwTQ=
google.golang.org/api v0.203.0 h1:SrEeuwU3S11Wlscsn+LA1kb/Y5xT8uggJSkIhD08NAU=
google.golang.org/api v0.203.0/go.mod h1:BuOVyCSYEPwJb3npWvDnNmFI92f3GeRnHNkETneT3SI=
google.golang.org/api v0.204.0 h1:3PjmQQEDkR/ENVZZwIYB4W/KzYtN8OrqnNcHWpeR8E4=
google.golang.org/api v0.204.0/go.mod h1:69y8QSoKIbL9F94bWgWAq6wGqGwyjBgi2y8rAK8zLag=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
@ -1123,14 +1185,26 @@ google.golang.org/genproto v0.0.0-20240903143218-8af14fe29dc1 h1:BulPr26Jqjnd4eY
google.golang.org/genproto v0.0.0-20240903143218-8af14fe29dc1/go.mod h1:hL97c3SYopEHblzpxRL4lSs523++l8DYxGM1FQiYmb4=
google.golang.org/genproto v0.0.0-20241007155032-5fefd90f89a9 h1:nFS3IivktIU5Mk6KQa+v6RKkHUpdQpphqGNLxqNnbEk=
google.golang.org/genproto v0.0.0-20241007155032-5fefd90f89a9/go.mod h1:tEzYTYZxbmVNOu0OAFH9HzdJtLn6h4Aj89zzlBCdHms=
google.golang.org/genproto v0.0.0-20241015192408-796eee8c2d53 h1:Df6WuGvthPzc+JiQ/G+m+sNX24kc0aTBqoDN/0yyykE=
google.golang.org/genproto v0.0.0-20241015192408-796eee8c2d53/go.mod h1:fheguH3Am2dGp1LfXkrvwqC/KlFq8F0nLq3LryOMrrE=
google.golang.org/genproto v0.0.0-20241021214115-324edc3d5d38 h1:Q3nlH8iSQSRUwOskjbcSMcF2jiYMNiQYZ0c2KEJLKKU=
google.golang.org/genproto v0.0.0-20241021214115-324edc3d5d38/go.mod h1:xBI+tzfqGGN2JBeSebfKXFSdBpWVQ7sLW40PTupVRm4=
google.golang.org/genproto/googleapis/api v0.0.0-20240903143218-8af14fe29dc1 h1:hjSy6tcFQZ171igDaN5QHOw2n6vx40juYbC/x67CEhc=
google.golang.org/genproto/googleapis/api v0.0.0-20240903143218-8af14fe29dc1/go.mod h1:qpvKtACPCQhAdu3PyQgV4l3LMXZEtft7y8QcarRsp9I=
google.golang.org/genproto/googleapis/api v0.0.0-20240930140551-af27646dc61f h1:jTm13A2itBi3La6yTGqn8bVSrc3ZZ1r8ENHlIXBfnRA=
google.golang.org/genproto/googleapis/api v0.0.0-20240930140551-af27646dc61f/go.mod h1:CLGoBuH1VHxAUXVPP8FfPwPEVJB6lz3URE5mY2SuayE=
google.golang.org/genproto/googleapis/api v0.0.0-20241007155032-5fefd90f89a9 h1:T6rh4haD3GVYsgEfWExoCZA2o2FmbNyKpTuAxbEFPTg=
google.golang.org/genproto/googleapis/api v0.0.0-20241007155032-5fefd90f89a9/go.mod h1:wp2WsuBYj6j8wUdo3ToZsdxxixbvQNAHqVJrTgi5E5M=
google.golang.org/genproto/googleapis/api v0.0.0-20241015192408-796eee8c2d53 h1:fVoAXEKA4+yufmbdVYv+SE73+cPZbbbe8paLsHfkK+U=
google.golang.org/genproto/googleapis/api v0.0.0-20241015192408-796eee8c2d53/go.mod h1:riSXTwQ4+nqmPGtobMFyW5FqVAmIs0St6VPp4Ug7CE4=
google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 h1:pPJltXNxVzT4pK9yD8vR9X75DaWYYmLGMsEvBfFQZzQ=
google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU=
google.golang.org/genproto/googleapis/rpc v0.0.0-20241007155032-5fefd90f89a9 h1:QCqS/PdaHTSWGvupk2F/ehwHtGc0/GYkT+3GAcR1CCc=
google.golang.org/genproto/googleapis/rpc v0.0.0-20241007155032-5fefd90f89a9/go.mod h1:GX3210XPVPUjJbTUbvwI8f2IpZDMZuPJWDzDuebbviI=
google.golang.org/genproto/googleapis/rpc v0.0.0-20241015192408-796eee8c2d53 h1:X58yt85/IXCx0Y3ZwN6sEIKZzQtDEYaBWrDvErdXrRE=
google.golang.org/genproto/googleapis/rpc v0.0.0-20241015192408-796eee8c2d53/go.mod h1:GX3210XPVPUjJbTUbvwI8f2IpZDMZuPJWDzDuebbviI=
google.golang.org/genproto/googleapis/rpc v0.0.0-20241021214115-324edc3d5d38 h1:zciRKQ4kBpFgpfC5QQCVtnnNAcLIqweL7plyZRQHVpI=
google.golang.org/genproto/googleapis/rpc v0.0.0-20241021214115-324edc3d5d38/go.mod h1:GX3210XPVPUjJbTUbvwI8f2IpZDMZuPJWDzDuebbviI=
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=

5
hack/generate/generate.go
View file

@ -43,6 +43,11 @@ func main() {
WritePath: filepath.Join(folderPath(), nameLower+"_test.go"),
ReplaceString: []string{"alchemy"},
},
{
TemplatePath: "pkg/detectors/alchemy/alchemy_integration_test.go",
WritePath: filepath.Join(folderPath(), nameLower+"_integration_test.go"),
ReplaceString: []string{"alchemy"},
},
})
// case "source":
// mustWriteTemplates([]templateJob{

71
main.go
View file

@ -27,6 +27,7 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/config"
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
"github.com/trufflesecurity/trufflehog/v3/pkg/engine"
"github.com/trufflesecurity/trufflehog/v3/pkg/feature"
"github.com/trufflesecurity/trufflehog/v3/pkg/handlers"
"github.com/trufflesecurity/trufflehog/v3/pkg/log"
"github.com/trufflesecurity/trufflehog/v3/pkg/output"
@ -70,6 +71,12 @@ var (
excludeDetectors = cli.Flag("exclude-detectors", "Comma separated list of detector types to exclude. Protobuf name or IDs may be used, as well as ranges. IDs defined here take precedence over the include list.").String()
jobReportFile = cli.Flag("output-report", "Write a scan report to the provided path.").Hidden().OpenFile(os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666)
// Add feature flags
forceSkipBinaries = cli.Flag("force-skip-binaries", "Force skipping binaries.").Bool()
forceSkipArchives = cli.Flag("force-skip-archives", "Force skipping archives.").Bool()
skipAdditionalRefs = cli.Flag("skip-additional-refs", "Skip additional references.").Bool()
userAgentSuffix = cli.Flag("user-agent-suffix", "Suffix to add to User-Agent.").String()
gitScan = cli.Command("git", "Find credentials in git repositories.")
gitScanURI = gitScan.Arg("uri", "Git repository URL. https://, file://, or ssh:// schema expected.").Required().String()
gitScanIncludePaths = gitScan.Flag("include-paths", "Path to file with newline separated regexes for files to include in scan.").Short('i').String()
@ -83,22 +90,22 @@ var (
_ = gitScan.Flag("entropy", "No-op flag for backwards compat.").Bool()
_ = gitScan.Flag("regex", "No-op flag for backwards compat.").Bool()
githubScan = cli.Command("github", "Find credentials in GitHub repositories.")
githubScanEndpoint = githubScan.Flag("endpoint", "GitHub endpoint.").Default("https://api.github.com").String()
githubScanRepos = githubScan.Flag("repo", `GitHub repository to scan. You can repeat this flag. Example: "https://github.com/dustin-decker/secretsandstuff"`).Strings()
githubScanOrgs = githubScan.Flag("org", `GitHub organization to scan. You can repeat this flag. Example: "trufflesecurity"`).Strings()
githubScanToken = githubScan.Flag("token", "GitHub token. Can be provided with environment variable GITHUB_TOKEN.").Envar("GITHUB_TOKEN").String()
githubIncludeForks = githubScan.Flag("include-forks", "Include forks in scan.").Bool()
githubIncludeMembers = githubScan.Flag("include-members", "Include organization member repositories in scan.").Bool()
githubIncludeRepos = githubScan.Flag("include-repos", `Repositories to include in an org scan. This can also be a glob pattern. You can repeat this flag. Must use Github repo full name. Example: "trufflesecurity/trufflehog", "trufflesecurity/t*"`).Strings()
githubIncludeWikis = githubScan.Flag("include-wikis", "Include repository wikisin scan.").Bool()
githubExcludeRepos = githubScan.Flag("exclude-repos", `Repositories to exclude in an org scan. This can also be a glob pattern. You can repeat this flag. Must use Github repo full name. Example: "trufflesecurity/driftwood", "trufflesecurity/d*"`).Strings()
githubScanIncludePaths = githubScan.Flag("include-paths", "Path to file with newline separated regexes for files to include in scan.").Short('i').String()
githubScanExcludePaths = githubScan.Flag("exclude-paths", "Path to file with newline separated regexes for files to exclude in scan.").Short('x').String()
githubScanIssueComments = githubScan.Flag("issue-comments", "Include issue descriptions and comments in scan.").Bool()
githubScanPRComments = githubScan.Flag("pr-comments", "Include pull request descriptions and comments in scan.").Bool()
githubScanGistComments = githubScan.Flag("gist-comments", "Include gist comments in scan.").Bool()
githubScan = cli.Command("github", "Find credentials in GitHub repositories.")
githubScanEndpoint = githubScan.Flag("endpoint", "GitHub endpoint.").Default("https://api.github.com").String()
githubScanRepos = githubScan.Flag("repo", `GitHub repository to scan. You can repeat this flag. Example: "https://github.com/dustin-decker/secretsandstuff"`).Strings()
githubScanOrgs = githubScan.Flag("org", `GitHub organization to scan. You can repeat this flag. Example: "trufflesecurity"`).Strings()
githubScanToken = githubScan.Flag("token", "GitHub token. Can be provided with environment variable GITHUB_TOKEN.").Envar("GITHUB_TOKEN").String()
githubIncludeForks = githubScan.Flag("include-forks", "Include forks in scan.").Bool()
githubIncludeMembers = githubScan.Flag("include-members", "Include organization member repositories in scan.").Bool()
githubIncludeRepos = githubScan.Flag("include-repos", `Repositories to include in an org scan. This can also be a glob pattern. You can repeat this flag. Must use Github repo full name. Example: "trufflesecurity/trufflehog", "trufflesecurity/t*"`).Strings()
githubIncludeWikis = githubScan.Flag("include-wikis", "Include repository wikisin scan.").Bool()
githubExcludeRepos = githubScan.Flag("exclude-repos", `Repositories to exclude in an org scan. This can also be a glob pattern. You can repeat this flag. Must use Github repo full name. Example: "trufflesecurity/driftwood", "trufflesecurity/d*"`).Strings()
githubScanIncludePaths = githubScan.Flag("include-paths", "Path to file with newline separated regexes for files to include in scan.").Short('i').String()
githubScanExcludePaths = githubScan.Flag("exclude-paths", "Path to file with newline separated regexes for files to exclude in scan.").Short('x').String()
githubScanIssueComments = githubScan.Flag("issue-comments", "Include issue descriptions and comments in scan.").Bool()
githubScanPRComments = githubScan.Flag("pr-comments", "Include pull request descriptions and comments in scan.").Bool()
githubScanGistComments = githubScan.Flag("gist-comments", "Include gist comments in scan.").Bool()
githubCommentsTimeframeDays = githubScan.Flag("comments-timeframe", "Number of days in the past to review when scanning issue, PR, and gist comments.").Uint32()
// GitHub Cross Fork Object Reference Experimental Feature
githubExperimentalScan = cli.Command("github-experimental", "Run an experimental GitHub scan. Must specify at least one experimental sub-module to run: object-discovery.")
@ -117,6 +124,8 @@ var (
gitlabScanToken = gitlabScan.Flag("token", "GitLab token. Can be provided with environment variable GITLAB_TOKEN.").Envar("GITLAB_TOKEN").Required().String()
gitlabScanIncludePaths = gitlabScan.Flag("include-paths", "Path to file with newline separated regexes for files to include in scan.").Short('i').String()
gitlabScanExcludePaths = gitlabScan.Flag("exclude-paths", "Path to file with newline separated regexes for files to exclude in scan.").Short('x').String()
gitlabScanIncludeRepos = gitlabScan.Flag("include-repos", `Repositories to include in an org scan. This can also be a glob pattern. You can repeat this flag. Must use Gitlab repo full name. Example: "trufflesecurity/trufflehog", "trufflesecurity/t*"`).Strings()
gitlabScanExcludeRepos = gitlabScan.Flag("exclude-repos", `Repositories to exclude in an org scan. This can also be a glob pattern. You can repeat this flag. Must use Gitlab repo full name. Example: "trufflesecurity/driftwood", "trufflesecurity/d*"`).Strings()
filesystemScan = cli.Command("filesystem", "Find credentials in a filesystem.")
filesystemPaths = filesystemScan.Arg("path", "Path to file or directory to scan.").Strings()
@ -278,7 +287,7 @@ func main() {
if *jsonOut {
logFormat = log.WithJSONSink
}
logger, sync := log.New("trufflehog", logFormat(os.Stderr))
logger, sync := log.New("trufflehog", logFormat(os.Stderr, log.WithGlobalRedaction()))
// make it the default logger for contexts
context.SetDefaultLogger(logger)
@ -368,6 +377,23 @@ func run(state overseer.State) {
}()
}
// Set feature configurations from CLI flags
if *forceSkipBinaries {
feature.ForceSkipBinaries.Store(true)
}
if *forceSkipArchives {
feature.ForceSkipArchives.Store(true)
}
if *skipAdditionalRefs {
feature.SkipAdditionalRefs.Store(true)
}
if *userAgentSuffix != "" {
feature.UserAgentSuffix.Store(*userAgentSuffix)
}
conf := &config.Config{}
if *configFilename != "" {
var err error
@ -626,6 +652,7 @@ func runSingleScan(ctx context.Context, cmd string, cfg engine.Config) (metrics,
IncludeIssueComments: *githubScanIssueComments,
IncludePullRequestComments: *githubScanPRComments,
IncludeGistComments: *githubScanGistComments,
CommentsTimeframeDays: *githubCommentsTimeframeDays,
Filter: filter,
}
if err := eng.ScanGitHub(ctx, cfg); err != nil {
@ -649,10 +676,12 @@ func runSingleScan(ctx context.Context, cmd string, cfg engine.Config) (metrics,
}
cfg := sources.GitlabConfig{
Endpoint: *gitlabScanEndpoint,
Token: *gitlabScanToken,
Repos: *gitlabScanRepos,
Filter: filter,
Endpoint: *gitlabScanEndpoint,
Token: *gitlabScanToken,
Repos: *gitlabScanRepos,
IncludeRepos: *gitlabScanIncludeRepos,
ExcludeRepos: *gitlabScanExcludeRepos,
Filter: filter,
}
if err := eng.ScanGitLab(ctx, cfg); err != nil {
return scanMetrics, fmt.Errorf("failed to scan GitLab: %v", err)

3
pkg/analyzer/analyzers/airbrake/airbrake.go
View file

@ -11,7 +11,6 @@ import (
"github.com/jedib0t/go-pretty/table"
"github.com/trufflesecurity/trufflehog/v3/pkg/analyzer/analyzers"
"github.com/trufflesecurity/trufflehog/v3/pkg/analyzer/config"
"github.com/trufflesecurity/trufflehog/v3/pkg/analyzer/pb/analyzerpb"
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
)
@ -21,7 +20,7 @@ type Analyzer struct {
Cfg *config.Config
}
func (Analyzer) Type() analyzerpb.AnalyzerType { return analyzerpb.AnalyzerType_Airbrake }
func (Analyzer) Type() analyzers.AnalyzerType { return analyzers.AnalyzerTypeAirbrake }
func (a Analyzer) Analyze(_ context.Context, credInfo map[string]string) (*analyzers.AnalyzerResult, error) {
info, err := AnalyzePermissions(a.Cfg, credInfo["key"])

106
pkg/analyzer/analyzers/analyzers.go
View file

@ -5,21 +5,23 @@ import (
"encoding/json"
"io"
"net/http"
"sort"
"github.com/fatih/color"
"github.com/trufflesecurity/trufflehog/v3/pkg/analyzer/pb/analyzerpb"
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
)
type (
Analyzer interface {
Type() analyzerpb.AnalyzerType
Type() AnalyzerType
Analyze(ctx context.Context, credentialInfo map[string]string) (*AnalyzerResult, error)
}
AnalyzerType int
// AnalyzerResult is the output of analysis.
AnalyzerResult struct {
AnalyzerType analyzerpb.AnalyzerType
AnalyzerType AnalyzerType
Bindings []Binding
UnboundedResources []Resource
Metadata map[string]any
@ -56,30 +58,80 @@ const (
FullAccess string = "full_access"
)
// Sorted list of all available analyzers. Used for valid sub-commands and TUI
// selection. TODO: Change slice type to Analyzer interface when all available
// analyzers implement it.
var AvailableAnalyzers = []string{
"Airbrake",
"Asana",
"Bitbucket",
"GitHub",
"GitLab",
"HuggingFace",
"Mailchimp",
"Mailgun",
"MySQL",
"OpenAI",
"Opsgenie",
"Postgres",
"Postman",
"Sendgrid",
"Shopify",
"Slack",
"Sourcegraph",
"Square",
"Stripe",
"Twilio",
const (
AnalyzerTypeInvalid AnalyzerType = iota
AnalyzerTypeAirbrake
AnalyzerTypeAsana
AnalyzerTypeBitbucket
AnalyzerTypeGitHub
AnalyzerTypeGitLab
AnalyzerTypeHuggingFace
AnalyzerTypeMailchimp
AnalyzerTypeMailgun
AnalyzerTypeMySQL
AnalyzerTypeOpenAI
AnalyzerTypeOpsgenie
AnalyzerTypePostgres
AnalyzerTypePostman
AnalyzerTypeSendgrid
AnalyzerTypeShopify
AnalyzerTypeSlack
AnalyzerTypeSourcegraph
AnalyzerTypeSquare
AnalyzerTypeStripe
AnalyzerTypeTwilio
// Add new items here with AnalyzerType prefix
)
// analyzerTypeStrings maps the enum to its string representation.
var analyzerTypeStrings = map[AnalyzerType]string{
AnalyzerTypeInvalid: "Invalid",
AnalyzerTypeAirbrake: "Airbrake",
AnalyzerTypeAsana: "Asana",
AnalyzerTypeBitbucket: "Bitbucket",
AnalyzerTypeGitHub: "GitHub",
AnalyzerTypeGitLab: "GitLab",
AnalyzerTypeHuggingFace: "HuggingFace",
AnalyzerTypeMailchimp: "Mailchimp",
AnalyzerTypeMailgun: "Mailgun",
AnalyzerTypeMySQL: "MySQL",
AnalyzerTypeOpenAI: "OpenAI",
AnalyzerTypeOpsgenie: "Opsgenie",
AnalyzerTypePostgres: "Postgres",
AnalyzerTypePostman: "Postman",
AnalyzerTypeSendgrid: "Sendgrid",
AnalyzerTypeShopify: "Shopify",
AnalyzerTypeSlack: "Slack",
AnalyzerTypeSourcegraph: "Sourcegraph",
AnalyzerTypeSquare: "Square",
AnalyzerTypeStripe: "Stripe",
AnalyzerTypeTwilio: "Twilio",
// Add new mappings here
}
// String method to get the string representation of an AnalyzerType.
func (a AnalyzerType) String() string {
if str, ok := analyzerTypeStrings[a]; ok {
return str
}
return "Unknown"
}
// GetSortedAnalyzerTypes returns a sorted slice of AnalyzerType strings, skipping "Invalid".
func AvailableAnalyzers() []string {
var analyzerStrings []string
// Iterate through the map to collect all string values except "Invalid".
for typ, str := range analyzerTypeStrings {
if typ != AnalyzerTypeInvalid {
analyzerStrings = append(analyzerStrings, str)
}
}
// Sort the slice alphabetically.
sort.Strings(analyzerStrings)
return analyzerStrings
}
type PermissionStatus struct {

3
pkg/analyzer/analyzers/asana/asana.go
View file

@ -14,7 +14,6 @@ import (
"github.com/jedib0t/go-pretty/table"
"github.com/trufflesecurity/trufflehog/v3/pkg/analyzer/analyzers"
"github.com/trufflesecurity/trufflehog/v3/pkg/analyzer/config"
"github.com/trufflesecurity/trufflehog/v3/pkg/analyzer/pb/analyzerpb"
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
)
@ -24,7 +23,7 @@ type Analyzer struct {
Cfg *config.Config
}
func (Analyzer) Type() analyzerpb.AnalyzerType { return analyzerpb.AnalyzerType_Asana }
func (Analyzer) Type() analyzers.AnalyzerType { return analyzers.AnalyzerTypeAsana }
func (a Analyzer) Analyze(_ context.Context, credInfo map[string]string) (*analyzers.AnalyzerResult, error) {
key, ok := credInfo["key"]

5
pkg/analyzer/analyzers/bitbucket/bitbucket.go
View file

@ -13,7 +13,6 @@ import (
"github.com/jedib0t/go-pretty/table"
"github.com/trufflesecurity/trufflehog/v3/pkg/analyzer/analyzers"
"github.com/trufflesecurity/trufflehog/v3/pkg/analyzer/config"
"github.com/trufflesecurity/trufflehog/v3/pkg/analyzer/pb/analyzerpb"
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
)
@ -59,7 +58,7 @@ type Analyzer struct {
Cfg *config.Config
}
func (Analyzer) Type() analyzerpb.AnalyzerType { return analyzerpb.AnalyzerType_Bitbucket }
func (Analyzer) Type() analyzers.AnalyzerType { return analyzers.AnalyzerTypeBitbucket }
func (a Analyzer) Analyze(_ context.Context, credInfo map[string]string) (*analyzers.AnalyzerResult, error) {
key, ok := credInfo["key"]
@ -79,7 +78,7 @@ func secretInfoToAnalyzerResult(info *SecretInfo) *analyzers.AnalyzerResult {
}
result := analyzers.AnalyzerResult{
AnalyzerType: analyzerpb.AnalyzerType_Bitbucket,
AnalyzerType: analyzers.AnalyzerTypeBitbucket,
}
// add unbounded resources

3
pkg/analyzer/analyzers/github/github.go
View file

@ -13,7 +13,6 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/analyzer/analyzers/github/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/analyzer/analyzers/github/finegrained"
"github.com/trufflesecurity/trufflehog/v3/pkg/analyzer/config"
"github.com/trufflesecurity/trufflehog/v3/pkg/analyzer/pb/analyzerpb"
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
)
@ -23,7 +22,7 @@ type Analyzer struct {
Cfg *config.Config
}
func (Analyzer) Type() analyzerpb.AnalyzerType { return analyzerpb.AnalyzerType_GitHub }
func (Analyzer) Type() analyzers.AnalyzerType { return analyzers.AnalyzerTypeGitHub }
func (a Analyzer) Analyze(_ context.Context, credInfo map[string]string) (*analyzers.AnalyzerResult, error) {
info, err := AnalyzePermissions(a.Cfg, credInfo["key"])

1
pkg/analyzer/analyzers/gitlab/expected_output.json Normal file
View file

@ -0,0 +1 @@
{"AnalyzerType":5,"Bindings":[{"Resource":{"Name":"gitlab.com/user/22466472","FullyQualifiedName":"gitlab.com/user/22466472","Type":"user","Metadata":{"token_created_at":"2024-08-15T06:33:00.337Z","token_expires_at":"2025-08-15","token_id":10470457,"token_name":"test-project-token","token_revoked":false},"Parent":null},"Permission":{"Value":"read_api","Parent":null}},{"Resource":{"Name":"gitlab.com/user/22466472","FullyQualifiedName":"gitlab.com/user/22466472","Type":"user","Metadata":{"token_created_at":"2024-08-15T06:33:00.337Z","token_expires_at":"2025-08-15","token_id":10470457,"token_name":"test-project-token","token_revoked":false},"Parent":null},"Permission":{"Value":"read_repository","Parent":null}},{"Resource":{"Name":"truffletester / trufflehog","FullyQualifiedName":"gitlab.com/project/60871295","Type":"project","Metadata":null,"Parent":null},"Permission":{"Value":"Developer","Parent":null}}],"UnboundedResources":null,"Metadata":{"enterprise":true,"version":"17.6.0-pre"}}

79
pkg/analyzer/analyzers/gitlab/gitlab.go
View file

@ -1,8 +1,11 @@
//go:generate generate_permissions permissions.yaml permissions.go gitlab
package gitlab
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
@ -13,7 +16,6 @@ import (
"github.com/jedib0t/go-pretty/table"
"github.com/trufflesecurity/trufflehog/v3/pkg/analyzer/analyzers"
"github.com/trufflesecurity/trufflehog/v3/pkg/analyzer/config"
"github.com/trufflesecurity/trufflehog/v3/pkg/analyzer/pb/analyzerpb"
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
)
@ -23,30 +25,96 @@ type Analyzer struct {
Cfg *config.Config
}
func (Analyzer) Type() analyzerpb.AnalyzerType { return analyzerpb.AnalyzerType_GitLab }
func (Analyzer) Type() analyzers.AnalyzerType { return analyzers.AnalyzerTypeGitLab }
func (a Analyzer) Analyze(_ context.Context, credInfo map[string]string) (*analyzers.AnalyzerResult, error) {
_, err := AnalyzePermissions(a.Cfg, credInfo["key"])
key, ok := credInfo["key"]
if !ok {
return nil, errors.New("key not found in credentialInfo")
}
info, err := AnalyzePermissions(a.Cfg, key)
if err != nil {
return nil, err
}
return nil, fmt.Errorf("not implemented")
return secretInfoToAnalyzerResult(info), nil
}
func secretInfoToAnalyzerResult(info *SecretInfo) *analyzers.AnalyzerResult {
result := analyzers.AnalyzerResult{
AnalyzerType: analyzers.AnalyzerTypeGitLab,
Metadata: map[string]any{
"version": info.Metadata.Version,
"enterprise": info.Metadata.Enterprise,
},
Bindings: []analyzers.Binding{},
}
// Add user and it's permissions to bindings
userFullyQualifiedName := fmt.Sprintf("gitlab.com/user/%d", info.AccessToken.UserID)
userResource := analyzers.Resource{
Name: userFullyQualifiedName,
FullyQualifiedName: userFullyQualifiedName,
Type: "user",
Metadata: map[string]any{
"token_name": info.AccessToken.Name,
"token_id": info.AccessToken.ID,
"token_created_at": info.AccessToken.CreatedAt,
"token_revoked": info.AccessToken.Revoked,
"token_expires_at": info.AccessToken.ExpiresAt,
},
}
for _, scope := range info.AccessToken.Scopes {
result.Bindings = append(result.Bindings, analyzers.Binding{
Resource: userResource,
Permission: analyzers.Permission{
Value: scope,
},
})
}
// append project and it's permissions to bindings
for _, project := range info.Projects {
projectResource := analyzers.Resource{
Name: project.NameWithNamespace,
FullyQualifiedName: fmt.Sprintf("gitlab.com/project/%d", project.ID),
Type: "project",
}
accessLevel, ok := access_level_map[project.Permissions.ProjectAccess.AccessLevel]
if !ok {
continue
}
result.Bindings = append(result.Bindings, analyzers.Binding{
Resource: projectResource,
Permission: analyzers.Permission{
Value: accessLevel,
},
})
}
return &result
}
// consider calling /api/v4/metadata to learn about gitlab instance version and whether neterrprises is enabled
// we'll call /api/v4/personal_access_tokens and /api/v4/user and then filter down to scopes.
// we'll call /api/v4/personal_access_tokens and then filter down to scopes.
type AccessTokenJSON struct {
ID int `json:"id"`
Name string `json:"name"`
Revoked bool `json:"revoked"`
CreatedAt string `json:"created_at"`
Scopes []string `json:"scopes"`
LastUsedAt string `json:"last_used_at"`
ExpiresAt string `json:"expires_at"`
UserID int `json:"user_id"`
}
type ProjectsJSON struct {
ID int `json:"id"`
NameWithNamespace string `json:"name_with_namespace"`
Permissions struct {
ProjectAccess struct {
@ -249,6 +317,7 @@ func printTokenInfo(token AccessTokenJSON) {
color.Green("Token Name: %s\n", token.Name)
color.Green("Created At: %s\n", token.CreatedAt)
color.Green("Last Used At: %s\n", token.LastUsedAt)
color.Green("User ID: %d\n", token.UserID)
color.Green("Expires At: %s (%v remaining)\n\n", token.ExpiresAt, getRemainingTime(token.ExpiresAt))
if token.Revoked {
color.Red("Token Revoked: %v\n", token.Revoked)

78
pkg/analyzer/analyzers/gitlab/gitlab_test.go Normal file
View file

@ -0,0 +1,78 @@
package gitlab
import (
_ "embed"
"encoding/json"
"testing"
"time"
"github.com/trufflesecurity/trufflehog/v3/pkg/analyzer/analyzers"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
)
//go:embed expected_output.json
var expectedOutput []byte
func TestAnalyzer_Analyze(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
tests := []struct {
name string
key string
want string // JSON string
wantErr bool
}{
{
name: "valid gitlab access token",
key: testSecrets.MustGetField("GITLABV2"),
want: string(expectedOutput),
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
a := Analyzer{}
got, err := a.Analyze(ctx, map[string]string{"key": tt.key})
if (err != nil) != tt.wantErr {
t.Errorf("Analyzer.Analyze() error = %v, wantErr %v", err, tt.wantErr)
return
}
// Marshal the actual result to JSON
gotJSON, err := json.Marshal(got)
if err != nil {
t.Fatalf("could not marshal got to JSON: %s", err)
}
// Parse the expected JSON string
var wantObj analyzers.AnalyzerResult
if err := json.Unmarshal([]byte(tt.want), &wantObj); err != nil {
t.Fatalf("could not unmarshal want JSON string: %s", err)
}
// Marshal the expected result to JSON (to normalize)
wantJSON, err := json.Marshal(wantObj)
if err != nil {
t.Fatalf("could not marshal want to JSON: %s", err)
}
// Compare the JSON strings
if string(gotJSON) != string(wantJSON) {
// Pretty-print both JSON strings for easier comparison
var gotIndented []byte
gotIndented, err = json.MarshalIndent(got, "", " ")
if err != nil {
t.Fatalf("could not marshal got to indented JSON: %s", err)
}
t.Errorf("Analyzer.Analyze() = \n%s", gotIndented)
}
})
}
}

126
pkg/analyzer/analyzers/gitlab/permissions.go Normal file
View file

@ -0,0 +1,126 @@
// Code generated by go generate; DO NOT EDIT.
package gitlab
import "errors"
type Permission int
const (
Invalid Permission = iota
Api Permission = iota
ReadUser Permission = iota
ReadApi Permission = iota
ReadRepository Permission = iota
WriteRepository Permission = iota
ReadRegistry Permission = iota
WriteRegistry Permission = iota
Sudo Permission = iota
AdminMode Permission = iota
CreateRunner Permission = iota
ManageRunner Permission = iota
AiFeatures Permission = iota
K8sProxy Permission = iota
ReadServicePing Permission = iota
)
var (
PermissionStrings = map[Permission]string{
Api: "api",
ReadUser: "read_user",
ReadApi: "read_api",
ReadRepository: "read_repository",
WriteRepository: "write_repository",
ReadRegistry: "read_registry",
WriteRegistry: "write_registry",
Sudo: "sudo",
AdminMode: "admin_mode",
CreateRunner: "create_runner",
ManageRunner: "manage_runner",
AiFeatures: "ai_features",
K8sProxy: "k8s_proxy",
ReadServicePing: "read_service_ping",
}
StringToPermission = map[string]Permission{
"api": Api,
"read_user": ReadUser,
"read_api": ReadApi,
"read_repository": ReadRepository,
"write_repository": WriteRepository,
"read_registry": ReadRegistry,
"write_registry": WriteRegistry,
"sudo": Sudo,
"admin_mode": AdminMode,
"create_runner": CreateRunner,
"manage_runner": ManageRunner,
"ai_features": AiFeatures,
"k8s_proxy": K8sProxy,
"read_service_ping": ReadServicePing,
}
PermissionIDs = map[Permission]int{
Api: 1,
ReadUser: 2,
ReadApi: 3,
ReadRepository: 4,
WriteRepository: 5,
ReadRegistry: 6,
WriteRegistry: 7,
Sudo: 8,
AdminMode: 9,
CreateRunner: 10,
ManageRunner: 11,
AiFeatures: 12,
K8sProxy: 13,
ReadServicePing: 14,
}
IdToPermission = map[int]Permission{
1: Api,
2: ReadUser,
3: ReadApi,
4: ReadRepository,
5: WriteRepository,
6: ReadRegistry,
7: WriteRegistry,
8: Sudo,
9: AdminMode,
10: CreateRunner,
11: ManageRunner,
12: AiFeatures,
13: K8sProxy,
14: ReadServicePing,
}
)
// ToString converts a Permission enum to its string representation
func (p Permission) ToString() (string, error) {
if str, ok := PermissionStrings[p]; ok {
return str, nil
}
return "", errors.New("invalid permission")
}
// ToID converts a Permission enum to its ID
func (p Permission) ToID() (int, error) {
if id, ok := PermissionIDs[p]; ok {
return id, nil
}
return 0, errors.New("invalid permission")
}
// PermissionFromString converts a string representation to its Permission enum
func PermissionFromString(s string) (Permission, error) {
if p, ok := StringToPermission[s]; ok {
return p, nil
}
return 0, errors.New("invalid permission string")
}
// PermissionFromID converts an ID to its Permission enum
func PermissionFromID(id int) (Permission, error) {
if p, ok := IdToPermission[id]; ok {
return p, nil
}
return 0, errors.New("invalid permission ID")
}

15
pkg/analyzer/analyzers/gitlab/permissions.yaml Normal file
View file

@ -0,0 +1,15 @@
permissions:
- api
- read_user
- read_api
- read_repository
- write_repository
- read_registry
- write_registry
- sudo
- admin_mode
- create_runner
- manage_runner
- ai_features
- k8s_proxy
- read_service_ping

5
pkg/analyzer/analyzers/huggingface/huggingface.go
View file

@ -13,7 +13,6 @@ import (
"github.com/jedib0t/go-pretty/table"
"github.com/trufflesecurity/trufflehog/v3/pkg/analyzer/analyzers"
"github.com/trufflesecurity/trufflehog/v3/pkg/analyzer/config"
"github.com/trufflesecurity/trufflehog/v3/pkg/analyzer/pb/analyzerpb"
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
)
@ -29,7 +28,7 @@ type Analyzer struct {
Cfg *config.Config
}
func (Analyzer) Type() analyzerpb.AnalyzerType { return analyzerpb.AnalyzerType_HuggingFace }
func (Analyzer) Type() analyzers.AnalyzerType { return analyzers.AnalyzerTypeHuggingFace }
func (a Analyzer) Analyze(_ context.Context, credInfo map[string]string) (*analyzers.AnalyzerResult, error) {
key, ok := credInfo["key"]
@ -239,7 +238,7 @@ func secretInfoToAnalyzerResult(info *SecretInfo) *analyzers.AnalyzerResult {
}
result := analyzers.AnalyzerResult{
AnalyzerType: analyzerpb.AnalyzerType_HuggingFace,
AnalyzerType: analyzers.AnalyzerTypeHuggingFace,
Metadata: map[string]interface{}{
"username": info.Token.Username,
"name": info.Token.Name,

5
pkg/analyzer/analyzers/mailchimp/mailchimp.go
View file

@ -13,7 +13,6 @@ import (
"github.com/jedib0t/go-pretty/table"
"github.com/trufflesecurity/trufflehog/v3/pkg/analyzer/analyzers"
"github.com/trufflesecurity/trufflehog/v3/pkg/analyzer/config"
"github.com/trufflesecurity/trufflehog/v3/pkg/analyzer/pb/analyzerpb"
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
)
@ -25,7 +24,7 @@ type Analyzer struct {
Cfg *config.Config
}
func (Analyzer) Type() analyzerpb.AnalyzerType { return analyzerpb.AnalyzerType_Mailchimp }
func (Analyzer) Type() analyzers.AnalyzerType { return analyzers.AnalyzerTypeMailchimp }
func (a Analyzer) Analyze(_ context.Context, credInfo map[string]string) (*analyzers.AnalyzerResult, error) {
key, ok := credInfo["key"]
@ -45,7 +44,7 @@ func secretInfoToAnalyzerResult(info *SecretInfo) *analyzers.AnalyzerResult {
return nil
}
result := analyzers.AnalyzerResult{
AnalyzerType: analyzerpb.AnalyzerType_Mailchimp,
AnalyzerType: analyzers.AnalyzerTypeMailchimp,
Bindings: make([]analyzers.Binding, 0, len(StringToPermission)),
UnboundedResources: make([]analyzers.Resource, 0, len(info.Domains.Domains)),
}

5
pkg/analyzer/analyzers/mailgun/mailgun.go
View file

@ -13,7 +13,6 @@ import (
"github.com/jedib0t/go-pretty/table"
"github.com/trufflesecurity/trufflehog/v3/pkg/analyzer/analyzers"
"github.com/trufflesecurity/trufflehog/v3/pkg/analyzer/config"
"github.com/trufflesecurity/trufflehog/v3/pkg/analyzer/pb/analyzerpb"
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
)
@ -23,7 +22,7 @@ type Analyzer struct {
Cfg *config.Config
}
func (Analyzer) Type() analyzerpb.AnalyzerType { return analyzerpb.AnalyzerType_Mailgun }
func (Analyzer) Type() analyzers.AnalyzerType { return analyzers.AnalyzerTypeMailgun }
func (a Analyzer) Analyze(_ context.Context, credInfo map[string]string) (*analyzers.AnalyzerResult, error) {
key, ok := credInfo["key"]
@ -43,7 +42,7 @@ func secretInfoToAnalyzerResult(info *DomainsJSON) *analyzers.AnalyzerResult {
return nil
}
result := analyzers.AnalyzerResult{
AnalyzerType: analyzerpb.AnalyzerType_Mailgun,
AnalyzerType: analyzers.AnalyzerTypeMailgun,
Bindings: make([]analyzers.Binding, len(info.Items)),
}

5
pkg/analyzer/analyzers/mysql/mysql.go
View file

@ -19,7 +19,6 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/analyzer/analyzers"
"github.com/trufflesecurity/trufflehog/v3/pkg/analyzer/config"
"github.com/trufflesecurity/trufflehog/v3/pkg/analyzer/pb/analyzerpb"
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
)
@ -29,7 +28,7 @@ type Analyzer struct {
Cfg *config.Config
}
func (Analyzer) Type() analyzerpb.AnalyzerType { return analyzerpb.AnalyzerType_MySQL }
func (Analyzer) Type() analyzers.AnalyzerType { return analyzers.AnalyzerTypeMySQL }
func (a Analyzer) Analyze(_ context.Context, credInfo map[string]string) (*analyzers.AnalyzerResult, error) {
uri, ok := credInfo["connection_string"]
@ -48,7 +47,7 @@ func secretInfoToAnalyzerResult(info *SecretInfo) *analyzers.AnalyzerResult {
return nil
}
result := analyzers.AnalyzerResult{
AnalyzerType: analyzerpb.AnalyzerType_MySQL,
AnalyzerType: analyzers.AnalyzerTypeMySQL,
Metadata: nil,
Bindings: []analyzers.Binding{},
}

3
pkg/analyzer/analyzers/openai/openai.go
View file

@ -16,7 +16,6 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/analyzer/analyzers"
"github.com/trufflesecurity/trufflehog/v3/pkg/analyzer/config"
"github.com/trufflesecurity/trufflehog/v3/pkg/analyzer/pb/analyzerpb"
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
)
@ -26,7 +25,7 @@ type Analyzer struct {
Cfg *config.Config
}
func (Analyzer) Type() analyzerpb.AnalyzerType { return analyzerpb.AnalyzerType_OpenAI }
func (Analyzer) Type() analyzers.AnalyzerType { return analyzers.AnalyzerTypeOpenAI }
func (a Analyzer) Analyze(_ context.Context, credInfo map[string]string) (*analyzers.AnalyzerResult, error) {
info, err := AnalyzePermissions(a.Cfg, credInfo["key"])

5
pkg/analyzer/analyzers/opsgenie/opsgenie.go
View file

@ -16,7 +16,6 @@ import (
"github.com/jedib0t/go-pretty/table"
"github.com/trufflesecurity/trufflehog/v3/pkg/analyzer/analyzers"
"github.com/trufflesecurity/trufflehog/v3/pkg/analyzer/config"
"github.com/trufflesecurity/trufflehog/v3/pkg/analyzer/pb/analyzerpb"
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
)
@ -26,7 +25,7 @@ type Analyzer struct {
Cfg *config.Config
}
func (Analyzer) Type() analyzerpb.AnalyzerType { return analyzerpb.AnalyzerType_Opsgenie }
func (Analyzer) Type() analyzers.AnalyzerType { return analyzers.AnalyzerTypeOpsgenie }
func (a Analyzer) Analyze(_ context.Context, credInfo map[string]string) (*analyzers.AnalyzerResult, error) {
key, ok := credInfo["key"]
@ -45,7 +44,7 @@ func secretInfoToAnalyzerResult(info *SecretInfo) *analyzers.AnalyzerResult {
return nil
}
result := analyzers.AnalyzerResult{
AnalyzerType: analyzerpb.AnalyzerType_Opsgenie,
AnalyzerType: analyzers.AnalyzerTypeOpsgenie,
Metadata: nil,
Bindings: make([]analyzers.Binding, len(info.Permissions)),
UnboundedResources: make([]analyzers.Resource, len(info.Users)),

5
pkg/analyzer/analyzers/postgres/postgres.go
View file

@ -16,7 +16,6 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/analyzer/analyzers"
"github.com/trufflesecurity/trufflehog/v3/pkg/analyzer/config"
"github.com/trufflesecurity/trufflehog/v3/pkg/analyzer/pb/analyzerpb"
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
)
@ -26,7 +25,7 @@ type Analyzer struct {
Cfg *config.Config
}
func (Analyzer) Type() analyzerpb.AnalyzerType { return analyzerpb.AnalyzerType_Postgres }
func (Analyzer) Type() analyzers.AnalyzerType { return analyzers.AnalyzerTypePostgres }
func (a Analyzer) Analyze(_ context.Context, credInfo map[string]string) (*analyzers.AnalyzerResult, error) {
uri, ok := credInfo["connection_string"]
@ -46,7 +45,7 @@ func secretInfoToAnalyzerResult(info *SecretInfo) *analyzers.AnalyzerResult {
return nil
}
result := analyzers.AnalyzerResult{
AnalyzerType: analyzerpb.AnalyzerType_Postgres,
AnalyzerType: analyzers.AnalyzerTypePostgres,
Metadata: nil,
Bindings: []analyzers.Binding{},
}

5
pkg/analyzer/analyzers/postman/postman.go
View file

@ -12,7 +12,6 @@ import (
"github.com/jedib0t/go-pretty/table"
"github.com/trufflesecurity/trufflehog/v3/pkg/analyzer/analyzers"
"github.com/trufflesecurity/trufflehog/v3/pkg/analyzer/config"
"github.com/trufflesecurity/trufflehog/v3/pkg/analyzer/pb/analyzerpb"
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
)
@ -22,7 +21,7 @@ type Analyzer struct {
Cfg *config.Config
}
func (Analyzer) Type() analyzerpb.AnalyzerType { return analyzerpb.AnalyzerType_Postman }
func (Analyzer) Type() analyzers.AnalyzerType { return analyzers.AnalyzerTypePostman }
func (a Analyzer) Analyze(_ context.Context, credInfo map[string]string) (*analyzers.AnalyzerResult, error) {
key, ok := credInfo["key"]
@ -42,7 +41,7 @@ func secretInfoToAnalyzerResult(info *SecretInfo) *analyzers.AnalyzerResult {
}
result := analyzers.AnalyzerResult{
AnalyzerType: analyzerpb.AnalyzerType_Postman,
AnalyzerType: analyzers.AnalyzerTypePostman,
Metadata: nil,
UnboundedResources: []analyzers.Resource{},
}

5
pkg/analyzer/analyzers/sendgrid/sendgrid.go
View file

@ -15,7 +15,6 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/analyzer/analyzers"
"github.com/trufflesecurity/trufflehog/v3/pkg/analyzer/config"
"github.com/trufflesecurity/trufflehog/v3/pkg/analyzer/pb/analyzerpb"
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
)
@ -25,7 +24,7 @@ type Analyzer struct {
Cfg *config.Config
}
func (Analyzer) Type() analyzerpb.AnalyzerType { return analyzerpb.AnalyzerType_Sendgrid }
func (Analyzer) Type() analyzers.AnalyzerType { return analyzers.AnalyzerTypeSendgrid }
func (a Analyzer) Analyze(_ context.Context, credInfo map[string]string) (*analyzers.AnalyzerResult, error) {
key, ok := credInfo["key"]
@ -54,7 +53,7 @@ func secretInfoToAnalyzerResult(info *SecretInfo) *analyzers.AnalyzerResult {
}
result := analyzers.AnalyzerResult{
AnalyzerType: analyzerpb.AnalyzerType_Sendgrid,
AnalyzerType: analyzers.AnalyzerTypeSendgrid,
Metadata: map[string]any{
"key_type": keyType,
"2fa_required": slices.Contains(info.RawScopes, "2fa_required"),

5
pkg/analyzer/analyzers/shopify/shopify.go
View file

@ -15,7 +15,6 @@ import (
"github.com/jedib0t/go-pretty/table"
"github.com/trufflesecurity/trufflehog/v3/pkg/analyzer/analyzers"
"github.com/trufflesecurity/trufflehog/v3/pkg/analyzer/config"
"github.com/trufflesecurity/trufflehog/v3/pkg/analyzer/pb/analyzerpb"
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
)
@ -30,7 +29,7 @@ var (
categoryOrder = []string{"Analytics", "Applications", "Assigned fulfillment orders", "Browsing behavior", "Custom pixels", "Customers", "Discounts", "Discovery", "Draft orders", "Files", "Fulfillment services", "Gift cards", "Inventory", "Legal policies", "Locations", "Marketing events", "Merchant-managed fulfillment orders", "Metaobject definitions", "Metaobject entries", "Online Store navigation", "Online Store pages", "Order editing", "Orders", "Packing slip management", "Payment customizations", "Payment terms", "Pixels", "Price rules", "Product feeds", "Product listings", "Products", "Publications", "Purchase options", "Reports", "Resource feedback", "Returns", "Sales channels", "Script tags", "Shipping", "Shop locales", "Shopify Markets", "Shopify Payments accounts", "Shopify Payments bank accounts", "Shopify Payments disputes", "Shopify Payments payouts", "Store content", "Store credit account transactions", "Store credit accounts", "Themes", "Third-party fulfillment orders", "Translations", "all_cart_transforms", "all_checkout_completion_target_customizations", "cart_transforms", "cash_tracking", "companies", "custom_fulfillment_services", "customer_data_erasure", "customer_merge", "delivery_customizations", "delivery_option_generators", "discounts_allocator_functions", "fulfillment_constraint_rules", "gates", "order_submission_rules", "privacy_settings", "shopify_payments_provider_accounts_sensitive", "validations"}
)
func (Analyzer) Type() analyzerpb.AnalyzerType { return analyzerpb.AnalyzerType_Shopify }
func (Analyzer) Type() analyzers.AnalyzerType { return analyzers.AnalyzerTypeShopify }
func (a Analyzer) Analyze(_ context.Context, credInfo map[string]string) (*analyzers.AnalyzerResult, error) {
key, ok := credInfo["key"]
@ -55,7 +54,7 @@ func secretInfoToAnalyzerResult(info *SecretInfo) *analyzers.AnalyzerResult {
return nil
}
result := analyzers.AnalyzerResult{
AnalyzerType: analyzerpb.AnalyzerType_Shopify,
AnalyzerType: analyzers.AnalyzerTypeShopify,
Metadata: map[string]any{
"status_code": info.StatusCode,
},

5
pkg/analyzer/analyzers/slack/slack.go
View file

@ -15,7 +15,6 @@ import (
"github.com/jedib0t/go-pretty/table"
"github.com/trufflesecurity/trufflehog/v3/pkg/analyzer/analyzers"
"github.com/trufflesecurity/trufflehog/v3/pkg/analyzer/config"
"github.com/trufflesecurity/trufflehog/v3/pkg/analyzer/pb/analyzerpb"
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
)
@ -25,7 +24,7 @@ type Analyzer struct {
Cfg *config.Config
}
func (Analyzer) Type() analyzerpb.AnalyzerType { return analyzerpb.AnalyzerType_Slack }
func (Analyzer) Type() analyzers.AnalyzerType { return analyzers.AnalyzerTypeSlack }
func (a Analyzer) Analyze(_ context.Context, credInfo map[string]string) (*analyzers.AnalyzerResult, error) {
key, ok := credInfo["key"]
@ -45,7 +44,7 @@ func secretInfoToAnalyzerResult(info *SecretInfo) *analyzers.AnalyzerResult {
return nil
}
result := analyzers.AnalyzerResult{
AnalyzerType: analyzerpb.AnalyzerType_Slack,
AnalyzerType: analyzers.AnalyzerTypeSlack,
Metadata: nil,
}

5
pkg/analyzer/analyzers/sourcegraph/sourcegraph.go
View file

@ -12,7 +12,6 @@ import (
"github.com/fatih/color"
"github.com/trufflesecurity/trufflehog/v3/pkg/analyzer/analyzers"
"github.com/trufflesecurity/trufflehog/v3/pkg/analyzer/config"
"github.com/trufflesecurity/trufflehog/v3/pkg/analyzer/pb/analyzerpb"
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
)
@ -22,7 +21,7 @@ type Analyzer struct {
Cfg *config.Config
}
func (Analyzer) Type() analyzerpb.AnalyzerType { return analyzerpb.AnalyzerType_Sourcegraph }
func (Analyzer) Type() analyzers.AnalyzerType { return analyzers.AnalyzerTypeSourcegraph }
func (a Analyzer) Analyze(_ context.Context, credInfo map[string]string) (*analyzers.AnalyzerResult, error) {
key, ok := credInfo["key"]
@ -46,7 +45,7 @@ func secretInfoToAnalyzerResult(info *SecretInfo) *analyzers.AnalyzerResult {
permission = PermissionStrings[SiteAdminFull]
}
result := analyzers.AnalyzerResult{
AnalyzerType: analyzerpb.AnalyzerType_Sourcegraph,
AnalyzerType: analyzers.AnalyzerTypeSourcegraph,
Metadata: nil,
Bindings: []analyzers.Binding{
{

5
pkg/analyzer/analyzers/square/square.go
View file

@ -14,7 +14,6 @@ import (
"github.com/jedib0t/go-pretty/table"
"github.com/trufflesecurity/trufflehog/v3/pkg/analyzer/analyzers"
"github.com/trufflesecurity/trufflehog/v3/pkg/analyzer/config"
"github.com/trufflesecurity/trufflehog/v3/pkg/analyzer/pb/analyzerpb"
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
)
@ -24,7 +23,7 @@ type Analyzer struct {
Cfg *config.Config
}
func (Analyzer) Type() analyzerpb.AnalyzerType { return analyzerpb.AnalyzerType_Square }
func (Analyzer) Type() analyzers.AnalyzerType { return analyzers.AnalyzerTypeSquare }
func (a Analyzer) Analyze(_ context.Context, credInfo map[string]string) (*analyzers.AnalyzerResult, error) {
key, ok := credInfo["key"]
@ -43,7 +42,7 @@ func secretInfoToAnalyzerResult(info *SecretInfo) *analyzers.AnalyzerResult {
return nil
}
result := analyzers.AnalyzerResult{
AnalyzerType: analyzerpb.AnalyzerType_Square,
AnalyzerType: analyzers.AnalyzerTypeSquare,
UnboundedResources: []analyzers.Resource{},
Metadata: map[string]any{
"expires_at": info.Permissions.ExpiresAt,

5
pkg/analyzer/analyzers/stripe/stripe.go
View file

@ -18,7 +18,6 @@ import (
"github.com/jedib0t/go-pretty/table"
"github.com/trufflesecurity/trufflehog/v3/pkg/analyzer/analyzers"
"github.com/trufflesecurity/trufflehog/v3/pkg/analyzer/config"
"github.com/trufflesecurity/trufflehog/v3/pkg/analyzer/pb/analyzerpb"
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
"gopkg.in/yaml.v2"
)
@ -29,7 +28,7 @@ type Analyzer struct {
Cfg *config.Config
}
func (Analyzer) Type() analyzerpb.AnalyzerType { return analyzerpb.AnalyzerType_Stripe }
func (Analyzer) Type() analyzers.AnalyzerType { return analyzers.AnalyzerTypeStripe }
func (a Analyzer) Analyze(_ context.Context, credInfo map[string]string) (*analyzers.AnalyzerResult, error) {
key, ok := credInfo["key"]
@ -49,7 +48,7 @@ func secretInfoToAnalyzerResult(info *SecretInfo) *analyzers.AnalyzerResult {
return nil
}
result := &analyzers.AnalyzerResult{
AnalyzerType: analyzerpb.AnalyzerType_Stripe,
AnalyzerType: analyzers.AnalyzerTypeStripe,
Metadata: map[string]any{
"key_type": info.KeyType,
"key_env": info.KeyEnv,

7
pkg/analyzer/analyzers/twilio/twilio.go
View file

@ -11,7 +11,6 @@ import (
"github.com/fatih/color"
"github.com/trufflesecurity/trufflehog/v3/pkg/analyzer/analyzers"
"github.com/trufflesecurity/trufflehog/v3/pkg/analyzer/config"
"github.com/trufflesecurity/trufflehog/v3/pkg/analyzer/pb/analyzerpb"
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
)
@ -19,8 +18,8 @@ type Analyzer struct {
Cfg *config.Config
}
func (a *Analyzer) Type() analyzerpb.AnalyzerType {
return analyzerpb.AnalyzerType_Twilio
func (a *Analyzer) Type() analyzers.AnalyzerType {
return analyzers.AnalyzerTypeTwilio
}
func (a *Analyzer) Analyze(ctx context.Context, credentialInfo map[string]string) (*analyzers.AnalyzerResult, error) {
@ -118,7 +117,7 @@ func (a *Analyzer) Analyze(ctx context.Context, credentialInfo map[string]string
}
return &analyzers.AnalyzerResult{
AnalyzerType: analyzerpb.AnalyzerType_Twilio,
AnalyzerType: analyzers.AnalyzerTypeTwilio,
Bindings: bindings,
}, nil
}

6
pkg/analyzer/cli.go
View file

@ -40,11 +40,11 @@ func Command(app *kingpin.Application) *kingpin.CmdClause {
keyTypeHelp := fmt.Sprintf(
"Type of key to analyze. Omit to interactively choose. Available key types: %s",
strings.Join(analyzers.AvailableAnalyzers, ", "),
strings.Join(analyzers.AvailableAnalyzers(), ", "),
)
// Lowercase the available analyzers.
availableAnalyzers := make([]string, len(analyzers.AvailableAnalyzers))
for i, a := range analyzers.AvailableAnalyzers {
availableAnalyzers := make([]string, len(analyzers.AvailableAnalyzers()))
for i, a := range analyzers.AvailableAnalyzers() {
availableAnalyzers[i] = strings.ToLower(a)
}
analyzeKeyType = cli.Arg("key-type", keyTypeHelp).Enum(availableAnalyzers...)

203
pkg/analyzer/pb/analyzerpb/analyzer.pb.go
View file

@ -1,203 +0,0 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.33.0
// protoc v4.25.3
// source: analyzer.proto
package analyzerpb
import (
protoreflect "google.golang.org/protobuf/reflect/protoreflect"
protoimpl "google.golang.org/protobuf/runtime/protoimpl"
reflect "reflect"
sync "sync"
)
const (
// Verify that this generated code is sufficiently up-to-date.
_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
// Verify that runtime/protoimpl is sufficiently up-to-date.
_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
)
type AnalyzerType int32
const (
AnalyzerType_Invalid AnalyzerType = 0
AnalyzerType_Airbrake AnalyzerType = 1
AnalyzerType_Asana AnalyzerType = 2
AnalyzerType_Bitbucket AnalyzerType = 3
AnalyzerType_GitHub AnalyzerType = 4
AnalyzerType_GitLab AnalyzerType = 5
AnalyzerType_HuggingFace AnalyzerType = 6
AnalyzerType_Mailchimp AnalyzerType = 7
AnalyzerType_Mailgun AnalyzerType = 8
AnalyzerType_MySQL AnalyzerType = 9
AnalyzerType_OpenAI AnalyzerType = 10
AnalyzerType_Opsgenie AnalyzerType = 11
AnalyzerType_Postgres AnalyzerType = 12
AnalyzerType_Postman AnalyzerType = 13
AnalyzerType_Sendgrid AnalyzerType = 14
AnalyzerType_Shopify AnalyzerType = 15
AnalyzerType_Slack AnalyzerType = 16
AnalyzerType_Sourcegraph AnalyzerType = 17
AnalyzerType_Square AnalyzerType = 18
AnalyzerType_Stripe AnalyzerType = 19
AnalyzerType_Twilio AnalyzerType = 20
)
// Enum value maps for AnalyzerType.
var (
AnalyzerType_name = map[int32]string{
0: "Invalid",
1: "Airbrake",
2: "Asana",
3: "Bitbucket",
4: "GitHub",
5: "GitLab",
6: "HuggingFace",
7: "Mailchimp",
8: "Mailgun",
9: "MySQL",
10: "OpenAI",
11: "Opsgenie",
12: "Postgres",
13: "Postman",
14: "Sendgrid",
15: "Shopify",
16: "Slack",
17: "Sourcegraph",
18: "Square",
19: "Stripe",
20: "Twilio",
}
AnalyzerType_value = map[string]int32{
"Invalid": 0,
"Airbrake": 1,
"Asana": 2,
"Bitbucket": 3,
"GitHub": 4,
"GitLab": 5,
"HuggingFace": 6,
"Mailchimp": 7,
"Mailgun": 8,
"MySQL": 9,
"OpenAI": 10,
"Opsgenie": 11,
"Postgres": 12,
"Postman": 13,
"Sendgrid": 14,
"Shopify": 15,
"Slack": 16,
"Sourcegraph": 17,
"Square": 18,
"Stripe": 19,
"Twilio": 20,
}
)
func (x AnalyzerType) Enum() *AnalyzerType {
p := new(AnalyzerType)
*p = x
return p
}
func (x AnalyzerType) String() string {
return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x))
}
func (AnalyzerType) Descriptor() protoreflect.EnumDescriptor {
return file_analyzer_proto_enumTypes[0].Descriptor()
}
func (AnalyzerType) Type() protoreflect.EnumType {
return &file_analyzer_proto_enumTypes[0]
}
func (x AnalyzerType) Number() protoreflect.EnumNumber {
return protoreflect.EnumNumber(x)
}
// Deprecated: Use AnalyzerType.Descriptor instead.
func (AnalyzerType) EnumDescriptor() ([]byte, []int) {
return file_analyzer_proto_rawDescGZIP(), []int{0}
}
var File_analyzer_proto protoreflect.FileDescriptor
var file_analyzer_proto_rawDesc = []byte{
0x0a, 0x0e, 0x61, 0x6e, 0x61, 0x6c, 0x79, 0x7a, 0x65, 0x72, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f,
0x12, 0x08, 0x61, 0x6e, 0x61, 0x6c, 0x79, 0x7a, 0x65, 0x72, 0x2a, 0xa3, 0x02, 0x0a, 0x0c, 0x41,
0x6e, 0x61, 0x6c, 0x79, 0x7a, 0x65, 0x72, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x49,
0x6e, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x10, 0x00, 0x12, 0x0c, 0x0a, 0x08, 0x41, 0x69, 0x72, 0x62,
0x72, 0x61, 0x6b, 0x65, 0x10, 0x01, 0x12, 0x09, 0x0a, 0x05, 0x41, 0x73, 0x61, 0x6e, 0x61, 0x10,
0x02, 0x12, 0x0d, 0x0a, 0x09, 0x42, 0x69, 0x74, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x10, 0x03,
0x12, 0x0a, 0x0a, 0x06, 0x47, 0x69, 0x74, 0x48, 0x75, 0x62, 0x10, 0x04, 0x12, 0x0a, 0x0a, 0x06,
0x47, 0x69, 0x74, 0x4c, 0x61, 0x62, 0x10, 0x05, 0x12, 0x0f, 0x0a, 0x0b, 0x48, 0x75, 0x67, 0x67,
0x69, 0x6e, 0x67, 0x46, 0x61, 0x63, 0x65, 0x10, 0x06, 0x12, 0x0d, 0x0a, 0x09, 0x4d, 0x61, 0x69,
0x6c, 0x63, 0x68, 0x69, 0x6d, 0x70, 0x10, 0x07, 0x12, 0x0b, 0x0a, 0x07, 0x4d, 0x61, 0x69, 0x6c,
0x67, 0x75, 0x6e, 0x10, 0x08, 0x12, 0x09, 0x0a, 0x05, 0x4d, 0x79, 0x53, 0x51, 0x4c, 0x10, 0x09,
0x12, 0x0a, 0x0a, 0x06, 0x4f, 0x70, 0x65, 0x6e, 0x41, 0x49, 0x10, 0x0a, 0x12, 0x0c, 0x0a, 0x08,
0x4f, 0x70, 0x73, 0x67, 0x65, 0x6e, 0x69, 0x65, 0x10, 0x0b, 0x12, 0x0c, 0x0a, 0x08, 0x50, 0x6f,
0x73, 0x74, 0x67, 0x72, 0x65, 0x73, 0x10, 0x0c, 0x12, 0x0b, 0x0a, 0x07, 0x50, 0x6f, 0x73, 0x74,
0x6d, 0x61, 0x6e, 0x10, 0x0d, 0x12, 0x0c, 0x0a, 0x08, 0x53, 0x65, 0x6e, 0x64, 0x67, 0x72, 0x69,
0x64, 0x10, 0x0e, 0x12, 0x0b, 0x0a, 0x07, 0x53, 0x68, 0x6f, 0x70, 0x69, 0x66, 0x79, 0x10, 0x0f,
0x12, 0x09, 0x0a, 0x05, 0x53, 0x6c, 0x61, 0x63, 0x6b, 0x10, 0x10, 0x12, 0x0f, 0x0a, 0x0b, 0x53,
0x6f, 0x75, 0x72, 0x63, 0x65, 0x67, 0x72, 0x61, 0x70, 0x68, 0x10, 0x11, 0x12, 0x0a, 0x0a, 0x06,
0x53, 0x71, 0x75, 0x61, 0x72, 0x65, 0x10, 0x12, 0x12, 0x0a, 0x0a, 0x06, 0x53, 0x74, 0x72, 0x69,
0x70, 0x65, 0x10, 0x13, 0x12, 0x0a, 0x0a, 0x06, 0x54, 0x77, 0x69, 0x6c, 0x69, 0x6f, 0x10, 0x14,
0x42, 0x45, 0x5a, 0x43, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x74,
0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x73, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x2f, 0x74,
0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x68, 0x6f, 0x67, 0x2f, 0x76, 0x33, 0x2f, 0x70, 0x6b, 0x67,
0x2f, 0x61, 0x6e, 0x61, 0x6c, 0x79, 0x7a, 0x65, 0x72, 0x2f, 0x70, 0x62, 0x2f, 0x61, 0x6e, 0x61,
0x6c, 0x79, 0x7a, 0x65, 0x72, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
}
var (
file_analyzer_proto_rawDescOnce sync.Once
file_analyzer_proto_rawDescData = file_analyzer_proto_rawDesc
)
func file_analyzer_proto_rawDescGZIP() []byte {
file_analyzer_proto_rawDescOnce.Do(func() {
file_analyzer_proto_rawDescData = protoimpl.X.CompressGZIP(file_analyzer_proto_rawDescData)
})
return file_analyzer_proto_rawDescData
}
var file_analyzer_proto_enumTypes = make([]protoimpl.EnumInfo, 1)
var file_analyzer_proto_goTypes = []interface{}{
(AnalyzerType)(0), // 0: analyzer.AnalyzerType
}
var file_analyzer_proto_depIdxs = []int32{
0, // [0:0] is the sub-list for method output_type
0, // [0:0] is the sub-list for method input_type
0, // [0:0] is the sub-list for extension type_name
0, // [0:0] is the sub-list for extension extendee
0, // [0:0] is the sub-list for field type_name
}
func init() { file_analyzer_proto_init() }
func file_analyzer_proto_init() {
if File_analyzer_proto != nil {
return
}
type x struct{}
out := protoimpl.TypeBuilder{
File: protoimpl.DescBuilder{
GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
RawDescriptor: file_analyzer_proto_rawDesc,
NumEnums: 1,
NumMessages: 0,
NumExtensions: 0,
NumServices: 0,
},
GoTypes: file_analyzer_proto_goTypes,
DependencyIndexes: file_analyzer_proto_depIdxs,
EnumInfos: file_analyzer_proto_enumTypes,
}.Build()
File_analyzer_proto = out.File
file_analyzer_proto_rawDesc = nil
file_analyzer_proto_goTypes = nil
file_analyzer_proto_depIdxs = nil
}

36
pkg/analyzer/pb/analyzerpb/analyzer.pb.validate.go
View file

@ -1,36 +0,0 @@
// Code generated by protoc-gen-validate. DO NOT EDIT.
// source: analyzer.proto
package analyzerpb
import (
"bytes"
"errors"
"fmt"
"net"
"net/mail"
"net/url"
"regexp"
"sort"
"strings"
"time"
"unicode/utf8"
"google.golang.org/protobuf/types/known/anypb"
)
// ensure the imports are used
var (
_ = bytes.MinRead
_ = errors.New("")
_ = fmt.Print
_ = utf8.UTFMax
_ = (*regexp.Regexp)(nil)
_ = (*strings.Reader)(nil)
_ = net.IPv4len
_ = time.Duration(0)
_ = (*url.URL)(nil)
_ = (*mail.Address)(nil)
_ = anypb.Any{}
_ = sort.Sort
)

29
pkg/analyzer/proto/analyzer.proto
View file

@ -1,29 +0,0 @@
syntax = "proto3";
package analyzer;
option go_package = "github.com/trufflesecurity/trufflehog/v3/pkg/analyzer/pb/analyzerpb";
enum AnalyzerType {
Invalid = 0;
Airbrake = 1;
Asana = 2;
Bitbucket = 3;
GitHub = 4;
GitLab = 5;
HuggingFace = 6;
Mailchimp = 7;
Mailgun = 8;
MySQL = 9;
OpenAI = 10;
Opsgenie = 11;
Postgres = 12;
Postman = 13;
Sendgrid = 14;
Shopify = 15;
Slack = 16;
Sourcegraph = 17;
Square = 18;
Stripe = 19;
Twilio = 20;
}

2
pkg/analyzer/tui/form.go
View file

@ -112,7 +112,7 @@ func (ui FormPage) View() string {
func (ui FormPage) PrevPage() (tea.Model, tea.Cmd) {
page := NewKeyTypePage(ui.Common)
// Select what was previously selected.
index, ok := slices.BinarySearch(analyzers.AvailableAnalyzers, ui.KeyType)
index, ok := slices.BinarySearch(analyzers.AvailableAnalyzers(), ui.KeyType)
if !ok {
// Should be impossible.
index = 0

4
pkg/analyzer/tui/keytype.go
View file

@ -34,8 +34,8 @@ func (ui KeyTypePage) Init() tea.Cmd {
}
func NewKeyTypePage(c *common.Common) KeyTypePage {
items := make([]list.Item, len(analyzers.AvailableAnalyzers))
for i, analyzerType := range analyzers.AvailableAnalyzers {
items := make([]list.Item, len(analyzers.AvailableAnalyzers()))
for i, analyzerType := range analyzers.AvailableAnalyzers() {
items[i] = KeyTypeItem(analyzerType)
}
delegate := list.NewDefaultDelegate()

2
pkg/analyzer/tui/tui.go
View file

@ -32,7 +32,7 @@ func Run(keyType string) (string, *SecretInfo, error) {
// If a keyType is provided, make sure it's in the list of AvailableAnalyzers.
if keyType != "" {
var found bool
for _, a := range analyzers.AvailableAnalyzers {
for _, a := range analyzers.AvailableAnalyzers() {
if strings.EqualFold(a, keyType) {
keyType = a
found = true

2
pkg/common/patterns.go
View file

@ -7,7 +7,7 @@ import (
"strings"
)
const EmailPattern = `\b(?:[a-z0-9!#$%&'*+/=?^_\x60{|}~-]+(?:\.[a-z0-9!#$%&'*+/=?^_\x60{|}~-]+)*|"(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21\x23-\x5b\x5d-\x7f]|\\[\x01-\x09\x0b\x0c\x0e-\x7f])*")@(?:(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.)+[a-z0-9](?:[a-z0-9-]*[a-z0-9])?|\[(?:(?:(2(5[0-5]|[0-4][0-9])|1[0-9][0-9]|[1-9]?[0-9]))\.){3}(?:(2(5[0-5]|[0-4][0-9])|1[0-9][0-9]|[1-9]?[0-9])|[a-z0-9-]*[a-z0-9]:(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21-\x5a\x53-\x7f]|\\[\x01-\x09\x0b\x0c\x0e-\x7f])+)\])\b`
const EmailPattern = `\b((?:[a-z0-9!#$%&'*+/=?^_\x60{|}~-]+(?:\.[a-z0-9!#$%&'*+/=?^_\x60{|}~-]+)*|"(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21\x23-\x5b\x5d-\x7f]|\\[\x01-\x09\x0b\x0c\x0e-\x7f])*")@(?:(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.)+[a-z0-9](?:[a-z0-9-]*[a-z0-9])?|\[(?:(?:(2(5[0-5]|[0-4][0-9])|1[0-9][0-9]|[1-9]?[0-9]))\.){3}(?:(2(5[0-5]|[0-4][0-9])|1[0-9][0-9]|[1-9]?[0-9])|[a-z0-9-]*[a-z0-9]:(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21-\x5a\x53-\x7f]|\\[\x01-\x09\x0b\x0c\x0e-\x7f])+)\]))\b`
const SubDomainPattern = `\b([A-Za-z0-9](?:[A-Za-z0-9\-]{0,61}[A-Za-z0-9])?)\b`
const UUIDPattern = `\b([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})\b`
const UUIDPatternUpperCase = `\b([0-9A-Z]{8}-[0-9A-Z]{4}-[0-9A-Z]{4}-[0-9A-Z]{4}-[0-9A-Z]{12})\b`

44
pkg/common/patterns_test.go
View file

@ -1,9 +1,10 @@
package common
import (
"github.com/stretchr/testify/assert"
"regexp"
"testing"
"github.com/stretchr/testify/assert"
)
const (
@ -13,6 +14,47 @@ const (
passwordRegex = `(?im)(?:pass|password)\S{0,40}?[:=\s]{1,3}[ '"=]{0,1}([^:^<>;.*&|£\n\s]{4,40})`
)
func TestEmailRegexCheck(t *testing.T) {
testEmails := `
// positive cases
standard email = john.doe@example.com
subdomain email = jane_doe123@sub.domain.co.us
organization email = alice.smith@test.org
test email = bob@test.name
with tag email = user.name+tag@domain.com
hyphen domain = info@my-site.net
service email = contact@web-service.io
underscore email = example_user@domain.info
departement email = first.last@department.company.edu
alphanumeric email = user1234@domain.co
local server email = admin@local-server.local
dot email = test.email@my-email-service.xyz
special char email = special@characters.com
support email = support@customer-service.org
// negative cases
not an email = abc.123@z
looks like email = test@user <- no domain
email but not = user12@service.COM <- capital letters not supported for domain
random text = here's some information about local-user@edu user
`
expectedStr := []string{
"john.doe@example.com", "jane_doe123@sub.domain.co.us",
"alice.smith@test.org", "bob@test.name", "user.name+tag@domain.com",
"info@my-site.net", "contact@web-service.io", "example_user@domain.info",
"first.last@department.company.edu", "user1234@domain.co", "admin@local-server.local",
"test.email@my-email-service.xyz", "special@characters.com", "support@customer-service.org",
}
emailRegex := regexp.MustCompile(EmailPattern)
emailMatches := emailRegex.FindAllString(testEmails, -1)
assert.Exactly(t, emailMatches, expectedStr)
}
func TestUsernameRegexCheck(t *testing.T) {
usernameRegexPat := UsernameRegexCheck(usernamePattern)

34
pkg/custom_detectors/custom_detectors.go
View file

@ -4,6 +4,7 @@ import (
"bytes"
"context"
"encoding/json"
"io"
"net/http"
"regexp"
"strings"
@ -101,10 +102,6 @@ func (c *CustomRegexWebhook) FromData(ctx context.Context, verify bool, data []b
close(resultsCh)
for result := range resultsCh {
// NOTE: I don't believe this is being set anywhere else, hence the map assignment.
result.ExtraData = map[string]string{
"name": c.GetName(),
}
results = append(results, result)
}
@ -129,6 +126,7 @@ func (c *CustomRegexWebhook) createResults(ctx context.Context, match map[string
DetectorType: detectorspb.DetectorType_CustomRegex,
DetectorName: c.GetName(),
Raw: []byte(raw),
ExtraData: map[string]string{},
}
if !verify {
@ -166,14 +164,34 @@ func (c *CustomRegexWebhook) createResults(ctx context.Context, match map[string
}
req.Header.Add(key, strings.TrimLeft(value, "\t\n\v\f\r "))
}
res, err := httpClient.Do(req)
resp, err := httpClient.Do(req)
if err != nil {
continue
}
// TODO: Read response body.
res.Body.Close()
if res.StatusCode == http.StatusOK {
defer func() {
_, _ = io.Copy(io.Discard, resp.Body)
_ = resp.Body.Close()
}()
if resp.StatusCode == http.StatusOK {
// mark the result as verified
result.Verified = true
body, err := io.ReadAll(resp.Body)
if err != nil {
continue
}
// TODO: handle different content-type responses seperatly when implement custom detector configurations
responseStr := string(body)
// truncate to 200 characters if response length exceeds 200
if len(responseStr) > 200 {
responseStr = responseStr[:200]
}
// store the processed response in ExtraData
result.ExtraData["response"] = responseStr
break
}
}

4
pkg/detectors/aeroworkflow/aeroworkflow.go
View file

@ -27,7 +27,7 @@ var (
defaultClient = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"aeroworkflow"}) + `([a-zA-Z0-9^!?#:*;]{20})\b`)
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"aeroworkflow"}) + `\b([a-zA-Z0-9^!?#:*;]{20})`)
idPat = regexp.MustCompile(detectors.PrefixRegex([]string{"aeroworkflow"}) + `\b([0-9]{1,})\b`)
)
@ -85,7 +85,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
}
func verifyAeroworkflow(ctx context.Context, client *http.Client, resMatch, resIdMatch string) (bool, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, aeroworkflowURL+"/api/"+resIdMatch+"/v1/AeroAppointments", nil)
req, err := http.NewRequestWithContext(ctx, http.MethodGet, aeroworkflowURL+"/api/"+resIdMatch+"/me", nil)
if err != nil {
return false, err
}

80
pkg/detectors/alegra/alegra.go
View file

@ -2,6 +2,8 @@ package alegra
import (
"context"
"fmt"
"io"
"net/http"
"strings"
@ -13,18 +15,17 @@ import (
)
type Scanner struct {
detectors.DefaultMultiPartCredentialProvider
client *http.Client
}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var (
client = common.SaneHttpClient()
defaultClient = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"alegra"}) + `\b([a-z0-9-]{20})\b`)
idPat = regexp.MustCompile(detectors.PrefixRegex([]string{"alegra"}) + `\b([a-zA-Z0-9\.\-\@]{25,30})\b`)
idPat = regexp.MustCompile(detectors.PrefixRegex([]string{"alegra"}) + common.EmailPattern)
)
// Keywords are used for efficiently pre-filtering chunks.
@ -37,41 +38,38 @@ func (s Scanner) Keywords() []string {
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
keyMatches := keyPat.FindAllStringSubmatch(dataStr, -1)
idMatches := idPat.FindAllStringSubmatch(dataStr, -1)
for _, match := range matches {
if len(match) != 2 {
continue
}
tokenPatMatch := strings.TrimSpace(match[1])
uniqueTokens := make(map[string]struct{})
uniqueIDs := make(map[string]struct{})
for _, idMatch := range idMatches {
if len(idMatch) != 2 {
continue
}
for _, match := range keyMatches {
uniqueTokens[match[1]] = struct{}{}
}
userPatMatch := strings.TrimSpace(idMatch[1])
for _, match := range idMatches {
id := match[0][strings.LastIndex(match[0], " ")+1:]
uniqueIDs[id] = struct{}{}
}
for token := range uniqueTokens {
for id := range uniqueIDs {
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_Alegra,
Raw: []byte(tokenPatMatch),
RawV2: []byte(tokenPatMatch + userPatMatch),
Raw: []byte(token),
RawV2: []byte(token + ":" + id),
}
if verify {
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.alegra.com/api/v1/users", nil)
if err != nil {
continue
}
req.SetBasicAuth(userPatMatch, tokenPatMatch)
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
}
client := s.client
if client == nil {
client = defaultClient
}
isVerified, verificationErr := verifyCredentials(ctx, client, id, token)
s1.Verified = isVerified
s1.SetVerificationError(verificationErr, token)
}
results = append(results, s1)
@ -81,6 +79,32 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
return results, nil
}
func verifyCredentials(ctx context.Context, client *http.Client, username, token string) (bool, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://api.alegra.com/api/v1/users/self", nil)
if err != nil {
return false, nil
}
req.SetBasicAuth(username, token)
res, err := client.Do(req)
if err != nil {
return false, err
}
defer func() {
_, _ = io.Copy(io.Discard, res.Body)
_ = res.Body.Close()
}()
switch res.StatusCode {
case http.StatusOK:
return true, nil
case http.StatusUnauthorized:
return false, nil
default:
return false, fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
}
}
func (s Scanner) Type() detectorspb.DetectorType {
return detectorspb.DetectorType_Alegra
}

1
pkg/detectors/alegra/alegra_integration_test.go
View file

@ -96,6 +96,7 @@ func TestAlegra_FromChunk(t *testing.T) {
t.Fatalf("no raw secret present: \n %+v", got[i])
}
got[i].Raw = nil
got[i].RawV2 = nil
}
if diff := pretty.Compare(got, tt.want); diff != "" {
t.Errorf("Alegra.FromData() %s diff: (-got +want)\n%s", tt.name, diff)

22
pkg/detectors/alegra/alegra_test.go
View file

@ -14,7 +14,7 @@ import (
var (
validPattern = "wdvn-usa87a-fxp9ioas/testUser.1005@example.com"
validSpecialCharPattern = "wdvn-usa87a-fxp9ioas / test-User.1005@example.com"
invalidPattern = "wdvn-usa87a-fxp9ioas/testUser$1005@example.com"
invalidPattern = "wdvn-usa87a-fxp9ioasQQsstestUsQQ@example"
)
func TestAlegra_Pattern(t *testing.T) {
@ -28,22 +28,22 @@ func TestAlegra_Pattern(t *testing.T) {
}{
{
name: "valid pattern",
input: fmt.Sprintf("alegra: '%s'", validPattern),
want: []string{"wdvn-usa87a-fxp9ioastestUser.1005@example.com"},
input: fmt.Sprintf("alegra: %s", validPattern),
want: []string{"wdvn-usa87a-fxp9ioas:wdvn-usa87a-fxp9ioas/testUser.1005@example.com"},
},
{
name: "valid pattern - with special characters",
input: fmt.Sprintf("alegra: '%s'", validSpecialCharPattern),
want: []string{"wdvn-usa87a-fxp9ioastest-User.1005@example.com"},
input: fmt.Sprintf("alegra: %s", validSpecialCharPattern),
want: []string{"wdvn-usa87a-fxp9ioas:test-User.1005@example.com"},
},
{
name: "valid pattern - key out of prefix range",
input: fmt.Sprintf("alegra keyword is not close to the real key and id = '%s'", validPattern),
input: fmt.Sprintf("alegra keyword is not close to the real key and id = %s", validPattern),
want: nil,
},
{
name: "invalid pattern",
input: fmt.Sprintf("alegra: '%s'", invalidPattern),
input: fmt.Sprintf("alegra: %s", invalidPattern),
want: nil,
},
}
@ -51,7 +51,7 @@ func TestAlegra_Pattern(t *testing.T) {
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
if len(matchedDetectors) == 0 {
if len(matchedDetectors) == 0 && test.want != nil {
t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
@ -63,11 +63,7 @@ func TestAlegra_Pattern(t *testing.T) {
}
if len(results) != len(test.want) {
if len(results) == 0 {
t.Errorf("did not receive result")
} else {
t.Errorf("expected %d results, only received %d", len(test.want), len(results))
}
t.Errorf("expected %d results, got %d", len(test.want), len(results))
return
}

91
pkg/detectors/algoliaadminkey/algoliaadminkey.go
View file

@ -2,6 +2,8 @@ package algoliaadminkey
import (
"context"
"fmt"
"encoding/json"
regexp "github.com/wasilibs/go-re2"
"net/http"
"strings"
@ -22,14 +24,14 @@ var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"algolia"}) + `\b([a-zA-Z0-9]{32})\b`)
idPat = regexp.MustCompile(detectors.PrefixRegex([]string{"algolia"}) + `\b([A-Z0-9]{10})\b`)
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"algolia", "docsearch", "apiKey"}) + `\b([a-zA-Z0-9]{32})\b`)
idPat = regexp.MustCompile(detectors.PrefixRegex([]string{"algolia", "docsearch", "appId"}) + `\b([A-Z0-9]{10})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"algolia"}
return []string{"algolia", "docsearch"}
}
// FromData will find and optionally verify AlgoliaAdminKey secrets in a given set of bytes.
@ -57,19 +59,16 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
}
if verify {
req, err := http.NewRequestWithContext(ctx, "GET", "https://"+resIdMatch+"-dsn.algolia.net/1/keys", nil)
if err != nil {
continue
}
req.Header.Add("X-Algolia-Application-Id", resIdMatch)
req.Header.Add("X-Algolia-API-Key", resMatch)
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
}
// Verify if the key is a valid Algolia Admin Key.
isVerified, verificationErr := verifyAlgoliaKey(ctx, resIdMatch, resMatch)
// Verify if the key has sensitive permissions, even if it's not an Admin Key.
if !isVerified {
isVerified, verificationErr = verifyAlgoliaKeyACL(ctx, resIdMatch, resMatch)
}
s1.SetVerificationError(verificationErr, resMatch)
s1.Verified = isVerified
}
results = append(results, s1)
@ -78,6 +77,68 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
return results, nil
}
func verifyAlgoliaKey(ctx context.Context, appId, apiKey string) (bool, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://"+appId+"-dsn.algolia.net/1/keys", nil)
if err != nil {
return false, err
}
req.Header.Add("X-Algolia-Application-Id", appId)
req.Header.Add("X-Algolia-API-Key", apiKey)
res, err := client.Do(req)
if err != nil {
return false, err
}
defer res.Body.Close()
if res.StatusCode == 403 {
return false, nil
} else if res.StatusCode < 200 || res.StatusCode > 299 {
return false, fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
}
return true, nil
}
func verifyAlgoliaKeyACL(ctx context.Context, appId, apiKey string) (bool, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://"+appId+".algolia.net/1/keys/"+apiKey, nil)
if err != nil {
return false, err
}
req.Header.Add("X-Algolia-Application-Id", appId)
req.Header.Add("X-Algolia-API-Key", apiKey)
res, err := client.Do(req)
if err != nil {
return false, err
}
defer res.Body.Close()
if res.StatusCode == 403 {
return false, nil
} else if res.StatusCode < 200 || res.StatusCode > 299 {
return false, fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
}
var jsonResponse struct {
ACL []string `json:"acl"`
}
if err := json.NewDecoder(res.Body).Decode(&jsonResponse); err != nil {
return false, err
}
for _, acl := range jsonResponse.ACL {
if acl != "search" && acl != "listIndexes" && acl != "settings" {
return true, nil // Other permissions are sensitive.
}
}
return false, nil
}
func (s Scanner) Type() detectorspb.DetectorType {
return detectorspb.DetectorType_AlgoliaAdminKey
}

2
pkg/detectors/algoliaadminkey/algoliaadminkey_integration_test.go
View file

@ -51,6 +51,7 @@ func TestAlgoliaAdminKey_FromChunk(t *testing.T) {
{
DetectorType: detectorspb.DetectorType_AlgoliaAdminKey,
Verified: true,
RawV2: []byte(fmt.Sprintf("%s%s", secret, id)),
},
},
wantErr: false,
@ -67,6 +68,7 @@ func TestAlgoliaAdminKey_FromChunk(t *testing.T) {
{
DetectorType: detectorspb.DetectorType_AlgoliaAdminKey,
Verified: false,
RawV2: []byte(fmt.Sprintf("%s%s", inactiveSecret, id)),
},
},
wantErr: false,

19
pkg/detectors/apiscience/apiscience.go → pkg/detectors/apimetrics/apimetrics.go
View file

@ -1,12 +1,13 @@
package apiscience
package apimetrics
import (
"context"
"fmt"
regexp "github.com/wasilibs/go-re2"
"net/http"
"strings"
regexp "github.com/wasilibs/go-re2"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
@ -21,16 +22,16 @@ var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"apiscience"}) + `\b([a-bA-Z0-9\S]{22})\b`)
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"apimetrics"}) + `\b([a-bA-Z0-9\S]{32})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"apiscience"}
return []string{"apimetrics"}
}
// FromData will find and optionally verify ApiScience secrets in a given set of bytes.
// FromData will find and optionally verify ApiMetrics secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
@ -43,12 +44,12 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
resMatch := strings.TrimSpace(match[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_ApiScience,
DetectorType: detectorspb.DetectorType_ApiMetrics,
Raw: []byte(resMatch),
}
if verify {
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.apiscience.com/v1/monitors", nil)
req, err := http.NewRequestWithContext(ctx, "GET", "https://client.apimetrics.io/api/2/calls/", nil)
if err != nil {
continue
}
@ -69,9 +70,9 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
}
func (s Scanner) Type() detectorspb.DetectorType {
return detectorspb.DetectorType_ApiScience
return detectorspb.DetectorType_ApiMetrics
}
func (s Scanner) Description() string {
return "ApiScience is a tool for monitoring the performance of APIs. ApiScience keys can be used to access and manage API monitors."
return "ApiMetrics is a tool for monitoring the performance of APIs. ApiMetrics keys can be used to access and manage API monitors."
}

20
pkg/detectors/apiscience/apiscience_test.go → pkg/detectors/apimetrics/apimetrics_test.go
View file

@ -1,7 +1,7 @@
//go:build detectors
// +build detectors
package apiscience
package apimetrics
import (
"context"
@ -16,15 +16,15 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
func TestApiScience_FromChunk(t *testing.T) {
func TestApiMetrics_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
secret := testSecrets.MustGetField("APISCIENCE")
inactiveSecret := testSecrets.MustGetField("APISCIENCE_INACTIVE")
secret := testSecrets.MustGetField("APIMETRICS")
inactiveSecret := testSecrets.MustGetField("APIMETRICS_INACTIVE")
type args struct {
ctx context.Context
@ -43,12 +43,12 @@ func TestApiScience_FromChunk(t *testing.T) {
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a apiscience secret %s within", secret)),
data: []byte(fmt.Sprintf("You can find a apimetrics secret %s within", secret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_ApiScience,
DetectorType: detectorspb.DetectorType_ApiMetrics,
Verified: true,
},
},
@ -59,12 +59,12 @@ func TestApiScience_FromChunk(t *testing.T) {
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a apiscience secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
data: []byte(fmt.Sprintf("You can find a apimetrics secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_ApiScience,
DetectorType: detectorspb.DetectorType_ApiMetrics,
Verified: false,
},
},
@ -87,7 +87,7 @@ func TestApiScience_FromChunk(t *testing.T) {
s := Scanner{}
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
if (err != nil) != tt.wantErr {
t.Errorf("ApiScience.FromData() error = %v, wantErr %v", err, tt.wantErr)
t.Errorf("ApiMetrics.FromData() error = %v, wantErr %v", err, tt.wantErr)
return
}
for i := range got {
@ -97,7 +97,7 @@ func TestApiScience_FromChunk(t *testing.T) {
got[i].Raw = nil
}
if diff := pretty.Compare(got, tt.want); diff != "" {
t.Errorf("ApiScience.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
t.Errorf("ApiMetrics.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
}
})
}

38
pkg/detectors/autoklose/autoklose.go
View file

@ -4,11 +4,12 @@ import (
"context"
"encoding/json"
"fmt"
regexp "github.com/wasilibs/go-re2"
"io"
"net/http"
"strings"
regexp "github.com/wasilibs/go-re2"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
@ -50,24 +51,37 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
}
if verify {
req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://api.autoklose.com/api/campaigns/?api_token=%s", resMatch), nil)
// API Documentation: https://api.aklab.xyz/#auth-info-fd71acd1-2e41-4991-8789-3edfd258479a
req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://api.autoklose.com/api/me/?api_token=%s", resMatch), nil)
if err != nil {
continue
}
req.Header.Add("Accept", "application/json")
res, err := client.Do(req)
if err == nil {
bodyBytes, err := io.ReadAll(res.Body)
if err != nil {
continue
}
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
if json.Valid(bodyBytes) {
s1.Verified = true
} else {
s1.Verified = false
defer func() {
_, _ = io.Copy(io.Discard, res.Body)
_ = res.Body.Close()
}()
if res.StatusCode == http.StatusOK {
s1.Verified = true
bodyBytes, err := io.ReadAll(res.Body)
if err != nil {
continue
}
var responseBody map[string]interface{}
if err := json.Unmarshal(bodyBytes, &responseBody); err == nil {
if email, ok := responseBody["email"].(string); ok {
s1.ExtraData = map[string]string{
"email": email,
}
}
}
}
} else {
s1.SetVerificationError(err, resMatch)
}
}

3
pkg/detectors/autoklose/autoklose_test.go
View file

@ -50,6 +50,9 @@ func TestAutoklose_FromChunk(t *testing.T) {
{
DetectorType: detectorspb.DetectorType_Autoklose,
Verified: true,
ExtraData: map[string]string{
"email": "mladen.stevanovic@vanillasoft.com",
},
},
},
wantErr: false,

2
pkg/detectors/aws/aws.go
View file

@ -458,5 +458,5 @@ func (s scanner) Type() detectorspb.DetectorType {
}
func (s scanner) Description() string {
return "AWS is a cloud service used offering over 200 API's to transact data and compute. AWS API keys can be used to access and modify this data and compute."
return "AWS (Amazon Web Services) is a comprehensive cloud computing platform offering a wide range of on-demand services like computing power, storage, databases. API keys for AWS can have varying amount of access to these services depending on the IAM policy attached."
}

2
pkg/detectors/awssessionkeys/awssessionkey.go
View file

@ -344,5 +344,5 @@ func (s scanner) Type() detectorspb.DetectorType {
}
func (s scanner) Description() string {
return "AWS is a cloud service used offering over 200 API's to transact data and compute. AWS API keys can be used to access and modify this data and compute."
return "AWS (Amazon Web Services) is a comprehensive cloud computing platform offering a wide range of on-demand services like computing power, storage, databases. API keys for AWS can have varying amount of access to these services depending on the IAM policy attached. AWS Session Tokens are short-lived keys."
}

32
pkg/detectors/ayrshare/ayrshare.go
View file

@ -2,11 +2,14 @@ package ayrshare
import (
"context"
"encoding/json"
"fmt"
regexp "github.com/wasilibs/go-re2"
"io"
"net/http"
"strings"
regexp "github.com/wasilibs/go-re2"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
@ -21,7 +24,7 @@ var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"ayrshare"}) + `\b([A-Z]{7}-[A-Z0-9]{7}-[A-Z0-9]{7}-[A-Z0-9]{7})\b`)
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"ayrshare"}) + `\b([A-Z0-9]{8}-[A-Z0-9]{8}-[A-Z0-9]{8}-[A-Z0-9]{8})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
@ -48,17 +51,36 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
}
if verify {
req, err := http.NewRequestWithContext(ctx, "GET", "https://app.ayrshare.com/api/analytics/links", nil)
req, err := http.NewRequestWithContext(ctx, "GET", "https://app.ayrshare.com/api/user", nil)
if err != nil {
continue
}
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
defer func() {
_, _ = io.Copy(io.Discard, res.Body)
_ = res.Body.Close()
}()
if res.StatusCode == http.StatusOK {
s1.Verified = true
bodyBytes, err := io.ReadAll(res.Body)
if err != nil {
continue
}
var responseBody map[string]interface{}
if err := json.Unmarshal(bodyBytes, &responseBody); err == nil {
if email, ok := responseBody["email"].(string); ok {
s1.ExtraData = map[string]string{
"email": email,
}
}
}
}
} else {
s1.SetVerificationError(err, resMatch)
}
}

127
pkg/detectors/box/box.go Normal file
View file

@ -0,0 +1,127 @@
package box
import (
"context"
"encoding/json"
"fmt"
"io"
"net/http"
regexp "github.com/wasilibs/go-re2"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
type Scanner struct {
client *http.Client
}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var (
defaultClient = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"box"}) + `\b([0-9a-zA-Z]{32})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"box"}
}
func (s Scanner) Description() string {
return "Box is a service offering various service for secure collaboration, content management, and workflow. Box token can be used to access and interact with this data."
}
// FromData will find and optionally verify Box secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
uniqueMatches := make(map[string]struct{})
for _, match := range keyPat.FindAllStringSubmatch(dataStr, -1) {
uniqueMatches[match[1]] = struct{}{}
}
for match := range uniqueMatches {
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_Box,
Raw: []byte(match),
}
if verify {
client := s.client
if client == nil {
client = defaultClient
}
isVerified, extraData, verificationErr := verifyMatch(ctx, client, match)
s1.Verified = isVerified
s1.ExtraData = extraData
s1.SetVerificationError(verificationErr, match)
}
results = append(results, s1)
}
return
}
func verifyMatch(ctx context.Context, client *http.Client, token string) (bool, map[string]string, error) {
url := "https://api.box.com/2.0/users/me"
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return false, nil, nil
}
req.Header = http.Header{"Authorization": []string{"Bearer " + token}}
req.Header.Add("content-type", "application/json")
res, err := client.Do(req)
if err != nil {
return false, nil, err
}
defer func() {
_, _ = io.Copy(io.Discard, res.Body)
_ = res.Body.Close()
}()
switch res.StatusCode {
case http.StatusOK:
{
var u user
if err := json.NewDecoder(res.Body).Decode(&u); err != nil {
return false, nil, err
}
return true, bakeExtraDataFromUser(u), nil
}
case http.StatusUnauthorized:
// 401 access token not found
// The secret is determinately not verified (nothing to do)
return false, nil, nil
default:
return false, nil, fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
}
}
func (s Scanner) Type() detectorspb.DetectorType {
return detectorspb.DetectorType_Box
}
func bakeExtraDataFromUser(u user) map[string]string {
return map[string]string{
"user_id": u.ID,
"username": u.Login,
"user_status": u.Status,
}
}
// struct to represent a Box user.
type user struct {
ID string `json:"id"`
Login string `json:"login"`
Status string `json:"status"`
}

165
pkg/detectors/box/box_integration_test.go Normal file
View file

@ -0,0 +1,165 @@
//go:build detectors
// +build detectors
package box
import (
"context"
"fmt"
"testing"
"time"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
func TestBox_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
token := testSecrets.MustGetField("BOX_ACCESS_TOKEN")
inactiveToken := testSecrets.MustGetField("BOX_ACCESS_TOKEN_INACTIVE")
type args struct {
ctx context.Context
data []byte
verify bool
}
tests := []struct {
name string
s Scanner
args args
want []detectors.Result
wantErr bool
wantVerificationErr bool
}{
{
name: "found, verified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a box token %s within", token)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_Box,
Verified: true,
Raw: []byte(token),
},
},
wantErr: false,
wantVerificationErr: false,
},
{
name: "found, unverified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a box token %s within but not valid", inactiveToken)), // the secret would satisfy the regex but not pass validation
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_Box,
Verified: false,
Raw: []byte(inactiveToken),
},
},
wantErr: false,
wantVerificationErr: false,
},
{
name: "not found",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte("You cannot find the secret within"),
verify: true,
},
want: nil,
wantErr: false,
wantVerificationErr: false,
},
{
name: "found, would be verified if not for timeout",
s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a box token %s within", token)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_Box,
Verified: false,
Raw: []byte(token),
},
},
wantErr: false,
wantVerificationErr: true,
},
{
name: "found, verified but unexpected api surface",
s: Scanner{client: common.ConstantResponseHttpClient(500, "")},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a box secret %s within", token)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_Box,
Verified: false,
Raw: []byte(token),
},
},
wantErr: false,
wantVerificationErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
if (err != nil) != tt.wantErr {
t.Errorf("Box.FromData() error = %v, wantErr %v", err, tt.wantErr)
return
}
for i := range got {
if len(got[i].Raw) == 0 {
t.Fatalf("no raw secret present: \n %+v", got[i])
}
if (got[i].VerificationError() != nil) != tt.wantVerificationErr {
t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())
}
}
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "verificationError", "ExtraData")
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
t.Errorf("Box.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
}
})
}
}
func BenchmarkFromData(benchmark *testing.B) {
ctx := context.Background()
s := Scanner{}
for name, data := range detectors.MustGetBenchmarkData() {
benchmark.Run(name, func(b *testing.B) {
b.ResetTimer()
for n := 0; n < b.N; n++ {
_, err := s.FromData(ctx, false, data)
if err != nil {
b.Fatal(err)
}
}
})
}
}

72
pkg/detectors/box/box_test.go Normal file
View file

@ -0,0 +1,72 @@
//go:build detectors
// +build detectors
package box
import (
"context"
"testing"
"github.com/google/go-cmp/cmp"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
func TestBox_Pattern(t *testing.T) {
d := Scanner{}
ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
name string
input string
want []string
}{
{
name: "typical pattern",
input: "box_access_token = 'Ogowv5cj5AJJjO5F3daNHbKJDdPud0CZ'",
want: []string{"Ogowv5cj5AJJjO5F3daNHbKJDdPud0CZ"},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
if len(matchedDetectors) == 0 {
t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
results, err := d.FromData(context.Background(), false, []byte(test.input))
if err != nil {
t.Errorf("error = %v", err)
return
}
if len(results) != len(test.want) {
if len(results) == 0 {
t.Errorf("did not receive result")
} else {
t.Errorf("expected %d results, only received %d", len(test.want), len(results))
}
return
}
actual := make(map[string]struct{}, len(results))
for _, r := range results {
if len(r.RawV2) > 0 {
actual[string(r.RawV2)] = struct{}{}
} else {
actual[string(r.Raw)] = struct{}{}
}
}
expected := make(map[string]struct{}, len(test.want))
for _, v := range test.want {
expected[v] = struct{}{}
}
if diff := cmp.Diff(expected, actual); diff != "" {
t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
}
})
}
}

138
pkg/detectors/boxoauth/boxoauth.go Normal file
View file

@ -0,0 +1,138 @@
package boxoauth
import (
"context"
"fmt"
"io"
"net/http"
"strings"
regexp "github.com/wasilibs/go-re2"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
type Scanner struct {
client *http.Client
}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var (
defaultClient = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
clientIdPat = regexp.MustCompile(detectors.PrefixRegex([]string{"id"}) + `\b([a-zA-Z0-9]{32})\b`)
clientSecretPat = regexp.MustCompile(detectors.PrefixRegex([]string{"secret"}) + `\b([a-zA-Z0-9]{32})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"box"}
}
func (s Scanner) Description() string {
return "Box is a service offering various service for secure collaboration, content management, and workflow. Box Oauth credentials can be used to access and interact with this data."
}
// FromData will find and optionally verify Box secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
uniqueIdMatches := make(map[string]struct{})
for _, match := range clientIdPat.FindAllStringSubmatch(dataStr, -1) {
uniqueIdMatches[match[1]] = struct{}{}
}
uniqueSecretMatches := make(map[string]struct{})
for _, match := range clientSecretPat.FindAllStringSubmatch(dataStr, -1) {
uniqueSecretMatches[match[1]] = struct{}{}
}
for resIdMatch := range uniqueIdMatches {
for resSecretMatch := range uniqueSecretMatches {
// ignore if the id and secret are the same
if resIdMatch == resSecretMatch {
continue
}
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_BoxOauth,
Raw: []byte(resIdMatch),
RawV2: []byte(resIdMatch + resSecretMatch),
}
if verify {
client := s.client
if client == nil {
client = defaultClient
}
isVerified, extraData, verificationErr := verifyMatch(ctx, client, resIdMatch, resSecretMatch)
s1.Verified = isVerified
s1.ExtraData = extraData
s1.SetVerificationError(verificationErr, resIdMatch)
}
results = append(results, s1)
// box client supportes only one client id and secret pair
if s1.Verified {
break
}
}
}
return
}
func verifyMatch(ctx context.Context, client *http.Client, id string, secret string) (bool, map[string]string, error) {
url := "https://api.box.com/oauth2/token"
payload := strings.NewReader("grant_type=client_credentials&client_id=" + id + "&client_secret=" + secret)
req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, payload)
if err != nil {
return false, nil, nil
}
req.Header = http.Header{"content-type": []string{"application/x-www-form-urlencoded"}}
res, err := client.Do(req)
if err != nil {
return false, nil, err
}
defer func() {
_, _ = io.Copy(io.Discard, res.Body)
_ = res.Body.Close()
}()
// We are using malformed request to check if the client id and secret are correct
// box oauth api returns 400 status code even if they are correct
// so we need to check the response body for the error message with 'unauthorized_client' keyword
// or if invalid_client keyword is present then the client id & secret are incorrect
switch res.StatusCode {
case http.StatusBadRequest:
{
bodyBytes, err := io.ReadAll(res.Body)
if err != nil {
return false, nil, err
}
body := string(bodyBytes)
if strings.Contains(body, "unauthorized_client") {
return true, nil, nil
}
return false, nil, nil
}
default:
return false, nil, fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
}
}
func (s Scanner) Type() detectorspb.DetectorType {
return detectorspb.DetectorType_BoxOauth
}

129
pkg/detectors/boxoauth/boxoauth_integration_test.go Normal file
View file

@ -0,0 +1,129 @@
//go:build detectors
// +build detectors
package boxoauth
import (
"context"
"fmt"
"testing"
"time"
"github.com/kylelemons/godebug/pretty"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
func TestBoxOauth_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
id := testSecrets.MustGetField("BOXOAUTH_ID")
secret := testSecrets.MustGetField("BOXOAUTH_SECRET")
invalidSecret := testSecrets.MustGetField("BOXOAUTH_INVALID_SECRET")
type args struct {
ctx context.Context
data []byte
verify bool
}
tests := []struct {
name string
s Scanner
args args
want []detectors.Result
wantErr bool
}{
{
name: "found, verified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a box id %s with secret %s", id, secret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_BoxOauth,
Verified: true,
Raw: []byte(id),
RawV2: []byte(id + secret),
},
},
wantErr: false,
},
{
name: "found, unverified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a box id %s with secret %s", id, invalidSecret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_BoxOauth,
Verified: false,
Raw: []byte(id),
RawV2: []byte(id + invalidSecret),
},
},
wantErr: false,
},
{
name: "not found",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte("You cannot find the secret within"),
verify: true,
},
want: nil,
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
s := Scanner{}
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
if (err != nil) != tt.wantErr {
t.Errorf("BoxOauth.FromData() error = %v, wantErr %v", err, tt.wantErr)
return
}
for i := range got {
if len(got[i].Raw) == 0 {
t.Fatalf("no raw secret present: \n %+v", got[i])
}
if len(got[i].Raw) == 0 {
t.Fatalf("no rawV2 secret present: \n %+v", got[i])
}
}
if diff := pretty.Compare(got, tt.want); diff != "" {
t.Errorf("BoxOauth.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
}
})
}
}
func BenchmarkFromData(benchmark *testing.B) {
ctx := context.Background()
s := Scanner{}
for name, data := range detectors.MustGetBenchmarkData() {
benchmark.Run(name, func(b *testing.B) {
b.ResetTimer()
for n := 0; n < b.N; n++ {
_, err := s.FromData(ctx, false, data)
if err != nil {
b.Fatal(err)
}
}
})
}
}

91
pkg/detectors/boxoauth/boxoauth_test.go Normal file
View file

@ -0,0 +1,91 @@
//go:build detectors
// +build detectors
package boxoauth
import (
"context"
"fmt"
"testing"
"github.com/google/go-cmp/cmp"
"github.com/brianvoe/gofakeit/v7"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
var (
clientId = gofakeit.Password(true, true, true, false, false, 32)
clientSecret = gofakeit.Password(true, true, true, false, false, 32)
invalidClientSecret = gofakeit.Password(true, true, true, true, false, 32)
)
func TestBoxOauth_Pattern(t *testing.T) {
d := Scanner{}
ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
name string
input string
want []string
}{
{
name: "valid pattern",
input: fmt.Sprintf("box id = '%s' box secret = '%s'", clientId, clientSecret),
want: []string{clientId + clientSecret},
},
{
name: "invalid pattern",
input: fmt.Sprintf("box id = '%s' box secret = '%s'", clientId, invalidClientSecret),
want: nil,
},
{
name: "invalid pattern",
input: fmt.Sprintf("box = '%s|%s'", clientId, invalidClientSecret),
want: nil,
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
if len(matchedDetectors) == 0 {
t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
results, err := d.FromData(context.Background(), false, []byte(test.input))
if err != nil {
t.Errorf("error = %v", err)
return
}
if len(results) != len(test.want) {
if len(results) == 0 {
t.Errorf("did not receive result")
} else {
t.Errorf("expected %d results, only received %d", len(test.want), len(results))
}
return
}
actual := make(map[string]struct{}, len(results))
for _, r := range results {
if len(r.RawV2) > 0 {
actual[string(r.RawV2)] = struct{}{}
} else {
actual[string(r.Raw)] = struct{}{}
}
}
expected := make(map[string]struct{}, len(test.want))
for _, v := range test.want {
expected[v] = struct{}{}
}
if diff := cmp.Diff(expected, actual); diff != "" {
t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
}
})
}
}

57
pkg/detectors/bulksms/bulksms.go
View file

@ -2,18 +2,17 @@ package bulksms
import (
"context"
b64 "encoding/base64"
"fmt"
regexp "github.com/wasilibs/go-re2"
"io"
"net/http"
"strings"
regexp "github.com/wasilibs/go-re2"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
type Scanner struct{
type Scanner struct {
detectors.DefaultMultiPartCredentialProvider
}
@ -24,12 +23,11 @@ var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"bulksms"}) + `\b([a-fA-Z0-9*]{29})\b`)
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"bulksms"}) + `\b([a-zA-Z0-9!@#$%^&*()]{29})\b`)
idPat = regexp.MustCompile(detectors.PrefixRegex([]string{"bulksms"}) + `\b([A-F0-9-]{37})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"bulksms"}
}
@ -38,46 +36,51 @@ func (s Scanner) Keywords() []string {
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
idMatches := idPat.FindAllStringSubmatch(dataStr, -1)
var uniqueIds = make(map[string]struct{})
var uniqueKeys = make(map[string]struct{})
for _, match := range matches {
if len(match) != 2 {
continue
}
resMatch := strings.TrimSpace(match[1])
for _, idmatch := range idMatches {
if len(match) != 2 {
continue
}
resIdMatch := strings.TrimSpace(idmatch[1])
for _, match := range idPat.FindAllStringSubmatch(dataStr, -1) {
uniqueIds[match[1]] = struct{}{}
}
for _, match := range keyPat.FindAllStringSubmatch(dataStr, -1) {
uniqueKeys[match[1]] = struct{}{}
}
for id := range uniqueIds {
for key := range uniqueKeys {
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_Bulksms,
Raw: []byte(resMatch),
RawV2: []byte(resMatch + resIdMatch),
Raw: []byte(key),
RawV2: []byte(key + id),
}
if verify {
data := fmt.Sprintf("%s:%s", resIdMatch, resMatch)
sEnc := b64.StdEncoding.EncodeToString([]byte(data))
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.bulksms.com/v1/messages", nil)
if err != nil {
continue
}
req.Header.Add("Authorization", fmt.Sprintf("Basic %s", sEnc))
req.SetBasicAuth(id, key)
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
defer func() {
_, _ = io.Copy(io.Discard, res.Body)
_ = res.Body.Close()
}()
if res.StatusCode == http.StatusOK {
s1.Verified = true
results = append(results, s1)
// move to next id, by skipping remaining key's
break
}
} else {
s1.SetVerificationError(err, key)
}
}
results = append(results, s1)
}
}
return results, nil

1
pkg/detectors/bulksms/bulksms_test.go
View file

@ -96,6 +96,7 @@ func TestBulksms_FromChunk(t *testing.T) {
t.Fatalf("no raw secret present: \n %+v", got[i])
}
got[i].Raw = nil
got[i].RawV2 = nil
}
if diff := pretty.Compare(got, tt.want); diff != "" {
t.Errorf("Bulksms.FromData() %s diff: (-got +want)\n%s", tt.name, diff)

5
pkg/detectors/calendarific/calendarific.go
View file

@ -2,10 +2,11 @@ package calendarific
import (
"context"
regexp "github.com/wasilibs/go-re2"
"net/http"
"strings"
regexp "github.com/wasilibs/go-re2"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
@ -20,7 +21,7 @@ var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"calendarific"}) + `\b([a-z0-9]{40})\b`)
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"calendarific"}) + `\b([a-zA-Z0-9]{32})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.

5
pkg/detectors/cannyio/cannyio.go
View file

@ -2,10 +2,11 @@ package cannyio
import (
"context"
regexp "github.com/wasilibs/go-re2"
"net/http"
"strings"
regexp "github.com/wasilibs/go-re2"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
@ -20,7 +21,7 @@ var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"canny"}) + `\b([a-z0-9]{8}-[a-z0-9]{4}-[a-z0-9]{4}-[0-9]{4}-[a-z0-9]{12})\b`)
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"canny"}) + `\b([a-z0-9]{8}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{12})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.

8
pkg/detectors/captaindata/captaindata.go → pkg/detectors/captaindata/v1/captaindata.go
View file

@ -2,21 +2,25 @@ package captaindata
import (
"context"
regexp "github.com/wasilibs/go-re2"
"net/http"
"strings"
regexp "github.com/wasilibs/go-re2"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
type Scanner struct{
type Scanner struct {
detectors.DefaultMultiPartCredentialProvider
}
func (s Scanner) Version() int { return 1 }
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var _ detectors.Versioner = (*Scanner)(nil)
var (
client = common.SaneHttpClient()

0
pkg/detectors/captaindata/captaindata_test.go → pkg/detectors/captaindata/v1/captaindata_test.go
View file

113
pkg/detectors/captaindata/v2/captaindata.go Normal file
View file

@ -0,0 +1,113 @@
package captaindata
import (
"context"
"fmt"
"io"
"net/http"
regexp "github.com/wasilibs/go-re2"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
type Scanner struct {
client *http.Client
}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var _ detectors.Versioner = (*Scanner)(nil)
func (Scanner) Version() int { return 2 }
var (
defaultClient = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"captaindata"}) + `\b([0-9a-f]{64})\b`)
projIdPat = regexp.MustCompile(detectors.PrefixRegex([]string{"captaindata"}) + `\b([0-9a-f]{8}\-[0-9a-f]{4}\-[0-9a-f]{4}\-[0-9a-f]{4}\-[0-9a-f]{12})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"captaindata"}
}
// FromData will find and optionally verify CaptainData secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
uniqueMatches := make(map[string]struct{})
for _, match := range keyPat.FindAllStringSubmatch(dataStr, -1) {
uniqueMatches[match[1]] = struct{}{}
}
uniqueProjIdMatches := make(map[string]struct{})
for _, match := range projIdPat.FindAllStringSubmatch(dataStr, -1) {
uniqueProjIdMatches[match[1]] = struct{}{}
}
for projId := range uniqueProjIdMatches {
for apiKey := range uniqueMatches {
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_CaptainData,
Raw: []byte(apiKey),
RawV2: []byte(projId + apiKey),
}
if verify {
client := s.client
if client == nil {
client = defaultClient
}
isVerified, extraData, verificationErr := verifyMatch(ctx, client, projId, apiKey)
s1.Verified = isVerified
s1.ExtraData = extraData
s1.SetVerificationError(verificationErr, apiKey)
}
results = append(results, s1)
}
}
return
}
func verifyMatch(ctx context.Context, client *http.Client, projId, apiKey string) (bool, map[string]string, error) {
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.captaindata.co/v3/workspace", nil)
if err != nil {
return false, nil, nil
}
req.Header.Set("Authorization", "x-api-key "+apiKey)
req.Header.Set("x-project-id", projId)
res, err := client.Do(req)
if err != nil {
return false, nil, err
}
defer func() {
_, _ = io.Copy(io.Discard, res.Body)
_ = res.Body.Close()
}()
switch res.StatusCode {
case http.StatusOK:
return true, nil, nil
case http.StatusUnauthorized:
return false, nil, nil
default:
return false, nil, fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
}
}
func (s Scanner) Type() detectorspb.DetectorType {
return detectorspb.DetectorType_CaptainData
}
func (s Scanner) Description() string {
return "CaptainData is a service for automating data extraction and processing. The API keys can be used to access and control these automation processes."
}

129
pkg/detectors/captaindata/v2/captaindata_integration_test.go Normal file
View file

@ -0,0 +1,129 @@
//go:build detectors
// +build detectors
package captaindata
import (
"context"
"fmt"
"testing"
"time"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
func TestCaptainData_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
projId := testSecrets.MustGetField("CAPTAINDATA_PROJID")
secret := testSecrets.MustGetField("CAPTAINDATA")
inactiveSecret := testSecrets.MustGetField("CAPTAINDATA_INACTIVE")
type args struct {
ctx context.Context
data []byte
verify bool
}
tests := []struct {
name string
s Scanner
args args
want []detectors.Result
wantErr bool
wantVerificationErr bool
}{
{
name: "found, verified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a captaindata project %s with captaindata secret %s within", projId, secret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_CaptainData,
Verified: true,
},
},
wantErr: false,
wantVerificationErr: false,
},
{
name: "found, unverified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a captaindata project %s with captaindata secret %s within but not valid", projId, inactiveSecret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_CaptainData,
Verified: false,
},
},
wantErr: false,
wantVerificationErr: true,
},
{
name: "not found",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte("You cannot find the secret within"),
verify: true,
},
want: nil,
wantErr: false,
wantVerificationErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
s := Scanner{}
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
if (err != nil) != tt.wantErr {
t.Errorf("CaptainData.FromData() error = %v, wantErr %v", err, tt.wantErr)
return
}
for i := range got {
if len(got[i].Raw) == 0 {
t.Fatalf("no raw secret present: \n %+v", got[i])
}
if (got[i].VerificationError() != nil) != tt.wantVerificationErr {
t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())
}
}
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "ExtraData", "verificationError")
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
t.Errorf("CaptainData.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
}
})
}
}
func BenchmarkFromData(benchmark *testing.B) {
ctx := context.Background()
s := Scanner{}
for name, data := range detectors.MustGetBenchmarkData() {
benchmark.Run(name, func(b *testing.B) {
b.ResetTimer()
for n := 0; n < b.N; n++ {
_, err := s.FromData(ctx, false, data)
if err != nil {
b.Fatal(err)
}
}
})
}
}

84
pkg/detectors/captaindata/v2/captaindata_test.go Normal file
View file

@ -0,0 +1,84 @@
package captaindata
import (
"context"
"testing"
"github.com/google/go-cmp/cmp"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
func TestCaptainData_Pattern(t *testing.T) {
d := Scanner{}
ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
name string
input string
want []string
}{
{
name: "typical pattern",
input: "captaindata_project = '12345678-1234-1234-1234-123456789012' captaindata_api_key = '1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef'",
want: []string{"12345678-1234-1234-1234-1234567890121234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef"},
},
{
name: "finds all matches",
input: `captaindata_project1 = '12345678-1234-1234-1234-123456789012' captaindata_api_key1 = '1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef'
captaindata_project2 = '87654321-4321-4321-4321-210987654321' captaindata_api_key2 = 'fedcba0987654321fedcba0987654321fedcba0987654321fedcba0987654321'`,
want: []string{
"12345678-1234-1234-1234-1234567890121234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef",
"12345678-1234-1234-1234-123456789012fedcba0987654321fedcba0987654321fedcba0987654321fedcba0987654321",
"87654321-4321-4321-4321-210987654321fedcba0987654321fedcba0987654321fedcba0987654321fedcba0987654321",
"87654321-4321-4321-4321-2109876543211234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef",
},
},
{
name: "invalid pattern",
input: "captaindata_project = '123456' captaindata_api_key = '1234567890'",
want: []string{},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
if len(matchedDetectors) == 0 {
t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
results, err := d.FromData(context.Background(), false, []byte(test.input))
if err != nil {
t.Errorf("error = %v", err)
return
}
if len(results) != len(test.want) {
if len(results) == 0 {
t.Errorf("did not receive result")
} else {
t.Errorf("expected %d results, only received %d", len(test.want), len(results))
}
return
}
actual := make(map[string]struct{}, len(results))
for _, r := range results {
if len(r.RawV2) > 0 {
actual[string(r.RawV2)] = struct{}{}
} else {
actual[string(r.Raw)] = struct{}{}
}
}
expected := make(map[string]struct{}, len(test.want))
for _, v := range test.want {
expected[v] = struct{}{}
}
if diff := cmp.Diff(expected, actual); diff != "" {
t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
}
})
}
}

22
pkg/detectors/checkvist/checkvist.go
View file

@ -2,17 +2,18 @@ package checkvist
import (
"context"
regexp "github.com/wasilibs/go-re2"
"net/http"
"net/url"
"strings"
regexp "github.com/wasilibs/go-re2"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
type Scanner struct{
type Scanner struct {
detectors.DefaultMultiPartCredentialProvider
}
@ -24,7 +25,7 @@ var (
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"checkvist"}) + `\b([0-9a-zA-Z]{14})\b`)
emailPat = regexp.MustCompile(detectors.PrefixRegex([]string{"checkvist"}) + `\b([\w\.-]+@[\w-]+\.[\w\.-]{2,5})\b`)
emailPat = regexp.MustCompile(detectors.PrefixRegex([]string{"checkvist"}) + common.EmailPattern)
)
// Keywords are used for efficiently pre-filtering chunks.
@ -38,14 +39,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
emailMatches := emailPat.FindAllStringSubmatch(dataStr, -1)
for _, emailMatch := range emailMatches {
if len(emailMatch) != 2 {
continue
}
resEmailMatch := strings.TrimSpace(emailMatch[1])
uniqueEmailMatches := make(map[string]struct{})
for _, match := range emailPat.FindAllStringSubmatch(dataStr, -1) {
uniqueEmailMatches[strings.TrimSpace(match[1])] = struct{}{}
}
for emailMatch := range uniqueEmailMatches {
for _, match := range matches {
if len(match) != 2 {
continue
@ -55,12 +55,12 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_Checkvist,
Raw: []byte(resMatch),
RawV2: []byte(resMatch + resEmailMatch),
RawV2: []byte(resMatch + emailMatch),
}
if verify {
payload := url.Values{}
payload.Add("username", resEmailMatch)
payload.Add("username", emailMatch)
payload.Add("remote_key", resMatch)
req, err := http.NewRequestWithContext(ctx, "GET", "https://checkvist.com/auth/login.json?version=2", strings.NewReader(payload.Encode()))

118
pkg/detectors/checkvist/checkvist_integration_test.go Normal file
View file

@ -0,0 +1,118 @@
//go:build detectors
// +build detectors
package checkvist
import (
"context"
"fmt"
"testing"
"time"
"github.com/kylelemons/godebug/pretty"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
func TestCheckvist_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
user := testSecrets.MustGetField("CHECKVIST_EMAIL")
secret := testSecrets.MustGetField("CHECKVIST")
inactiveSecret := testSecrets.MustGetField("CHECKVIST_INACTIVE")
type args struct {
ctx context.Context
data []byte
verify bool
}
tests := []struct {
name string
s Scanner
args args
want []detectors.Result
wantErr bool
}{
{
name: "found, verified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a checkvist user %s with checkvist secret %s within", user, secret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_Checkvist,
Verified: true,
},
},
wantErr: false,
},
{
name: "found, unverified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a checkvist user %s with checkvist secret %s within but not valid", user, inactiveSecret)), // the secret would satisfy the regex but not pass validation
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_Checkvist,
Verified: false,
},
},
wantErr: false,
},
{
name: "not found",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte("You cannot find the secret within"),
verify: true,
},
want: nil,
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
s := Scanner{}
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
if (err != nil) != tt.wantErr {
t.Errorf("Checkvist.FromData() error = %v, wantErr %v", err, tt.wantErr)
return
}
for i := range got {
if len(got[i].Raw) == 0 {
t.Fatalf("no raw secret present: \n %+v", got[i])
}
got[i].Raw = nil
}
if diff := pretty.Compare(got, tt.want); diff != "" {
t.Errorf("Checkvist.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
}
})
}
}
func BenchmarkFromData(benchmark *testing.B) {
ctx := context.Background()
s := Scanner{}
for name, data := range detectors.MustGetBenchmarkData() {
benchmark.Run(name, func(b *testing.B) {
b.ResetTimer()
for n := 0; n < b.N; n++ {
s.FromData(ctx, false, data)
}
})
}
}

144
pkg/detectors/checkvist/checkvist_test.go
View file

@ -1,117 +1,81 @@
//go:build detectors
// +build detectors
package checkvist
import (
"context"
"fmt"
"testing"
"time"
"github.com/kylelemons/godebug/pretty"
"github.com/google/go-cmp/cmp"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
"github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
func TestCheckvist_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
user := testSecrets.MustGetField("CHECKVIST_EMAIL")
secret := testSecrets.MustGetField("CHECKVIST")
inactiveSecret := testSecrets.MustGetField("CHECKVIST_INACTIVE")
var (
validPattern = "wdvnusa87afxYn / testuser1005@example.com"
invalidPattern = "wdvn-usa87a-fxp9ioasQQsstestUsQQ@example"
)
func TestCheckvist_Pattern(t *testing.T) {
d := Scanner{}
ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
type args struct {
ctx context.Context
data []byte
verify bool
}
tests := []struct {
name string
s Scanner
args args
want []detectors.Result
wantErr bool
name string
input string
want []string
}{
{
name: "found, verified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a checkvist user %s with checkvist secret %s within", user, secret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_Checkvist,
Verified: true,
},
},
wantErr: false,
name: "valid pattern",
input: fmt.Sprintf("checkvist: %s", validPattern),
want: []string{"wdvnusa87afxYntestuser1005@example.com"},
},
{
name: "found, unverified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a checkvist user %s with checkvist secret %s within but not valid", user, inactiveSecret)), // the secret would satisfy the regex but not pass validation
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_Checkvist,
Verified: false,
},
},
wantErr: false,
name: "valid pattern - key out of prefix range",
input: fmt.Sprintf("checkvist keyword is not close to the real key and id = %s", validPattern),
want: nil,
},
{
name: "not found",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte("You cannot find the secret within"),
verify: true,
},
want: nil,
wantErr: false,
name: "invalid pattern",
input: fmt.Sprintf("checkvist: %s", invalidPattern),
want: nil,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
s := Scanner{}
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
if (err != nil) != tt.wantErr {
t.Errorf("Checkvist.FromData() error = %v, wantErr %v", err, tt.wantErr)
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
if len(matchedDetectors) == 0 && test.want != nil {
t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
for i := range got {
if len(got[i].Raw) == 0 {
t.Fatalf("no raw secret present: \n %+v", got[i])
}
got[i].Raw = nil
}
if diff := pretty.Compare(got, tt.want); diff != "" {
t.Errorf("Checkvist.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
}
})
}
}
func BenchmarkFromData(benchmark *testing.B) {
ctx := context.Background()
s := Scanner{}
for name, data := range detectors.MustGetBenchmarkData() {
benchmark.Run(name, func(b *testing.B) {
b.ResetTimer()
for n := 0; n < b.N; n++ {
s.FromData(ctx, false, data)
results, err := d.FromData(context.Background(), false, []byte(test.input))
if err != nil {
t.Errorf("error = %v", err)
return
}
if len(results) != len(test.want) {
t.Errorf("expected %d results, got %d", len(test.want), len(results))
return
}
actual := make(map[string]struct{}, len(results))
for _, r := range results {
if len(r.RawV2) > 0 {
actual[string(r.RawV2)] = struct{}{}
} else {
actual[string(r.Raw)] = struct{}{}
}
}
expected := make(map[string]struct{}, len(test.want))
for _, v := range test.want {
expected[v] = struct{}{}
}
if diff := cmp.Diff(expected, actual); diff != "" {
t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
}
})
}

23
pkg/detectors/cloudflareglobalapikey/cloudflareglobalapikey.go
View file

@ -12,7 +12,7 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
type Scanner struct{
type Scanner struct {
detectors.DefaultMultiPartCredentialProvider
}
@ -24,7 +24,7 @@ var (
apiKeyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"cloudflare"}) + `\b([A-Za-z0-9_-]{37})\b`)
emailPat = regexp.MustCompile(`\b([A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}(\.[A-Za-z]{2})?)\b`)
emailPat = regexp.MustCompile(common.EmailPattern)
)
// Keywords are used for efficiently pre-filtering chunks.
@ -38,7 +38,11 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
dataStr := string(data)
apiKeyMatches := apiKeyPat.FindAllStringSubmatch(dataStr, -1)
emailMatches := emailPat.FindAllStringSubmatch(dataStr, -1)
uniqueEmailMatches := make(map[string]struct{})
for _, match := range emailPat.FindAllStringSubmatch(dataStr, -1) {
uniqueEmailMatches[strings.TrimSpace(match[1])] = struct{}{}
}
for _, apiKeyMatch := range apiKeyMatches {
if len(apiKeyMatch) != 2 {
@ -46,17 +50,12 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
}
apiKeyRes := strings.TrimSpace(apiKeyMatch[1])
for _, emailMatch := range emailMatches {
if len(emailMatch) != 2 {
continue
}
emailRes := strings.TrimSpace(emailMatch[1])
for emailMatch := range uniqueEmailMatches {
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_CloudflareGlobalApiKey,
Redacted: emailRes,
Redacted: emailMatch,
Raw: []byte(apiKeyRes),
RawV2: []byte(apiKeyRes + emailRes),
RawV2: []byte(apiKeyRes + emailMatch),
}
if verify {
@ -64,7 +63,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
if err != nil {
continue
}
req.Header.Add("X-Auth-Email", emailRes)
req.Header.Add("X-Auth-Email", emailMatch)
req.Header.Add("X-Auth-Key", apiKeyRes)
req.Header.Add("Content-Type", "application/json")

124
pkg/detectors/cloudflareglobalapikey/cloudflareglobalapikey_integration_test.go Normal file
View file

@ -0,0 +1,124 @@
//go:build detectors
// +build detectors
package cloudflareglobalapikey
import (
"context"
"fmt"
"testing"
"time"
"github.com/kylelemons/godebug/pretty"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
func TestCloudflareGlobalApiKey_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
globalApiKey := testSecrets.MustGetField("CLOUDFLARE_GLOBAL_API_KEY")
globalApiKeyEmail := testSecrets.MustGetField("CLOUDFLARE_GLOBAL_API_KEY_EMAIL")
inactiveglobalApiKey := testSecrets.MustGetField("CLOUDFLARE_GLOBAL_API_KEY_INACTIVE")
type args struct {
ctx context.Context
data []byte
verify bool
}
tests := []struct {
name string
s Scanner
args args
want []detectors.Result
wantErr bool
}{
{
name: "found, verified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a cloudflare globalapikey secret %s within with email %s", globalApiKey, globalApiKeyEmail)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_CloudflareGlobalApiKey,
Redacted: globalApiKeyEmail,
Verified: true,
},
},
wantErr: false,
},
{
name: "found, unverified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a cloudflare globalapikey secret %s with email %s within but unverified", inactiveglobalApiKey, globalApiKeyEmail)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_CloudflareGlobalApiKey,
Redacted: globalApiKeyEmail,
Verified: false,
},
},
wantErr: false,
},
{
name: "not found",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte("You cannot find the secret within"),
verify: true,
},
want: nil,
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
s := Scanner{}
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
if (err != nil) != tt.wantErr {
t.Errorf("CloudflareGlobalApiKey.FromData() error = %v, wantErr %v", err, tt.wantErr)
return
}
for i := range got {
if len(got[i].Raw) == 0 {
t.Fatalf("no raw secret present: \n %+v", got[i])
}
got[i].Raw = nil
}
if diff := pretty.Compare(got, tt.want); diff != "" {
t.Errorf("CloudflareGlobalApiKey.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
}
})
}
}
func BenchmarkFromData(benchmark *testing.B) {
ctx := context.Background()
s := Scanner{}
for name, data := range detectors.MustGetBenchmarkData() {
benchmark.Run(name, func(b *testing.B) {
b.ResetTimer()
for n := 0; n < b.N; n++ {
_, err := s.FromData(ctx, false, data)
if err != nil {
b.Fatal(err)
}
}
})
}
}

148
pkg/detectors/cloudflareglobalapikey/cloudflareglobalapikey_test.go
View file

@ -1,124 +1,82 @@
//go:build detectors
// +build detectors
package cloudflareglobalapikey
import (
"context"
"fmt"
"testing"
"time"
"github.com/kylelemons/godebug/pretty"
"github.com/google/go-cmp/cmp"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
"github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
func TestCloudflareGlobalApiKey_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
var (
validPattern = "abcD123efg456HIJklmn789OPQ_rstUVWxYZ-012 / testuser1005@example.com"
invalidPattern = "abcD123efg456HIJklmn789OPQ_rstUVWxYZ-012/testing@go"
)
globalApiKey := testSecrets.MustGetField("CLOUDFLARE_GLOBAL_API_KEY")
globalApiKeyEmail := testSecrets.MustGetField("CLOUDFLARE_GLOBAL_API_KEY_EMAIL")
inactiveglobalApiKey := testSecrets.MustGetField("CLOUDFLARE_GLOBAL_API_KEY_INACTIVE")
func TestCloudFlareGlobalAPIKey_Pattern(t *testing.T) {
d := Scanner{}
ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
type args struct {
ctx context.Context
data []byte
verify bool
}
tests := []struct {
name string
s Scanner
args args
want []detectors.Result
wantErr bool
name string
input string
want []string
}{
{
name: "found, verified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a cloudflare globalapikey secret %s within with email %s", globalApiKey, globalApiKeyEmail)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_CloudflareGlobalApiKey,
Redacted: globalApiKeyEmail,
Verified: true,
},
},
wantErr: false,
name: "valid pattern",
input: fmt.Sprintf("cloudflare: %s", validPattern),
want: []string{"abcD123efg456HIJklmn789OPQ_rstUVWxYZ-testuser1005@example.com"},
},
{
name: "found, unverified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a cloudflare globalapikey secret %s with email %s within but unverified", inactiveglobalApiKey, globalApiKeyEmail)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_CloudflareGlobalApiKey,
Redacted: globalApiKeyEmail,
Verified: false,
},
},
wantErr: false,
name: "valid pattern - key out of prefix range",
input: fmt.Sprintf("cloudflare keyword is not close to the real key and id = %s", validPattern),
want: nil,
},
{
name: "not found",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte("You cannot find the secret within"),
verify: true,
},
want: nil,
wantErr: false,
name: "invalid pattern",
input: fmt.Sprintf("cloudflare: %s", invalidPattern),
want: nil,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
s := Scanner{}
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
if (err != nil) != tt.wantErr {
t.Errorf("CloudflareGlobalApiKey.FromData() error = %v, wantErr %v", err, tt.wantErr)
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
if len(matchedDetectors) == 0 && test.want != nil {
t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
for i := range got {
if len(got[i].Raw) == 0 {
t.Fatalf("no raw secret present: \n %+v", got[i])
}
got[i].Raw = nil
}
if diff := pretty.Compare(got, tt.want); diff != "" {
t.Errorf("CloudflareGlobalApiKey.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
}
})
}
}
func BenchmarkFromData(benchmark *testing.B) {
ctx := context.Background()
s := Scanner{}
for name, data := range detectors.MustGetBenchmarkData() {
benchmark.Run(name, func(b *testing.B) {
b.ResetTimer()
for n := 0; n < b.N; n++ {
_, err := s.FromData(ctx, false, data)
if err != nil {
b.Fatal(err)
results, err := d.FromData(context.Background(), false, []byte(test.input))
if err != nil {
t.Errorf("error = %v", err)
return
}
if len(results) != len(test.want) {
t.Errorf("expected %d results, got %d", len(test.want), len(results))
return
}
actual := make(map[string]struct{}, len(results))
for _, r := range results {
if len(r.RawV2) > 0 {
actual[string(r.RawV2)] = struct{}{}
} else {
actual[string(r.Raw)] = struct{}{}
}
}
expected := make(map[string]struct{}, len(test.want))
for _, v := range test.want {
expected[v] = struct{}{}
}
if diff := cmp.Diff(expected, actual); diff != "" {
t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
}
})
}
}

15
pkg/detectors/cloze/cloze.go
View file

@ -25,7 +25,7 @@ var (
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"cloze"}) + `\b([0-9a-f]{32})\b`)
emailPat = regexp.MustCompile(detectors.PrefixRegex([]string{"cloze"}) + `\b([\w\.-]+@[\w-]+\.[\w\.-]{2,5})\b`)
emailPat = regexp.MustCompile(detectors.PrefixRegex([]string{"cloze"}) + common.EmailPattern)
)
// Keywords are used for efficiently pre-filtering chunks.
@ -39,14 +39,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
emailMatches := emailPat.FindAllStringSubmatch(dataStr, -1)
for _, emailMatch := range emailMatches {
if len(emailMatch) != 2 {
continue
}
resEmailMatch := strings.TrimSpace(emailMatch[1])
uniqueEmailMatches := make(map[string]struct{})
for _, match := range emailPat.FindAllStringSubmatch(dataStr, -1) {
uniqueEmailMatches[strings.TrimSpace(match[1])] = struct{}{}
}
for emailMatch := range uniqueEmailMatches {
for _, match := range matches {
if len(match) != 2 {
continue
@ -60,7 +59,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
if verify {
payload := url.Values{}
payload.Add("user", resEmailMatch)
payload.Add("user", emailMatch)
payload.Add("api_key", resMatch)
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.cloze.com/v1/profile?"+payload.Encode(), nil)

118
pkg/detectors/cloze/cloze_integration_test.go Normal file
View file

@ -0,0 +1,118 @@
//go:build detectors
// +build detectors
package cloze
import (
"context"
"fmt"
"testing"
"time"
"github.com/kylelemons/godebug/pretty"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
func TestCloze_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
email := testSecrets.MustGetField("CLOZE_EMAIL")
secret := testSecrets.MustGetField("CLOZE")
inactiveSecret := testSecrets.MustGetField("CLOZE_INACTIVE")
type args struct {
ctx context.Context
data []byte
verify bool
}
tests := []struct {
name string
s Scanner
args args
want []detectors.Result
wantErr bool
}{
{
name: "found, verified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a cloze user %s with cloze secret %s within", email, secret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_Cloze,
Verified: true,
},
},
wantErr: false,
},
{
name: "found, unverified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a cloze user %s with cloze secret %s within but not valid", email, inactiveSecret)), // the secret would satisfy the regex but not pass validation
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_Cloze,
Verified: false,
},
},
wantErr: false,
},
{
name: "not found",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte("You cannot find the secret within"),
verify: true,
},
want: nil,
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
s := Scanner{}
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
if (err != nil) != tt.wantErr {
t.Errorf("Cloze.FromData() error = %v, wantErr %v", err, tt.wantErr)
return
}
for i := range got {
if len(got[i].Raw) == 0 {
t.Fatalf("no raw secret present: \n %+v", got[i])
}
got[i].Raw = nil
}
if diff := pretty.Compare(got, tt.want); diff != "" {
t.Errorf("Cloze.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
}
})
}
}
func BenchmarkFromData(benchmark *testing.B) {
ctx := context.Background()
s := Scanner{}
for name, data := range detectors.MustGetBenchmarkData() {
benchmark.Run(name, func(b *testing.B) {
b.ResetTimer()
for n := 0; n < b.N; n++ {
s.FromData(ctx, false, data)
}
})
}
}

144
pkg/detectors/cloze/cloze_test.go
View file

@ -1,117 +1,81 @@
//go:build detectors
// +build detectors
package cloze
import (
"context"
"fmt"
"testing"
"time"
"github.com/kylelemons/godebug/pretty"
"github.com/google/go-cmp/cmp"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
"github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
func TestCloze_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
email := testSecrets.MustGetField("CLOZE_EMAIL")
secret := testSecrets.MustGetField("CLOZE")
inactiveSecret := testSecrets.MustGetField("CLOZE_INACTIVE")
var (
validPattern = "1a2b3c4d5e6f7a8b9c0d1e2f3a4b5c6d / testuser1005@example.com"
invalidPattern = "abcD123efg456HIJklmn789OPQ_rstUVWxYZ-012/testing@go"
)
func TestCloze_Pattern(t *testing.T) {
d := Scanner{}
ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
type args struct {
ctx context.Context
data []byte
verify bool
}
tests := []struct {
name string
s Scanner
args args
want []detectors.Result
wantErr bool
name string
input string
want []string
}{
{
name: "found, verified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a cloze user %s with cloze secret %s within", email, secret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_Cloze,
Verified: true,
},
},
wantErr: false,
name: "valid pattern",
input: fmt.Sprintf("cloze: %s", validPattern),
want: []string{"1a2b3c4d5e6f7a8b9c0d1e2f3a4b5c6d"},
},
{
name: "found, unverified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a cloze user %s with cloze secret %s within but not valid", email, inactiveSecret)), // the secret would satisfy the regex but not pass validation
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_Cloze,
Verified: false,
},
},
wantErr: false,
name: "valid pattern - key out of prefix range",
input: fmt.Sprintf("cloze keyword is not close to the real key and id = %s", validPattern),
want: nil,
},
{
name: "not found",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte("You cannot find the secret within"),
verify: true,
},
want: nil,
wantErr: false,
name: "invalid pattern",
input: fmt.Sprintf("cloze: %s", invalidPattern),
want: nil,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
s := Scanner{}
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
if (err != nil) != tt.wantErr {
t.Errorf("Cloze.FromData() error = %v, wantErr %v", err, tt.wantErr)
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
if len(matchedDetectors) == 0 && test.want != nil {
t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
for i := range got {
if len(got[i].Raw) == 0 {
t.Fatalf("no raw secret present: \n %+v", got[i])
}
got[i].Raw = nil
}
if diff := pretty.Compare(got, tt.want); diff != "" {
t.Errorf("Cloze.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
}
})
}
}
func BenchmarkFromData(benchmark *testing.B) {
ctx := context.Background()
s := Scanner{}
for name, data := range detectors.MustGetBenchmarkData() {
benchmark.Run(name, func(b *testing.B) {
b.ResetTimer()
for n := 0; n < b.N; n++ {
s.FromData(ctx, false, data)
results, err := d.FromData(context.Background(), false, []byte(test.input))
if err != nil {
t.Errorf("error = %v", err)
return
}
if len(results) != len(test.want) {
t.Errorf("expected %d results, got %d", len(test.want), len(results))
return
}
actual := make(map[string]struct{}, len(results))
for _, r := range results {
if len(r.RawV2) > 0 {
actual[string(r.RawV2)] = struct{}{}
} else {
actual[string(r.Raw)] = struct{}{}
}
}
expected := make(map[string]struct{}, len(test.want))
for _, v := range test.want {
expected[v] = struct{}{}
}
if diff := cmp.Diff(expected, actual); diff != "" {
t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
}
})
}

22
pkg/detectors/currencycloud/currencycloud.go
View file

@ -3,17 +3,18 @@ package currencycloud
import (
"context"
"fmt"
regexp "github.com/wasilibs/go-re2"
"io"
"net/http"
"strings"
regexp "github.com/wasilibs/go-re2"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
type Scanner struct{
type Scanner struct {
detectors.DefaultMultiPartCredentialProvider
}
@ -25,7 +26,7 @@ var (
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"currencycloud"}) + `\b([0-9a-z]{64})\b`)
emailPat = regexp.MustCompile(`\b([a-zA-Z0-9._-]+@[a-zA-Z0-9._-]+\.[a-z]+)\b`)
emailPat = regexp.MustCompile(common.EmailPattern)
)
// Keywords are used for efficiently pre-filtering chunks.
@ -39,7 +40,11 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
emailMatches := emailPat.FindAllStringSubmatch(dataStr, -1)
uniqueEmailMatches := make(map[string]struct{})
for _, match := range emailPat.FindAllStringSubmatch(dataStr, -1) {
uniqueEmailMatches[strings.TrimSpace(match[1])] = struct{}{}
}
for _, match := range matches {
if len(match) != 2 {
@ -47,12 +52,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
}
resMatch := strings.TrimSpace(match[1])
for _, emailmatch := range emailMatches {
if len(emailmatch) != 2 {
continue
}
resEmailMatch := strings.TrimSpace(emailmatch[1])
for emailmatch := range uniqueEmailMatches {
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_CurrencyCloud,
Raw: []byte(resMatch),
@ -61,7 +61,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
if verify {
for _, env := range environments {
// Get authentication token
payload := strings.NewReader(`{"login_id":"` + resEmailMatch + `","api_key":"` + resMatch + `"`)
payload := strings.NewReader(`{"login_id":"` + emailmatch + `","api_key":"` + resMatch + `"`)
req, err := http.NewRequestWithContext(ctx, "POST", "https://"+env+".currencycloud.com/v2/authenticate/api", payload)
if err != nil {
continue

121
pkg/detectors/currencycloud/currencycloud_integration_test.go Normal file
View file

@ -0,0 +1,121 @@
//go:build detectors
// +build detectors
package currencycloud
import (
"context"
"fmt"
"testing"
"time"
"github.com/kylelemons/godebug/pretty"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
func TestCurrencycloud_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
secret := testSecrets.MustGetField("CURRENCYCLOUD")
email := testSecrets.MustGetField("SCANNERS_EMAIL")
inactiveSecret := testSecrets.MustGetField("CURRENCYCLOUD_INACTIVE")
type args struct {
ctx context.Context
data []byte
verify bool
}
tests := []struct {
name string
s Scanner
args args
want []detectors.Result
wantErr bool
}{
{
name: "found, verified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a currencycloud secret %s within %s", secret, email)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_CurrencyCloud,
Verified: true,
},
},
wantErr: false,
},
{
name: "found, unverified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a currencycloud secret %s within %s but not valid", inactiveSecret, email)), // the secret would satisfy the regex but not pass validation
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_CurrencyCloud,
Verified: false,
},
},
wantErr: false,
},
{
name: "not found",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte("You cannot find the secret within"),
verify: true,
},
want: nil,
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
s := Scanner{}
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
if (err != nil) != tt.wantErr {
t.Errorf("Currencycloud.FromData() error = %v, wantErr %v", err, tt.wantErr)
return
}
for i := range got {
if len(got[i].Raw) == 0 {
t.Fatalf("no raw secret present: \n %+v", got[i])
}
got[i].Raw = nil
}
if diff := pretty.Compare(got, tt.want); diff != "" {
t.Errorf("Currencycloud.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
}
})
}
}
func BenchmarkFromData(benchmark *testing.B) {
ctx := context.Background()
s := Scanner{}
for name, data := range detectors.MustGetBenchmarkData() {
benchmark.Run(name, func(b *testing.B) {
b.ResetTimer()
for n := 0; n < b.N; n++ {
_, err := s.FromData(ctx, false, data)
if err != nil {
b.Fatal(err)
}
}
})
}
}

145
pkg/detectors/currencycloud/currencycloud_test.go
View file

@ -1,121 +1,82 @@
//go:build detectors
// +build detectors
package currencycloud
import (
"context"
"fmt"
"testing"
"time"
"github.com/kylelemons/godebug/pretty"
"github.com/google/go-cmp/cmp"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
"github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
func TestCurrencycloud_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
secret := testSecrets.MustGetField("CURRENCYCLOUD")
email := testSecrets.MustGetField("SCANNERS_EMAIL")
inactiveSecret := testSecrets.MustGetField("CURRENCYCLOUD_INACTIVE")
var (
validPattern = "1a2b3c4d5e6f7a8b9c0d1e2f3a4b5c6d7e8f9a0b1c2d3e4f5a6b7c8d9e0f1a2b / testuser1005@example.com"
invalidPattern = "abcD123efg456HIJklmn789OPQ_rstUVWxYZ-012/testing@go"
)
func TestCurrencyCloud_Pattern(t *testing.T) {
d := Scanner{}
ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
type args struct {
ctx context.Context
data []byte
verify bool
}
tests := []struct {
name string
s Scanner
args args
want []detectors.Result
wantErr bool
name string
input string
want []string
}{
{
name: "found, verified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a currencycloud secret %s within %s", secret, email)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_CurrencyCloud,
Verified: true,
},
},
wantErr: false,
name: "valid pattern",
input: fmt.Sprintf("currencycloud: %s", validPattern),
want: []string{"1a2b3c4d5e6f7a8b9c0d1e2f3a4b5c6d7e8f9a0b1c2d3e4f5a6b7c8d9e0f1a2b"},
},
{
name: "found, unverified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a currencycloud secret %s within %s but not valid", inactiveSecret, email)), // the secret would satisfy the regex but not pass validation
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_CurrencyCloud,
Verified: false,
},
},
wantErr: false,
name: "valid pattern - key out of prefix range",
input: fmt.Sprintf("currencycloud keyword is not close to the real key and id = %s", validPattern),
want: nil,
},
{
name: "not found",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte("You cannot find the secret within"),
verify: true,
},
want: nil,
wantErr: false,
name: "invalid pattern",
input: fmt.Sprintf("currencycloud: %s", invalidPattern),
want: nil,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
s := Scanner{}
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
if (err != nil) != tt.wantErr {
t.Errorf("Currencycloud.FromData() error = %v, wantErr %v", err, tt.wantErr)
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
if len(matchedDetectors) == 0 && test.want != nil {
t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
for i := range got {
if len(got[i].Raw) == 0 {
t.Fatalf("no raw secret present: \n %+v", got[i])
}
got[i].Raw = nil
}
if diff := pretty.Compare(got, tt.want); diff != "" {
t.Errorf("Currencycloud.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
}
})
}
}
func BenchmarkFromData(benchmark *testing.B) {
ctx := context.Background()
s := Scanner{}
for name, data := range detectors.MustGetBenchmarkData() {
benchmark.Run(name, func(b *testing.B) {
b.ResetTimer()
for n := 0; n < b.N; n++ {
_, err := s.FromData(ctx, false, data)
if err != nil {
b.Fatal(err)
results, err := d.FromData(context.Background(), false, []byte(test.input))
if err != nil {
t.Errorf("error = %v", err)
return
}
if len(results) != len(test.want) {
t.Errorf("expected %d results, got %d", len(test.want), len(results))
return
}
actual := make(map[string]struct{}, len(results))
for _, r := range results {
if len(r.RawV2) > 0 {
actual[string(r.RawV2)] = struct{}{}
} else {
actual[string(r.Raw)] = struct{}{}
}
}
expected := make(map[string]struct{}, len(test.want))
for _, v := range test.want {
expected[v] = struct{}{}
}
if diff := cmp.Diff(expected, actual); diff != "" {
t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
}
})
}
}

7
pkg/detectors/detectors.go
View file

@ -89,10 +89,7 @@ type Result struct {
DetectorType detectorspb.DetectorType
// DetectorName is the name of the Detector. Used for custom detectors.
DetectorName string
// DecoderType is the type of Decoder.
DecoderType detectorspb.DecoderType
// Verified indicates whether the secret was verified.
Verified bool
Verified bool
// Raw contains the raw secret identifier data. Prefer IDs over secrets since it is used for deduping after hashing.
Raw []byte
// RawV2 contains the raw secret identifier that is a combination of both the ID and the secret.
@ -171,6 +168,8 @@ type ResultWithMetadata struct {
Data []byte
// DetectorDescription is the description of the Detector.
DetectorDescription string
// DecoderType is the type of decoder that was used to generate this result's data.
DecoderType detectorspb.DecoderType
}
// CopyMetadata returns a detector result with included metadata from the source chunk.

5
pkg/detectors/gcpapplicationdefaultcredentials/gcpapplicationdefaultcredentials.go
View file

@ -79,6 +79,11 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_GCPApplicationDefaultCredentials,
Raw: []byte(detectedClientID),
RawV2: []byte(detectedClientID + creds.RefreshToken),
}
if len(creds.RefreshToken) > 3 {
s1.Redacted = creds.RefreshToken[:3] + "..." + creds.RefreshToken[min(len(creds.RefreshToken)-1, 47):]
}
if verify {

7
pkg/detectors/gcpapplicationdefaultcredentials/gcpapplicationdefaultcredentials_test.go
View file

@ -2,10 +2,11 @@ package gcpapplicationdefaultcredentials
import (
"context"
"testing"
"github.com/google/go-cmp/cmp"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
"testing"
)
func TestGcpapplicationdefaultcredentials_Pattern(t *testing.T) {
@ -24,7 +25,9 @@ func TestGcpapplicationdefaultcredentials_Pattern(t *testing.T) {
"refresh_token": "1//0_joijgor3i4ut98579862709342j3kjJOIE02834jijfewoifjowiejfhghyzznfoiwejfwnvuhewiufnwinciwu_-o2i3jjfcc",
"type": "authorized_user"
}`,
want: []string{"191375729402-oiuj2498ry3497gjveoierj8294jfj41"},
want: []string{
"191375729402-oiuj2498ry3497gjveoierj8294jfj411//0_joijgor3i4ut98579862709342j3kjJOIE02834jijfewoifjowiejfhghyzznfoiwejfwnvuhewiufnwinciwu_-o2i3jjfcc",
},
},
}

40
pkg/detectors/gitlab/v1/gitlab.go
View file

@ -34,6 +34,8 @@ func (Scanner) CloudEndpoint() string { return "https://gitlab.com" }
var (
defaultClient = common.SaneHttpClient()
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"gitlab"}) + `\b([a-zA-Z0-9\-=_]{20,22})\b`)
BlockedUserMessage = "403 Forbidden - Your account has been blocked"
)
// Keywords are used for efficiently pre-filtering chunks.
@ -60,6 +62,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_Gitlab,
Raw: []byte(resMatch),
ExtraData: map[string]string{},
}
s1.ExtraData = map[string]string{
"rotation_guide": "https://howtorotate.com/docs/tutorials/gitlab/",
@ -67,9 +70,16 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
}
if verify {
isVerified, verificationErr := s.verifyGitlab(ctx, resMatch)
isVerified, extraData, verificationErr := s.verifyGitlab(ctx, resMatch)
s1.Verified = isVerified
for key, value := range extraData {
s1.ExtraData[key] = value
}
s1.SetVerificationError(verificationErr, resMatch)
s1.AnalysisInfo = map[string]string{
"key": resMatch,
}
}
results = append(results, s1)
@ -78,7 +88,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
return results, nil
}
func (s Scanner) verifyGitlab(ctx context.Context, resMatch string) (bool, error) {
func (s Scanner) verifyGitlab(ctx context.Context, resMatch string) (bool, map[string]string, error) {
// there are 4 read 'scopes' for a gitlab token: api, read_user, read_repo, and read_registry
// they all grant access to different parts of the API. I couldn't find an endpoint that every
// one of these scopes has access to, so we just check an example endpoint for each scope. If any
@ -98,13 +108,14 @@ func (s Scanner) verifyGitlab(ctx context.Context, resMatch string) (bool, error
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))
res, err := client.Do(req)
if err != nil {
return false, err
return false, nil, err
}
defer res.Body.Close()
body, err := io.ReadAll(res.Body)
bodyBytes, err := io.ReadAll(res.Body)
if err != nil {
return false, err
return false, nil, err
}
// 200 means good key and has `read_user` scope
@ -112,19 +123,28 @@ func (s Scanner) verifyGitlab(ctx context.Context, resMatch string) (bool, error
// 401 is bad key
switch res.StatusCode {
case http.StatusOK:
return json.Valid(body), nil
return json.Valid(bodyBytes), nil, nil
case http.StatusForbidden:
// check if the user account is blocked or not
stringBody := string(bodyBytes)
if strings.Contains(stringBody, BlockedUserMessage) {
return true, map[string]string{
"blocked": "True",
}, nil
}
// Good key but not the right scope
return true, nil
return true, nil, nil
case http.StatusUnauthorized:
// Nothing to do; zero values are the ones we want
return false, nil
return false, nil, nil
default:
return false, fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
return false, nil, fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
}
}
return false, nil
return false, nil, nil
}
func (s Scanner) Type() detectorspb.DetectorType {

1
pkg/detectors/gitlab/v1/gitlab_v1_test.go
View file

@ -187,6 +187,7 @@ func TestGitlab_FromChunk(t *testing.T) {
if (got[i].VerificationError() != nil) != tt.wantVerificationErr {
t.Fatalf(" wantVerificationError = %v, verification error = %v,", tt.wantVerificationErr, got[i].VerificationError())
}
got[i].AnalysisInfo = nil
}
opts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")
if diff := cmp.Diff(got, tt.want, opts); diff != "" {

1
pkg/detectors/gitlab/v1/gitlab_v2_test.go
View file

@ -87,6 +87,7 @@ func TestGitlab_FromChunk_WithV2Secrets(t *testing.T) {
if (got[i].VerificationError() != nil) != tt.wantVerificationErr {
t.Fatalf(" wantVerificationError = %v, verification error = %v,", tt.wantVerificationErr, got[i].VerificationError())
}
got[i].AnalysisInfo = nil
}
opts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")
if diff := cmp.Diff(got, tt.want, opts); diff != "" {

1
pkg/detectors/gitlab/v2/gitlab_v1_test.go
View file

@ -98,6 +98,7 @@ func TestGitlabV2_FromChunk_WithV1Secrets(t *testing.T) {
if (got[i].VerificationError() != nil) != tt.wantVerificationErr {
t.Fatalf(" wantVerificationError = %v, verification error = %v,", tt.wantVerificationErr, got[i].VerificationError())
}
got[i].AnalysisInfo = nil
}
opts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")
if diff := cmp.Diff(got, tt.want, opts); diff != "" {

41
pkg/detectors/gitlab/v2/gitlab_v2.go
View file

@ -3,6 +3,7 @@ package gitlab
import (
"context"
"fmt"
"io"
"net/http"
"strings"
@ -10,6 +11,7 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
v1 "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/gitlab/v1"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
@ -49,6 +51,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_Gitlab,
Raw: []byte(resMatch),
ExtraData: map[string]string{},
}
s1.ExtraData = map[string]string{
"rotation_guide": "https://howtorotate.com/docs/tutorials/gitlab/",
@ -56,9 +59,16 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
}
if verify {
isVerified, verificationErr := s.verifyGitlab(ctx, resMatch)
isVerified, extraData, verificationErr := s.verifyGitlab(ctx, resMatch)
s1.Verified = isVerified
for key, value := range extraData {
s1.ExtraData[key] = value
}
s1.SetVerificationError(verificationErr, resMatch)
s1.AnalysisInfo = map[string]string{
"key": resMatch,
}
}
results = append(results, s1)
@ -67,7 +77,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
return results, nil
}
func (s Scanner) verifyGitlab(ctx context.Context, resMatch string) (bool, error) {
func (s Scanner) verifyGitlab(ctx context.Context, resMatch string) (bool, map[string]string, error) {
// there are 4 read 'scopes' for a gitlab token: api, read_user, read_repo, and read_registry
// they all grant access to different parts of the API. I couldn't find an endpoint that every
// one of these scopes has access to, so we just check an example endpoint for each scope. If any
@ -86,28 +96,41 @@ func (s Scanner) verifyGitlab(ctx context.Context, resMatch string) (bool, error
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))
res, err := client.Do(req)
if err != nil {
return false, err
return false, nil, err
}
defer res.Body.Close()
bodyBytes, err := io.ReadAll(res.Body)
if err != nil {
return false, nil, err
}
defer res.Body.Close() // The request body is unused.
// 200 means good key and has `read_user` scope
// 403 means good key but not the right scope
// 401 is bad key
switch res.StatusCode {
case http.StatusOK:
return true, nil
return true, nil, nil
case http.StatusForbidden:
// check if the user account is blocked or not
stringBody := string(bodyBytes)
if strings.Contains(stringBody, v1.BlockedUserMessage) {
return true, map[string]string{
"blocked": "True",
}, nil
}
// Good key but not the right scope
return true, nil
return true, nil, nil
case http.StatusUnauthorized:
// Nothing to do; zero values are the ones we want
return false, nil
return false, nil, nil
default:
return false, fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
return false, nil, fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
}
}
return false, nil
return false, nil, nil
}
func (s Scanner) Type() detectorspb.DetectorType {

1
pkg/detectors/gitlab/v2/gitlab_v2_test.go
View file

@ -167,6 +167,7 @@ func TestGitlabV2_FromChunk(t *testing.T) {
if (got[i].VerificationError() != nil) != tt.wantVerificationErr {
t.Fatalf(" wantVerificationError = %v, verification error = %v,", tt.wantVerificationErr, got[i].VerificationError())
}
got[i].AnalysisInfo = nil
}
opts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")
if diff := cmp.Diff(got, tt.want, opts); diff != "" {

20
pkg/detectors/gocanvas/gocanvas.go
View file

@ -4,18 +4,19 @@ import (
"context"
"encoding/xml"
"fmt"
regexp "github.com/wasilibs/go-re2"
"io"
"net/http"
"net/url"
"strings"
regexp "github.com/wasilibs/go-re2"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
type Scanner struct{
type Scanner struct {
detectors.DefaultMultiPartCredentialProvider
}
@ -27,7 +28,7 @@ var (
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"gocanvas"}) + `\b([0-9A-Za-z/+]{43}=[ \r\n]{1})`)
emailPat = regexp.MustCompile(detectors.PrefixRegex([]string{"gocanvas"}) + `\b([\w\.-]+@[\w-]+\.[\w\.-]{2,5})\b`)
emailPat = regexp.MustCompile(detectors.PrefixRegex([]string{"gocanvas"}) + common.EmailPattern)
)
// Keywords are used for efficiently pre-filtering chunks.
@ -41,14 +42,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
emailMatches := emailPat.FindAllStringSubmatch(dataStr, -1)
for _, emailMatch := range emailMatches {
if len(emailMatch) != 2 {
continue
}
resEmailMatch := strings.TrimSpace(emailMatch[1])
uniqueEmailMatches := make(map[string]struct{})
for _, match := range emailPat.FindAllStringSubmatch(dataStr, -1) {
uniqueEmailMatches[strings.TrimSpace(match[1])] = struct{}{}
}
for emailMatch := range uniqueEmailMatches {
for _, match := range matches {
if len(match) != 2 {
continue
@ -62,7 +62,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
if verify {
payload := url.Values{}
payload.Add("username", resEmailMatch)
payload.Add("username", emailMatch)
req, err := http.NewRequestWithContext(ctx, "GET", "https://www.gocanvas.com/apiv2/forms.xml", strings.NewReader(payload.Encode()))
if err != nil {

118
pkg/detectors/gocanvas/gocanvas_integration_test.go Normal file
View file

@ -0,0 +1,118 @@
//go:build detectors
// +build detectors
package gocanvas
import (
"context"
"fmt"
"testing"
"time"
"github.com/kylelemons/godebug/pretty"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
func TestGoCanvas_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
username := testSecrets.MustGetField("SCANNERS_EMAIL")
secret := testSecrets.MustGetField("GOCANVAS")
inactiveSecret := testSecrets.MustGetField("GOCANVAS_INACTIVE")
type args struct {
ctx context.Context
data []byte
verify bool
}
tests := []struct {
name string
s Scanner
args args
want []detectors.Result
wantErr bool
}{
{
name: "found, verified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a gocanvas username %s with gocanvas secret %s within", username, secret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_GoCanvas,
Verified: true,
},
},
wantErr: false,
},
{
name: "found, unverified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a gocanvas username %s with gocanvas secret %s within but not valid", username, inactiveSecret)), // the secret would satisfy the regex but not pass validation
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_GoCanvas,
Verified: false,
},
},
wantErr: false,
},
{
name: "not found",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte("You cannot find the secret within"),
verify: true,
},
want: nil,
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
s := Scanner{}
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
if (err != nil) != tt.wantErr {
t.Errorf("GoCanvas.FromData() error = %v, wantErr %v", err, tt.wantErr)
return
}
for i := range got {
if len(got[i].Raw) == 0 {
t.Fatalf("no raw secret present: \n %+v", got[i])
}
got[i].Raw = nil
}
if diff := pretty.Compare(got, tt.want); diff != "" {
t.Errorf("GoCanvas.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
}
})
}
}
func BenchmarkFromData(benchmark *testing.B) {
ctx := context.Background()
s := Scanner{}
for name, data := range detectors.MustGetBenchmarkData() {
benchmark.Run(name, func(b *testing.B) {
b.ResetTimer()
for n := 0; n < b.N; n++ {
s.FromData(ctx, false, data)
}
})
}
}

144
pkg/detectors/gocanvas/gocanvas_test.go
View file

@ -1,117 +1,81 @@
//go:build detectors
// +build detectors
package gocanvas
import (
"context"
"fmt"
"testing"
"time"
"github.com/kylelemons/godebug/pretty"
"github.com/google/go-cmp/cmp"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
"github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
func TestGoCanvas_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
username := testSecrets.MustGetField("SCANNERS_EMAIL")
secret := testSecrets.MustGetField("GOCANVAS")
inactiveSecret := testSecrets.MustGetField("GOCANVAS_INACTIVE")
var (
validPattern = "Abc123+/Xyz456mnopQRStuvw89YZ12345678ABad6C= / gocanvasemail = testuser1005@example.com"
invalidPattern = "abcD123efg456HIJklmn789OPQ_rstUVWxYZ-012/testing@go"
)
func TestGoCanvas_Pattern(t *testing.T) {
d := Scanner{}
ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
type args struct {
ctx context.Context
data []byte
verify bool
}
tests := []struct {
name string
s Scanner
args args
want []detectors.Result
wantErr bool
name string
input string
want []string
}{
{
name: "found, verified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a gocanvas username %s with gocanvas secret %s within", username, secret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_GoCanvas,
Verified: true,
},
},
wantErr: false,
name: "valid pattern",
input: fmt.Sprintf("gocanvas: %s", validPattern),
want: []string{"Abc123+/Xyz456mnopQRStuvw89YZ12345678ABad6C="},
},
{
name: "found, unverified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a gocanvas username %s with gocanvas secret %s within but not valid", username, inactiveSecret)), // the secret would satisfy the regex but not pass validation
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_GoCanvas,
Verified: false,
},
},
wantErr: false,
name: "valid pattern - key out of prefix range",
input: fmt.Sprintf("gocanvas keyword is not close to the real key and id = %s", validPattern),
want: nil,
},
{
name: "not found",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte("You cannot find the secret within"),
verify: true,
},
want: nil,
wantErr: false,
name: "invalid pattern",
input: fmt.Sprintf("gocanvas: %s", invalidPattern),
want: nil,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
s := Scanner{}
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
if (err != nil) != tt.wantErr {
t.Errorf("GoCanvas.FromData() error = %v, wantErr %v", err, tt.wantErr)
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
if len(matchedDetectors) == 0 && test.want != nil {
t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
for i := range got {
if len(got[i].Raw) == 0 {
t.Fatalf("no raw secret present: \n %+v", got[i])
}
got[i].Raw = nil
}
if diff := pretty.Compare(got, tt.want); diff != "" {
t.Errorf("GoCanvas.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
}
})
}
}
func BenchmarkFromData(benchmark *testing.B) {
ctx := context.Background()
s := Scanner{}
for name, data := range detectors.MustGetBenchmarkData() {
benchmark.Run(name, func(b *testing.B) {
b.ResetTimer()
for n := 0; n < b.N; n++ {
s.FromData(ctx, false, data)
results, err := d.FromData(context.Background(), false, []byte(test.input))
if err != nil {
t.Errorf("error = %v", err)
return
}
if len(results) != len(test.want) {
t.Errorf("expected %d results, got %d", len(test.want), len(results))
return
}
actual := make(map[string]struct{}, len(results))
for _, r := range results {
if len(r.RawV2) > 0 {
actual[string(r.RawV2)] = struct{}{}
} else {
actual[string(r.Raw)] = struct{}{}
}
}
expected := make(map[string]struct{}, len(test.want))
for _, v := range test.want {
expected[v] = struct{}{}
}
if diff := cmp.Diff(expected, actual); diff != "" {
t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
}
})
}

22
pkg/detectors/magicbell/magicbell.go
View file

@ -2,16 +2,17 @@ package magicbell
import (
"context"
regexp "github.com/wasilibs/go-re2"
"net/http"
"strings"
regexp "github.com/wasilibs/go-re2"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
type Scanner struct{
type Scanner struct {
detectors.DefaultMultiPartCredentialProvider
}
@ -23,7 +24,7 @@ var (
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"magicbell"}) + `\b([a-zA-Z-0-9]{40})\b`)
emailPat = regexp.MustCompile(`\b([a-zA-Z0-9+._-]+@[a-zA-Z0-9._-]+\.[a-zA-Z0-9_-]+)\b`)
emailPat = regexp.MustCompile(common.EmailPattern)
)
// Keywords are used for efficiently pre-filtering chunks.
@ -37,7 +38,11 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
dataStr := string(data)
apiKeyMatches := keyPat.FindAllStringSubmatch(dataStr, -1)
emailMatches := emailPat.FindAllStringSubmatch(dataStr, -1)
uniqueEmailMatches := make(map[string]struct{})
for _, match := range emailPat.FindAllStringSubmatch(dataStr, -1) {
uniqueEmailMatches[strings.TrimSpace(match[1])] = struct{}{}
}
for _, keyMatch := range apiKeyMatches {
if len(keyMatch) != 2 {
@ -45,12 +50,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
}
apiKeyRes := strings.TrimSpace(keyMatch[1])
for _, emailMatch := range emailMatches {
if len(emailMatch) != 2 {
continue
}
emailRes := strings.TrimSpace(emailMatch[1])
for emailMatch := range uniqueEmailMatches {
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_MagicBell,
Raw: []byte(apiKeyRes),
@ -62,7 +62,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
continue
}
req.Header.Add("X-MAGICBELL-API-KEY", apiKeyRes)
req.Header.Add("X-MAGICBELL-USER-EMAIL", emailRes)
req.Header.Add("X-MAGICBELL-USER-EMAIL", emailMatch)
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()

121
pkg/detectors/magicbell/magicbell_integration_test.go Normal file
View file

@ -0,0 +1,121 @@
//go:build detectors
// +build detectors
package magicbell
import (
"context"
"fmt"
"testing"
"time"
"github.com/kylelemons/godebug/pretty"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
func TestMagicBell_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
secret := testSecrets.MustGetField("MAGICBELL")
inactiveSecret := testSecrets.MustGetField("MAGICBELL_INACTIVE")
userEmail := testSecrets.MustGetField("MAGICBELL_USER_EMAIL")
type args struct {
ctx context.Context
data []byte
verify bool
}
tests := []struct {
name string
s Scanner
args args
want []detectors.Result
wantErr bool
}{
{
name: "found, verified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a magicbell secret %s with email %s within", secret, userEmail)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_MagicBell,
Verified: true,
},
},
wantErr: false,
},
{
name: "found, unverified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a magicbell secret %s with email %s within but not valid", inactiveSecret, userEmail)), // the secret would satisfy the regex but not pass validation
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_MagicBell,
Verified: false,
},
},
wantErr: false,
},
{
name: "not found",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte("You cannot find the secret within"),
verify: true,
},
want: nil,
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
s := Scanner{}
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
if (err != nil) != tt.wantErr {
t.Errorf("MagicBell.FromData() error = %v, wantErr %v", err, tt.wantErr)
return
}
for i := range got {
if len(got[i].Raw) == 0 {
t.Fatalf("no raw secret present: \n %+v", got[i])
}
got[i].Raw = nil
}
if diff := pretty.Compare(got, tt.want); diff != "" {
t.Errorf("MagicBell.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
}
})
}
}
func BenchmarkFromData(benchmark *testing.B) {
ctx := context.Background()
s := Scanner{}
for name, data := range detectors.MustGetBenchmarkData() {
benchmark.Run(name, func(b *testing.B) {
b.ResetTimer()
for n := 0; n < b.N; n++ {
_, err := s.FromData(ctx, false, data)
if err != nil {
b.Fatal(err)
}
}
})
}
}

Some files were not shown because too many files have changed in this diff Show more