Automatically update trufflehog (#121)

Co-authored-by: Dustin Decker <dustin@trufflesec.com>
This commit is contained in:
Bill Rich 2022-04-03 10:51:56 -08:00 committed by GitHub
parent aea937422e
commit a8f07c8e97
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 232 additions and 2 deletions

View file

@ -1,4 +1,6 @@
builds:
ldflags:
- -X 'github.com/trufflesecurity/trufflehog/v3/pkg/version.BuildVersion={{ .Version }}'
- env: [CGO_ENABLED=0]
binary: trufflehog
goos:
@ -42,4 +44,4 @@ docker_manifests:
- name_template: ghcr.io/trufflesecurity/{{ .ProjectName }}:latest
image_templates:
- ghcr.io/trufflesecurity/{{ .ProjectName }}:{{ .Version }}-amd64
- ghcr.io/trufflesecurity/{{ .ProjectName }}:{{ .Version }}-arm64v8
- ghcr.io/trufflesecurity/{{ .ProjectName }}:{{ .Version }}-arm64v8

7
go.mod
View file

@ -4,6 +4,8 @@ go 1.17
replace github.com/gitleaks/go-gitdiff => github.com/bill-rich/go-gitdiff v0.7.6-custom1
replace github.com/jpillora/overseer => github.com/dustin-decker/overseer v1.1.7-custom2
require (
cloud.google.com/go/secretmanager v1.3.0
github.com/Azure/go-autorest/autorest/azure/auth v0.5.11
@ -13,6 +15,7 @@ require (
github.com/aws/aws-sdk-go-v2/service/sts v1.16.2
github.com/bitfinexcom/bitfinex-api-go v0.0.0-20210608095005-9e0b26f200fb
github.com/bradleyfalzon/ghinstallation/v2 v2.0.4
github.com/denisbrodbeck/machineid v1.0.1
github.com/envoyproxy/protoc-gen-validate v0.6.7
github.com/fatih/color v1.13.0
github.com/felixge/fgprof v0.9.2
@ -25,6 +28,7 @@ require (
github.com/h2non/filetype v1.1.3
github.com/hashicorp/go-retryablehttp v0.7.0
github.com/joho/godotenv v1.4.0
github.com/jpillora/overseer v1.1.6
github.com/kylelemons/godebug v1.1.0
github.com/mattn/go-colorable v0.1.12
github.com/paulbellamy/ratecounter v0.2.0
@ -70,6 +74,7 @@ require (
github.com/emirpasic/gods v1.12.0 // indirect
github.com/go-git/gcfg v1.5.0 // indirect
github.com/go-git/go-billy/v5 v5.3.1 // indirect
github.com/go-ole/go-ole v1.2.6 // indirect
github.com/golang-jwt/jwt/v4 v4.2.0 // indirect
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect
github.com/golang/protobuf v1.5.2 // indirect
@ -82,6 +87,7 @@ require (
github.com/imdario/mergo v0.3.12 // indirect
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/jpillora/s3 v1.1.4 // indirect
github.com/json-iterator/go v1.1.11 // indirect
github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351 // indirect
github.com/mattn/go-isatty v0.0.14 // indirect
@ -90,6 +96,7 @@ require (
github.com/modern-go/reflect2 v1.0.1 // indirect
github.com/pkg/diff v0.0.0-20200914180035-5b29258ca4f7 // indirect
github.com/xanzy/ssh-agent v0.3.0 // indirect
github.com/yusufpapurcu/wmi v1.2.2 // indirect
go.opencensus.io v0.23.0 // indirect
golang.org/x/mod v0.5.0 // indirect
golang.org/x/sys v0.0.0-20220209214540-3681064d5158 // indirect

14
go.sum
View file

@ -146,8 +146,12 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/denisbrodbeck/machineid v1.0.1 h1:geKr9qtkB876mXguW2X6TU4ZynleN6ezuMSRhl4D7AQ=
github.com/denisbrodbeck/machineid v1.0.1/go.mod h1:dJUwb7PTidGDeYyUBmXZ2GphQBbjJCrnectwCyxcUSI=
github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U=
github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE=
github.com/dustin-decker/overseer v1.1.7-custom2 h1:k6lSAUkdxnICmN5/sEsCKFVYchv2wBfogAdwONk39y0=
github.com/dustin-decker/overseer v1.1.7-custom2/go.mod h1:nT9w37AiO1Nop2VhVhNfzAFaPjthvxgpDV3XKsxYkcI=
github.com/emirpasic/gods v1.12.0 h1:QAUIPSaCu4G+POclxeqb3F+WPpdKqFGlw36+yOzGlrg=
github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o=
github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
@ -186,6 +190,8 @@ github.com/go-git/go-git/v5 v5.4.2/go.mod h1:gQ1kArt6d+n+BGd+/B/I74HwRTLhth2+zti
github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY=
github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
github.com/gobwas/httphead v0.0.0-20200921212729-da3d93bc3c58/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo=
github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw=
github.com/gobwas/ws v1.0.4/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM=
@ -331,6 +337,8 @@ github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGw
github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
github.com/joho/godotenv v1.4.0 h1:3l4+N6zfMWnkbPEXKng2o2/MR5mSwTrBih4ZEkkz1lg=
github.com/joho/godotenv v1.4.0/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
github.com/jpillora/s3 v1.1.4 h1:YCCKDWzb/Ye9EBNd83ATRF/8wPEy0xd43Rezb6u6fzc=
github.com/jpillora/s3 v1.1.4/go.mod h1:yedE603V+crlFi1Kl/5vZJaBu9pUzE9wvKegU/lF2zs=
github.com/json-iterator/go v1.1.5/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
github.com/json-iterator/go v1.1.11 h1:uVUAXhF2To8cbw/3xN3pxj6kk7TYKs98NIrTqPlMWAQ=
github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
@ -416,7 +424,11 @@ github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMB
github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE=
github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
github.com/smartystreets/assertions v1.0.1 h1:voD4ITNjPL5jjBfgR/r8fPIIBrliWrWHeiJApdr3r4w=
github.com/smartystreets/assertions v1.0.1/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM=
github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
github.com/smartystreets/gunit v1.1.3 h1:32x+htJCu3aMswhPw3teoJ+PnWPONqdNgaGs6Qt8ZaU=
github.com/smartystreets/gunit v1.1.3/go.mod h1:EH5qMBab2UclzXUcpR8b93eHsIlp9u+pDQIRp5DZNzQ=
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
github.com/spf13/afero v1.3.3/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY520V4=
github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I=
@ -447,6 +459,8 @@ github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg=
github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
github.com/zricethezav/gitleaks/v8 v8.5.2 h1:+IiLvmBiMGetpgNtzC71EHktTXc1Zt7m9AwnqhhJ04g=
github.com/zricethezav/gitleaks/v8 v8.5.2/go.mod h1:TxBuxH8eB/1la8Mc7I6j/ZZYNG/mHGpoi09N2oHx5nQ=
go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs=

94
main.go
View file

@ -4,7 +4,6 @@ import (
"context"
"encoding/json"
"fmt"
"gopkg.in/alecthomas/kingpin.v2"
"log"
"net/http"
_ "net/http/pprof"
@ -12,8 +11,14 @@ import (
"runtime"
"strconv"
"strings"
"syscall"
"time"
"github.com/jpillora/overseer"
"github.com/trufflesecurity/trufflehog/v3/pkg/updater"
"github.com/trufflesecurity/trufflehog/v3/pkg/version"
"gopkg.in/alecthomas/kingpin.v2"
"github.com/felixge/fgprof"
"github.com/gorilla/mux"
@ -25,9 +30,91 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/sources/git"
)
var (
cli = kingpin.New("TruffleHog", "TruffleHog is a tool for finding credentials.")
cmd string
debug = cli.Flag("debug", "Run in debug mode").Bool()
versionFlag = cli.Flag("version", "Prints trufflehog version.").Bool()
jsonOut = cli.Flag("json", "Output in JSON format.").Short('j').Bool()
jsonLegacy = cli.Flag("json-legacy", "Use the pre-v3.0 JSON format. Only works with git, gitlab, and github sources.").Bool()
concurrency = cli.Flag("concurrency", "Number of concurrent workers.").Default(strconv.Itoa(runtime.NumCPU())).Int()
noVerification = cli.Flag("no-verification", "Don't verify the results.").Bool()
onlyVerified = cli.Flag("only-verified", "Only output verified results.").Bool()
// rules = cli.Flag("rules", "Path to file with custom rules.").String()
printAvgDetectorTime = cli.Flag("print-avg-detector-time", "Print the average time spent on each detector.").Bool()
noUpdate = cli.Flag("no-update", "Don't check for updates.").Bool()
gitScan = cli.Command("git", "Find credentials in git repositories.")
gitScanURI = gitScan.Arg("uri", "Git repository URL. https:// or file:// schema expected.").Required().String()
gitScanIncludePaths = gitScan.Flag("include-paths", "Path to file with newline separated regexes for files to include in scan.").Short('i').String()
gitScanExcludePaths = gitScan.Flag("exclude-paths", "Path to file with newline separated regexes for files to exclude in scan.").Short('x').String()
gitScanSinceCommit = gitScan.Flag("since-commit", "Commit to start scan from.").String()
gitScanBranch = gitScan.Flag("branch", "Branch to scan.").String()
gitScanMaxDepth = gitScan.Flag("max-depth", "Maximum depth of commits to scan.").Int()
_ = gitScan.Flag("allow", "No-op flag for backwards compat.").Bool()
_ = gitScan.Flag("entropy", "No-op flag for backwards compat.").Bool()
_ = gitScan.Flag("regex", "No-op flag for backwards compat.").Bool()
githubScan = cli.Command("github", "Find credentials in GitHub repositories.")
githubScanEndpoint = githubScan.Flag("endpoint", "GitHub endpoint.").Default("https://api.github.com").String()
githubScanRepos = githubScan.Flag("repo", `GitHub repository to scan. You can repeat this flag. Example: "https://github.com/dustin-decker/secretsandstuff"`).Strings()
githubScanOrgs = githubScan.Flag("org", `GitHub organization to scan. You can repeat this flag. Example: "trufflesecurity"`).Strings()
githubScanToken = githubScan.Flag("token", "GitHub token.").String()
githubIncludeForks = githubScan.Flag("include-forks", "Include forks in scan.").Bool()
githubIncludeMembers = githubScan.Flag("include-members", "Include organization member repositories in scan.").Bool()
gitlabScan = cli.Command("gitlab", "Find credentials in GitLab repositories.")
// TODO: Add more GitLab options
gitlabScanEndpoint = gitlabScan.Flag("endpoint", "GitLab endpoint.").Default("https://gitlab.com").String()
gitlabScanRepos = gitlabScan.Flag("repo", "GitLab repo url. You can repeat this flag. Leave empty to scan all repos accessible with provided credential. Example: https://gitlab.com/org/repo.git").Strings()
gitlabScanToken = gitlabScan.Flag("token", "GitLab token.").Required().String()
filesystemScan = cli.Command("filesystem", "Find credentials in a filesystem.")
filesystemDirectories = filesystemScan.Flag("directory", "Path to directory to scan. You can repeat this flag.").Required().Strings()
// TODO: Add more filesystem scan options. Currently only supports scanning a list of directories.
// filesystemScanRecursive = filesystemScan.Flag("recursive", "Scan recursively.").Short('r').Bool()
// filesystemScanIncludePaths = filesystemScan.Flag("include-paths", "Path to file with newline separated regexes for files to include in scan.").Short('i').String()
// filesystemScanExcludePaths = filesystemScan.Flag("exclude-paths", "Path to file with newline separated regexes for files to exclude in scan.").Short('x').String()
s3Scan = cli.Command("s3", "Find credentials in S3 buckets.")
s3ScanKey = s3Scan.Flag("key", "S3 key used to authenticate.").String()
s3ScanSecret = s3Scan.Flag("secret", "S3 secret used to authenticate.").String()
s3ScanCloudEnv = s3Scan.Flag("cloud-environment", "Use IAM credentials in cloud environment.").Bool()
s3ScanBuckets = s3Scan.Flag("bucket", "Name of S3 bucket to scan. You can repeat this flag.").Strings()
)
func init() {
for i, arg := range os.Args {
if strings.HasPrefix(arg, "--") {
os.Args[i] = strings.ReplaceAll(arg, "_", "-")
}
}
cmd = kingpin.MustParse(cli.Parse(os.Args[1:]))
}
func main() {
updateCfg := overseer.Config{
Program: run,
Debug: *debug,
RestartSignal: syscall.SIGTERM,
// TODO: Eventually add a PreUpgrade func for signature check w/ x509 PKCS1v15
// PreUpgrade: checkUpdateSignature(binaryPath string),
}
if !*noUpdate {
updateCfg.Fetcher = updater.Fetcher(version.BuildVersion)
}
err := overseer.RunErr(updateCfg)
if err != nil {
logrus.WithError(err).Fatal("error occured with trufflehog updater 🐷")
}
}
func run(state overseer.State) {
cli := kingpin.New("TruffleHog", "TruffleHog is a tool for finding credentials.")
debug := cli.Flag("debug", "Run in debug mode").Bool()
versionFlag := cli.Flag("version", "Prints trufflehog version.").Bool()
jsonOut := cli.Flag("json", "Output in JSON format.").Short('j').Bool()
jsonLegacy := cli.Flag("json-legacy", "Use the pre-v3.0 JSON format. Only works with git, gitlab, and github sources.").Bool()
concurrency := cli.Flag("concurrency", "Number of concurrent workers.").Default(strconv.Itoa(runtime.NumCPU())).Int()
@ -82,6 +169,11 @@ func main() {
cmd := kingpin.MustParse(cli.Parse(os.Args[1:]))
if *versionFlag {
fmt.Println("trufflehog " + version.BuildVersion)
return
}
// When setting a base commit, chunks must be scanned in order.
if *gitScanSinceCommit != "" {
*concurrency = 1

112
pkg/updater/updater.go Normal file
View file

@ -0,0 +1,112 @@
package updater
import (
"archive/tar"
"archive/zip"
"bytes"
"compress/gzip"
"encoding/json"
"io"
"io/ioutil"
"net/http"
"runtime"
"strings"
"time"
"github.com/go-errors/errors"
"github.com/jpillora/overseer/fetcher"
log "github.com/sirupsen/logrus"
"github.com/trufflesecurity/trufflehog/v3/pkg/version"
)
func Fetcher(version string) fetcher.Interface {
return &OSS{
CurrentVersion: version,
}
}
type OSS struct {
Interval time.Duration
CurrentVersion string
}
// Init validates the provided config
func (g *OSS) Init() error {
//initiate OSS connection
return nil
}
const url = "https://oss.trufflehog.org/updates"
type FormData struct {
OS string
Arch string
CurrentVersion string
Timezone string
Binary string
}
// Fetch binary from URL via OSS client
func (g *OSS) Fetch() (io.Reader, error) {
log.Debug("fetching trufflehog update")
zone, _ := time.Now().Zone()
data := &FormData{
OS: runtime.GOOS,
Arch: runtime.GOARCH,
CurrentVersion: version.BuildVersion,
Timezone: zone,
Binary: "trufflehog",
}
dataByte, err := json.Marshal(data)
if err != nil {
return nil, err
}
reader := bytes.NewReader(dataByte)
resp, err := http.Post(url, "application/json", reader)
if err != nil || resp == nil {
return nil, err
}
defer resp.Body.Close()
newBinBytes, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, err
}
buffer := bytes.NewReader(newBinBytes)
switch runtime.GOOS {
case "windows":
zipReader, err := zip.NewReader(buffer, int64(len(newBinBytes)))
if err != nil {
return nil, errors.Errorf("Failed to read zip archive: %s", err)
}
for _, f := range zipReader.File {
if strings.HasPrefix(f.Name, "trufflehog") {
return f.Open()
}
}
default:
gzipReader, err := gzip.NewReader(buffer)
if err != nil {
return nil, errors.Errorf("Failed to read gzip archive: %s", err)
}
defer gzipReader.Close()
tarReader := tar.NewReader(gzipReader)
for {
header, err := tarReader.Next()
if err == io.EOF {
return nil, errors.New("unable to get update")
}
if header.Typeflag == tar.TypeReg {
if strings.HasPrefix(header.Name, "trufflehog") {
return tarReader, nil
}
}
}
}
return nil, errors.New("unable to get update")
}

3
pkg/version/version.go Normal file
View file

@ -0,0 +1,3 @@
package version
var BuildVersion = "dev"