From 794a082b6c76167cc54d39c484db0777cb5f4da9 Mon Sep 17 00:00:00 2001 From: Dustin Decker Date: Sat, 12 Mar 2022 07:36:42 -0800 Subject: [PATCH] Fix Dockerfiles, readme example, and github rate limit handling --- Dockerfile | 2 ++ Dockerfile.goreleaser | 2 ++ README.md | 2 +- pkg/sources/github/github.go | 46 ++++++++++++++++++++---------------- 4 files changed, 30 insertions(+), 22 deletions(-) diff --git a/Dockerfile b/Dockerfile index 6e2a6e859..5439bcf39 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,8 +3,10 @@ RUN mkdir /build COPY . /build WORKDIR /build RUN CGO_ENABLED=0 go build -a -o trufflehog main.go +RUN mkdir /empty FROM scratch +COPY --from=builder /empty /tmp COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt COPY --from=builder /build/trufflehog /usr/bin/trufflehog ENTRYPOINT ["/usr/bin/trufflehog"] \ No newline at end of file diff --git a/Dockerfile.goreleaser b/Dockerfile.goreleaser index a9bda863c..7a8ef0ba6 100644 --- a/Dockerfile.goreleaser +++ b/Dockerfile.goreleaser @@ -1,6 +1,8 @@ FROM golang:bullseye as builder +RUN mkdir /empty FROM scratch +COPY --from=builder /empty /tmp COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt COPY trufflehog /usr/bin/trufflehog ENTRYPOINT ["/usr/bin/trufflehog"] \ No newline at end of file diff --git a/README.md b/README.md index 372287a2d..dbdb40268 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ Several options: ### 3. Docker ```bash -$ docker run -v "$PWD:/pwd" ghcr.io/trufflesecurity/trufflehog2:latest github --repo https://github.com/dustin-decker/secretsandstuff.git +$ docker run -it -v "$PWD:/pwd" trufflesecurity/trufflehog2:latest github --repo https://github.com/dustin-decker/secretsandstuff --debug 🐷🔑🐷 TruffleHog. Unearth your secrets. 🐷🔑🐷 Found verified result 🐷🔑 diff --git a/pkg/sources/github/github.go b/pkg/sources/github/github.go index dc3ac3fdd..23db939ee 100644 --- a/pkg/sources/github/github.go +++ b/pkg/sources/github/github.go @@ -354,32 +354,36 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err } // handleRateLimit returns true if a rate limit was handled -//unauthed github has a rate limit of 60 requests per hour. This will likely only be exhausted if many users/orgs are scanned without auth +// Unauthenticated access to most github endpoints has a rate limit of 60 requests per hour. +// This will likely only be exhausted if many users/orgs are scanned without auth func handleRateLimit(errIn error, res *github.Response) bool { - knownWait := true - remaining, err := strconv.Atoi(res.Header.Get("x-ratelimit-remaining")) - if err != nil { - knownWait = false - } - resetTime, err := strconv.Atoi(res.Header.Get("x-ratelimit-reset")) - if err != nil || resetTime == 0 { - knownWait = false - } - - if knownWait && remaining == 0 { - waitTime := int64(resetTime) - time.Now().Unix() - if waitTime > 0 { - duration := time.Duration(waitTime+1) * time.Second - log.WithField("resumeTime", time.Now().Add(duration).String()).Debugf("rate limited") - time.Sleep(duration) - return true - } - } - limit, ok := errIn.(*github.RateLimitError) if !ok { return false } + + if res != nil { + knownWait := true + remaining, err := strconv.Atoi(res.Header.Get("x-ratelimit-remaining")) + if err != nil { + knownWait = false + } + resetTime, err := strconv.Atoi(res.Header.Get("x-ratelimit-reset")) + if err != nil || resetTime == 0 { + knownWait = false + } + + if knownWait && remaining == 0 { + waitTime := int64(resetTime) - time.Now().Unix() + if waitTime > 0 { + duration := time.Duration(waitTime+1) * time.Second + log.WithField("resumeTime", time.Now().Add(duration).String()).Debugf("rate limited") + time.Sleep(duration) + return true + } + } + } + log.WithField("retry-after", limit.Message).Debug("handling rate limit (5 minutes retry)") time.Sleep(time.Minute * 5) return true