Mirrors/trufflehog
2
0
Fork 0
mirror of https://github.com/trufflesecurity/trufflehog.git synced 2024-11-10 07:04:24 +00:00
Code Issues Projects Releases Packages Wiki Activity

add rate limit and consumption metrics for GitHub (#1651)

Browse source 
* add rate limit and consumption metrics

* incrment after each repo scanned

* update repo scanned label name
This commit is contained in:
Zubair Khan 2023-08-22 15:01:59 -04:00 committed by GitHub
parent 059ea23a72
commit fd00d2b30b
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
2 changed files with 53 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
pkg/sources/github/github.go
View file

@ -416,6 +416,11 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
apiEndpoint = "https://api.github.com"
}
// Reset consumption and rate limit metrics on each run.
githubNumRateLimitEncountered.WithLabelValues(s.name).Set(0)
githubSecondsSpentRateLimited.WithLabelValues(s.name).Set(0)
githubReposScanned.WithLabelValues(s.name).Set(0)
installationClient, err := s.enumerate(ctx, apiEndpoint)
if err != nil {
return err
@ -451,6 +456,7 @@ func (s *Source) enumerate(ctx context.Context, apiEndpoint string) (*github.Cli
}
s.repos = s.filteredRepoCache.Values()
githubReposEnumerated.WithLabelValues(s.name).Set(float64(len(s.repos)))
s.log.Info("Completed enumeration", "num_repos", len(s.repos), "num_orgs", s.orgsCache.Count(), "num_members", len(s.memberCache))
// We must sort the repos so we can resume later if necessary.
@ -781,6 +787,8 @@ func (s *Source) scan(ctx context.Context, installationClient *github.Client, ch
return nil
}
githubReposScanned.WithLabelValues(s.name).Inc()
if err = s.scanComments(ctx, repoURL, chunksChan); err != nil {
scanErrs.Add(fmt.Errorf("error scanning comments in repo %s: %w", repoURL, err))
return nil
@ -810,6 +818,8 @@ func (s *Source) handleRateLimit(errIn error, res *github.Response) bool {
return false
}
githubNumRateLimitEncountered.WithLabelValues(s.name).Inc()
if res != nil {
knownWait := true
remaining, err := strconv.Atoi(res.Header.Get("x-ratelimit-remaining"))
@ -827,6 +837,7 @@ func (s *Source) handleRateLimit(errIn error, res *github.Response) bool {
duration := time.Duration(waitTime+1) * time.Second
s.log.V(2).Info("rate limited", "resumeTime", time.Now().Add(duration).String())
time.Sleep(duration)
githubSecondsSpentRateLimited.WithLabelValues(s.name).Add(duration.Seconds())
return true
}
}

42
pkg/sources/github/metrics.go Normal file
View file

@ -0,0 +1,42 @@
package github
import (
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
)
var (
githubNumRateLimitEncountered = promauto.NewGaugeVec(prometheus.GaugeOpts{
Namespace: common.MetricsNamespace,
Subsystem: common.MetricsSubsystem,
Name: "github_num_rate_limit_encountered",
Help: "Total number of times Github Rate Limit was encountered",
},
[]string{"source_name"})
githubSecondsSpentRateLimited = promauto.NewGaugeVec(prometheus.GaugeOpts{
Namespace: common.MetricsNamespace,
Subsystem: common.MetricsSubsystem,
Name: "github_seconds_spent_rate_limited",
Help: "Total number of seconds spent idle due to GitHub rate limits.",
},
[]string{"source_name"})
githubReposEnumerated = promauto.NewGaugeVec(prometheus.GaugeOpts{
Namespace: common.MetricsNamespace,
Subsystem: common.MetricsSubsystem,
Name: "github_repos_enumerated",
Help: "Total number of GitHub repositories enumerated.",
},
[]string{"source_name"})
githubReposScanned = promauto.NewGaugeVec(prometheus.GaugeOpts{
Namespace: common.MetricsNamespace,
Subsystem: common.MetricsSubsystem,
Name: "github_repos_scanned",
Help: "Total number of GitHub repositories scanned.",
},
[]string{"source_name"})
)