Initial docs and release automation (#5)

This commit is contained in:
Dustin Decker 2022-01-18 16:59:18 -08:00 committed by GitHub
parent 15c6105be1
commit d6ffadb1ee
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
18 changed files with 368 additions and 14 deletions

33
.github/ISSUE_TEMPLATE/bug_report.md vendored Normal file
View file

@ -0,0 +1,33 @@
---
name: Bug report
about: Create a report to help us improve
title: ""
labels: bug, needs triage
assignees: dustin-decker, bill-rich
---
**Describe the bug**
A clear and concise description of what the bug is.
**To Reproduce**
Steps to reproduce the behavior:
1. Go to '...'
2. Click on '....'
3. Scroll down to '....'
4. See error
**Expected behavior**
A clear and concise description of what you expected to happen.
**Output**
Please include relevant log output, and run with the `--debug` flag.
**Screenshots**
If applicable, add screenshots to help explain your problem.
**Desktop (please complete the following information):**
- OS: [e.g. iOS]
- Version [e.g. 22]
**Additional context**
Add any other context about the problem here.

View file

@ -0,0 +1,19 @@
---
name: Feature request
about: Suggest an idea for this project
title: ""
labels: enhancement, needs triage
assignees: dustin-decker, bill-rich
---
**Is your feature request related to a problem? Please describe.**
A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
**Describe the solution you'd like**
A clear and concise description of what you want to happen.
**Describe alternatives you've considered**
A clear and concise description of any alternative solutions or features you've considered.
**Additional context**
Add any other context or screenshots about the feature request here.

4
.github/PULL_REQUEST_TEMPLATE.md vendored Normal file
View file

@ -0,0 +1,4 @@
<!--
Please create an issue to collect feedback prior to feature additions. Please also reference that issue in any PRs.
If possible try to keep PRs scoped to one feature, and add tests for new features.
-->

70
.github/workflows/codeql-analysis.yml vendored Normal file
View file

@ -0,0 +1,70 @@
# For most projects, this workflow file will not need changing; you simply need
# to commit it to your repository.
#
# You may wish to alter this file to override the set of languages analyzed,
# or to provide custom queries or build logic.
#
# ******** NOTE ********
# We have attempted to detect the languages in your repository. Please check
# the `language` matrix defined below to confirm you have the correct set of
# supported CodeQL languages.
#
name: "CodeQL"
on:
push:
branches: [ main ]
pull_request:
# The branches below must be a subset of the branches above
branches: [ main ]
schedule:
- cron: '35 11 * * 2'
jobs:
analyze:
name: Analyze
runs-on: ubuntu-latest
permissions:
actions: read
contents: read
security-events: write
strategy:
fail-fast: false
matrix:
language: [ 'go' ]
# CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
# Learn more about CodeQL language support at https://git.io/codeql-language-support
steps:
- name: Checkout repository
uses: actions/checkout@v2
# Initializes the CodeQL tools for scanning.
- name: Initialize CodeQL
uses: github/codeql-action/init@v1
with:
languages: ${{ matrix.language }}
# If you wish to specify custom queries, you can do so here or in a config file.
# By default, queries listed here will override any specified in a config file.
# Prefix the list here with "+" to use these queries and those in the config file.
# queries: ./path/to/local/query, your-org/your-repo/queries@main
# Autobuild attempts to build any compiled languages (C/C++, C#, or Java).
# If this step fails, then you should remove it and run the build manually (see below)
- name: Autobuild
uses: github/codeql-action/autobuild@v1
# Command-line programs to run using the OS shell.
# 📚 https://git.io/JvXDl
# ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
# and modify them (or add more) to build your code if your project
# uses a compiled language
#- run: |
# make bootstrap
# make release
- name: Perform CodeQL Analysis
uses: github/codeql-action/analyze@v1

30
.github/workflows/release.yml vendored Normal file
View file

@ -0,0 +1,30 @@
name: release
on:
push:
tags:
- 'v*'
jobs:
goreleaser:
runs-on: ubuntu-latest
steps:
-
name: Checkout
uses: actions/checkout@v2
with:
fetch-depth: 0
-
name: Set up Go
uses: actions/setup-go@v2
with:
go-version: 1.17
-
name: Run GoReleaser
uses: goreleaser/goreleaser-action@v2
with:
distribution: goreleaser
version: latest
args: release --rm-dist
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

26
.github/workflows/test.yml vendored Normal file
View file

@ -0,0 +1,26 @@
on: [push, pull_request]
name: Test
jobs:
test:
strategy:
matrix:
go-version: [1.17.x]
os: [ubuntu-latest]
runs-on: ${{ matrix.os }}
permissions:
contents: 'read'
id-token: 'write'
steps:
- name: Install Go
uses: actions/setup-go@v2
with:
go-version: ${{ matrix.go-version }}
- name: Checkout code
uses: actions/checkout@v2
- id: 'auth'
uses: 'google-github-actions/auth@v0'
with:
workload_identity_provider: 'projects/811013774421/locations/global/workloadIdentityPools/github-pool/providers/github-provider'
service_account: 'github-ci-external@trufflehog-testing.iam.gserviceaccount.com'
- name: Test
run: go test -timeout 30s ./...

76
CODE_OF_CONDUCT.md Normal file
View file

@ -0,0 +1,76 @@
# Contributor Covenant Code of Conduct
## Our Pledge
In the interest of fostering an open and welcoming environment, we as
contributors and maintainers pledge to making participation in our project and
our community a harassment-free experience for everyone, regardless of age, body
size, disability, ethnicity, sex characteristics, gender identity and expression,
level of experience, education, socio-economic status, nationality, personal
appearance, race, religion, or sexual identity and orientation.
## Our Standards
Examples of behavior that contributes to creating a positive environment
include:
* Using welcoming and inclusive language
* Being respectful of differing viewpoints and experiences
* Gracefully accepting constructive criticism
* Focusing on what is best for the community
* Showing empathy towards other community members
Examples of unacceptable behavior by participants include:
* The use of sexualized language or imagery and unwelcome sexual attention or
advances
* Trolling, insulting/derogatory comments, and personal or political attacks
* Public or private harassment
* Publishing others' private information, such as a physical or electronic
address, without explicit permission
* Other conduct which could reasonably be considered inappropriate in a
professional setting
## Our Responsibilities
Project maintainers are responsible for clarifying the standards of acceptable
behavior and are expected to take appropriate and fair corrective action in
response to any instances of unacceptable behavior.
Project maintainers have the right and responsibility to remove, edit, or
reject comments, commits, code, wiki edits, issues, and other contributions
that are not aligned to this Code of Conduct, or to ban temporarily or
permanently any contributor for other behaviors that they deem inappropriate,
threatening, offensive, or harmful.
## Scope
This Code of Conduct applies both within project spaces and in public spaces
when an individual is representing the project or its community. Examples of
representing a project or community include using an official project e-mail
address, posting via an official social media account, or acting as an appointed
representative at an online or offline event. Representation of a project may be
further defined and clarified by project maintainers.
## Enforcement
Instances of abusive, harassing, or otherwise unacceptable behavior may be
reported by contacting the project team at community@trufflesec.com. All
complaints will be reviewed and investigated and will result in a response that
is deemed necessary and appropriate to the circumstances. The project team is
obligated to maintain confidentiality with regard to the reporter of an incident.
Further details of specific enforcement policies may be posted separately.
Project maintainers who do not follow or enforce the Code of Conduct in good
faith may face temporary or permanent repercussions as determined by other
members of the project's leadership.
## Attribution
This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html
[homepage]: https://www.contributor-covenant.org
For answers to common questions about this code of conduct, see
https://www.contributor-covenant.org/faq

7
CONTRIBUTING.md Normal file
View file

@ -0,0 +1,7 @@
# Contribution guidelines
Please create an issue to collect feedback prior to feature additions. If possible try to keep PR's scoped to one feature, and add tests for new features.
When showing intereste, in a bug, enhancement, PR, or issue, please use the thumbs up/thumbs down emoji on the original message rather than adding comments expressing the same.
Contributors need to sign our CLA before we are able to accept contributions:

View file

@ -0,0 +1,10 @@
FROM golang:bullseye as builder
RUN mkdir /build
COPY . /build
WORKDIR /build
RUN CGO_ENABLED=0 go build -a -o trufflehog main.go
FROM scratch
COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
COPY --from=builder /build/trufflehog /usr/bin/trufflehog
ENTRYPOINT ["/usr/bin/trufflehog"]

View file

@ -26,7 +26,10 @@ bench:
CGO_ENABLED=0 go test $(shell go list ./pkg/secrets/... | grep -v /vendor/) -benchmem -run=xxx -bench .
run:
CGO_ENABLED=0 go run . git file://.
CGO_ENABLED=0 go run . git file://. --json
run-debug:
CGO_ENABLED=0 go run . git file://. --json --debug
protos:
docker run -u "$(shell id -u)" -v "$(shell dirname $(realpath $(firstword $(MAKEFILE_LIST))))":/pwd "${PROTOS_IMAGE}" bash -c "cd /pwd; /pwd/scripts/gen_proto.sh"

View file

@ -0,0 +1,71 @@
# TruffleHog
# Join The Slack
Have questions? Feedback? Jump in slack and hang out with us
https://join.slack.com/t/trufflehog-community/shared_invite/zt-pw2qbi43-Aa86hkiimstfdKH9UCpPzQ
## Installation
Several options:
### 1. Go
`go install github.com/trufflesecurity/trufflehog2.git@latest`
### 2. [Release binaries](https://github.com/trufflesecurity/trufflehog2/releases)
### 3. Docker (TODO)
`docker run ...`
### 4. Pip (TODO)
pip install trufflehog
### 5. Brew (TODO)
brew install trufflehog
## Usage
TruffleHog has a sub-command for each source of data that you may want to scan:
- git
- github
- gitlab
- slack
- S3
Each subcommand can have options that you can see with the `-h` flag provided to the sub command:
```
$ trufflehog git --help
usage: TruffleHog git [<flags>] <uri>
Find credentials in git repositories.
Flags:
--help Show context-sensitive help (also try --help-long and --help-man).
--debug Run in debug mode
--json Output in JSON format.
--concurrency=8 Number of concurrent workers.
--verification Verify the results.
-i, --include_paths=INCLUDE_PATHS
Path to file with newline separated regexes for files to include in scan.
-x, --exclude_paths=EXCLUDE_PATHS
Path to file with newline separated regexes for files to exclude in scan.
--branch=BRANCH Branch to scan.
--allow No-op flag for backwards compat.
--entropy No-op flag for backwards compat.
--regex No-op flag for backwards compat.
Args:
<uri> Git repository URL. https:// or file:// schema expected.
```
For example, to scan a `git` repository, start with
```
$ trufflehog git https://github.com/trufflesecurity/trufflehog2.git
```
# License Change
Since v3.0, TruffleHog is released under a AGPL 3 license, included in [`LICENSE`](LICENSE). TruffleHog v3.0 uses none of the previous codebase, but care was taken to preserve backwards compatibility on the command line interface. The work previous to this release is still available licensed under GPL 2.0 in the history of this repository and the previous package releases and tags. A completed CLA is required for us to accept contributions going forward.

1
SECURITY.md Normal file
View file

@ -0,0 +1 @@
Please report any security issues to security@trufflesec.com and include `trufflehog` in the subject

View file

@ -4,13 +4,14 @@ import (
"context"
"encoding/json"
"fmt"
"github.com/trufflesecurity/trufflehog/pkg/common"
"github.com/trufflesecurity/trufflehog/pkg/sources/git"
"log"
"os"
"runtime"
"strconv"
"github.com/trufflesecurity/trufflehog/pkg/common"
"github.com/trufflesecurity/trufflehog/pkg/sources/git"
"github.com/sirupsen/logrus"
"github.com/trufflesecurity/trufflehog/pkg/decoders"
"github.com/trufflesecurity/trufflehog/pkg/engine"
@ -21,7 +22,7 @@ func main() {
cli := kingpin.New("TruffleHog", "TruffleHog is a tool for finding credentials.")
debug := cli.Flag("debug", "Run in debug mode").Bool()
jsonOut := cli.Flag("json", "Output in JSON format.").Bool()
jsonOut := cli.Flag("json", "Output in JSON format.").Short('j').Bool()
concurrency := cli.Flag("concurrency", "Number of concurrent workers.").Default(strconv.Itoa(runtime.NumCPU())).Int()
verification := cli.Flag("verification", "Verify the results.").Bool()
// rules := cli.Flag("rules", "Path to file with custom rules.").String()

View file

@ -3,9 +3,10 @@ package common
import (
"bufio"
"fmt"
log "github.com/sirupsen/logrus"
"os"
"regexp"
log "github.com/sirupsen/logrus"
)
type Filter struct {
@ -82,7 +83,7 @@ func FilterRulesFromFile(source string) (*FilterRuleSet, error) {
func (filter *Filter) Pass(object string) bool {
excluded := filter.exclude.Matches(object)
included := filter.include.Matches(object)
log.Debugf("test PathFilter: file: %s, included: %t, excluded: %t, pass: %t", object, included, excluded, !excluded && included)
// log.Debugf("test PathFilter: file: %s, included: %t, excluded: %t, pass: %t", object, included, excluded, !excluded && included)
return !excluded && included
}

View file

@ -3,15 +3,16 @@ package engine
import (
"context"
"fmt"
"runtime"
gogit "github.com/go-git/go-git/v5"
"github.com/go-git/go-git/v5/plumbing"
"github.com/go-git/go-git/v5/plumbing/object"
"github.com/sirupsen/logrus"
"github.com/trufflesecurity/trufflehog/pkg/common"
"github.com/trufflesecurity/trufflehog/pkg/pb/source_metadatapb"
"github.com/trufflesecurity/trufflehog/pkg/pb/sourcespb"
"github.com/trufflesecurity/trufflehog/pkg/sources/git"
"runtime"
)
func (e *Engine) ScanGit(ctx context.Context, repoPath, gitScanBranch, headRef string, filter *common.Filter) error {
@ -90,7 +91,7 @@ func (e *Engine) ScanGit(ctx context.Context, repoPath, gitScanBranch, headRef s
})
go func() {
err := gitSource.ScanRepo(ctx, repo, scanOptions, &object.Commit{}, filter, e.ChunksChan())
err := gitSource.ScanRepo(ctx, repo, scanOptions, nil, filter, e.ChunksChan())
if err != nil {
logrus.WithError(err).Fatal("could not scan repo")
}

View file

@ -34,7 +34,7 @@ func TestSource_Scan(t *testing.T) {
init: init{
name: "this repo",
connection: &sourcespb.Git{
Directories: []string{"../../../../"},
Directories: []string{"../../../"},
Credential: &sourcespb.Git_Unauthenticated{
Unauthenticated: &credentialspb.Unauthenticated{},
},

View file

@ -313,7 +313,7 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
log.WithError(err).Warnf("unable to clone repo, continuing")
continue
}
err = s.git.ScanRepo(ctx, repo, &gogit.LogOptions{All: true}, nil, chunksChan)
err = s.git.ScanRepo(ctx, repo, &gogit.LogOptions{All: true}, nil, common.FilterEmpty(), chunksChan)
if err != nil {
log.WithError(err).Warnf("unable to scan repo")
}
@ -407,7 +407,7 @@ func (s *Source) paginateGists(ctx context.Context, user string, chunksChan chan
s.log.WithField("repo", *gist.HTMLURL).Debugf("attempting to clone gist from user %s", user)
scanCtx := context.Background()
err = s.git.ScanRepo(scanCtx, repo, &gogit.LogOptions{All: true}, nil, chunksChan)
err = s.git.ScanRepo(scanCtx, repo, &gogit.LogOptions{All: true}, nil, common.FilterEmpty(), chunksChan)
if err != nil {
log.WithError(err).Warnf("Could not scan after clone: %s", *gist.HTMLURL)
continue

View file

@ -14,6 +14,7 @@ import (
"google.golang.org/protobuf/proto"
"google.golang.org/protobuf/types/known/anypb"
"github.com/trufflesecurity/trufflehog/pkg/common"
"github.com/trufflesecurity/trufflehog/pkg/pb/source_metadatapb"
"github.com/trufflesecurity/trufflehog/pkg/pb/sourcespb"
@ -215,7 +216,7 @@ func (s *Source) scanRepos(ctx context.Context, chunksChan chan *sources.Chunk,
errors = append(errors, err)
continue
}
err = s.git.ScanRepo(ctx, repo, &gogit.LogOptions{All: true}, nil, chunksChan)
err = s.git.ScanRepo(ctx, repo, &gogit.LogOptions{All: true}, nil, common.FilterEmpty(), chunksChan)
if err != nil {
errors = append(errors, err)
continue
@ -236,7 +237,7 @@ func (s *Source) scanRepos(ctx context.Context, chunksChan chan *sources.Chunk,
errors = append(errors, err)
continue
}
err = s.git.ScanRepo(ctx, repo, &gogit.LogOptions{All: true}, nil, chunksChan)
err = s.git.ScanRepo(ctx, repo, &gogit.LogOptions{All: true}, nil, common.FilterEmpty(), chunksChan)
if err != nil {
errors = append(errors, err)
continue