Initial docs and release automation (#5)

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,33 @@
+---
+name: Bug report
+about: Create a report to help us improve
+title: ""
+labels: bug, needs triage
+assignees: dustin-decker, bill-rich
+---
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**To Reproduce**
+Steps to reproduce the behavior:
+1. Go to '...'
+2. Click on '....'
+3. Scroll down to '....'
+4. See error
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+**Output**
+Please include relevant log output, and run with the `--debug` flag.
+
+**Screenshots**
+If applicable, add screenshots to help explain your problem.
+
+**Desktop (please complete the following information):**
+ - OS: [e.g. iOS]
+ - Version [e.g. 22]
+
+**Additional context**
+Add any other context about the problem here.

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,19 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+title: ""
+labels: enhancement, needs triage
+assignees: dustin-decker, bill-rich
+---
+
+**Is your feature request related to a problem? Please describe.**
+A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+
+**Describe the solution you'd like**
+A clear and concise description of what you want to happen.
+
+**Describe alternatives you've considered**
+A clear and concise description of any alternative solutions or features you've considered.
+
+**Additional context**
+Add any other context or screenshots about the feature request here.

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,4 @@
+<!--
+Please create an issue to collect feedback prior to feature additions. Please also reference that issue in any PRs.
+If possible try to keep PRs scoped to one feature, and add tests for new features.
+-->

.github/workflows/codeql-analysis.yml

@@ -0,0 +1,70 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
+name: "CodeQL"
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [ main ]
+  schedule:
+    - cron: '35 11 * * 2'
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ 'go' ]
+        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
+        # Learn more about CodeQL language support at https://git.io/codeql-language-support
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v1
+      with:
+        languages: ${{ matrix.language }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+        # queries: ./path/to/local/query, your-org/your-repo/queries@main
+
+    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
+    # If this step fails, then you should remove it and run the build manually (see below)
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@v1
+
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 https://git.io/JvXDl
+
+    # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
+    #    and modify them (or add more) to build your code if your project
+    #    uses a compiled language
+
+    #- run: |
+    #   make bootstrap
+    #   make release
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v1

.github/workflows/release.yml

@@ -0,0 +1,30 @@
+name: release
+
+on:
+  push:
+    tags:
+      - 'v*'
+
+jobs:
+  goreleaser:
+    runs-on: ubuntu-latest
+    steps:
+      -
+        name: Checkout
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+      -
+        name: Set up Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.17
+      -
+        name: Run GoReleaser
+        uses: goreleaser/goreleaser-action@v2
+        with:
+          distribution: goreleaser
+          version: latest
+          args: release --rm-dist
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/test.yml

@@ -0,0 +1,26 @@
+on: [push, pull_request]
+name: Test
+jobs:
+  test:
+    strategy:
+      matrix:
+        go-version: [1.17.x]
+        os: [ubuntu-latest]
+    runs-on: ${{ matrix.os }}
+    permissions:
+      contents: 'read'
+      id-token: 'write'
+    steps:
+    - name: Install Go
+      uses: actions/setup-go@v2
+      with:
+        go-version: ${{ matrix.go-version }}
+    - name: Checkout code
+      uses: actions/checkout@v2
+    - id: 'auth'
+      uses: 'google-github-actions/auth@v0'
+      with:
+        workload_identity_provider: 'projects/811013774421/locations/global/workloadIdentityPools/github-pool/providers/github-provider'
+        service_account: 'github-ci-external@trufflehog-testing.iam.gserviceaccount.com'
+    - name: Test
+      run: go test -timeout 30s ./...

CODE_OF_CONDUCT.md

@@ -0,0 +1,76 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+In the interest of fostering an open and welcoming environment, we as
+contributors and maintainers pledge to making participation in our project and
+our community a harassment-free experience for everyone, regardless of age, body
+size, disability, ethnicity, sex characteristics, gender identity and expression,
+level of experience, education, socio-economic status, nationality, personal
+appearance, race, religion, or sexual identity and orientation.
+
+## Our Standards
+
+Examples of behavior that contributes to creating a positive environment
+include:
+
+* Using welcoming and inclusive language
+* Being respectful of differing viewpoints and experiences
+* Gracefully accepting constructive criticism
+* Focusing on what is best for the community
+* Showing empathy towards other community members
+
+Examples of unacceptable behavior by participants include:
+
+* The use of sexualized language or imagery and unwelcome sexual attention or
+ advances
+* Trolling, insulting/derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or electronic
+ address, without explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+ professional setting
+
+## Our Responsibilities
+
+Project maintainers are responsible for clarifying the standards of acceptable
+behavior and are expected to take appropriate and fair corrective action in
+response to any instances of unacceptable behavior.
+
+Project maintainers have the right and responsibility to remove, edit, or
+reject comments, commits, code, wiki edits, issues, and other contributions
+that are not aligned to this Code of Conduct, or to ban temporarily or
+permanently any contributor for other behaviors that they deem inappropriate,
+threatening, offensive, or harmful.
+
+## Scope
+
+This Code of Conduct applies both within project spaces and in public spaces
+when an individual is representing the project or its community. Examples of
+representing a project or community include using an official project e-mail
+address, posting via an official social media account, or acting as an appointed
+representative at an online or offline event. Representation of a project may be
+further defined and clarified by project maintainers.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported by contacting the project team at community@trufflesec.com. All
+complaints will be reviewed and investigated and will result in a response that
+is deemed necessary and appropriate to the circumstances. The project team is
+obligated to maintain confidentiality with regard to the reporter of an incident.
+Further details of specific enforcement policies may be posted separately.
+
+Project maintainers who do not follow or enforce the Code of Conduct in good
+faith may face temporary or permanent repercussions as determined by other
+members of the project's leadership.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
+available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html
+
+[homepage]: https://www.contributor-covenant.org
+
+For answers to common questions about this code of conduct, see
+https://www.contributor-covenant.org/faq

CONTRIBUTING.md

@@ -0,0 +1,7 @@
+# Contribution guidelines
+
+Please create an issue to collect feedback prior to feature additions. If possible try to keep PR's scoped to one feature, and add tests for new features.
+
+When showing intereste, in a bug, enhancement, PR, or issue, please use the thumbs up/thumbs down emoji on the original message rather than adding comments expressing the same.
+
+Contributors need to sign our CLA before we are able to accept contributions: 

Dockerfile

@@ -0,0 +1,10 @@
+FROM golang:bullseye as builder
+RUN mkdir /build
+COPY . /build
+WORKDIR /build
+RUN CGO_ENABLED=0 go build -a -o trufflehog main.go
+
+FROM scratch
+COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
+COPY --from=builder /build/trufflehog /usr/bin/trufflehog
+ENTRYPOINT ["/usr/bin/trufflehog"]

Makefile

@@ -26,7 +26,10 @@ bench:
 	CGO_ENABLED=0 go test $(shell go list ./pkg/secrets/... | grep -v /vendor/) -benchmem -run=xxx -bench .
 
 run:
-	CGO_ENABLED=0 go run . git file://.
+	CGO_ENABLED=0 go run . git file://. --json
+
+run-debug:
+	CGO_ENABLED=0 go run . git file://. --json --debug
 
 protos:
 	docker run -u "$(shell id -u)" -v "$(shell dirname $(realpath $(firstword $(MAKEFILE_LIST))))":/pwd "${PROTOS_IMAGE}" bash -c "cd /pwd; /pwd/scripts/gen_proto.sh"

README.md

@@ -0,0 +1,71 @@
+# TruffleHog
+
+# Join The Slack
+Have questions? Feedback? Jump in slack and hang out with us
+
+https://join.slack.com/t/trufflehog-community/shared_invite/zt-pw2qbi43-Aa86hkiimstfdKH9UCpPzQ
+
+## Installation
+
+Several options:
+
+### 1. Go
+`go install github.com/trufflesecurity/trufflehog2.git@latest`
+
+### 2. [Release binaries](https://github.com/trufflesecurity/trufflehog2/releases)
+
+### 3. Docker (TODO)
+`docker run ...`
+
+### 4. Pip (TODO)
+pip install trufflehog
+
+### 5. Brew (TODO)
+brew install trufflehog
+
+## Usage
+
+TruffleHog has a sub-command for each source of data that you may want to scan:
+
+- git
+- github
+- gitlab
+- slack
+- S3
+
+Each subcommand can have options that you can see with the `-h` flag provided to the sub command:
+
+```
+$ trufflehog git --help
+usage: TruffleHog git [<flags>] <uri>
+
+Find credentials in git repositories.
+
+Flags:
+      --help           Show context-sensitive help (also try --help-long and --help-man).
+      --debug          Run in debug mode
+      --json           Output in JSON format.
+      --concurrency=8  Number of concurrent workers.
+      --verification   Verify the results.
+  -i, --include_paths=INCLUDE_PATHS  
+                       Path to file with newline separated regexes for files to include in scan.
+  -x, --exclude_paths=EXCLUDE_PATHS  
+                       Path to file with newline separated regexes for files to exclude in scan.
+      --branch=BRANCH  Branch to scan.
+      --allow          No-op flag for backwards compat.
+      --entropy        No-op flag for backwards compat.
+      --regex          No-op flag for backwards compat.
+
+Args:
+  <uri>  Git repository URL. https:// or file:// schema expected.
+```
+
+For example, to scan a  `git` repository, start with
+
+```
+$ trufflehog git https://github.com/trufflesecurity/trufflehog2.git
+```
+
+# License Change
+
+Since v3.0, TruffleHog is released under a AGPL 3 license, included in [`LICENSE`](LICENSE). TruffleHog v3.0 uses none of the previous codebase, but care was taken to preserve backwards compatibility on the command line interface. The work previous to this release is still available licensed under GPL 2.0 in the history of this repository and the previous package releases and tags. A completed CLA is required for us to accept contributions going forward.

SECURITY.md

@@ -0,0 +1 @@
+Please report any security issues to security@trufflesec.com and include `trufflehog` in the subject

main.go

@@ -4,13 +4,14 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"github.com/trufflesecurity/trufflehog/pkg/common"
-	"github.com/trufflesecurity/trufflehog/pkg/sources/git"
 	"log"
 	"os"
 	"runtime"
 	"strconv"
 
+	"github.com/trufflesecurity/trufflehog/pkg/common"
+	"github.com/trufflesecurity/trufflehog/pkg/sources/git"
+
 	"github.com/sirupsen/logrus"
 	"github.com/trufflesecurity/trufflehog/pkg/decoders"
 	"github.com/trufflesecurity/trufflehog/pkg/engine"
@@ -21,7 +22,7 @@ func main() {
 
 	cli := kingpin.New("TruffleHog", "TruffleHog is a tool for finding credentials.")
 	debug := cli.Flag("debug", "Run in debug mode").Bool()
-	jsonOut := cli.Flag("json", "Output in JSON format.").Bool()
+	jsonOut := cli.Flag("json", "Output in JSON format.").Short('j').Bool()
 	concurrency := cli.Flag("concurrency", "Number of concurrent workers.").Default(strconv.Itoa(runtime.NumCPU())).Int()
 	verification := cli.Flag("verification", "Verify the results.").Bool()
 	// rules := cli.Flag("rules", "Path to file with custom rules.").String()

pkg/common/filter.go

@@ -3,9 +3,10 @@ package common
 import (
 	"bufio"
 	"fmt"
-	log "github.com/sirupsen/logrus"
 	"os"
 	"regexp"
+
+	log "github.com/sirupsen/logrus"
 )
 
 type Filter struct {
@@ -82,7 +83,7 @@ func FilterRulesFromFile(source string) (*FilterRuleSet, error) {
 func (filter *Filter) Pass(object string) bool {
 	excluded := filter.exclude.Matches(object)
 	included := filter.include.Matches(object)
-	log.Debugf("test PathFilter: file: %s, included: %t, excluded: %t, pass: %t", object, included, excluded, !excluded && included)
+	// log.Debugf("test PathFilter: file: %s, included: %t, excluded: %t, pass: %t", object, included, excluded, !excluded && included)
 	return !excluded && included
 }
 

pkg/engine/git.go

@@ -3,15 +3,16 @@ package engine
 import (
 	"context"
 	"fmt"
+	"runtime"
+
 	gogit "github.com/go-git/go-git/v5"
 	"github.com/go-git/go-git/v5/plumbing"
-	"github.com/go-git/go-git/v5/plumbing/object"
 	"github.com/sirupsen/logrus"
+
 	"github.com/trufflesecurity/trufflehog/pkg/common"
 	"github.com/trufflesecurity/trufflehog/pkg/pb/source_metadatapb"
 	"github.com/trufflesecurity/trufflehog/pkg/pb/sourcespb"
 	"github.com/trufflesecurity/trufflehog/pkg/sources/git"
-	"runtime"
 )
 
 func (e *Engine) ScanGit(ctx context.Context, repoPath, gitScanBranch, headRef string, filter *common.Filter) error {
@@ -90,7 +91,7 @@ func (e *Engine) ScanGit(ctx context.Context, repoPath, gitScanBranch, headRef s
 		})
 
 	go func() {
-		err := gitSource.ScanRepo(ctx, repo, scanOptions, &object.Commit{}, filter, e.ChunksChan())
+		err := gitSource.ScanRepo(ctx, repo, scanOptions, nil, filter, e.ChunksChan())
 		if err != nil {
 			logrus.WithError(err).Fatal("could not scan repo")
 		}

pkg/sources/git/git_test.go

@@ -34,7 +34,7 @@ func TestSource_Scan(t *testing.T) {
 			init: init{
 				name: "this repo",
 				connection: &sourcespb.Git{
-					Directories: []string{"../../../../"},
+					Directories: []string{"../../../"},
 					Credential: &sourcespb.Git_Unauthenticated{
 						Unauthenticated: &credentialspb.Unauthenticated{},
 					},

pkg/sources/github/github.go

@@ -313,7 +313,7 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
 			log.WithError(err).Warnf("unable to clone repo, continuing")
 			continue
 		}
-		err = s.git.ScanRepo(ctx, repo, &gogit.LogOptions{All: true}, nil, chunksChan)
+		err = s.git.ScanRepo(ctx, repo, &gogit.LogOptions{All: true}, nil, common.FilterEmpty(), chunksChan)
 		if err != nil {
 			log.WithError(err).Warnf("unable to scan repo")
 		}
@@ -407,7 +407,7 @@ func (s *Source) paginateGists(ctx context.Context, user string, chunksChan chan
 		s.log.WithField("repo", *gist.HTMLURL).Debugf("attempting to clone gist from user %s", user)
 
 		scanCtx := context.Background()
-		err = s.git.ScanRepo(scanCtx, repo, &gogit.LogOptions{All: true}, nil, chunksChan)
+		err = s.git.ScanRepo(scanCtx, repo, &gogit.LogOptions{All: true}, nil, common.FilterEmpty(), chunksChan)
 		if err != nil {
 			log.WithError(err).Warnf("Could not scan after clone: %s", *gist.HTMLURL)
 			continue

pkg/sources/gitlab/gitlab.go

@@ -14,6 +14,7 @@ import (
 	"google.golang.org/protobuf/proto"
 	"google.golang.org/protobuf/types/known/anypb"
 
+	"github.com/trufflesecurity/trufflehog/pkg/common"
 	"github.com/trufflesecurity/trufflehog/pkg/pb/source_metadatapb"
 	"github.com/trufflesecurity/trufflehog/pkg/pb/sourcespb"
 
@@ -215,7 +216,7 @@ func (s *Source) scanRepos(ctx context.Context, chunksChan chan *sources.Chunk,
 				errors = append(errors, err)
 				continue
 			}
-			err = s.git.ScanRepo(ctx, repo, &gogit.LogOptions{All: true}, nil, chunksChan)
+			err = s.git.ScanRepo(ctx, repo, &gogit.LogOptions{All: true}, nil, common.FilterEmpty(), chunksChan)
 			if err != nil {
 				errors = append(errors, err)
 				continue
@@ -236,7 +237,7 @@ func (s *Source) scanRepos(ctx context.Context, chunksChan chan *sources.Chunk,
 				errors = append(errors, err)
 				continue
 			}
-			err = s.git.ScanRepo(ctx, repo, &gogit.LogOptions{All: true}, nil, chunksChan)
+			err = s.git.ScanRepo(ctx, repo, &gogit.LogOptions{All: true}, nil, common.FilterEmpty(), chunksChan)
 			if err != nil {
 				errors = append(errors, err)
 				continue