Add --json-legacy flag to make output match pre-v3.0

.gitignore

@@ -0,0 +1 @@
+.idea

go.mod

@@ -23,6 +23,7 @@ require (
 	github.com/mattn/go-colorable v0.1.12
 	github.com/pkg/errors v0.9.1
 	github.com/razorpay/razorpay-go v0.0.0-20210728161131-0341409a6ab2
+	github.com/sergi/go-diff v1.2.0
 	github.com/sirupsen/logrus v1.8.1
 	github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502
 	github.com/xanzy/go-gitlab v0.54.3
@@ -75,7 +76,6 @@ require (
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.1 // indirect
 	github.com/pkg/diff v0.0.0-20200914180035-5b29258ca4f7 // indirect
-	github.com/sergi/go-diff v1.1.0 // indirect
 	github.com/xanzy/ssh-agent v0.3.0 // indirect
 	go.opencensus.io v0.23.0 // indirect
 	golang.org/x/mod v0.5.0 // indirect

go.sum

@@ -317,8 +317,9 @@ github.com/razorpay/razorpay-go v0.0.0-20210728161131-0341409a6ab2/go.mod h1:Vcl
 github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
-github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0=
 github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
+github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ=
+github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
 github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
 github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE=
 github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=

main.go

@@ -17,15 +17,16 @@ import (
 	"github.com/trufflesecurity/trufflehog/pkg/common"
 	"github.com/trufflesecurity/trufflehog/pkg/decoders"
 	"github.com/trufflesecurity/trufflehog/pkg/engine"
+	"github.com/trufflesecurity/trufflehog/pkg/output"
 	"github.com/trufflesecurity/trufflehog/pkg/pb/source_metadatapb"
 	"github.com/trufflesecurity/trufflehog/pkg/sources/git"
 )
 
 func main() {
-
 	cli := kingpin.New("TruffleHog", "TruffleHog is a tool for finding credentials.")
 	debug := cli.Flag("debug", "Run in debug mode").Bool()
 	jsonOut := cli.Flag("json", "Output in JSON format.").Short('j').Bool()
+	jsonLegacy := cli.Flag("json-legacy", "Use the pre-v3.0 JSON format. Only works with git, gitlab, and github sources.").Bool()
 	concurrency := cli.Flag("concurrency", "Number of concurrent workers.").Default(strconv.Itoa(runtime.NumCPU())).Int()
 	noVerification := cli.Flag("no-verification", "Don't verify the results.").Bool()
 	// rules := cli.Flag("rules", "Path to file with custom rules.").String()
@@ -86,9 +87,11 @@ func main() {
 		logrus.WithError(err)
 	}
 
+	var repoPath string
 	switch cmd {
 	case gitScan.FullCommand():
-		repoPath, remote, err := git.PrepareRepo(*gitScanURI)
+		var remote bool
+		repoPath, remote, err = git.PrepareRepo(*gitScanURI)
 		if err != nil || repoPath == "" {
 			logrus.WithError(err).Fatal("error preparing git repo for scanning")
 		}
@@ -122,37 +125,40 @@ func main() {
 	redPrinter := color.New(color.FgRed)
 	whitePrinter := color.New(color.FgWhite)
 
-	fmt.Printf("🐷🔑🐷  TruffleHog. Unearth your secrets. 🐷🔑🐷\n\n")
+	if !*jsonLegacy && !*jsonOut {
+		fmt.Printf("🐷🔑🐷  TruffleHog. Unearth your secrets. 🐷🔑🐷\n\n")
+	}
 
 	for r := range e.ResultsChan() {
-		type outputFormat struct {
-			DetectorType string
-			Verified     bool
-			*source_metadatapb.MetaData
-		}
-		output := outputFormat{
+		out := outputFormat{
 			DetectorType: r.Result.DetectorType.String(),
 			Verified:     r.Result.Verified,
 			MetaData:     r.SourceMetadata,
 		}
 
-		if *jsonOut {
-			// todo - add parity to trufflehog's existing output for git
-			// source
-			out, err := json.Marshal(output)
+		switch {
+		case *jsonLegacy:
+			legacy := output.ConvertToLegacyJSON(&r, repoPath)
+			out, err := json.Marshal(legacy)
 			if err != nil {
 				logrus.WithError(err).Fatal("could not marshal result")
 			}
 			fmt.Println(string(out))
-		} else {
-			meta, err := structToMap(output.MetaData.Data)
+		case *jsonOut:
+			out, err := json.Marshal(r)
+			if err != nil {
+				logrus.WithError(err).Fatal("could not marshal result")
+			}
+			fmt.Println(string(out))
+		default:
+			meta, err := structToMap(out.MetaData.Data)
 			if err != nil {
 				logrus.WithError(err).Fatal("could not marshal result")
 			}
 
 			yellowPrinter.Print("Found result 🐷🔑\n")
-			greenPrinter.Printf("Detector Type: %s\n", output.DetectorType)
-			if output.Verified {
+			greenPrinter.Printf("Detector Type: %s\n", out.DetectorType)
+			if out.Verified {
 				redPrinter.Print("Verified: true\n")
 			} else {
 				whitePrinter.Print("Verified: false\n")
@@ -176,3 +182,9 @@ func structToMap(obj interface{}) (m map[string]map[string]interface{}, err erro
 	err = json.Unmarshal(data, &m)
 	return
 }
+
+type outputFormat struct {
+	DetectorType string
+	Verified     bool
+	*source_metadatapb.MetaData
+}

pkg/output/legacy_json.go

@@ -0,0 +1,188 @@
+package output
+
+import (
+	"fmt"
+	gogit "github.com/go-git/go-git/v5"
+	"github.com/go-git/go-git/v5/plumbing"
+	"github.com/go-git/go-git/v5/plumbing/object"
+	"github.com/sergi/go-diff/diffmatchpatch"
+	"github.com/sirupsen/logrus"
+	"github.com/trufflesecurity/trufflehog/pkg/detectors"
+	"github.com/trufflesecurity/trufflehog/pkg/pb/sourcespb"
+	"log"
+	"net/url"
+	"strings"
+)
+
+func ConvertToLegacyJSON(r *detectors.ResultWithMetadata, repoPath string) *LegacyJSONOutput {
+	var source LegacyJSONCompatibleSource
+	switch r.SourceType {
+	case sourcespb.SourceType_SOURCE_TYPE_GIT:
+		source = r.SourceMetadata.GetGit()
+	case sourcespb.SourceType_SOURCE_TYPE_GITHUB:
+		source = r.SourceMetadata.GetGithub()
+	case sourcespb.SourceType_SOURCE_TYPE_GITLAB:
+		source = r.SourceMetadata.GetGitlab()
+	default:
+		log.Fatalf("legacy JSON output can not be used with this source: %s", r.SourceName)
+	}
+
+	// The repo will be needed to gather info needed for the legacy output that isn't included in the new
+	// output format.
+	repo, err := gogit.PlainOpenWithOptions(repoPath, &gogit.PlainOpenOptions{DetectDotGit: true})
+	if err != nil {
+		logrus.WithError(err).Fatalf("could open repo: %s", repoPath)
+	}
+
+	fileName := source.GetFile()
+	commitHash := plumbing.NewHash(source.GetCommit())
+	commit, err := repo.CommitObject(commitHash)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	diff := GenerateDiff(commit, fileName)
+
+	foundString := string(r.Result.Raw)
+
+	// Add highlighting to the offending bit of string.
+	printableDiff := strings.ReplaceAll(diff, foundString, fmt.Sprintf("\u001b[93m%s\u001b[0m", foundString))
+
+	// Load up the struct to match the old JSON format
+	output := &LegacyJSONOutput{
+		Branch:       FindBranch(commit, repo),
+		Commit:       commit.Message,
+		CommitHash:   commitHash.String(),
+		Date:         commit.Committer.When.Format("2006-01-02 15:04:05"),
+		Diff:         diff,
+		Path:         fileName,
+		PrintDiff:    printableDiff,
+		Reason:       r.Result.DetectorType.String(),
+		StringsFound: []string{foundString},
+	}
+	return output
+}
+
+// BranchHeads creates a map of branch names to their head commit. This can be used to find if a commit is an ancestor
+// of a branch head.
+func BranchHeads(repo *gogit.Repository) (map[string]*object.Commit, error) {
+	branches := map[string]*object.Commit{}
+	branchIter, err := repo.Branches()
+	if err != nil {
+		return branches, err
+	}
+
+	branchIter.ForEach(func(branchRef *plumbing.Reference) error {
+		branchName := branchRef.Name().String()
+		headHash, err := repo.ResolveRevision(plumbing.Revision(branchName))
+		if err != nil {
+			logrus.WithError(err).Errorf("unable to resolve head of branch: %s", branchRef.Name().String())
+			return nil
+		}
+		headCommit, err := repo.CommitObject(*headHash)
+		if err != nil {
+			logrus.WithError(err).Errorf("unable to get commit: %s", headCommit.String())
+			return nil
+		}
+		branches[branchName] = headCommit
+		return nil
+	})
+	return branches, nil
+}
+
+// FindBranch returns the first branch a commit is a part of. Not the most accurate, but it should work similar to pre v3.0.
+func FindBranch(commit *object.Commit, repo *gogit.Repository) string {
+	branches, err := BranchHeads(repo)
+	if err != nil {
+		logrus.WithError(err).Fatal("could not list branches")
+	}
+
+	for name, head := range branches {
+		isAncestor, err := commit.IsAncestor(head)
+		if err != nil {
+			logrus.WithError(err).Errorf("could not determine if %s is an ancestor of %s", commit.Hash.String(), head.Hash.String())
+			continue
+		}
+		if isAncestor {
+			return name
+		}
+	}
+	return ""
+}
+
+// GenerateDiff will take a commit and create a string diff between the commit and its first parent.
+func GenerateDiff(commit *object.Commit, fileName string) string {
+	var diff string
+
+	// First grab the first parent of the commit. If there are none, we are at the first commit and should diff against
+	// an empty file.
+	parent, err := commit.Parent(0)
+	if err != object.ErrParentNotFound && err != nil {
+		logrus.WithError(err).Errorf("could not find parent of %s", commit.Hash.String())
+	}
+
+	// Now get the files from the commit and its parent.
+	var parentFile *object.File
+	if parent != nil {
+		parentFile, err = parent.File(fileName)
+		if err != nil && err != object.ErrFileNotFound {
+			logrus.WithError(err).Errorf("could not get previous version of file: %q", fileName)
+			return diff
+		}
+	}
+	commitFile, err := commit.File(fileName)
+	if err != nil {
+		logrus.WithError(err).Errorf("could not get current version of file: %q", fileName)
+		return diff
+	}
+
+	// go-git doesn't support creating a diff for just one file in a commit, so another package is needed to generate
+	// the diff.
+	dmp := diffmatchpatch.New()
+	var oldContent, newContent string
+	if parentFile != nil {
+		oldContent, err = parentFile.Contents()
+		if err != nil {
+			logrus.WithError(err).Errorf("could not get contents of previous version of file: %q", fileName)
+		}
+	}
+	// commitFile should never be nil at this point, but double-checking so we don't get a nil error.
+	if commitFile != nil {
+		newContent, _ = commitFile.Contents()
+		if err != nil {
+			logrus.WithError(err).Errorf("could not get contents of current version of file: %q", fileName)
+		}
+	}
+
+	// If anything has gone wrong here, we'll just be diffing two empty files.
+	diffs := dmp.DiffMain(oldContent, newContent, false)
+	patches := dmp.PatchMake(diffs)
+
+	// Put all the pieces of the diff together into one string.
+	for _, patch := range patches {
+		// The String() method URL escapes the diff, so it needs to be undone.
+		patchDiff, err := url.QueryUnescape(patch.String())
+		if err != nil {
+			logrus.WithError(err).Error("unable to unescape diff")
+		}
+		diff += patchDiff
+	}
+	return diff
+}
+
+type LegacyJSONOutput struct {
+	Branch       string   `json:"branch"`
+	Commit       string   `json:"commit"`
+	CommitHash   string   `json:"commitHash"`
+	Date         string   `json:"date"`
+	Diff         string   `json:"diff"`
+	Path         string   `json:"path"`
+	PrintDiff    string   `json:"printDiff"`
+	Reason       string   `json:"reason"`
+	StringsFound []string `json:"stringsFound"`
+}
+
+type LegacyJSONCompatibleSource interface {
+	GetCommit() string
+	GetFile() string
+}