Mirrors/trufflehog
2
0
Fork 0
mirror of https://github.com/trufflesecurity/trufflehog.git synced 2024-11-10 07:04:24 +00:00
Code Issues Projects Releases Packages Wiki Activity

Pull gitparse config options out of pkg consts (#1072)

Browse source 
* Pull gitparse config options out of pkg consts.

* Adjust naming
This commit is contained in:
Bill Rich 2023-02-04 13:19:23 -08:00 committed by GitHub
parent 2d35edfb23
commit af6e3f8fdf
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 71 additions and 25 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

71
pkg/gitparse/gitparse.go
View file

@ -18,11 +18,14 @@ import (
)
const (
// DateFormat is the standard date format for git.
DateFormat = "Mon Jan 02 15:04:05 2006 -0700"
// defaultDateFormat is the standard date format for git.
defaultDateFormat = "Mon Jan 02 15:04:05 2006 -0700"
// MaxDiffSize is the maximum size for a diff. Larger diffs will be cut off.
MaxDiffSize = 1 * 1024 * 1024 * 1024 // 1GB
// defaultMaxDiffSize is the maximum size for a diff. Larger diffs will be cut off.
defaultMaxDiffSize = 1 * 1024 * 1024 * 1024 // 1GB
// defaultMaxCommitSize is the maximum size for a commit. Larger commits will be cut off.
defaultMaxCommitSize = 1 * 1024 * 1024 * 1024 // 1GB
)
// Commit contains commit header info and diffs.
@ -42,6 +45,46 @@ type Diff struct {
IsBinary bool
}
// Parser sets values used in GitParse.
type Parser struct {
maxDiffSize int
maxCommitSize int
dateFormat string
}
// WithMaxDiffSize sets maxDiffSize option. Diffs larger than maxDiffSize will
// be truncated.
func WithMaxDiffSize(maxDiffSize int) Option {
return func(parser *Parser) {
parser.maxDiffSize = maxDiffSize
}
}
// WithMaxCommitSize sets maxCommitSize option. Commits larger than maxCommitSize
// will be put in the commit channel and additional diffs will be added to a
// new commit.
func WithMaxCommitSize(maxCommitSize int) Option {
return func(parser *Parser) {
parser.maxCommitSize = maxCommitSize
}
}
// Option is used for adding options to Config.
type Option func(*Parser)
// NewParser creates a GitParse config from options and sets defaults.
func NewParser(options ...Option) *Parser {
parser := &Parser{
dateFormat: defaultDateFormat,
maxDiffSize: defaultMaxDiffSize,
maxCommitSize: defaultMaxCommitSize,
}
for _, option := range options {
option(parser)
}
return parser
}
// Equal compares the content of two Commits to determine if they are the same.
func (c1 *Commit) Equal(c2 *Commit) bool {
switch {
@ -75,7 +118,7 @@ func (c1 *Commit) Equal(c2 *Commit) bool {
}
// RepoPath parses the output of the `git log` command for the `source` path.
func RepoPath(ctx context.Context, source string, head string, abbreviatedLog bool) (chan Commit, error) {
func (c *Parser) RepoPath(ctx context.Context, source string, head string, abbreviatedLog bool) (chan Commit, error) {
args := []string{"-C", source, "log", "-p", "-U5", "--full-history", "--date=format:%a %b %d %H:%M:%S %Y %z"}
if abbreviatedLog {
args = append(args, "--diff-filter=AM")
@ -93,11 +136,11 @@ func RepoPath(ctx context.Context, source string, head string, abbreviatedLog bo
cmd.Env = append(cmd.Env, fmt.Sprintf("GIT_DIR=%s", filepath.Join(absPath, ".git")))
}
return executeCommand(ctx, cmd)
return c.executeCommand(ctx, cmd)
}
// Unstaged parses the output of the `git diff` command for the `source` path.
func Unstaged(ctx context.Context, source string) (chan Commit, error) {
func (c *Parser) Unstaged(ctx context.Context, source string) (chan Commit, error) {
args := []string{"-C", source, "diff", "-p", "-U5", "--full-history", "--diff-filter=AM", "--date=format:%a %b %d %H:%M:%S %Y %z", "HEAD"}
cmd := exec.Command("git", args...)
@ -107,11 +150,11 @@ func Unstaged(ctx context.Context, source string) (chan Commit, error) {
cmd.Env = append(cmd.Env, fmt.Sprintf("GIT_DIR=%s", filepath.Join(absPath, ".git")))
}
return executeCommand(ctx, cmd)
return c.executeCommand(ctx, cmd)
}
// executeCommand runs an exec.Cmd, reads stdout and stderr, and waits for the Cmd to complete.
func executeCommand(ctx context.Context, cmd *exec.Cmd) (chan Commit, error) {
func (c *Parser) executeCommand(ctx context.Context, cmd *exec.Cmd) (chan Commit, error) {
commitChan := make(chan Commit, 64)
stdOut, err := cmd.StdoutPipe()
@ -136,7 +179,7 @@ func executeCommand(ctx context.Context, cmd *exec.Cmd) (chan Commit, error) {
}()
go func() {
FromReader(ctx, stdOut, commitChan)
c.fromReader(ctx, stdOut, commitChan)
if err := cmd.Wait(); err != nil {
log.WithError(err).Debugf("Error waiting for git command to complete.")
}
@ -145,7 +188,7 @@ func executeCommand(ctx context.Context, cmd *exec.Cmd) (chan Commit, error) {
return commitChan, nil
}
func FromReader(ctx context.Context, stdOut io.Reader, commitChan chan Commit) {
func (c *Parser) fromReader(ctx context.Context, stdOut io.Reader, commitChan chan Commit) {
outReader := bufio.NewReader(stdOut)
var currentCommit *Commit
var currentDiff *Diff
@ -178,7 +221,7 @@ func FromReader(ctx context.Context, stdOut io.Reader, commitChan chan Commit) {
case isAuthorLine(line):
currentCommit.Author = strings.TrimRight(string(line[8:]), "\n")
case isDateLine(line):
date, err := time.Parse(DateFormat, strings.TrimSpace(string(line[6:])))
date, err := time.Parse(c.dateFormat, strings.TrimSpace(string(line[6:])))
if err != nil {
log.WithError(err).Debug("Could not parse date from git stream.")
}
@ -230,8 +273,8 @@ func FromReader(ctx context.Context, stdOut io.Reader, commitChan chan Commit) {
}
}
}
if currentDiff.Content.Len() > MaxDiffSize {
log.Debugf("Diff for %s exceeded MaxDiffSize(%d)", currentDiff.PathB, MaxDiffSize)
if currentDiff.Content.Len() > c.maxDiffSize {
log.Debugf("Diff for %s exceeded MaxDiffSize(%d)", currentDiff.PathB, c.maxDiffSize)
break
}
}

21
pkg/gitparse/gitparse_test.go
View file

@ -87,7 +87,8 @@ func TestBinaryPathParse(t *testing.T) {
func TestSingleCommitSingleDiff(t *testing.T) {
r := bytes.NewReader([]byte(singleCommitSingleDiff))
commitChan := make(chan Commit)
date, _ := time.Parse(DateFormat, "Mon Mar 15 23:27:16 2021 -0700")
parser := NewParser()
date, _ := time.Parse(parser.dateFormat, "Mon Mar 15 23:27:16 2021 -0700")
content := bytes.NewBuffer([]byte(singleCommitSingleDiffDiff))
builder := strings.Builder{}
builder.Write([]byte(singleCommitSingleDiffMessage))
@ -108,7 +109,7 @@ func TestSingleCommitSingleDiff(t *testing.T) {
},
}
go func() {
FromReader(context.TODO(), r, commitChan)
parser.fromReader(context.TODO(), r, commitChan)
}()
i := 0
for commit := range commitChan {
@ -125,9 +126,10 @@ func TestSingleCommitSingleDiff(t *testing.T) {
func TestMultiCommitContextDiff(t *testing.T) {
r := bytes.NewReader([]byte(singleCommitContextDiff))
parser := NewParser()
commitChan := make(chan Commit)
dateOne, _ := time.Parse(DateFormat, "Mon Mar 15 23:27:16 2021 -0700")
dateTwo, _ := time.Parse(DateFormat, "Wed Dec 12 18:19:21 2018 -0800")
dateOne, _ := time.Parse(parser.dateFormat, "Mon Mar 15 23:27:16 2021 -0700")
dateTwo, _ := time.Parse(parser.dateFormat, "Wed Dec 12 18:19:21 2018 -0800")
diffOneA := bytes.NewBuffer([]byte(singleCommitContextDiffDiffOneA))
diffTwoA := bytes.NewBuffer([]byte(singleCommitContextDiffDiffTwoA))
// diffTwoB := bytes.NewBuffer([]byte(singleCommitContextDiffDiffTwoB))
@ -166,7 +168,7 @@ func TestMultiCommitContextDiff(t *testing.T) {
},
}
go func() {
FromReader(context.TODO(), r, commitChan)
NewParser().fromReader(context.TODO(), r, commitChan)
}()
i := 0
for commit := range commitChan {
@ -182,9 +184,10 @@ func TestMultiCommitContextDiff(t *testing.T) {
}
func TestMaxDiffSize(t *testing.T) {
parser := NewParser()
bigBytes := bytes.Buffer{}
bigBytes.WriteString(singleCommitSingleDiff)
for i := 0; i <= MaxDiffSize/1024+10; i++ {
for i := 0; i <= parser.maxDiffSize/1024+10; i++ {
bigBytes.WriteString("+")
for n := 0; n < 1024; n++ {
bigBytes.Write([]byte("0"))
@ -195,12 +198,12 @@ func TestMaxDiffSize(t *testing.T) {
commitChan := make(chan Commit)
go func() {
FromReader(context.TODO(), bigReader, commitChan)
parser.fromReader(context.TODO(), bigReader, commitChan)
}()
commit := <-commitChan
if commit.Diffs[0].Content.Len() > MaxDiffSize+1024 {
t.Errorf("diff did not match MaxDiffSize. Got: %d, expected (max): %d", commit.Diffs[0].Content.Len(), MaxDiffSize+1024)
if commit.Diffs[0].Content.Len() > parser.maxDiffSize+1024 {
t.Errorf("diff did not match MaxDiffSize. Got: %d, expected (max): %d", commit.Diffs[0].Content.Len(), parser.maxDiffSize+1024)
}
}

4
pkg/sources/git/git.go
View file

@ -330,7 +330,7 @@ func (s *Git) ScanCommits(ctx context.Context, repo *git.Repository, path string
return err
}
commitChan, err := gitparse.RepoPath(ctx, path, scanOptions.HeadHash, scanOptions.BaseHash == "")
commitChan, err := gitparse.NewParser().RepoPath(ctx, path, scanOptions.HeadHash, scanOptions.BaseHash == "")
if err != nil {
return err
}
@ -469,7 +469,7 @@ func (s *Git) ScanUnstaged(ctx context.Context, repo *git.Repository, path strin
// get the URL metadata for reporting (may be empty)
urlMetadata := getSafeRemoteURL(repo, "origin")
commitChan, err := gitparse.Unstaged(ctx, path)
commitChan, err := gitparse.NewParser().Unstaged(ctx, path)
if err != nil {
return err
}