[feat] - tmp file diffs (#2306)

* Write large diffs to tmp files * address comments * Move bufferedfilewriter to own pkg * update test * swallow write err * use buffer pool * use size vs len * use interface * fix test * update comments * fix test * remove unused * remove * remove unused * move parser and commit struct closer to where they are used * linter change * add more kvp pairs to error * fix test * update * address comments * remove bufferedfile writer * address comments * adjust interface * fix finalize * address comments * lint * remove guard * fix * add TODO
2024-11-10 07:04:24 +00:00 · 2024-01-30 12:30:51 -08:00 · 2024-01-30 12:30:51 -08:00 · 7c59ff95d5
commit 7c59ff95d5
parent 6824eb41ea
3 changed files with 455 additions and 225 deletions
--- a/pkg/gitparse/gitparse.go
+++ b/pkg/gitparse/gitparse.go
@ -4,7 +4,6 @@ import (
 	"bufio"
 	"bytes"
 	"fmt"
-	"github.com/go-logr/logr"
 	"io"
 	"os"
 	"os/exec"
@ -13,6 +12,8 @@ import (
 	"strings"
 	"time"

+	"github.com/go-logr/logr"
+
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
 )
@ -22,12 +23,122 @@ const (
 	defaultDateFormat = "Mon Jan 02 15:04:05 2006 -0700"

 	// defaultMaxDiffSize is the maximum size for a diff. Larger diffs will be cut off.
-	defaultMaxDiffSize = 1 * 1024 * 1024 * 1024 // 1GB
+	defaultMaxDiffSize = 2 * 1024 * 1024 * 1024 // 2GB

 	// defaultMaxCommitSize is the maximum size for a commit. Larger commits will be cut off.
-	defaultMaxCommitSize = 1 * 1024 * 1024 * 1024 // 1GB
+	defaultMaxCommitSize = 2 * 1024 * 1024 * 1024 // 2GB
 )

+// contentWriter defines a common interface for writing, reading, and managing diff content.
+// It abstracts the underlying storage mechanism, allowing flexibility in how content is handled.
+// This interface enables the use of different content storage strategies (e.g., in-memory buffer, file-based storage)
+// based on performance needs or resource constraints, providing a unified way to interact with different content types.
+type contentWriter interface { // Write appends data to the content storage.
+	// Write appends data to the content storage.
+	Write(ctx context.Context, data []byte) (int, error)
+	// ReadCloser provides a reader for accessing stored content.
+	ReadCloser() (io.ReadCloser, error)
+	// CloseForWriting closes the content storage for writing.
+	CloseForWriting() error
+	// Len returns the current size of the content.
+	Len() int
+	// String returns the content as a string or an error if the content cannot be converted to a string.
+	String() (string, error)
+}
+
+// state represents the current mode of buffer.
+type state uint8
+
+const (
+	// writeOnly indicates the buffer is in write-only mode.
+	writeOnly state = iota
+	// readOnly indicates the buffer has been closed and is in read-only mode.
+	readOnly
+)
+
+// buffer is a wrapper around bytes.Buffer, implementing the contentWriter interface.
+// This allows bytes.Buffer to be used wherever a contentWriter is required, ensuring compatibility
+// with the contentWriter interface while leveraging the existing implementation of bytes.Buffer.
+type buffer struct {
+	state state // current state of the buffer (writeOnly or readOnly)
+	bytes.Buffer
+}
+
+func newBuffer() *buffer { return &buffer{state: writeOnly} }
+
+// Write delegates the writing operation to the underlying bytes.Buffer, ignoring the context.
+// The context is included to satisfy the contentWriter interface, allowing for future extensions
+// where context handling might be necessary (e.g., for timeouts or cancellation).
+func (b *buffer) Write(_ context.Context, data []byte) (int, error) {
+	if b.state == readOnly {
+		return 0, fmt.Errorf("buffer is in read-only mode")
+	}
+	return b.Buffer.Write(data)
+}
+
+// ReadCloser provides a read-closer for the buffer's content.
+// It wraps the buffer's content in a NopCloser to provide a ReadCloser without additional closing behavior,
+// as closing a bytes.Buffer is a no-op.
+func (b *buffer) ReadCloser() (io.ReadCloser, error) {
+	if b.state == writeOnly {
+		return nil, fmt.Errorf("buffer is in write-only mode")
+	}
+	return io.NopCloser(bytes.NewReader(b.Bytes())), nil
+}
+
+// CloseForWriting is a no-op for buffer, as there is no resource cleanup needed for bytes.Buffer.
+func (b *buffer) CloseForWriting() error {
+	b.state = readOnly
+	return nil
+}
+
+// String returns the buffer's content as a string.
+func (b *buffer) String() (string, error) { return b.Buffer.String(), nil }
+
+// Diff contains the information about a file diff in a commit.
+// It abstracts the underlying content representation, allowing for flexible handling of diff content.
+// The use of contentWriter enables the management of diff data either in memory or on disk,
+// based on its size, optimizing resource usage and performance.
+type Diff struct {
+	PathB         string
+	LineStart     int
+	contentWriter contentWriter
+	IsBinary      bool
+}
+
+type diffOption func(*Diff)
+
+// withPathB sets the PathB option.
+func withPathB(pathB string) diffOption { return func(d *Diff) { d.PathB = pathB } }
+
+// NewDiff creates a new Diff with a threshold.
+func NewDiff(opts ...diffOption) *Diff {
+	diff := new(Diff)
+	diff.contentWriter = newBuffer()
+	for _, opt := range opts {
+		opt(diff)
+	}
+
+	return diff
+}
+
+// Len returns the length of the storage.
+func (d *Diff) Len() int { return d.contentWriter.Len() }
+
+// ReadCloser returns a ReadCloser for the contentWriter.
+func (d *Diff) ReadCloser() (io.ReadCloser, error) { return d.contentWriter.ReadCloser() }
+
+// write delegates to the contentWriter.
+func (d *Diff) write(ctx context.Context, p []byte) error {
+	_, err := d.contentWriter.Write(ctx, p)
+	return err
+}
+
+// finalize ensures proper closure of resources associated with the Diff.
+// handle the final flush in the finalize method, in case there's data remaining in the buffer.
+// This method should be called to release resources, especially when writing to a file.
+func (d *Diff) finalize() error { return d.contentWriter.CloseForWriting() }
+
 // Commit contains commit header info and diffs.
 type Commit struct {
 	Hash    string
@ -38,12 +149,53 @@ type Commit struct {
 	Size    int // in bytes
 }

-// Diff contains the info about a file diff in a commit.
-type Diff struct {
-	PathB     string
-	LineStart int
-	Content   bytes.Buffer
-	IsBinary  bool
+// Equal compares the content of two Commits to determine if they are the same.
+func (c1 *Commit) Equal(ctx context.Context, c2 *Commit) bool {
+	switch {
+	case c1.Hash != c2.Hash:
+		return false
+	case c1.Author != c2.Author:
+		return false
+	case !c1.Date.Equal(c2.Date):
+		return false
+	case c1.Message.String() != c2.Message.String():
+		return false
+	case len(c1.Diffs) != len(c2.Diffs):
+		return false
+	}
+
+	// isEqualString handles the error-prone String() method calls and compares the results.
+	isEqualContentString := func(s1, s2 contentWriter) (bool, error) {
+		str1, err := s1.String()
+		if err != nil {
+			return false, err
+		}
+		str2, err := s2.String()
+		if err != nil {
+			return false, err
+		}
+		return str1 == str2, nil
+	}
+
+	for i := range c1.Diffs {
+		d1 := c1.Diffs[i]
+		d2 := c2.Diffs[i]
+		switch {
+		case d1.PathB != d2.PathB:
+			return false
+		case d1.LineStart != d2.LineStart:
+			return false
+		case d1.IsBinary != d2.IsBinary:
+			return false
+		default:
+			equal, err := isEqualContentString(d1.contentWriter, d2.contentWriter)
+			if err != nil || !equal {
+				ctx.Logger().Error(err, "failed to compare diff content")
+				return false
+			}
+		}
+	}
+	return true
 }

 // Parser sets values used in GitParse.
@ -51,6 +203,7 @@ type Parser struct {
 	maxDiffSize   int
 	maxCommitSize int
 	dateFormat    string
+	contentWriter contentWriter
 }

 type ParseState int
@ -123,6 +276,7 @@ func NewParser(options ...Option) *Parser {
 		dateFormat:    defaultDateFormat,
 		maxDiffSize:   defaultMaxDiffSize,
 		maxCommitSize: defaultMaxCommitSize,
+		contentWriter: newBuffer(),
 	}
 	for _, option := range options {
 		option(parser)
@ -130,38 +284,6 @@ func NewParser(options ...Option) *Parser {
 	return parser
 }

-// Equal compares the content of two Commits to determine if they are the same.
-func (c1 *Commit) Equal(c2 *Commit) bool {
-	switch {
-	case c1.Hash != c2.Hash:
-		return false
-	case c1.Author != c2.Author:
-		return false
-	case !c1.Date.Equal(c2.Date):
-		return false
-	case c1.Message.String() != c2.Message.String():
-		return false
-	case len(c1.Diffs) != len(c2.Diffs):
-		return false
-	}
-	for i := range c1.Diffs {
-		d1 := c1.Diffs[i]
-		d2 := c2.Diffs[i]
-		switch {
-		case d1.PathB != d2.PathB:
-			return false
-		case d1.LineStart != d2.LineStart:
-			return false
-		case d1.Content.String() != d2.Content.String():
-			return false
-		case d1.IsBinary != d2.IsBinary:
-			return false
-		}
-	}
-	return true
-
-}
-
 // RepoPath parses the output of the `git log` command for the `source` path.
 func (c *Parser) RepoPath(ctx context.Context, source string, head string, abbreviatedLog bool, excludedGlobs []string, isBare bool) (chan Commit, error) {
 	args := []string{"-C", source, "log", "-p", "--full-history", "--date=format:%a %b %d %H:%M:%S %Y %z"}
@ -254,11 +376,11 @@ func (c *Parser) FromReader(ctx context.Context, stdOut io.Reader, commitChan ch
 	outReader := bufio.NewReader(stdOut)
 	var (
 		currentCommit *Commit
-		currentDiff   Diff

 		totalLogSize int
 	)
 	var latestState = Initial
+	currentDiff := NewDiff()

 	defer common.RecoverWithExit(ctx)
 	defer close(commitChan)
@ -277,9 +399,23 @@ func (c *Parser) FromReader(ctx context.Context, stdOut io.Reader, commitChan ch
 			latestState = CommitLine

 			// If there is a currentDiff, add it to currentCommit.
-			if currentDiff.Content.Len() > 0 || currentDiff.IsBinary {
-				currentCommit.Diffs = append(currentCommit.Diffs, currentDiff)
-				currentCommit.Size += currentDiff.Content.Len()
+			if currentDiff.Len() > 0 || currentDiff.IsBinary {
+				// TODO: Consider modifying the diffs field in the Commit struct to be a []*Diff.
+				// Otherwise, we end up with this temporal coupling where we have to finalize the diff
+				// before we can add it to the commit. I found this out the hard way when I tried to
+				// test this.
+				if err := currentDiff.finalize(); err != nil {
+					ctx.Logger().Error(
+						err,
+						"failed to finalize diff",
+						"commit", currentCommit.Hash,
+						"diff", currentDiff.PathB,
+						"size", currentDiff.Len(),
+						"latest_state", latestState.String(),
+					)
+				}
+				currentCommit.Diffs = append(currentCommit.Diffs, *currentDiff)
+				currentCommit.Size += currentDiff.Len()
 			}
 			// If there is a currentCommit, send it to the channel.
 			if currentCommit != nil {
@ -287,10 +423,8 @@ func (c *Parser) FromReader(ctx context.Context, stdOut io.Reader, commitChan ch
 				totalLogSize += currentCommit.Size
 			}
 			// Create a new currentDiff and currentCommit
-			currentDiff = Diff{}
-			currentCommit = &Commit{
-				Message: strings.Builder{},
-			}
+			currentDiff = NewDiff()
+			currentCommit = &Commit{Message: strings.Builder{}}
 			// Check that the commit line contains a hash and set it.
 			if len(line) >= 47 {
 				currentCommit.Hash = string(line[7:47])
@ -326,12 +460,21 @@ func (c *Parser) FromReader(ctx context.Context, stdOut io.Reader, commitChan ch
 			if currentCommit == nil {
 				currentCommit = &Commit{}
 			}
-			if currentDiff.Content.Len() > 0 || currentDiff.IsBinary {
-				currentCommit.Diffs = append(currentCommit.Diffs, currentDiff)
+			if currentDiff.Len() > 0 || currentDiff.IsBinary {
+				if err := currentDiff.finalize(); err != nil {
+					ctx.Logger().Error(err,
+						"failed to finalize diff",
+						"commit", currentCommit.Hash,
+						"diff", currentDiff.PathB,
+						"size", currentDiff.Len(),
+						"latest_state", latestState.String(),
+					)
+				}
+				currentCommit.Diffs = append(currentCommit.Diffs, *currentDiff)
 				// If the currentDiff is over 1GB, drop it into the channel so it isn't held in memory waiting for more commits.
 				totalSize := 0
 				for _, diff := range currentCommit.Diffs {
-					totalSize += diff.Content.Len()
+					totalSize += diff.Len()
 				}
 				if totalSize > c.maxCommitSize {
 					oldCommit := currentCommit
@ -348,7 +491,7 @@ func (c *Parser) FromReader(ctx context.Context, stdOut io.Reader, commitChan ch
 					currentCommit.Message.WriteString(oldCommit.Message.String())
 				}
 			}
-			currentDiff = Diff{}
+			currentDiff = NewDiff()
 		case isModeLine(isStaged, latestState, line):
 			latestState = ModeLine
 			// NoOp
@ -375,12 +518,20 @@ func (c *Parser) FromReader(ctx context.Context, stdOut io.Reader, commitChan ch
 		case isHunkLineNumberLine(isStaged, latestState, line):
 			latestState = HunkLineNumberLine

-			if currentDiff.Content.Len() > 0 || currentDiff.IsBinary {
-				currentCommit.Diffs = append(currentCommit.Diffs, currentDiff)
-			}
-			currentDiff = Diff{
-				PathB: currentDiff.PathB,
+			if currentDiff.Len() > 0 || currentDiff.IsBinary {
+				if err := currentDiff.finalize(); err != nil {
+					ctx.Logger().Error(
+						err,
+						"failed to finalize diff",
+						"commit", currentCommit.Hash,
+						"diff", currentDiff.PathB,
+						"size", currentDiff.Len(),
+						"latest_state", latestState.String(),
+					)
+				}
+				currentCommit.Diffs = append(currentCommit.Diffs, *currentDiff)
 			}
+			currentDiff = NewDiff(withPathB(currentDiff.PathB))

 			words := bytes.Split(line, []byte(" "))
 			if len(words) >= 3 {
@ -395,24 +546,21 @@ func (c *Parser) FromReader(ctx context.Context, stdOut io.Reader, commitChan ch
 				latestState = HunkContentLine
 			}
 			// TODO: Why do we care about this? It creates empty lines in the diff. If there are no plusLines, it's just newlines.
-			currentDiff.Content.Write([]byte("\n"))
+			if err := currentDiff.write(ctx, []byte("\n")); err != nil {
+				ctx.Logger().Error(err, "failed to write to diff")
+			}
 		case isHunkPlusLine(isStaged, latestState, line):
 			if latestState != HunkContentLine {
 				latestState = HunkContentLine
 			}

-			currentDiff.Content.Write(line[1:])
-		case isHunkMinusLine(isStaged, latestState, line):
-			if latestState != HunkContentLine {
-				latestState = HunkContentLine
+			if err := currentDiff.write(ctx, line[1:]); err != nil {
+				ctx.Logger().Error(err, "failed to write to diff")
 			}
 			// NoOp. We only care about additions.
-		case isHunkNewlineWarningLine(isStaged, latestState, line):
-			if latestState != HunkContentLine {
-				latestState = HunkContentLine
-			}
-			// NoOp
-		case isHunkEmptyLine(isStaged, latestState, line):
+		case isHunkMinusLine(isStaged, latestState, line),
+			isHunkNewlineWarningLine(isStaged, latestState, line),
+			isHunkEmptyLine(isStaged, latestState, line):
 			if latestState != HunkContentLine {
 				latestState = HunkContentLine
 			}
@ -439,14 +587,14 @@ func (c *Parser) FromReader(ctx context.Context, stdOut io.Reader, commitChan ch
 			latestState = ParseFailure
 		}

-		if currentDiff.Content.Len() > c.maxDiffSize {
+		if currentDiff.Len() > c.maxDiffSize {
 			ctx.Logger().V(2).Info(fmt.Sprintf(
 				"Diff for %s exceeded MaxDiffSize(%d)", currentDiff.PathB, c.maxDiffSize,
 			))
 			break
 		}
 	}
-	cleanupParse(currentCommit, &currentDiff, commitChan, &totalLogSize)
+	cleanupParse(ctx, currentCommit, currentDiff, commitChan, &totalLogSize)

 	ctx.Logger().V(2).Info("finished parsing git log.", "total_log_size", totalLogSize)
 }
@ -716,9 +864,13 @@ func isCommitSeparatorLine(isStaged bool, latestState ParseState, line []byte) b
 	return false
 }

-func cleanupParse(currentCommit *Commit, currentDiff *Diff, commitChan chan Commit, totalLogSize *int) {
+func cleanupParse(ctx context.Context, currentCommit *Commit, currentDiff *Diff, commitChan chan Commit, totalLogSize *int) {
+	if err := currentDiff.finalize(); err != nil {
+		ctx.Logger().Error(err, "failed to finalize diff")
+		return
+	}
 	// Ignore empty or binary diffs (this condition may be redundant).
-	if currentDiff != nil && (currentDiff.Content.Len() > 0 || currentDiff.IsBinary) {
+	if currentDiff != nil && (currentDiff.Len() > 0 || currentDiff.IsBinary) {
 		currentCommit.Diffs = append(currentCommit.Diffs, *currentDiff)
 	}
 	if currentCommit != nil {
--- a/pkg/gitparse/gitparse_test.go
+++ b/pkg/gitparse/gitparse_test.go
@ -6,6 +6,8 @@ import (
 	"testing"
 	"time"

+	"github.com/stretchr/testify/assert"
+
 	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
 )

@ -703,7 +705,7 @@ func TestCommitParsing(t *testing.T) {
 			break
 		}

-		if !commit.Equal(&expected[i]) {
+		if !commit.Equal(context.Background(), &expected[i]) {
 			t.Errorf("Commit does not match.\nexpected: %+v\n%s\nactual  : %+v\n%s", expected[i], expected[i].Message.String(), commit, commit.Message.String())
 		}
 		i++
@ -736,7 +738,7 @@ func TestIndividualCommitParsing(t *testing.T) {
 			}

 			// Assert
-			if !commit.Equal(&expected[i]) {
+			if !commit.Equal(context.Background(), &expected[i]) {
 				t.Errorf("Commit does not match.\nexpected: %+v\n%s\nactual  : %+v\n%s", expected[i], expected[j].Message.String(), commit, commit.Message.String())
 			}
 			j++
@ -744,6 +746,12 @@ func TestIndividualCommitParsing(t *testing.T) {
 	}
 }

+func newBufferWithContent(content []byte) *buffer {
+	var b buffer
+	_, _ = b.Write(context.Background(), content) // Using Write method to add content
+	return &b
+}
+
 func TestStagedDiffParsing(t *testing.T) {
 	expected := []Commit{
 		{
@ -753,44 +761,45 @@ func TestStagedDiffParsing(t *testing.T) {
 			Message: strings.Builder{},
 			Diffs: []Diff{
 				{
-					PathB:     "aws",
-					LineStart: 1,
-					Content:   *bytes.NewBuffer([]byte("[default]\naws_access_key_id = AKIAXYZDQCEN4B6JSJQI\naws_secret_access_key = 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\noutput = json\nregion = us-east-2\n")),
-					IsBinary:  false,
+					PathB:         "aws",
+					LineStart:     1,
+					contentWriter: newBufferWithContent([]byte("[default]\naws_access_key_id = AKIAXYZDQCEN4B6JSJQI\naws_secret_access_key = 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\noutput = json\nregion = us-east-2\n")),
+					IsBinary:      false,
 				},
 				{
-					PathB:     "aws2",
-					LineStart: 1,
-					Content:   *bytes.NewBuffer([]byte("\n\nthis is the secret: [Default]\nAccess key Id: AKIAILE3JG6KMS3HZGCA\nSecret Access Key: 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\n\nokay thank you bye\n")),
-					IsBinary:  false,
+					PathB:         "aws2",
+					LineStart:     1,
+					contentWriter: newBufferWithContent([]byte("\n\nthis is the secret: [Default]\nAccess key Id: AKIAILE3JG6KMS3HZGCA\nSecret Access Key: 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\n\nokay thank you bye\n")),
+					IsBinary:      false,
 				},
 				{
-					PathB:     "core/runtime/src/main/java/io/quarkus/runtime/QuarkusApplication.java",
-					LineStart: 3,
-					Content:   *bytes.NewBuffer([]byte("/**\n * This is usually used for command mode applications with a startup logic. The logic is executed inside\n * {@link QuarkusApplication#run} method before the main application exits.\n */\n")),
-					IsBinary:  false,
+					PathB:         "core/runtime/src/main/java/io/quarkus/runtime/QuarkusApplication.java",
+					LineStart:     3,
+					contentWriter: newBufferWithContent([]byte("/**\n * This is usually used for command mode applications with a startup logic. The logic is executed inside\n * {@link QuarkusApplication#run} method before the main application exits.\n */\n")),
+					IsBinary:      false,
 				},
 				{
-					PathB:    "trufflehog_3.42.0_linux_arm64.tar.gz",
-					IsBinary: true,
+					PathB:         "trufflehog_3.42.0_linux_arm64.tar.gz",
+					IsBinary:      true,
+					contentWriter: newBufferWithContent(nil),
 				},
 				{
-					PathB:     "tzu",
-					LineStart: 11,
-					Content:   *bytes.NewBuffer([]byte("\n\n\n\nSource: https://www.gnu.org/software/diffutils/manual/diffutils.html#An-Example-of-Unified-Format\n")),
-					IsBinary:  false,
+					PathB:         "tzu",
+					LineStart:     11,
+					contentWriter: newBufferWithContent([]byte("\n\n\n\nSource: https://www.gnu.org/software/diffutils/manual/diffutils.html#An-Example-of-Unified-Format\n")),
+					IsBinary:      false,
 				},
 				{
-					PathB:     "lao",
-					LineStart: 1,
-					Content:   *bytes.NewBuffer([]byte("The Way that can be told of is not the eternal Way;\nThe name that can be named is not the eternal name.\nThe Nameless is the origin of Heaven and Earth;\nThe Named is the mother of all things.\nTherefore let there always be non-being,\n  so we may see their subtlety,\nAnd let there always be being,\n  so we may see their outcome.\nThe two are the same,\nBut after they are produced,\n  they have different names.\n")),
-					IsBinary:  false,
+					PathB:         "lao",
+					LineStart:     1,
+					contentWriter: newBufferWithContent([]byte("The Way that can be told of is not the eternal Way;\nThe name that can be named is not the eternal name.\nThe Nameless is the origin of Heaven and Earth;\nThe Named is the mother of all things.\nTherefore let there always be non-being,\n  so we may see their subtlety,\nAnd let there always be being,\n  so we may see their outcome.\nThe two are the same,\nBut after they are produced,\n  they have different names.\n")),
+					IsBinary:      false,
 				},
 				{
-					PathB:     "tzu",
-					LineStart: 1,
-					Content:   *bytes.NewBuffer([]byte("The Nameless is the origin of Heaven and Earth;\nThe named is the mother of all things.\n\nTherefore let there always be non-being,\n  so we may see their subtlety,\nAnd let there always be being,\n  so we may see their outcome.\nThe two are the same,\nBut after they are produced,\n  they have different names.\nThey both may be called deep and profound.\nDeeper and more profound,\nThe door of all subtleties!\n")),
-					IsBinary:  false,
+					PathB:         "tzu",
+					LineStart:     1,
+					contentWriter: newBufferWithContent([]byte("The Nameless is the origin of Heaven and Earth;\nThe named is the mother of all things.\n\nTherefore let there always be non-being,\n  so we may see their subtlety,\nAnd let there always be being,\n  so we may see their outcome.\nThe two are the same,\nBut after they are produced,\n  they have different names.\nThey both may be called deep and profound.\nDeeper and more profound,\nThe door of all subtleties!\n")),
+					IsBinary:      false,
 				},
 			},
 		},
@ -809,7 +818,7 @@ func TestStagedDiffParsing(t *testing.T) {
 			break
 		}

-		if !commit.Equal(&expected[i]) {
+		if !commit.Equal(context.Background(), &expected[i]) {
 			t.Errorf("Commit does not match.\nexpected:\n%+v\n\nactual:\n%+v\n", expected[i], commit)
 		}
 		i++
@ -825,10 +834,10 @@ func TestCommitParseFailureRecovery(t *testing.T) {
 			Message: newStringBuilderValue("Add travis testing\n"),
 			Diffs: []Diff{
 				{
-					PathB:     ".travis.yml",
-					LineStart: 1,
-					Content:   *bytes.NewBuffer([]byte("language: python\npython:\n  - \"2.6\"\n  - \"2.7\"\n  - \"3.2\"\n  - \"3.3\"\n  - \"3.4\"\n  - \"3.5\"\n  - \"3.5-dev\" # 3.5 development branch\n  - \"3.6\"\n  - \"3.6-dev\" # 3.6 development branch\n  - \"3.7-dev\" # 3.7 development branch\n  - \"nightly\"\n")),
-					IsBinary:  false,
+					PathB:         ".travis.yml",
+					LineStart:     1,
+					contentWriter: newBufferWithContent([]byte("language: python\npython:\n  - \"2.6\"\n  - \"2.7\"\n  - \"3.2\"\n  - \"3.3\"\n  - \"3.4\"\n  - \"3.5\"\n  - \"3.5-dev\" # 3.5 development branch\n  - \"3.6\"\n  - \"3.6-dev\" # 3.6 development branch\n  - \"3.7-dev\" # 3.7 development branch\n  - \"nightly\"\n")),
+					IsBinary:      false,
 				},
 			},
 		},
@ -846,10 +855,10 @@ func TestCommitParseFailureRecovery(t *testing.T) {
 			Message: newStringBuilderValue("Change file\n"),
 			Diffs: []Diff{
 				{
-					PathB:     "tzu",
-					LineStart: 11,
-					Content:   *bytes.NewBuffer([]byte("\n\n\n\nSource: https://www.gnu.org/software/diffutils/manual/diffutils.html#An-Example-of-Unified-Format\n")),
-					IsBinary:  false,
+					PathB:         "tzu",
+					LineStart:     11,
+					contentWriter: newBufferWithContent([]byte("\n\n\n\nSource: https://www.gnu.org/software/diffutils/manual/diffutils.html#An-Example-of-Unified-Format\n")),
+					IsBinary:      false,
 				},
 			},
 		},
@ -868,7 +877,7 @@ func TestCommitParseFailureRecovery(t *testing.T) {
 			break
 		}

-		if !commit.Equal(&expected[i]) {
+		if !commit.Equal(context.Background(), &expected[i]) {
 			t.Errorf("Commit does not match.\nexpected: %+v\n\nactual  : %+v\n", expected[i], commit)
 		}
 		i++
@ -954,22 +963,22 @@ func TestDiffParseFailureRecovery(t *testing.T) {
 			Message: strings.Builder{},
 			Diffs: []Diff{
 				{
-					PathB:     "aws",
-					LineStart: 1,
-					Content:   *bytes.NewBuffer([]byte("[default]\naws_access_key_id = AKIAXYZDQCEN4B6JSJQI\naws_secret_access_key = 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\noutput = json\nregion = us-east-2\n")),
-					IsBinary:  false,
+					PathB:         "aws",
+					LineStart:     1,
+					contentWriter: newBufferWithContent([]byte("[default]\naws_access_key_id = AKIAXYZDQCEN4B6JSJQI\naws_secret_access_key = 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\noutput = json\nregion = us-east-2\n")),
+					IsBinary:      false,
 				},
 				{
-					PathB:     "tzu",
-					LineStart: 11,
-					Content:   *bytes.NewBuffer([]byte("\n\n\n\nSource: https://www.gnu.org/software/diffutils/manual/diffutils.html#An-Example-of-Unified-Format\n")),
-					IsBinary:  false,
+					PathB:         "tzu",
+					LineStart:     11,
+					contentWriter: newBufferWithContent([]byte("\n\n\n\nSource: https://www.gnu.org/software/diffutils/manual/diffutils.html#An-Example-of-Unified-Format\n")),
+					IsBinary:      false,
 				},
 				{
-					PathB:     "tzu",
-					LineStart: 1,
-					Content:   *bytes.NewBuffer([]byte("The Nameless is the origin of Heaven and Earth;\nThe named is the mother of all things.\n\nTherefore let there always be non-being,\n  so we may see their subtlety,\nAnd let there always be being,\n  so we may see their outcome.\nThe two are the same,\nBut after they are produced,\n  they have different names.\nThey both may be called deep and profound.\nDeeper and more profound,\nThe door of all subtleties!\n")),
-					IsBinary:  false,
+					PathB:         "tzu",
+					LineStart:     1,
+					contentWriter: newBufferWithContent([]byte("The Nameless is the origin of Heaven and Earth;\nThe named is the mother of all things.\n\nTherefore let there always be non-being,\n  so we may see their subtlety,\nAnd let there always be being,\n  so we may see their outcome.\nThe two are the same,\nBut after they are produced,\n  they have different names.\nThey both may be called deep and profound.\nDeeper and more profound,\nThe door of all subtleties!\n")),
+					IsBinary:      false,
 				},
 			},
 		},
@ -988,7 +997,7 @@ func TestDiffParseFailureRecovery(t *testing.T) {
 			break
 		}

-		if !commit.Equal(&expected[i]) {
+		if !commit.Equal(context.Background(), &expected[i]) {
 			t.Errorf("Commit does not match.\nexpected: %+v\n\nactual  : %+v\n", expected[i], commit)
 		}
 		i++
@ -1103,8 +1112,8 @@ func TestMaxDiffSize(t *testing.T) {

 	select {
 	case commit := <-commitChan:
-		if commit.Diffs[0].Content.Len() > parser.maxDiffSize+1024 {
-			t.Errorf("diff did not match MaxDiffSize. Got: %d, expected (max): %d", commit.Diffs[0].Content.Len(), parser.maxDiffSize+1024)
+		if commit.Diffs[0].Len() > parser.maxDiffSize+1024 {
+			t.Errorf("diff did not match MaxDiffSize. Got: %d, expected (max): %d", commit.Diffs[0].Len(), parser.maxDiffSize+1024)
 		}
 	case <-ctx.Done():
 		t.Fatal("Test timed out")
@ -1568,22 +1577,22 @@ func expectedCommits() []Commit {
 			Message: newStringBuilderValue("Added Unusable coloring\n"),
 			Diffs: []Diff{
 				{
-					PathB:     "components/item.lua",
-					LineStart: 9,
-					Content:   *bytes.NewBuffer([]byte("\n\nlocal Unfit = LibStub('Unfit-1.0')\n\n\n")),
-					IsBinary:  false,
+					PathB:         "components/item.lua",
+					LineStart:     9,
+					contentWriter: newBufferWithContent([]byte("\n\nlocal Unfit = LibStub('Unfit-1.0')\n\n\n")),
+					IsBinary:      false,
 				},
 				{
-					PathB:     "embeds.xml",
-					LineStart: 6,
-					Content:   *bytes.NewBuffer([]byte("\n\n       <Script file=\"libs\\Unfit-1.0\\Unfit-1.0.lua\"/>\n\n\n\n")),
-					IsBinary:  false,
+					PathB:         "embeds.xml",
+					LineStart:     6,
+					contentWriter: newBufferWithContent([]byte("\n\n       <Script file=\"libs\\Unfit-1.0\\Unfit-1.0.lua\"/>\n\n\n\n")),
+					IsBinary:      false,
 				},
 				{
-					PathB:     "libs/Unfit-1.0",
-					LineStart: 1,
-					Content:   *bytes.NewBuffer([]byte("Subproject commit 0000000000000000000000000000000000000000\n")),
-					IsBinary:  false,
+					PathB:         "libs/Unfit-1.0",
+					LineStart:     1,
+					contentWriter: newBufferWithContent([]byte("Subproject commit 0000000000000000000000000000000000000000\n")),
+					IsBinary:      false,
 				},
 			},
 		},
@ -1610,10 +1619,10 @@ func expectedCommits() []Commit {
 			Message: strings.Builder{},
 			Diffs: []Diff{
 				{
-					PathB:     "sample.txt",
-					LineStart: 1,
-					Content:   *bytes.NewBuffer([]byte("Hello, world!\n")),
-					IsBinary:  false,
+					PathB:         "sample.txt",
+					LineStart:     1,
+					contentWriter: newBufferWithContent([]byte("Hello, world!\n")),
+					IsBinary:      false,
 				},
 			},
 		},
@ -1624,15 +1633,15 @@ func expectedCommits() []Commit {
 			Message: newStringBuilderValue("Add travis testing\n"),
 			Diffs: []Diff{
 				{
-					PathB:     ".gitignore",
-					LineStart: 1,
-					Content:   *bytes.NewBuffer([]byte("\n\n\n**/__pycache__/\n**/*.pyc\n")),
-					IsBinary:  false,
+					PathB:         ".gitignore",
+					LineStart:     1,
+					contentWriter: newBufferWithContent([]byte("\n\n\n**/__pycache__/\n**/*.pyc\n")),
+					IsBinary:      false,
 				},
 				{
 					PathB:     ".travis.yml",
 					LineStart: 1,
-					Content: *bytes.NewBuffer([]byte(`language: python
+					contentWriter: newBufferWithContent([]byte(`language: python
 python:
  - "2.6"
  - "2.7"
@ -1659,7 +1668,7 @@ python:
 				{
 					PathB:     "Makefile",
 					LineStart: 1,
-					Content: *bytes.NewBuffer([]byte(`PROTOS_IMAGE=us-docker.pkg.dev/thog-artifacts/public/go-ci-1.17-1
+					contentWriter: newBufferWithContent([]byte(`PROTOS_IMAGE=us-docker.pkg.dev/thog-artifacts/public/go-ci-1.17-1

 .PHONY: check
 .PHONY: test
@ -1703,10 +1712,10 @@ protos:
 			Message: newStringBuilderValue("Test toFile/plusLine parsing\n"),
 			Diffs: []Diff{
 				{
-					PathB:     "plusLine.txt",
-					LineStart: 1,
-					Content:   *bytes.NewBuffer([]byte("-- test\n++ test\n\n")),
-					IsBinary:  false,
+					PathB:         "plusLine.txt",
+					LineStart:     1,
+					contentWriter: newBufferWithContent([]byte("-- test\n++ test\n\n")),
+					IsBinary:      false,
 				},
 			},
 		},
@ -1717,41 +1726,41 @@ protos:
 			Message: newStringBuilderValue("Do not refresh OIDC session if the user is requesting logout\n"),
 			Diffs: []Diff{
 				{
-					PathB:     "extensions/oidc/runtime/src/main/java/io/quarkus/oidc/runtime/BackChannelLogoutTokenCache.java",
-					LineStart: 45,
-					Content:   *bytes.NewBuffer([]byte("\n\n    public boolean containsTokenVerification(String token) {\n        return cacheMap.containsKey(token);\n    }\n\n\n\n\n")),
-					IsBinary:  false,
+					PathB:         "extensions/oidc/runtime/src/main/java/io/quarkus/oidc/runtime/BackChannelLogoutTokenCache.java",
+					LineStart:     45,
+					contentWriter: newBufferWithContent([]byte("\n\n    public boolean containsTokenVerification(String token) {\n        return cacheMap.containsKey(token);\n    }\n\n\n\n\n")),
+					IsBinary:      false,
 				},
 				{
-					PathB:     "extensions/oidc/runtime/src/main/java/io/quarkus/oidc/runtime/CodeAuthenticationMechanism.java",
-					LineStart: 1023,
-					Content:   *bytes.NewBuffer([]byte("\n\n    private boolean isRpInitiatedLogout(RoutingContext context, TenantConfigContext configContext) {\n\n\n")),
-					IsBinary:  false,
+					PathB:         "extensions/oidc/runtime/src/main/java/io/quarkus/oidc/runtime/CodeAuthenticationMechanism.java",
+					LineStart:     1023,
+					contentWriter: newBufferWithContent([]byte("\n\n    private boolean isRpInitiatedLogout(RoutingContext context, TenantConfigContext configContext) {\n\n\n")),
+					IsBinary:      false,
 				},
 				{
-					PathB:     "extensions/oidc/runtime/src/main/java/io/quarkus/oidc/runtime/CodeAuthenticationMechanism.java",
-					LineStart: 1214,
-					Content:   *bytes.NewBuffer([]byte("\n\n\n\n    private class LogoutCall implements Function<SecurityIdentity, Uni<?>> {\n        RoutingContext context;\n        TenantConfigContext configContext;\n        String idToken;\n\n        LogoutCall(RoutingContext context, TenantConfigContext configContext, String idToken) {\n            this.context = context;\n            this.configContext = configContext;\n            this.idToken = idToken;\n        }\n\n        @Override\n        public Uni<Void> apply(SecurityIdentity identity) {\n            if (isRpInitiatedLogout(context, configContext)) {\n                LOG.debug(\"Performing an RP initiated logout\");\n                fireEvent(SecurityEvent.Type.OIDC_LOGOUT_RP_INITIATED, identity);\n                return buildLogoutRedirectUriUni(context, configContext, idToken);\n            }\n            if (isBackChannelLogoutPendingAndValid(configContext, identity)\n                    || isFrontChannelLogoutValid(context, configContext,\n                            identity)) {\n                return removeSessionCookie(context, configContext.oidcConfig)\n                        .map(new Function<Void, Void>() {\n                            @Override\n                            public Void apply(Void t) {\n                                throw new LogoutException();\n                            }\n                        });\n\n            }\n            return VOID_UNI;\n        }\n    }\n\n")),
-					IsBinary:  false,
+					PathB:         "extensions/oidc/runtime/src/main/java/io/quarkus/oidc/runtime/CodeAuthenticationMechanism.java",
+					LineStart:     1214,
+					contentWriter: newBufferWithContent([]byte("\n\n\n\n    private class LogoutCall implements Function<SecurityIdentity, Uni<?>> {\n        RoutingContext context;\n        TenantConfigContext configContext;\n        String idToken;\n\n        LogoutCall(RoutingContext context, TenantConfigContext configContext, String idToken) {\n            this.context = context;\n            this.configContext = configContext;\n            this.idToken = idToken;\n        }\n\n        @Override\n        public Uni<Void> apply(SecurityIdentity identity) {\n            if (isRpInitiatedLogout(context, configContext)) {\n                LOG.debug(\"Performing an RP initiated logout\");\n                fireEvent(SecurityEvent.Type.OIDC_LOGOUT_RP_INITIATED, identity);\n                return buildLogoutRedirectUriUni(context, configContext, idToken);\n            }\n            if (isBackChannelLogoutPendingAndValid(configContext, identity)\n                    || isFrontChannelLogoutValid(context, configContext,\n                            identity)) {\n                return removeSessionCookie(context, configContext.oidcConfig)\n                        .map(new Function<Void, Void>() {\n                            @Override\n                            public Void apply(Void t) {\n                                throw new LogoutException();\n                            }\n                        });\n\n            }\n            return VOID_UNI;\n        }\n    }\n\n")),
+					IsBinary:      false,
 				},
 				{
-					PathB:     "integration-tests/oidc-wiremock/src/main/resources/application.properties",
-					LineStart: 20,
-					Content:   *bytes.NewBuffer([]byte("\n\n\nquarkus.oidc.code-flow.token.refresh-expired=true\nquarkus.oidc.code-flow.token.refresh-token-time-skew=5M\n\n\n")),
-					IsBinary:  false,
+					PathB:         "integration-tests/oidc-wiremock/src/main/resources/application.properties",
+					LineStart:     20,
+					contentWriter: newBufferWithContent([]byte("\n\n\nquarkus.oidc.code-flow.token.refresh-expired=true\nquarkus.oidc.code-flow.token.refresh-token-time-skew=5M\n\n\n")),
+					IsBinary:      false,
 				},
 				// WTF, shouldn't this be filtered out?
 				{
-					PathB:     "integration-tests/oidc-wiremock/src/test/java/io/quarkus/it/keycloak/CodeFlowAuthorizationTest.java",
-					LineStart: 6,
-					Content:   *bytes.NewBuffer([]byte("\n\n\n\n\n\n")),
-					IsBinary:  false,
+					PathB:         "integration-tests/oidc-wiremock/src/test/java/io/quarkus/it/keycloak/CodeFlowAuthorizationTest.java",
+					LineStart:     6,
+					contentWriter: newBufferWithContent([]byte("\n\n\n\n\n\n")),
+					IsBinary:      false,
 				},
 				{
-					PathB:     "integration-tests/oidc-wiremock/src/test/java/io/quarkus/it/keycloak/CodeFlowAuthorizationTest.java",
-					LineStart: 76,
-					Content:   *bytes.NewBuffer([]byte("\n\n            // Logout\n\n\n\n")),
-					IsBinary:  false,
+					PathB:         "integration-tests/oidc-wiremock/src/test/java/io/quarkus/it/keycloak/CodeFlowAuthorizationTest.java",
+					LineStart:     76,
+					contentWriter: newBufferWithContent([]byte("\n\n            // Logout\n\n\n\n")),
+					IsBinary:      false,
 				},
 			},
 		},
@ -1762,10 +1771,10 @@ protos:
 			Message: newStringBuilderValue("Add QuarkusApplication javadoc\n\n* Fix #34463\n"),
 			Diffs: []Diff{
 				{
-					PathB:     "core/runtime/src/main/java/io/quarkus/runtime/QuarkusApplication.java",
-					LineStart: 3,
-					Content:   *bytes.NewBuffer([]byte("/**\n * This is usually used for command mode applications with a startup logic. The logic is executed inside\n * {@link QuarkusApplication#run} method before the main application exits.\n */\n")),
-					IsBinary:  false,
+					PathB:         "core/runtime/src/main/java/io/quarkus/runtime/QuarkusApplication.java",
+					LineStart:     3,
+					contentWriter: newBufferWithContent([]byte("/**\n * This is usually used for command mode applications with a startup logic. The logic is executed inside\n * {@link QuarkusApplication#run} method before the main application exits.\n */\n")),
+					IsBinary:      false,
 				},
 			},
 		},
@ -1783,8 +1792,9 @@ protos:
 			Message: newStringBuilderValue("Change binary file\n"),
 			Diffs: []Diff{
 				{
-					PathB:    "trufflehog_3.42.0_linux_arm64.tar.gz",
-					IsBinary: true,
+					PathB:         "trufflehog_3.42.0_linux_arm64.tar.gz",
+					contentWriter: newBufferWithContent([]byte("")),
+					IsBinary:      true,
 				},
 			},
 		},
@ -1795,8 +1805,9 @@ protos:
 			Message: newStringBuilderValue("Add binary file\n"),
 			Diffs: []Diff{
 				{
-					PathB:    "trufflehog_3.42.0_linux_arm64.tar.gz",
-					IsBinary: true,
+					PathB:         "trufflehog_3.42.0_linux_arm64.tar.gz",
+					contentWriter: newBufferWithContent([]byte("")),
+					IsBinary:      true,
 				},
 			},
 		},
@ -1821,10 +1832,10 @@ protos:
 			Message: newStringBuilderValue("Change file\n"),
 			Diffs: []Diff{
 				{
-					PathB:     "tzu",
-					LineStart: 11,
-					Content:   *bytes.NewBuffer([]byte("\n\n\n\nSource: https://www.gnu.org/software/diffutils/manual/diffutils.html#An-Example-of-Unified-Format\n")),
-					IsBinary:  false,
+					PathB:         "tzu",
+					LineStart:     11,
+					contentWriter: newBufferWithContent([]byte("\n\n\n\nSource: https://www.gnu.org/software/diffutils/manual/diffutils.html#An-Example-of-Unified-Format\n")),
+					IsBinary:      false,
 				},
 			},
 		},
@ -1835,16 +1846,16 @@ protos:
 			Message: newStringBuilderValue("Create files\n"),
 			Diffs: []Diff{
 				{
-					PathB:     "lao",
-					LineStart: 1,
-					Content:   *bytes.NewBuffer([]byte("The Way that can be told of is not the eternal Way;\nThe name that can be named is not the eternal name.\nThe Nameless is the origin of Heaven and Earth;\nThe Named is the mother of all things.\nTherefore let there always be non-being,\n  so we may see their subtlety,\nAnd let there always be being,\n  so we may see their outcome.\nThe two are the same,\nBut after they are produced,\n  they have different names.\n")),
-					IsBinary:  false,
+					PathB:         "lao",
+					LineStart:     1,
+					contentWriter: newBufferWithContent([]byte("The Way that can be told of is not the eternal Way;\nThe name that can be named is not the eternal name.\nThe Nameless is the origin of Heaven and Earth;\nThe Named is the mother of all things.\nTherefore let there always be non-being,\n  so we may see their subtlety,\nAnd let there always be being,\n  so we may see their outcome.\nThe two are the same,\nBut after they are produced,\n  they have different names.\n")),
+					IsBinary:      false,
 				},
 				{
-					PathB:     "tzu",
-					LineStart: 1,
-					Content:   *bytes.NewBuffer([]byte("The Nameless is the origin of Heaven and Earth;\nThe named is the mother of all things.\n\nTherefore let there always be non-being,\n  so we may see their subtlety,\nAnd let there always be being,\n  so we may see their outcome.\nThe two are the same,\nBut after they are produced,\n  they have different names.\nThey both may be called deep and profound.\nDeeper and more profound,\nThe door of all subtleties!\n")),
-					IsBinary:  false,
+					PathB:         "tzu",
+					LineStart:     1,
+					contentWriter: newBufferWithContent([]byte("The Nameless is the origin of Heaven and Earth;\nThe named is the mother of all things.\n\nTherefore let there always be non-being,\n  so we may see their subtlety,\nAnd let there always be being,\n  so we may see their outcome.\nThe two are the same,\nBut after they are produced,\n  they have different names.\nThey both may be called deep and profound.\nDeeper and more profound,\nThe door of all subtleties!\n")),
+					IsBinary:      false,
 				},
 			},
 		},
@ -2041,3 +2052,32 @@ index 2ee133b..12b4843 100644
 +output = json
 +region = us-east-2
 `
+
+func TestBufferWriterStateTransitionOnClose(t *testing.T) {
+	t.Parallel()
+	writer := new(buffer)
+
+	// Initially, the writer should be in write-only mode.
+	assert.Equal(t, writeOnly, writer.state)
+
+	// Perform some write operation.
+	_, err := writer.Write(context.Background(), []byte("test data"))
+	assert.NoError(t, err)
+
+	// Close the writer.
+	err = writer.CloseForWriting()
+	assert.NoError(t, err)
+
+	// After closing, the writer should be in read-only mode.
+	assert.Equal(t, readOnly, writer.state)
+}
+
+func TestBufferWriterWriteInReadOnlyState(t *testing.T) {
+	t.Parallel()
+	writer := new(buffer)
+	_ = writer.CloseForWriting() // Transition to read-only mode
+
+	// Attempt to write in read-only mode.
+	_, err := writer.Write(context.Background(), []byte("should fail"))
+	assert.Error(t, err)
+}
--- a/pkg/sources/git/git.go
+++ b/pkg/sources/git/git.go
@ -19,13 +19,12 @@ import (
 	"github.com/go-git/go-git/v5/plumbing"
 	"github.com/go-git/go-git/v5/plumbing/object"
 	"github.com/google/go-github/v42/github"
+	diskbufferreader "github.com/trufflesecurity/disk-buffer-reader"
 	"golang.org/x/oauth2"
 	"golang.org/x/sync/semaphore"
 	"google.golang.org/protobuf/proto"
 	"google.golang.org/protobuf/types/known/anypb"

-	diskbufferreader "github.com/trufflesecurity/disk-buffer-reader"
-
 	"github.com/trufflesecurity/trufflehog/v3/pkg/cleantemp"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
@ -508,6 +507,7 @@ func (s *Git) ScanCommits(ctx context.Context, repo *git.Repository, path string
 		atomic.AddUint64(&s.metrics.commitsScanned, 1)
 		logger.V(5).Info("scanning commit", "commit", commit.Hash)
 		for _, diff := range commit.Diffs {
+			diff := diff
 			if !scanOptions.Filter.Pass(diff.PathB) {
 				continue
 			}
@ -539,21 +539,38 @@ func (s *Git) ScanCommits(ctx context.Context, repo *git.Repository, path string
 				continue
 			}

-			if diff.Content.Len() > sources.ChunkSize+sources.PeekSize {
-				s.gitChunk(ctx, diff, fileName, email, hash, when, remoteURL, reporter)
+			if diff.Len() > sources.ChunkSize+sources.PeekSize {
+				s.gitChunk(ctx, &diff, fileName, email, hash, when, remoteURL, reporter)
 				continue
 			}
-			metadata := s.sourceMetadataFunc(fileName, email, hash, when, remoteURL, int64(diff.LineStart))
-			chunk := sources.Chunk{
-				SourceName:     s.sourceName,
-				SourceID:       s.sourceID,
-				JobID:          s.jobID,
-				SourceType:     s.sourceType,
-				SourceMetadata: metadata,
-				Data:           diff.Content.Bytes(),
-				Verify:         s.verify,
+
+			chunkData := func(d *gitparse.Diff) error {
+				metadata := s.sourceMetadataFunc(fileName, email, hash, when, remoteURL, int64(diff.LineStart))
+
+				reader, err := d.ReadCloser()
+				if err != nil {
+					ctx.Logger().Error(err, "error creating reader for commits", "filename", fileName, "commit", hash, "file", diff.PathB)
+					return nil
+				}
+				defer reader.Close()
+
+				data := make([]byte, diff.Len())
+				if _, err := reader.Read(data); err != nil {
+					ctx.Logger().Error(err, "error reading diff content for commit", "filename", fileName, "commit", hash, "file", diff.PathB)
+					return nil
+				}
+				chunk := sources.Chunk{
+					SourceName:     s.sourceName,
+					SourceID:       s.sourceID,
+					JobID:          s.jobID,
+					SourceType:     s.sourceType,
+					SourceMetadata: metadata,
+					Data:           data,
+					Verify:         s.verify,
+				}
+				return reporter.ChunkOk(ctx, chunk)
 			}
-			if err := reporter.ChunkOk(ctx, chunk); err != nil {
+			if err := chunkData(&diff); err != nil {
 				return err
 			}
 		}
@ -561,8 +578,15 @@ func (s *Git) ScanCommits(ctx context.Context, repo *git.Repository, path string
 	return nil
 }

-func (s *Git) gitChunk(ctx context.Context, diff gitparse.Diff, fileName, email, hash, when, urlMetadata string, reporter sources.ChunkReporter) {
-	originalChunk := bufio.NewScanner(&diff.Content)
+func (s *Git) gitChunk(ctx context.Context, diff *gitparse.Diff, fileName, email, hash, when, urlMetadata string, reporter sources.ChunkReporter) {
+	reader, err := diff.ReadCloser()
+	if err != nil {
+		ctx.Logger().Error(err, "error creating reader for chunk", "filename", fileName, "commit", hash, "file", diff.PathB)
+		return
+	}
+	defer reader.Close()
+
+	originalChunk := bufio.NewScanner(reader)
 	newChunkBuffer := bytes.Buffer{}
 	lastOffset := 0
 	for offset := 0; originalChunk.Scan(); offset++ {
@ -654,6 +678,7 @@ func (s *Git) ScanStaged(ctx context.Context, repo *git.Repository, path string,
 	ctx.Logger().V(1).Info("scanning staged changes", "path", path)
 	for commit := range commitChan {
 		for _, diff := range commit.Diffs {
+			diff := diff
 			logger := ctx.Logger().WithValues("filename", diff.PathB, "commit", commit.Hash, "file", diff.PathB)
 			logger.V(2).Info("scanning staged changes from git")

@ -703,17 +728,33 @@ func (s *Git) ScanStaged(ctx context.Context, repo *git.Repository, path string,
 				continue
 			}

-			metadata := s.sourceMetadataFunc(fileName, email, "Staged", when, urlMetadata, int64(diff.LineStart))
-			chunk := sources.Chunk{
-				SourceName:     s.sourceName,
-				SourceID:       s.sourceID,
-				JobID:          s.jobID,
-				SourceType:     s.sourceType,
-				SourceMetadata: metadata,
-				Data:           diff.Content.Bytes(),
-				Verify:         s.verify,
+			chunkData := func(d *gitparse.Diff) error {
+				metadata := s.sourceMetadataFunc(fileName, email, "Staged", when, urlMetadata, int64(diff.LineStart))
+
+				reader, err := diff.ReadCloser()
+				if err != nil {
+					ctx.Logger().Error(err, "error creating reader for staged", "filename", fileName, "commit", hash, "file", diff.PathB)
+					return nil
+				}
+				defer reader.Close()
+
+				data := make([]byte, diff.Len())
+				if _, err := reader.Read(data); err != nil {
+					ctx.Logger().Error(err, "error reading diff content for staged", "filename", fileName, "commit", hash, "file", diff.PathB)
+					return nil
+				}
+				chunk := sources.Chunk{
+					SourceName:     s.sourceName,
+					SourceID:       s.sourceID,
+					JobID:          s.jobID,
+					SourceType:     s.sourceType,
+					SourceMetadata: metadata,
+					Data:           data,
+					Verify:         s.verify,
+				}
+				return reporter.ChunkOk(ctx, chunk)
 			}
-			if err := reporter.ChunkOk(ctx, chunk); err != nil {
+			if err := chunkData(&diff); err != nil {
 				return err
 			}
 		}
@ -1048,9 +1089,6 @@ func (s *Git) handleBinary(ctx context.Context, gitDir string, reporter sources.
 		return err
 	}
 	defer func() {
-		if err := fileReader.Close(); err != nil {
-			ctx.Logger().Error(err, "error closing fileReader")
-		}
 		if err := cmd.Wait(); err != nil {
 			ctx.Logger().Error(
 				err, "error waiting for command",