feat: support docker image history scanning (#2882)

* feat: support docker image history scanning

* refactor: collapse error handling into return

Style suggestion from review feedback.

* fix: associate layers with history entries

Where possible, add the associated layer to the history entry record. This may help tracing any issues discovered.

This also changes the entry reference format to `image-metadata:history:%d:created-by` which _may_ be more self-explanatory.

pkg/sources/docker/docker.go

@@ -80,6 +80,14 @@ type imageInfo struct {
 	tag   string
 }
 
+type historyEntryInfo struct {
+	index       int
+	entry       v1.History
+	layerDigest string
+	base        string
+	tag         string
+}
+
 type layerInfo struct {
 	digest v1.Hash
 	base   string
@@ -108,7 +116,24 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ .
 			}
 
 			ctx = context.WithValues(ctx, "image", imgInfo.base, "tag", imgInfo.tag)
-			ctx.Logger().V(2).Info("scanning image")
+
+			ctx.Logger().V(2).Info("scanning image history")
+
+			historyEntries, err := getHistoryEntries(ctx, imgInfo)
+			if err != nil {
+				scanErrs.Add(err)
+				return nil
+			}
+
+			for _, historyEntry := range historyEntries {
+				if err := s.processHistoryEntry(ctx, historyEntry, chunksChan); err != nil {
+					scanErrs.Add(err)
+					return nil
+				}
+				dockerHistoryEntriesScanned.WithLabelValues(s.name).Inc()
+			}
+
+			ctx.Logger().V(2).Info("scanning image layers")
 
 			layers, err := imgInfo.image.Layers()
 			if err != nil {
@@ -181,6 +206,86 @@ func (s *Source) processImage(ctx context.Context, image string) (imageInfo, err
 	return imgInfo, nil
 }
 
+// getHistoryEntries collates an image's configuration history together with the
+// corresponding layer digests for any non-empty layers.
+func getHistoryEntries(ctx context.Context, imgInfo imageInfo) ([]historyEntryInfo, error) {
+	config, err := imgInfo.image.ConfigFile()
+	if err != nil {
+		return nil, err
+	}
+
+	layers, err := imgInfo.image.Layers()
+	if err != nil {
+		return nil, err
+	}
+
+	history := config.History
+	entries := make([]historyEntryInfo, len(history))
+
+	layerIndex := 0
+	for historyIndex, entry := range history {
+		e := historyEntryInfo{
+			base:  imgInfo.base,
+			tag:   imgInfo.tag,
+			entry: entry,
+			index: historyIndex,
+		}
+
+		// Associate with a layer if possible -- failing to do this will not affect
+		// the scan, just remove some traceability.
+		if !entry.EmptyLayer {
+			if layerIndex < len(layers) {
+				digest, err := layers[layerIndex].Digest()
+
+				if err == nil {
+					e.layerDigest = digest.String()
+				} else {
+					ctx.Logger().V(2).Error(err, "cannot associate layer with history entry: layer digest failed",
+						"layerIndex", layerIndex, "historyIndex", historyIndex)
+				}
+			} else {
+				ctx.Logger().V(2).Info("cannot associate layer with history entry: no correlated layer exists at this index",
+					"layerIndex", layerIndex, "historyIndex", historyIndex)
+			}
+
+			layerIndex++
+		}
+
+		entries[historyIndex] = e
+	}
+
+	return entries, nil
+}
+
+// processHistoryEntry processes a history entry from the image configuration metadata.
+func (s *Source) processHistoryEntry(ctx context.Context, historyInfo historyEntryInfo, chunksChan chan *sources.Chunk) error {
+	// Make up an identifier for this entry that is moderately sensible. There is
+	// no file name to use here, so the path tries to be a little descriptive.
+	entryPath := fmt.Sprintf("image-metadata:history:%d:created-by", historyInfo.index)
+
+	chunk := &sources.Chunk{
+		SourceType: s.Type(),
+		SourceName: s.name,
+		SourceID:   s.SourceID(),
+		SourceMetadata: &source_metadatapb.MetaData{
+			Data: &source_metadatapb.MetaData_Docker{
+				Docker: &source_metadatapb.Docker{
+					File:  entryPath,
+					Image: historyInfo.base,
+					Tag:   historyInfo.tag,
+					Layer: historyInfo.layerDigest,
+				},
+			},
+		},
+		Verify: s.verify,
+		Data:   []byte(historyInfo.entry.CreatedBy),
+	}
+
+	ctx.Logger().V(2).Info("scanning image history entry", "index", historyInfo.index, "layer", historyInfo.layerDigest)
+
+	return common.CancellableWrite(ctx, chunksChan, chunk)
+}
+
 // processLayer processes an individual layer of an image.
 func (s *Source) processLayer(ctx context.Context, layer v1.Layer, imgInfo imageInfo, chunksChan chan *sources.Chunk) error {
 	layerInfo := layerInfo{

pkg/sources/docker/docker_test.go

@@ -1,6 +1,7 @@
 package docker
 
 import (
+	"strings"
 	"sync"
 	"testing"
 
@@ -9,6 +10,7 @@ import (
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/credentialspb"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
 )
@@ -32,12 +34,21 @@ func TestDockerImageScan(t *testing.T) {
 	var wg sync.WaitGroup
 	chunksChan := make(chan *sources.Chunk, 1)
 	chunkCounter := 0
+	layerCounter := 0
+	historyCounter := 0
+
 	wg.Add(1)
 	go func() {
 		defer wg.Done()
 		for chunk := range chunksChan {
 			assert.NotEmpty(t, chunk)
 			chunkCounter++
+
+			if isHistoryChunk(t, chunk) {
+				historyCounter++
+			} else {
+				layerCounter++
+			}
 		}
 	}()
 
@@ -47,7 +58,9 @@ func TestDockerImageScan(t *testing.T) {
 	close(chunksChan)
 	wg.Wait()
 
-	assert.Equal(t, 1, chunkCounter)
+	assert.Equal(t, 2, chunkCounter)
+	assert.Equal(t, 1, layerCounter)
+	assert.Equal(t, 1, historyCounter)
 }
 
 func TestDockerImageScanWithDigest(t *testing.T) {
@@ -69,12 +82,27 @@ func TestDockerImageScanWithDigest(t *testing.T) {
 	var wg sync.WaitGroup
 	chunksChan := make(chan *sources.Chunk, 1)
 	chunkCounter := 0
+	layerCounter := 0
+	historyCounter := 0
+
+	var historyChunk *source_metadatapb.Docker
+	var layerChunk *source_metadatapb.Docker
+
 	wg.Add(1)
 	go func() {
 		defer wg.Done()
 		for chunk := range chunksChan {
 			assert.NotEmpty(t, chunk)
 			chunkCounter++
+
+			if isHistoryChunk(t, chunk) {
+				// save last for later comparison
+				historyChunk = chunk.SourceMetadata.GetDocker()
+				historyCounter++
+			} else {
+				layerChunk = chunk.SourceMetadata.GetDocker()
+				layerCounter++
+			}
 		}
 	}()
 
@@ -84,7 +112,26 @@ func TestDockerImageScanWithDigest(t *testing.T) {
 	close(chunksChan)
 	wg.Wait()
 
-	assert.Equal(t, 1, chunkCounter)
+	// Since this test pins the layer by digest, layers will have consistent
+	// hashes. This allows layer digest comparison as they will be stable for
+	// given image digest.
+	assert.Equal(t, &source_metadatapb.Docker{
+		Image: "trufflesecurity/secrets",
+		Tag:   "sha256:864f6d41209462d8e37fc302ba1532656e265f7c361f11e29fed6ca1f4208e11",
+		File:  "image-metadata:history:0:created-by",
+		Layer: "sha256:a794864de8c4ff087813fd66cff74601b84cbef8fe1a1f17f9923b40cf051b59",
+	}, historyChunk)
+
+	assert.Equal(t, &source_metadatapb.Docker{
+		Image: "trufflesecurity/secrets",
+		Tag:   "sha256:864f6d41209462d8e37fc302ba1532656e265f7c361f11e29fed6ca1f4208e11",
+		File:  "/aws",
+		Layer: "sha256:a794864de8c4ff087813fd66cff74601b84cbef8fe1a1f17f9923b40cf051b59",
+	}, layerChunk)
+
+	assert.Equal(t, 2, chunkCounter)
+	assert.Equal(t, 1, layerCounter)
+	assert.Equal(t, 1, historyCounter)
 }
 
 func TestBaseAndTagFromImage(t *testing.T) {
@@ -110,3 +157,12 @@ func TestBaseAndTagFromImage(t *testing.T) {
 		}
 	}
 }
+
+func isHistoryChunk(t *testing.T, chunk *sources.Chunk) bool {
+	t.Helper()
+
+	metadata := chunk.SourceMetadata.GetDocker()
+
+	return metadata != nil &&
+		strings.HasPrefix(metadata.File, "image-metadata:history:")
+}

pkg/sources/docker/metrics.go

@@ -16,6 +16,14 @@ var (
 	},
 		[]string{"source_name"})
 
+	dockerHistoryEntriesScanned = promauto.NewGaugeVec(prometheus.GaugeOpts{
+		Namespace: common.MetricsNamespace,
+		Subsystem: common.MetricsSubsystem,
+		Name:      "docker_history_entries_scanned",
+		Help:      "Total number of Docker image history entries scanned.",
+	},
+		[]string{"source_name"})
+
 	dockerImagesScanned = promauto.NewGaugeVec(prometheus.GaugeOpts{
 		Namespace: common.MetricsNamespace,
 		Subsystem: common.MetricsSubsystem,