trufflehog/pkg/sources/gcs/gcs_integration_test.go

//go:build integration
// +build integration

package gcs

import (
	"sort"
	"testing"

	"github.com/google/go-cmp/cmp"
	"github.com/google/go-cmp/cmp/cmpopts"
	"github.com/stretchr/testify/assert"

	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
)

func TestChunks(t *testing.T) {
	ctx := context.Background()

	source, conn := createTestSource(&sourcespb.GCS{
		ProjectId:      testProjectID,
		Credential:     &sourcespb.GCS_Adc{},
		ExcludeBuckets: []string{perfTestBucketGlob, publicBucket},
	})

	err := source.Init(ctx, "test", 1, 1, true, conn, 8)
	assert.Nil(t, err)

	chunksCh := make(chan *sources.Chunk, 1)

	go func() {
		defer close(chunksCh)
		err := source.Chunks(ctx, chunksCh)
		assert.Nil(t, err)
	}()

	want := createTestChunks()

	got := make([]*sources.Chunk, 0, len(want))
	for chunk := range chunksCh {
		got = append(got, chunk)
	}
	sort.Slice(got, func(i, j int) bool {
		return got[i].SourceMetadata.GetGcs().Filename < got[j].SourceMetadata.GetGcs().Filename
	})

	assert.Equal(t, len(want), len(got))

	for i, chunk := range got {
		if diff := cmp.Diff(want[i].SourceMetadata.GetGcs(), chunk.SourceMetadata.GetGcs(),
			cmpopts.IgnoreFields(source_metadatapb.GCS{}, "state", "sizeCache", "unknownFields", "CreatedAt", "UpdatedAt"),
		); diff != "" {
			t.Errorf("chunk mismatch (-want +got):\n%s", diff)
		}
	}
}

func TestChunks_PublicBucket(t *testing.T) {
	ctx := context.Background()

	source, conn := createTestSource(&sourcespb.GCS{
		Credential:     &sourcespb.GCS_Unauthenticated{},
		IncludeBuckets: []string{publicBucket},
	})

	err := source.Init(ctx, "test", 1, 1, true, conn, 8)
	assert.Nil(t, err)

	chunksCh := make(chan *sources.Chunk, 1)

	go func() {
		defer close(chunksCh)
		err := source.Chunks(ctx, chunksCh)
		assert.Nil(t, err)
	}()

	want := []*sources.Chunk{
		{
			SourceName: "test",
			SourceType: sourcespb.SourceType_SOURCE_TYPE_GCS,
			SourceID:   0,
			Verify:     true,
			SourceMetadata: &source_metadatapb.MetaData{
				Data: &source_metadatapb.MetaData_Gcs{
					Gcs: &source_metadatapb.GCS{
						Filename:    "aws1.txt",
						Bucket:      publicBucket,
						ContentType: "text/plain",
						Email:       "",
						Link:        "https://storage.googleapis.com/download/storage/v1/b/public-trufflehog-test-bucket/o/aws1.txt?generation=1678334408999764&alt=media",
						Acls:        []string{},
					},
				},
			},
		},
	}

	got := make([]*sources.Chunk, 0, len(want))
	for chunk := range chunksCh {
		got = append(got, chunk)
	}
	sort.Slice(got, func(i, j int) bool {
		return got[i].SourceMetadata.GetGcs().Filename < got[j].SourceMetadata.GetGcs().Filename
	})

	assert.Equal(t, len(want), len(got))

	for i, chunk := range got {
		if diff := cmp.Diff(want[i].SourceMetadata.GetGcs(), chunk.SourceMetadata.GetGcs(),
			cmpopts.IgnoreFields(source_metadatapb.GCS{}, "state", "sizeCache", "unknownFields", "CreatedAt", "UpdatedAt"),
		); diff != "" {
			t.Errorf("chunk mismatch (-want +got):\n%s", diff)
		}
	}
}

func createTestChunks() []*sources.Chunk {
	objects := []object{
		{
			name:        "aws1.txt",
			bucket:      testBucket,
			contentType: "text/plain",
			size:        150,
			link:        "https://storage.googleapis.com/download/storage/v1/b/test-bkt-th/o/aws1.txt?generation=1677870994890594&alt=media",
			acl:         []string{},
		},
		{
			name:        "moar2.txt",
			bucket:      testBucket,
			contentType: "text/plain",
			size:        12,
			link:        "https://storage.googleapis.com/download/storage/v1/b/test-bkt-th/o/moar2.txt?generation=1677871000378542&alt=media",
			acl:         []string{},
		},
		{
			name:        "aws3.txt",
			bucket:      testBucket2,
			contentType: "text/plain",
			size:        150,
			link:        "https://storage.googleapis.com/download/storage/v1/b/test-bkt-th2/o/aws3.txt?generation=1677871022489611&alt=media",
			acl:         []string{},
		},
		{
			name:        "moar.txt",
			bucket:      testBucket3,
			contentType: "text/plain",
			size:        6,
			link:        "https://storage.googleapis.com/download/storage/v1/b/test-bkt-th3/o/moar.txt?generation=1677871042896804&alt=media",
			acl:         []string{},
		},
		{
			name:        "AMAZON_FASHION_5.json",
			bucket:      testBucket4,
			contentType: "application/json",
			size:        1413469,
			link:        "https://storage.googleapis.com/download/storage/v1/b/test-bkt-th4/o/AMAZON_FASHION_5.json?generation=1677871063457469&alt=media",
			acl:         []string{},
		},
	}

	chunks := make([]*sources.Chunk, 0, len(objects))
	for _, o := range objects {
		chunks = append(chunks, &sources.Chunk{
			SourceName: "test",
			SourceType: sourcespb.SourceType_SOURCE_TYPE_GCS,
			SourceID:   0,
			Verify:     true,
			SourceMetadata: &source_metadatapb.MetaData{
				Data: &source_metadatapb.MetaData_Gcs{
					Gcs: &source_metadatapb.GCS{
						Filename:    o.name,
						Bucket:      o.bucket,
						ContentType: o.contentType,
						Email:       o.owner,
						Link:        o.link,
						Acls:        o.acl,
					},
				},
			},
		})
	}
	sort.Slice(chunks, func(i, j int) bool {
		return chunks[i].SourceMetadata.GetGcs().Filename < chunks[j].SourceMetadata.GetGcs().Filename
	})

	return chunks
}