trufflehog/pkg/engine/gcs.go

97 lines
2.6 KiB
Go
Raw Normal View History

Add gcs scanning integration (#1153) * Setup for GCS scanning. * Update GCS engine w/ projectID req. * Add concurrency field to gcsManager. * add errgroup to gcsManager. * Update gcs manager. * Use defautl ADC. * use ADC.' * Add TOOD. * add log to iterator completion. * use a BinaryReader instead of concrete object for channel type. * initial test for Chunks. * Add tests for chunking objects. * Add concurrency. * update metadata to include content type and acls. * Add object reading code. * Add integration test. * Add entrypoint. * Add removed wg.Wait(). * remove dead code. * remove build. * Remove period from file extension. * remove used. * Add comment. * Setup for GCS scanning. * Update GCS engine w/ projectID req. * Add concurrency field to gcsManager. * add errgroup to gcsManager. * Update gcs manager. * Use defautl ADC. * use ADC.' * Add TOOD. * add log to iterator completion. * use a BinaryReader instead of concrete object for channel type. * initial test for Chunks. * Add tests for chunking objects. * Add concurrency. * update metadata to include content type and acls. * Add object reading code. * Add integration test. * Add entrypoint. * Add removed wg.Wait(). * remove dead code. * remove build. * remove used. * Add file type for objects. * Add check for file type and size. * Add default file size. * Add additinoal auth options and remaining CLI flags. * Handle errors in go routines. * Handle resuming for buckets. * Remove redundant words in comment. * remove ok check on bool check. * remove extra blank line. * Add return if handler handles chunk. * Add comment. * remove extra blank line. * cleanup comment. * Add comment. * move up fxn. * go mod tidy. * Add exclusion to perf testing buckets. * Handle blocking the channel. * remove unused const. * fix tests. * fix tests. * Handle gcs manger options better. * update fxn name. * Remove arg name. * ignore buckets in gcsManager test. * fix test. * propulate gsManagerOpts. * inline err check. * Add readme. * update readme spelling. * fix test.
2023-03-08 01:32:04 +00:00
package engine
import (
"fmt"
"google.golang.org/protobuf/proto"
"google.golang.org/protobuf/types/known/anypb"
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources/gcs"
)
// ScanGCS with the provided options.
func (e *Engine) ScanGCS(ctx context.Context, c sources.GCSConfig) error {
// Project ID is required if using any authenticated access.
if c.ProjectID == "" && !c.WithoutAuth {
Add gcs scanning integration (#1153) * Setup for GCS scanning. * Update GCS engine w/ projectID req. * Add concurrency field to gcsManager. * add errgroup to gcsManager. * Update gcs manager. * Use defautl ADC. * use ADC.' * Add TOOD. * add log to iterator completion. * use a BinaryReader instead of concrete object for channel type. * initial test for Chunks. * Add tests for chunking objects. * Add concurrency. * update metadata to include content type and acls. * Add object reading code. * Add integration test. * Add entrypoint. * Add removed wg.Wait(). * remove dead code. * remove build. * Remove period from file extension. * remove used. * Add comment. * Setup for GCS scanning. * Update GCS engine w/ projectID req. * Add concurrency field to gcsManager. * add errgroup to gcsManager. * Update gcs manager. * Use defautl ADC. * use ADC.' * Add TOOD. * add log to iterator completion. * use a BinaryReader instead of concrete object for channel type. * initial test for Chunks. * Add tests for chunking objects. * Add concurrency. * update metadata to include content type and acls. * Add object reading code. * Add integration test. * Add entrypoint. * Add removed wg.Wait(). * remove dead code. * remove build. * remove used. * Add file type for objects. * Add check for file type and size. * Add default file size. * Add additinoal auth options and remaining CLI flags. * Handle errors in go routines. * Handle resuming for buckets. * Remove redundant words in comment. * remove ok check on bool check. * remove extra blank line. * Add return if handler handles chunk. * Add comment. * remove extra blank line. * cleanup comment. * Add comment. * move up fxn. * go mod tidy. * Add exclusion to perf testing buckets. * Handle blocking the channel. * remove unused const. * fix tests. * fix tests. * Handle gcs manger options better. * update fxn name. * Remove arg name. * ignore buckets in gcsManager test. * fix test. * propulate gsManagerOpts. * inline err check. * Add readme. * update readme spelling. * fix test.
2023-03-08 01:32:04 +00:00
return fmt.Errorf("project ID is required")
}
// If using unauthenticated access, the project ID is not used.
if c.ProjectID != "" && c.WithoutAuth {
c.ProjectID = ""
ctx.Logger().Info("project ID is not used when using unauthenticated access, ignoring provided project ID")
}
Add gcs scanning integration (#1153) * Setup for GCS scanning. * Update GCS engine w/ projectID req. * Add concurrency field to gcsManager. * add errgroup to gcsManager. * Update gcs manager. * Use defautl ADC. * use ADC.' * Add TOOD. * add log to iterator completion. * use a BinaryReader instead of concrete object for channel type. * initial test for Chunks. * Add tests for chunking objects. * Add concurrency. * update metadata to include content type and acls. * Add object reading code. * Add integration test. * Add entrypoint. * Add removed wg.Wait(). * remove dead code. * remove build. * Remove period from file extension. * remove used. * Add comment. * Setup for GCS scanning. * Update GCS engine w/ projectID req. * Add concurrency field to gcsManager. * add errgroup to gcsManager. * Update gcs manager. * Use defautl ADC. * use ADC.' * Add TOOD. * add log to iterator completion. * use a BinaryReader instead of concrete object for channel type. * initial test for Chunks. * Add tests for chunking objects. * Add concurrency. * update metadata to include content type and acls. * Add object reading code. * Add integration test. * Add entrypoint. * Add removed wg.Wait(). * remove dead code. * remove build. * remove used. * Add file type for objects. * Add check for file type and size. * Add default file size. * Add additinoal auth options and remaining CLI flags. * Handle errors in go routines. * Handle resuming for buckets. * Remove redundant words in comment. * remove ok check on bool check. * remove extra blank line. * Add return if handler handles chunk. * Add comment. * remove extra blank line. * cleanup comment. * Add comment. * move up fxn. * go mod tidy. * Add exclusion to perf testing buckets. * Handle blocking the channel. * remove unused const. * fix tests. * fix tests. * Handle gcs manger options better. * update fxn name. * Remove arg name. * ignore buckets in gcsManager test. * fix test. * propulate gsManagerOpts. * inline err check. * Add readme. * update readme spelling. * fix test.
2023-03-08 01:32:04 +00:00
connection := &sourcespb.GCS{
ProjectId: c.ProjectID,
IncludeBuckets: c.IncludeBuckets,
ExcludeBuckets: c.ExcludeBuckets,
IncludeObjects: c.IncludeObjects,
ExcludeObjects: c.ExcludeObjects,
}
// Make sure only one auth method is selected.
if !isAuthValid(ctx, c, connection) {
return fmt.Errorf("multiple auth methods selected, please select only one")
}
var conn anypb.Any
err := anypb.MarshalFrom(&conn, connection, proto.MarshalOptions{})
if err != nil {
return fmt.Errorf("failed to marshal GCS connection: %w", err)
}
sourceName := "trufflehog - gcs"
sourceID, jobID, _ := e.sourceManager.GetIDs(ctx, sourceName, gcs.SourceType)
gcsSource := &gcs.Source{}
if err := gcsSource.Init(ctx, sourceName, jobID, sourceID, true, &conn, int(c.Concurrency)); err != nil {
return err
Add gcs scanning integration (#1153) * Setup for GCS scanning. * Update GCS engine w/ projectID req. * Add concurrency field to gcsManager. * add errgroup to gcsManager. * Update gcs manager. * Use defautl ADC. * use ADC.' * Add TOOD. * add log to iterator completion. * use a BinaryReader instead of concrete object for channel type. * initial test for Chunks. * Add tests for chunking objects. * Add concurrency. * update metadata to include content type and acls. * Add object reading code. * Add integration test. * Add entrypoint. * Add removed wg.Wait(). * remove dead code. * remove build. * Remove period from file extension. * remove used. * Add comment. * Setup for GCS scanning. * Update GCS engine w/ projectID req. * Add concurrency field to gcsManager. * add errgroup to gcsManager. * Update gcs manager. * Use defautl ADC. * use ADC.' * Add TOOD. * add log to iterator completion. * use a BinaryReader instead of concrete object for channel type. * initial test for Chunks. * Add tests for chunking objects. * Add concurrency. * update metadata to include content type and acls. * Add object reading code. * Add integration test. * Add entrypoint. * Add removed wg.Wait(). * remove dead code. * remove build. * remove used. * Add file type for objects. * Add check for file type and size. * Add default file size. * Add additinoal auth options and remaining CLI flags. * Handle errors in go routines. * Handle resuming for buckets. * Remove redundant words in comment. * remove ok check on bool check. * remove extra blank line. * Add return if handler handles chunk. * Add comment. * remove extra blank line. * cleanup comment. * Add comment. * move up fxn. * go mod tidy. * Add exclusion to perf testing buckets. * Handle blocking the channel. * remove unused const. * fix tests. * fix tests. * Handle gcs manger options better. * update fxn name. * Remove arg name. * ignore buckets in gcsManager test. * fix test. * propulate gsManagerOpts. * inline err check. * Add readme. * update readme spelling. * fix test.
2023-03-08 01:32:04 +00:00
}
_, err = e.sourceManager.Run(ctx, sourceName, gcsSource)
return err
Add gcs scanning integration (#1153) * Setup for GCS scanning. * Update GCS engine w/ projectID req. * Add concurrency field to gcsManager. * add errgroup to gcsManager. * Update gcs manager. * Use defautl ADC. * use ADC.' * Add TOOD. * add log to iterator completion. * use a BinaryReader instead of concrete object for channel type. * initial test for Chunks. * Add tests for chunking objects. * Add concurrency. * update metadata to include content type and acls. * Add object reading code. * Add integration test. * Add entrypoint. * Add removed wg.Wait(). * remove dead code. * remove build. * Remove period from file extension. * remove used. * Add comment. * Setup for GCS scanning. * Update GCS engine w/ projectID req. * Add concurrency field to gcsManager. * add errgroup to gcsManager. * Update gcs manager. * Use defautl ADC. * use ADC.' * Add TOOD. * add log to iterator completion. * use a BinaryReader instead of concrete object for channel type. * initial test for Chunks. * Add tests for chunking objects. * Add concurrency. * update metadata to include content type and acls. * Add object reading code. * Add integration test. * Add entrypoint. * Add removed wg.Wait(). * remove dead code. * remove build. * remove used. * Add file type for objects. * Add check for file type and size. * Add default file size. * Add additinoal auth options and remaining CLI flags. * Handle errors in go routines. * Handle resuming for buckets. * Remove redundant words in comment. * remove ok check on bool check. * remove extra blank line. * Add return if handler handles chunk. * Add comment. * remove extra blank line. * cleanup comment. * Add comment. * move up fxn. * go mod tidy. * Add exclusion to perf testing buckets. * Handle blocking the channel. * remove unused const. * fix tests. * fix tests. * Handle gcs manger options better. * update fxn name. * Remove arg name. * ignore buckets in gcsManager test. * fix test. * propulate gsManagerOpts. * inline err check. * Add readme. * update readme spelling. * fix test.
2023-03-08 01:32:04 +00:00
}
func isAuthValid(ctx context.Context, c sources.GCSConfig, connection *sourcespb.GCS) bool {
var isAuthSelected bool
if c.WithoutAuth {
isAuthSelected = true
connection.Credential = &sourcespb.GCS_Unauthenticated{}
}
if c.CloudCred {
if isAuthSelected {
return false
}
isAuthSelected = true
connection.Credential = &sourcespb.GCS_Adc{}
}
if c.ServiceAccount != "" {
if isAuthSelected {
return false
}
isAuthSelected = true
connection.Credential = &sourcespb.GCS_ServiceAccountFile{
ServiceAccountFile: c.ServiceAccount,
Add gcs scanning integration (#1153) * Setup for GCS scanning. * Update GCS engine w/ projectID req. * Add concurrency field to gcsManager. * add errgroup to gcsManager. * Update gcs manager. * Use defautl ADC. * use ADC.' * Add TOOD. * add log to iterator completion. * use a BinaryReader instead of concrete object for channel type. * initial test for Chunks. * Add tests for chunking objects. * Add concurrency. * update metadata to include content type and acls. * Add object reading code. * Add integration test. * Add entrypoint. * Add removed wg.Wait(). * remove dead code. * remove build. * Remove period from file extension. * remove used. * Add comment. * Setup for GCS scanning. * Update GCS engine w/ projectID req. * Add concurrency field to gcsManager. * add errgroup to gcsManager. * Update gcs manager. * Use defautl ADC. * use ADC.' * Add TOOD. * add log to iterator completion. * use a BinaryReader instead of concrete object for channel type. * initial test for Chunks. * Add tests for chunking objects. * Add concurrency. * update metadata to include content type and acls. * Add object reading code. * Add integration test. * Add entrypoint. * Add removed wg.Wait(). * remove dead code. * remove build. * remove used. * Add file type for objects. * Add check for file type and size. * Add default file size. * Add additinoal auth options and remaining CLI flags. * Handle errors in go routines. * Handle resuming for buckets. * Remove redundant words in comment. * remove ok check on bool check. * remove extra blank line. * Add return if handler handles chunk. * Add comment. * remove extra blank line. * cleanup comment. * Add comment. * move up fxn. * go mod tidy. * Add exclusion to perf testing buckets. * Handle blocking the channel. * remove unused const. * fix tests. * fix tests. * Handle gcs manger options better. * update fxn name. * Remove arg name. * ignore buckets in gcsManager test. * fix test. * propulate gsManagerOpts. * inline err check. * Add readme. * update readme spelling. * fix test.
2023-03-08 01:32:04 +00:00
}
}
if c.ApiKey != "" {
if isAuthSelected {
return false
}
isAuthSelected = true
connection.Credential = &sourcespb.GCS_ApiKey{
ApiKey: c.ApiKey,
}
}
if !isAuthSelected {
ctx.Logger().Info("no auth method selected, using unauthenticated")
connection.Credential = &sourcespb.GCS_Unauthenticated{}
}
return true
}