trufflehog/proto/sources.proto
Charlie Gunyon 311494e86e
Elastic adapter (#2727)
* Add stub source and elastic API funcs

* Spawn workers and ship chunks

* Now successfully detects a credential

- Added tests
- Added some documentation comments
- Threaded the passed context through to all the API requests

* Linting fixes

* Add integration tests and resolve some bugs they uncovered

* Logstash -> Elasticsearch

* Add support for --index-pattern

* Add support for --query-json

* Use structs instead of string building to construct a search body

* Support --since-timestamp

* Implement additional authentication methods

* Fix some small bugs

* Refactoring to support --best-effort-scan

* Finish implementation of --best-effort-scan

* Implement scan catch-up

* Finish connecting support for nodes CLI arg

* Add some integration tests around the catchup mechanism

* go mod tidy

* Fix some linting issues

* Remove some debugging Prints

* Move off of _doc

* Remove informational Printf and add informational logging

* Remove debugging logging

* Copy the index from the outer loop as well

* Don't burn up the ES API with rapid requests if there's no work to do in subsequent scans

* No need to export UnitOfWork.AddSearch

* Use a better name for the range query variable when building the timestamp range clause in searches

* Replace some unlocking defers with explicit unlocks to make the synchronized part of the code clearer

* found -> ok

* Remove superfluous buildElasticClient method

---------

Co-authored-by: Charlie Gunyon <charlie@spectral.energy>
2024-05-24 09:38:20 -05:00

423 lines
11 KiB
Protocol Buffer

syntax = "proto3";
package sources;
option go_package = "github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb";
import "validate/validate.proto";
import "credentials.proto";
import "google/protobuf/any.proto";
import "google/protobuf/duration.proto";
import "google/protobuf/timestamp.proto";
enum SourceType {
SOURCE_TYPE_AZURE_STORAGE = 0;
SOURCE_TYPE_BITBUCKET = 1;
SOURCE_TYPE_CIRCLECI = 2;
SOURCE_TYPE_CONFLUENCE = 3;
SOURCE_TYPE_DOCKER = 4;
SOURCE_TYPE_ECR = 5;
SOURCE_TYPE_GCS = 6;
SOURCE_TYPE_GITHUB = 7;
SOURCE_TYPE_PUBLIC_GIT = 8;
SOURCE_TYPE_GITLAB = 9;
SOURCE_TYPE_JIRA = 10;
SOURCE_TYPE_NPM_UNAUTHD_PACKAGES = 11;
SOURCE_TYPE_PYPI_UNAUTHD_PACKAGES = 12;
SOURCE_TYPE_S3 = 13;
SOURCE_TYPE_SLACK = 14;
SOURCE_TYPE_FILESYSTEM = 15;
SOURCE_TYPE_GIT = 16;
SOURCE_TYPE_TEST = 17;
SOURCE_TYPE_S3_UNAUTHED = 18;
SOURCE_TYPE_GITHUB_UNAUTHENTICATED_ORG = 19;
SOURCE_TYPE_BUILDKITE = 20;
SOURCE_TYPE_GERRIT = 21;
SOURCE_TYPE_JENKINS = 22;
SOURCE_TYPE_TEAMS = 23;
SOURCE_TYPE_JFROG_ARTIFACTORY = 24;
SOURCE_TYPE_SYSLOG = 25;
SOURCE_TYPE_PUBLIC_EVENT_MONITORING = 26;
SOURCE_TYPE_SLACK_REALTIME = 27;
SOURCE_TYPE_GOOGLE_DRIVE = 28;
SOURCE_TYPE_SHAREPOINT = 29;
SOURCE_TYPE_GCS_UNAUTHED = 30;
SOURCE_TYPE_AZURE_REPOS = 31;
SOURCE_TYPE_TRAVISCI = 32;
SOURCE_TYPE_POSTMAN = 33;
SOURCE_TYPE_WEBHOOK = 34;
SOURCE_TYPE_ELASTICSEARCH = 35;
}
message LocalSource {
string type = 1 ;
string name = 2 ;
// DEPRECATED: scan_interval is deprecated and can be removed when we no longer depend on the name.
// Deprecating in favor of scan_period due to the fact that scan_interval is a duration
// which is a fixed-length span of time represented as a count of seconds and fractions of seconds
// at nanosecond resolution. Most of the time, we want to be able to specify a scan interval in
// human-readable format (e.g. 45s, 30m, 12h, etc.) which is not possible with a duration.
// https://protobuf.dev/reference/protobuf/google.protobuf/#duration
google.protobuf.Duration scan_interval = 3 [deprecated = true];
bool verify = 4;
google.protobuf.Any connection = 5;
string scan_period = 6;
}
// https://www.jfrog.com/confluence/display/JFROG/Artifactory+REST+API#ArtifactoryRESTAPI-RetrieveFolderorRepositoryArchive
message Artifactory {
string endpoint = 1 [(validate.rules).string.uri_ref = true];
oneof credential {
credentials.BasicAuth basic_auth = 2;
string access_token = 3;
credentials.Unauthenticated unauthenticated = 7;
}
repeated string repositories = 4;
repeated string include_paths = 5;
repeated string ignore_paths = 6;
}
message AzureStorage {
oneof credential {
string connection_string = 1;
credentials.BasicAuth basic_auth = 2;
string client_certificate = 3;
credentials.Unauthenticated unauthenticated = 4;
}
repeated string storage_containers = 5;
}
message Bitbucket {
string endpoint = 1 [(validate.rules).string.uri_ref = true];
oneof credential {
string token = 2;
credentials.Oauth2 oauth = 3;
credentials.BasicAuth basic_auth = 4;
}
repeated string repositories = 5;
repeated string ignore_repos = 6;
bool skip_binaries = 7;
bool skip_archives = 8;
}
message CircleCI {
string endpoint = 1 [(validate.rules).string.uri_ref = true];
oneof credential {
string token = 2;
}
}
message TravisCI {
string endpoint = 1 [(validate.rules).string.uri_ref = true];
oneof credential {
string token = 2;
}
}
message Confluence {
string endpoint = 1 [(validate.rules).string.uri_ref = true];
oneof credential {
credentials.Unauthenticated unauthenticated = 2;
credentials.BasicAuth basic_auth = 3;
string token = 4;
}
enum GetAllSpacesScope {
ALL = 0;
GLOBAL = 1;
PERSONAL = 2;
}
GetAllSpacesScope spaces_scope = 5;
bool insecure_skip_verify_tls = 6;
repeated string spaces = 7;
repeated string ignore_spaces = 8;
bool include_attachments = 9;
bool skip_history = 10;
}
message Docker {
oneof credential {
credentials.Unauthenticated unauthenticated = 1;
credentials.BasicAuth basic_auth = 2;
string bearer_token = 3;
bool docker_keychain = 4;
}
repeated string images = 5;
}
message ECR {
oneof credential {
credentials.KeySecret access_key = 1;
}
repeated string registries = 2;
}
message Filesystem {
// DEPRECATED: directories is deprecated and can be removed / renamed to
// paths when we no longer depend on the name in enterprise configs.
repeated string directories = 1;
repeated string paths = 2;
string include_paths_file = 3; // path to file containing newline separated list of paths
string exclude_paths_file = 4; // path to file containing newline separated list of paths
}
message GCS {
oneof credential {
string json_service_account = 1;
string api_key = 2;
credentials.Unauthenticated unauthenticated = 3;
credentials.CloudEnvironment adc = 4;
string service_account_file = 11;
credentials.Oauth2 oauth = 12;
}
string project_id = 5;
repeated string include_buckets = 6;
repeated string exclude_buckets = 7;
repeated string include_objects = 8;
repeated string exclude_objects = 9;
int64 max_object_size = 10;
}
message Git {
oneof credential {
credentials.BasicAuth basic_auth = 1;
credentials.Unauthenticated unauthenticated = 2;
credentials.SSHAuth ssh_auth = 5;
}
repeated string directories = 3;
repeated string repositories = 4;
string head = 6;
string base = 7;
bool bare = 8;
string include_paths_file = 9; // path to file containing newline separated list of paths
string exclude_paths_file = 10; // path to file containing newline separated list of paths
string exclude_globs = 11; // comma separated list of globs
int64 max_depth = 12;
// This field is generally used by the CLI or within CI/CD systems to specify a single repository,
// whereas the repositories field is used by the enterprise config to specify multiple repositories.
// Passing a single repository via the uri field also allows for additional options to be specified
// like head, base, bare, etc.
string uri = 13; // repository URL. https://, file://, or ssh://
bool skip_binaries = 14;
bool skip_archives = 15;
}
message GitLab {
string endpoint = 1 [(validate.rules).string.uri_ref = true];
oneof credential {
string token = 2;
credentials.Oauth2 oauth = 3;
credentials.BasicAuth basic_auth = 4;
}
repeated string repositories = 5;
repeated string ignore_repos = 6;
bool skip_binaries = 7;
bool skip_archives = 8;
}
message GitHub {
string endpoint = 1 [(validate.rules).string.uri_ref = true];
oneof credential {
credentials.GitHubApp github_app = 2;
string token = 3;
credentials.Unauthenticated unauthenticated = 4;
credentials.BasicAuth basic_auth = 13;
}
repeated string repositories = 5;
repeated string organizations = 6;
bool scanUsers = 7;
bool includeForks = 8;
string head = 9;
string base = 10;
repeated string ignore_repos = 11;
repeated string include_repos = 12;
bool include_pull_request_comments = 14;
bool include_issue_comments = 15;
bool include_gist_comments = 16;
bool skip_binaries = 17;
bool skip_archives = 18;
bool include_wikis = 19;
}
message GoogleDrive {
oneof credential {
string refresh_token = 1;
}
}
message JIRA {
string endpoint = 1 [(validate.rules).string.uri_ref = true];
oneof credential {
credentials.BasicAuth basic_auth = 2;
credentials.Unauthenticated unauthenticated = 3;
credentials.Oauth2 oauth = 4;
string token = 6;
}
repeated string projects = 5;
repeated string ignore_projects = 7;
bool insecure_skip_verify_tls = 8;
}
message NPMUnauthenticatedPackage {
oneof credential {
credentials.Unauthenticated unauthenticated = 1;
}
}
message PyPIUnauthenticatedPackage {
oneof credential {
credentials.Unauthenticated unauthenticated = 1;
}
}
message S3 {
oneof credential {
credentials.KeySecret access_key = 1;
credentials.Unauthenticated unauthenticated = 2;
credentials.CloudEnvironment cloud_environment = 4;
credentials.AWSSessionTokenSecret session_token = 5;
}
repeated string buckets = 3;
int64 max_object_size = 6;
repeated string roles = 7;
repeated string ignore_buckets = 8;
}
message Slack {
string endpoint = 1 [(validate.rules).string.uri_ref = true];
oneof credential {
string token = 2;
credentials.SlackTokens tokens = 5;
}
repeated string channels = 3;
repeated string ignore_list = 4;
}
message Test{}
message Buildkite {
oneof credential {
string token = 1;
}
}
message Gerrit {
string endpoint = 1 [(validate.rules).string.uri_ref = true];
oneof credential {
credentials.BasicAuth basic_auth = 2;
credentials.Unauthenticated unauthenticated = 3;
}
repeated string projects = 4;
bool skip_binaries = 5;
bool skip_archives = 6;
}
message Jenkins {
string endpoint = 1 [(validate.rules).string.uri_ref = true];
oneof credential {
credentials.BasicAuth basic_auth = 2;
credentials.Header header = 3;
}
bool insecure_skip_verify_tls = 4;
}
message Teams {
string endpoint = 1 [(validate.rules).string.uri_ref = true];
oneof credential {
string token = 2;
credentials.ClientCredentials authenticated = 3;
credentials.Oauth2 oauth = 7;
}
repeated string channels = 4;
repeated string ignore_list = 5;
repeated string team_ids = 6;
}
message Syslog {
string protocol = 1;
string listen_address = 2;
string tlsCert = 3;
string tlsKey = 4;
string format = 5;
}
message Forager {
oneof credential {
credentials.Unauthenticated unauthenticated = 1;
}
repeated string domains = 2;
int64 max_depth = 3;
google.protobuf.Timestamp since = 4;
}
message SlackRealtime {
oneof credential {
credentials.SlackTokens tokens = 1;
}
}
message Sharepoint {
oneof credential {
credentials.Oauth2 oauth = 1;
}
string site_url = 2;
}
message AzureRepos {
string endpoint = 1 [(validate.rules).string.uri_ref = true];
oneof credential {
string token = 2;
credentials.Oauth2 oauth = 3;
}
repeated string repositories = 4;
repeated string organizations = 5;
repeated string projects = 6;
bool include_forks = 7;
repeated string ignore_repos = 8;
repeated string include_repos = 9;
repeated string include_projects = 10;
repeated string ignore_projects = 11;
bool skip_binaries = 12;
bool skip_archives = 13;
}
message Postman {
oneof credential {
credentials.Unauthenticated unauthenticated = 1;
string token = 2;
}
repeated string workspaces = 3;
repeated string collections = 4;
repeated string environments = 5;
repeated string exclude_collections = 6;
repeated string exclude_environments = 7;
repeated string include_collections = 8;
repeated string include_environments = 9;
repeated string detector_keywords = 10;
repeated string workspace_paths = 11;
repeated string collection_paths = 12;
repeated string environment_paths = 13;
}
message Webhook {
string listen_address = 1 [(validate.rules).string.hostname = true];
oneof credential {
credentials.Header header = 2;
}
}
message Elasticsearch {
repeated string nodes = 1;
string username = 2;
string password = 3;
string cloud_id = 4;
string api_key = 5;
string service_token = 6;
string index_pattern = 7;
string query_json = 8;
string since_timestamp = 9;
bool best_effort_scan = 10;
}