From 166be2995ee76e51a343f04e41d348cfeb1cc8d9 Mon Sep 17 00:00:00 2001 From: Tyera Date: Wed, 10 Jan 2024 21:20:15 -0700 Subject: [PATCH] Bigtable: update google proto files and allow configuration of max_message_size (#34740) * Update proto files with tonic-build v0.9.2 * Manually ignore invalid doc-tests * Add new ReadRowsRequest fields * Add LedgerStorageConfig::max_message_size and default value * Add BigtableConnection::max_message_size and use on client creation * Add max_message_size to RpcBigtableConfig and make const pub * Add solana-validator cli arg --- ledger-tool/src/bigtable.rs | 1 + rpc/src/rpc.rs | 2 + rpc/src/rpc_service.rs | 2 + storage-bigtable/proto/google.api.rs | 1047 ++++++++++++++++-- storage-bigtable/proto/google.bigtable.v2.rs | 788 ++++++++++++- storage-bigtable/proto/google.rpc.rs | 6 +- storage-bigtable/src/bigtable.rs | 27 +- storage-bigtable/src/lib.rs | 6 + validator/src/bin/solana-test-validator.rs | 1 + validator/src/cli.rs | 12 + validator/src/main.rs | 1 + 11 files changed, 1777 insertions(+), 116 deletions(-) diff --git a/ledger-tool/src/bigtable.rs b/ledger-tool/src/bigtable.rs index cf153aae6ce2cb..89a0cb6e2d75fa 100644 --- a/ledger-tool/src/bigtable.rs +++ b/ledger-tool/src/bigtable.rs @@ -653,6 +653,7 @@ async fn get_bigtable( credential_type: CredentialType::Filepath(Some(args.crediential_path.unwrap())), instance_name: args.instance_name, app_profile_id: args.app_profile_id, + max_message_size: solana_storage_bigtable::DEFAULT_MAX_MESSAGE_SIZE, }, ) .await diff --git a/rpc/src/rpc.rs b/rpc/src/rpc.rs index cf73a224589dec..fcbe726f07bd91 100644 --- a/rpc/src/rpc.rs +++ b/rpc/src/rpc.rs @@ -170,6 +170,7 @@ pub struct RpcBigtableConfig { pub bigtable_instance_name: String, pub bigtable_app_profile_id: String, pub timeout: Option, + pub max_message_size: usize, } impl Default for RpcBigtableConfig { @@ -181,6 +182,7 @@ impl Default for RpcBigtableConfig { bigtable_instance_name, bigtable_app_profile_id, timeout: None, + max_message_size: solana_storage_bigtable::DEFAULT_MAX_MESSAGE_SIZE, } } } diff --git a/rpc/src/rpc_service.rs b/rpc/src/rpc_service.rs index 2703a5f0af9a50..8597394f102325 100644 --- a/rpc/src/rpc_service.rs +++ b/rpc/src/rpc_service.rs @@ -406,6 +406,7 @@ impl JsonRpcService { ref bigtable_instance_name, ref bigtable_app_profile_id, timeout, + max_message_size, }) = config.rpc_bigtable_config { let bigtable_config = solana_storage_bigtable::LedgerStorageConfig { @@ -414,6 +415,7 @@ impl JsonRpcService { credential_type: CredentialType::Filepath(None), instance_name: bigtable_instance_name.clone(), app_profile_id: bigtable_app_profile_id.clone(), + max_message_size, }; runtime .block_on(solana_storage_bigtable::LedgerStorage::new_with_config( diff --git a/storage-bigtable/proto/google.api.rs b/storage-bigtable/proto/google.api.rs index 16b9ba5f26295b..96a336431a35e9 100644 --- a/storage-bigtable/proto/google.api.rs +++ b/storage-bigtable/proto/google.api.rs @@ -187,15 +187,18 @@ pub struct Http { /// 1. Leaf request fields (recursive expansion nested messages in the request /// message) are classified into three categories: /// - Fields referred by the path template. They are passed via the URL path. -/// - Fields referred by the \[HttpRule.body][google.api.HttpRule.body\]. They are passed via the HTTP +/// - Fields referred by the \[HttpRule.body][google.api.HttpRule.body\]. They +/// are passed via the HTTP /// request body. /// - All other fields are passed via the URL query parameters, and the /// parameter name is the field path in the request message. A repeated /// field can be represented as multiple query parameters under the same /// name. -/// 2. If \[HttpRule.body][google.api.HttpRule.body\] is "*", there is no URL query parameter, all fields +/// 2. If \[HttpRule.body][google.api.HttpRule.body\] is "*", there is no URL +/// query parameter, all fields /// are passed via URL path and HTTP request body. -/// 3. If \[HttpRule.body][google.api.HttpRule.body\] is omitted, there is no HTTP request body, all +/// 3. If \[HttpRule.body][google.api.HttpRule.body\] is omitted, there is no HTTP +/// request body, all /// fields are passed via URL path and URL query parameters. /// /// ### Path template syntax @@ -292,7 +295,8 @@ pub struct Http { pub struct HttpRule { /// Selects a method to which this rule applies. /// - /// Refer to \[selector][google.api.DocumentationRule.selector\] for syntax details. + /// Refer to \[selector][google.api.DocumentationRule.selector\] for syntax + /// details. #[prost(string, tag = "1")] pub selector: ::prost::alloc::string::String, /// The name of the request field whose value is mapped to the HTTP request @@ -365,6 +369,479 @@ pub struct CustomHttpPattern { #[prost(string, tag = "2")] pub path: ::prost::alloc::string::String, } +/// The launch stage as defined by [Google Cloud Platform +/// Launch Stages](). +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] +#[repr(i32)] +pub enum LaunchStage { + /// Do not use this default value. + Unspecified = 0, + /// The feature is not yet implemented. Users can not use it. + Unimplemented = 6, + /// Prelaunch features are hidden from users and are only visible internally. + Prelaunch = 7, + /// Early Access features are limited to a closed group of testers. To use + /// these features, you must sign up in advance and sign a Trusted Tester + /// agreement (which includes confidentiality provisions). These features may + /// be unstable, changed in backward-incompatible ways, and are not + /// guaranteed to be released. + EarlyAccess = 1, + /// Alpha is a limited availability test for releases before they are cleared + /// for widespread use. By Alpha, all significant design issues are resolved + /// and we are in the process of verifying functionality. Alpha customers + /// need to apply for access, agree to applicable terms, and have their + /// projects allowlisted. Alpha releases don't have to be feature complete, + /// no SLAs are provided, and there are no technical support obligations, but + /// they will be far enough along that customers can actually use them in + /// test environments or for limited-use tests -- just like they would in + /// normal production cases. + Alpha = 2, + /// Beta is the point at which we are ready to open a release for any + /// customer to use. There are no SLA or technical support obligations in a + /// Beta release. Products will be complete from a feature perspective, but + /// may have some open outstanding issues. Beta releases are suitable for + /// limited production use cases. + Beta = 3, + /// GA features are open to all developers and are considered stable and + /// fully qualified for production use. + Ga = 4, + /// Deprecated features are scheduled to be shut down and removed. For more + /// information, see the "Deprecation Policy" section of our [Terms of + /// Service]() + /// and the [Google Cloud Platform Subject to the Deprecation + /// Policy]() documentation. + Deprecated = 5, +} +impl LaunchStage { + /// String value of the enum field names used in the ProtoBuf definition. + /// + /// The values are not transformed in any way and thus are considered stable + /// (if the ProtoBuf definition does not change) and safe for programmatic use. + pub fn as_str_name(&self) -> &'static str { + match self { + LaunchStage::Unspecified => "LAUNCH_STAGE_UNSPECIFIED", + LaunchStage::Unimplemented => "UNIMPLEMENTED", + LaunchStage::Prelaunch => "PRELAUNCH", + LaunchStage::EarlyAccess => "EARLY_ACCESS", + LaunchStage::Alpha => "ALPHA", + LaunchStage::Beta => "BETA", + LaunchStage::Ga => "GA", + LaunchStage::Deprecated => "DEPRECATED", + } + } + /// Creates an enum from field names used in the ProtoBuf definition. + pub fn from_str_name(value: &str) -> ::core::option::Option { + match value { + "LAUNCH_STAGE_UNSPECIFIED" => Some(Self::Unspecified), + "UNIMPLEMENTED" => Some(Self::Unimplemented), + "PRELAUNCH" => Some(Self::Prelaunch), + "EARLY_ACCESS" => Some(Self::EarlyAccess), + "ALPHA" => Some(Self::Alpha), + "BETA" => Some(Self::Beta), + "GA" => Some(Self::Ga), + "DEPRECATED" => Some(Self::Deprecated), + _ => None, + } + } +} +/// Required information for every language. +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct CommonLanguageSettings { + /// Link to automatically generated reference documentation. Example: + /// + #[deprecated] + #[prost(string, tag = "1")] + pub reference_docs_uri: ::prost::alloc::string::String, + /// The destination where API teams want this client library to be published. + #[prost(enumeration = "ClientLibraryDestination", repeated, tag = "2")] + pub destinations: ::prost::alloc::vec::Vec, +} +/// Details about how and where to publish client libraries. +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ClientLibrarySettings { + /// Version of the API to apply these settings to. This is the full protobuf + /// package for the API, ending in the version element. + /// Examples: "google.cloud.speech.v1" and "google.spanner.admin.database.v1". + #[prost(string, tag = "1")] + pub version: ::prost::alloc::string::String, + /// Launch stage of this version of the API. + #[prost(enumeration = "LaunchStage", tag = "2")] + pub launch_stage: i32, + /// When using transport=rest, the client request will encode enums as + /// numbers rather than strings. + #[prost(bool, tag = "3")] + pub rest_numeric_enums: bool, + /// Settings for legacy Java features, supported in the Service YAML. + #[prost(message, optional, tag = "21")] + pub java_settings: ::core::option::Option, + /// Settings for C++ client libraries. + #[prost(message, optional, tag = "22")] + pub cpp_settings: ::core::option::Option, + /// Settings for PHP client libraries. + #[prost(message, optional, tag = "23")] + pub php_settings: ::core::option::Option, + /// Settings for Python client libraries. + #[prost(message, optional, tag = "24")] + pub python_settings: ::core::option::Option, + /// Settings for Node client libraries. + #[prost(message, optional, tag = "25")] + pub node_settings: ::core::option::Option, + /// Settings for .NET client libraries. + #[prost(message, optional, tag = "26")] + pub dotnet_settings: ::core::option::Option, + /// Settings for Ruby client libraries. + #[prost(message, optional, tag = "27")] + pub ruby_settings: ::core::option::Option, + /// Settings for Go client libraries. + #[prost(message, optional, tag = "28")] + pub go_settings: ::core::option::Option, +} +/// This message configures the settings for publishing [Google Cloud Client +/// libraries]() +/// generated from the service config. +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct Publishing { + /// A list of API method settings, e.g. the behavior for methods that use the + /// long-running operation pattern. + #[prost(message, repeated, tag = "2")] + pub method_settings: ::prost::alloc::vec::Vec, + /// Link to a *public* URI where users can report issues. Example: + /// + #[prost(string, tag = "101")] + pub new_issue_uri: ::prost::alloc::string::String, + /// Link to product home page. Example: + /// + #[prost(string, tag = "102")] + pub documentation_uri: ::prost::alloc::string::String, + /// Used as a tracking tag when collecting data about the APIs developer + /// relations artifacts like docs, packages delivered to package managers, + /// etc. Example: "speech". + #[prost(string, tag = "103")] + pub api_short_name: ::prost::alloc::string::String, + /// GitHub label to apply to issues and pull requests opened for this API. + #[prost(string, tag = "104")] + pub github_label: ::prost::alloc::string::String, + /// GitHub teams to be added to CODEOWNERS in the directory in GitHub + /// containing source code for the client libraries for this API. + #[prost(string, repeated, tag = "105")] + pub codeowner_github_teams: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, + /// A prefix used in sample code when demarking regions to be included in + /// documentation. + #[prost(string, tag = "106")] + pub doc_tag_prefix: ::prost::alloc::string::String, + /// For whom the client library is being published. + #[prost(enumeration = "ClientLibraryOrganization", tag = "107")] + pub organization: i32, + /// Client library settings. If the same version string appears multiple + /// times in this list, then the last one wins. Settings from earlier + /// settings with the same version string are discarded. + #[prost(message, repeated, tag = "109")] + pub library_settings: ::prost::alloc::vec::Vec, + /// Optional link to proto reference documentation. Example: + /// + #[prost(string, tag = "110")] + pub proto_reference_documentation_uri: ::prost::alloc::string::String, +} +/// Settings for Java client libraries. +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct JavaSettings { + /// The package name to use in Java. Clobbers the java_package option + /// set in the protobuf. This should be used **only** by APIs + /// who have already set the language_settings.java.package_name" field + /// in gapic.yaml. API teams should use the protobuf java_package option + /// where possible. + /// + /// Example of a YAML configuration:: + /// + /// publishing: + /// java_settings: + /// library_package: com.google.cloud.pubsub.v1 + #[prost(string, tag = "1")] + pub library_package: ::prost::alloc::string::String, + /// Configure the Java class name to use instead of the service's for its + /// corresponding generated GAPIC client. Keys are fully-qualified + /// service names as they appear in the protobuf (including the full + /// the language_settings.java.interface_names" field in gapic.yaml. API + /// teams should otherwise use the service name as it appears in the + /// protobuf. + /// + /// Example of a YAML configuration:: + /// + /// publishing: + /// java_settings: + /// service_class_names: + /// - google.pubsub.v1.Publisher: TopicAdmin + /// - google.pubsub.v1.Subscriber: SubscriptionAdmin + #[prost(map = "string, string", tag = "2")] + pub service_class_names: ::std::collections::HashMap< + ::prost::alloc::string::String, + ::prost::alloc::string::String, + >, + /// Some settings. + #[prost(message, optional, tag = "3")] + pub common: ::core::option::Option, +} +/// Settings for C++ client libraries. +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct CppSettings { + /// Some settings. + #[prost(message, optional, tag = "1")] + pub common: ::core::option::Option, +} +/// Settings for Php client libraries. +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct PhpSettings { + /// Some settings. + #[prost(message, optional, tag = "1")] + pub common: ::core::option::Option, +} +/// Settings for Python client libraries. +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct PythonSettings { + /// Some settings. + #[prost(message, optional, tag = "1")] + pub common: ::core::option::Option, +} +/// Settings for Node client libraries. +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct NodeSettings { + /// Some settings. + #[prost(message, optional, tag = "1")] + pub common: ::core::option::Option, +} +/// Settings for Dotnet client libraries. +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct DotnetSettings { + /// Some settings. + #[prost(message, optional, tag = "1")] + pub common: ::core::option::Option, + /// Map from original service names to renamed versions. + /// This is used when the default generated types + /// would cause a naming conflict. (Neither name is + /// fully-qualified.) + /// Example: Subscriber to SubscriberServiceApi. + #[prost(map = "string, string", tag = "2")] + pub renamed_services: ::std::collections::HashMap< + ::prost::alloc::string::String, + ::prost::alloc::string::String, + >, + /// Map from full resource types to the effective short name + /// for the resource. This is used when otherwise resource + /// named from different services would cause naming collisions. + /// Example entry: + /// "datalabeling.googleapis.com/Dataset": "DataLabelingDataset" + #[prost(map = "string, string", tag = "3")] + pub renamed_resources: ::std::collections::HashMap< + ::prost::alloc::string::String, + ::prost::alloc::string::String, + >, + /// List of full resource types to ignore during generation. + /// This is typically used for API-specific Location resources, + /// which should be handled by the generator as if they were actually + /// the common Location resources. + /// Example entry: "documentai.googleapis.com/Location" + #[prost(string, repeated, tag = "4")] + pub ignored_resources: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, + /// Namespaces which must be aliased in snippets due to + /// a known (but non-generator-predictable) naming collision + #[prost(string, repeated, tag = "5")] + pub forced_namespace_aliases: ::prost::alloc::vec::Vec< + ::prost::alloc::string::String, + >, + /// Method signatures (in the form "service.method(signature)") + /// which are provided separately, so shouldn't be generated. + /// Snippets *calling* these methods are still generated, however. + #[prost(string, repeated, tag = "6")] + pub handwritten_signatures: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, +} +/// Settings for Ruby client libraries. +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct RubySettings { + /// Some settings. + #[prost(message, optional, tag = "1")] + pub common: ::core::option::Option, +} +/// Settings for Go client libraries. +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct GoSettings { + /// Some settings. + #[prost(message, optional, tag = "1")] + pub common: ::core::option::Option, +} +/// Describes the generator configuration for a method. +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct MethodSettings { + /// The fully qualified name of the method, for which the options below apply. + /// This is used to find the method to apply the options. + #[prost(string, tag = "1")] + pub selector: ::prost::alloc::string::String, + /// Describes settings to use for long-running operations when generating + /// API methods for RPCs. Complements RPCs that use the annotations in + /// google/longrunning/operations.proto. + /// + /// Example of a YAML configuration:: + /// + /// publishing: + /// method_settings: + /// - selector: google.cloud.speech.v2.Speech.BatchRecognize + /// long_running: + /// initial_poll_delay: + /// seconds: 60 # 1 minute + /// poll_delay_multiplier: 1.5 + /// max_poll_delay: + /// seconds: 360 # 6 minutes + /// total_poll_timeout: + /// seconds: 54000 # 90 minutes + #[prost(message, optional, tag = "2")] + pub long_running: ::core::option::Option, + /// List of top-level fields of the request message, that should be + /// automatically populated by the client libraries based on their + /// (google.api.field_info).format. Currently supported format: UUID4. + /// + /// Example of a YAML configuration: + /// + /// publishing: + /// method_settings: + /// - selector: google.example.v1.ExampleService.CreateExample + /// auto_populated_fields: + /// - request_id + #[prost(string, repeated, tag = "3")] + pub auto_populated_fields: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, +} +/// Nested message and enum types in `MethodSettings`. +pub mod method_settings { + /// Describes settings to use when generating API methods that use the + /// long-running operation pattern. + /// All default values below are from those used in the client library + /// generators (e.g. + /// \[Java\]()). + #[allow(clippy::derive_partial_eq_without_eq)] + #[derive(Clone, PartialEq, ::prost::Message)] + pub struct LongRunning { + /// Initial delay after which the first poll request will be made. + /// Default value: 5 seconds. + #[prost(message, optional, tag = "1")] + pub initial_poll_delay: ::core::option::Option<::prost_types::Duration>, + /// Multiplier to gradually increase delay between subsequent polls until it + /// reaches max_poll_delay. + /// Default value: 1.5. + #[prost(float, tag = "2")] + pub poll_delay_multiplier: f32, + /// Maximum time between two subsequent poll requests. + /// Default value: 45 seconds. + #[prost(message, optional, tag = "3")] + pub max_poll_delay: ::core::option::Option<::prost_types::Duration>, + /// Total polling timeout. + /// Default value: 5 minutes. + #[prost(message, optional, tag = "4")] + pub total_poll_timeout: ::core::option::Option<::prost_types::Duration>, + } +} +/// The organization for which the client libraries are being published. +/// Affects the url where generated docs are published, etc. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] +#[repr(i32)] +pub enum ClientLibraryOrganization { + /// Not useful. + Unspecified = 0, + /// Google Cloud Platform Org. + Cloud = 1, + /// Ads (Advertising) Org. + Ads = 2, + /// Photos Org. + Photos = 3, + /// Street View Org. + StreetView = 4, + /// Shopping Org. + Shopping = 5, + /// Geo Org. + Geo = 6, + /// Generative AI - + GenerativeAi = 7, +} +impl ClientLibraryOrganization { + /// String value of the enum field names used in the ProtoBuf definition. + /// + /// The values are not transformed in any way and thus are considered stable + /// (if the ProtoBuf definition does not change) and safe for programmatic use. + pub fn as_str_name(&self) -> &'static str { + match self { + ClientLibraryOrganization::Unspecified => { + "CLIENT_LIBRARY_ORGANIZATION_UNSPECIFIED" + } + ClientLibraryOrganization::Cloud => "CLOUD", + ClientLibraryOrganization::Ads => "ADS", + ClientLibraryOrganization::Photos => "PHOTOS", + ClientLibraryOrganization::StreetView => "STREET_VIEW", + ClientLibraryOrganization::Shopping => "SHOPPING", + ClientLibraryOrganization::Geo => "GEO", + ClientLibraryOrganization::GenerativeAi => "GENERATIVE_AI", + } + } + /// Creates an enum from field names used in the ProtoBuf definition. + pub fn from_str_name(value: &str) -> ::core::option::Option { + match value { + "CLIENT_LIBRARY_ORGANIZATION_UNSPECIFIED" => Some(Self::Unspecified), + "CLOUD" => Some(Self::Cloud), + "ADS" => Some(Self::Ads), + "PHOTOS" => Some(Self::Photos), + "STREET_VIEW" => Some(Self::StreetView), + "SHOPPING" => Some(Self::Shopping), + "GEO" => Some(Self::Geo), + "GENERATIVE_AI" => Some(Self::GenerativeAi), + _ => None, + } + } +} +/// To where should client libraries be published? +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] +#[repr(i32)] +pub enum ClientLibraryDestination { + /// Client libraries will neither be generated nor published to package + /// managers. + Unspecified = 0, + /// Generate the client library in a repo under github.com/googleapis, + /// but don't publish it to package managers. + Github = 10, + /// Publish the library to package managers like nuget.org and npmjs.com. + PackageManager = 20, +} +impl ClientLibraryDestination { + /// String value of the enum field names used in the ProtoBuf definition. + /// + /// The values are not transformed in any way and thus are considered stable + /// (if the ProtoBuf definition does not change) and safe for programmatic use. + pub fn as_str_name(&self) -> &'static str { + match self { + ClientLibraryDestination::Unspecified => { + "CLIENT_LIBRARY_DESTINATION_UNSPECIFIED" + } + ClientLibraryDestination::Github => "GITHUB", + ClientLibraryDestination::PackageManager => "PACKAGE_MANAGER", + } + } + /// Creates an enum from field names used in the ProtoBuf definition. + pub fn from_str_name(value: &str) -> ::core::option::Option { + match value { + "CLIENT_LIBRARY_DESTINATION_UNSPECIFIED" => Some(Self::Unspecified), + "GITHUB" => Some(Self::Github), + "PACKAGE_MANAGER" => Some(Self::PackageManager), + _ => None, + } + } +} /// An indicator of the behavior of a given field (for example, that a field /// is required in requests, or given as output but ignored as input). /// This **does not** change the behavior in protocol buffers itself; it only @@ -407,6 +884,19 @@ pub enum FieldBehavior { /// a non-empty value will be returned. The user will not be aware of what /// non-empty value to expect. NonEmptyDefault = 7, + /// Denotes that the field in a resource (a message annotated with + /// google.api.resource) is used in the resource name to uniquely identify the + /// resource. For AIP-compliant APIs, this should only be applied to the + /// `name` field on the resource. + /// + /// This behavior should not be applied to references to other resources within + /// the message. + /// + /// The identifier field of resources often have different field behavior + /// depending on the request it is embedded in (e.g. for Create methods name + /// is optional and unused, while for Update methods it is required). Instead + /// of method-specific annotations, only `IDENTIFIER` is required. + Identifier = 8, } impl FieldBehavior { /// String value of the enum field names used in the ProtoBuf definition. @@ -423,6 +913,22 @@ impl FieldBehavior { FieldBehavior::Immutable => "IMMUTABLE", FieldBehavior::UnorderedList => "UNORDERED_LIST", FieldBehavior::NonEmptyDefault => "NON_EMPTY_DEFAULT", + FieldBehavior::Identifier => "IDENTIFIER", + } + } + /// Creates an enum from field names used in the ProtoBuf definition. + pub fn from_str_name(value: &str) -> ::core::option::Option { + match value { + "FIELD_BEHAVIOR_UNSPECIFIED" => Some(Self::Unspecified), + "OPTIONAL" => Some(Self::Optional), + "REQUIRED" => Some(Self::Required), + "OUTPUT_ONLY" => Some(Self::OutputOnly), + "INPUT_ONLY" => Some(Self::InputOnly), + "IMMUTABLE" => Some(Self::Immutable), + "UNORDERED_LIST" => Some(Self::UnorderedList), + "NON_EMPTY_DEFAULT" => Some(Self::NonEmptyDefault), + "IDENTIFIER" => Some(Self::Identifier), + _ => None, } } } @@ -440,11 +946,7 @@ impl FieldBehavior { /// // For Kubernetes resources, the format is {api group}/{kind}. /// option (google.api.resource) = { /// type: "pubsub.googleapis.com/Topic" -/// name_descriptor: { -/// pattern: "projects/{project}/topics/{topic}" -/// parent_type: "cloudresourcemanager.googleapis.com/Project" -/// parent_name_extractor: "projects/{project}" -/// } +/// pattern: "projects/{project}/topics/{topic}" /// }; /// } /// @@ -452,10 +954,7 @@ impl FieldBehavior { /// /// resources: /// - type: "pubsub.googleapis.com/Topic" -/// name_descriptor: -/// - pattern: "projects/{project}/topics/{topic}" -/// parent_type: "cloudresourcemanager.googleapis.com/Project" -/// parent_name_extractor: "projects/{project}" +/// pattern: "projects/{project}/topics/{topic}" /// /// Sometimes, resources have multiple patterns, typically because they can /// live under multiple parents. @@ -465,26 +964,10 @@ impl FieldBehavior { /// message LogEntry { /// option (google.api.resource) = { /// type: "logging.googleapis.com/LogEntry" -/// name_descriptor: { -/// pattern: "projects/{project}/logs/{log}" -/// parent_type: "cloudresourcemanager.googleapis.com/Project" -/// parent_name_extractor: "projects/{project}" -/// } -/// name_descriptor: { -/// pattern: "folders/{folder}/logs/{log}" -/// parent_type: "cloudresourcemanager.googleapis.com/Folder" -/// parent_name_extractor: "folders/{folder}" -/// } -/// name_descriptor: { -/// pattern: "organizations/{organization}/logs/{log}" -/// parent_type: "cloudresourcemanager.googleapis.com/Organization" -/// parent_name_extractor: "organizations/{organization}" -/// } -/// name_descriptor: { -/// pattern: "billingAccounts/{billing_account}/logs/{log}" -/// parent_type: "billing.googleapis.com/BillingAccount" -/// parent_name_extractor: "billingAccounts/{billing_account}" -/// } +/// pattern: "projects/{project}/logs/{log}" +/// pattern: "folders/{folder}/logs/{log}" +/// pattern: "organizations/{organization}/logs/{log}" +/// pattern: "billingAccounts/{billing_account}/logs/{log}" /// }; /// } /// @@ -492,48 +975,10 @@ impl FieldBehavior { /// /// resources: /// - type: 'logging.googleapis.com/LogEntry' -/// name_descriptor: -/// - pattern: "projects/{project}/logs/{log}" -/// parent_type: "cloudresourcemanager.googleapis.com/Project" -/// parent_name_extractor: "projects/{project}" -/// - pattern: "folders/{folder}/logs/{log}" -/// parent_type: "cloudresourcemanager.googleapis.com/Folder" -/// parent_name_extractor: "folders/{folder}" -/// - pattern: "organizations/{organization}/logs/{log}" -/// parent_type: "cloudresourcemanager.googleapis.com/Organization" -/// parent_name_extractor: "organizations/{organization}" -/// - pattern: "billingAccounts/{billing_account}/logs/{log}" -/// parent_type: "billing.googleapis.com/BillingAccount" -/// parent_name_extractor: "billingAccounts/{billing_account}" -/// -/// For flexible resources, the resource name doesn't contain parent names, but -/// the resource itself has parents for policy evaluation. -/// -/// Example: -/// -/// message Shelf { -/// option (google.api.resource) = { -/// type: "library.googleapis.com/Shelf" -/// name_descriptor: { -/// pattern: "shelves/{shelf}" -/// parent_type: "cloudresourcemanager.googleapis.com/Project" -/// } -/// name_descriptor: { -/// pattern: "shelves/{shelf}" -/// parent_type: "cloudresourcemanager.googleapis.com/Folder" -/// } -/// }; -/// } -/// -/// The ResourceDescriptor Yaml config will look like: -/// -/// resources: -/// - type: 'library.googleapis.com/Shelf' -/// name_descriptor: -/// - pattern: "shelves/{shelf}" -/// parent_type: "cloudresourcemanager.googleapis.com/Project" -/// - pattern: "shelves/{shelf}" -/// parent_type: "cloudresourcemanager.googleapis.com/Folder" +/// pattern: "projects/{project}/logs/{log}" +/// pattern: "folders/{folder}/logs/{log}" +/// pattern: "organizations/{organization}/logs/{log}" +/// pattern: "billingAccounts/{billing_account}/logs/{log}" #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct ResourceDescriptor { @@ -651,6 +1096,15 @@ pub mod resource_descriptor { History::FutureMultiPattern => "FUTURE_MULTI_PATTERN", } } + /// Creates an enum from field names used in the ProtoBuf definition. + pub fn from_str_name(value: &str) -> ::core::option::Option { + match value { + "HISTORY_UNSPECIFIED" => Some(Self::Unspecified), + "ORIGINALLY_SINGLE_PATTERN" => Some(Self::OriginallySinglePattern), + "FUTURE_MULTI_PATTERN" => Some(Self::FutureMultiPattern), + _ => None, + } + } } /// A flag representing a specific style that a resource claims to conform to. #[derive( @@ -689,6 +1143,14 @@ pub mod resource_descriptor { Style::DeclarativeFriendly => "DECLARATIVE_FRIENDLY", } } + /// Creates an enum from field names used in the ProtoBuf definition. + pub fn from_str_name(value: &str) -> ::core::option::Option { + match value { + "STYLE_UNSPECIFIED" => Some(Self::Unspecified), + "DECLARATIVE_FRIENDLY" => Some(Self::DeclarativeFriendly), + _ => None, + } + } } } /// Defines a proto annotation that describes a string field that refers to @@ -732,3 +1194,438 @@ pub struct ResourceReference { #[prost(string, tag = "2")] pub child_type: ::prost::alloc::string::String, } +/// Specifies the routing information that should be sent along with the request +/// in the form of routing header. +/// **NOTE:** All service configuration rules follow the "last one wins" order. +/// +/// The examples below will apply to an RPC which has the following request type: +/// +/// Message Definition: +/// +/// message Request { +/// // The name of the Table +/// // Values can be of the following formats: +/// // - `projects//tables/` +/// // - `projects//instances//tables/
` +/// // - `region//zones//tables/
` +/// string table_name = 1; +/// +/// // This value specifies routing for replication. +/// // It can be in the following formats: +/// // - `profiles/` +/// // - a legacy `profile_id` that can be any string +/// string app_profile_id = 2; +/// } +/// +/// Example message: +/// +/// { +/// table_name: projects/proj_foo/instances/instance_bar/table/table_baz, +/// app_profile_id: profiles/prof_qux +/// } +/// +/// The routing header consists of one or multiple key-value pairs. Every key +/// and value must be percent-encoded, and joined together in the format of +/// `key1=value1&key2=value2`. +/// In the examples below I am skipping the percent-encoding for readablity. +/// +/// Example 1 +/// +/// Extracting a field from the request to put into the routing header +/// unchanged, with the key equal to the field name. +/// +/// annotation: +/// +/// option (google.api.routing) = { +/// // Take the `app_profile_id`. +/// routing_parameters { +/// field: "app_profile_id" +/// } +/// }; +/// +/// result: +/// +/// x-goog-request-params: app_profile_id=profiles/prof_qux +/// +/// Example 2 +/// +/// Extracting a field from the request to put into the routing header +/// unchanged, with the key different from the field name. +/// +/// annotation: +/// +/// option (google.api.routing) = { +/// // Take the `app_profile_id`, but name it `routing_id` in the header. +/// routing_parameters { +/// field: "app_profile_id" +/// path_template: "{routing_id=**}" +/// } +/// }; +/// +/// result: +/// +/// x-goog-request-params: routing_id=profiles/prof_qux +/// +/// Example 3 +/// +/// Extracting a field from the request to put into the routing +/// header, while matching a path template syntax on the field's value. +/// +/// NB: it is more useful to send nothing than to send garbage for the purpose +/// of dynamic routing, since garbage pollutes cache. Thus the matching. +/// +/// Sub-example 3a +/// +/// The field matches the template. +/// +/// annotation: +/// +/// option (google.api.routing) = { +/// // Take the `table_name`, if it's well-formed (with project-based +/// // syntax). +/// routing_parameters { +/// field: "table_name" +/// path_template: "{table_name=projects/*/instances/*/**}" +/// } +/// }; +/// +/// result: +/// +/// x-goog-request-params: +/// table_name=projects/proj_foo/instances/instance_bar/table/table_baz +/// +/// Sub-example 3b +/// +/// The field does not match the template. +/// +/// annotation: +/// +/// option (google.api.routing) = { +/// // Take the `table_name`, if it's well-formed (with region-based +/// // syntax). +/// routing_parameters { +/// field: "table_name" +/// path_template: "{table_name=regions/*/zones/*/**}" +/// } +/// }; +/// +/// result: +/// +/// +/// +/// Sub-example 3c +/// +/// Multiple alternative conflictingly named path templates are +/// specified. The one that matches is used to construct the header. +/// +/// annotation: +/// +/// option (google.api.routing) = { +/// // Take the `table_name`, if it's well-formed, whether +/// // using the region- or projects-based syntax. +/// +/// routing_parameters { +/// field: "table_name" +/// path_template: "{table_name=regions/*/zones/*/**}" +/// } +/// routing_parameters { +/// field: "table_name" +/// path_template: "{table_name=projects/*/instances/*/**}" +/// } +/// }; +/// +/// result: +/// +/// x-goog-request-params: +/// table_name=projects/proj_foo/instances/instance_bar/table/table_baz +/// +/// Example 4 +/// +/// Extracting a single routing header key-value pair by matching a +/// template syntax on (a part of) a single request field. +/// +/// annotation: +/// +/// option (google.api.routing) = { +/// // Take just the project id from the `table_name` field. +/// routing_parameters { +/// field: "table_name" +/// path_template: "{routing_id=projects/*}/**" +/// } +/// }; +/// +/// result: +/// +/// x-goog-request-params: routing_id=projects/proj_foo +/// +/// Example 5 +/// +/// Extracting a single routing header key-value pair by matching +/// several conflictingly named path templates on (parts of) a single request +/// field. The last template to match "wins" the conflict. +/// +/// annotation: +/// +/// option (google.api.routing) = { +/// // If the `table_name` does not have instances information, +/// // take just the project id for routing. +/// // Otherwise take project + instance. +/// +/// routing_parameters { +/// field: "table_name" +/// path_template: "{routing_id=projects/*}/**" +/// } +/// routing_parameters { +/// field: "table_name" +/// path_template: "{routing_id=projects/*/instances/*}/**" +/// } +/// }; +/// +/// result: +/// +/// x-goog-request-params: +/// routing_id=projects/proj_foo/instances/instance_bar +/// +/// Example 6 +/// +/// Extracting multiple routing header key-value pairs by matching +/// several non-conflicting path templates on (parts of) a single request field. +/// +/// Sub-example 6a +/// +/// Make the templates strict, so that if the `table_name` does not +/// have an instance information, nothing is sent. +/// +/// annotation: +/// +/// option (google.api.routing) = { +/// // The routing code needs two keys instead of one composite +/// // but works only for the tables with the "project-instance" name +/// // syntax. +/// +/// routing_parameters { +/// field: "table_name" +/// path_template: "{project_id=projects/*}/instances/*/**" +/// } +/// routing_parameters { +/// field: "table_name" +/// path_template: "projects/*/{instance_id=instances/*}/**" +/// } +/// }; +/// +/// result: +/// +/// x-goog-request-params: +/// project_id=projects/proj_foo&instance_id=instances/instance_bar +/// +/// Sub-example 6b +/// +/// Make the templates loose, so that if the `table_name` does not +/// have an instance information, just the project id part is sent. +/// +/// annotation: +/// +/// option (google.api.routing) = { +/// // The routing code wants two keys instead of one composite +/// // but will work with just the `project_id` for tables without +/// // an instance in the `table_name`. +/// +/// routing_parameters { +/// field: "table_name" +/// path_template: "{project_id=projects/*}/**" +/// } +/// routing_parameters { +/// field: "table_name" +/// path_template: "projects/*/{instance_id=instances/*}/**" +/// } +/// }; +/// +/// result (is the same as 6a for our example message because it has the instance +/// information): +/// +/// x-goog-request-params: +/// project_id=projects/proj_foo&instance_id=instances/instance_bar +/// +/// Example 7 +/// +/// Extracting multiple routing header key-value pairs by matching +/// several path templates on multiple request fields. +/// +/// NB: note that here there is no way to specify sending nothing if one of the +/// fields does not match its template. E.g. if the `table_name` is in the wrong +/// format, the `project_id` will not be sent, but the `routing_id` will be. +/// The backend routing code has to be aware of that and be prepared to not +/// receive a full complement of keys if it expects multiple. +/// +/// annotation: +/// +/// option (google.api.routing) = { +/// // The routing needs both `project_id` and `routing_id` +/// // (from the `app_profile_id` field) for routing. +/// +/// routing_parameters { +/// field: "table_name" +/// path_template: "{project_id=projects/*}/**" +/// } +/// routing_parameters { +/// field: "app_profile_id" +/// path_template: "{routing_id=**}" +/// } +/// }; +/// +/// result: +/// +/// x-goog-request-params: +/// project_id=projects/proj_foo&routing_id=profiles/prof_qux +/// +/// Example 8 +/// +/// Extracting a single routing header key-value pair by matching +/// several conflictingly named path templates on several request fields. The +/// last template to match "wins" the conflict. +/// +/// annotation: +/// +/// option (google.api.routing) = { +/// // The `routing_id` can be a project id or a region id depending on +/// // the table name format, but only if the `app_profile_id` is not set. +/// // If `app_profile_id` is set it should be used instead. +/// +/// routing_parameters { +/// field: "table_name" +/// path_template: "{routing_id=projects/*}/**" +/// } +/// routing_parameters { +/// field: "table_name" +/// path_template: "{routing_id=regions/*}/**" +/// } +/// routing_parameters { +/// field: "app_profile_id" +/// path_template: "{routing_id=**}" +/// } +/// }; +/// +/// result: +/// +/// x-goog-request-params: routing_id=profiles/prof_qux +/// +/// Example 9 +/// +/// Bringing it all together. +/// +/// annotation: +/// +/// option (google.api.routing) = { +/// // For routing both `table_location` and a `routing_id` are needed. +/// // +/// // table_location can be either an instance id or a region+zone id. +/// // +/// // For `routing_id`, take the value of `app_profile_id` +/// // - If it's in the format `profiles/`, send +/// // just the `` part. +/// // - If it's any other literal, send it as is. +/// // If the `app_profile_id` is empty, and the `table_name` starts with +/// // the project_id, send that instead. +/// +/// routing_parameters { +/// field: "table_name" +/// path_template: "projects/*/{table_location=instances/*}/tables/*" +/// } +/// routing_parameters { +/// field: "table_name" +/// path_template: "{table_location=regions/*/zones/*}/tables/*" +/// } +/// routing_parameters { +/// field: "table_name" +/// path_template: "{routing_id=projects/*}/**" +/// } +/// routing_parameters { +/// field: "app_profile_id" +/// path_template: "{routing_id=**}" +/// } +/// routing_parameters { +/// field: "app_profile_id" +/// path_template: "profiles/{routing_id=*}" +/// } +/// }; +/// +/// result: +/// +/// x-goog-request-params: +/// table_location=instances/instance_bar&routing_id=prof_qux +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct RoutingRule { + /// A collection of Routing Parameter specifications. + /// **NOTE:** If multiple Routing Parameters describe the same key + /// (via the `path_template` field or via the `field` field when + /// `path_template` is not provided), "last one wins" rule + /// determines which Parameter gets used. + /// See the examples for more details. + #[prost(message, repeated, tag = "2")] + pub routing_parameters: ::prost::alloc::vec::Vec, +} +/// A projection from an input message to the GRPC or REST header. +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct RoutingParameter { + /// A request field to extract the header key-value pair from. + #[prost(string, tag = "1")] + pub field: ::prost::alloc::string::String, + /// A pattern matching the key-value field. Optional. + /// If not specified, the whole field specified in the `field` field will be + /// taken as value, and its name used as key. If specified, it MUST contain + /// exactly one named segment (along with any number of unnamed segments) The + /// pattern will be matched over the field specified in the `field` field, then + /// if the match is successful: + /// - the name of the single named segment will be used as a header name, + /// - the match value of the segment will be used as a header value; + /// if the match is NOT successful, nothing will be sent. + /// + /// Example: + /// + /// -- This is a field in the request message + /// | that the header value will be extracted from. + /// | + /// | -- This is the key name in the + /// | | routing header. + /// V | + /// field: "table_name" v + /// path_template: "projects/*/{table_location=instances/*}/tables/*" + /// ^ ^ + /// | | + /// In the {} brackets is the pattern that -- | + /// specifies what to extract from the | + /// field as a value to be sent. | + /// | + /// The string in the field must match the whole pattern -- + /// before brackets, inside brackets, after brackets. + /// + /// When looking at this specific example, we can see that: + /// - A key-value pair with the key `table_location` + /// and the value matching `instances/*` should be added + /// to the x-goog-request-params routing header. + /// - The value is extracted from the request message's `table_name` field + /// if it matches the full pattern specified: + /// `projects/*/instances/*/tables/*`. + /// + /// **NB:** If the `path_template` field is not provided, the key name is + /// equal to the field name, and the whole field should be sent as a value. + /// This makes the pattern for the field and the value functionally equivalent + /// to `**`, and the configuration + /// + /// { + /// field: "table_name" + /// } + /// + /// is a functionally equivalent shorthand to: + /// + /// { + /// field: "table_name" + /// path_template: "{table_name=**}" + /// } + /// + /// See Example 1 for more details. + #[prost(string, tag = "2")] + pub path_template: ::prost::alloc::string::String, +} diff --git a/storage-bigtable/proto/google.bigtable.v2.rs b/storage-bigtable/proto/google.bigtable.v2.rs index 362d06070dcb6f..4b23bf0e77b1e5 100644 --- a/storage-bigtable/proto/google.bigtable.v2.rs +++ b/storage-bigtable/proto/google.bigtable.v2.rs @@ -246,7 +246,7 @@ pub mod value_range { /// RowFilter.Chain and RowFilter.Interleave documentation. /// /// The total serialized size of a RowFilter message must not -/// exceed 4096 bytes, and RowFilters may not be nested within each other +/// exceed 20480 bytes, and RowFilters may not be nested within each other /// (in Chains or Interleaves) to a depth of more than 20. #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -623,6 +623,130 @@ pub mod read_modify_write_rule { IncrementAmount(i64), } } +/// NOTE: This API is intended to be used by Apache Beam BigtableIO. +/// A partition of a change stream. +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct StreamPartition { + /// The row range covered by this partition and is specified by + /// [`start_key_closed`, `end_key_open`). + #[prost(message, optional, tag = "1")] + pub row_range: ::core::option::Option, +} +/// NOTE: This API is intended to be used by Apache Beam BigtableIO. +/// The information required to continue reading the data from multiple +/// `StreamPartitions` from where a previous read left off. +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct StreamContinuationTokens { + /// List of continuation tokens. + #[prost(message, repeated, tag = "1")] + pub tokens: ::prost::alloc::vec::Vec, +} +/// NOTE: This API is intended to be used by Apache Beam BigtableIO. +/// The information required to continue reading the data from a +/// `StreamPartition` from where a previous read left off. +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct StreamContinuationToken { + /// The partition that this token applies to. + #[prost(message, optional, tag = "1")] + pub partition: ::core::option::Option, + /// An encoded position in the stream to restart reading from. + #[prost(string, tag = "2")] + pub token: ::prost::alloc::string::String, +} +/// ReadIterationStats captures information about the iteration of rows or cells +/// over the course of a read, e.g. how many results were scanned in a read +/// operation versus the results returned. +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ReadIterationStats { + /// The rows seen (scanned) as part of the request. This includes the count of + /// rows returned, as captured below. + #[prost(int64, tag = "1")] + pub rows_seen_count: i64, + /// The rows returned as part of the request. + #[prost(int64, tag = "2")] + pub rows_returned_count: i64, + /// The cells seen (scanned) as part of the request. This includes the count of + /// cells returned, as captured below. + #[prost(int64, tag = "3")] + pub cells_seen_count: i64, + /// The cells returned as part of the request. + #[prost(int64, tag = "4")] + pub cells_returned_count: i64, +} +/// RequestLatencyStats provides a measurement of the latency of the request as +/// it interacts with different systems over its lifetime, e.g. how long the +/// request took to execute within a frontend server. +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct RequestLatencyStats { + /// The latency measured by the frontend server handling this request, from + /// when the request was received, to when this value is sent back in the + /// response. For more context on the component that is measuring this latency, + /// see: + /// + /// Note: This value may be slightly shorter than the value reported into + /// aggregate latency metrics in Monitoring for this request + /// () as this value + /// needs to be sent in the response before the latency measurement including + /// that transmission is finalized. + /// + /// Note: This value includes the end-to-end latency of contacting nodes in + /// the targeted cluster, e.g. measuring from when the first byte arrives at + /// the frontend server, to when this value is sent back as the last value in + /// the response, including any latency incurred by contacting nodes, waiting + /// for results from nodes, and finally sending results from nodes back to the + /// caller. + #[prost(message, optional, tag = "1")] + pub frontend_server_latency: ::core::option::Option<::prost_types::Duration>, +} +/// FullReadStatsView captures all known information about a read. +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct FullReadStatsView { + /// Iteration stats describe how efficient the read is, e.g. comparing + /// rows seen vs. rows returned or cells seen vs cells returned can provide an + /// indication of read efficiency (the higher the ratio of seen to retuned the + /// better). + #[prost(message, optional, tag = "1")] + pub read_iteration_stats: ::core::option::Option, + /// Request latency stats describe the time taken to complete a request, from + /// the server side. + #[prost(message, optional, tag = "2")] + pub request_latency_stats: ::core::option::Option, +} +/// RequestStats is the container for additional information pertaining to a +/// single request, helpful for evaluating the performance of the sent request. +/// Currently, there are the following supported methods: +/// * google.bigtable.v2.ReadRows +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct RequestStats { + /// Information pertaining to each request type received. The type is chosen + /// based on the requested view. + /// + /// See the messages above for additional context. + #[prost(oneof = "request_stats::StatsView", tags = "1")] + pub stats_view: ::core::option::Option, +} +/// Nested message and enum types in `RequestStats`. +pub mod request_stats { + /// Information pertaining to each request type received. The type is chosen + /// based on the requested view. + /// + /// See the messages above for additional context. + #[allow(clippy::derive_partial_eq_without_eq)] + #[derive(Clone, PartialEq, ::prost::Oneof)] + pub enum StatsView { + /// Available with the ReadRowsRequest.RequestStatsView.REQUEST_STATS_FULL + /// view, see package google.bigtable.v2. + #[prost(message, tag = "1")] + FullReadStatsView(super::FullReadStatsView), + } +} /// Request message for Bigtable.ReadRows. #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -636,17 +760,85 @@ pub struct ReadRowsRequest { /// "default" application profile will be used. #[prost(string, tag = "5")] pub app_profile_id: ::prost::alloc::string::String, - /// The row keys and/or ranges to read. If not specified, reads from all rows. + /// The row keys and/or ranges to read sequentially. If not specified, reads + /// from all rows. #[prost(message, optional, tag = "2")] pub rows: ::core::option::Option, /// The filter to apply to the contents of the specified row(s). If unset, /// reads the entirety of each row. #[prost(message, optional, tag = "3")] pub filter: ::core::option::Option, - /// The read will terminate after committing to N rows' worth of results. The + /// The read will stop after committing to N rows' worth of results. The /// default (zero) is to return all results. #[prost(int64, tag = "4")] pub rows_limit: i64, + /// The view into RequestStats, as described above. + #[prost(enumeration = "read_rows_request::RequestStatsView", tag = "6")] + pub request_stats_view: i32, + /// Experimental API - Please note that this API is currently experimental + /// and can change in the future. + /// + /// Return rows in lexiographical descending order of the row keys. The row + /// contents will not be affected by this flag. + /// + /// Example result set: + ///```ignore + /// [ + /// {key: "k2", "f:col1": "v1", "f:col2": "v1"}, + /// {key: "k1", "f:col1": "v2", "f:col2": "v2"} + /// ] + #[prost(bool, tag = "7")] + pub reversed: bool, +} +/// Nested message and enum types in `ReadRowsRequest`. +pub mod read_rows_request { + /// The desired view into RequestStats that should be returned in the response. + /// + /// See also: RequestStats message. + #[derive( + Clone, + Copy, + Debug, + PartialEq, + Eq, + Hash, + PartialOrd, + Ord, + ::prost::Enumeration + )] + #[repr(i32)] + pub enum RequestStatsView { + /// The default / unset value. The API will default to the NONE option below. + Unspecified = 0, + /// Do not include any RequestStats in the response. This will leave the + /// RequestStats embedded message unset in the response. + RequestStatsNone = 1, + /// Include the full set of available RequestStats in the response, + /// applicable to this read. + RequestStatsFull = 2, + } + impl RequestStatsView { + /// String value of the enum field names used in the ProtoBuf definition. + /// + /// The values are not transformed in any way and thus are considered stable + /// (if the ProtoBuf definition does not change) and safe for programmatic use. + pub fn as_str_name(&self) -> &'static str { + match self { + RequestStatsView::Unspecified => "REQUEST_STATS_VIEW_UNSPECIFIED", + RequestStatsView::RequestStatsNone => "REQUEST_STATS_NONE", + RequestStatsView::RequestStatsFull => "REQUEST_STATS_FULL", + } + } + /// Creates an enum from field names used in the ProtoBuf definition. + pub fn from_str_name(value: &str) -> ::core::option::Option { + match value { + "REQUEST_STATS_VIEW_UNSPECIFIED" => Some(Self::Unspecified), + "REQUEST_STATS_NONE" => Some(Self::RequestStatsNone), + "REQUEST_STATS_FULL" => Some(Self::RequestStatsFull), + _ => None, + } + } + } } /// Response message for Bigtable.ReadRows. #[allow(clippy::derive_partial_eq_without_eq)] @@ -664,6 +856,28 @@ pub struct ReadRowsResponse { /// key, allowing the client to skip that work on a retry. #[prost(bytes = "vec", tag = "2")] pub last_scanned_row_key: ::prost::alloc::vec::Vec, + /// + /// If requested, provide enhanced query performance statistics. The semantics + /// dictate: + /// * request_stats is empty on every (streamed) response, except + /// * request_stats has non-empty information after all chunks have been + /// streamed, where the ReadRowsResponse message only contains + /// request_stats. + /// * For example, if a read request would have returned an empty + /// response instead a single ReadRowsResponse is streamed with empty + /// chunks and request_stats filled. + /// + /// Visually, response messages will stream as follows: + /// ... -> {chunks: \[...\]} -> {chunks: [], request_stats: {...}} + /// \______________________/ \________________________________/ + /// Primary response Trailer of RequestStats info + /// + /// Or if the read did not return any values: + /// {chunks: [], request_stats: {...}} + /// \________________________________/ + /// Trailer of RequestStats info + #[prost(message, optional, tag = "3")] + pub request_stats: ::core::option::Option, } /// Nested message and enum types in `ReadRowsResponse`. pub mod read_rows_response { @@ -780,8 +994,8 @@ pub struct SampleRowKeysResponse { #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct MutateRowRequest { - /// Required. The unique name of the table to which the mutation should be applied. - /// Values are of the form + /// Required. The unique name of the table to which the mutation should be + /// applied. Values are of the form /// `projects//instances//tables/
`. #[prost(string, tag = "1")] pub table_name: ::prost::alloc::string::String, @@ -792,9 +1006,9 @@ pub struct MutateRowRequest { /// Required. The key of the row to which the mutation should be applied. #[prost(bytes = "vec", tag = "2")] pub row_key: ::prost::alloc::vec::Vec, - /// Required. Changes to be atomically applied to the specified row. Entries are applied - /// in order, meaning that earlier mutations can be masked by later ones. - /// Must contain at least one entry and at most 100000. + /// Required. Changes to be atomically applied to the specified row. Entries + /// are applied in order, meaning that earlier mutations can be masked by later + /// ones. Must contain at least one entry and at most 100000. #[prost(message, repeated, tag = "3")] pub mutations: ::prost::alloc::vec::Vec, } @@ -806,7 +1020,8 @@ pub struct MutateRowResponse {} #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct MutateRowsRequest { - /// Required. The unique name of the table to which the mutations should be applied. + /// Required. The unique name of the table to which the mutations should be + /// applied. #[prost(string, tag = "1")] pub table_name: ::prost::alloc::string::String, /// This value specifies routing for replication. If not specified, the @@ -830,10 +1045,9 @@ pub mod mutate_rows_request { /// The key of the row to which the `mutations` should be applied. #[prost(bytes = "vec", tag = "1")] pub row_key: ::prost::alloc::vec::Vec, - /// Required. Changes to be atomically applied to the specified row. Mutations are - /// applied in order, meaning that earlier mutations can be masked by - /// later ones. - /// You must specify at least one mutation. + /// Required. Changes to be atomically applied to the specified row. + /// Mutations are applied in order, meaning that earlier mutations can be + /// masked by later ones. You must specify at least one mutation. #[prost(message, repeated, tag = "2")] pub mutations: ::prost::alloc::vec::Vec, } @@ -845,6 +1059,11 @@ pub struct MutateRowsResponse { /// One or more results for Entries from the batch request. #[prost(message, repeated, tag = "1")] pub entries: ::prost::alloc::vec::Vec, + /// Information about how client should limit the rate (QPS). Primirily used by + /// supported official Cloud Bigtable clients. If unset, the rate limit info is + /// not provided by the server. + #[prost(message, optional, tag = "3")] + pub rate_limit_info: ::core::option::Option, } /// Nested message and enum types in `MutateRowsResponse`. pub mod mutate_rows_response { @@ -864,13 +1083,36 @@ pub mod mutate_rows_response { pub status: ::core::option::Option, } } +/// Information about how client should adjust the load to Bigtable. +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct RateLimitInfo { + /// Time that clients should wait before adjusting the target rate again. + /// If clients adjust rate too frequently, the impact of the previous + /// adjustment may not have been taken into account and may + /// over-throttle or under-throttle. If clients adjust rate too slowly, they + /// will not be responsive to load changes on server side, and may + /// over-throttle or under-throttle. + #[prost(message, optional, tag = "1")] + pub period: ::core::option::Option<::prost_types::Duration>, + /// If it has been at least one `period` since the last load adjustment, the + /// client should multiply the current load by this value to get the new target + /// load. For example, if the current load is 100 and `factor` is 0.8, the new + /// target load should be 80. After adjusting, the client should ignore + /// `factor` until another `period` has passed. + /// + /// The client can measure its load using any unit that's comparable over time + /// For example, QPS can be used as long as each request involves a similar + /// amount of work. + #[prost(double, tag = "2")] + pub factor: f64, +} /// Request message for Bigtable.CheckAndMutateRow. #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct CheckAndMutateRowRequest { - /// Required. The unique name of the table to which the conditional mutation should be - /// applied. - /// Values are of the form + /// Required. The unique name of the table to which the conditional mutation + /// should be applied. Values are of the form /// `projects//instances//tables/
`. #[prost(string, tag = "1")] pub table_name: ::prost::alloc::string::String, @@ -878,7 +1120,8 @@ pub struct CheckAndMutateRowRequest { /// "default" application profile will be used. #[prost(string, tag = "7")] pub app_profile_id: ::prost::alloc::string::String, - /// Required. The key of the row to which the conditional mutation should be applied. + /// Required. The key of the row to which the conditional mutation should be + /// applied. #[prost(bytes = "vec", tag = "2")] pub row_key: ::prost::alloc::vec::Vec, /// The filter to be applied to the contents of the specified row. Depending @@ -911,13 +1154,30 @@ pub struct CheckAndMutateRowResponse { #[prost(bool, tag = "1")] pub predicate_matched: bool, } +/// Request message for client connection keep-alive and warming. +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct PingAndWarmRequest { + /// Required. The unique name of the instance to check permissions for as well + /// as respond. Values are of the form + /// `projects//instances/`. + #[prost(string, tag = "1")] + pub name: ::prost::alloc::string::String, + /// This value specifies routing for replication. If not specified, the + /// "default" application profile will be used. + #[prost(string, tag = "2")] + pub app_profile_id: ::prost::alloc::string::String, +} +/// Response message for Bigtable.PingAndWarm connection keepalive and warming. +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct PingAndWarmResponse {} /// Request message for Bigtable.ReadModifyWriteRow. #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct ReadModifyWriteRowRequest { - /// Required. The unique name of the table to which the read/modify/write rules should be - /// applied. - /// Values are of the form + /// Required. The unique name of the table to which the read/modify/write rules + /// should be applied. Values are of the form /// `projects//instances//tables/
`. #[prost(string, tag = "1")] pub table_name: ::prost::alloc::string::String, @@ -925,12 +1185,13 @@ pub struct ReadModifyWriteRowRequest { /// "default" application profile will be used. #[prost(string, tag = "4")] pub app_profile_id: ::prost::alloc::string::String, - /// Required. The key of the row to which the read/modify/write rules should be applied. + /// Required. The key of the row to which the read/modify/write rules should be + /// applied. #[prost(bytes = "vec", tag = "2")] pub row_key: ::prost::alloc::vec::Vec, - /// Required. Rules specifying how the specified row's contents are to be transformed - /// into writes. Entries are applied in order, meaning that earlier rules will - /// affect the results of later ones. + /// Required. Rules specifying how the specified row's contents are to be + /// transformed into writes. Entries are applied in order, meaning that earlier + /// rules will affect the results of later ones. #[prost(message, repeated, tag = "3")] pub rules: ::prost::alloc::vec::Vec, } @@ -942,6 +1203,312 @@ pub struct ReadModifyWriteRowResponse { #[prost(message, optional, tag = "1")] pub row: ::core::option::Option, } +/// NOTE: This API is intended to be used by Apache Beam BigtableIO. +/// Request message for Bigtable.GenerateInitialChangeStreamPartitions. +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct GenerateInitialChangeStreamPartitionsRequest { + /// Required. The unique name of the table from which to get change stream + /// partitions. Values are of the form + /// `projects//instances//tables/
`. + /// Change streaming must be enabled on the table. + #[prost(string, tag = "1")] + pub table_name: ::prost::alloc::string::String, + /// This value specifies routing for replication. If not specified, the + /// "default" application profile will be used. + /// Single cluster routing must be configured on the profile. + #[prost(string, tag = "2")] + pub app_profile_id: ::prost::alloc::string::String, +} +/// NOTE: This API is intended to be used by Apache Beam BigtableIO. +/// Response message for Bigtable.GenerateInitialChangeStreamPartitions. +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct GenerateInitialChangeStreamPartitionsResponse { + /// A partition of the change stream. + #[prost(message, optional, tag = "1")] + pub partition: ::core::option::Option, +} +/// NOTE: This API is intended to be used by Apache Beam BigtableIO. +/// Request message for Bigtable.ReadChangeStream. +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ReadChangeStreamRequest { + /// Required. The unique name of the table from which to read a change stream. + /// Values are of the form + /// `projects//instances//tables/
`. + /// Change streaming must be enabled on the table. + #[prost(string, tag = "1")] + pub table_name: ::prost::alloc::string::String, + /// This value specifies routing for replication. If not specified, the + /// "default" application profile will be used. + /// Single cluster routing must be configured on the profile. + #[prost(string, tag = "2")] + pub app_profile_id: ::prost::alloc::string::String, + /// The partition to read changes from. + #[prost(message, optional, tag = "3")] + pub partition: ::core::option::Option, + /// If specified, OK will be returned when the stream advances beyond + /// this time. Otherwise, changes will be continuously delivered on the stream. + /// This value is inclusive and will be truncated to microsecond granularity. + #[prost(message, optional, tag = "5")] + pub end_time: ::core::option::Option<::prost_types::Timestamp>, + /// If specified, the duration between `Heartbeat` messages on the stream. + /// Otherwise, defaults to 5 seconds. + #[prost(message, optional, tag = "7")] + pub heartbeat_duration: ::core::option::Option<::prost_types::Duration>, + /// Options for describing where we want to start reading from the stream. + #[prost(oneof = "read_change_stream_request::StartFrom", tags = "4, 6")] + pub start_from: ::core::option::Option, +} +/// Nested message and enum types in `ReadChangeStreamRequest`. +pub mod read_change_stream_request { + /// Options for describing where we want to start reading from the stream. + #[allow(clippy::derive_partial_eq_without_eq)] + #[derive(Clone, PartialEq, ::prost::Oneof)] + pub enum StartFrom { + /// Start reading the stream at the specified timestamp. This timestamp must + /// be within the change stream retention period, less than or equal to the + /// current time, and after change stream creation, whichever is greater. + /// This value is inclusive and will be truncated to microsecond granularity. + #[prost(message, tag = "4")] + StartTime(::prost_types::Timestamp), + /// Tokens that describe how to resume reading a stream where reading + /// previously left off. If specified, changes will be read starting at the + /// the position. Tokens are delivered on the stream as part of `Heartbeat` + /// and `CloseStream` messages. + /// + /// If a single token is provided, the token’s partition must exactly match + /// the request’s partition. If multiple tokens are provided, as in the case + /// of a partition merge, the union of the token partitions must exactly + /// cover the request’s partition. Otherwise, INVALID_ARGUMENT will be + /// returned. + #[prost(message, tag = "6")] + ContinuationTokens(super::StreamContinuationTokens), + } +} +/// NOTE: This API is intended to be used by Apache Beam BigtableIO. +/// Response message for Bigtable.ReadChangeStream. +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ReadChangeStreamResponse { + /// The data or control message on the stream. + #[prost(oneof = "read_change_stream_response::StreamRecord", tags = "1, 2, 3")] + pub stream_record: ::core::option::Option, +} +/// Nested message and enum types in `ReadChangeStreamResponse`. +pub mod read_change_stream_response { + /// A partial or complete mutation. + #[allow(clippy::derive_partial_eq_without_eq)] + #[derive(Clone, PartialEq, ::prost::Message)] + pub struct MutationChunk { + /// If set, then the mutation is a `SetCell` with a chunked value across + /// multiple messages. + #[prost(message, optional, tag = "1")] + pub chunk_info: ::core::option::Option, + /// If this is a continuation of a chunked message (`chunked_value_offset` > + /// 0), ignore all fields except the `SetCell`'s value and merge it with + /// the previous message by concatenating the value fields. + #[prost(message, optional, tag = "2")] + pub mutation: ::core::option::Option, + } + /// Nested message and enum types in `MutationChunk`. + pub mod mutation_chunk { + /// Information about the chunking of this mutation. + /// Only `SetCell` mutations can be chunked, and all chunks for a `SetCell` + /// will be delivered contiguously with no other mutation types interleaved. + #[allow(clippy::derive_partial_eq_without_eq)] + #[derive(Clone, PartialEq, ::prost::Message)] + pub struct ChunkInfo { + /// The total value size of all the chunks that make up the `SetCell`. + #[prost(int32, tag = "1")] + pub chunked_value_size: i32, + /// The byte offset of this chunk into the total value size of the + /// mutation. + #[prost(int32, tag = "2")] + pub chunked_value_offset: i32, + /// When true, this is the last chunk of a chunked `SetCell`. + #[prost(bool, tag = "3")] + pub last_chunk: bool, + } + } + /// A message corresponding to one or more mutations to the partition + /// being streamed. A single logical `DataChange` message may also be split + /// across a sequence of multiple individual messages. Messages other than + /// the first in a sequence will only have the `type` and `chunks` fields + /// populated, with the final message in the sequence also containing `done` + /// set to true. + #[allow(clippy::derive_partial_eq_without_eq)] + #[derive(Clone, PartialEq, ::prost::Message)] + pub struct DataChange { + /// The type of the mutation. + #[prost(enumeration = "data_change::Type", tag = "1")] + pub r#type: i32, + /// The cluster where the mutation was applied. + /// Not set when `type` is `GARBAGE_COLLECTION`. + #[prost(string, tag = "2")] + pub source_cluster_id: ::prost::alloc::string::String, + /// The row key for all mutations that are part of this `DataChange`. + /// If the `DataChange` is chunked across multiple messages, then this field + /// will only be set for the first message. + #[prost(bytes = "vec", tag = "3")] + pub row_key: ::prost::alloc::vec::Vec, + /// The timestamp at which the mutation was applied on the Bigtable server. + #[prost(message, optional, tag = "4")] + pub commit_timestamp: ::core::option::Option<::prost_types::Timestamp>, + /// A value that lets stream consumers reconstruct Bigtable's + /// conflict resolution semantics. + /// + /// In the event that the same row key, column family, column qualifier, + /// timestamp are modified on different clusters at the same + /// `commit_timestamp`, the mutation with the larger `tiebreaker` will be the + /// one chosen for the eventually consistent state of the system. + #[prost(int32, tag = "5")] + pub tiebreaker: i32, + /// The mutations associated with this change to the partition. + /// May contain complete mutations or chunks of a multi-message chunked + /// `DataChange` record. + #[prost(message, repeated, tag = "6")] + pub chunks: ::prost::alloc::vec::Vec, + /// When true, indicates that the entire `DataChange` has been read + /// and the client can safely process the message. + #[prost(bool, tag = "8")] + pub done: bool, + /// An encoded position for this stream's partition to restart reading from. + /// This token is for the StreamPartition from the request. + #[prost(string, tag = "9")] + pub token: ::prost::alloc::string::String, + /// An estimate of the commit timestamp that is usually lower than or equal + /// to any timestamp for a record that will be delivered in the future on the + /// stream. It is possible that, under particular circumstances that a future + /// record has a timestamp is is lower than a previously seen timestamp. For + /// an example usage see + /// + #[prost(message, optional, tag = "10")] + pub estimated_low_watermark: ::core::option::Option<::prost_types::Timestamp>, + } + /// Nested message and enum types in `DataChange`. + pub mod data_change { + /// The type of mutation. + #[derive( + Clone, + Copy, + Debug, + PartialEq, + Eq, + Hash, + PartialOrd, + Ord, + ::prost::Enumeration + )] + #[repr(i32)] + pub enum Type { + /// The type is unspecified. + Unspecified = 0, + /// A user-initiated mutation. + User = 1, + /// A system-initiated mutation as part of garbage collection. + /// + GarbageCollection = 2, + /// This is a continuation of a multi-message change. + Continuation = 3, + } + impl Type { + /// String value of the enum field names used in the ProtoBuf definition. + /// + /// The values are not transformed in any way and thus are considered stable + /// (if the ProtoBuf definition does not change) and safe for programmatic use. + pub fn as_str_name(&self) -> &'static str { + match self { + Type::Unspecified => "TYPE_UNSPECIFIED", + Type::User => "USER", + Type::GarbageCollection => "GARBAGE_COLLECTION", + Type::Continuation => "CONTINUATION", + } + } + /// Creates an enum from field names used in the ProtoBuf definition. + pub fn from_str_name(value: &str) -> ::core::option::Option { + match value { + "TYPE_UNSPECIFIED" => Some(Self::Unspecified), + "USER" => Some(Self::User), + "GARBAGE_COLLECTION" => Some(Self::GarbageCollection), + "CONTINUATION" => Some(Self::Continuation), + _ => None, + } + } + } + } + /// A periodic message with information that can be used to checkpoint + /// the state of a stream. + #[allow(clippy::derive_partial_eq_without_eq)] + #[derive(Clone, PartialEq, ::prost::Message)] + pub struct Heartbeat { + /// A token that can be provided to a subsequent `ReadChangeStream` call + /// to pick up reading at the current stream position. + #[prost(message, optional, tag = "1")] + pub continuation_token: ::core::option::Option, + /// An estimate of the commit timestamp that is usually lower than or equal + /// to any timestamp for a record that will be delivered in the future on the + /// stream. It is possible that, under particular circumstances that a future + /// record has a timestamp is is lower than a previously seen timestamp. For + /// an example usage see + /// + #[prost(message, optional, tag = "2")] + pub estimated_low_watermark: ::core::option::Option<::prost_types::Timestamp>, + } + /// A message indicating that the client should stop reading from the stream. + /// If status is OK and `continuation_tokens` & `new_partitions` are empty, the + /// stream has finished (for example if there was an `end_time` specified). + /// If `continuation_tokens` & `new_partitions` are present, then a change in + /// partitioning requires the client to open a new stream for each token to + /// resume reading. Example: + /// [B, D) ends + /// | + /// v + /// new_partitions: [A, C) [C, E) + /// continuation_tokens.partitions: [B,C) [C,D) + /// ^---^ ^---^ + /// ^ ^ + /// | | + /// | StreamContinuationToken 2 + /// | + /// StreamContinuationToken 1 + /// To read the new partition [A,C), supply the continuation tokens whose + /// ranges cover the new partition, for example ContinuationToken[A,B) & + /// ContinuationToken[B,C). + #[allow(clippy::derive_partial_eq_without_eq)] + #[derive(Clone, PartialEq, ::prost::Message)] + pub struct CloseStream { + /// The status of the stream. + #[prost(message, optional, tag = "1")] + pub status: ::core::option::Option, + /// If non-empty, contains the information needed to resume reading their + /// associated partitions. + #[prost(message, repeated, tag = "2")] + pub continuation_tokens: ::prost::alloc::vec::Vec< + super::StreamContinuationToken, + >, + /// If non-empty, contains the new partitions to start reading from, which + /// are related to but not necessarily identical to the partitions for the + /// above `continuation_tokens`. + #[prost(message, repeated, tag = "3")] + pub new_partitions: ::prost::alloc::vec::Vec, + } + /// The data or control message on the stream. + #[allow(clippy::derive_partial_eq_without_eq)] + #[derive(Clone, PartialEq, ::prost::Oneof)] + pub enum StreamRecord { + /// A mutation to the partition. + #[prost(message, tag = "1")] + DataChange(DataChange), + /// A periodic heartbeat message. + #[prost(message, tag = "2")] + Heartbeat(Heartbeat), + /// An indication that the stream should be closed. + #[prost(message, tag = "3")] + CloseStream(CloseStream), + } +} /// Generated client implementations. pub mod bigtable_client { #![allow(unused_variables, dead_code, missing_docs, clippy::let_unit_value)] @@ -956,7 +1523,7 @@ pub mod bigtable_client { /// Attempt to create a new client by connecting to a given endpoint. pub async fn connect(dst: D) -> Result where - D: std::convert::TryInto, + D: TryInto, D::Error: Into, { let conn = tonic::transport::Endpoint::new(dst)?.connect().await?; @@ -1012,6 +1579,22 @@ pub mod bigtable_client { self.inner = self.inner.accept_compressed(encoding); self } + /// Limits the maximum size of a decoded message. + /// + /// Default: `4MB` + #[must_use] + pub fn max_decoding_message_size(mut self, limit: usize) -> Self { + self.inner = self.inner.max_decoding_message_size(limit); + self + } + /// Limits the maximum size of an encoded message. + /// + /// Default: `usize::MAX` + #[must_use] + pub fn max_encoding_message_size(mut self, limit: usize) -> Self { + self.inner = self.inner.max_encoding_message_size(limit); + self + } /// Streams back the contents of all requested rows in key order, optionally /// applying the same Reader filter to each. Depending on their size, /// rows and cells may be broken up across multiple responses, but @@ -1020,7 +1603,7 @@ pub mod bigtable_client { pub async fn read_rows( &mut self, request: impl tonic::IntoRequest, - ) -> Result< + ) -> std::result::Result< tonic::Response>, tonic::Status, > { @@ -1037,7 +1620,10 @@ pub mod bigtable_client { let path = http::uri::PathAndQuery::from_static( "/google.bigtable.v2.Bigtable/ReadRows", ); - self.inner.server_streaming(request.into_request(), path, codec).await + let mut req = request.into_request(); + req.extensions_mut() + .insert(GrpcMethod::new("google.bigtable.v2.Bigtable", "ReadRows")); + self.inner.server_streaming(req, path, codec).await } /// Returns a sample of row keys in the table. The returned row keys will /// delimit contiguous sections of the table of approximately equal size, @@ -1046,7 +1632,7 @@ pub mod bigtable_client { pub async fn sample_row_keys( &mut self, request: impl tonic::IntoRequest, - ) -> Result< + ) -> std::result::Result< tonic::Response>, tonic::Status, > { @@ -1063,14 +1649,20 @@ pub mod bigtable_client { let path = http::uri::PathAndQuery::from_static( "/google.bigtable.v2.Bigtable/SampleRowKeys", ); - self.inner.server_streaming(request.into_request(), path, codec).await + let mut req = request.into_request(); + req.extensions_mut() + .insert(GrpcMethod::new("google.bigtable.v2.Bigtable", "SampleRowKeys")); + self.inner.server_streaming(req, path, codec).await } /// Mutates a row atomically. Cells already present in the row are left /// unchanged unless explicitly changed by `mutation`. pub async fn mutate_row( &mut self, request: impl tonic::IntoRequest, - ) -> Result, tonic::Status> { + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { self.inner .ready() .await @@ -1084,7 +1676,10 @@ pub mod bigtable_client { let path = http::uri::PathAndQuery::from_static( "/google.bigtable.v2.Bigtable/MutateRow", ); - self.inner.unary(request.into_request(), path, codec).await + let mut req = request.into_request(); + req.extensions_mut() + .insert(GrpcMethod::new("google.bigtable.v2.Bigtable", "MutateRow")); + self.inner.unary(req, path, codec).await } /// Mutates multiple rows in a batch. Each individual row is mutated /// atomically as in MutateRow, but the entire batch is not executed @@ -1092,7 +1687,7 @@ pub mod bigtable_client { pub async fn mutate_rows( &mut self, request: impl tonic::IntoRequest, - ) -> Result< + ) -> std::result::Result< tonic::Response>, tonic::Status, > { @@ -1109,13 +1704,19 @@ pub mod bigtable_client { let path = http::uri::PathAndQuery::from_static( "/google.bigtable.v2.Bigtable/MutateRows", ); - self.inner.server_streaming(request.into_request(), path, codec).await + let mut req = request.into_request(); + req.extensions_mut() + .insert(GrpcMethod::new("google.bigtable.v2.Bigtable", "MutateRows")); + self.inner.server_streaming(req, path, codec).await } /// Mutates a row atomically based on the output of a predicate Reader filter. pub async fn check_and_mutate_row( &mut self, request: impl tonic::IntoRequest, - ) -> Result, tonic::Status> { + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { self.inner .ready() .await @@ -1129,7 +1730,39 @@ pub mod bigtable_client { let path = http::uri::PathAndQuery::from_static( "/google.bigtable.v2.Bigtable/CheckAndMutateRow", ); - self.inner.unary(request.into_request(), path, codec).await + let mut req = request.into_request(); + req.extensions_mut() + .insert( + GrpcMethod::new("google.bigtable.v2.Bigtable", "CheckAndMutateRow"), + ); + self.inner.unary(req, path, codec).await + } + /// Warm up associated instance metadata for this connection. + /// This call is not required but may be useful for connection keep-alive. + pub async fn ping_and_warm( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/google.bigtable.v2.Bigtable/PingAndWarm", + ); + let mut req = request.into_request(); + req.extensions_mut() + .insert(GrpcMethod::new("google.bigtable.v2.Bigtable", "PingAndWarm")); + self.inner.unary(req, path, codec).await } /// Modifies a row atomically on the server. The method reads the latest /// existing timestamp and value from the specified columns and writes a new @@ -1139,7 +1772,10 @@ pub mod bigtable_client { pub async fn read_modify_write_row( &mut self, request: impl tonic::IntoRequest, - ) -> Result, tonic::Status> { + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { self.inner .ready() .await @@ -1153,7 +1789,85 @@ pub mod bigtable_client { let path = http::uri::PathAndQuery::from_static( "/google.bigtable.v2.Bigtable/ReadModifyWriteRow", ); - self.inner.unary(request.into_request(), path, codec).await + let mut req = request.into_request(); + req.extensions_mut() + .insert( + GrpcMethod::new("google.bigtable.v2.Bigtable", "ReadModifyWriteRow"), + ); + self.inner.unary(req, path, codec).await + } + /// NOTE: This API is intended to be used by Apache Beam BigtableIO. + /// Returns the current list of partitions that make up the table's + /// change stream. The union of partitions will cover the entire keyspace. + /// Partitions can be read with `ReadChangeStream`. + pub async fn generate_initial_change_stream_partitions( + &mut self, + request: impl tonic::IntoRequest< + super::GenerateInitialChangeStreamPartitionsRequest, + >, + ) -> std::result::Result< + tonic::Response< + tonic::codec::Streaming< + super::GenerateInitialChangeStreamPartitionsResponse, + >, + >, + tonic::Status, + > { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/google.bigtable.v2.Bigtable/GenerateInitialChangeStreamPartitions", + ); + let mut req = request.into_request(); + req.extensions_mut() + .insert( + GrpcMethod::new( + "google.bigtable.v2.Bigtable", + "GenerateInitialChangeStreamPartitions", + ), + ); + self.inner.server_streaming(req, path, codec).await + } + /// NOTE: This API is intended to be used by Apache Beam BigtableIO. + /// Reads changes from a table's change stream. Changes will + /// reflect both user-initiated mutations and mutations that are caused by + /// garbage collection. + pub async fn read_change_stream( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result< + tonic::Response< + tonic::codec::Streaming, + >, + tonic::Status, + > { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/google.bigtable.v2.Bigtable/ReadChangeStream", + ); + let mut req = request.into_request(); + req.extensions_mut() + .insert( + GrpcMethod::new("google.bigtable.v2.Bigtable", "ReadChangeStream"), + ); + self.inner.server_streaming(req, path, codec).await } } } diff --git a/storage-bigtable/proto/google.rpc.rs b/storage-bigtable/proto/google.rpc.rs index 8f3d2b98eee10e..e20cb14834952c 100644 --- a/storage-bigtable/proto/google.rpc.rs +++ b/storage-bigtable/proto/google.rpc.rs @@ -8,12 +8,14 @@ #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct Status { - /// The status code, which should be an enum value of \[google.rpc.Code][google.rpc.Code\]. + /// The status code, which should be an enum value of + /// \[google.rpc.Code][google.rpc.Code\]. #[prost(int32, tag = "1")] pub code: i32, /// A developer-facing error message, which should be in English. Any /// user-facing error message should be localized and sent in the - /// \[google.rpc.Status.details][google.rpc.Status.details\] field, or localized by the client. + /// \[google.rpc.Status.details][google.rpc.Status.details\] field, or localized + /// by the client. #[prost(string, tag = "2")] pub message: ::prost::alloc::string::String, /// A list of messages that carry the error details. There is a common set of diff --git a/storage-bigtable/src/bigtable.rs b/storage-bigtable/src/bigtable.rs index 3eeee6f6eb2cda..5dc452896d2188 100644 --- a/storage-bigtable/src/bigtable.rs +++ b/storage-bigtable/src/bigtable.rs @@ -121,6 +121,7 @@ pub struct BigTableConnection { table_prefix: String, app_profile_id: String, timeout: Option, + max_message_size: usize, } impl BigTableConnection { @@ -141,11 +142,18 @@ impl BigTableConnection { read_only: bool, timeout: Option, credential_type: CredentialType, + max_message_size: usize, ) -> Result { match std::env::var("BIGTABLE_EMULATOR_HOST") { Ok(endpoint) => { info!("Connecting to bigtable emulator at {}", endpoint); - Self::new_for_emulator(instance_name, app_profile_id, &endpoint, timeout) + Self::new_for_emulator( + instance_name, + app_profile_id, + &endpoint, + timeout, + max_message_size, + ) } Err(_) => { @@ -210,6 +218,7 @@ impl BigTableConnection { table_prefix, app_profile_id: app_profile_id.to_string(), timeout, + max_message_size, }) } } @@ -220,6 +229,7 @@ impl BigTableConnection { app_profile_id: &str, endpoint: &str, timeout: Option, + max_message_size: usize, ) -> Result { Ok(Self { access_token: None, @@ -229,6 +239,7 @@ impl BigTableConnection { table_prefix: format!("projects/emulator/instances/{instance_name}/tables/"), app_profile_id: app_profile_id.to_string(), timeout, + max_message_size, }) } @@ -254,7 +265,9 @@ impl BigTableConnection { } Ok(req) }, - ); + ) + .max_decoding_message_size(self.max_message_size) + .max_encoding_message_size(self.max_message_size); BigTable { access_token: self.access_token.clone(), client, @@ -469,6 +482,8 @@ impl) -> InterceptedRequestResult> BigTable { ], })), }), + request_stats_view: 0, + reversed: false, }) .await? .into_inner(); @@ -494,6 +509,8 @@ impl) -> InterceptedRequestResult> BigTable { filter: Some(RowFilter { filter: Some(row_filter::Filter::StripValueTransformer(true)), }), + request_stats_view: 0, + reversed: false, }) .await? .into_inner(); @@ -545,6 +562,8 @@ impl) -> InterceptedRequestResult> BigTable { // Only return the latest version of each cell filter: Some(row_filter::Filter::CellsPerColumnLimitFilter(1)), }), + request_stats_view: 0, + reversed: false, }) .await? .into_inner(); @@ -577,6 +596,8 @@ impl) -> InterceptedRequestResult> BigTable { // Only return the latest version of each cell filter: Some(row_filter::Filter::CellsPerColumnLimitFilter(1)), }), + request_stats_view: 0, + reversed: false, }) .await? .into_inner(); @@ -610,6 +631,8 @@ impl) -> InterceptedRequestResult> BigTable { // Only return the latest version of each cell filter: Some(row_filter::Filter::CellsPerColumnLimitFilter(1)), }), + request_stats_view: 0, + reversed: false, }) .await? .into_inner(); diff --git a/storage-bigtable/src/lib.rs b/storage-bigtable/src/lib.rs index 9f6667cece7a76..85c714c635b5b4 100644 --- a/storage-bigtable/src/lib.rs +++ b/storage-bigtable/src/lib.rs @@ -381,6 +381,7 @@ impl From for TransactionByAddrInfo { pub const DEFAULT_INSTANCE_NAME: &str = "solana-ledger"; pub const DEFAULT_APP_PROFILE_ID: &str = "default"; +pub const DEFAULT_MAX_MESSAGE_SIZE: usize = 64 * 1024 * 1024; // 64MB #[derive(Debug)] pub enum CredentialType { @@ -395,6 +396,7 @@ pub struct LedgerStorageConfig { pub credential_type: CredentialType, pub instance_name: String, pub app_profile_id: String, + pub max_message_size: usize, } impl Default for LedgerStorageConfig { @@ -405,6 +407,7 @@ impl Default for LedgerStorageConfig { credential_type: CredentialType::Filepath(None), instance_name: DEFAULT_INSTANCE_NAME.to_string(), app_profile_id: DEFAULT_APP_PROFILE_ID.to_string(), + max_message_size: DEFAULT_MAX_MESSAGE_SIZE, } } } @@ -471,6 +474,7 @@ impl LedgerStorage { app_profile_id, endpoint, timeout, + LedgerStorageConfig::default().max_message_size, )?, stats, }) @@ -484,6 +488,7 @@ impl LedgerStorage { instance_name, app_profile_id, credential_type, + max_message_size, } = config; let connection = bigtable::BigTableConnection::new( instance_name.as_str(), @@ -491,6 +496,7 @@ impl LedgerStorage { read_only, timeout, credential_type, + max_message_size, ) .await?; Ok(Self { stats, connection }) diff --git a/validator/src/bin/solana-test-validator.rs b/validator/src/bin/solana-test-validator.rs index 39f206116ecaf7..aee5fc039df410 100644 --- a/validator/src/bin/solana-test-validator.rs +++ b/validator/src/bin/solana-test-validator.rs @@ -430,6 +430,7 @@ fn main() { String ), timeout: None, + ..RpcBigtableConfig::default() }) } else { None diff --git a/validator/src/cli.rs b/validator/src/cli.rs index b77993fe54954c..d065a3524f4078 100644 --- a/validator/src/cli.rs +++ b/validator/src/cli.rs @@ -878,6 +878,15 @@ pub fn app<'a>(version: &'a str, default_args: &'a DefaultArgs) -> App<'a, 'a> { .default_value(&default_args.rpc_bigtable_app_profile_id) .help("Bigtable application profile id to use in requests") ) + .arg( + Arg::with_name("rpc_bigtable_max_message_size") + .long("rpc-bigtable-max-message-size") + .value_name("BYTES") + .validator(is_parsable::) + .takes_value(true) + .default_value(&default_args.rpc_bigtable_max_message_size) + .help("Max encoding and decoding message size used in Bigtable Grpc client"), + ) .arg( Arg::with_name("rpc_pubsub_worker_threads") .long("rpc-pubsub-worker-threads") @@ -1925,6 +1934,7 @@ pub struct DefaultArgs { pub rpc_bigtable_timeout: String, pub rpc_bigtable_instance_name: String, pub rpc_bigtable_app_profile_id: String, + pub rpc_bigtable_max_message_size: String, pub rpc_max_request_body_size: String, pub rpc_pubsub_worker_threads: String, @@ -2010,6 +2020,8 @@ impl DefaultArgs { rpc_bigtable_instance_name: solana_storage_bigtable::DEFAULT_INSTANCE_NAME.to_string(), rpc_bigtable_app_profile_id: solana_storage_bigtable::DEFAULT_APP_PROFILE_ID .to_string(), + rpc_bigtable_max_message_size: solana_storage_bigtable::DEFAULT_MAX_MESSAGE_SIZE + .to_string(), rpc_pubsub_worker_threads: "4".to_string(), accountsdb_repl_threads: num_cpus::get().to_string(), maximum_full_snapshot_archives_to_retain: DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN diff --git a/validator/src/main.rs b/validator/src/main.rs index a3c8d170dab784..902a73df45604d 100644 --- a/validator/src/main.rs +++ b/validator/src/main.rs @@ -1230,6 +1230,7 @@ pub fn main() { timeout: value_t!(matches, "rpc_bigtable_timeout", u64) .ok() .map(Duration::from_secs), + max_message_size: value_t_or_exit!(matches, "rpc_bigtable_max_message_size", usize), }) } else { None