Merge branch 'master' of github.com:pingcap/kvproto into feat-resolve…

…d-ts-for-large-txn

proto/brpb.proto

@@ -277,6 +277,14 @@ enum CompressionType {
     ZSTD = 3;
 }
 
+message StreamBackupTaskSecurityConfig {
+    oneof encryption {
+        // not recommended in production. configure KMS based or locally managed master key instead in TiKV StreamBackupConfig
+        CipherInfo plaintext_data_key = 1;
+        MasterKeyConfig master_key_config = 2;
+    }
+}
+
 // BackupMpde represents the mode of this whole backup request to the cluster.
 // and we need to store it in `backupmeta`.
 enum BackupMode {
@@ -295,6 +303,11 @@ message CipherInfo {
     bytes cipher_key = 2;
 }
 
+message MasterKeyConfig {
+    encryptionpb.EncryptionMethod encryption_type = 1;
+    repeated encryptionpb.MasterKey master_keys = 2;
+}
+
 message BackupRequest {
     uint64 cluster_id = 1;
 
@@ -356,7 +369,7 @@ message StreamBackupTaskInfo {
     uint64 start_ts = 2;
     uint64 end_ts = 3;
 
-    // Misc meta datas.
+    // Misc meta data.
     // The name of the task, also the ID of the task.
     string name = 4;
     // The table filter of the task.
@@ -366,7 +379,10 @@ message StreamBackupTaskInfo {
     // compression type
     CompressionType compression_type = 6;
 
-    // The last timestamp of the task has been updated. 
+    // security config for backup files
+    StreamBackupTaskSecurityConfig security_config = 7;
+
+    // The last timestamp of the task has been updated.
     // This is a simple solution for unfrequent config changing: 
     // When we watched a config change(via polling or etcd watching),
     // We perform a incremental scan between [last_update_ts, now), 
@@ -665,7 +681,7 @@ message DataFileGroup {
 }
 
 message DataFileInfo {
-    // SHA256 of the file.
+    // Checksum of the plaintext file, i.e., pre-compression, pre-encryption.
     bytes sha256 = 1;
     // Path of the file.
     string path = 2;
@@ -715,18 +731,23 @@ message DataFileInfo {
 
     // The original region range the log file belongs to.
     // In older versions, this might be empty.
+    // This may not be accurative: file may contain keys less than this.
     bytes region_start_key = 20;
     // The original region range the log file belongs to.
     // In older versions, this might be empty.
+    // This may not be accurative: file may contain keys greater than this.
     bytes region_end_key = 21;
     // The region epoch that the log file belongs to.
     // In older versions, this might be empty.
-    // If a region get split or merged during observing, 
-    // the file may contain multi epoches.
+    // If a region get split or merged during observing, the file may contain multi epoches.
+    // This may not be complete: file may contain records from other versions.
+    //
+    // If there is exactly one epoch, 
+    // `region_start_key` and `region_end_key` must match this epoch.
     repeated metapb.RegionEpoch region_epoch = 22;
 
-    // It may support encrypting at future.
-    reserved "iv";
+    // Encryption information of this data file, not set if plaintext.
+    encryptionpb.FileEncryptionInfo file_encryption_info = 23;
 }
 
 message StreamBackupError {
@@ -784,6 +805,21 @@ message LogFileSubcompaction {
     LogFileSubcompactionMeta meta = 1;
     // The generated SSTs.
     repeated File sst_outputs = 2;
+    // The hint for the history of the region that contains the keys being compacted.
+    // This may be helpful for splitting, but it does not guarantee completeness and accuracy.
+    repeated RegionMetaHint region_meta_hints = 3;
+}
+
+message RegionMetaHint {
+    // You may get region id from `LogFileSubcompactionMeta`.
+
+    // The left boundary of the region of this epoch.
+    // It is an byte-comparable encoded key without TS.
+    bytes start_key = 1;
+    // The right boundary of the region of the epoch.
+    bytes end_key = 2;
+    // The region epoch.
+    metapb.RegionEpoch region_epoch = 3;
 }
 
 // Batched version of `LogFileSubcompaction`.
@@ -812,12 +848,12 @@ message LogFileCompaction {
 }
 
 message MetaEdit {
-    // Path to the meta file. 
+    // Path to the meta file.
     string path = 1;
     // Delete the physical files (MetaFileGroup) in the meta file.
     repeated string delete_physical_files = 2;
     // Delete the logical files (MetaFileInfo) in the meta file.
-    // Note: Even the operation have been performed in the meta, 
+    // Note: Even the operation have been performed in the meta,
     // this modification should be kept as long as the corresponding physical
     // file not deleted. Or we may cannot know when to delete the physical file.
     // Then the file will be leak until truncated.
@@ -826,7 +862,7 @@ message MetaEdit {
     bool destruct_self = 4;
 }
 
-// An extended version of `SpansOfFile`, added more metadata for the 
+// An extended version of `SpansOfFile`, added more metadata for the
 // execution of delayed deletion.
 message DeleteSpansOfFile {
     string path = 1;

proto/cdcpb.proto

@@ -38,6 +38,13 @@ message ClusterIDMismatch {
     uint64 request = 2;
 }
 
+// Congested is an error variable that
+// tells people that the TiKV-CDC is congested.
+message Congested {
+    // The region ID that triggers the congestion.
+    uint64 region_id = 1;
+}
+
 message Error {
     errorpb.NotLeader not_leader = 1;
     errorpb.RegionNotFound region_not_found = 2;
@@ -46,6 +53,7 @@ message Error {
     Compatibility compatibility = 5;
     ClusterIDMismatch cluster_id_mismatch = 6;
     errorpb.ServerIsBusy server_is_busy = 7;
+    Congested congested = 8;
 }
 
 message TxnInfo {

proto/encryptionpb.proto

@@ -96,6 +96,36 @@ message MasterKeyKms {
     string region = 3;
     // KMS endpoint. Normally not needed.
     string endpoint = 4;
+    // optional, used to set up azure master key backend
+    AzureKms azure_kms = 5;
+    // optional, used to set up gcp master key backend
+    GcpKms gcp_kms = 6;
+    // optional, used to set up aws master key backend
+    AwsKms aws_kms = 7;
+}
+
+message AzureKms {
+    string tenant_id = 1;
+    string client_id = 2;
+    string client_secret = 3;
+    // Key vault to encrypt/decrypt data key.
+    string key_vault_url = 4;
+    // optional hsm used to generate data key
+    string hsm_name = 5;
+    string hsm_url = 6;
+    string client_certificate = 7;
+    string client_certificate_path = 8;
+    string client_certificate_password = 9;
+
+}
+
+message GcpKms {
+    string credential = 1;
+}
+
+message AwsKms {
+    string access_key = 1;
+    string secret_access_key = 2;
 }
 
 message EncryptedContent {
@@ -113,3 +143,25 @@ message EncryptedContent {
     // Valid only when KMS is used.
     bytes ciphertext_key = 5;
 }
+
+message FileEncryptionInfo {
+    oneof mode {
+        PlainTextDataKey plain_text_data_key = 1;
+        MasterKeyBased master_key_based = 2;
+    }
+    // file encryption method
+    encryptionpb.EncryptionMethod encryption_method = 3;
+    // iv to encrypt the file by data key
+    bytes file_iv = 4;
+    // file checksum after encryption, optional if using GCM
+    bytes checksum = 5;
+}
+
+// not recommended in production.
+// user needs to pass back the same data key for restore.
+message PlainTextDataKey {}
+
+message MasterKeyBased {
+    // encrypted data key with metadata
+    repeated encryptionpb.EncryptedContent data_key_encrypted_content = 1;
+}

proto/import_sstpb.proto

@@ -8,6 +8,7 @@ import "kvrpcpb.proto";
 import "gogoproto/gogo.proto";
 import "rustproto.proto";
 import "brpb.proto";
+import "encryptionpb.proto";
 
 option (gogoproto.sizer_all) = true;
 option (gogoproto.marshaler_all) = true;
@@ -394,6 +395,9 @@ message KVMeta {
 
     // the compression type for the file.
     backup.CompressionType compression_type = 13;
+
+    // encryption information of the kv file, not set if encryption is not enabled.
+    encryptionpb.FileEncryptionInfo file_encryption_info = 14;
 }
 
 
@@ -425,8 +429,11 @@ message ApplyRequest {
     // context represents region info and it used to build raft commands.
     kvrpcpb.Context context = 4;
 
-    // cipher_info is used to decrypt kv file when download file.
+    // plaintext data key to decrypt kv file if configured during log backup.
     backup.CipherInfo cipher_info = 11;
+
+    // master keys config used to decrypt data keys in restore if configured during log backup.
+    repeated encryptionpb.MasterKey master_keys = 14;
 }
 
 message ApplyResponse {

proto/raft_serverpb.proto

@@ -255,6 +255,12 @@ enum ExtraMessageType {
     // Messages for the snapshot gen precheck process.
     MsgSnapGenPrecheckRequest = 15;
     MsgSnapGenPrecheckResponse = 16;
+    // Used in transfer leader process for leader to inform follower to load the 
+    // region into in-memory engine if the relevant region is cached.
+    MsgPreLoadRegionRequest = 17;
+    // Used in transfer leader process for follower to inform leader the completes 
+    // of the region cache
+    MsgPreLoadRegionResponse = 18;
 }
 
 message FlushMemtable {

proto/resource_manager.proto

@@ -211,6 +211,8 @@ enum RunawayWatchType {
 
 message RunawayRule {
   uint64 exec_elapsed_time_ms = 1;
+  int64 processed_keys = 2;
+  int64 request_unit = 3;
 }
 
 message RunawayWatch {
@@ -229,7 +231,10 @@ message RunawaySettings {
 }
 
 message BackgroundSettings {
-  repeated string job_types = 1; 
+  // background task types.
+  repeated string job_types = 1;
+  // the percentage limit of total resource(cpu/io) that background tasks can use.
+  uint64 utilization_limit = 2;
 }
 
 message Participant {

scripts/check.sh

@@ -1,12 +1,17 @@
 #!/usr/bin/env bash
 
 check_protoc_version() {
-    version=$(protoc --version)
-    major=$(echo ${version} | sed -n -e 's/.*\([0-9]\{1,\}\)\.[0-9]\{1,\}\.[0-9]\{1,\}.*/\1/p')
-    minor=$(echo ${version} | sed -n -e 's/.*[0-9]\{1,\}\.\([0-9]\{1,\}\)\.[0-9]\{1,\}.*/\1/p')
+    version=$(protoc --version | awk '{print $NF}')
+    major=$(echo ${version} | cut -d '.' -f 1)
+    minor=$(echo ${version} | cut -d '.' -f 2)
     if [ "$major" -eq 3 ] && [ "$minor" -ge 8 ]; then
         return 0
     fi
+    # protobuf bumps the major version to 21 after 3.
+    # https://github.com/protocolbuffers/protobuf/releases/tag/v21.7
+    if [ "$major" -ge 21 ]; then
+        return 0
+    fi
     echo "protoc version not match, version 3.8.x+ is needed, current version: ${version}"
     return 1
 }