Add '--tamper-proof' flag and Azure Confidential Ledger Logic

azure-pipelines.yml

@@ -396,6 +396,7 @@ jobs:
           S2S_SRC_S3_SERVICE_URL: $(S2S_SRC_S3_SERVICE_URL)
           S2S_SRC_GCP_SERVICE_URL: $(S2S_SRC_GCP_SERVICE_URL)
           SHARE_SAS_URL: $(SHARE_SAS_URL)
+          TAMPER_PROOF_ENDPOINT: "https://testazcopy.confidential-ledger.azure.com"
           GOCOVERDIR: '$(System.DefaultWorkingDirectory)/coverage'
         condition: succeededOrFailed()
       # Smoke Tests Publishing

cmd/copy.go

@@ -113,6 +113,7 @@ type rawCopyCmdArgs struct {
 	noGuessMimeType          bool
 	preserveLastModifiedTime bool
 	putMd5                   bool
+	tamperProof              string
 	md5ValidationOption      string
 	CheckLength              bool
 	deleteSnapshotsOption    string
@@ -642,6 +643,7 @@ func (raw rawCopyCmdArgs) cook() (CookedCopyCmdArgs, error) {
 	}
 
 	cooked.putMd5 = raw.putMd5
+	cooked.tamperProof = raw.tamperProof
 	err = cooked.md5ValidationOption.Parse(raw.md5ValidationOption)
 	if err != nil {
 		return cooked, err
@@ -834,6 +836,9 @@ func (raw rawCopyCmdArgs) cook() (CookedCopyCmdArgs, error) {
 	if err = validateMd5Option(cooked.md5ValidationOption, cooked.FromTo); err != nil {
 		return cooked, err
 	}
+	if err = validateTamperProofOption(cooked.tamperProof, cooked.putMd5, cooked.FromTo); err != nil {
+		return cooked, err
+	}
 
 	// Because of some of our defaults, these must live down here and can't be properly checked.
 	// TODO: Remove the above checks where they can't be done.
@@ -1032,6 +1037,14 @@ func validateMd5Option(option common.HashValidationOption, fromTo common.FromTo)
 	return nil
 }
 
+func validateTamperProofOption(tamperProof string, putMd5 bool, fromTo common.FromTo) error {
+	// Check to ensure put-md5 flag is present, check-md5 is set by default.
+	if len(tamperProof) > 0 && !putMd5 && fromTo.IsUpload() {
+		return fmt.Errorf("put-md5 is false but tamper-proof is set")
+	}
+	return nil
+}
+
 // Valid tag key and value characters include:
 // 1. Lowercase and uppercase letters (a-z, A-Z)
 // 2. Digits (0-9)
@@ -1153,6 +1166,7 @@ type CookedCopyCmdArgs struct {
 	preserveLastModifiedTime bool
 	deleteSnapshotsOption    common.DeleteSnapshotsOption
 	putMd5                   bool
+	tamperProof              string
 	md5ValidationOption      common.HashValidationOption
 	CheckLength              bool
 	// commandString hold the user given command which is logged to the Job log file
@@ -1529,6 +1543,7 @@ func (cca *CookedCopyCmdArgs) processCopyJobPartOrders() (err error) {
 			NoGuessMimeType:          cca.noGuessMimeType,
 			PreserveLastModifiedTime: cca.preserveLastModifiedTime,
 			PutMd5:                   cca.putMd5,
+			TamperProof:              cca.tamperProof,
 			MD5ValidationOption:      cca.md5ValidationOption,
 			DeleteSnapshotsOption:    cca.deleteSnapshotsOption,
 			// Setting tags when tags explicitly provided by the user through blob-tags flag
@@ -2085,6 +2100,7 @@ func init() {
 	cpCmd.PersistentFlags().BoolVar(&raw.forceIfReadOnly, "force-if-read-only", false, "False by default. When overwriting an existing file on Windows or Azure Files, force the overwrite to work even if the existing file has its read-only attribute set")
 	cpCmd.PersistentFlags().BoolVar(&raw.backupMode, common.BackupModeFlagName, false, "False by default. Activates Windows' SeBackupPrivilege for uploads, or SeRestorePrivilege for downloads, to allow AzCopy to see read all files, regardless of their file system permissions, and to restore all permissions. Requires that the account running AzCopy already has these permissions (e.g. has Administrator rights or is a member of the 'Backup Operators' group). All this flag does is activate privileges that the account already has.")
 	cpCmd.PersistentFlags().BoolVar(&raw.putMd5, "put-md5", false, "Create an MD5 hash of each file, and save the hash as the Content-MD5 property of the destination blob or file. By default the hash is NOT created. Only available when uploading.")
+	cpCmd.PersistentFlags().StringVar(&raw.tamperProof, "tamper-proof", "", "Uploads/ downloads the MD5 hash of each file to a specified tamper-proof storage.")
 	cpCmd.PersistentFlags().StringVar(&raw.md5ValidationOption, "check-md5", common.DefaultHashValidationOption.String(), "Specifies how strictly MD5 hashes should be validated when downloading. Only available when downloading. Available options: NoCheck, LogOnly, FailIfDifferent, FailIfDifferentOrMissing (default 'FailIfDifferent').")
 	cpCmd.PersistentFlags().StringVar(&raw.includeFileAttributes, "include-attributes", "", "(Windows only) Include files whose attributes match the attribute list. For example: A;S;R")
 	cpCmd.PersistentFlags().StringVar(&raw.excludeFileAttributes, "exclude-attributes", "", "(Windows only) Exclude files whose attributes match the attribute list. For example: A;S;R")

cmd/sync.go

@@ -69,6 +69,7 @@ type rawSyncCmdArgs struct {
 	preserveSymlinks        bool
 	backupMode              bool
 	putMd5                  bool
+	tamperProof             string
 	md5ValidationOption     string
 	// this flag indicates the user agreement with respect to deleting the extra files at the destination
 	// which do not exists at source. With this flag turned on/off, users will not be asked for permission.
@@ -301,6 +302,8 @@ func (raw *rawSyncCmdArgs) cook() (cookedSyncCmdArgs, error) {
 		return cooked, err
 	}
 
+	cooked.tamperProof = raw.tamperProof
+
 	err = cooked.md5ValidationOption.Parse(raw.md5ValidationOption)
 	if err != nil {
 		return cooked, err
@@ -412,6 +415,7 @@ type cookedSyncCmdArgs struct {
 	preserveSMBInfo         bool
 	preservePOSIXProperties bool
 	putMd5                  bool
+	tamperProof             string
 	md5ValidationOption     common.HashValidationOption
 	blockSize               int64
 	putBlobSize             int64
@@ -818,6 +822,7 @@ func init() {
 	syncCmd.PersistentFlags().StringVar(&raw.deleteDestination, "delete-destination", "false", "Defines whether to delete extra files from the destination that are not present at the source. Could be set to true, false, or prompt. "+
 		"If set to prompt, the user will be asked a question before scheduling files and blobs for deletion. (default 'false').")
 	syncCmd.PersistentFlags().BoolVar(&raw.putMd5, "put-md5", false, "Create an MD5 hash of each file, and save the hash as the Content-MD5 property of the destination blob or file. (By default the hash is NOT created.) Only available when uploading.")
+	syncCmd.PersistentFlags().StringVar(&raw.tamperProof, "tamper-proof", "", "Writes the MD5 hash of each file to a specified tamper-proof storage.")
 	syncCmd.PersistentFlags().StringVar(&raw.md5ValidationOption, "check-md5", common.DefaultHashValidationOption.String(), "Specifies how strictly MD5 hashes should be validated when downloading. This option is only available when downloading. Available values include: NoCheck, LogOnly, FailIfDifferent, FailIfDifferentOrMissing. (default 'FailIfDifferent').")
 	syncCmd.PersistentFlags().BoolVar(&raw.s2sPreserveAccessTier, "s2s-preserve-access-tier", true, "Preserve access tier during service to service copy. "+
 		"Please refer to [Azure Blob storage: hot, cool, and archive access tiers](https://docs.microsoft.com/azure/storage/blobs/storage-blob-storage-tiers) to ensure destination storage account supports setting access tier. "+

cmd/syncEnumerator.go

@@ -155,6 +155,7 @@ func (cca *cookedSyncCmdArgs) initEnumerator(ctx context.Context) (enumerator *s
 		BlobAttributes: common.BlobTransferAttributes{
 			PreserveLastModifiedTime:         cca.preserveSMBInfo, // true by default for sync so that future syncs have this information available
 			PutMd5:                           cca.putMd5,
+			TamperProof:                      cca.tamperProof,
 			MD5ValidationOption:              cca.md5ValidationOption,
 			BlockSizeInBytes:                 cca.blockSize,
 			PutBlobSizeInBytes:               cca.putBlobSize,

common/rpc-models.go

@@ -239,6 +239,7 @@ type BlobTransferAttributes struct {
 	NoGuessMimeType                  bool                  // represents user decision to interpret the content-encoding from source file
 	PreserveLastModifiedTime         bool                  // when downloading, tell engine to set file's timestamp to timestamp of blob
 	PutMd5                           bool                  // when uploading, should we create and PUT Content-MD5 hashes
+	TamperProof                      string                // when uploading/downloading should we upload/download hash from a tamper-proof ledger
 	MD5ValidationOption              HashValidationOption  // when downloading, how strictly should we validate MD5 hashes?
 	BlockSizeInBytes                 int64                 // when uploading/downloading/copying, specify the size of each chunk
 	PutBlobSizeInBytes               int64                 // when uploading, specify the threshold to determine if the blob should be uploaded in a single PUT request

ste/JobPartPlan.go

@@ -13,7 +13,7 @@ import (
 // dataSchemaVersion defines the data schema version of JobPart order files supported by
 // current version of azcopy
 // To be Incremented every time when we release azcopy with changed dataSchema
-const DataSchemaVersion common.Version = 18
+const DataSchemaVersion common.Version = 19
 
 const (
 	CustomHeaderMaxBytes = 256
@@ -302,6 +302,9 @@ type JobPartPlanDstBlob struct {
 	// Controls uploading of MD5 hashes
 	PutMd5 bool
 
+	// Specifies tamper proof storage to store MD5 hash
+	TamperProof string
+
 	MetadataLength uint16
 	Metadata       [MetadataMaxBytes]byte
 

ste/JobPartPlanFileName.go

@@ -194,6 +194,7 @@ func (jpfn JobPartPlanFileName) Create(order common.CopyJobPartOrderRequest) {
 			ContentLanguageLength:            uint16(len(order.BlobAttributes.ContentLanguage)),
 			CacheControlLength:               uint16(len(order.BlobAttributes.CacheControl)),
 			PutMd5:                           order.BlobAttributes.PutMd5, // here because it relates to uploads (blob destination)
+			TamperProof:                      order.BlobAttributes.TamperProof,
 			BlockBlobTier:                    order.BlobAttributes.BlockBlobTier,
 			PageBlobTier:                     order.BlobAttributes.PageBlobTier,
 			MetadataLength:                   uint16(len(order.BlobAttributes.Metadata)),