From dd28262eb0502eab11b7f4856747505a51904ce7 Mon Sep 17 00:00:00 2001 From: seebees Date: Wed, 15 Nov 2023 11:56:24 -0800 Subject: [PATCH 1/5] =?UTF-8?q?fix:=20issue=20when=20a=20DynamoDB=20Set=20?= =?UTF-8?q?attribute=20is=20marked=20as=20SIGN=5FONLY=20in=20th=E2=80=A6?= =?UTF-8?q?=20(#560)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit DB-ESDK for DynamoDB supports SIGN_ONLY and ENCRYPT_AND_SIGN attribute actions. In version 3.1.0 and below, when a Set type is assigned a SIGN_ONLY attribute action, there is a chance that signature validation of the record containing a Set will fail on read, even if the Set attributes contain the same values. The probability of a failure depends on the order of the elements in the Set combined with how DynamoDB returns this data, which is undefined. This update addresses the issue by ensuring that any Set values are canonicalized in the same order while written to DynamoDB as when read back from DynamoDB. See: https://github.com/aws/aws-database-encryption-sdk-dynamodb-java/tree/v3.1.1/DecryptWithPermute for additional details --- CHANGELOG.md | 13 + .../DynamoDbEncryption/src/ConfigToInfo.dfy | 2 + .../DynamoDbEncryption/src/DDBSupport.dfy | 2 + .../DynamoDbEncryption/src/DynamoToStruct.dfy | 60 ++-- .../DynamoDbEncryption/src/SearchInfo.dfy | 15 +- .../test/DynamoToStruct.dfy | 264 +++++++++++++++++- ...tionSdkDynamoDbItemEncryptorOperations.dfy | 1 + .../dafny/DynamoDbItemEncryptor/src/Index.dfy | 1 + README.md | 4 +- TestVectors/dafny/DDBEncryption/src/Index.dfy | 4 + .../dafny/DDBEncryption/src/Permute.dfy | 72 +++++ .../dafny/DDBEncryption/src/TestVectors.dfy | 108 ++++++- .../src/WriteSetPermutations.dfy | 142 ++++++++++ TestVectors/runtimes/java/build.gradle.kts | 4 + TestVectors/runtimes/java/data.json | 257 ++++++++++++++++- .../ddb-attribute-serialization.md | 19 +- .../ddb-encryption-branch-key-id-supplier.md | 3 + .../ddb-item-conversion.md | 12 +- .../dynamodb-encryption-client/ddb-support.md | 3 + .../string-ordering.md | 103 +++++++ specification/structured-encryption/footer.md | 2 + specification/structured-encryption/header.md | 2 + submodules/MaterialProviders | 2 +- 23 files changed, 1043 insertions(+), 52 deletions(-) create mode 100644 TestVectors/dafny/DDBEncryption/src/Permute.dfy create mode 100644 TestVectors/dafny/DDBEncryption/src/WriteSetPermutations.dfy create mode 100644 specification/dynamodb-encryption-client/string-ordering.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 2a246fbbd..19d91bb95 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,18 @@ # Changelog +## 3.1.1 2023-11-07 + +### Fix + +Issue when a DynamoDB Set attribute is marked as SIGN_ONLY in the AWS Database Encryption SDK (DB-ESDK) for DynamoDB. + +DB-ESDK for DynamoDB supports SIGN_ONLY and ENCRYPT_AND_SIGN attribute actions. In version 3.1.0 and below, when a Set type is assigned a SIGN_ONLY attribute action, there is a chance that signature validation of the record containing a Set will fail on read, even if the Set attributes contain the same values. The probability of a failure depends on the order of the elements in the Set combined with how DynamoDB returns this data, which is undefined. + +This update addresses the issue by ensuring that any Set values are canonicalized in the same order while written to DynamoDB as when read back from DynamoDB. + +See: https://github.com/aws/aws-database-encryption-sdk-dynamodb-java/tree/v3.1.1/DecryptWithPermute for additional details for additional details + + ## 3.1.0 2023-09-07 ### Features diff --git a/DynamoDbEncryption/dafny/DynamoDbEncryption/src/ConfigToInfo.dfy b/DynamoDbEncryption/dafny/DynamoDbEncryption/src/ConfigToInfo.dfy index 4dcd2033f..69e80e25a 100644 --- a/DynamoDbEncryption/dafny/DynamoDbEncryption/src/ConfigToInfo.dfy +++ b/DynamoDbEncryption/dafny/DynamoDbEncryption/src/ConfigToInfo.dfy @@ -328,6 +328,7 @@ module SearchConfigToInfo { if |badNames| == 0 then None else + // We happen to order these values, but this ordering MUST NOT be relied upon. var badSeq := SortedSets.ComputeSetToOrderedSequence2(badNames, CharLess); Some(badSeq[0]) } @@ -399,6 +400,7 @@ module SearchConfigToInfo { if |badNames| == 0 then None else + // We happen to order these values, but this ordering MUST NOT be relied upon. var badSeq := SortedSets.ComputeSetToOrderedSequence2(badNames, CharLess); Some(badSeq[0]) } diff --git a/DynamoDbEncryption/dafny/DynamoDbEncryption/src/DDBSupport.dfy b/DynamoDbEncryption/dafny/DynamoDbEncryption/src/DDBSupport.dfy index 65a299690..c609d5ac2 100644 --- a/DynamoDbEncryption/dafny/DynamoDbEncryption/src/DDBSupport.dfy +++ b/DynamoDbEncryption/dafny/DynamoDbEncryption/src/DDBSupport.dfy @@ -44,6 +44,7 @@ module DynamoDBSupport { Success(true) else var bad := set k <- item | ReservedPrefix <= k; + // We happen to order these values, but this ordering MUST NOT be relied upon. var badSeq := SortedSets.ComputeSetToOrderedSequence2(bad, CharLess); if |badSeq| == 0 then Failure("") @@ -173,6 +174,7 @@ module DynamoDBSupport { var both := newAttrs.Keys * item.Keys; var bad := set k <- both | newAttrs[k] != item[k]; if 0 < |bad| { + // We happen to order these values, but this ordering MUST NOT be relied upon. var badSeq := SortedSets.ComputeSetToOrderedSequence2(bad, CharLess); return Failure(E("Supplied Beacons do not match calculated beacons : " + Join(badSeq, ", "))); } diff --git a/DynamoDbEncryption/dafny/DynamoDbEncryption/src/DynamoToStruct.dfy b/DynamoDbEncryption/dafny/DynamoDbEncryption/src/DynamoToStruct.dfy index 41fb54d4b..6e592d3ee 100644 --- a/DynamoDbEncryption/dafny/DynamoDbEncryption/src/DynamoToStruct.dfy +++ b/DynamoDbEncryption/dafny/DynamoDbEncryption/src/DynamoToStruct.dfy @@ -23,15 +23,8 @@ module DynamoToStruct { type StructuredDataTerminalType = x : StructuredData | x.content.Terminal? witness * type TerminalDataMap = map -//= specification/dynamodb-encryption-client/ddb-item-conversion.md#overview -//= type=TODO -//# The conversion from DDB Item to Structured Data must be lossless, -//# meaning that converting a DDB Item to -//# a Structured Data and back to a DDB Item again -//# MUST result in the exact same DDB Item. - // This file exists for these two functions : ItemToStructured and StructuredToItem - // which provide lossless conversion between an AttributeMap and a StructuredDataMap + // which provide conversion between an AttributeMap and a StructuredDataMap // Convert AttributeMap to StructuredDataMap //= specification/dynamodb-encryption-client/ddb-item-conversion.md#convert-ddb-item-to-structured-data @@ -107,6 +100,7 @@ module DynamoToStruct { else var badNames := set k <- s | !IsValid_AttributeName(k) :: k; OneBadKey(s, badNames, IsValid_AttributeName); + // We happen to order these values, but this ordering MUST NOT be relied upon. var orderedAttrNames := SetToOrderedSequence(badNames, CharLess); var attrNameList := Join(orderedAttrNames, ","); MakeError("Not valid attribute names : " + attrNameList) @@ -317,7 +311,11 @@ module DynamoToStruct { //= specification/dynamodb-encryption-client/ddb-attribute-serialization.md#number //= type=implication - //# Number MUST be serialized as UTF-8 encoded bytes. + //# This value MUST be normalized in the same way as DynamoDB normalizes numbers. + + //= specification/dynamodb-encryption-client/ddb-attribute-serialization.md#number + //= type=implication + //# This normalized value MUST then be serialized as UTF-8 encoded bytes. ensures a.N? && ret.Success? && !prefix ==> && Norm.NormalizeNumber(a.N).Success? && var nn := Norm.NormalizeNumber(a.N).value; @@ -488,30 +486,52 @@ module DynamoToStruct { function method StringSetAttrToBytes(ss: StringSetAttributeValue): (ret: Result, string>) ensures ret.Success? ==> Seq.HasNoDuplicates(ss) { - :- Need(|Seq.ToSet(ss)| == |ss|, "String Set had duplicate values"); + var asSet := Seq.ToSet(ss); + :- Need(|asSet| == |ss|, "String Set had duplicate values"); Seq.LemmaNoDuplicatesCardinalityOfSet(ss); - var count :- U32ToBigEndian(|ss|); - var body :- CollectString(ss); + + //= specification/dynamodb-encryption-client/ddb-attribute-serialization.md#set-entries + //# Entries in a String Set MUST be ordered in ascending [UTF-16 binary order](./string-ordering.md#utf-16-binary-order). + var sortedList := SortedSets.ComputeSetToOrderedSequence2(asSet, CharLess); + var count :- U32ToBigEndian(|sortedList|); + var body :- CollectString(sortedList); Success(count + body) } function method NumberSetAttrToBytes(ns: NumberSetAttributeValue): (ret: Result, string>) ensures ret.Success? ==> Seq.HasNoDuplicates(ns) { - :- Need(|Seq.ToSet(ns)| == |ns|, "Number Set had duplicate values"); + var asSet := Seq.ToSet(ns); + :- Need(|asSet| == |ns|, "Number Set had duplicate values"); Seq.LemmaNoDuplicatesCardinalityOfSet(ns); - var count :- U32ToBigEndian(|ns|); - var body :- CollectString(ns); + + var normList :- Seq.MapWithResult(n => Norm.NormalizeNumber(n), ns); + var asSet := Seq.ToSet(normList); + :- Need(|asSet| == |normList|, "Number Set had duplicate values after normalization."); + + //= specification/dynamodb-encryption-client/ddb-attribute-serialization.md#set-entries + //# Entries in a Number Set MUST be ordered in ascending [UTF-16 binary order](./string-ordering.md#utf-16-binary-order). + + //= specification/dynamodb-encryption-client/ddb-attribute-serialization.md#set-entries + //# This ordering MUST be applied after normalization of the number value. + var sortedList := SortedSets.ComputeSetToOrderedSequence2(asSet, CharLess); + var count :- U32ToBigEndian(|sortedList|); + var body :- CollectString(sortedList); Success(count + body) } function method BinarySetAttrToBytes(bs: BinarySetAttributeValue): (ret: Result, string>) ensures ret.Success? ==> Seq.HasNoDuplicates(bs) { - :- Need(|Seq.ToSet(bs)| == |bs|, "Binary Set had duplicate values"); + var asSet := Seq.ToSet(bs); + :- Need(|asSet| == |bs|, "Binary Set had duplicate values"); Seq.LemmaNoDuplicatesCardinalityOfSet(bs); - var count :- U32ToBigEndian(|bs|); - var body :- CollectBinary(bs); + + //= specification/dynamodb-encryption-client/ddb-attribute-serialization.md#set-entries + //# Entries in a Binary Set MUST be ordered lexicographically by their underlying bytes in ascending order. + var sortedList := SortedSets.ComputeSetToOrderedSequence2(asSet, ByteLess); + var count :- U32ToBigEndian(|sortedList|); + var body :- CollectBinary(sortedList); Success(count + body) } @@ -749,6 +769,9 @@ module DynamoToStruct { ensures (ret.Success? && |mapToSerialize| == 0) ==> (ret.value == serialized) ensures (ret.Success? && |mapToSerialize| == 0) ==> (|ret.value| == |serialized|) { + //= specification/dynamodb-encryption-client/ddb-attribute-serialization.md#key-value-pair-entries + //# Entries in a serialized Map MUST be ordered by key value, + //# ordered in ascending [UTF-16 binary order](./string-ordering.md#utf-16-binary-order). var keys := SortedSets.ComputeSetToOrderedSequence2(mapToSerialize.Keys, CharLess); CollectOrderedMapSubset(keys, mapToSerialize, serialized) } @@ -1136,6 +1159,7 @@ module DynamoToStruct { OneBadResult(m); var badValues := FlattenErrors(m); assert(|badValues| > 0); + // We happen to order these values, but this ordering MUST NOT be relied upon. var badValueSeq := SetToOrderedSequence(badValues, CharLess); Failure(Join(badValueSeq, "\n")) } diff --git a/DynamoDbEncryption/dafny/DynamoDbEncryption/src/SearchInfo.dfy b/DynamoDbEncryption/dafny/DynamoDbEncryption/src/SearchInfo.dfy index 700318273..bee6418a7 100644 --- a/DynamoDbEncryption/dafny/DynamoDbEncryption/src/SearchInfo.dfy +++ b/DynamoDbEncryption/dafny/DynamoDbEncryption/src/SearchInfo.dfy @@ -344,12 +344,6 @@ module SearchableEncryptionInfo { versions[currWrite].IsVirtualField(field) } - function method GenerateClosure(fields : seq) : seq - requires ValidState() - { - versions[currWrite].GenerateClosure(fields) - } - method GeneratePlainBeacons(item : DDB.AttributeMap) returns (output : Result) requires ValidState() { @@ -559,6 +553,7 @@ module SearchableEncryptionInfo { requires version == 1 requires keySource.ValidState() { + // We happen to order these values, but this ordering MUST NOT be relied upon. var beaconNames := SortedSets.ComputeSetToOrderedSequence2(beacons.Keys, CharLess); var stdKeys := Seq.Filter((k : string) => k in beacons && beacons[k].Standard?, beaconNames); FilterPreservesHasNoDuplicates((k : string) => k in beacons && beacons[k].Standard?, beaconNames); @@ -575,6 +570,7 @@ module SearchableEncryptionInfo { keySource : KeySource, virtualFields : VirtualFieldMap, beacons : BeaconMap, + // The ordering of `beaconNames` MUST NOT be relied upon. beaconNames : seq, stdNames : seq, encryptedFields : set @@ -614,13 +610,6 @@ module SearchableEncryptionInfo { [field] } - function method GenerateClosure(fields : seq) : seq - { - var fieldLists := Seq.Map((s : string) => GetFields(s), fields); - var fieldSet := set f <- fieldLists, g <- f :: g; - SortedSets.ComputeSetToOrderedSequence2(fieldSet, CharLess) - } - method getKeyMap(keyId : MaybeKeyId) returns (output : Result) requires ValidState() ensures ValidState() diff --git a/DynamoDbEncryption/dafny/DynamoDbEncryption/test/DynamoToStruct.dfy b/DynamoDbEncryption/dafny/DynamoDbEncryption/test/DynamoToStruct.dfy index c7290bb8e..ffecf051e 100644 --- a/DynamoDbEncryption/dafny/DynamoDbEncryption/test/DynamoToStruct.dfy +++ b/DynamoDbEncryption/dafny/DynamoDbEncryption/test/DynamoToStruct.dfy @@ -282,14 +282,264 @@ module DynamoToStructTest { expect newMapValue.value == mapValue; } - //= specification/dynamodb-encryption-client/ddb-item-conversion.md#overview + method {:test} TestNormalizeNAttr() { + var numberValue := AttributeValue.N("000123.000"); + var encodedNumberData := StructuredDataTerminal(value := [49,50,51], typeId := [0,2]); + var encodedNumberValue := StructuredData(content := Terminal(encodedNumberData), attributes := None); + var numberStruct := AttrToStructured(numberValue); + expect numberStruct.Success?; + expect numberStruct.value == encodedNumberValue; + + var newNumberValue := StructuredToAttr(encodedNumberValue); + expect newNumberValue.Success?; + expect newNumberValue.value == AttributeValue.N("123"); + } + + method {:test} TestNormalizeNInSet() { + var numberSetValue := AttributeValue.NS(["001.00"]); + var encodedNumberSetData := StructuredDataTerminal(value := [0,0,0,1, 0,0,0,1, 49], typeId := [1,2]); + var encodedNumberSetValue := StructuredData(content := Terminal(encodedNumberSetData), attributes := None); + var numberSetStruct := AttrToStructured(numberSetValue); + expect numberSetStruct.Success?; + expect numberSetStruct.value == encodedNumberSetValue; + + var newNumberSetValue := StructuredToAttr(encodedNumberSetValue); + expect newNumberSetValue.Success?; + expect newNumberSetValue.value == AttributeValue.NS(["1"]); + } + + method {:test} TestNormalizeNInList() { + var nValue := AttributeValue.N("001.00"); + var normalizedNValue := AttributeValue.N("1"); + + var listValue := AttributeValue.L([nValue]); + var encodedListData := StructuredDataTerminal(value := [ + 0,0,0,1, // 1 member in list + 0,2, 0,0,0,1, 49 // 1st member is N("1") + ], + typeId := [3,0]); + var encodedListValue := StructuredData(content := Terminal(encodedListData), attributes := None); + var listStruct := AttrToStructured(listValue); + expect listStruct.Success?; + expect listStruct.value == encodedListValue; + + var newListValue := StructuredToAttr(listStruct.value); + expect newListValue.Success?; + expect newListValue.value == AttributeValue.L([normalizedNValue]); + } + + method {:test} TestNormalizeNInMap() { + var nValue := AttributeValue.N("001.00"); + var normalizedNValue := AttributeValue.N("1"); + + var mapValue := AttributeValue.M(map["keyA" := nValue]); + var k := 'k' as uint8; + var e := 'e' as uint8; + var y := 'y' as uint8; + var A := 'A' as uint8; + + var encodedMapData := StructuredDataTerminal( + value := [ + 0,0,0,1, // there is 1 entry in the map + 0,1, 0,0,0,4, k,e,y,A, // 1st entry's key + 0,2, 0,0,0,1, // 1st entry's value is a N and is 1 byte long + 49 // "1" + ], + typeId := [2,0]); + + var encodedMapValue := StructuredData(content := Terminal(encodedMapData), attributes := None); + var mapStruct := AttrToStructured(mapValue); + expect mapStruct.Success?; + expect mapStruct.value == encodedMapValue; + + var newMapValue := StructuredToAttr(mapStruct.value); + expect newMapValue.Success?; + expect newMapValue.value == AttributeValue.M(map["keyA" := normalizedNValue]); + } + + //= specification/dynamodb-encryption-client/ddb-attribute-serialization.md#set-entries //= type=test - //# The conversion from DDB Item to Structured Data must be lossless, - //# meaning that converting a DDB Item to - //# a Structured Data and back to a DDB Item again - //# MUST result in the exact same DDB Item. - method {:test} TestRoundTrip() { + //# Entries in a Number Set MUST be ordered in ascending [UTF-16 binary order](./string-ordering.md#utf-16-binary-order). + method {:test} TestSortNSAttr() { + var numberSetValue := AttributeValue.NS(["1","2","10"]); + var encodedNumberSetData := StructuredDataTerminal(value := [0,0,0,3, 0,0,0,1, 49, 0,0,0,2, 49,48, 0,0,0,1, 50], typeId := [1,2]); + var encodedNumberSetValue := StructuredData(content := Terminal(encodedNumberSetData), attributes := None); + var numberSetStruct := AttrToStructured(numberSetValue); + expect numberSetStruct.Success?; + expect numberSetStruct.value == encodedNumberSetValue; + + var newNumberSetValue := StructuredToAttr(encodedNumberSetValue); + expect newNumberSetValue.Success?; + expect newNumberSetValue.value == AttributeValue.NS(["1","10","2"]); + } + + //= specification/dynamodb-encryption-client/ddb-attribute-serialization.md#set-entries + //= type=test + //# This ordering MUST be applied after normalization of the number value. + method {:test} TestSortNSAfterNormalize() { + var numberSetValue := AttributeValue.NS(["1","02","10"]); + var encodedNumberSetData := StructuredDataTerminal(value := [0,0,0,3, 0,0,0,1, 49, 0,0,0,2, 49,48, 0,0,0,1, 50], typeId := [1,2]); + var encodedNumberSetValue := StructuredData(content := Terminal(encodedNumberSetData), attributes := None); + var numberSetStruct := AttrToStructured(numberSetValue); + expect numberSetStruct.Success?; + expect numberSetStruct.value == encodedNumberSetValue; + + var newNumberSetValue := StructuredToAttr(encodedNumberSetValue); + expect newNumberSetValue.Success?; + expect newNumberSetValue.value == AttributeValue.NS(["1","10","2"]); + } + + //= specification/dynamodb-encryption-client/ddb-attribute-serialization.md#set-entries + //= type=test + //# Entries in a String Set MUST be ordered in ascending [UTF-16 binary order](./string-ordering.md#utf-16-binary-order). + method {:test} TestSortSSAttr() { + var stringSetValue := AttributeValue.SS(["&","｡","𐀂"]); + // Note that string values are UTF-8 encoded, but sorted by UTF-16 encoding. + var encodedStringSetData := StructuredDataTerminal(value := [ + 0,0,0,3, // 3 entries in set + 0,0,0,1, // 1st entry is 1 byte + 0x26, // "&" in UTF-8 encoding + 0,0,0,4, // 2nd entry is 4 bytes + 0xF0,0x90,0x80,0x82, // "𐀂" in UTF-8 encoding + 0,0,0,3, // 3rd entry is 3 bytes + 0xEF,0xBD,0xA1 // "｡" in UTF-8 encoding + ], + typeId := [1,1] + ); + var encodedStringSetValue := StructuredData(content := Terminal(encodedStringSetData), attributes := None); + var stringSetStruct := AttrToStructured(stringSetValue); + expect stringSetStruct.Success?; + expect stringSetStruct.value == encodedStringSetValue; + + var newStringSetValue := StructuredToAttr(encodedStringSetValue); + expect newStringSetValue.Success?; + expect newStringSetValue.value == AttributeValue.SS(["&","𐀂","｡"]); + } + + //= specification/dynamodb-encryption-client/ddb-attribute-serialization.md#set-entries + //= type=test + //# Entries in a Binary Set MUST be ordered lexicographically by their underlying bytes in ascending order. + method {:test} TestSortBSAttr() { + var binarySetValue := AttributeValue.BS([[1],[2],[1,0]]); + var encodedBinarySetData := StructuredDataTerminal(value := [0,0,0,3, 0,0,0,1, 1, 0,0,0,2, 1,0, 0,0,0,1, 2], typeId := [1,0xff]); + var encodedBinarySetValue := StructuredData(content := Terminal(encodedBinarySetData), attributes := None); + var binarySetStruct := AttrToStructured(binarySetValue); + expect binarySetStruct.Success?; + expect binarySetStruct.value == encodedBinarySetValue; + + var newBinarySetValue := StructuredToAttr(encodedBinarySetValue); + expect newBinarySetValue.Success?; + expect newBinarySetValue.value == AttributeValue.BS([[1],[1,0],[2]]); + } + + method {:test} TestSetsInListAreSorted() { + var nSetValue := AttributeValue.NS(["2","1","10"]); + var sSetValue := AttributeValue.SS(["&","｡","𐀂"]); + var bSetValue := AttributeValue.BS([[1,0],[1],[2]]); + + var sortedNSetValue := AttributeValue.NS(["1","10","2"]); + var sortedSSetValue := AttributeValue.SS(["&","𐀂","｡"]); + var sortedBSetValue := AttributeValue.BS([[1],[1,0],[2]]); + + var listValue := AttributeValue.L([nSetValue, sSetValue, bSetValue]); + var encodedListData := StructuredDataTerminal(value := [ + 0,0,0,3, // 3 members in list + 1,2, 0,0,0,20, // 1st member is a NS and is 20 bytes long + 0,0,0,3, 0,0,0,1, 49, 0,0,0,2, 49,48, 0,0,0,1, 50, // NS + 1,1, 0,0,0,24, // 2nd member is a SS and is 24 bytes long + 0,0,0,3, 0,0,0,1, 0x26, 0,0,0,4, 0xF0,0x90,0x80,0x82, 0,0,0,3, 0xEF,0xBD,0xA1, // SS + 1,0xFF, 0,0,0,20, // 3rd member is a BS and is 20 bytes long + 0,0,0,3, 0,0,0,1, 1, 0,0,0,2, 1,0, 0,0,0,1, 2 // BS + ], + typeId := [3,0]); + var encodedListValue := StructuredData(content := Terminal(encodedListData), attributes := None); + var listStruct := AttrToStructured(listValue); + expect listStruct.Success?; + expect listStruct.value == encodedListValue; + + var newListValue := StructuredToAttr(listStruct.value); + expect newListValue.Success?; + expect newListValue.value == AttributeValue.L([sortedNSetValue, sortedSSetValue, sortedBSetValue]); + } + + method {:test} TestSetsInMapAreSorted() { + var nSetValue := AttributeValue.NS(["2","1","10"]); + var sSetValue := AttributeValue.SS(["&","｡","𐀂"]); + var bSetValue := AttributeValue.BS([[1,0],[1],[2]]); + var sortedNSetValue := AttributeValue.NS(["1","10","2"]); + var sortedSSetValue := AttributeValue.SS(["&","𐀂","｡"]); + var sortedBSetValue := AttributeValue.BS([[1],[1,0],[2]]); + + var mapValue := AttributeValue.M(map["keyA" := sSetValue, "keyB" := nSetValue, "keyC" := bSetValue]); + var k := 'k' as uint8; + var e := 'e' as uint8; + var y := 'y' as uint8; + var A := 'A' as uint8; + var B := 'B' as uint8; + var C := 'C' as uint8; + + var encodedMapData := StructuredDataTerminal( + value := [ + 0,0,0,3, // there are 3 entries in the map + 0,1, 0,0,0,4, k,e,y,A, // 1st entry's key + 1,1, 0,0,0,24, // 1st entry's value is a SS and is 24 bytes long + 0,0,0,3, 0,0,0,1, 0x26, 0,0,0,4, 0xF0,0x90,0x80,0x82, 0,0,0,3, 0xEF,0xBD,0xA1, // SS + 0,1, 0,0,0,4, k,e,y,B, // 2nd entry's key + 1,2, 0,0,0,20, // 2nd entry's value is a NS and is 20 bytes long + 0,0,0,3, 0,0,0,1, 49, 0,0,0,2, 49,48, 0,0,0,1, 50, // NS + 0,1, 0,0,0,4, k,e,y,C, // 3rd entry's key + 1,0xFF, 0,0,0,20, // 3rd entry's value is a BS and is 20 bytes long + 0,0,0,3, 0,0,0,1, 1, 0,0,0,2, 1,0, 0,0,0,1, 2 // BS + ], + typeId := [2,0]); + + var encodedMapValue := StructuredData(content := Terminal(encodedMapData), attributes := None); + var mapStruct := AttrToStructured(mapValue); + expect mapStruct.Success?; + expect mapStruct.value == encodedMapValue; + + var newMapValue := StructuredToAttr(mapStruct.value); + expect newMapValue.Success?; + expect newMapValue.value == AttributeValue.M(map["keyA" := sortedSSetValue, "keyB" := sortedNSetValue, "keyC" := sortedBSetValue]); + } + + //= specification/dynamodb-encryption-client/ddb-attribute-serialization.md#key-value-pair-entries + //= type=test + //# Entries in a serialized Map MUST be ordered by key value, + //# ordered in ascending [UTF-16 binary order](./string-ordering.md#utf-16-binary-order). + method {:test} TestSortMapKeys() { + var nullValue := AttributeValue.NULL(true); + + var mapValue := AttributeValue.M(map["&" := nullValue, "｡" := nullValue, "𐀂" := nullValue]); + + // Note that the string values are encoded as UTF-8, but are sorted according to UTF-16 encoding. + var encodedMapData := StructuredDataTerminal( + value := [ + 0,0,0,3, // 3 entries + 0,1, 0,0,0,1, // 1st key is a string 1 byte long + 0x26, // "&" UTF-8 encoded + 0,0, 0,0,0,0, // null value + 0,1, 0,0,0,4, // 2nd key is a string 4 bytes long + 0xF0, 0x90, 0x80, 0x82, // "𐀂" UTF-8 encoded + 0,0, 0,0,0,0, // null value + 0,1, 0,0,0,3, // 3rd key is a string 3 bytes long + 0xEF, 0xBD, 0xA1, // "｡" + 0,0, 0,0,0,0 // null value + ], + typeId := [2,0]); + var encodedMapValue := StructuredData(content := Terminal(encodedMapData), attributes := None); + var mapStruct := AttrToStructured(mapValue); + expect mapStruct.Success?; + expect mapStruct.value == encodedMapValue; + + var newMapValue := StructuredToAttr(mapStruct.value); + expect newMapValue.Success?; + expect newMapValue.value == mapValue; + } + + method {:test} TestRoundTrip() { + // Note - set and number values are carefully pre-normalized. var val1 := AttributeValue.S("astring"); var val2 := AttributeValue.N("12345"); var val3 := AttributeValue.B([1,2,3,4,5]); @@ -297,7 +547,7 @@ module DynamoToStructTest { var val5 := AttributeValue.NULL(true); var val6 := AttributeValue.BS([[1,2,3,4,5],[2,3,4,5,6],[3,4,5,6,7]]); var val7 := AttributeValue.SS(["ab","cdef","ghijk"]); - var val8 := AttributeValue.NS(["1","234.567","0"]); + var val8 := AttributeValue.NS(["0", "1","234.567"]); var val9a := AttributeValue.L([val8, val7, val6]); var val9b := AttributeValue.L([val5, val4, val3]); diff --git a/DynamoDbEncryption/dafny/DynamoDbItemEncryptor/src/AwsCryptographyDbEncryptionSdkDynamoDbItemEncryptorOperations.dfy b/DynamoDbEncryption/dafny/DynamoDbItemEncryptor/src/AwsCryptographyDbEncryptionSdkDynamoDbItemEncryptorOperations.dfy index 7d56fd9cd..5ea299388 100644 --- a/DynamoDbEncryption/dafny/DynamoDbItemEncryptor/src/AwsCryptographyDbEncryptionSdkDynamoDbItemEncryptorOperations.dfy +++ b/DynamoDbEncryption/dafny/DynamoDbItemEncryptor/src/AwsCryptographyDbEncryptionSdkDynamoDbItemEncryptorOperations.dfy @@ -525,6 +525,7 @@ module AwsCryptographyDbEncryptionSdkDynamoDbItemEncryptorOperations refines Abs function method GetItemNames(item : ComAmazonawsDynamodbTypes.AttributeMap) : string { + // We happen to order these values, but this ordering MUST NOT be relied upon. var keys := SortedSets.ComputeSetToOrderedSequence2(item.Keys, CharLess); if |keys| == 0 then "item is empty" diff --git a/DynamoDbEncryption/dafny/DynamoDbItemEncryptor/src/Index.dfy b/DynamoDbEncryption/dafny/DynamoDbItemEncryptor/src/Index.dfy index d226f16ac..9f88ab83e 100644 --- a/DynamoDbEncryption/dafny/DynamoDbItemEncryptor/src/Index.dfy +++ b/DynamoDbEncryption/dafny/DynamoDbItemEncryptor/src/Index.dfy @@ -93,6 +93,7 @@ module message := "Sort key attribute action MUST be SIGN_ONLY" )); + // We happen to order these values, but this ordering MUST NOT be relied upon. var attributeNames : seq := SortedSets.ComputeSetToOrderedSequence2(config.attributeActionsOnEncrypt.Keys, CharLess); for i := 0 to |attributeNames| invariant forall j | 0 <= j < i :: diff --git a/README.md b/README.md index cfbd48588..5441851b3 100644 --- a/README.md +++ b/README.md @@ -59,7 +59,7 @@ To use the DB-ESDK for DynamoDB in Java, you must have: * **Via Gradle Kotlin** In a Gradle Java Project, add the following to the _dependencies_ section: ```kotlin - implementation("software.amazon.cryptography:aws-database-encryption-sdk-dynamodb:3.1.0") + implementation("software.amazon.cryptography:aws-database-encryption-sdk-dynamodb:3.1.1") implementation("software.amazon.cryptography:aws-cryptographic-material-providers:1.0.0") implementation(platform("software.amazon.awssdk:bom:2.19.1")) implementation("software.amazon.awssdk:dynamodb") @@ -92,7 +92,7 @@ To use the DB-ESDK for DynamoDB in Java, you must have: software.amazon.cryptography aws-database-encryption-sdk-dynamodb - 3.1.0 + 3.1.1 software.amazon.cryptography diff --git a/TestVectors/dafny/DDBEncryption/src/Index.dfy b/TestVectors/dafny/DDBEncryption/src/Index.dfy index a982f81c1..8a67f6039 100644 --- a/TestVectors/dafny/DDBEncryption/src/Index.dfy +++ b/TestVectors/dafny/DDBEncryption/src/Index.dfy @@ -3,11 +3,13 @@ include "LibraryIndex.dfy" include "TestVectors.dfy" +include "WriteSetPermutations.dfy" module WrappedDDBEncryptionMain { import opened Wrappers import opened DdbEncryptionTestVectors + import WriteSetPermutations import CreateInterceptedDDBClient import FileIO import JSON.API @@ -25,11 +27,13 @@ module WrappedDDBEncryptionMain { } method ASDF() { + WriteSetPermutations.WriteSetPermutations(); var config := MakeEmptyTestVector(); config :- expect AddJson(config, "records.json"); config :- expect AddJson(config, "configs.json"); config :- expect AddJson(config, "data.json"); config :- expect AddJson(config, "iotest.json"); + config :- expect AddJson(config, "PermTest.json"); config.RunAllTests(); } } diff --git a/TestVectors/dafny/DDBEncryption/src/Permute.dfy b/TestVectors/dafny/DDBEncryption/src/Permute.dfy new file mode 100644 index 000000000..1c030a2f3 --- /dev/null +++ b/TestVectors/dafny/DDBEncryption/src/Permute.dfy @@ -0,0 +1,72 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +include "../../../../submodules/MaterialProviders/libraries/src/Wrappers.dfy" + +module {:options "-functionSyntax:4"} Permutations { + import opened Wrappers + + method GeneratePermutations(source : seq) returns (result : seq>) + { + if |source| == 0 { + return []; + } + if |source| == 1 { + return [source]; + } + var A := new T[|source|](i requires 0 <= i < |source| => source[i]); + result := Permute(A.Length, A); + } + + method Swap(A : array, x : nat, y : nat) + requires 0 <= x < A.Length + requires 0 <= y < A.Length + modifies A + { + var tmp := A[x]; + A[x] := A[y]; + A[y] := tmp; + } + + // https://en.wikipedia.org/wiki/Heap%27s_algorithm + // Each step generates the k! permutations that end with the same n-k final elements + method Permute(k : nat, A : array) returns (result : seq>) + requires 0 < k <= A.Length + modifies A + { + if k == 1 { + return [A[..]]; + } else { + var result : seq> := []; + for i := 0 to k { + var next := Permute(k - 1, A); + result := result + next; + if (k % 1) == 0 { + Swap(A, i, k-1); + } else { + Swap(A, 0, k-1); + } + } + return result; + } + } + + method {:test} BasicTests() { + var zero := GeneratePermutations([]); + var one := GeneratePermutations([1]); + var two := GeneratePermutations([1,2]); + var three := GeneratePermutations([1,2,3]); + var four := GeneratePermutations([1,2,3,4]); + expect zero == []; + expect one == [[1]]; + expect two == [[1,2],[2,1]]; + expect three == [[1, 2, 3], [2, 1, 3], [3, 1, 2], [1, 3, 2], [1, 2, 3], [2, 1, 3]]; + expect four == [ + [1, 2, 3, 4], [2, 1, 3, 4], [3, 1, 2, 4], [1, 3, 2, 4], [1, 2, 3, 4], [2, 1, 3, 4], + [4, 1, 3, 2], [1, 4, 3, 2], [3, 4, 1, 2], [4, 3, 1, 2], [4, 1, 3, 2], [1, 4, 3, 2], + [1, 2, 3, 4], [2, 1, 3, 4], [3, 1, 2, 4], [1, 3, 2, 4], [1, 2, 3, 4], [2, 1, 3, 4], + [2, 1, 4, 3], [1, 2, 4, 3], [4, 2, 1, 3], [2, 4, 1, 3], [2, 1, 4, 3], [1, 2, 4, 3] + ]; + } +} + diff --git a/TestVectors/dafny/DDBEncryption/src/TestVectors.dfy b/TestVectors/dafny/DDBEncryption/src/TestVectors.dfy index 1bf899e93..abd94d234 100644 --- a/TestVectors/dafny/DDBEncryption/src/TestVectors.dfy +++ b/TestVectors/dafny/DDBEncryption/src/TestVectors.dfy @@ -39,6 +39,8 @@ module {:options "-functionSyntax:4"} DdbEncryptionTestVectors { import KeyVectorsTypes = AwsCryptographyMaterialProvidersTestVectorKeysTypes import KeyVectors import CreateInterceptedDDBClient + import SortedSets + import Seq predicate IsValidInt32(x: int) { -0x8000_0000 <= x < 0x8000_0000} type ConfigName = string @@ -73,6 +75,11 @@ module {:options "-functionSyntax:4"} DdbEncryptionTestVectors { failures : seq ) + datatype RoundTripTest = RoundTripTest ( + configs : map, + records : seq + ) + datatype WriteTest = WriteTest ( config : TableConfig, records : seq, @@ -113,6 +120,7 @@ module {:options "-functionSyntax:4"} DdbEncryptionTestVectors { configsForIoTest : PairList, configsForModTest : PairList, writeTests : seq, + roundTripTests : seq, decryptTests : seq ) { @@ -129,6 +137,12 @@ module {:options "-functionSyntax:4"} DdbEncryptionTestVectors { print |ioTests|, " ioTests.\n"; print |configsForIoTest|, " configsForIoTest.\n"; print |configsForModTest|, " configsForModTest.\n"; + if |roundTripTests| != 0 { + print |roundTripTests[0].configs|, " configs and ", |roundTripTests[0].records|, " records for round trip.\n"; + } + if |roundTripTests| > 1 { + print |roundTripTests[1].configs|, " configs and ", |roundTripTests[1].records|, " records for round trip.\n"; + } Validate(); BasicIoTest(); RunIoTests(); @@ -136,11 +150,13 @@ module {:options "-functionSyntax:4"} DdbEncryptionTestVectors { ConfigModTest(); ComplexTests(); WriteTests(); + RoundTripTests(); DecryptTests(); var client :- expect CreateInterceptedDDBClient.CreateVanillaDDBClient(); DeleteTable(client); } + method Validate() { var bad := false; for i := 0 to |globalRecords| { @@ -496,6 +512,57 @@ module {:options "-functionSyntax:4"} DdbEncryptionTestVectors { } } + method RoundTripTests() + { + print "RoundTripTests\n"; + for i := 0 to |roundTripTests| { + + var configs := roundTripTests[i].configs; + var records := roundTripTests[i].records; + var keys := SortedSets.ComputeSetToOrderedSequence2(configs.Keys, CharLess); + + for j := 0 to |keys| { + var client :- expect newGazelle(configs[keys[j]]); + for k := 0 to |records| { + OneRoundTripTest(client, records[k]); + } + } + } + } + + method OneRoundTripTest(client : DDB.IDynamoDBClient, record : Record) { + var putInput := DDB.PutItemInput( + TableName := TableName, + Item := record.item, + Expected := None, + ReturnValues := None, + ReturnConsumedCapacity := None, + ReturnItemCollectionMetrics := None, + ConditionalOperator := None, + ConditionExpression := None, + ExpressionAttributeNames := None, + ExpressionAttributeValues := None + ); + var _ :- expect client.PutItem(putInput); + + var getInput := DDB.GetItemInput( + TableName := TableName, + Key := map[HashName := record.item[HashName]], + AttributesToGet := None, + ConsistentRead := None, + ReturnConsumedCapacity := None, + ProjectionExpression := None, + ExpressionAttributeNames := None + ); + var out :- expect client.GetItem(getInput); + expect out.Item.Some?; + if NormalizeItem(out.Item.value) != NormalizeItem(record.item) { + print "\n", NormalizeItem(out.Item.value), "\n", NormalizeItem(record.item), "\n"; + } + expect NormalizeItem(out.Item.value) == NormalizeItem(record.item); + } + + method DecryptTests() { print "DecryptTests\n"; @@ -685,7 +752,7 @@ module {:options "-functionSyntax:4"} DdbEncryptionTestVectors { { var exp := NormalizeItem(expected); for i := 0 to |actual| { - if actual[i] == exp { + if NormalizeItem(actual[i]) == exp { return true; } } @@ -781,6 +848,23 @@ module {:options "-functionSyntax:4"} DdbEncryptionTestVectors { DDB.AttributeValue.N(nn.value) else value + case SS(s) => + var asSet := Seq.ToSet(s); + DDB.AttributeValue.SS(SortedSets.ComputeSetToOrderedSequence2(asSet, CharLess)) + case NS(s) => + var normList := Seq.MapWithResult(n => Norm.NormalizeNumber(n), s); + if normList.Success? then + var asSet := Seq.ToSet(normList.value); + DDB.AttributeValue.NS(SortedSets.ComputeSetToOrderedSequence2(asSet, CharLess)) + else + value + case BS(s) => + var asSet := Seq.ToSet(s); + DDB.AttributeValue.BS(SortedSets.ComputeSetToOrderedSequence2(asSet, ByteLess)) + case L(list) => + DDB.AttributeValue.L(Seq.Map(n => Normalize(n), list)) + case M(m) => + DDB.AttributeValue.M(map k <- m :: k := Normalize(m[k])) case _ => value } } @@ -962,7 +1046,7 @@ module {:options "-functionSyntax:4"} DdbEncryptionTestVectors { function MakeEmptyTestVector() : TestVectorConfig { - TestVectorConfig(MakeCreateTableInput(), [], map[], [], map[], map[], [], [], [], [], [], [], []) + TestVectorConfig(MakeCreateTableInput(), [], map[], [], map[], map[], [], [], [], [], [], [], [], []) } method ParseTestVector(data : JSON, prev : TestVectorConfig) returns (output : Result) @@ -980,6 +1064,7 @@ module {:options "-functionSyntax:4"} DdbEncryptionTestVectors { var gsi : seq := []; var tableEncryptionConfigs : map := map[]; var writeTests : seq := []; + var roundTripTests : seq := []; var decryptTests : seq := []; for i := 0 to |data.obj| { @@ -996,6 +1081,7 @@ module {:options "-functionSyntax:4"} DdbEncryptionTestVectors { case "GSI" => gsi :- GetGSIs(data.obj[i].1); case "tableEncryptionConfigs" => tableEncryptionConfigs :- GetTableConfigs(data.obj[i].1); case "WriteTests" => writeTests :- GetWriteTests(data.obj[i].1); + case "RoundTripTest" => roundTripTests :- GetRoundTripTests(data.obj[i].1); case "DecryptTests" => decryptTests :- GetDecryptTests(data.obj[i].1); case _ => return Failure("Unexpected top level tag " + data.obj[i].0); } @@ -1016,11 +1102,29 @@ module {:options "-functionSyntax:4"} DdbEncryptionTestVectors { configsForIoTest := prev.configsForIoTest + ioPairs, configsForModTest := prev.configsForModTest + queryPairs, writeTests := prev.writeTests + writeTests, + roundTripTests := prev.roundTripTests + roundTripTests, decryptTests := prev.decryptTests + decryptTests ) ); } + method GetRoundTripTests(data : JSON) returns (output : Result, string>) + { + :- Need(data.Object?, "RoundTripTest Test must be an object."); + var configs : map := map[]; + var records : seq := []; + + for i := 0 to |data.obj| { + var obj := data.obj[i]; + match obj.0 { + case "Configs" => var src :- GetTableConfigs(obj.1); configs := src; + case "Records" => var src :- GetRecords(obj.1); records := src; + case _ => return Failure("Unexpected part of a write test : '" + obj.0 + "'"); + } + } + return Success([RoundTripTest(configs, records)]); + } + method GetWriteTests(data : JSON) returns (output : Result , string>) { :- Need(data.Array?, "Write Test list must be an array."); diff --git a/TestVectors/dafny/DDBEncryption/src/WriteSetPermutations.dfy b/TestVectors/dafny/DDBEncryption/src/WriteSetPermutations.dfy new file mode 100644 index 000000000..e3e3b313d --- /dev/null +++ b/TestVectors/dafny/DDBEncryption/src/WriteSetPermutations.dfy @@ -0,0 +1,142 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +include "../Model/AwsCryptographyDynamoDbEncryptionTypesWrapped.dfy" +include "CreateInterceptedDDBClient.dfy" +include "JsonItem.dfy" +include "Permute.dfy" + +module {:options "-functionSyntax:4"} WriteSetPermutations { + import opened JSON.Values + import BoundedInts + import JSON.API + import FileIO + import opened StandardLibrary.String + import opened Permutations + import Base64 + + type Bytes = seq + type BytesList = seq + type StringList = seq + + function GetConfigs() : JSON + { + Object([("AllSign", + Object([("attributeActionsOnEncrypt", + Object([ + ("RecNum", String("SIGN_ONLY")), + ("StringSet", String("SIGN_ONLY")), + ("NumberSet", String("SIGN_ONLY")), + ("BinarySet", String("SIGN_ONLY")) + ]) + )])), + ("AllEncrypt", + Object([("attributeActionsOnEncrypt", + Object([ + ("RecNum", String("SIGN_ONLY")), + ("StringSet", String("ENCRYPT_AND_SIGN")), + ("NumberSet", String("ENCRYPT_AND_SIGN")), + ("BinarySet", String("ENCRYPT_AND_SIGN")) + ]) + )]) + )]) + } + + function {:tailrecursion} GetStringArray(str : StringList, acc : seq := []) : JSON + { + if |str| == 0 then + Array(acc) + else + GetStringArray(str[1..], acc + [String(str[0])]) + } + + function {:tailrecursion} EncodeStrings(bytes : BytesList, acc : seq := []) : seq + { + if |bytes| == 0 then + acc + else + EncodeStrings(bytes[1..], acc + [Base64.Encode(bytes[0])]) + } + + function GetBinaryArray(bytes : BytesList) : JSON + { + var strs := EncodeStrings(bytes); + GetStringArray(strs) + } + + + function {:opaque} GetRecord(recNum : int, str : StringList, num : StringList, bytes : BytesList) : JSON + { + var numStr := Base10Int2String(recNum); + Object([ + ("RecNum", Object([("N", String(numStr))])), + ("StringSet", Object([("SS", GetStringArray(str))])), + ("NumberSet", Object([("NS", GetStringArray(num))])), + ("BinarySet", Object([("BS", GetBinaryArray(bytes))])) + ]) + } + + + function {:opaque} {:tailrecursion} GetRecords2( + recNum : int, + str : seq, + num : seq, + bytes : seq, + acc : seq := []) : seq + decreases str + decreases num + decreases bytes + { + if |str| == 0 || |num| == 0 || |bytes| == 0 then + acc + else + var newRec := GetRecord(recNum, str[0], num[0], bytes[0]); + GetRecords2(recNum+1, str[1..], num[1..], bytes[1..], acc + [newRec]) + } + + method GetRecords() returns (result : seq) + { + var recs : seq := []; + var recs1 := [GetRecord(200, ["aaa"], ["111"], [[1,2,3]])]; + + var p2s := GeneratePermutations(["aaa", "bbb"]); + var p2n := GeneratePermutations(["111", "222"]); + var p2b := GeneratePermutations([[1,2,3], [2,3,4]]); + var recs2 := GetRecords2(201, p2s, p2n, p2b); + + var p3s := GeneratePermutations(["aaa", "bbb", "ccc"]); + var p3n := GeneratePermutations(["111", "222", "333"]); + var p3b := GeneratePermutations([[1,2,3], [2,3,4], [3,4,5]]); + var recs3 := GetRecords2(203, p3s, p3n, p3b); + + var p4s := GeneratePermutations(["aaa", "bbb", "ccc", "ddd"]); + var p4n := GeneratePermutations(["111", "222", "333", "444"]); + var p4b := GeneratePermutations([[1,2,3], [2,3,4], [3,4,5], [4,5,6]]); + var recs4 := GetRecords2(209, p4s, p4n, p4b); + + return recs1 + recs2 + recs3 + recs4; + } + + function BytesBv(bits: seq): seq + { + seq(|bits|, i requires 0 <= i < |bits| => bits[i] as bv8) + } + + method WriteSetPermutations() + { + var configs := GetConfigs(); + var records := GetRecords(); + var whole := Object([("RoundTripTest", Object([ + ("Records", Array(records)), + ("Configs", configs) + ]))]); + + var jsonBytes :- expect API.Serialize(whole); + var jsonBv := BytesBv(jsonBytes); + + var _ :- expect FileIO.WriteBytesToFile( + "PermTest.json", + jsonBv + ); + } +} diff --git a/TestVectors/runtimes/java/build.gradle.kts b/TestVectors/runtimes/java/build.gradle.kts index e26dfab41..f0e5d15d7 100644 --- a/TestVectors/runtimes/java/build.gradle.kts +++ b/TestVectors/runtimes/java/build.gradle.kts @@ -1,3 +1,6 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + import java.io.File import java.io.FileInputStream import java.util.Properties @@ -64,6 +67,7 @@ repositories { name = "DynamoDB Local Release Repository - US West (Oregon) Region" url = URI.create("https://s3-us-west-2.amazonaws.com/dynamodb-local/release") } + mavenLocal() mavenCentral() if (caUrl != null && caPassword != null) { maven { diff --git a/TestVectors/runtimes/java/data.json b/TestVectors/runtimes/java/data.json index 4bbc334ad..24fe5fc60 100644 --- a/TestVectors/runtimes/java/data.json +++ b/TestVectors/runtimes/java/data.json @@ -1,4 +1,251 @@ { + "RoundTripTest": { + "Configs": { + "AllSign": { + "attributeActionsOnEncrypt": { + "RecNum": "SIGN_ONLY", + "String": "SIGN_ONLY", + "Number": "SIGN_ONLY", + "Bytes": "SIGN_ONLY", + "StringSet": "SIGN_ONLY", + "NumberSet": "SIGN_ONLY", + "BinarySet": "SIGN_ONLY", + "Map": "SIGN_ONLY", + "List": "SIGN_ONLY", + "Null": "SIGN_ONLY", + "Bool": "SIGN_ONLY" + } + }, + "AllNothing": { + "attributeActionsOnEncrypt": { + "RecNum": "SIGN_ONLY", + "String": "DO_NOTHING", + "Number": "DO_NOTHING", + "Bytes": "DO_NOTHING", + "StringSet": "DO_NOTHING", + "NumberSet": "DO_NOTHING", + "BinarySet": "DO_NOTHING", + "Map": "DO_NOTHING", + "List": "DO_NOTHING", + "Null": "DO_NOTHING", + "Bool": "DO_NOTHING" + }, + "allowedUnsignedAttributes": [ + "String", + "Number", + "Bytes", + "StringSet", + "NumberSet", + "BinarySet", + "Map", + "List", + "Null", + "Bool" + ] + }, + "AllEncrypt": { + "attributeActionsOnEncrypt": { + "RecNum": "SIGN_ONLY", + "String": "ENCRYPT_AND_SIGN", + "Number": "ENCRYPT_AND_SIGN", + "Bytes": "ENCRYPT_AND_SIGN", + "StringSet": "ENCRYPT_AND_SIGN", + "NumberSet": "ENCRYPT_AND_SIGN", + "BinarySet": "ENCRYPT_AND_SIGN", + "Map": "ENCRYPT_AND_SIGN", + "List": "ENCRYPT_AND_SIGN", + "Null": "ENCRYPT_AND_SIGN", + "Bool": "ENCRYPT_AND_SIGN" + } + } + }, + "Records": [ + { + "RecNum": { + "N": "100" + }, + "StringSet": { + "SS": [ + "aaa", + "bbb" + ] + }, + "NumberSet": { + "NS": [ + "1.0", + "2.0" + ] + }, + "BinarySet": { + "BS": [ + "b25l", + "dHdv" + ] + } + }, + { + "RecNum": { + "N": "101" + }, + "StringSet": { + "SS": [ + "bbb", + "aaa" + ] + }, + "NumberSet": { + "NS": [ + "2.0", + "1.0" + ] + }, + "BinarySet": { + "BS": [ + "dHdv", + "b25l" + ] + }, + "Bool": { + "BOOL": true + } + }, + { + "RecNum": { + "N": "102" + }, + "StringSet": { + "SS": [ + "ddd", + "bbb", + "aaa", + "ccc", + "eee" + ] + }, + "NumberSet": { + "NS": [ + "2.0", + "1.0", + "-000.000", + "+123.456", + "123.456e5", + ".99999999999999999999999999999999999999E+126", + "1234567890123456789012345678901234567800000000000000000000000000000" + ] + }, + "BinarySet": { + "BS": [ + "dHdv", + "b25l" + ] + }, + "Bool": { + "BOOL": true + }, + "Map": { + "M": { + "eee": { + "SS": [ + "ddd", + "bbb", + "aaa", + "ccc", + "eee" + ] + }, + "aaa": { + "NS": [ + "2.0", + "1.0", + "-000.000", + "+123.456", + "123.456e5", + ".99999999999999999999999999999999999999E+126", + "1234567890123456789012345678901234567800000000000000000000000000000" + ] + }, + "ccc": { + "BS": [ + "dHdv", + "b25l" + ] + } + } + }, + "List": { + "L": [ + { + "SS": [ + "ddd", + "bbb", + "aaa", + "ccc", + "eee" + ] + }, + { + "NS": [ + "2.0", + "1.0", + "-000.000", + "+123.456", + "123.456e5", + ".99999999999999999999999999999999999999E+126", + "1234567890123456789012345678901234567800000000000000000000000000000" + ] + }, + { + "BS": [ + "dHdv", + "b25l" + ] + } + ] + } + }, + { + "RecNum": { + "N": "103" + }, + "StringSet": { + "SS": [ + "aaa" + ] + }, + "NumberSet": { + "NS": [ + "1.0" + ] + }, + "BinarySet": { + "BS": [ + "b25l" + ] + }, + "String": { + "S": "" + }, + "Number": { + "N": "0" + }, + "Bytes": { + "B": "" + }, + "Map": { + "M": {} + }, + "List": { + "L": [] + }, + "Null": { + "NULL": "" + }, + "Bool": { + "BOOL": false + } + } + ] + }, "IoPairs": [ [ "1", @@ -31,7 +278,9 @@ ":six": "Seis", ":seven": "Siete", ":eight": "Ocho", - ":NumberTest" : {"N" : "0800.000e0"}, + ":NumberTest": { + "N": "0800.000e0" + }, ":nine": "Nueve", ":cmp1a": "F_Cuatro.S_Junk", ":cmp1b": "F_444.S_Junk", @@ -195,13 +444,15 @@ { "Query": "cmp1c < Comp1", "Fail": [ - 0,1 + 0, + 1 ] }, { "Query": "cmp1c = Comp1", "Fail": [ - 0,1 + 0, + 1 ] }, { diff --git a/specification/dynamodb-encryption-client/ddb-attribute-serialization.md b/specification/dynamodb-encryption-client/ddb-attribute-serialization.md index 87c8b4307..a0be3ec80 100644 --- a/specification/dynamodb-encryption-client/ddb-attribute-serialization.md +++ b/specification/dynamodb-encryption-client/ddb-attribute-serialization.md @@ -60,9 +60,11 @@ String MUST be serialized as UTF-8 encoded bytes. #### Number -Number MUST be serialized as UTF-8 encoded bytes. Note that DynamoDB Number Attribute Values are strings. +This value MUST be normalized in the same way as DynamoDB normalizes numbers. +This normalized value MUST then be serialized as UTF-8 encoded bytes. + #### Binary Binary MUST be serialized with the identity function; @@ -107,8 +109,18 @@ Each of these entries MUST be serialized as: All [Set Entry Values](#set-entry-value) are the same type. Binary Sets MUST NOT contain duplicate entries. +Entries in a Binary Set MUST be ordered lexicographically by their underlying bytes in ascending order. + Number Sets MUST NOT contain duplicate entries. +Entries in a Number Set MUST be ordered in ascending [UTF-16 binary order](./string-ordering.md#utf-16-binary-order). +This ordering MUST be applied after normalization of the number value. +Note that because normalized number characters are all in the ASCII range (U+0000 to U+007F), +this ordering is equivalent to the [code point ordering](./string-ordering.md#code-point-order). + String Sets MUST NOT contain duplicate entries. +Entries in a String Set MUST be ordered in ascending [UTF-16 binary order](./string-ordering.md#utf-16-binary-order). +Note that though the entries are sorted by UTF016 binary order, +the values are serialized in the set with UTF-8 encoding. ###### Set Entry Length @@ -157,6 +169,11 @@ Each key-value pair MUST be serialized as: This sequence MUST NOT contain duplicate [Map Keys](#map-key). +Entries in a serialized Map MUST be ordered by key value, +ordered in ascending [UTF-16 binary order](./string-ordering.md#utf-16-binary-order). +Note that even though the values are sorted according to UTF-16 binary order, +string values are actually encoded within the map as UTF-8. + ###### Key Type Key Type MUST be the [Type ID](#type-id) for Strings. diff --git a/specification/dynamodb-encryption-client/ddb-encryption-branch-key-id-supplier.md b/specification/dynamodb-encryption-client/ddb-encryption-branch-key-id-supplier.md index 43462aa33..f9a48124e 100644 --- a/specification/dynamodb-encryption-client/ddb-encryption-branch-key-id-supplier.md +++ b/specification/dynamodb-encryption-client/ddb-encryption-branch-key-id-supplier.md @@ -1,3 +1,6 @@ +[//]: # "Copyright Amazon.com Inc. or its affiliates. All Rights Reserved." +[//]: # "SPDX-License-Identifier: CC-BY-SA-4.0" + # DynamoDb Encryption Branch Key Supplier ## Overview diff --git a/specification/dynamodb-encryption-client/ddb-item-conversion.md b/specification/dynamodb-encryption-client/ddb-item-conversion.md index 76168263b..644624d58 100644 --- a/specification/dynamodb-encryption-client/ddb-item-conversion.md +++ b/specification/dynamodb-encryption-client/ddb-item-conversion.md @@ -26,10 +26,12 @@ This document describes how a DynamoDB Item is converted to the Structured Encryption Library's [Structured Data](../structured-encryption/structures.md#structured-data), and vice versa. -The conversion from DDB Item to Structured Data must be lossless, -meaning that converting a DDB Item to -a Structured Data and back to a DDB Item again -MUST result in the exact same DDB Item. +Round Trip conversion between DDB Item and Structured Data is technically lossless, but it is not identity. +The conversion normalizes some values, the same way that +DynamoDB PuItem followed by GetItem normalizes some values. +The sets still have the same members, and the numbers still have the same values, +but the members of the set might appear in a different order, +and the numeric value might be formatted differently. ## Convert DDB Item to Structured Data @@ -73,4 +75,4 @@ has the following requirements: - Conversion from a Structured Data Map MUST fail if it has duplicate keys - Conversion from a Structured Data Number Set MUST fail if it has duplicate values - Conversion from a Structured Data String Set MUST fail if it has duplicate values -- Conversion from a Structured Data Binary Set MUST fail if it has duplicate values \ No newline at end of file +- Conversion from a Structured Data Binary Set MUST fail if it has duplicate values diff --git a/specification/dynamodb-encryption-client/ddb-support.md b/specification/dynamodb-encryption-client/ddb-support.md index 7697759e7..b276942f4 100644 --- a/specification/dynamodb-encryption-client/ddb-support.md +++ b/specification/dynamodb-encryption-client/ddb-support.md @@ -1,3 +1,6 @@ +[//]: # "Copyright Amazon.com Inc. or its affiliates. All Rights Reserved." +[//]: # "SPDX-License-Identifier: CC-BY-SA-4.0" + # DynamoDB Support Layer The DynamoDB Support Layer provides everything necessary to the middleware interceptors, diff --git a/specification/dynamodb-encryption-client/string-ordering.md b/specification/dynamodb-encryption-client/string-ordering.md new file mode 100644 index 000000000..7b2cb8bc8 --- /dev/null +++ b/specification/dynamodb-encryption-client/string-ordering.md @@ -0,0 +1,103 @@ +[//]: # "Copyright Amazon.com Inc. or its affiliates. All Rights Reserved." +[//]: # "SPDX-License-Identifier: CC-BY-SA-4.0" + +# String Ordering + +## Version + +1.0.0 + +### Changelog + +- 1.0.0 + + - Initial record + +## Definitions + +### Conventions used in this document + +The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" +in this document are to be interpreted as described in [RFC 2119](https://tools.ietf.org/html/rfc2119). + +### Unicode + +For the latest version see: +https://www.unicode.org/versions/latest/ + +For Version 15.0 see: +https://www.unicode.org/versions/Unicode15.0.0/ + +### Unicode scalar value + +Any Unicode code point except [surrogate](#surrogates) code points. + +See [section 3.9 Unicode Encoding Forms](https://www.unicode.org/versions/Unicode15.0.0/ch03.pdf) of the Unicode specification. + +### UTF-16 code unit + +A 16-bit value representing a Unicode code point in a UTF-16 encoding. +Includes [surrogate](#surrogates) code points. + +See [section 3.9 Unicode Encoding Forms](https://www.unicode.org/versions/Unicode15.0.0/ch03.pdf) of the Unicode specification. + +### Surrogates + +Unicode code points in the range U+D800 to U+DFFF. +These code points are only used in UTF-16 encodings +to represent Unicode values above U+FFFF. + +See [section 3.8 Surrogates](https://www.unicode.org/versions/Unicode15.0.0/ch03.pdf) of the Unicode specification. + +## Overview + +There are several instances throughout this specification where an order must be +imposed on an unordered data structure during serialization for the purposes of canonicalization. +This means that we must clearly specify the canonical ordering wherever +such serialization is required. +This is especially importing when ordering strings, +as different encodings may lend themselves to slightly different orderings. + +Wherever strings need to be ordered, +this specification will require either a [code point order](#code-point-order) +or a [UTF-16 binary order](#utf-16-binary-order). + +## UTF-16 Binary Order + +When ordering strings, +these strings MUST be compared according to their UTF-16 encoding, +lexicographically per [UTF-16 code unit](#utf-16-code-unit). +UTF-16 code units for [high or low surrogates](#surrogates) MUST be compared individually, +and the [Unicode scalar value](#unicode-scalar-value) represented by a surrogate pair +MUST NOT be compared. + +Note that this is not equivalent to the [code point order](#code-point-order). +Specifically, the range of characters with Unicode code point U+E000 to U+0xFFFF +(code points representable by 16 bits, but after the surrogate range) +MUST be considered "greater than" any character with a Unicode code point of U+10000 to U+10FFFF. + +As an example, consider the following two characters: + +| char | Unicode code point | UTF-16 encoding | +| ---- | ------------------ | --------------- | +| `｡` | U+FF61 | 0xFF61 | +| `𐀂` | U+10002 | 0xD800 0xDC02 | + +This ordering will order `｡` _after_ `𐀂`, despite `𐀂` having a higher Unicode code point. + +## Code Point Order + +This is the ordering referred to in the Unicode specification as a [code point order](https://www.unicode.org/versions/Unicode15.0.0/ch05.pdf). + +When ordering strings, +these strings are compared lexicographically per [Unicode scalar value](#unicode-scalar-value) represented by the string. +This means that if a string is UTF-16 encoded, +higher order Unicode characters, encoded as a surrogate pair, +must be handled as the Unicode scalar value represented by that surrogate pair, +instead of each surrogate code point being handled individually. + +Note that this is equivalent to lexicographically comparing a UTF-8 encoded string per byte. +This is also equivalent to lexicographically comparing a UTF-32 encoded string per 32-bit code unit. + +Currently, this specification does not directly use code point order for sorting string values, +but may use this ordering for new behaviors in the future. diff --git a/specification/structured-encryption/footer.md b/specification/structured-encryption/footer.md index 560259c42..b3dfc0444 100644 --- a/specification/structured-encryption/footer.md +++ b/specification/structured-encryption/footer.md @@ -1,3 +1,5 @@ +[//]: # "Copyright Amazon.com Inc. or its affiliates. All Rights Reserved." +[//]: # "SPDX-License-Identifier: CC-BY-SA-4.0" # Structured Encryption Footer diff --git a/specification/structured-encryption/header.md b/specification/structured-encryption/header.md index 5d8ec6212..ec0593ff5 100644 --- a/specification/structured-encryption/header.md +++ b/specification/structured-encryption/header.md @@ -1,3 +1,5 @@ +[//]: # "Copyright Amazon.com Inc. or its affiliates. All Rights Reserved." +[//]: # "SPDX-License-Identifier: CC-BY-SA-4.0" # Structured Encryption Header diff --git a/submodules/MaterialProviders b/submodules/MaterialProviders index cce342923..13e0ac3c3 160000 --- a/submodules/MaterialProviders +++ b/submodules/MaterialProviders @@ -1 +1 @@ -Subproject commit cce342923ce5602f2e6162dccca44ecefaffca68 +Subproject commit 13e0ac3c3c5eea83494706e4a96f40126d8f38a8 From 000585c24da7b76f4963d41875ed9eb6f455b534 Mon Sep 17 00:00:00 2001 From: seebees Date: Wed, 15 Nov 2023 11:38:35 -0800 Subject: [PATCH 2/5] chore: Examples run serially. Examples wait for GSI. No extra checkout (#561) --- .../CompoundBeaconSearchableEncryptionExample.java | 3 +++ .../VirtualBeaconSearchableEncryptionExample.java | 3 +++ codebuild/release/release-prod.yml | 3 --- codebuild/release/release.yml | 8 ++++---- 4 files changed, 10 insertions(+), 7 deletions(-) diff --git a/Examples/runtimes/java/DynamoDbEncryption/src/main/java/software/amazon/cryptography/examples/searchableencryption/CompoundBeaconSearchableEncryptionExample.java b/Examples/runtimes/java/DynamoDbEncryption/src/main/java/software/amazon/cryptography/examples/searchableencryption/CompoundBeaconSearchableEncryptionExample.java index baabb5e03..e148d9d7e 100644 --- a/Examples/runtimes/java/DynamoDbEncryption/src/main/java/software/amazon/cryptography/examples/searchableencryption/CompoundBeaconSearchableEncryptionExample.java +++ b/Examples/runtimes/java/DynamoDbEncryption/src/main/java/software/amazon/cryptography/examples/searchableencryption/CompoundBeaconSearchableEncryptionExample.java @@ -328,6 +328,9 @@ public static void PutAndQueryItemWithCompoundBeacon(DynamoDbClient ddb, String .expressionAttributeValues(expressionAttributeValues) .build(); + // GSIs do not update instantly + // so if the results come back empty + // we retry after a short sleep for (int i=0; i<10; ++i) { QueryResponse queryResponse = ddb.query(queryRequest); List> attributeValues = queryResponse.items(); diff --git a/Examples/runtimes/java/DynamoDbEncryption/src/main/java/software/amazon/cryptography/examples/searchableencryption/VirtualBeaconSearchableEncryptionExample.java b/Examples/runtimes/java/DynamoDbEncryption/src/main/java/software/amazon/cryptography/examples/searchableencryption/VirtualBeaconSearchableEncryptionExample.java index d6ae06c77..d4379a3d6 100644 --- a/Examples/runtimes/java/DynamoDbEncryption/src/main/java/software/amazon/cryptography/examples/searchableencryption/VirtualBeaconSearchableEncryptionExample.java +++ b/Examples/runtimes/java/DynamoDbEncryption/src/main/java/software/amazon/cryptography/examples/searchableencryption/VirtualBeaconSearchableEncryptionExample.java @@ -436,6 +436,9 @@ public static void PutItemQueryItemWithVirtualBeacon(String ddbTableName, String .expressionAttributeValues(expressionAttributeValues) .build(); + // GSIs do not update instantly + // so if the results come back empty + // we retry after a short sleep for (int i=0; i<10; ++i) { final QueryResponse queryResponse = ddb.query(queryRequest); List> attributeValues = queryResponse.items(); diff --git a/codebuild/release/release-prod.yml b/codebuild/release/release-prod.yml index 095f7e6ed..b8f2a3263 100644 --- a/codebuild/release/release-prod.yml +++ b/codebuild/release/release-prod.yml @@ -4,8 +4,6 @@ version: 0.2 env: - variables: - BRANCH: "main" parameter-store: ACCOUNT: /CodeBuild/AccountId secrets-manager: @@ -31,7 +29,6 @@ phases: - cd aws-database-encryption-sdk-dynamodb-java/ pre_build: commands: - - git checkout $BRANCH - aws secretsmanager get-secret-value --region us-west-2 --secret-id Maven-GPG-Keys-Release --query SecretBinary --output text | base64 -d > ~/mvn_gpg.tgz - tar -xvf ~/mvn_gpg.tgz -C ~ # Create default location where GPG looks for creds and keys diff --git a/codebuild/release/release.yml b/codebuild/release/release.yml index 9111a41cc..149b84661 100644 --- a/codebuild/release/release.yml +++ b/codebuild/release/release.yml @@ -28,7 +28,7 @@ batch: - identifier: validate_staging_corretto11 depend-on: - - release_staging + - validate_staging_corretto8 buildspec: codebuild/staging/validate-staging.yml env: variables: @@ -38,7 +38,7 @@ batch: - identifier: validate_staging_corretto17 depend-on: - - release_staging + - validate_staging_corretto11 buildspec: codebuild/staging/validate-staging.yml env: variables: @@ -73,7 +73,7 @@ batch: - identifier: validate_release_corretto11 depend-on: - - upload_to_sonatype + - validate_release_corretto8 buildspec: codebuild/release/validate-release.yml env: variables: @@ -83,7 +83,7 @@ batch: - identifier: validate_release_corretto17 depend-on: - - upload_to_sonatype + - validate_release_corretto11 buildspec: codebuild/release/validate-release.yml env: variables: From 4998e1dc6d42c007817ae4d989ae37aa57b0b0f9 Mon Sep 17 00:00:00 2001 From: seebees Date: Wed, 15 Nov 2023 11:39:44 -0800 Subject: [PATCH 3/5] fix: Decrypt attributes returned by all DDB APIs (#578) --- CHANGELOG.md | 8 + .../src/DeleteItemTransform.dfy | 75 +++- .../src/DynamoDbMiddlewareSupport.dfy | 9 + .../src/PutItemTransform.dfy | 75 +++- .../src/UpdateItemTransform.dfy | 138 ++++++- .../DynamoDbEncryptionInterceptor.java | 11 + .../datamodeling/encryption/DoNotEncrypt.java | 4 + .../datamodeling/encryption/DoNotTouch.java | 4 + ...EncryptionInterceptorIntegrationTests.java | 155 ++++++- ...ryptionEnhancedClientIntegrationTests.java | 179 +++++--- .../validdatamodels/AllTypesClass.java | 386 ++++++++++++++++++ .../ddb-sdk-integration.md | 89 ++++ 12 files changed, 1076 insertions(+), 57 deletions(-) create mode 100644 DynamoDbEncryption/runtimes/java/src/test/java/software/amazon/cryptography/dbencryptionsdk/dynamodb/enhancedclient/validdatamodels/AllTypesClass.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 19d91bb95..5f0eafce4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## 3.1.2 2023-11-13 + +### Fix + +Fixed an issue where, when using the DynamoDbEncryptionInterceptor, +an encrypted item in the Attributes field of a DeleteItem, PutItem, or UpdateItem +response was passed through unmodified instead of being decrypted. + ## 3.1.1 2023-11-07 ### Fix diff --git a/DynamoDbEncryption/dafny/DynamoDbEncryptionTransforms/src/DeleteItemTransform.dfy b/DynamoDbEncryption/dafny/DynamoDbEncryptionTransforms/src/DeleteItemTransform.dfy index 2042879e7..ee971d997 100644 --- a/DynamoDbEncryption/dafny/DynamoDbEncryptionTransforms/src/DeleteItemTransform.dfy +++ b/DynamoDbEncryption/dafny/DynamoDbEncryptionTransforms/src/DeleteItemTransform.dfy @@ -66,8 +66,79 @@ module DeleteItemTransform { method Output(config: Config, input: DeleteItemOutputTransformInput) returns (output: Result) - ensures output.Success? && output.value.transformedOutput == input.sdkOutput + + //= specification/dynamodb-encryption-client/ddb-sdk-integration.md#decrypt-after-deleteitem + //= type=implication + //# After a [DeleteItem](https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_DeleteItem.html) + //# call is made to DynamoDB, + //# the resulting response MUST be modified before + //# being returned to the caller if: + // - there exists an Item Encryptor specified within the + // [DynamoDB Encryption Client Config](#dynamodb-encryption-client-configuration) + // with a [DynamoDB Table Name](./ddb-item-encryptor.md#dynamodb-table-name) + // equal to the `TableName` on the DeleteItem request. + // - the response contains [Attributes](https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_DeleteItem.html#DDB-DeleteItem-response-Attributes). + // The response will contain Attributes if the related DeleteItem request's + // [ReturnValues](https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_DeleteItem.html#DDB-DeleteItem-request-ReturnValues) + // had a value of `ALL_OLD` and an item was deleted. + ensures ( + && output.Success? + && input.originalInput.TableName in config.tableEncryptionConfigs + && input.sdkOutput.Attributes.Some? + ) ==> + && var tableConfig := config.tableEncryptionConfigs[input.originalInput.TableName]; + && var oldHistory := old(tableConfig.itemEncryptor.History.DecryptItem); + && var newHistory := tableConfig.itemEncryptor.History.DecryptItem; + + && |newHistory| == |oldHistory|+1 + && Seq.Last(newHistory).output.Success? + + //= specification/dynamodb-encryption-client/ddb-sdk-integration.md#decrypt-after-deleteitem + //= type=implication + //# In this case, the [Item Encryptor](./ddb-item-encryptor.md) MUST perform + //# [Decrypt Item](./decrypt-item.md) where the input + //# [DynamoDB Item](./decrypt-item.md#dynamodb-item) + //# is the `Attributes` field in the original response + && Seq.Last(newHistory).input.encryptedItem == input.sdkOutput.Attributes.value + + //= specification/dynamodb-encryption-client/ddb-sdk-integration.md#decrypt-after-deleteitem + //= type=implication + //# Beacons MUST be [removed](ddb-support.md#removebeacons) from the result. + && RemoveBeacons(tableConfig, Seq.Last(newHistory).output.value.plaintextItem).Success? + && var item := RemoveBeacons(tableConfig, Seq.Last(newHistory).output.value.plaintextItem).value; + + //= specification/dynamodb-encryption-client/ddb-sdk-integration.md#decrypt-after-deleteitem + //= type=implication + //# The DeleteItem response's `Attributes` field MUST be + //# replaced by the encrypted DynamoDb Item outputted above. + && output.value.transformedOutput.Attributes.Some? + && (item == output.value.transformedOutput.Attributes.value) + + // Passthrough the response if the above specification is not met + ensures ( + && output.Success? + && ( + || input.originalInput.TableName !in config.tableEncryptionConfigs + || input.sdkOutput.Attributes.None? + ) + ) ==> + output.value.transformedOutput == input.sdkOutput + + requires ValidConfig?(config) + ensures ValidConfig?(config) + modifies ModifiesConfig(config) { - return Success(DeleteItemOutputTransformOutput(transformedOutput := input.sdkOutput)); + var tableName := input.originalInput.TableName; + if tableName !in config.tableEncryptionConfigs || input.sdkOutput.Attributes.None? + { + return Success(DeleteItemOutputTransformOutput(transformedOutput := input.sdkOutput)); + } + var tableConfig := config.tableEncryptionConfigs[tableName]; + var decryptRes := tableConfig.itemEncryptor.DecryptItem( + EncTypes.DecryptItemInput(encryptedItem:=input.sdkOutput.Attributes.value) + ); + var decrypted :- MapError(decryptRes); + var item :- RemoveBeacons(tableConfig, decrypted.plaintextItem); + return Success(DeleteItemOutputTransformOutput(transformedOutput := input.sdkOutput.(Attributes := Some(item)))); } } diff --git a/DynamoDbEncryption/dafny/DynamoDbEncryptionTransforms/src/DynamoDbMiddlewareSupport.dfy b/DynamoDbEncryption/dafny/DynamoDbEncryptionTransforms/src/DynamoDbMiddlewareSupport.dfy index 7dadca820..91432e7f7 100644 --- a/DynamoDbEncryption/dafny/DynamoDbEncryptionTransforms/src/DynamoDbMiddlewareSupport.dfy +++ b/DynamoDbEncryption/dafny/DynamoDbEncryptionTransforms/src/DynamoDbMiddlewareSupport.dfy @@ -32,6 +32,15 @@ module DynamoDbMiddlewareSupport { .MapFailure(e => E(e)) } + // IsSigned returned whether this attribute is signed according to this config + predicate method {:opaque} IsSigned( + config : ValidTableConfig, + attr : string + ) + { + BS.IsSigned(config.itemEncryptor.config.attributeActionsOnEncrypt, attr) + } + // TestConditionExpression fails if a condition expression is not suitable for the // given encryption schema. // Generally this means no encrypted attribute is referenced. diff --git a/DynamoDbEncryption/dafny/DynamoDbEncryptionTransforms/src/PutItemTransform.dfy b/DynamoDbEncryption/dafny/DynamoDbEncryptionTransforms/src/PutItemTransform.dfy index 7cec499cf..5b2e96ff9 100644 --- a/DynamoDbEncryption/dafny/DynamoDbEncryptionTransforms/src/PutItemTransform.dfy +++ b/DynamoDbEncryption/dafny/DynamoDbEncryptionTransforms/src/PutItemTransform.dfy @@ -83,8 +83,79 @@ module PutItemTransform { method Output(config: Config, input: PutItemOutputTransformInput) returns (output: Result) - ensures output.Success? && output.value.transformedOutput == input.sdkOutput + + //= specification/dynamodb-encryption-client/ddb-sdk-integration.md#decrypt-after-putitem + //= type=implication + //# After a [PutItem](https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_PutItem.html) + //# call is made to DynamoDB, + //# the resulting response MUST be modified before + //# being returned to the caller if: + // - there exists an Item Encryptor specified within the + // [DynamoDB Encryption Client Config](#dynamodb-encryption-client-configuration) + // with a [DynamoDB Table Name](./ddb-item-encryptor.md#dynamodb-table-name) + // equal to the `TableName` on the PutItem request. + // - the response contains [Attributes](https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_PutItem.html#DDB-PutItem-response-Attributes). + // The response will contain Attributes if the related PutItem request's + // [ReturnValues](https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_PutItem.html#DDB-PutItem-request-ReturnValues) + // had a value of `ALL_OLD` and the PutItem call replaced a pre-existing item. + ensures ( + && output.Success? + && input.originalInput.TableName in config.tableEncryptionConfigs + && input.sdkOutput.Attributes.Some? + ) ==> + && var tableConfig := config.tableEncryptionConfigs[input.originalInput.TableName]; + && var oldHistory := old(tableConfig.itemEncryptor.History.DecryptItem); + && var newHistory := tableConfig.itemEncryptor.History.DecryptItem; + + && |newHistory| == |oldHistory|+1 + && Seq.Last(newHistory).output.Success? + + //= specification/dynamodb-encryption-client/ddb-sdk-integration.md#decrypt-after-putitem + //= type=implication + //# In this case, the [Item Encryptor](./ddb-item-encryptor.md) MUST perform + //# [Decrypt Item](./decrypt-item.md) where the input + //# [DynamoDB Item](./decrypt-item.md#dynamodb-item) + //# is the `Attributes` field in the original response + && Seq.Last(newHistory).input.encryptedItem == input.sdkOutput.Attributes.value + + //= specification/dynamodb-encryption-client/ddb-sdk-integration.md#decrypt-after-putitem + //= type=implication + //# Beacons MUST be [removed](ddb-support.md#removebeacons) from the result. + && RemoveBeacons(tableConfig, Seq.Last(newHistory).output.value.plaintextItem).Success? + && var item := RemoveBeacons(tableConfig, Seq.Last(newHistory).output.value.plaintextItem).value; + + //= specification/dynamodb-encryption-client/ddb-sdk-integration.md#decrypt-after-putitem + //= type=implication + //# The PutItem response's `Attributes` field MUST be + //# replaced by the encrypted DynamoDb Item outputted above. + && output.value.transformedOutput.Attributes.Some? + && (item == output.value.transformedOutput.Attributes.value) + + // Passthrough the response if the above specification is not met + ensures ( + && output.Success? + && ( + || input.originalInput.TableName !in config.tableEncryptionConfigs + || input.sdkOutput.Attributes.None? + ) + ) ==> + output.value.transformedOutput == input.sdkOutput + + requires ValidConfig?(config) + ensures ValidConfig?(config) + modifies ModifiesConfig(config) { - return Success(PutItemOutputTransformOutput(transformedOutput := input.sdkOutput)); + var tableName := input.originalInput.TableName; + if tableName !in config.tableEncryptionConfigs || input.sdkOutput.Attributes.None? + { + return Success(PutItemOutputTransformOutput(transformedOutput := input.sdkOutput)); + } + var tableConfig := config.tableEncryptionConfigs[tableName]; + var decryptRes := tableConfig.itemEncryptor.DecryptItem( + EncTypes.DecryptItemInput(encryptedItem:=input.sdkOutput.Attributes.value) + ); + var decrypted :- MapError(decryptRes); + var item :- RemoveBeacons(tableConfig, decrypted.plaintextItem); + return Success(PutItemOutputTransformOutput(transformedOutput := input.sdkOutput.(Attributes := Some(item)))); } } diff --git a/DynamoDbEncryption/dafny/DynamoDbEncryptionTransforms/src/UpdateItemTransform.dfy b/DynamoDbEncryption/dafny/DynamoDbEncryptionTransforms/src/UpdateItemTransform.dfy index 0e66fba1d..6e51ec902 100644 --- a/DynamoDbEncryption/dafny/DynamoDbEncryptionTransforms/src/UpdateItemTransform.dfy +++ b/DynamoDbEncryption/dafny/DynamoDbEncryptionTransforms/src/UpdateItemTransform.dfy @@ -65,8 +65,142 @@ module UpdateItemTransform { method Output(config: Config, input: UpdateItemOutputTransformInput) returns (output: Result) - ensures output.Success? && output.value.transformedOutput == input.sdkOutput + + //= specification/dynamodb-encryption-client/ddb-sdk-integration.md#decrypt-after-updateitem + //= type=implication + //# After a [UpdateItem](https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_UpdateItem.html) + //# call is made to DynamoDB, + //# the resulting response MUST be modified before + //# being returned to the caller if: + // - there exists an Item Encryptor specified within the + // [DynamoDB Encryption Client Config](#dynamodb-encryption-client-configuration) + // with a [DynamoDB Table Name](./ddb-item-encryptor.md#dynamodb-table-name) + // equal to the `TableName` on the UpdateItem request. + // - the response contains [Attributes](https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_UpdateItem.html#DDB-UpdateItem-response-Attributes). + // - the original UpdateItem request had a + // [ReturnValues](https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_UpdateItem.html#DDB-UpdateItem-request-ReturnValues) + // with a value of `ALL_OLD` or `ALL_NEW`. + ensures ( + && output.Success? + && input.originalInput.TableName in config.tableEncryptionConfigs + && input.sdkOutput.Attributes.Some? + && input.originalInput.ReturnValues.Some? + && ( + || input.originalInput.ReturnValues.value.ALL_OLD? + || input.originalInput.ReturnValues.value.ALL_NEW? + ) + ) ==> + && var tableConfig := config.tableEncryptionConfigs[input.originalInput.TableName]; + && var oldHistory := old(tableConfig.itemEncryptor.History.DecryptItem); + && var newHistory := tableConfig.itemEncryptor.History.DecryptItem; + + && |newHistory| == |oldHistory|+1 + && Seq.Last(newHistory).output.Success? + + //= specification/dynamodb-encryption-client/ddb-sdk-integration.md#decrypt-after-updateitem + //= type=implication + //# In this case, the [Item Encryptor](./ddb-item-encryptor.md) MUST perform + //# [Decrypt Item](./decrypt-item.md) where the input + //# [DynamoDB Item](./decrypt-item.md#dynamodb-item) + //# is the `Attributes` field in the original response + && Seq.Last(newHistory).input.encryptedItem == input.sdkOutput.Attributes.value + + //= specification/dynamodb-encryption-client/ddb-sdk-integration.md#decrypt-after-updateitem + //= type=implication + //# Beacons MUST be [removed](ddb-support.md#removebeacons) from the result. + && RemoveBeacons(tableConfig, Seq.Last(newHistory).output.value.plaintextItem).Success? + && var item := RemoveBeacons(tableConfig, Seq.Last(newHistory).output.value.plaintextItem).value; + + //= specification/dynamodb-encryption-client/ddb-sdk-integration.md#decrypt-after-updateitem + //= type=implication + //# The UpdateItem response's `Attributes` field MUST be + //# replaced by the encrypted DynamoDb Item outputted above. + && output.value.transformedOutput.Attributes.Some? + && (item == output.value.transformedOutput.Attributes.value) + + //= specification/dynamodb-encryption-client/ddb-sdk-integration.md#decrypt-after-updateitem + //= type=implication + //# In all other cases, the UpdateItem response MUST NOT be modified. + ensures ( + && output.Success? + && ( + || input.originalInput.TableName !in config.tableEncryptionConfigs + || input.sdkOutput.Attributes.None? + ) + ) ==> ( + && output.value.transformedOutput == input.sdkOutput + ) + + ensures ( + && output.Success? + && input.originalInput.TableName in config.tableEncryptionConfigs + && input.sdkOutput.Attributes.Some? + && (input.originalInput.ReturnValues.Some? ==> ( + || input.originalInput.ReturnValues.value.UPDATED_NEW? + || input.originalInput.ReturnValues.value.UPDATED_OLD? + ) + ) + ) ==> ( + && var tableConfig := config.tableEncryptionConfigs[input.originalInput.TableName]; + && output.value.transformedOutput == input.sdkOutput + && forall k <- input.sdkOutput.Attributes.value.Keys :: !IsSigned(tableConfig, k) + ) + + //= specification/dynamodb-encryption-client/ddb-sdk-integration.md#decrypt-after-updateitem + //= type=implication + //# Additionally, if a value of `UPDATED_OLD` or `UPDATED_NEW` was used, + //# and any Attributes in the response are authenticated + //# per the [DynamoDB Encryption Client Config](#dynamodb-encryption-client-configuration), + //# an error MUST be raised. + ensures ( + && input.originalInput.TableName in config.tableEncryptionConfigs + && var tableConfig := config.tableEncryptionConfigs[input.originalInput.TableName]; + && input.sdkOutput.Attributes.Some? + && (input.originalInput.ReturnValues.Some? ==> ( + || input.originalInput.ReturnValues.value.UPDATED_NEW? + || input.originalInput.ReturnValues.value.UPDATED_OLD? + ) + ) + && exists k <- input.sdkOutput.Attributes.value.Keys :: IsSigned(tableConfig, k) + ) ==> + output.Failure? + + requires ValidConfig?(config) + ensures ValidConfig?(config) + modifies ModifiesConfig(config) { - return Success(UpdateItemOutputTransformOutput(transformedOutput := input.sdkOutput)); + var tableName := input.originalInput.TableName; + + if + || tableName !in config.tableEncryptionConfigs + || input.sdkOutput.Attributes.None? + { + return Success(UpdateItemOutputTransformOutput(transformedOutput := input.sdkOutput)); + } + + var tableConfig := config.tableEncryptionConfigs[tableName]; + var attributes := input.sdkOutput.Attributes.value; + + if !( + && input.originalInput.ReturnValues.Some? + && ( + || input.originalInput.ReturnValues.value.ALL_NEW? + || input.originalInput.ReturnValues.value.ALL_OLD?) + ) + { + // This error should not be possible to reach if we assume the DDB API contract is correct. + // We include this runtime check for defensive purposes. + :- Need(forall k <- attributes.Keys :: !IsSigned(tableConfig, k), + E("UpdateItems response contains signed attributes, but does not include the entire item which is required for verification.")); + + return Success(UpdateItemOutputTransformOutput(transformedOutput := input.sdkOutput)); + } + + var decryptRes := tableConfig.itemEncryptor.DecryptItem( + EncTypes.DecryptItemInput(encryptedItem:=attributes) + ); + var decrypted :- MapError(decryptRes); + var item :- RemoveBeacons(tableConfig, decrypted.plaintextItem); + return Success(UpdateItemOutputTransformOutput(transformedOutput := input.sdkOutput.(Attributes := Some(item)))); } } diff --git a/DynamoDbEncryption/runtimes/java/src/main/java/software/amazon/cryptography/dbencryptionsdk/dynamodb/DynamoDbEncryptionInterceptor.java b/DynamoDbEncryption/runtimes/java/src/main/java/software/amazon/cryptography/dbencryptionsdk/dynamodb/DynamoDbEncryptionInterceptor.java index 78e8848be..bdfa8306a 100644 --- a/DynamoDbEncryption/runtimes/java/src/main/java/software/amazon/cryptography/dbencryptionsdk/dynamodb/DynamoDbEncryptionInterceptor.java +++ b/DynamoDbEncryption/runtimes/java/src/main/java/software/amazon/cryptography/dbencryptionsdk/dynamodb/DynamoDbEncryptionInterceptor.java @@ -252,6 +252,17 @@ public SdkResponse modifyResponse(Context.ModifyResponse context, ExecutionAttri .sdkHttpResponse(originalResponse.sdkHttpResponse()) .build(); break; + } case "DeleteItem": { + DeleteItemResponse transformedResponse = transformer.DeleteItemOutputTransform( + DeleteItemOutputTransformInput.builder() + .sdkOutput((DeleteItemResponse) originalResponse) + .originalInput((DeleteItemRequest) originalRequest) + .build()).transformedOutput(); + outgoingResponse = transformedResponse.toBuilder() + .responseMetadata(((DeleteItemResponse) originalResponse).responseMetadata()) + .sdkHttpResponse(originalResponse.sdkHttpResponse()) + .build(); + break; } case "ExecuteStatement": { ExecuteStatementResponse transformedResponse = transformer.ExecuteStatementOutputTransform( ExecuteStatementOutputTransformInput.builder() diff --git a/DynamoDbEncryption/runtimes/java/src/main/sdkv1/com/amazonaws/services/dynamodbv2/datamodeling/encryption/DoNotEncrypt.java b/DynamoDbEncryption/runtimes/java/src/main/sdkv1/com/amazonaws/services/dynamodbv2/datamodeling/encryption/DoNotEncrypt.java index 45f490968..a6c11bdba 100644 --- a/DynamoDbEncryption/runtimes/java/src/main/sdkv1/com/amazonaws/services/dynamodbv2/datamodeling/encryption/DoNotEncrypt.java +++ b/DynamoDbEncryption/runtimes/java/src/main/sdkv1/com/amazonaws/services/dynamodbv2/datamodeling/encryption/DoNotEncrypt.java @@ -21,6 +21,9 @@ import java.lang.annotation.Target; /** + * Warning: This annotation only works with the DynamoDBMapper for AWS SDK for Java 1.x. + * If you are using the AWS SDK for Java 2.x, use @DynamoDbEncryptionSignOnly instead. + * * Prevents the associated item (class or attribute) from being encrypted. * *

For guidance on performing a safe data model change procedure, please see For guidance on performing a safe data model change procedure, please see putItem = putResponse.attributes(); + assertNotNull(putItem); + assertEquals(partitionValue, putItem.get(TEST_PARTITION_NAME).s()); + assertEquals(sortValue, putItem.get(TEST_SORT_NAME).n()); + assertEquals(attrValue, putItem.get(TEST_ATTR_NAME).s()); // Get Item back from table Map keyToGet = createTestKey(partitionValue, sortValue); @@ -87,6 +99,147 @@ public void TestPutItemGetItem() { assertEquals(attrValue, returnedItem.get(TEST_ATTR_NAME).s()); } + // Test that if we update a DO_NOTHING attribute, + // we correctly decrypt the ALL_* return values + @Test + public void TestUpdateItemReturnAll() { + // Put item into table + String partitionValue = "update_ALL"; + String sortValue = randomNum; + String attrValue = "bar"; + String attrValue2 = "hello world"; + Map item = createTestItem(partitionValue, sortValue, attrValue, attrValue2); + + PutItemRequest putRequest = PutItemRequest.builder() + .tableName(TEST_TABLE_NAME) + .item(item) + .returnValues(ReturnValue.ALL_OLD) + .build(); + + ddbKmsKeyring.putItem(putRequest); + + // Update unsigned attribute, and return ALL_OLD + Map keyToGet = createTestKey(partitionValue, sortValue); + UpdateItemRequest updateRequest = UpdateItemRequest.builder() + .key(keyToGet) + .returnValues(ReturnValue.ALL_OLD) + .updateExpression("SET #D = :d") + .expressionAttributeNames(Collections.singletonMap("#D", "attr2")) + .expressionAttributeValues(Collections.singletonMap(":d", AttributeValue.fromS("updated"))) + .tableName(TEST_TABLE_NAME) + .build(); + + UpdateItemResponse updateResponse = ddbKmsKeyring.updateItem(updateRequest); + assertEquals(200, updateResponse.sdkHttpResponse().statusCode()); + Map returnedItem = updateResponse.attributes(); + assertNotNull(returnedItem); + assertEquals(partitionValue, returnedItem.get(TEST_PARTITION_NAME).s()); + assertEquals(sortValue, returnedItem.get(TEST_SORT_NAME).n()); + assertEquals(attrValue, returnedItem.get(TEST_ATTR_NAME).s()); + assertEquals("hello world", returnedItem.get(TEST_ATTR2_NAME).s()); + + // Update unsigned attribute, and return ALL_NEW + UpdateItemRequest updateRequest2 = updateRequest.toBuilder() + .returnValues(ReturnValue.ALL_NEW) + .expressionAttributeValues(Collections.singletonMap(":d", AttributeValue.fromS("updated2"))) + .build(); + + UpdateItemResponse updateResponse2 = ddbKmsKeyring.updateItem(updateRequest2); + assertEquals(200, updateResponse2.sdkHttpResponse().statusCode()); + Map returnedItem2 = updateResponse2.attributes(); + assertNotNull(returnedItem2); + assertEquals(partitionValue, returnedItem2.get(TEST_PARTITION_NAME).s()); + assertEquals(sortValue, returnedItem2.get(TEST_SORT_NAME).n()); + assertEquals(attrValue, returnedItem2.get(TEST_ATTR_NAME).s()); + assertEquals("updated2", returnedItem2.get(TEST_ATTR2_NAME).s()); + } + + // Test that if we update a DO_NOTHING attribute, + // we correctly pass through the UPDATED_* return values + @Test + public void TestUpdateItemReturnUpdated() { + // Put item into table + String partitionValue = "update_UPDATE"; + String sortValue = randomNum; + String attrValue = "bar"; + String attrValue2 = "hello world"; + Map item = createTestItem(partitionValue, sortValue, attrValue, attrValue2); + + PutItemRequest putRequest = PutItemRequest.builder() + .tableName(TEST_TABLE_NAME) + .item(item) + .returnValues(ReturnValue.ALL_OLD) + .build(); + + ddbKmsKeyring.putItem(putRequest); + + // Update unsigned attribute, and return UPDATED_OLD + Map keyToGet = createTestKey(partitionValue, sortValue); + UpdateItemRequest updateRequest = UpdateItemRequest.builder() + .key(keyToGet) + .returnValues(ReturnValue.UPDATED_OLD) + .updateExpression("SET #D = :d") + .expressionAttributeNames(Collections.singletonMap("#D", "attr2")) + .expressionAttributeValues(Collections.singletonMap(":d", AttributeValue.fromS("updated"))) + .tableName(TEST_TABLE_NAME) + .build(); + + UpdateItemResponse updateResponse = ddbKmsKeyring.updateItem(updateRequest); + assertEquals(200, updateResponse.sdkHttpResponse().statusCode()); + Map returnedItem = updateResponse.attributes(); + assertNotNull(returnedItem); + assertFalse(returnedItem.containsKey(TEST_ATTR_NAME)); + assertEquals("hello world", returnedItem.get(TEST_ATTR2_NAME).s()); + + // Update unsigned attribute, and return ALL_NEW + UpdateItemRequest updateRequest2 = updateRequest.toBuilder() + .returnValues(ReturnValue.UPDATED_NEW) + .expressionAttributeValues(Collections.singletonMap(":d", AttributeValue.fromS("updated2"))) + .build(); + + UpdateItemResponse updateResponse2 = ddbKmsKeyring.updateItem(updateRequest2); + assertEquals(200, updateResponse2.sdkHttpResponse().statusCode()); + Map returnedItem2 = updateResponse2.attributes(); + assertNotNull(returnedItem2); + assertFalse(returnedItem2.containsKey(TEST_ATTR_NAME)); + assertEquals("updated2", returnedItem2.get(TEST_ATTR2_NAME).s()); + } + + @Test + public void TestDeleteItem() { + // Put item into table + String partitionValue = "delete"; + String sortValue = randomNum; + String attrValue = "bar"; + String attrValue2 = "hello world"; + Map item = createTestItem(partitionValue, sortValue, attrValue, attrValue2); + + PutItemRequest putRequest = PutItemRequest.builder() + .tableName(TEST_TABLE_NAME) + .item(item) + .build(); + + PutItemResponse putResponse = ddbKmsKeyring.putItem(putRequest); + assertEquals(200, putResponse.sdkHttpResponse().statusCode()); + + // Delete item from table, set ReturnValues to ALL_OLD to return deleted item + Map keyToGet = createTestKey(partitionValue, sortValue); + + DeleteItemRequest deleteRequest = DeleteItemRequest.builder() + .key(keyToGet) + .tableName(TEST_TABLE_NAME) + .returnValues(ReturnValue.ALL_OLD) + .build(); + + DeleteItemResponse deleteResponse = ddbKmsKeyring.deleteItem(deleteRequest); + assertEquals(200, deleteResponse.sdkHttpResponse().statusCode()); + Map returnedItem = deleteResponse.attributes(); + assertNotNull(returnedItem); + assertEquals(partitionValue, returnedItem.get(TEST_PARTITION_NAME).s()); + assertEquals(sortValue, returnedItem.get(TEST_SORT_NAME).n()); + assertEquals(attrValue, returnedItem.get(TEST_ATTR_NAME).s()); + } + @Test public void TestBatchWriteBatchGet() { // Batch write items to table diff --git a/DynamoDbEncryption/runtimes/java/src/test/java/software/amazon/cryptography/dbencryptionsdk/dynamodb/enhancedclient/DynamoDbEncryptionEnhancedClientIntegrationTests.java b/DynamoDbEncryption/runtimes/java/src/test/java/software/amazon/cryptography/dbencryptionsdk/dynamodb/enhancedclient/DynamoDbEncryptionEnhancedClientIntegrationTests.java index 207fb41f4..ae84494ba 100644 --- a/DynamoDbEncryption/runtimes/java/src/test/java/software/amazon/cryptography/dbencryptionsdk/dynamodb/enhancedclient/DynamoDbEncryptionEnhancedClientIntegrationTests.java +++ b/DynamoDbEncryption/runtimes/java/src/test/java/software/amazon/cryptography/dbencryptionsdk/dynamodb/enhancedclient/DynamoDbEncryptionEnhancedClientIntegrationTests.java @@ -15,8 +15,12 @@ import software.amazon.awssdk.enhanced.dynamodb.Key; import software.amazon.awssdk.enhanced.dynamodb.TableSchema; import software.amazon.awssdk.enhanced.dynamodb.model.GetItemEnhancedRequest; +import software.amazon.awssdk.enhanced.dynamodb.model.PutItemEnhancedRequest; +import software.amazon.awssdk.enhanced.dynamodb.model.PutItemEnhancedResponse; +import software.amazon.awssdk.enhanced.dynamodb.model.UpdateItemEnhancedRequest; import software.amazon.awssdk.services.dynamodb.DynamoDbClient; import software.amazon.awssdk.services.dynamodb.model.DynamoDbException; +import software.amazon.awssdk.services.dynamodb.model.ReturnValue; import software.amazon.awssdk.services.kms.model.KmsException; import software.amazon.cryptography.dbencryptionsdk.dynamodb.enhancedclient.validdatamodels.*; @@ -31,6 +35,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.ThreadLocalRandom; import static org.testng.Assert.assertEquals; import static software.amazon.cryptography.dbencryptionsdk.dynamodb.TestUtils.*; @@ -41,6 +46,11 @@ public class DynamoDbEncryptionEnhancedClientIntegrationTests { + // Some integration tests MUST mutate the state of the DDB table. + // For such tests, include a random number in the primary key + // to avoid conflicts between distributed test runners sharing a table. + private int randomNum = ThreadLocalRandom.current().nextInt(Integer.MIN_VALUE, Integer.MAX_VALUE ); + private static DynamoDbEnhancedClient initEnhancedClientWithInterceptor( final TableSchema schemaOnEncrypt, final List allowedUnsignedAttributes, @@ -75,6 +85,46 @@ private static DynamoDbEnhancedClient initEnhancedClientWithInterceptor( .build(); } + private static DynamoDbEnhancedClient createEnhancedClientForLegacyClass(DynamoDBEncryptor oldEncryptor, TableSchema schemaOnEncrypt) { + Map legacyActions = new HashMap<>(); + legacyActions.put("partition_key", CryptoAction.SIGN_ONLY); + legacyActions.put("sort_key", CryptoAction.SIGN_ONLY); + legacyActions.put("encryptAndSign", CryptoAction.ENCRYPT_AND_SIGN); + legacyActions.put("signOnly", CryptoAction.SIGN_ONLY); + legacyActions.put("doNothing", CryptoAction.DO_NOTHING); + LegacyOverride legacyOverride = LegacyOverride + .builder() + .encryptor(oldEncryptor) + .policy(LegacyPolicy.FORCE_LEGACY_ENCRYPT_ALLOW_LEGACY_DECRYPT) + .attributeActionsOnEncrypt(legacyActions) + .build(); + + Map tableConfigs = new HashMap<>(); + tableConfigs.put(TEST_TABLE_NAME, + DynamoDbEnhancedTableEncryptionConfig.builder() + .logicalTableName(TEST_TABLE_NAME) + .keyring(createKmsKeyring()) + .allowedUnsignedAttributes(Arrays.asList("doNothing")) + .schemaOnEncrypt(schemaOnEncrypt) + .legacyOverride(legacyOverride) + .build()); + DynamoDbEncryptionInterceptor interceptor = + DynamoDbEnhancedClientEncryption.CreateDynamoDbEncryptionInterceptor( + CreateDynamoDbEncryptionInterceptorInput.builder() + .tableEncryptionConfigs(tableConfigs) + .build() + ); + DynamoDbClient ddb = DynamoDbClient.builder() + .overrideConfiguration( + ClientOverrideConfiguration.builder() + .addExecutionInterceptor(interceptor) + .build()) + .build(); + return DynamoDbEnhancedClient.builder() + .dynamoDbClient(ddb) + .build(); + } + @Test public void TestPutAndGet() { TableSchema schemaOnEncrypt = TableSchema.fromBean(SimpleClass.class); @@ -109,6 +159,35 @@ public void TestPutAndGet() { assertEquals(result.getDoNothing(), "fizzbuzz"); } + @Test + public void TestPutAndGetAllTypes() { + TableSchema schemaOnEncrypt = TableSchema.fromBean(AllTypesClass.class); + List allowedUnsignedAttributes = Collections.singletonList("doNothing"); + DynamoDbEnhancedClient enhancedClient = + initEnhancedClientWithInterceptor(schemaOnEncrypt, allowedUnsignedAttributes, null, null); + + DynamoDbTable table = enhancedClient.table(TEST_TABLE_NAME, schemaOnEncrypt); + + AllTypesClass record = AllTypesClass.createTestItem("EnhancedPutGetAllTypes", 1); + + // Put an item into DDB such that it also returns back the item. + PutItemEnhancedResponse putItemResp = table.putItemWithResponse( + (PutItemEnhancedRequest.Builder requestBuilder) + -> requestBuilder.item(record) + .returnValues(ReturnValue.ALL_OLD)); + assertEquals(putItemResp.attributes(), record); + + // Get the item back from the table + Key key = Key.builder() + .partitionValue("EnhancedPutGetAllTypes").sortValue(1) + .build(); + + // Get the item by using the key. + AllTypesClass result = table.getItem( + (GetItemEnhancedRequest.Builder requestBuilder) -> requestBuilder.key(key)); + assertEquals(result, record); + } + @Test public void TestPutAndGetAnnotatedFlattenedBean() { final String PARTITION = "AnnotatedFlattenedBean"; @@ -236,20 +315,6 @@ public void TestGetLegacyItem() { mapper.save(record); - // Configure EnhancedClient with Legacy behavior - Map legacyActions = new HashMap<>(); - legacyActions.put("partition_key", CryptoAction.SIGN_ONLY); - legacyActions.put("sort_key", CryptoAction.SIGN_ONLY); - legacyActions.put("encryptAndSign", CryptoAction.ENCRYPT_AND_SIGN); - legacyActions.put("signOnly", CryptoAction.SIGN_ONLY); - legacyActions.put("doNothing", CryptoAction.DO_NOTHING); - LegacyOverride legacyOverride = LegacyOverride - .builder() - .encryptor(oldEncryptor) - .policy(LegacyPolicy.FORCE_LEGACY_ENCRYPT_ALLOW_LEGACY_DECRYPT) - .attributeActionsOnEncrypt(legacyActions) - .build(); - TableSchema schemaOnEncrypt = TableSchema.fromBean(LegacyClass.class); DynamoDbEnhancedClient enhancedClient = createEnhancedClientForLegacyClass(oldEncryptor, schemaOnEncrypt); @@ -300,44 +365,58 @@ public void TestWriteLegacyItem() { assertEquals("fizzbuzz", result.getDoNothing()); } - private static DynamoDbEnhancedClient createEnhancedClientForLegacyClass(DynamoDBEncryptor oldEncryptor, TableSchema schemaOnEncrypt) { - Map legacyActions = new HashMap<>(); - legacyActions.put("partition_key", CryptoAction.SIGN_ONLY); - legacyActions.put("sort_key", CryptoAction.SIGN_ONLY); - legacyActions.put("encryptAndSign", CryptoAction.ENCRYPT_AND_SIGN); - legacyActions.put("signOnly", CryptoAction.SIGN_ONLY); - legacyActions.put("doNothing", CryptoAction.DO_NOTHING); - LegacyOverride legacyOverride = LegacyOverride - .builder() - .encryptor(oldEncryptor) - .policy(LegacyPolicy.FORCE_LEGACY_ENCRYPT_ALLOW_LEGACY_DECRYPT) - .attributeActionsOnEncrypt(legacyActions) - .build(); + @Test + public void TestDelete() { + TableSchema schemaOnEncrypt = TableSchema.fromBean(AllTypesClass.class); + List allowedUnsignedAttributes = Collections.singletonList("doNothing"); + DynamoDbEnhancedClient enhancedClient = + initEnhancedClientWithInterceptor(schemaOnEncrypt, allowedUnsignedAttributes, null, null); - Map tableConfigs = new HashMap<>(); - tableConfigs.put(TEST_TABLE_NAME, - DynamoDbEnhancedTableEncryptionConfig.builder() - .logicalTableName(TEST_TABLE_NAME) - .keyring(createKmsKeyring()) - .allowedUnsignedAttributes(Arrays.asList("doNothing")) - .schemaOnEncrypt(schemaOnEncrypt) - .legacyOverride(legacyOverride) - .build()); - DynamoDbEncryptionInterceptor interceptor = - DynamoDbEnhancedClientEncryption.CreateDynamoDbEncryptionInterceptor( - CreateDynamoDbEncryptionInterceptorInput.builder() - .tableEncryptionConfigs(tableConfigs) - .build() - ); - DynamoDbClient ddb = DynamoDbClient.builder() - .overrideConfiguration( - ClientOverrideConfiguration.builder() - .addExecutionInterceptor(interceptor) - .build()) - .build(); - return DynamoDbEnhancedClient.builder() - .dynamoDbClient(ddb) + DynamoDbTable table = enhancedClient.table(TEST_TABLE_NAME, schemaOnEncrypt); + + AllTypesClass record = AllTypesClass.createTestItem("EnhancedDelete", randomNum); + + // Put an item into an Amazon DynamoDB table. + table.putItem(record); + + // Get the item back from the table + Key key = Key.builder() + .partitionValue("EnhancedDelete").sortValue(randomNum) .build(); + + // Get the item by using the key. + AllTypesClass result = table.deleteItem(key); + assertEquals(result, record); + } + + @Test + public void TestUpdate() { + TableSchema schemaOnEncrypt = TableSchema.fromBean(AllTypesClass.class); + List allowedUnsignedAttributes = Collections.singletonList("doNothing"); + DynamoDbEnhancedClient enhancedClient = + initEnhancedClientWithInterceptor(schemaOnEncrypt, allowedUnsignedAttributes, null, null); + + DynamoDbTable table = enhancedClient.table(TEST_TABLE_NAME, schemaOnEncrypt); + + AllTypesClass record = AllTypesClass.createTestItem("EnhancedUpdate", 1); + + // Put an item into an Amazon DynamoDB table. + table.putItem(record); + + AllTypesClass doNothingValue = new AllTypesClass(); + doNothingValue.setDoNothing("updatedDoNothing"); + doNothingValue.setPartitionKey("EnhancedUpdate"); + doNothingValue.setSortKey(1); + + // Perform an update only on "doNothing" attribute + AllTypesClass result = table.updateItem( + (UpdateItemEnhancedRequest.Builder requestBuilder) + -> requestBuilder.item(doNothingValue) + .ignoreNulls(true) + ); + // EnhancedClient uses ReturnValues of ALL_NEW, so compare against put item with update + record.setDoNothing("updatedDoNothing"); + assertEquals(result, record); } @Test( diff --git a/DynamoDbEncryption/runtimes/java/src/test/java/software/amazon/cryptography/dbencryptionsdk/dynamodb/enhancedclient/validdatamodels/AllTypesClass.java b/DynamoDbEncryption/runtimes/java/src/test/java/software/amazon/cryptography/dbencryptionsdk/dynamodb/enhancedclient/validdatamodels/AllTypesClass.java new file mode 100644 index 000000000..ef50fd2fe --- /dev/null +++ b/DynamoDbEncryption/runtimes/java/src/test/java/software/amazon/cryptography/dbencryptionsdk/dynamodb/enhancedclient/validdatamodels/AllTypesClass.java @@ -0,0 +1,386 @@ +package software.amazon.cryptography.dbencryptionsdk.dynamodb.enhancedclient.validdatamodels; + +import software.amazon.awssdk.enhanced.dynamodb.mapper.annotations.DynamoDbAttribute; +import software.amazon.awssdk.enhanced.dynamodb.mapper.annotations.DynamoDbBean; +import software.amazon.awssdk.enhanced.dynamodb.mapper.annotations.DynamoDbIgnoreNulls; +import software.amazon.awssdk.enhanced.dynamodb.mapper.annotations.DynamoDbPartitionKey; +import software.amazon.awssdk.enhanced.dynamodb.mapper.annotations.DynamoDbSortKey; +import software.amazon.cryptography.dbencryptionsdk.dynamodb.enhancedclient.DynamoDbEncryptionDoNothing; +import software.amazon.cryptography.dbencryptionsdk.dynamodb.enhancedclient.DynamoDbEncryptionSignOnly; + +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; + +/** + * This class is used by the Enhanced Client Tests + */ + +@DynamoDbBean +public class AllTypesClass { + + private String partitionKey; + private int sortKey; + + // One attribute for every DDB attribute type and ENCRYPT_AND_SIGN/SIGN_ONLY pair + private String encryptString; + private String signString; + private Double encryptNum; + private Double signNum; + private ByteBuffer encryptBinary; + private ByteBuffer signBinary; + private Boolean encryptBool; + private Boolean signBool; + private String encryptExpectedNull; // This should always be null, define no setters + private String signExpectedNull; // This should always be null, define no setters + private List encryptList; + private List signList; + private Map encryptMap; + private Map signMap; + private Set encryptStringSet; + private Set signStringSet; + private Set encryptNumSet; + private Set signNumSet; + private Set encryptBinarySet; + private Set signBinarySet; + + // And one doNothing for good measure + private String doNothing; + + @DynamoDbPartitionKey + @DynamoDbAttribute(value = "partition_key") + public String getPartitionKey() { + return this.partitionKey; + } + + @DynamoDbSortKey + @DynamoDbAttribute(value = "sort_key") + public int getSortKey() { + return this.sortKey; + } + + @DynamoDbIgnoreNulls + public String getEncryptString() { + return this.encryptString; + } + + @DynamoDbEncryptionSignOnly + @DynamoDbIgnoreNulls + public String getSignString() { + return this.signString; + } + + @DynamoDbIgnoreNulls + public Double getEncryptNum() { + return encryptNum; + } + + @DynamoDbIgnoreNulls + @DynamoDbEncryptionSignOnly + public Double getSignNum() { + return signNum; + } + + @DynamoDbIgnoreNulls + public ByteBuffer getEncryptBinary() { + return encryptBinary; + } + + @DynamoDbIgnoreNulls + @DynamoDbEncryptionSignOnly + public ByteBuffer getSignBinary() { + return signBinary; + } + + @DynamoDbIgnoreNulls + public Boolean getEncryptBool() { + return encryptBool; + } + + @DynamoDbIgnoreNulls + @DynamoDbEncryptionSignOnly + public Boolean getSignBool() { + return signBool; + } + + // This should always return null + public String getEncryptExpectedNull() { + return encryptExpectedNull; + } + + // This should always return null + @DynamoDbEncryptionSignOnly + public String getSignExpectedNull() { + return signExpectedNull; + } + + @DynamoDbIgnoreNulls + public List getEncryptList() { + return encryptList; + } + + @DynamoDbIgnoreNulls + @DynamoDbEncryptionSignOnly + public List getSignList() { + return signList; + } + + @DynamoDbIgnoreNulls + public Map getEncryptMap() { + return encryptMap; + } + + @DynamoDbIgnoreNulls + @DynamoDbEncryptionSignOnly + public Map getSignMap() { + return signMap; + } + + @DynamoDbIgnoreNulls + public Set getEncryptStringSet() { + return encryptStringSet; + } + + @DynamoDbIgnoreNulls + @DynamoDbEncryptionSignOnly + public Set getSignStringSet() { + return signStringSet; + } + + @DynamoDbIgnoreNulls + public Set getEncryptNumSet() { + return encryptNumSet; + } + + @DynamoDbIgnoreNulls + @DynamoDbEncryptionSignOnly + public Set getSignNumSet() { + return signNumSet; + } + + @DynamoDbIgnoreNulls + public Set getEncryptBinarySet() { + return encryptBinarySet; + } + + @DynamoDbIgnoreNulls + @DynamoDbEncryptionSignOnly + public Set getSignBinarySet() { + return signBinarySet; + } + + @DynamoDbIgnoreNulls + @DynamoDbEncryptionDoNothing + public String getDoNothing() { + return this.doNothing; + } + + public void setPartitionKey(String partitionKey) { + this.partitionKey = partitionKey; + } + + public void setSortKey(int sortKey) { + this.sortKey = sortKey; + } + + public void setEncryptString(String encryptString) { + this.encryptString = encryptString; + } + + public void setSignString(String signString) { + this.signString = signString; + } + + public void setEncryptNum(Double encryptNum) { + this.encryptNum = encryptNum; + } + + public void setSignNum(Double signNum) { + this.signNum = signNum; + } + + public void setEncryptBinary(ByteBuffer encryptBinary) { + this.encryptBinary = encryptBinary; + } + + public void setSignBinary(ByteBuffer signBinary) { + this.signBinary = signBinary; + } + + public void setEncryptBool(Boolean encryptBool) { + this.encryptBool = encryptBool; + } + + public void setSignBool(Boolean signBool) { + this.signBool = signBool; + } + + public void setEncryptList(List encryptList) { + this.encryptList = encryptList; + } + + public void setSignList(List signList) { + this.signList = signList; + } + + public void setEncryptMap(Map encryptMap) { + this.encryptMap = encryptMap; + } + + public void setSignMap(Map signMap) { + this.signMap = signMap; + } + + public void setEncryptStringSet(Set encryptStringSet) { + this.encryptStringSet = encryptStringSet; + } + + public void setSignStringSet(Set signStringSet) { + this.signStringSet = signStringSet; + } + + public void setEncryptNumSet(Set encryptNumSet) { + this.encryptNumSet = encryptNumSet; + } + + public void setSignNumSet(Set signNumSet) { + this.signNumSet = signNumSet; + } + + public void setEncryptBinarySet(Set encryptBinarySet) { + this.encryptBinarySet = encryptBinarySet; + } + + public void setSignBinarySet(Set signBinarySet) { + this.signBinarySet = signBinarySet; + } + + public void setDoNothing(String doNothing) { + this.doNothing = doNothing; + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + + if (obj.getClass() != this.getClass()) { + return false; + } + + final AllTypesClass other = (AllTypesClass) obj; + + if (!(Objects.equals(other.getPartitionKey(), this.partitionKey))) { + return false; + } + if (other.getSortKey() != this.sortKey) { + return false; + } + if (!(Objects.equals(other.getEncryptString(), this.encryptString))) { + return false; + } + if (!(Objects.equals(other.getSignString(), this.signString))) { + return false; + } + if (!(Objects.equals(other.getEncryptNum(), this.encryptNum))) { + return false; + } + if (!(Objects.equals(other.getSignNum(), this.signNum))) { + return false; + } + if (!(Objects.equals(other.getEncryptBinary(), this.encryptBinary))) { + return false; + } + if (!(Objects.equals(other.getSignBinary(), this.signBinary))) { + return false; + } + if (!(Objects.equals(other.getEncryptBool(), this.encryptBool))) { + return false; + } + if (!(Objects.equals(other.getSignBool(), this.signBool))) { + return false; + } + if (other.getEncryptExpectedNull() != null) { + return false; + } + if (other.getSignExpectedNull() != null) { + return false; + } + if (!(Objects.equals(other.getEncryptList(), this.encryptList))) { + return false; + } + if (!(Objects.equals(other.getSignList(), this.signList))) { + return false; + } + if (!(Objects.equals(other.getEncryptMap(), this.encryptMap))) { + return false; + } + if (!(Objects.equals(other.getSignMap(), this.signMap))) { + return false; + } + if (!(Objects.equals(other.getEncryptStringSet(), this.encryptStringSet))) { + return false; + } + if (!(Objects.equals(other.getSignStringSet(), this.signStringSet))) { + return false; + } + if (!(Objects.equals(other.getEncryptNumSet(), this.encryptNumSet))) { + return false; + } + if (!(Objects.equals(other.getSignNumSet(), this.signNumSet))) { + return false; + } + if (!(Objects.equals(other.getEncryptBinarySet(), this.encryptBinarySet))) { + return false; + } + if (!(Objects.equals(other.getSignBinarySet(), this.signBinarySet))) { + return false; + } + if (!(Objects.equals(other.getDoNothing(), this.doNothing))) { + return false; + } + + return true; + } + + public static AllTypesClass createTestItem(String partitionValue, int sortValue) { + AllTypesClass testItem = new AllTypesClass(); + testItem.setPartitionKey(partitionValue); + testItem.setSortKey(sortValue); + testItem.setEncryptString("encryptString"); + testItem.setSignString("signString"); + testItem.setEncryptNum(111.111); + testItem.setSignNum(999.999); + testItem.setEncryptBinary(StandardCharsets.UTF_8.encode("encryptBinary")); + testItem.setSignBinary(StandardCharsets.UTF_8.encode("sortBinary")); + testItem.setEncryptBool(true); + testItem.setSignBool(false); + testItem.setEncryptList(Arrays.asList("encrypt1", "encrypt2", "encrypt3")); + testItem.setSignList(Arrays.asList("sort1", "sort2", "sort3")); + testItem.setEncryptMap(Collections.singletonMap("encryptMap", 1)); + testItem.setSignMap(Collections.singletonMap("sortMap", 2)); + testItem.setEncryptStringSet(new HashSet<>(Arrays.asList("encrypt1", "encrypt2", "encrypt3"))); + testItem.setSignStringSet(new HashSet<>(Arrays.asList("sort1", "sort2", "sort3"))); + testItem.setEncryptNumSet(new HashSet<>(Arrays.asList(1, 2, 3))); + testItem.setSignNumSet(new HashSet<>(Arrays.asList(4, 5, 6))); + testItem.setEncryptBinarySet(new HashSet<>(Arrays.asList( + StandardCharsets.UTF_8.encode("encrypt1"), + StandardCharsets.UTF_8.encode("encrypt2"), + StandardCharsets.UTF_8.encode("encrypt3") + ))); + testItem.setSignBinarySet(new HashSet<>(Arrays.asList( + StandardCharsets.UTF_8.encode("sort1"), + StandardCharsets.UTF_8.encode("sort2"), + StandardCharsets.UTF_8.encode("sort3") + ))); + testItem.setDoNothing("doNothing"); + return testItem; + } +} diff --git a/specification/dynamodb-encryption-client/ddb-sdk-integration.md b/specification/dynamodb-encryption-client/ddb-sdk-integration.md index 0cb2716fb..3f48a692a 100644 --- a/specification/dynamodb-encryption-client/ddb-sdk-integration.md +++ b/specification/dynamodb-encryption-client/ddb-sdk-integration.md @@ -124,6 +124,9 @@ MUST have the following modified behavior: - [Encrypt before BatchWriteItem](#encrypt-before-batchwriteitem) - [Encrypt before TransactWriteItems](#encrypt-before-transactwriteitems) - [Decrypt after GetItem](#decrypt-after-getitem) +- [Decrypt after PutItem](#decrypt-after-putitem) +- [Decrypt after UpdateItem](#decrypt-after-updateitem) +- [Decrypt after DeleteItem](#decrypt-after-deleteitem) - [Decrypt after BatchGetItem](#decrypt-after-batchgetitem) - [Decrypt after Scan](#decrypt-after-scan) - [Decrypt after Query](#decrypt-after-query) @@ -334,6 +337,92 @@ Beacons MUST be [removed](ddb-support.md#removebeacons) from the result. The GetItem response's `Item` field MUST be replaced by the encrypted DynamoDb Item outputted above. +### Decrypt after PutItem + +After a [PutItem](https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_PutItem.html) +call is made to DynamoDB, +the resulting response MUST be modified before +being returned to the caller if: +- there exists an Item Encryptor specified within the + [DynamoDB Encryption Client Config](#dynamodb-encryption-client-configuration) + with a [DynamoDB Table Name](./ddb-item-encryptor.md#dynamodb-table-name) + equal to the `TableName` on the PutItem request. +- the response contains [Attributes](https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_PutItem.html#DDB-PutItem-response-Attributes). + The response will contain Attributes if the related PutItem request's + [ReturnValues](https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_PutItem.html#DDB-PutItem-request-ReturnValues) + had a value of `ALL_OLD` and the PutItem call replaced a pre-existing item. + +In this case, the [Item Encryptor](./ddb-item-encryptor.md) MUST perform +[Decrypt Item](./decrypt-item.md) where the input +[DynamoDB Item](./decrypt-item.md#dynamodb-item) +is the `Attributes` field in the original response + +Beacons MUST be [removed](ddb-support.md#removebeacons) from the result. + +The PutItem response's `Attributes` field MUST be +replaced by the encrypted DynamoDb Item outputted above. + +### Decrypt after DeleteItem + +After a [DeleteItem](https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_DeleteItem.html) +call is made to DynamoDB, +the resulting response MUST be modified before +being returned to the caller if: +- there exists an Item Encryptor specified within the + [DynamoDB Encryption Client Config](#dynamodb-encryption-client-configuration) + with a [DynamoDB Table Name](./ddb-item-encryptor.md#dynamodb-table-name) + equal to the `TableName` on the DeleteItem request. +- the response contains [Attributes](https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_DeleteItem.html#DDB-DeleteItem-response-Attributes). + The response will contain Attributes if the related DeleteItem request's + [ReturnValues](https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_DeleteItem.html#DDB-DeleteItem-request-ReturnValues) + had a value of `ALL_OLD` and an item was deleted. + +In this case, the [Item Encryptor](./ddb-item-encryptor.md) MUST perform +[Decrypt Item](./decrypt-item.md) where the input +[DynamoDB Item](./decrypt-item.md#dynamodb-item) +is the `Attributes` field in the original response + +Beacons MUST be [removed](ddb-support.md#removebeacons) from the result. + +The DeleteItem response's `Attributes` field MUST be +replaced by the encrypted DynamoDb Item outputted above. + +### Decrypt after UpdateItem + +After a [UpdateItem](https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_UpdateItem.html) +call is made to DynamoDB, +the resulting response MUST be modified before +being returned to the caller if: +- there exists an Item Encryptor specified within the + [DynamoDB Encryption Client Config](#dynamodb-encryption-client-configuration) + with a [DynamoDB Table Name](./ddb-item-encryptor.md#dynamodb-table-name) + equal to the `TableName` on the UpdateItem request. +- the response contains [Attributes](https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_UpdateItem.html#DDB-UpdateItem-response-Attributes). +- the original UpdateItem request had a + [ReturnValues](https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_UpdateItem.html#DDB-UpdateItem-request-ReturnValues) + with a value of `ALL_OLD` or `ALL_NEW`. + +In this case, the [Item Encryptor](./ddb-item-encryptor.md) MUST perform +[Decrypt Item](./decrypt-item.md) where the input +[DynamoDB Item](./decrypt-item.md#dynamodb-item) +is the `Attributes` field in the original response + +Beacons MUST be [removed](ddb-support.md#removebeacons) from the result. + +The UpdateItem response's `Attributes` field MUST be +replaced by the encrypted DynamoDb Item outputted above. + +In all other cases, the UpdateItem response MUST NOT be modified. + +Additionally, if a value of `UPDATED_OLD` or `UPDATED_NEW` was used, +and any Attributes in the response are authenticated +per the [DynamoDB Encryption Client Config](#dynamodb-encryption-client-configuration), +an error MUST be raised. +Given that we [validate UpdateItem requests](#validate-before-updateitem), +and thus updates will not modify any signed field, +an error here would indicate a bug in +our library or a bug within DynamoDB. + ### Decrypt after BatchGetItem After a [BatchGetItem](https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_BatchGetItem.html) From 44e5fae226be58158dd1f744d809baa10bd76a12 Mon Sep 17 00:00:00 2001 From: Andy Jewell Date: Tue, 14 Nov 2023 15:47:03 -0500 Subject: [PATCH 4/5] fix: repair examples The examples relied on order that has been fixed --- ...aconStylesSearchableEncryptionExample.java | 8 ++-- .../complexexample/QueryRequests.java | 42 ++++++++++++------- 2 files changed, 31 insertions(+), 19 deletions(-) diff --git a/Examples/runtimes/java/DynamoDbEncryption/src/main/java/software/amazon/cryptography/examples/searchableencryption/BeaconStylesSearchableEncryptionExample.java b/Examples/runtimes/java/DynamoDbEncryption/src/main/java/software/amazon/cryptography/examples/searchableencryption/BeaconStylesSearchableEncryptionExample.java index d3ec132e8..4700374cd 100644 --- a/Examples/runtimes/java/DynamoDbEncryption/src/main/java/software/amazon/cryptography/examples/searchableencryption/BeaconStylesSearchableEncryptionExample.java +++ b/Examples/runtimes/java/DynamoDbEncryption/src/main/java/software/amazon/cryptography/examples/searchableencryption/BeaconStylesSearchableEncryptionExample.java @@ -195,8 +195,8 @@ public static void PutItemQueryItemWithBeaconStyles(String ddbTableName, String item1.put("dessert", AttributeValue.builder().s("cake").build()); item1.put("fruit", AttributeValue.builder().s("banana").build()); ArrayList basket = new ArrayList(); - basket.add("banana"); basket.add("apple"); + basket.add("banana"); basket.add("pear"); item1.put("basket", AttributeValue.builder().ss(basket).build()); @@ -207,9 +207,9 @@ public static void PutItemQueryItemWithBeaconStyles(String ddbTableName, String item2.put("fruit", AttributeValue.builder().s("orange").build()); item2.put("dessert", AttributeValue.builder().s("orange").build()); basket = new ArrayList(); - basket.add("strawberry"); - basket.add("blueberry"); basket.add("blackberry"); + basket.add("blueberry"); + basket.add("strawberry"); item2.put("basket", AttributeValue.builder().ss(basket).build()); // 10. Create the DynamoDb Encryption Interceptor @@ -283,8 +283,8 @@ public static void PutItemQueryItemWithBeaconStyles(String ddbTableName, String // Select records where the fruit attribute exists in a particular set ArrayList basket3 = new ArrayList(); basket3.add("boysenberry"); - basket3.add("orange"); basket3.add("grape"); + basket3.add("orange"); expressionAttributeValues.put(":value", AttributeValue.builder().ss(basket3).build()); scanRequest = ScanRequest.builder() diff --git a/Examples/runtimes/java/DynamoDbEncryption/src/main/java/software/amazon/cryptography/examples/searchableencryption/complexexample/QueryRequests.java b/Examples/runtimes/java/DynamoDbEncryption/src/main/java/software/amazon/cryptography/examples/searchableencryption/complexexample/QueryRequests.java index c91354121..4075a3994 100644 --- a/Examples/runtimes/java/DynamoDbEncryption/src/main/java/software/amazon/cryptography/examples/searchableencryption/complexexample/QueryRequests.java +++ b/Examples/runtimes/java/DynamoDbEncryption/src/main/java/software/amazon/cryptography/examples/searchableencryption/complexexample/QueryRequests.java @@ -582,22 +582,34 @@ public static void runQuery14(String ddbTableName, DynamoDbClient ddb) { .expressionAttributeValues(query14AttributeValues) .build(); - QueryResponse query14Response = ddb.query(query14Request); - // Validate query was returned successfully - assert 200 == query14Response.sdkHttpResponse().statusCode(); - - // Assert 1 item was returned: `employee1` - assert query14Response.items().size() == 1; - // Known value test: Assert some properties on one of the items - boolean foundKnownValueItemQuery14 = false; - for (Map item : query14Response.items()) { - if (item.get("partition_key").s().equals("employee1")) { - foundKnownValueItemQuery14 = true; - assert item.get("EmployeeID").s().equals("emp_001"); - assert item.get("Location").m().get("Desk").s().equals("3"); - } + // GSIs do not update instantly + // so if the results come back empty + // we retry after a short sleep + for (int i=0; i<10; ++i) { + QueryResponse query14Response = ddb.query(query14Request); + // Validate query was returned successfully + assert 200 == query14Response.sdkHttpResponse().statusCode(); + + // if no results, sleep and try again + if (query14Response.items().size() == 0) { + try {Thread.sleep(20);} catch (Exception e) {} + continue; + } + + // Assert 1 item was returned: `employee1` + assert query14Response.items().size() == 1; + // Known value test: Assert some properties on one of the items + boolean foundKnownValueItemQuery14 = false; + for (Map item : query14Response.items()) { + if (item.get("partition_key").s().equals("employee1")) { + foundKnownValueItemQuery14 = true; + assert item.get("EmployeeID").s().equals("emp_001"); + assert item.get("Location").m().get("Desk").s().equals("3"); + } + } + assert foundKnownValueItemQuery14; + break; } - assert foundKnownValueItemQuery14; } public static void runQuery15(String ddbTableName, DynamoDbClient ddb) { From f68ec3c61b94f7b6641548e95448a719e1ba3d08 Mon Sep 17 00:00:00 2001 From: seebees Date: Wed, 15 Nov 2023 13:21:37 -0800 Subject: [PATCH 5/5] chore: Update README.md Co-authored-by: lavaleri <49660121+lavaleri@users.noreply.github.com> --- README.md | 4 ++-- project.properties | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 5441851b3..59ab51d3e 100644 --- a/README.md +++ b/README.md @@ -59,7 +59,7 @@ To use the DB-ESDK for DynamoDB in Java, you must have: * **Via Gradle Kotlin** In a Gradle Java Project, add the following to the _dependencies_ section: ```kotlin - implementation("software.amazon.cryptography:aws-database-encryption-sdk-dynamodb:3.1.1") + implementation("software.amazon.cryptography:aws-database-encryption-sdk-dynamodb:3.1.2") implementation("software.amazon.cryptography:aws-cryptographic-material-providers:1.0.0") implementation(platform("software.amazon.awssdk:bom:2.19.1")) implementation("software.amazon.awssdk:dynamodb") @@ -92,7 +92,7 @@ To use the DB-ESDK for DynamoDB in Java, you must have: software.amazon.cryptography aws-database-encryption-sdk-dynamodb - 3.1.1 + 3.1.2 software.amazon.cryptography diff --git a/project.properties b/project.properties index aa393fed9..f3fd2a886 100644 --- a/project.properties +++ b/project.properties @@ -1,4 +1,4 @@ -projectJavaVersion=3.1.0 +projectJavaVersion=3.1.2 mplDependencyJavaVersion=1.0.0 dafnyRuntimeJavaVersion=4.2.0 smithyDafnyJavaConversionVersion=0.1