diff --git a/cli/collection_create.go b/cli/collection_create.go index f4c36fbd53..eecdfef2d8 100644 --- a/cli/collection_create.go +++ b/cli/collection_create.go @@ -17,25 +17,34 @@ import ( "github.com/spf13/cobra" "github.com/sourcenetwork/defradb/client" + "github.com/sourcenetwork/defradb/datastore" "github.com/sourcenetwork/defradb/internal/db" + "github.com/sourcenetwork/defradb/internal/encryption" ) func MakeCollectionCreateCommand() *cobra.Command { var file string - var shouldEncrypt bool + var shouldEncryptDoc bool + var encryptedFields []string var cmd = &cobra.Command{ - Use: "create [-i --identity] [-e --encrypt] ", + Use: "create [-i --identity] [-e --encrypt] [--encrypt-fields] ", Short: "Create a new document.", Long: `Create a new document. Options: - -i, --identity - Marks the document as private and set the identity as the owner. The access to the document + -i, --identity + Marks the document as private and set the identity as the owner. The access to the document and permissions are controlled by ACP (Access Control Policy). -e, --encrypt Encrypt flag specified if the document needs to be encrypted. If set, DefraDB will generate a symmetric key for encryption using AES-GCM. + + --encrypt-fields + Comma-separated list of fields to encrypt. If set, DefraDB will encrypt only the specified fields + and for every field in the list it will generate a symmetric key for encryption using AES-GCM. + If combined with '--encrypt' flag, all the fields in the document not listed in '--encrypt-fields' + will be encrypted with the same key. Example: create from string: defradb client collection create --name User '{ "name": "Bob" }' @@ -81,7 +90,7 @@ Example: create from stdin: } txn, _ := db.TryGetContextTxn(cmd.Context()) - setContextDocEncryption(cmd, shouldEncrypt, txn) + setContextDocEncryption(cmd, shouldEncryptDoc, encryptedFields, txn) if client.IsJSONArray(docData) { docs, err := client.NewDocsFromJSON(docData, col.Definition()) @@ -98,8 +107,23 @@ Example: create from stdin: return col.Create(cmd.Context(), doc) }, } - cmd.PersistentFlags().BoolVarP(&shouldEncrypt, "encrypt", "e", false, + cmd.PersistentFlags().BoolVarP(&shouldEncryptDoc, "encrypt", "e", false, "Flag to enable encryption of the document") + cmd.PersistentFlags().StringSliceVar(&encryptedFields, "encrypt-fields", nil, + "Comma-separated list of fields to encrypt") cmd.Flags().StringVarP(&file, "file", "f", "", "File containing document(s)") return cmd } + +// setContextDocEncryption sets doc encryption for the current command context. +func setContextDocEncryption(cmd *cobra.Command, shouldEncryptDoc bool, encryptFields []string, txn datastore.Txn) { + if !shouldEncryptDoc && len(encryptFields) == 0 { + return + } + ctx := cmd.Context() + if txn != nil { + ctx = encryption.ContextWithStore(ctx, txn) + } + ctx = encryption.SetContextConfigFromParams(ctx, shouldEncryptDoc, encryptFields) + cmd.SetContext(ctx) +} diff --git a/cli/utils.go b/cli/utils.go index b2d4c076bc..d1ee09962b 100644 --- a/cli/utils.go +++ b/cli/utils.go @@ -25,10 +25,8 @@ import ( acpIdentity "github.com/sourcenetwork/defradb/acp/identity" "github.com/sourcenetwork/defradb/client" - "github.com/sourcenetwork/defradb/datastore" "github.com/sourcenetwork/defradb/http" "github.com/sourcenetwork/defradb/internal/db" - "github.com/sourcenetwork/defradb/internal/encryption" "github.com/sourcenetwork/defradb/keyring" ) @@ -162,19 +160,6 @@ func setContextIdentity(cmd *cobra.Command, privateKeyHex string) error { return nil } -// setContextDocEncryption sets doc encryption for the current command context. -func setContextDocEncryption(cmd *cobra.Command, shouldEncrypt bool, txn datastore.Txn) { - if !shouldEncrypt { - return - } - ctx := cmd.Context() - if txn != nil { - ctx = encryption.ContextWithStore(ctx, txn) - } - ctx = encryption.SetContextConfig(ctx, encryption.DocEncConfig{IsEncrypted: true}) - cmd.SetContext(ctx) -} - // setContextRootDir sets the rootdir for the current command context. func setContextRootDir(cmd *cobra.Command) error { rootdir, err := cmd.Root().PersistentFlags().GetString("rootdir") diff --git a/client/request/consts.go b/client/request/consts.go index cba609b788..ed39cfd4a7 100644 --- a/client/request/consts.go +++ b/client/request/consts.go @@ -26,7 +26,8 @@ const ( FieldIDName = "fieldId" ShowDeleted = "showDeleted" - EncryptArgName = "encrypt" + EncryptDocArgName = "encrypt" + EncryptFieldsArgName = "encryptFields" FilterClause = "filter" GroupByClause = "groupBy" diff --git a/client/request/mutation.go b/client/request/mutation.go index 70d0bed1d9..4f4f06fe7b 100644 --- a/client/request/mutation.go +++ b/client/request/mutation.go @@ -50,6 +50,9 @@ type ObjectMutation struct { // Encrypt is a boolean flag that indicates whether the input data should be encrypted. Encrypt bool + + // EncryptFields is a list of doc fields from input data that should be encrypted. + EncryptFields []string } // ToSelect returns a basic Select object, with the same Name, Alias, and Fields as diff --git a/datastore/prefix_query.go b/datastore/prefix_query.go index 7150aebe48..13d055893c 100644 --- a/datastore/prefix_query.go +++ b/datastore/prefix_query.go @@ -13,6 +13,7 @@ package datastore import ( "context" "encoding/json" + "errors" ds "github.com/ipfs/go-datastore" @@ -35,15 +36,13 @@ func DeserializePrefix[T any]( elements := make([]T, 0) for res := range q.Next() { if res.Error != nil { - _ = q.Close() - return nil, nil, res.Error + return nil, nil, errors.Join(res.Error, q.Close()) } var element T err = json.Unmarshal(res.Value, &element) if err != nil { - _ = q.Close() - return nil, nil, NewErrInvalidStoredValue(err) + return nil, nil, errors.Join(NewErrInvalidStoredValue(err), q.Close()) } keys = append(keys, res.Key) elements = append(elements, element) @@ -68,8 +67,7 @@ func FetchKeysForPrefix( keys := make([]ds.Key, 0) for res := range q.Next() { if res.Error != nil { - _ = q.Close() - return nil, res.Error + return nil, errors.Join(res.Error, q.Close()) } keys = append(keys, ds.NewKey(res.Key)) } diff --git a/docs/data_format_changes/i2817-doc-field-encryption.md b/docs/data_format_changes/i2817-doc-field-encryption.md new file mode 100644 index 0000000000..0d54f4e872 --- /dev/null +++ b/docs/data_format_changes/i2817-doc-field-encryption.md @@ -0,0 +1,3 @@ +# Doc field encryption +Changed hard-coded constant test encryption key to be dependant on docID and fieldName. +This produces different CIDs. \ No newline at end of file diff --git a/docs/website/references/cli/defradb_client_collection_create.md b/docs/website/references/cli/defradb_client_collection_create.md index 5425e3f860..862d30cf2e 100644 --- a/docs/website/references/cli/defradb_client_collection_create.md +++ b/docs/website/references/cli/defradb_client_collection_create.md @@ -7,13 +7,19 @@ Create a new document. Create a new document. Options: - -i, --identity - Marks the document as private and set the identity as the owner. The access to the document + -i, --identity + Marks the document as private and set the identity as the owner. The access to the document and permissions are controlled by ACP (Access Control Policy). -e, --encrypt Encrypt flag specified if the document needs to be encrypted. If set, DefraDB will generate a symmetric key for encryption using AES-GCM. + + --encrypt-fields + Comma-separated list of fields to encrypt. If set, DefraDB will encrypt only the specified fields + and for every field in the list it will generate a symmetric key for encryption using AES-GCM. + If combined with '--encrypt' flag, all the fields in the document not listed in '--encrypt-fields' + will be encrypted with the same key. Example: create from string: defradb client collection create --name User '{ "name": "Bob" }' @@ -33,15 +39,16 @@ Example: create from stdin: ``` -defradb client collection create [-i --identity] [-e --encrypt] [flags] +defradb client collection create [-i --identity] [-e --encrypt] [--encrypt-fields] [flags] ``` ### Options ``` - -e, --encrypt Flag to enable encryption of the document - -f, --file string File containing document(s) - -h, --help help for create + -e, --encrypt Flag to enable encryption of the document + --encrypt-fields strings Comma-separated list of fields to encrypt + -f, --file string File containing document(s) + -h, --help help for create ``` ### Options inherited from parent commands diff --git a/http/client_collection.go b/http/client_collection.go index 8df094f5fc..c13e4c68e9 100644 --- a/http/client_collection.go +++ b/http/client_collection.go @@ -132,9 +132,14 @@ func (c *Collection) CreateMany( func setDocEncryptionFlagIfNeeded(ctx context.Context, req *http.Request) { encConf := encryption.GetContextConfig(ctx) - if encConf.HasValue() && encConf.Value().IsEncrypted { + if encConf.HasValue() { q := req.URL.Query() - q.Set(docEncryptParam, "true") + if encConf.Value().IsDocEncrypted { + q.Set(docEncryptParam, "true") + } + if len(encConf.Value().EncryptedFields) > 0 { + q.Set(docEncryptFieldsParam, strings.Join(encConf.Value().EncryptedFields, ",")) + } req.URL.RawQuery = q.Encode() } } diff --git a/http/handler_collection.go b/http/handler_collection.go index 412f486602..57202a4b9f 100644 --- a/http/handler_collection.go +++ b/http/handler_collection.go @@ -16,6 +16,7 @@ import ( "io" "net/http" "strconv" + "strings" "github.com/getkin/kin-openapi/openapi3" "github.com/go-chi/chi/v5" @@ -25,6 +26,7 @@ import ( ) const docEncryptParam = "encrypt" +const docEncryptFieldsParam = "encryptFields" type collectionHandler struct{} @@ -47,8 +49,16 @@ func (s *collectionHandler) Create(rw http.ResponseWriter, req *http.Request) { } ctx := req.Context() - if req.URL.Query().Get(docEncryptParam) == "true" { - ctx = encryption.SetContextConfig(ctx, encryption.DocEncConfig{IsEncrypted: true}) + q := req.URL.Query() + encConf := encryption.DocEncConfig{} + if q.Get(docEncryptParam) == "true" { + encConf.IsDocEncrypted = true + } + if q.Get(docEncryptFieldsParam) != "" { + encConf.EncryptedFields = strings.Split(q.Get(docEncryptFieldsParam), ",") + } + if encConf.IsDocEncrypted || len(encConf.EncryptedFields) > 0 { + ctx = encryption.SetContextConfig(ctx, encConf) } switch { diff --git a/internal/core/key.go b/internal/core/key.go index 9b56c421fc..8f0ab3fd4e 100644 --- a/internal/core/key.go +++ b/internal/core/key.go @@ -795,25 +795,25 @@ func bytesPrefixEnd(b []byte) []byte { // EncStoreDocKey is a key for the encryption store. type EncStoreDocKey struct { - DocID string - FieldID uint32 + DocID string + FieldName string } var _ Key = (*EncStoreDocKey)(nil) // NewEncStoreDocKey creates a new EncStoreDocKey from a docID and fieldID. -func NewEncStoreDocKey(docID string, fieldID uint32) EncStoreDocKey { +func NewEncStoreDocKey(docID string, fieldName string) EncStoreDocKey { return EncStoreDocKey{ - DocID: docID, - FieldID: fieldID, + DocID: docID, + FieldName: fieldName, } } func (k EncStoreDocKey) ToString() string { - if k.FieldID == 0 { + if k.FieldName == "" { return k.DocID } - return fmt.Sprintf("%s/%d", k.DocID, k.FieldID) + return fmt.Sprintf("%s/%s", k.DocID, k.FieldName) } func (k EncStoreDocKey) Bytes() []byte { diff --git a/internal/db/collection.go b/internal/db/collection.go index 64f90960cc..d501afc4b7 100644 --- a/internal/db/collection.go +++ b/internal/db/collection.go @@ -33,6 +33,7 @@ import ( "github.com/sourcenetwork/defradb/internal/db/base" "github.com/sourcenetwork/defradb/internal/db/description" "github.com/sourcenetwork/defradb/internal/db/fetcher" + "github.com/sourcenetwork/defradb/internal/encryption" "github.com/sourcenetwork/defradb/internal/lens" merklecrdt "github.com/sourcenetwork/defradb/internal/merkle/crdt" ) @@ -561,6 +562,27 @@ func (c *collection) Save( return txn.Commit(ctx) } +func (c *collection) validateEncryptedFields(ctx context.Context) error { + encConf := encryption.GetContextConfig(ctx) + if !encConf.HasValue() { + return nil + } + fields := encConf.Value().EncryptedFields + if len(fields) == 0 { + return nil + } + + for _, field := range fields { + if _, exists := c.Schema().GetFieldByName(field); !exists { + return client.NewErrFieldNotExist(field) + } + if strings.HasPrefix(field, "_") { + return NewErrCanNotEncryptBuiltinField(field) + } + } + return nil +} + // save saves the document state. save MUST not be called outside the `c.create` // and `c.update` methods as we wrap the acp logic within those methods. Calling // save elsewhere could cause the omission of acp checks. @@ -569,6 +591,10 @@ func (c *collection) save( doc *client.Document, isCreate bool, ) (cid.Cid, error) { + if err := c.validateEncryptedFields(ctx); err != nil { + return cid.Undef, err + } + if !isCreate { err := c.updateIndexedDoc(ctx, doc) if err != nil { @@ -657,7 +683,7 @@ func (c *collection) save( return cid.Undef, err } - link, _, err := merkleCRDT.Save(ctx, &merklecrdt.DocField{DocID: primaryKey.DocID, FieldValue: val}) + link, _, err := merkleCRDT.Save(ctx, merklecrdt.NewDocField(primaryKey.DocID, k, val)) if err != nil { return cid.Undef, err } diff --git a/internal/db/errors.go b/internal/db/errors.go index 603cee8130..87828b78a8 100644 --- a/internal/db/errors.go +++ b/internal/db/errors.go @@ -100,6 +100,7 @@ const ( errReplicatorDocID string = "failed to get docID for replicator" errReplicatorCollections string = "failed to get collections for replicator" errReplicatorNotFound string = "replicator not found" + errCanNotEncryptBuiltinField string = "can not encrypt build-in field" ) var ( @@ -140,6 +141,7 @@ var ( ErrSelfTargetForReplicator = errors.New("can't target ourselves as a replicator") ErrReplicatorCollections = errors.New(errReplicatorCollections) ErrReplicatorNotFound = errors.New(errReplicatorNotFound) + ErrCanNotEncryptBuiltinField = errors.New(errCanNotEncryptBuiltinField) ) // NewErrFailedToGetHeads returns a new error indicating that the heads of a document @@ -330,6 +332,10 @@ func NewErrCannotMoveField(name string, proposedIndex, existingIndex int) error ) } +func NewErrCanNotEncryptBuiltinField(name string) error { + return errors.New(errCanNotEncryptBuiltinField, errors.NewKV("Name", name)) +} + func NewErrCannotDeleteField(name string) error { return errors.New( errCannotDeleteField, diff --git a/internal/encryption/config.go b/internal/encryption/config.go index ddb4a3815a..91edb98824 100644 --- a/internal/encryption/config.go +++ b/internal/encryption/config.go @@ -12,5 +12,8 @@ package encryption // DocEncConfig is the configuration for document encryption. type DocEncConfig struct { - IsEncrypted bool + // IsDocEncrypted is a flag to indicate if the document should be encrypted. + IsDocEncrypted bool + // EncryptedFields is a list of fields individual that should be encrypted. + EncryptedFields []string } diff --git a/internal/encryption/context.go b/internal/encryption/context.go index 10a03c89c1..96e90a7e0c 100644 --- a/internal/encryption/context.go +++ b/internal/encryption/context.go @@ -29,16 +29,13 @@ type configContextKey struct{} func TryGetContextEncryptor(ctx context.Context) (*DocEncryptor, bool) { enc, ok := ctx.Value(docEncContextKey{}).(*DocEncryptor) if ok { - checkKeyGenerationFlag(ctx, enc) + setConfig(ctx, enc) } return enc, ok } -func checkKeyGenerationFlag(ctx context.Context, enc *DocEncryptor) { - encConfig := GetContextConfig(ctx) - if encConfig.HasValue() && encConfig.Value().IsEncrypted { - enc.EnableKeyGeneration() - } +func setConfig(ctx context.Context, enc *DocEncryptor) { + enc.SetConfig(GetContextConfig(ctx)) } func ensureContextWithDocEnc(ctx context.Context) (context.Context, *DocEncryptor) { @@ -71,3 +68,16 @@ func GetContextConfig(ctx context.Context) immutable.Option[DocEncConfig] { func SetContextConfig(ctx context.Context, encConfig DocEncConfig) context.Context { return context.WithValue(ctx, configContextKey{}, encConfig) } + +// SetContextConfigFromParams returns a new context with the doc encryption config created from given params. +// If encryptDoc is false, and encryptFields is empty, the context is returned as is. +func SetContextConfigFromParams(ctx context.Context, encryptDoc bool, encryptFields []string) context.Context { + if encryptDoc || len(encryptFields) > 0 { + conf := DocEncConfig{EncryptedFields: encryptFields} + if encryptDoc { + conf.IsDocEncrypted = true + } + ctx = SetContextConfig(ctx, conf) + } + return ctx +} diff --git a/internal/encryption/encryptor.go b/internal/encryption/encryptor.go index 596e9f9903..9a6cb8f6f0 100644 --- a/internal/encryption/encryptor.go +++ b/internal/encryption/encryptor.go @@ -18,6 +18,8 @@ import ( ds "github.com/ipfs/go-datastore" + "github.com/sourcenetwork/immutable" + "github.com/sourcenetwork/defradb/datastore" "github.com/sourcenetwork/defradb/internal/core" ) @@ -29,7 +31,7 @@ const keyLength = 32 // 32 bytes for AES-256 const testEncryptionKey = "examplekey1234567890examplekey12" // generateEncryptionKey generates a random AES key. -func generateEncryptionKey() ([]byte, error) { +func generateEncryptionKey(_, _ string) ([]byte, error) { key := make([]byte, keyLength) if _, err := io.ReadFull(rand.Reader, key); err != nil { return nil, err @@ -38,44 +40,89 @@ func generateEncryptionKey() ([]byte, error) { } // generateTestEncryptionKey generates a deterministic encryption key for testing. -func generateTestEncryptionKey() ([]byte, error) { - return []byte(testEncryptionKey), nil +// While testing, we also want to make sure different keys are generated for different docs and fields +// and that's why we use the docID and fieldName to generate the key. +func generateTestEncryptionKey(docID, fieldName string) ([]byte, error) { + return []byte(fieldName + docID + testEncryptionKey)[0:keyLength], nil } +// DocEncryptor is a document encryptor that encrypts and decrypts individual document fields. +// It acts based on the configuration [DocEncConfig] provided and data stored in the provided store. +// It uses [core.EncStoreDocKey] to store and retrieve encryption keys. type DocEncryptor struct { - shouldGenerateKey bool - ctx context.Context - store datastore.DSReaderWriter + conf immutable.Option[DocEncConfig] + ctx context.Context + store datastore.DSReaderWriter } func newDocEncryptor(ctx context.Context) *DocEncryptor { return &DocEncryptor{ctx: ctx} } -func (d *DocEncryptor) EnableKeyGeneration() { - d.shouldGenerateKey = true +// SetConfig sets the configuration for the document encryptor. +func (d *DocEncryptor) SetConfig(conf immutable.Option[DocEncConfig]) { + d.conf = conf } +// SetStore sets the store for the document encryptor. func (d *DocEncryptor) SetStore(store datastore.DSReaderWriter) { d.store = store } -func (d *DocEncryptor) Encrypt(docID string, fieldID uint32, plainText []byte) ([]byte, error) { - encryptionKey, storeKey, err := d.fetchEncryptionKey(docID, fieldID) +func shouldEncryptIndividualField(conf immutable.Option[DocEncConfig], fieldName string) bool { + if !conf.HasValue() || fieldName == "" { + return false + } + for _, field := range conf.Value().EncryptedFields { + if field == fieldName { + return true + } + } + return false +} + +func shouldEncryptField(conf immutable.Option[DocEncConfig], fieldName string) bool { + if !conf.HasValue() { + return false + } + if conf.Value().IsDocEncrypted { + return true + } + if fieldName == "" { + return false + } + for _, field := range conf.Value().EncryptedFields { + if field == fieldName { + return true + } + } + return false +} + +// Encrypt encrypts the given plainText that is associated with the given docID and fieldName. +// If the current configuration is set to encrypt the given key individually, it will encrypt it with a new key. +// Otherwise, it will use document-level encryption key. +func (d *DocEncryptor) Encrypt(docID, fieldName string, plainText []byte) ([]byte, error) { + encryptionKey, err := d.fetchEncryptionKey(docID, fieldName) if err != nil { return nil, err } if len(encryptionKey) == 0 { - if !d.shouldGenerateKey { + if !shouldEncryptIndividualField(d.conf, fieldName) { + fieldName = "" + } + + if !shouldEncryptField(d.conf, fieldName) { return plainText, nil } - encryptionKey, err = generateEncryptionKeyFunc() + encryptionKey, err = generateEncryptionKeyFunc(docID, fieldName) if err != nil { return nil, err } + storeKey := core.NewEncStoreDocKey(docID, fieldName) err = d.store.Put(d.ctx, storeKey.ToDS(), encryptionKey) if err != nil { return nil, err @@ -84,8 +131,10 @@ func (d *DocEncryptor) Encrypt(docID string, fieldID uint32, plainText []byte) ( return EncryptAES(plainText, encryptionKey) } -func (d *DocEncryptor) Decrypt(docID string, fieldID uint32, cipherText []byte) ([]byte, error) { - encKey, _, err := d.fetchEncryptionKey(docID, fieldID) +// Decrypt decrypts the given cipherText that is associated with the given docID and fieldName. +// If the corresponding encryption key is not found, it returns nil. +func (d *DocEncryptor) Decrypt(docID, fieldName string, cipherText []byte) ([]byte, error) { + encKey, err := d.fetchEncryptionKey(docID, fieldName) if err != nil { return nil, err } @@ -95,33 +144,61 @@ func (d *DocEncryptor) Decrypt(docID string, fieldID uint32, cipherText []byte) return DecryptAES(cipherText, encKey) } -// fetchEncryptionKey fetches the encryption key for the given docID and fieldID. +// fetchEncryptionKey fetches the encryption key for the given docID and fieldName. // If the key is not found, it returns an empty key. -func (d *DocEncryptor) fetchEncryptionKey(docID string, fieldID uint32) ([]byte, core.EncStoreDocKey, error) { - storeKey := core.NewEncStoreDocKey(docID, fieldID) +func (d *DocEncryptor) fetchEncryptionKey(docID string, fieldName string) ([]byte, error) { if d.store == nil { - return nil, core.EncStoreDocKey{}, ErrNoStorageProvided + return nil, ErrNoStorageProvided } + // first we try to find field-level key + storeKey := core.NewEncStoreDocKey(docID, fieldName) encryptionKey, err := d.store.Get(d.ctx, storeKey.ToDS()) isNotFound := errors.Is(err, ds.ErrNotFound) - if err != nil && !isNotFound { - return nil, core.EncStoreDocKey{}, err + if err != nil { + if !isNotFound { + return nil, err + } + // if previous fetch was for doc-level, there is nothing else to look for + if fieldName == "" { + return nil, nil + } + if shouldEncryptIndividualField(d.conf, fieldName) { + return nil, nil + } + // try to find doc-level key + storeKey.FieldName = "" + encryptionKey, err = d.store.Get(d.ctx, storeKey.ToDS()) + isNotFound = errors.Is(err, ds.ErrNotFound) + if err != nil && !isNotFound { + return nil, err + } } - return encryptionKey, storeKey, nil + return encryptionKey, nil } -func EncryptDoc(ctx context.Context, docID string, fieldID uint32, plainText []byte) ([]byte, error) { +// EncryptDoc encrypts the given plainText that is associated with the given docID and fieldName with +// encryptor in the context. +// If the current configuration is set to encrypt the given key individually, it will encrypt it with a new key. +// Otherwise, it will use document-level encryption key. +func EncryptDoc(ctx context.Context, docID string, fieldName string, plainText []byte) ([]byte, error) { enc, ok := TryGetContextEncryptor(ctx) if !ok { return nil, nil } - return enc.Encrypt(docID, fieldID, plainText) + return enc.Encrypt(docID, fieldName, plainText) } -func DecryptDoc(ctx context.Context, docID string, fieldID uint32, cipherText []byte) ([]byte, error) { +// DecryptDoc decrypts the given cipherText that is associated with the given docID and fieldName with +// encryptor in the context. +func DecryptDoc(ctx context.Context, docID string, fieldName string, cipherText []byte) ([]byte, error) { enc, ok := TryGetContextEncryptor(ctx) if !ok { return nil, nil } - return enc.Decrypt(docID, fieldID, cipherText) + return enc.Decrypt(docID, fieldName, cipherText) +} + +// ShouldEncryptField returns true if the given field should be encrypted based on the context config. +func ShouldEncryptField(ctx context.Context, fieldName string) bool { + return shouldEncryptField(GetContextConfig(ctx), fieldName) } diff --git a/internal/encryption/encryptor_test.go b/internal/encryption/encryptor_test.go index 10abd1f062..76888ed4f1 100644 --- a/internal/encryption/encryptor_test.go +++ b/internal/encryption/encryptor_test.go @@ -19,157 +19,204 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" + "github.com/sourcenetwork/immutable" + "github.com/sourcenetwork/defradb/datastore/mocks" "github.com/sourcenetwork/defradb/internal/core" ) var testErr = errors.New("test error") -var docID = "bae-c9fb0fa4-1195-589c-aa54-e68333fb90b3" +const docID = "bae-c9fb0fa4-1195-589c-aa54-e68333fb90b3" + +const fieldName = "name" func getPlainText() []byte { return []byte("test") } -func getCipherText(t *testing.T) []byte { - cipherText, err := EncryptAES(getPlainText(), []byte(testEncryptionKey)) +func getEncKey(fieldName string) []byte { + key, _ := generateTestEncryptionKey(docID, fieldName) + return key +} + +func getCipherText(t *testing.T, fieldName string) []byte { + cipherText, err := EncryptAES(getPlainText(), getEncKey(fieldName)) assert.NoError(t, err) return cipherText } -func TestEncryptorEncrypt_IfStorageReturnsError_Error(t *testing.T) { +func newDefaultEncryptor(t *testing.T) (*DocEncryptor, *mocks.DSReaderWriter) { + return newEncryptorWithConfig(t, DocEncConfig{IsDocEncrypted: true}) +} + +func newEncryptorWithConfig(t *testing.T, conf DocEncConfig) (*DocEncryptor, *mocks.DSReaderWriter) { enc := newDocEncryptor(context.Background()) st := mocks.NewDSReaderWriter(t) + enc.SetConfig(immutable.Some(conf)) enc.SetStore(st) + return enc, st +} + +func TestEncryptorEncrypt_IfStorageReturnsError_Error(t *testing.T) { + enc, st := newDefaultEncryptor(t) st.EXPECT().Get(mock.Anything, mock.Anything).Return(nil, testErr) - _, err := enc.Encrypt(docID, 0, []byte("test")) + _, err := enc.Encrypt(docID, fieldName, []byte("test")) assert.ErrorIs(t, err, testErr) } -func TestEncryptorEncrypt_IfNoKeyFoundInStorage_ShouldGenerateKeyStoreItAndReturnCipherText(t *testing.T) { - enc := newDocEncryptor(context.Background()) - st := mocks.NewDSReaderWriter(t) - enc.EnableKeyGeneration() - enc.SetStore(st) +func TestEncryptorEncrypt_IfStorageReturnsErrorOnSecondCall_Error(t *testing.T) { + enc, st := newDefaultEncryptor(t) - st.EXPECT().Get(mock.Anything, mock.Anything).Return(nil, ds.ErrNotFound) + st.EXPECT().Get(mock.Anything, mock.Anything).Return(nil, ds.ErrNotFound).Once() + st.EXPECT().Get(mock.Anything, mock.Anything).Return(nil, testErr) + + _, err := enc.Encrypt(docID, fieldName, []byte("test")) + + assert.ErrorIs(t, err, testErr) +} + +func TestEncryptorEncrypt_WithEmptyFieldNameIfNoKeyFoundInStorage_ShouldGenerateKeyStoreItAndReturnCipherText(t *testing.T) { + enc, st := newDefaultEncryptor(t) + + storeKey := core.NewEncStoreDocKey(docID, "") + + st.EXPECT().Get(mock.Anything, storeKey.ToDS()).Return(nil, ds.ErrNotFound) + st.EXPECT().Put(mock.Anything, storeKey.ToDS(), getEncKey("")).Return(nil) + + cipherText, err := enc.Encrypt(docID, "", getPlainText()) + + assert.NoError(t, err) + assert.Equal(t, getCipherText(t, ""), cipherText) +} + +func TestEncryptorEncrypt_IfNoFieldEncRequestedAndNoKeyInStorage_GenerateKeyStoreItAndReturnCipherText(t *testing.T) { + enc, st := newDefaultEncryptor(t) + + docStoreKey := core.NewEncStoreDocKey(docID, "").ToDS() + fieldStoreKey := core.NewEncStoreDocKey(docID, fieldName).ToDS() + + st.EXPECT().Get(mock.Anything, fieldStoreKey).Return(nil, ds.ErrNotFound) + st.EXPECT().Get(mock.Anything, docStoreKey).Return(nil, ds.ErrNotFound) + st.EXPECT().Put(mock.Anything, docStoreKey, getEncKey("")).Return(nil) + + cipherText, err := enc.Encrypt(docID, fieldName, getPlainText()) + + assert.NoError(t, err) + assert.Equal(t, getCipherText(t, ""), cipherText) +} + +func TestEncryptorEncrypt_IfNoKeyWithFieldFoundInStorage_ShouldGenerateKeyStoreItAndReturnCipherText(t *testing.T) { + enc, st := newEncryptorWithConfig(t, DocEncConfig{EncryptedFields: []string{fieldName}}) - storeKey := core.NewEncStoreDocKey(docID, 0) + storeKey := core.NewEncStoreDocKey(docID, fieldName) - st.EXPECT().Put(mock.Anything, storeKey.ToDS(), []byte(testEncryptionKey)).Return(nil) + st.EXPECT().Get(mock.Anything, storeKey.ToDS()).Return(nil, ds.ErrNotFound) + st.EXPECT().Put(mock.Anything, storeKey.ToDS(), getEncKey(fieldName)).Return(nil) - cipherText, err := enc.Encrypt(docID, 0, getPlainText()) + cipherText, err := enc.Encrypt(docID, fieldName, getPlainText()) assert.NoError(t, err) - assert.Equal(t, getCipherText(t), cipherText) + assert.Equal(t, getCipherText(t, fieldName), cipherText) +} + +func TestEncryptorEncrypt_IfKeyWithFieldFoundInStorage_ShouldUseItToReturnCipherText(t *testing.T) { + enc, st := newEncryptorWithConfig(t, DocEncConfig{EncryptedFields: []string{fieldName}}) + + storeKey := core.NewEncStoreDocKey(docID, fieldName) + st.EXPECT().Get(mock.Anything, storeKey.ToDS()).Return(getEncKey(fieldName), nil) + + cipherText, err := enc.Encrypt(docID, fieldName, getPlainText()) + + assert.NoError(t, err) + assert.Equal(t, getCipherText(t, fieldName), cipherText) } func TestEncryptorEncrypt_IfKeyFoundInStorage_ShouldUseItToReturnCipherText(t *testing.T) { - enc := newDocEncryptor(context.Background()) - st := mocks.NewDSReaderWriter(t) - enc.EnableKeyGeneration() - enc.SetStore(st) + enc, st := newDefaultEncryptor(t) - st.EXPECT().Get(mock.Anything, mock.Anything).Return([]byte(testEncryptionKey), nil) + st.EXPECT().Get(mock.Anything, mock.Anything).Return(getEncKey(""), nil) - cipherText, err := enc.Encrypt(docID, 0, getPlainText()) + cipherText, err := enc.Encrypt(docID, "", getPlainText()) assert.NoError(t, err) - assert.Equal(t, getCipherText(t), cipherText) + assert.Equal(t, getCipherText(t, ""), cipherText) } func TestEncryptorEncrypt_IfStorageFailsToStoreEncryptionKey_ReturnError(t *testing.T) { - enc := newDocEncryptor(context.Background()) - st := mocks.NewDSReaderWriter(t) - enc.EnableKeyGeneration() - enc.SetStore(st) + enc, st := newDefaultEncryptor(t) st.EXPECT().Get(mock.Anything, mock.Anything).Return(nil, ds.ErrNotFound) st.EXPECT().Put(mock.Anything, mock.Anything, mock.Anything).Return(testErr) - _, err := enc.Encrypt(docID, 0, getPlainText()) + _, err := enc.Encrypt(docID, fieldName, getPlainText()) assert.ErrorIs(t, err, testErr) } func TestEncryptorEncrypt_IfKeyGenerationIsNotEnabled_ShouldReturnPlainText(t *testing.T) { - enc := newDocEncryptor(context.Background()) - st := mocks.NewDSReaderWriter(t) - // we don call enc.EnableKeyGeneration() - enc.SetStore(st) + enc, st := newDefaultEncryptor(t) + enc.SetConfig(immutable.None[DocEncConfig]()) st.EXPECT().Get(mock.Anything, mock.Anything).Return(nil, ds.ErrNotFound) - cipherText, err := enc.Encrypt(docID, 0, getPlainText()) + cipherText, err := enc.Encrypt(docID, fieldName, getPlainText()) assert.NoError(t, err) assert.Equal(t, getPlainText(), cipherText) } func TestEncryptorEncrypt_IfNoStorageProvided_Error(t *testing.T) { - enc := newDocEncryptor(context.Background()) - enc.EnableKeyGeneration() - // we don call enc.SetStore(st) + enc, _ := newDefaultEncryptor(t) + enc.SetStore(nil) - _, err := enc.Encrypt(docID, 0, getPlainText()) + _, err := enc.Encrypt(docID, fieldName, getPlainText()) assert.ErrorIs(t, err, ErrNoStorageProvided) } func TestEncryptorDecrypt_IfNoStorageProvided_Error(t *testing.T) { - enc := newDocEncryptor(context.Background()) - enc.EnableKeyGeneration() - // we don call enc.SetStore(st) + enc, _ := newDefaultEncryptor(t) + enc.SetStore(nil) - _, err := enc.Decrypt(docID, 0, getPlainText()) + _, err := enc.Decrypt(docID, fieldName, getPlainText()) assert.ErrorIs(t, err, ErrNoStorageProvided) } func TestEncryptorDecrypt_IfStorageReturnsError_Error(t *testing.T) { - enc := newDocEncryptor(context.Background()) - st := mocks.NewDSReaderWriter(t) - enc.SetStore(st) + enc, st := newDefaultEncryptor(t) st.EXPECT().Get(mock.Anything, mock.Anything).Return(nil, testErr) - _, err := enc.Decrypt(docID, 0, []byte("test")) + _, err := enc.Decrypt(docID, fieldName, []byte("test")) assert.ErrorIs(t, err, testErr) } func TestEncryptorDecrypt_IfKeyFoundInStorage_ShouldUseItToReturnPlainText(t *testing.T) { - enc := newDocEncryptor(context.Background()) - st := mocks.NewDSReaderWriter(t) - enc.EnableKeyGeneration() - enc.SetStore(st) + enc, st := newDefaultEncryptor(t) - st.EXPECT().Get(mock.Anything, mock.Anything).Return([]byte(testEncryptionKey), nil) + st.EXPECT().Get(mock.Anything, mock.Anything).Return(getEncKey(""), nil) - plainText, err := enc.Decrypt(docID, 0, getCipherText(t)) + plainText, err := enc.Decrypt(docID, fieldName, getCipherText(t, "")) assert.NoError(t, err) assert.Equal(t, getPlainText(), plainText) } -func TestEncryptorDecrypt_IfNoKeyFoundInStorage_ShouldGenerateKeyStoreItAndReturnCipherText(t *testing.T) { - enc := newDocEncryptor(context.Background()) - st := mocks.NewDSReaderWriter(t) - enc.EnableKeyGeneration() - enc.SetStore(st) - - st.EXPECT().Get(mock.Anything, mock.Anything).Return(nil, ds.ErrNotFound) - - storeKey := core.NewEncStoreDocKey(docID, 0) - - st.EXPECT().Put(mock.Anything, storeKey.ToDS(), []byte(testEncryptionKey)).Return(nil) - - cipherText, err := enc.Encrypt(docID, 0, getPlainText()) +func TestEncryptDoc_IfContextHasNoEncryptor_ReturnNil(t *testing.T) { + data, err := EncryptDoc(context.Background(), docID, fieldName, getPlainText()) + assert.Nil(t, data, "data should be nil") + assert.NoError(t, err, "error should be nil") +} - assert.NoError(t, err) - assert.Equal(t, getCipherText(t), cipherText) +func TestDecryptDoc_IfContextHasNoEncryptor_ReturnNil(t *testing.T) { + data, err := DecryptDoc(context.Background(), docID, fieldName, getCipherText(t, fieldName)) + assert.Nil(t, data, "data should be nil") + assert.NoError(t, err, "error should be nil") } diff --git a/internal/merkle/clock/clock.go b/internal/merkle/clock/clock.go index 1cb79ed756..b5b5f2631c 100644 --- a/internal/merkle/clock/clock.go +++ b/internal/merkle/clock/clock.go @@ -86,8 +86,7 @@ func (mc *MerkleClock) AddDelta( delta.SetPriority(height) block := coreblock.New(delta, links, heads...) - // Write the new block to the dag store. - isEncrypted, err := mc.checkIfBlockEncryptionEnabled(ctx, heads) + isEncrypted, err := mc.checkIfBlockEncryptionEnabled(ctx, block.Delta.GetFieldName(), heads) if err != nil { return cidlink.Link{}, nil, err } @@ -130,10 +129,10 @@ func (mc *MerkleClock) AddDelta( func (mc *MerkleClock) checkIfBlockEncryptionEnabled( ctx context.Context, + fieldName string, heads []cid.Cid, ) (bool, error) { - encConf := encryption.GetContextConfig(ctx) - if encConf.HasValue() && encConf.Value().IsEncrypted { + if encryption.ShouldEncryptField(ctx, fieldName) { return true, nil } @@ -156,7 +155,8 @@ func (mc *MerkleClock) checkIfBlockEncryptionEnabled( func encryptBlock(ctx context.Context, block *coreblock.Block) (*coreblock.Block, error) { clonedCRDT := block.Delta.Clone() - bytes, err := encryption.EncryptDoc(ctx, string(clonedCRDT.GetDocID()), 0, clonedCRDT.GetData()) + bytes, err := encryption.EncryptDoc(ctx, string(clonedCRDT.GetDocID()), + clonedCRDT.GetFieldName(), clonedCRDT.GetData()) if err != nil { return nil, err } diff --git a/internal/merkle/crdt/field.go b/internal/merkle/crdt/field.go index 6426165f49..10795942f0 100644 --- a/internal/merkle/crdt/field.go +++ b/internal/merkle/crdt/field.go @@ -10,13 +10,28 @@ package merklecrdt -import "github.com/sourcenetwork/defradb/client" +import ( + "github.com/sourcenetwork/defradb/client" +) // DocField is a struct that holds the document ID and the field value. // This is used to have a link between the document and the field value. // For example, to check if the field value needs to be encrypted depending on the document-level // encryption is enabled or not. type DocField struct { - DocID string + // DocID is the ID of a document associated with the field value. + DocID string + // FieldName is the name of the field. + FieldName string + // FieldValue is the field value. FieldValue *client.FieldValue } + +// NewDocField creates a new DocField instance. +func NewDocField(docID, fieldName string, fieldValue *client.FieldValue) *DocField { + return &DocField{ + DocID: docID, + FieldName: fieldName, + FieldValue: fieldValue, + } +} diff --git a/internal/planner/create.go b/internal/planner/create.go index b03f2c1765..89acfabc5d 100644 --- a/internal/planner/create.go +++ b/internal/planner/create.go @@ -159,9 +159,7 @@ func (p *Planner) CreateDocs(parsed *mapper.Mutation) (planNode, error) { create.input = []map[string]any{parsed.Input} } - if parsed.Encrypt { - p.ctx = encryption.SetContextConfig(p.ctx, encryption.DocEncConfig{IsEncrypted: true}) - } + p.ctx = encryption.SetContextConfigFromParams(p.ctx, parsed.Encrypt, parsed.EncryptFields) // get collection col, err := p.db.GetCollectionByName(p.ctx, parsed.Name) diff --git a/internal/planner/mapper/mapper.go b/internal/planner/mapper/mapper.go index 858ef0e0ae..55e2358f3e 100644 --- a/internal/planner/mapper/mapper.go +++ b/internal/planner/mapper/mapper.go @@ -1165,11 +1165,12 @@ func ToMutation(ctx context.Context, store client.Store, mutationRequest *reques } return &Mutation{ - Select: *underlyingSelect, - Type: MutationType(mutationRequest.Type), - Input: mutationRequest.Input, - Inputs: mutationRequest.Inputs, - Encrypt: mutationRequest.Encrypt, + Select: *underlyingSelect, + Type: MutationType(mutationRequest.Type), + Input: mutationRequest.Input, + Inputs: mutationRequest.Inputs, + Encrypt: mutationRequest.Encrypt, + EncryptFields: mutationRequest.EncryptFields, }, nil } diff --git a/internal/planner/mapper/mutation.go b/internal/planner/mapper/mutation.go index 251d01298f..f02bc9d067 100644 --- a/internal/planner/mapper/mutation.go +++ b/internal/planner/mapper/mutation.go @@ -35,4 +35,7 @@ type Mutation struct { // Encrypt is a flag to indicate if the input data should be encrypted. Encrypt bool + + // EncryptFields is a list of fields from the input data that should be encrypted. + EncryptFields []string } diff --git a/internal/request/graphql/parser/mutation.go b/internal/request/graphql/parser/mutation.go index 2ed4ebf539..b329eeec88 100644 --- a/internal/request/graphql/parser/mutation.go +++ b/internal/request/graphql/parser/mutation.go @@ -140,8 +140,15 @@ func parseMutation(schema gql.Schema, parent *gql.Object, field *ast.Field) (*re ids[i] = id.Value } mut.DocIDs = immutable.Some(ids) - } else if prop == request.EncryptArgName { + } else if prop == request.EncryptDocArgName { mut.Encrypt = argument.Value.(*ast.BooleanValue).Value + } else if prop == request.EncryptFieldsArgName { + raw := argument.Value.(*ast.ListValue) + fieldNames := make([]string, len(raw.Values)) + for i, val := range raw.Values { + fieldNames[i] = val.GetValue().(string) + } + mut.EncryptFields = fieldNames } } diff --git a/internal/request/graphql/schema/descriptions.go b/internal/request/graphql/schema/descriptions.go index 281002366a..526e41470e 100644 --- a/internal/request/graphql/schema/descriptions.go +++ b/internal/request/graphql/schema/descriptions.go @@ -158,7 +158,15 @@ Returns the head commit for this document. ` encryptArgDescription string = ` -Encrypt flag specified if the input document(s) needs to be encrypted. If set, DefraDB will generate a -symmetric key for encryption using AES-GCM. +Encrypt flag specifies if the input document(s) needs to be encrypted. If set, + DefraDB will generate a symmetric key for encryption using AES-GCM, and will + use it to encrypt all fields' values. +` + + encryptFieldsArgDescription string = ` +An optional list of individual fields that should be encrypted. For every field + in the list DefraDB will generate a symmetric key for encryption using AES-GCM. + If 'encrypt' is set to true, it all fields not listed in 'encryptedFields' will + be encrypted with the same key. ` ) diff --git a/internal/request/graphql/schema/generate.go b/internal/request/graphql/schema/generate.go index 82ff15d057..8bf9b74d72 100644 --- a/internal/request/graphql/schema/generate.go +++ b/internal/request/graphql/schema/generate.go @@ -33,6 +33,11 @@ const ( mutationInputsNameSuffix = "MutationInputsArg" ) +const ( + typeFieldEnumSuffix = "Field" + typeExplicitFieldEnumSuffix = "ExplicitField" +) + // Generator creates all the necessary typed schema definitions from an AST Document // and adds them to the Schema via the SchemaManager type Generator struct { @@ -385,7 +390,7 @@ func (g *Generator) createExpandedFieldList( listFieldFilterArgDescription, ), "groupBy": schemaTypes.NewArgConfig( - gql.NewList(gql.NewNonNull(g.manager.schema.TypeMap()[typeName+"Fields"])), + gql.NewList(gql.NewNonNull(g.manager.schema.TypeMap()[typeName+typeFieldEnumSuffix])), schemaTypes.GroupByArgDescription, ), "order": schemaTypes.NewArgConfig( @@ -1046,14 +1051,21 @@ func (g *Generator) GenerateMutationInputForGQLType(obj *gql.Object) ([]*gql.Fie return nil, NewErrTypeNotFound(mutationInputName) } + explicitUserFieldsEnum := g.genUserExplicitTypeFieldsEnum(obj) + + g.manager.schema.TypeMap()[explicitUserFieldsEnum.Name()] = explicitUserFieldsEnum + create := &gql.Field{ Name: "create_" + obj.Name(), Description: createDocumentDescription, Type: obj, Args: gql.FieldConfigArgument{ - "input": schemaTypes.NewArgConfig(mutationInput, "Create a "+obj.Name()+" document"), - "inputs": schemaTypes.NewArgConfig(gql.NewList(mutationInput), "Create "+obj.Name()+" documents"), - "encrypt": schemaTypes.NewArgConfig(gql.Boolean, encryptArgDescription), + request.Input: schemaTypes.NewArgConfig(mutationInput, "Create a "+obj.Name()+" document"), + request.Inputs: schemaTypes.NewArgConfig(gql.NewList(gql.NewNonNull(mutationInput)), + "Create "+obj.Name()+" documents"), + request.EncryptDocArgName: schemaTypes.NewArgConfig(gql.Boolean, encryptArgDescription), + request.EncryptFieldsArgName: schemaTypes.NewArgConfig(gql.NewList(gql.NewNonNull(explicitUserFieldsEnum)), + encryptFieldsArgDescription), }, } @@ -1065,7 +1077,7 @@ func (g *Generator) GenerateMutationInputForGQLType(obj *gql.Object) ([]*gql.Fie request.DocIDArgName: schemaTypes.NewArgConfig(gql.ID, updateIDArgDescription), request.DocIDsArgName: schemaTypes.NewArgConfig(gql.NewList(gql.ID), updateIDsArgDescription), "filter": schemaTypes.NewArgConfig(filterInput, updateFilterArgDescription), - "input": schemaTypes.NewArgConfig(mutationInput, "Update field values"), + request.Input: schemaTypes.NewArgConfig(mutationInput, "Update field values"), }, } @@ -1085,11 +1097,27 @@ func (g *Generator) GenerateMutationInputForGQLType(obj *gql.Object) ([]*gql.Fie func (g *Generator) genTypeFieldsEnum(obj *gql.Object) *gql.Enum { enumFieldsCfg := gql.EnumConfig{ - Name: genTypeName(obj, "Fields"), + Name: genTypeName(obj, typeFieldEnumSuffix), + Values: gql.EnumValueConfigMap{}, + } + + for f, field := range obj.Fields() { + enumFieldsCfg.Values[field.Name] = &gql.EnumValueConfig{Value: f} + } + + return gql.NewEnum(enumFieldsCfg) +} + +func (g *Generator) genUserExplicitTypeFieldsEnum(obj *gql.Object) *gql.Enum { + enumFieldsCfg := gql.EnumConfig{ + Name: genTypeName(obj, typeExplicitFieldEnumSuffix), Values: gql.EnumValueConfigMap{}, } for f, field := range obj.Fields() { + if strings.HasPrefix(field.Name, "_") { + continue + } enumFieldsCfg.Values[field.Name] = &gql.EnumValueConfig{Value: f} } diff --git a/tests/clients/cli/wrapper_collection.go b/tests/clients/cli/wrapper_collection.go index f26142c8e9..9ef71e8ce7 100644 --- a/tests/clients/cli/wrapper_collection.go +++ b/tests/clients/cli/wrapper_collection.go @@ -63,13 +63,7 @@ func (c *Collection) Create( return client.ErrOperationNotPermittedOnNamelessCols } - args := []string{"client", "collection", "create"} - args = append(args, "--name", c.Description().Name.Value()) - - encConf := encryption.GetContextConfig(ctx) - if encConf.HasValue() && encConf.Value().IsEncrypted { - args = append(args, "--encrypt") - } + args := makeDocCreateArgs(ctx, c) document, err := doc.String() if err != nil { @@ -93,13 +87,7 @@ func (c *Collection) CreateMany( return client.ErrOperationNotPermittedOnNamelessCols } - args := []string{"client", "collection", "create"} - args = append(args, "--name", c.Description().Name.Value()) - - encConf := encryption.GetContextConfig(ctx) - if encConf.HasValue() && encConf.Value().IsEncrypted { - args = append(args, "--encrypt") - } + args := makeDocCreateArgs(ctx, c) docStrings := make([]string, len(docs)) for i, doc := range docs { @@ -121,6 +109,26 @@ func (c *Collection) CreateMany( return nil } +func makeDocCreateArgs( + ctx context.Context, + c *Collection, +) []string { + args := []string{"client", "collection", "create"} + args = append(args, "--name", c.Description().Name.Value()) + + encConf := encryption.GetContextConfig(ctx) + if encConf.HasValue() { + if encConf.Value().IsDocEncrypted { + args = append(args, "--encrypt") + } + if len(encConf.Value().EncryptedFields) > 0 { + args = append(args, "--encrypt-fields", strings.Join(encConf.Value().EncryptedFields, ",")) + } + } + + return args +} + func (c *Collection) Update( ctx context.Context, doc *client.Document, diff --git a/tests/integration/encryption/commit_relation_test.go b/tests/integration/encryption/commit_relation_test.go new file mode 100644 index 0000000000..848afa5fd3 --- /dev/null +++ b/tests/integration/encryption/commit_relation_test.go @@ -0,0 +1,102 @@ +// Copyright 2024 Democratized Data Foundation +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package encryption + +import ( + "testing" + + testUtils "github.com/sourcenetwork/defradb/tests/integration" +) + +func TestDocEncryption_WithEncryptionSecondaryRelations_ShouldStoreEncryptedCommit(t *testing.T) { + const userDocID = "bae-4d563681-e131-5e01-8ab4-6c65ac0d0478" + const deviceDocID = "bae-50211587-fde7-5d75-8034-e7040dfba203" + + test := testUtils.TestCase{ + Actions: []any{ + testUtils.SchemaUpdate{ + Schema: ` + type User { + name: String + devices: [Device] + } + + type Device { + model: String + manufacturer: String + owner: User + } + `, + }, + testUtils.CreateDoc{ + CollectionID: 0, + Doc: `{ + "name": "Chris" + }`, + IsDocEncrypted: true, + }, + testUtils.CreateDoc{ + CollectionID: 1, + DocMap: map[string]any{ + "model": "Walkman", + "manufacturer": "Sony", + "owner": testUtils.NewDocIndex(0, 0), + }, + IsDocEncrypted: true, + }, + testUtils.Request{ + Request: ` + query { + commits { + delta + docID + fieldName + } + } + `, + Results: []map[string]any{ + { + "delta": encrypt(testUtils.CBORValue("Chris"), userDocID, ""), + "docID": userDocID, + "fieldName": "name", + }, + { + "delta": nil, + "docID": userDocID, + "fieldName": nil, + }, + { + "delta": encrypt(testUtils.CBORValue("Sony"), deviceDocID, ""), + "docID": deviceDocID, + "fieldName": "manufacturer", + }, + { + "delta": encrypt(testUtils.CBORValue("Walkman"), deviceDocID, ""), + "docID": deviceDocID, + "fieldName": "model", + }, + { + "delta": encrypt(testUtils.CBORValue(userDocID), deviceDocID, ""), + "docID": deviceDocID, + "fieldName": "owner_id", + }, + { + "delta": nil, + "docID": deviceDocID, + "fieldName": nil, + }, + }, + }, + }, + } + + testUtils.ExecuteTestCase(t, test) +} diff --git a/tests/integration/encryption/commit_test.go b/tests/integration/encryption/commit_test.go index 6a94621b3a..ffa5333748 100644 --- a/tests/integration/encryption/commit_test.go +++ b/tests/integration/encryption/commit_test.go @@ -13,27 +13,19 @@ package encryption import ( "testing" - "github.com/sourcenetwork/defradb/internal/encryption" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + testUtils "github.com/sourcenetwork/defradb/tests/integration" ) -func encrypt(plaintext []byte) []byte { - val, _ := encryption.EncryptAES(plaintext, []byte("examplekey1234567890examplekey12")) - return val -} - func TestDocEncryption_WithEncryptionOnLWWCRDT_ShouldStoreCommitsDeltaEncrypted(t *testing.T) { - const docID = "bae-c9fb0fa4-1195-589c-aa54-e68333fb90b3" - test := testUtils.TestCase{ Actions: []any{ updateUserCollectionSchema(), testUtils.CreateDoc{ - Doc: `{ - "name": "John", - "age": 21 - }`, - IsEncrypted: true, + Doc: john21Doc, + IsDocEncrypted: true, }, testUtils.Request{ Request: ` @@ -55,41 +47,41 @@ func TestDocEncryption_WithEncryptionOnLWWCRDT_ShouldStoreCommitsDeltaEncrypted( `, Results: []map[string]any{ { - "cid": "bafyreih7ry7ef26xn3lm2rhxusf2rbgyvl535tltrt6ehpwtvdnhlmptiu", + "cid": "bafyreibdjepzhhiez4o27srv33xcd52yr336tpzqtkv36rdf3h3oue2l5m", "collectionID": int64(1), - "delta": encrypt(testUtils.CBORValue(21)), - "docID": docID, + "delta": encrypt(testUtils.CBORValue(21), john21DocID, ""), + "docID": john21DocID, "fieldId": "1", "fieldName": "age", "height": int64(1), "links": []map[string]any{}, }, { - "cid": "bafyreifusejlwidaqswasct37eorazlfix6vyyn5af42pmjvktilzj5cty", + "cid": "bafyreihkiua7jpwkye3xlex6s5hh2azckcaljfi2h3iscgub5sikacyrbu", "collectionID": int64(1), - "delta": encrypt(testUtils.CBORValue("John")), - "docID": docID, + "delta": encrypt(testUtils.CBORValue("John"), john21DocID, ""), + "docID": john21DocID, "fieldId": "2", "fieldName": "name", "height": int64(1), "links": []map[string]any{}, }, { - "cid": "bafyreicvxlfxeqghmc3gy56rp5rzfejnbng4nu77x5e3wjinfydl6wvycq", + "cid": "bafyreidxdhzhwjrv5s4x6cho5drz6xq2tc7oymzupf4p4gfk6eelsnc7ke", "collectionID": int64(1), "delta": nil, - "docID": docID, + "docID": john21DocID, "fieldId": "C", "fieldName": nil, "height": int64(1), "links": []map[string]any{ { - "cid": "bafyreifusejlwidaqswasct37eorazlfix6vyyn5af42pmjvktilzj5cty", - "name": "name", + "cid": "bafyreibdjepzhhiez4o27srv33xcd52yr336tpzqtkv36rdf3h3oue2l5m", + "name": "age", }, { - "cid": "bafyreih7ry7ef26xn3lm2rhxusf2rbgyvl535tltrt6ehpwtvdnhlmptiu", - "name": "age", + "cid": "bafyreihkiua7jpwkye3xlex6s5hh2azckcaljfi2h3iscgub5sikacyrbu", + "name": "name", }, }, }, @@ -106,11 +98,8 @@ func TestDocEncryption_UponUpdateOnLWWCRDT_ShouldEncryptCommitDelta(t *testing.T Actions: []any{ updateUserCollectionSchema(), testUtils.CreateDoc{ - Doc: `{ - "name": "John", - "age": 21 - }`, - IsEncrypted: true, + Doc: john21Doc, + IsDocEncrypted: true, }, testUtils.UpdateDoc{ Doc: `{ @@ -127,10 +116,10 @@ func TestDocEncryption_UponUpdateOnLWWCRDT_ShouldEncryptCommitDelta(t *testing.T `, Results: []map[string]any{ { - "delta": encrypt(testUtils.CBORValue(22)), + "delta": encrypt(testUtils.CBORValue(22), john21DocID, ""), }, { - "delta": encrypt(testUtils.CBORValue(21)), + "delta": encrypt(testUtils.CBORValue(21), john21DocID, ""), }, }, }, @@ -141,24 +130,15 @@ func TestDocEncryption_UponUpdateOnLWWCRDT_ShouldEncryptCommitDelta(t *testing.T } func TestDocEncryption_WithMultipleDocsUponUpdate_ShouldEncryptOnlyRelevantDocs(t *testing.T) { - const johnDocID = "bae-c9fb0fa4-1195-589c-aa54-e68333fb90b3" - const islamDocID = "bae-d55bd956-1cc4-5d26-aa71-b98807ad49d6" - test := testUtils.TestCase{ Actions: []any{ updateUserCollectionSchema(), testUtils.CreateDoc{ - Doc: `{ - "name": "John", - "age": 21 - }`, - IsEncrypted: true, + Doc: john21Doc, + IsDocEncrypted: true, }, testUtils.CreateDoc{ - Doc: `{ - "name": "Islam", - "age": 33 - }`, + Doc: islam33Doc, }, testUtils.UpdateDoc{ DocID: 0, @@ -183,20 +163,20 @@ func TestDocEncryption_WithMultipleDocsUponUpdate_ShouldEncryptOnlyRelevantDocs( `, Results: []map[string]any{ { - "delta": encrypt(testUtils.CBORValue(22)), - "docID": johnDocID, + "delta": encrypt(testUtils.CBORValue(22), john21DocID, ""), + "docID": john21DocID, }, { - "delta": encrypt(testUtils.CBORValue(21)), - "docID": johnDocID, + "delta": encrypt(testUtils.CBORValue(21), john21DocID, ""), + "docID": john21DocID, }, { "delta": testUtils.CBORValue(34), - "docID": islamDocID, + "docID": islam33DocID, }, { "delta": testUtils.CBORValue(33), - "docID": islamDocID, + "docID": islam33DocID, }, }, }, @@ -218,14 +198,13 @@ func TestDocEncryption_WithEncryptionOnCounterCRDT_ShouldStoreCommitsDeltaEncryp } `}, testUtils.CreateDoc{ - Doc: `{ "points": 5 }`, - IsEncrypted: true, + Doc: `{ "points": 5 }`, + IsDocEncrypted: true, }, testUtils.Request{ Request: ` query { commits { - cid delta docID } @@ -233,12 +212,10 @@ func TestDocEncryption_WithEncryptionOnCounterCRDT_ShouldStoreCommitsDeltaEncryp `, Results: []map[string]any{ { - "cid": "bafyreieb6owsoljj4vondkx35ngxmhliauwvphicz4edufcy7biexij7mu", - "delta": encrypt(testUtils.CBORValue(5)), + "delta": encrypt(testUtils.CBORValue(5), docID, ""), "docID": docID, }, { - "cid": "bafyreif2lejhvdja2rmo237lrwpj45usrm55h6gzr4ewl6gajq3cl4ppsi", "delta": nil, "docID": docID, }, @@ -251,6 +228,8 @@ func TestDocEncryption_WithEncryptionOnCounterCRDT_ShouldStoreCommitsDeltaEncryp } func TestDocEncryption_UponUpdateOnCounterCRDT_ShouldEncryptedCommitDelta(t *testing.T) { + const docID = "bae-d3cc98b4-38d5-5c50-85a3-d3045d44094e" + test := testUtils.TestCase{ Actions: []any{ testUtils.SchemaUpdate{ @@ -260,8 +239,8 @@ func TestDocEncryption_UponUpdateOnCounterCRDT_ShouldEncryptedCommitDelta(t *tes } `}, testUtils.CreateDoc{ - Doc: `{ "points": 5 }`, - IsEncrypted: true, + Doc: `{ "points": 5 }`, + IsDocEncrypted: true, }, testUtils.UpdateDoc{ Doc: `{ @@ -278,10 +257,10 @@ func TestDocEncryption_UponUpdateOnCounterCRDT_ShouldEncryptedCommitDelta(t *tes `, Results: []map[string]any{ { - "delta": encrypt(testUtils.CBORValue(3)), + "delta": encrypt(testUtils.CBORValue(3), docID, ""), }, { - "delta": encrypt(testUtils.CBORValue(5)), + "delta": encrypt(testUtils.CBORValue(5), docID, ""), }, }, }, @@ -296,21 +275,13 @@ func TestDocEncryption_UponEncryptionSeveralDocs_ShouldStoreAllCommitsDeltaEncry Actions: []any{ updateUserCollectionSchema(), testUtils.CreateDoc{ - Doc: `[{ - "name": "John", - "age": 21 - }, - { - "name": "Islam", - "age": 33 - }]`, - IsEncrypted: true, + Doc: "[" + john21Doc + ", " + islam33Doc + "]", + IsDocEncrypted: true, }, testUtils.Request{ Request: ` query { commits { - cid delta docID } @@ -318,32 +289,26 @@ func TestDocEncryption_UponEncryptionSeveralDocs_ShouldStoreAllCommitsDeltaEncry `, Results: []map[string]any{ { - "cid": "bafyreih7ry7ef26xn3lm2rhxusf2rbgyvl535tltrt6ehpwtvdnhlmptiu", - "delta": encrypt(testUtils.CBORValue(21)), + "delta": encrypt(testUtils.CBORValue(21), john21DocID, ""), "docID": testUtils.NewDocIndex(0, 0), }, { - "cid": "bafyreifusejlwidaqswasct37eorazlfix6vyyn5af42pmjvktilzj5cty", - "delta": encrypt(testUtils.CBORValue("John")), + "delta": encrypt(testUtils.CBORValue("John"), john21DocID, ""), "docID": testUtils.NewDocIndex(0, 0), }, { - "cid": "bafyreicvxlfxeqghmc3gy56rp5rzfejnbng4nu77x5e3wjinfydl6wvycq", "delta": nil, "docID": testUtils.NewDocIndex(0, 0), }, { - "cid": "bafyreibe24bo67owxewoso3ekinera2bhusguij5qy2ahgyufaq3fbvaxa", - "delta": encrypt(testUtils.CBORValue(33)), + "delta": encrypt(testUtils.CBORValue(33), islam33DocID, ""), "docID": testUtils.NewDocIndex(0, 1), }, { - "cid": "bafyreie2fddpidgc62fhd2fjrsucq3spgh2mgvto2xwolcdmdhb5pdeok4", - "delta": encrypt(testUtils.CBORValue("Islam")), + "delta": encrypt(testUtils.CBORValue("Islam"), islam33DocID, ""), "docID": testUtils.NewDocIndex(0, 1), }, { - "cid": "bafyreifulxdkf4m3wmmdxjg43l4mw7uuxl5il27eabklc22nptilrh64sa", "delta": nil, "docID": testUtils.NewDocIndex(0, 1), }, @@ -354,3 +319,45 @@ func TestDocEncryption_UponEncryptionSeveralDocs_ShouldStoreAllCommitsDeltaEncry testUtils.ExecuteTestCase(t, test) } + +func TestDocEncryption_IfTwoDocsHaveSameFieldValue_CipherTextShouldBeDifferent(t *testing.T) { + test := testUtils.TestCase{ + Actions: []any{ + updateUserCollectionSchema(), + testUtils.CreateDoc{ + Doc: `{ + "name": "John", + "age": 21 + }`, + IsDocEncrypted: true, + }, + testUtils.CreateDoc{ + Doc: `{ + "name": "Islam", + "age": 21 + }`, + IsDocEncrypted: true, + }, + testUtils.Request{ + Request: ` + query { + commits(fieldId: "1") { + delta + fieldName + } + } + `, + Asserter: testUtils.ResultAsserterFunc(func(_ testing.TB, result []map[string]any) (bool, string) { + require.Equal(t, 2, len(result), "Expected 2 commits") + require.Equal(t, result[0]["fieldName"], "age") + delta1 := result[0]["delta"] + delta2 := result[1]["delta"] + assert.NotEqual(t, delta1, delta2, "docs should be encrypted with different encryption keys") + return true, "" + }), + }, + }, + } + + testUtils.ExecuteTestCase(t, test) +} diff --git a/tests/integration/encryption/field_commit_test.go b/tests/integration/encryption/field_commit_test.go new file mode 100644 index 0000000000..5a956a42b2 --- /dev/null +++ b/tests/integration/encryption/field_commit_test.go @@ -0,0 +1,190 @@ +// Copyright 2024 Democratized Data Foundation +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package encryption + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + testUtils "github.com/sourcenetwork/defradb/tests/integration" +) + +func TestDocEncryptionField_WithEncryptionOnField_ShouldStoreOnlyFieldsDeltaEncrypted(t *testing.T) { + test := testUtils.TestCase{ + Actions: []any{ + updateUserCollectionSchema(), + testUtils.CreateDoc{ + Doc: john21Doc, + EncryptedFields: []string{"age"}, + }, + testUtils.Request{ + Request: ` + query { + commits { + delta + docID + fieldName + } + } + `, + Results: []map[string]any{ + { + "delta": encrypt(testUtils.CBORValue(21), john21DocID, "age"), + "docID": john21DocID, + "fieldName": "age", + }, + { + "delta": testUtils.CBORValue("John"), + "docID": john21DocID, + "fieldName": "name", + }, + { + "delta": nil, + "docID": john21DocID, + "fieldName": nil, + }, + }, + }, + }, + } + + testUtils.ExecuteTestCase(t, test) +} + +func TestDocEncryptionField_WithDocAndFieldEncryption_ShouldUseDedicatedEncKeyForIndividualFields(t *testing.T) { + deltaForField := func(fieldName string, result []map[string]any) any { + for _, r := range result { + if r["fieldName"] == fieldName { + return r["delta"] + } + } + t.Fatalf("Field %s not found in results %v", fieldName, result) + return nil + } + + test := testUtils.TestCase{ + Actions: []any{ + testUtils.SchemaUpdate{ + Schema: ` + type Users { + name1: String + name2: String + name3: String + name4: String + }`, + }, + testUtils.CreateDoc{ + Doc: `{ + "name1": "John", + "name2": "John", + "name3": "John", + "name4": "John" + }`, + IsDocEncrypted: true, + EncryptedFields: []string{"name1", "name3"}, + }, + testUtils.Request{ + Request: ` + query { + commits { + cid + delta + fieldName + } + } + `, + Asserter: testUtils.ResultAsserterFunc(func(_ testing.TB, result []map[string]any) (bool, string) { + name1 := deltaForField("name1", result) + name2 := deltaForField("name2", result) + name3 := deltaForField("name3", result) + name4 := deltaForField("name4", result) + assert.Equal(t, name2, name4, "name2 and name4 should have the same encryption key") + assert.NotEqual(t, name2, name1, "name2 and name1 should have different encryption keys") + assert.NotEqual(t, name2, name3, "name2 and name3 should have different encryption keys") + assert.NotEqual(t, name1, name3, "name1 and name3 should have different encryption keys") + return true, "" + }), + }, + }, + } + + testUtils.ExecuteTestCase(t, test) +} + +func TestDocEncryptionField_UponUpdateWithDocAndFieldEncryption_ShouldUseDedicatedEncKeyForIndividualFields(t *testing.T) { + deltaForField := func(fieldName string, result []map[string]any) any { + for _, r := range result { + if r["fieldName"] == fieldName { + return r["delta"] + } + } + t.Fatalf("Field %s not found in results %v", fieldName, result) + return nil + } + + test := testUtils.TestCase{ + Actions: []any{ + testUtils.SchemaUpdate{ + Schema: ` + type Users { + name1: String + name2: String + name3: String + name4: String + }`, + }, + testUtils.CreateDoc{ + Doc: `{ + "name1": "John", + "name2": "John", + "name3": "John", + "name4": "John" + }`, + IsDocEncrypted: true, + EncryptedFields: []string{"name1", "name3"}, + }, + testUtils.UpdateDoc{ + Doc: `{ + "name1": "Andy", + "name2": "Andy", + "name3": "Andy", + "name4": "Andy" + }`, + }, + testUtils.Request{ + Request: ` + query { + commits(order: {height: DESC}, limit: 5) { + cid + delta + fieldName + height + } + } + `, + Asserter: testUtils.ResultAsserterFunc(func(_ testing.TB, result []map[string]any) (bool, string) { + name1 := deltaForField("name1", result) + name2 := deltaForField("name2", result) + name3 := deltaForField("name3", result) + name4 := deltaForField("name4", result) + assert.Equal(t, name2, name4, "name2 and name4 should have the same encryption key") + assert.NotEqual(t, name2, name1, "name2 and name1 should have different encryption keys") + assert.NotEqual(t, name2, name3, "name2 and name3 should have different encryption keys") + assert.NotEqual(t, name1, name3, "name1 and name3 should have different encryption keys") + return true, "" + }), + }, + }, + } + + testUtils.ExecuteTestCase(t, test) +} diff --git a/tests/integration/encryption/field_query_test.go b/tests/integration/encryption/field_query_test.go new file mode 100644 index 0000000000..d008960448 --- /dev/null +++ b/tests/integration/encryption/field_query_test.go @@ -0,0 +1,54 @@ +// Copyright 2024 Democratized Data Foundation +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package encryption + +import ( + "testing" + + testUtils "github.com/sourcenetwork/defradb/tests/integration" +) + +func TestDocEncryptionField_WithEncryption_ShouldFetchDecrypted(t *testing.T) { + test := testUtils.TestCase{ + Actions: []any{ + testUtils.SchemaUpdate{ + Schema: ` + type Users { + name: String + age: Int + } + `}, + testUtils.CreateDoc{ + Doc: john21Doc, + EncryptedFields: []string{"name"}, + }, + testUtils.Request{ + Request: ` + query { + Users { + _docID + name + age + } + }`, + Results: []map[string]any{ + { + "_docID": testUtils.NewDocIndex(0, 0), + "name": "John", + "age": int64(21), + }, + }, + }, + }, + } + + testUtils.ExecuteTestCase(t, test) +} diff --git a/tests/integration/encryption/field_test.go b/tests/integration/encryption/field_test.go new file mode 100644 index 0000000000..839e97274f --- /dev/null +++ b/tests/integration/encryption/field_test.go @@ -0,0 +1,129 @@ +// Copyright 2024 Democratized Data Foundation +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package encryption + +import ( + "testing" + + "github.com/sourcenetwork/immutable" + + "github.com/sourcenetwork/defradb/client" + "github.com/sourcenetwork/defradb/internal/db" + testUtils "github.com/sourcenetwork/defradb/tests/integration" +) + +func TestDocEncryptionField_IfFieldDoesNotExistInGQLSchema_ReturnError(t *testing.T) { + test := testUtils.TestCase{ + SupportedMutationTypes: immutable.Some([]testUtils.MutationType{ + testUtils.GQLRequestMutationType, + }), + Actions: []any{ + testUtils.SchemaUpdate{ + Schema: ` + type Users { + name: String + age: Int + } + `}, + testUtils.CreateDoc{ + Doc: john21Doc, + EncryptedFields: []string{"points"}, + ExpectedError: "Argument \"encryptFields\" has invalid value [points].", + }, + }, + } + + testUtils.ExecuteTestCase(t, test) +} + +func TestDocEncryptionField_IfAttemptToEncryptBuiltinFieldInGQLSchema_ReturnError(t *testing.T) { + test := testUtils.TestCase{ + SupportedMutationTypes: immutable.Some([]testUtils.MutationType{ + testUtils.GQLRequestMutationType, + }), + Actions: []any{ + testUtils.SchemaUpdate{ + Schema: ` + type Users { + name: String + age: Int + } + `}, + testUtils.CreateDoc{ + Doc: john21Doc, + EncryptedFields: []string{"_docID"}, + ExpectedError: "Argument \"encryptFields\" has invalid value [_docID].", + }, + testUtils.CreateDoc{ + Doc: john21Doc, + EncryptedFields: []string{"_version"}, + ExpectedError: "Argument \"encryptFields\" has invalid value [_version].", + }, + }, + } + + testUtils.ExecuteTestCase(t, test) +} + +func TestDocEncryptionField_IfFieldDoesNotExist_ReturnError(t *testing.T) { + test := testUtils.TestCase{ + SupportedMutationTypes: immutable.Some([]testUtils.MutationType{ + testUtils.CollectionSaveMutationType, + testUtils.CollectionNamedMutationType, + }), + Actions: []any{ + testUtils.SchemaUpdate{ + Schema: ` + type Users { + name: String + age: Int + } + `}, + testUtils.CreateDoc{ + Doc: john21Doc, + EncryptedFields: []string{"points"}, + ExpectedError: client.NewErrFieldNotExist("points").Error(), + }, + }, + } + + testUtils.ExecuteTestCase(t, test) +} + +func TestDocEncryptionField_IfAttemptToEncryptBuiltinField_ReturnError(t *testing.T) { + test := testUtils.TestCase{ + SupportedMutationTypes: immutable.Some([]testUtils.MutationType{ + testUtils.CollectionSaveMutationType, + testUtils.CollectionNamedMutationType, + }), + Actions: []any{ + testUtils.SchemaUpdate{ + Schema: ` + type Users { + name: String + age: Int + } + `}, + testUtils.CreateDoc{ + Doc: john21Doc, + EncryptedFields: []string{"_docID"}, + ExpectedError: db.NewErrCanNotEncryptBuiltinField("_docID").Error(), + }, + testUtils.CreateDoc{ + Doc: john21Doc, + EncryptedFields: []string{"_version"}, + ExpectedError: client.NewErrFieldNotExist("_version").Error(), + }, + }, + } + + testUtils.ExecuteTestCase(t, test) +} diff --git a/tests/integration/encryption/peer_test.go b/tests/integration/encryption/peer_test.go index 6d9c937278..4eb0ddc19a 100644 --- a/tests/integration/encryption/peer_test.go +++ b/tests/integration/encryption/peer_test.go @@ -33,12 +33,9 @@ func TestDocEncryptionPeer_IfPeerHasNoKey_ShouldNotFetch(t *testing.T) { CollectionIDs: []int{0}, }, testUtils.CreateDoc{ - NodeID: immutable.Some(0), - Doc: `{ - "name": "John", - "age": 21 - }`, - IsEncrypted: true, + NodeID: immutable.Some(0), + Doc: john21Doc, + IsDocEncrypted: true, }, testUtils.WaitForSync{}, testUtils.Request{ @@ -71,12 +68,9 @@ func TestDocEncryptionPeer_UponSync_ShouldSyncEncryptedDAG(t *testing.T) { CollectionIDs: []int{0}, }, testUtils.CreateDoc{ - NodeID: immutable.Some(0), - Doc: `{ - "name": "John", - "age": 21 - }`, - IsEncrypted: true, + NodeID: immutable.Some(0), + Doc: john21Doc, + IsDocEncrypted: true, }, testUtils.WaitForSync{}, testUtils.Request{ @@ -100,41 +94,41 @@ func TestDocEncryptionPeer_UponSync_ShouldSyncEncryptedDAG(t *testing.T) { `, Results: []map[string]any{ { - "cid": "bafyreih7ry7ef26xn3lm2rhxusf2rbgyvl535tltrt6ehpwtvdnhlmptiu", + "cid": "bafyreibdjepzhhiez4o27srv33xcd52yr336tpzqtkv36rdf3h3oue2l5m", "collectionID": int64(1), - "delta": encrypt(testUtils.CBORValue(21)), - "docID": "bae-c9fb0fa4-1195-589c-aa54-e68333fb90b3", + "delta": encrypt(testUtils.CBORValue(21), john21DocID, ""), + "docID": john21DocID, "fieldId": "1", "fieldName": "age", "height": int64(1), "links": []map[string]any{}, }, { - "cid": "bafyreifusejlwidaqswasct37eorazlfix6vyyn5af42pmjvktilzj5cty", + "cid": "bafyreihkiua7jpwkye3xlex6s5hh2azckcaljfi2h3iscgub5sikacyrbu", "collectionID": int64(1), - "delta": encrypt(testUtils.CBORValue("John")), - "docID": "bae-c9fb0fa4-1195-589c-aa54-e68333fb90b3", + "delta": encrypt(testUtils.CBORValue("John"), john21DocID, ""), + "docID": john21DocID, "fieldId": "2", "fieldName": "name", "height": int64(1), "links": []map[string]any{}, }, { - "cid": "bafyreicvxlfxeqghmc3gy56rp5rzfejnbng4nu77x5e3wjinfydl6wvycq", + "cid": "bafyreidxdhzhwjrv5s4x6cho5drz6xq2tc7oymzupf4p4gfk6eelsnc7ke", "collectionID": int64(1), "delta": nil, - "docID": "bae-c9fb0fa4-1195-589c-aa54-e68333fb90b3", + "docID": john21DocID, "fieldId": "C", "fieldName": nil, "height": int64(1), "links": []map[string]any{ { - "cid": "bafyreifusejlwidaqswasct37eorazlfix6vyyn5af42pmjvktilzj5cty", - "name": "name", + "cid": "bafyreibdjepzhhiez4o27srv33xcd52yr336tpzqtkv36rdf3h3oue2l5m", + "name": "age", }, { - "cid": "bafyreih7ry7ef26xn3lm2rhxusf2rbgyvl535tltrt6ehpwtvdnhlmptiu", - "name": "age", + "cid": "bafyreihkiua7jpwkye3xlex6s5hh2azckcaljfi2h3iscgub5sikacyrbu", + "name": "name", }, }, }, diff --git a/tests/integration/encryption/query_relation_test.go b/tests/integration/encryption/query_relation_test.go new file mode 100644 index 0000000000..8ca51c03aa --- /dev/null +++ b/tests/integration/encryption/query_relation_test.go @@ -0,0 +1,198 @@ +// Copyright 2024 Democratized Data Foundation +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package encryption + +import ( + "testing" + + testUtils "github.com/sourcenetwork/defradb/tests/integration" +) + +func TestDocEncryption_WithEncryptionOnBothRelations_ShouldFetchDecrypted(t *testing.T) { + test := testUtils.TestCase{ + Actions: []any{ + testUtils.SchemaUpdate{ + Schema: ` + type User { + name: String + devices: [Device] + } + + type Device { + model: String + manufacturer: String + owner: User + } + `, + }, + testUtils.CreateDoc{ + CollectionID: 0, + Doc: `{ + "name": "Chris" + }`, + IsDocEncrypted: true, + }, + testUtils.CreateDoc{ + CollectionID: 1, + DocMap: map[string]any{ + "model": "Walkman", + "manufacturer": "Sony", + "owner": testUtils.NewDocIndex(0, 0), + }, + IsDocEncrypted: true, + }, + testUtils.Request{ + Request: `query { + User { + name + devices { + model + manufacturer + } + } + }`, + Results: []map[string]any{ + { + "name": "Chris", + "devices": []map[string]any{ + { + "model": "Walkman", + "manufacturer": "Sony", + }, + }, + }, + }, + }, + }, + } + + testUtils.ExecuteTestCase(t, test) +} + +func TestDocEncryption_WithEncryptionOnPrimaryRelations_ShouldFetchDecrypted(t *testing.T) { + test := testUtils.TestCase{ + Actions: []any{ + testUtils.SchemaUpdate{ + Schema: ` + type User { + name: String + devices: [Device] + } + + type Device { + model: String + manufacturer: String + owner: User + } + `, + }, + testUtils.CreateDoc{ + CollectionID: 0, + Doc: `{ + "name": "Chris" + }`, + }, + testUtils.CreateDoc{ + CollectionID: 1, + DocMap: map[string]any{ + "model": "Walkman", + "manufacturer": "Sony", + "owner": testUtils.NewDocIndex(0, 0), + }, + IsDocEncrypted: true, + }, + testUtils.Request{ + Request: `query { + User { + name + devices { + model + manufacturer + } + } + }`, + Results: []map[string]any{ + { + "name": "Chris", + "devices": []map[string]any{ + { + "model": "Walkman", + "manufacturer": "Sony", + }, + }, + }, + }, + }, + }, + } + + testUtils.ExecuteTestCase(t, test) +} + +func TestDocEncryption_WithEncryptionOnSecondaryRelations_ShouldFetchDecrypted(t *testing.T) { + test := testUtils.TestCase{ + Actions: []any{ + testUtils.SchemaUpdate{ + Schema: ` + type User { + name: String + devices: [Device] + } + + type Device { + model: String + manufacturer: String + owner: User + } + `, + }, + testUtils.CreateDoc{ + CollectionID: 0, + Doc: `{ + "name": "Chris" + }`, + IsDocEncrypted: true, + }, + testUtils.CreateDoc{ + CollectionID: 1, + DocMap: map[string]any{ + "model": "Walkman", + "manufacturer": "Sony", + "owner": testUtils.NewDocIndex(0, 0), + }, + }, + testUtils.Request{ + Request: `query { + User { + name + devices { + model + manufacturer + } + } + }`, + Results: []map[string]any{ + { + "name": "Chris", + "devices": []map[string]any{ + { + "model": "Walkman", + "manufacturer": "Sony", + }, + }, + }, + }, + }, + }, + } + + testUtils.ExecuteTestCase(t, test) +} diff --git a/tests/integration/encryption/query_test.go b/tests/integration/encryption/query_test.go index 32d9bd2c94..05e7b1dcdd 100644 --- a/tests/integration/encryption/query_test.go +++ b/tests/integration/encryption/query_test.go @@ -27,11 +27,8 @@ func TestDocEncryption_WithEncryption_ShouldFetchDecrypted(t *testing.T) { } `}, testUtils.CreateDoc{ - Doc: `{ - "name": "John", - "age": 21 - }`, - IsEncrypted: true, + Doc: john21Doc, + IsDocEncrypted: true, }, testUtils.Request{ Request: ` @@ -79,7 +76,7 @@ func TestDocEncryption_WithEncryptionOnCounterCRDT_ShouldFetchDecrypted(t *testi "name": "John", "points": 5 }`, - IsEncrypted: true, + IsDocEncrypted: true, }, testUtils.Request{ Request: query, diff --git a/tests/integration/encryption/utils.go b/tests/integration/encryption/utils.go index 400a0d34c3..fd9c1d17c0 100644 --- a/tests/integration/encryption/utils.go +++ b/tests/integration/encryption/utils.go @@ -11,6 +11,7 @@ package encryption import ( + "github.com/sourcenetwork/defradb/internal/encryption" testUtils "github.com/sourcenetwork/defradb/tests/integration" ) @@ -24,8 +25,31 @@ const userCollectionGQLSchema = (` } `) +const ( + john21Doc = `{ + "name": "John", + "age": 21 + }` + islam33Doc = `{ + "name": "Islam", + "age": 33 + }` + john21DocID = "bae-c9fb0fa4-1195-589c-aa54-e68333fb90b3" + islam33DocID = "bae-d55bd956-1cc4-5d26-aa71-b98807ad49d6" +) + func updateUserCollectionSchema() testUtils.SchemaUpdate { return testUtils.SchemaUpdate{ Schema: userCollectionGQLSchema, } } + +// encrypt encrypts the given plain text with a deterministic encryption key. +// We also want to make sure different keys are generated for different docs and fields +// and that's why we use the docID and fieldName to generate the key. +func encrypt(plaintext []byte, docID, fieldName string) []byte { + const keyLength = 32 + const testEncKey = "examplekey1234567890examplekey12" + val, _ := encryption.EncryptAES(plaintext, []byte(fieldName + docID + testEncKey)[0:keyLength]) + return val +} diff --git a/tests/integration/query/one_to_many/with_group_related_id_test.go b/tests/integration/query/one_to_many/with_group_related_id_test.go index 673ccfddad..509c38fc27 100644 --- a/tests/integration/query/one_to_many/with_group_related_id_test.go +++ b/tests/integration/query/one_to_many/with_group_related_id_test.go @@ -371,7 +371,7 @@ func TestQueryOneToManyWithParentGroupByOnRelatedTypeFromSingleSide(t *testing.T }, }, - ExpectedError: "Argument \"groupBy\" has invalid value [published_id].\nIn element #1: Expected type \"AuthorFields\", found published_id.", + ExpectedError: "Argument \"groupBy\" has invalid value [published_id].\nIn element #1: Expected type \"AuthorField\", found published_id.", } executeTestCase(t, test) @@ -449,7 +449,7 @@ func TestQueryOneToManyWithParentGroupByOnRelatedTypeWithIDSelectionFromSingleSi }, }, - ExpectedError: "Argument \"groupBy\" has invalid value [published_id].\nIn element #1: Expected type \"AuthorFields\", found published_id.", + ExpectedError: "Argument \"groupBy\" has invalid value [published_id].\nIn element #1: Expected type \"AuthorField\", found published_id.", } executeTestCase(t, test) diff --git a/tests/integration/schema/group_test.go b/tests/integration/schema/group_test.go index 5ac89b95ec..04a8ccfde8 100644 --- a/tests/integration/schema/group_test.go +++ b/tests/integration/schema/group_test.go @@ -41,7 +41,7 @@ func TestGroupByFieldForTheManySideInSchema(t *testing.T) { testUtils.IntrospectionRequest{ Request: ` { - __type(name: "BookFields") { + __type(name: "BookField") { name kind enumValues { @@ -53,7 +53,7 @@ func TestGroupByFieldForTheManySideInSchema(t *testing.T) { ContainsData: map[string]any{ "__type": map[string]any{ "kind": "ENUM", - "name": "BookFields", + "name": "BookField", "enumValues": []any{ // Internal related object fields. map[string]any{"name": "author"}, @@ -103,7 +103,7 @@ func TestGroupByFieldForTheSingleSideInSchema(t *testing.T) { testUtils.IntrospectionRequest{ Request: ` { - __type(name: "AuthorFields") { + __type(name: "AuthorField") { name kind enumValues { @@ -115,7 +115,7 @@ func TestGroupByFieldForTheSingleSideInSchema(t *testing.T) { ContainsData: map[string]any{ "__type": map[string]any{ "kind": "ENUM", - "name": "AuthorFields", + "name": "AuthorField", "enumValues": []any{ // Internal related object fields. map[string]any{"name": "published"}, diff --git a/tests/integration/schema/type_explicit_fields_test.go b/tests/integration/schema/type_explicit_fields_test.go new file mode 100644 index 0000000000..844bed4e52 --- /dev/null +++ b/tests/integration/schema/type_explicit_fields_test.go @@ -0,0 +1,60 @@ +// Copyright 2024 Democratized Data Foundation +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package schema + +import ( + "testing" + + testUtils "github.com/sourcenetwork/defradb/tests/integration" +) + +func TestEncryptFieldsForCreateMutation(t *testing.T) { + test := testUtils.TestCase{ + + Description: "Test that type explicit (or user-defined) fields are generated.", + + Actions: []any{ + testUtils.SchemaUpdate{ + Schema: ` + type User { + name: String + age: Int + } + `, + }, + testUtils.IntrospectionRequest{ + Request: ` + { + __type(name: "UserField") { + name + kind + enumValues { + name + } + } + } + `, + ContainsData: map[string]any{ + "__type": map[string]any{ + "kind": "ENUM", + "name": "UserField", + "enumValues": []any{ + map[string]any{"name": "name"}, + map[string]any{"name": "age"}, + }, + }, + }, + }, + }, + } + + testUtils.ExecuteTestCase(t, test) +} diff --git a/tests/integration/test_case.go b/tests/integration/test_case.go index 3af3303446..ad5a86a665 100644 --- a/tests/integration/test_case.go +++ b/tests/integration/test_case.go @@ -230,7 +230,10 @@ type CreateDoc struct { Identity immutable.Option[int] // Specifies whether the document should be encrypted. - IsEncrypted bool + IsDocEncrypted bool + + // Individual fields of the document to encrypt. + EncryptedFields []string // The collection in which this document should be created. CollectionID int diff --git a/tests/integration/utils2.go b/tests/integration/utils2.go index 1dc8006b1b..82ab5620c8 100644 --- a/tests/integration/utils2.go +++ b/tests/integration/utils2.go @@ -1252,9 +1252,7 @@ func createDocViaColSave( func makeContextForDocCreate(s *state, ctx context.Context, action *CreateDoc) context.Context { identity := getIdentity(s, action.Identity) ctx = db.SetContextIdentity(ctx, identity) - if action.IsEncrypted { - ctx = encryption.SetContextConfig(ctx, encryption.DocEncConfig{IsEncrypted: true}) - } + ctx = encryption.SetContextConfigFromParams(ctx, action.IsDocEncrypted, action.EncryptedFields) return ctx } @@ -1322,8 +1320,12 @@ func createDocViaGQL( params := paramName + ": " + input - if action.IsEncrypted { - params = params + ", " + request.EncryptArgName + ": true" + if action.IsDocEncrypted { + params = params + ", " + request.EncryptDocArgName + ": true" + } + if len(action.EncryptedFields) > 0 { + params = params + ", " + request.EncryptFieldsArgName + ": [" + + strings.Join(action.EncryptedFields, ", ") + "]" } req := fmt.Sprintf( @@ -1337,8 +1339,7 @@ func createDocViaGQL( ) txn := getTransaction(s, node, immutable.None[int](), action.ExpectedError) - - ctx := makeContextForDocCreate(s, db.SetContextTxn(s.ctx, txn), &action) + ctx := db.SetContextIdentity(db.SetContextTxn(s.ctx, txn), getIdentity(s, action.Identity)) result := node.ExecRequest(ctx, req) if len(result.GQL.Errors) > 0 {