Add an experimental command for importing and exporting collections of contracts

client/MANIFEST

@@ -7,6 +7,7 @@
 ./pdo/client/builder/shell.py
 ./pdo/client/builder/state.py
 ./pdo/client/commands/__init__.py
+./pdo/client/commands/collection.py
 ./pdo/client/commands/contract.py
 ./pdo/client/commands/eservice.py
 ./pdo/client/commands/ledger.py

client/docs/collection.md

@@ -0,0 +1,141 @@
+<!---
+Licensed under Creative Commons Attribution 4.0 International License
+https://creativecommons.org/licenses/by/4.0/
+--->
+
+# Contract Collection Files #
+
+**THIS FILE DESCRIBES AN EXPERIMENTAL FACILITY FOR PDO**
+
+Often a contract is meaningful only in the context of other
+contracts. For example, an asset issuer contract object may depend
+upon an asset type contract object. The type object is necessary to
+establish trust in the scope of the asset being issued.
+
+In order to simplify sharing, a contract collection file packages
+information about a collection of contracts and their relationship to
+each other into a single bundle. The file that is created includes a
+context file that describes the relationships between the contracts
+and a list of contract description files that can be used to operate
+on the contract objects.
+
+## What's the Problem ##
+
+Ultimately, the contract collection file operations are designed to
+simplify sharing of complex sets of inter-related contracts.
+
+Currently, context files are the means through which relationships
+between contract objects are captured. For example, the following
+context file uses context relative links to describe the relationship
+between an asset type, a vetting organization, and an asset issuer. In
+this case, the blue marble issuer depends on specific asset type and
+vetting contract object. That is, many operations on the blue marble
+issuer contract object require operations on the corresponding asset
+type and vetting contract objects.
+
+```
+[marbles.blue.asset_type]
+identity = "blue_type"
+source = "@{ContractFamily.Exchange.asset_type.source}"
+name = "blue marble"
+description = "blue marble asset type"
+link = "http://"
+
+[marbles.blue.vetting]
+identity = "blue_vetting"
+source = "@{ContractFamily.Exchange.vetting.source}"
+asset_type_context = "@{..asset_type}"
+
+[marbles.blue.issuer]
+identity = "blue_issuer"
+source = "${ContractFamily.Exchange.issuer.source}"
+asset_type_context = "@{..asset_type}"
+vetting_context = "@{..vetting}"
+```
+
+When the asset type, vetting organization and issuer contracts have
+been created, common PDO tools will add a reference to the contract
+description file (the `save_file` attribute).
+
+```
+[marbles.blue.asset_type]
+identity = "blue_type"
+source = "@{ContractFamily.Exchange.asset_type.source}"
+name = "blue marble"
+description = "blue marble asset type"
+link = "http://"
+save_file = "asset_type_5057b384b77f99cd.pdo"
+
+[marbles.blue.vetting]
+identity = "blue_vetting"
+source = "@{ContractFamily.Exchange.vetting.source}"
+asset_type_context = "@{..asset_type}"
+save_file = "vetting_6e3338599072eecd.pdo"
+
+[marbles.blue.issuer]
+identity = "blue_issuer"
+source = "${ContractFamily.Exchange.issuer.source}"
+asset_type_context = "@{..asset_type}"
+vetting_context = "@{..vetting}"
+save_file = "issuer_contract_6926ae75188c1954.pdo"
+```
+
+Contract collection file operations are intended to provide a simple
+way to share complete collections of contract object and the
+relationships between them.
+
+## Format of the Contract Collection File ##
+
+A contract collection file is a compressed archive that includes a
+collection of contract description files and a context file that
+describes the relationships between the contract objects in the
+description files.
+
+The context file, `context.toml`, is a TOML formatted file that
+captures the relative relationships between the contract objects. All
+context relative paths (e.g. `@{..vetting}`) must be resolved within
+the context defined in `context.toml`. In addition, the context
+contains a list of the high level contexts to simplify enumeration of
+the objects in the context file.
+
+Each of the contract description files is stored separately in the
+bundle. While the operations provided by PDO currently attach unique
+identifiers to the file names, ultimately the file names should not be
+assumed to be globally unique.
+
+## Export a Contract Collection File ##
+
+The function `export_contract_collection` creates a contract collection file:
+```
+    export_contract_collection(context, context_paths, contract_cache, export_file)
+```
+
+  * **context**: current context
+  * **context_paths** : list of path expressions to retrieve values from a context
+  * **contract_cache** : name of the directory where contract save files are stored
+  * **export_file** : name of the file where the contract family will be written
+
+```python
+    export_contract_collection(
+        context.get_context('marbles.blue'),
+        ['asset_type', 'vetting', 'issuer'],
+        '__contract_cache__',
+        'blue_marble_collection.zip')
+```
+
+## Import a Contract Collection File ##
+
+The function `import_contract_collection` creates a context and save
+contract save files from a contract collection file:
+```
+import_contract_collection(context_file_name, contract_cache, import_file)
+```
+
+  * **context_file_name** : name of the file to save imported context
+  * **contract_cache** : name of the directory where contract save files are stored
+  * **import_file** : name of the contract collection file to import
+
+```python
+    import_contract_collection('blue_marble.toml', '__contract_cache__', 'blue_marble_collection.zip')
+    context.LoadContextFile(state, bindings, 'blue_marble.toml', prefix='marbles.blue')
+```

client/pdo/client/commands/__init__.py

@@ -13,6 +13,7 @@
 # limitations under the License.
 
 __all__ = [
+    'collection',
     'context',
     'contract',
     'eservice',

client/pdo/client/commands/collection.py

@@ -0,0 +1,205 @@
+# Copyright 2024 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# collections are a package for sharing multiple, interrelated
+# contracts. more information is available in the file
+# $PDO_SOURCE_ROOT/client/docs/collection.md
+
+import argparse
+import copy
+import logging
+import os
+import toml
+import typing
+
+from zipfile import ZipFile
+
+import pdo.client.builder.shell as pshell
+import pdo.client.builder.script as pscript
+import pdo.client.builder as pbuilder
+from pdo.common.utility import experimental
+
+logger = logging.getLogger(__name__)
+
+__all__ = [
+    'export',
+    'import',
+    'script_command_export',
+    'script_command_import',
+    'do_collection',
+    'load_commands',
+]
+
+# -----------------------------------------------------------------
+# -----------------------------------------------------------------
+def __find_contracts__(context : dict) -> typing.List[str] :
+    """Find all contract save files in a context dictionary
+
+    @type context : dict
+    @param context : the export context
+    """
+    save_files = []
+    for k, v in context.items() :
+        if k == 'save_file' :
+            save_files.append(v)
+        elif isinstance(v, dict):
+            save_files.extend(__find_contracts__(v))
+
+    return save_files
+
+# -----------------------------------------------------------------
+# -----------------------------------------------------------------
+@experimental
+def export_contract_collection(
+        context : pbuilder.context.Context,
+        context_paths : typing.List[str],
+        contract_cache : str,
+        export_file : str) :
+    """Export the context and associated contract files to a zip file that
+    can be shared with others who want to use the contract
+
+    @type context: pbuilder.context.Context
+    @param context: current context
+    @param context_paths : list of path expressions to retrieve values from a context
+    @param contract_cache : name of the directory where contract save files are stored
+    @param export_file : name of the file where the contract family will be written
+    """
+
+    # the context we create is initialized, mark it so
+    export_context = {
+        'contexts' : context_paths,
+        'initialized' : True,
+    }
+
+    # copy the portions of the context specified in the context_paths
+
+    # note: while there are fields in the context that are unnecessary for future use of the
+    # contract, it is far easier to simply copy them here. at some point, this may be smarter about
+    # only copying the fields that are necessary.
+
+    for c in context_paths :
+        # since the incoming contexts are paths, we need to make sure
+        # we copy the context from/to the right location
+        (*prefix, key) = c.split('.')
+        ec = export_context
+        for p in prefix :
+            if p not in ec :
+                ec[p] = {}
+            ec = ec[p]
+        ec[key] = copy.deepcopy(context.get(c))
+
+    # now find all of the contract references in the exported context
+    save_files = __find_contracts__(export_context)
+
+    # and write the contract collection into the zip file
+    with ZipFile(export_file, 'w') as zf :
+        # add the context to the package, this has a canonical name
+        zf.writestr('context.toml', toml.dumps(export_context))
+
+        # add the contract save files to the package
+        for s in save_files :
+            contract_file_name = os.path.join(contract_cache, s)
+            zf.write(contract_file_name, arcname=s)
+
+# -----------------------------------------------------------------
+# -----------------------------------------------------------------
+@experimental
+def import_contract_collection(context_file_name : str, contract_cache : str, import_file : str) -> dict :
+    """Import the context and contract files from a collections zip file
+
+    @param context_file_name : name of the file to save imported context
+    @param contract_cache : name of the directory where contract save files are stored
+    @param import_file : name of the contract collection file to import
+    @rtype: dict
+    @return: the initialized context
+    """
+    with ZipFile(import_file, 'r') as zf :
+        # extract the context file from the package and save it
+        # in the specified file
+        import_context = toml.loads(zf.read('context.toml').decode())
+        with open(context_file_name, 'w') as cf :
+            toml.dump(import_context, cf)
+
+        # find all of the contract references in the exported context
+        save_files = __find_contracts__(import_context)
+
+        # extract the contract save files into the standard directory
+        for save_file in save_files :
+            zf.extract(save_file, contract_cache)
+
+    return import_context
+
+## -----------------------------------------------------------------
+## -----------------------------------------------------------------
+class script_command_export(pscript.script_command_base) :
+    name = "export"
+    help = "Export a context and associated contract files to a contract collection file"
+
+    @classmethod
+    def add_arguments(cls, subparser) :
+        subparser.add_argument('--collection-file', help="file where the collection will be saved", required=True, type=str)
+        subparser.add_argument('--path', help="path to the context key", required=True, nargs='+', type=str)
+        subparser.add_argument('--prefix', help="prefix for new contract context", type=str)
+
+    @classmethod
+    def invoke(cls, state, bindings, path, collection_file, prefix='', **kwargs) :
+        data_directory = bindings.get('data', state.get(['Contract', 'DataDirectory']))
+        contract_cache = bindings.get('save', os.path.join(data_directory, '__contract_cache__'))
+        export_file = bindings.expand(collection_file)
+
+        context = pbuilder.Context(state, prefix)
+        export_contract_collection(context, path, contract_cache, export_file)
+
+        return export_file
+
+## -----------------------------------------------------------------
+## -----------------------------------------------------------------
+class script_command_import(pscript.script_command_base) :
+    name = "import"
+    help = "Import a context and associated contract files from a contract collection file"
+
+    @classmethod
+    def add_arguments(cls, subparser) :
+        subparser.add_argument('--collection-file', help="file where the collection will be saved", required=True, type=str)
+        subparser.add_argument('--context-file', help="file where context will be saved", required=True, type=str)
+        subparser.add_argument('--prefix', help="prefix for new contract context", type=str)
+
+    @classmethod
+    def invoke(cls, state, bindings, collection_file, context_file, prefix='', **kwargs) :
+        data_directory = bindings.get('data', state.get(['Contract', 'DataDirectory']))
+        contract_cache = bindings.get('save', os.path.join(data_directory, '__contract_cache__'))
+        import_file = bindings.expand(collection_file)
+        context_file = bindings.expand(context_file)
+
+        if import_contract_collection(context_file, contract_cache, import_file) :
+            prefix = prefix or []
+            pbuilder.Context.LoadContextFile(state, bindings, context_file, prefix=prefix)
+            return True
+
+        return False
+
+## -----------------------------------------------------------------
+## Create the generic, shell independent version of the aggregate command
+## -----------------------------------------------------------------
+__subcommands__ = [
+    script_command_export,
+    script_command_import,
+]
+do_collection = pscript.create_shell_command('collection', __subcommands__)
+
+## -----------------------------------------------------------------
+## Enable binding of the shell independent version to a pdo-shell command
+## -----------------------------------------------------------------
+def load_commands(cmdclass) :
+    pshell.bind_shell_command(cmdclass, 'collection', do_collection)

client/pdo/client/scripts/EntryPoints.py

@@ -22,6 +22,10 @@
 warnings.catch_warnings()
 warnings.simplefilter("ignore")
 
+# -----------------------------------------------------------------
+def run_shell_collection() :
+    run_shell_command('do_collection', 'pdo.client.commands.collection')
+
 # -----------------------------------------------------------------
 def run_shell_context() :
     run_shell_command('do_context', 'pdo.client.commands.context')

client/setup.py

@@ -62,6 +62,7 @@
       entry_points = {
           'console_scripts': [
               'pdo-shell = pdo.client.scripts.ShellCLI:Main',
+              'pdo-collection = pdo.client.scripts.EntryPoints:run_shell_collection',
               'pdo-contract = pdo.client.scripts.EntryPoints:run_shell_contract',
               'pdo-context = pdo.client.scripts.EntryPoints:run_shell_context',
               'pdo-ledger = pdo.client.scripts.EntryPoints:run_shell_ledger',