From 7d14a254169ae48ce02e07df0d89d86df9eea1c9 Mon Sep 17 00:00:00 2001 From: Mic Bowman Date: Thu, 22 Feb 2024 17:10:06 -0800 Subject: [PATCH 1/2] Add an experimental command for importing and exporting collections of contracts Often a contract is meaningful only in the context of other contracts. For example, an asset issuer contract object may depend upon an asset type contract object. The type object is necessary to establish trust in the scope of the asset being issued. In order to simplify sharing, a contract collection file packages information about a collection of contracts and their relationship to each other into a single bundle. The file that is created includes a context file that describes the relationships between the contracts and a list of contract description files that can be used to operate on the contract objects. Signed-off-by: Mic Bowman --- client/MANIFEST | 1 + client/pdo/client/commands/__init__.py | 1 + client/pdo/client/commands/collection.py | 197 +++++++++++++++++++++++ python/pdo/common/utility.py | 12 ++ 4 files changed, 211 insertions(+) create mode 100644 client/pdo/client/commands/collection.py diff --git a/client/MANIFEST b/client/MANIFEST index 4a58821e..35cc16e4 100644 --- a/client/MANIFEST +++ b/client/MANIFEST @@ -7,6 +7,7 @@ ./pdo/client/builder/shell.py ./pdo/client/builder/state.py ./pdo/client/commands/__init__.py +./pdo/client/commands/collection.py ./pdo/client/commands/contract.py ./pdo/client/commands/eservice.py ./pdo/client/commands/ledger.py diff --git a/client/pdo/client/commands/__init__.py b/client/pdo/client/commands/__init__.py index 55fa9464..b259693d 100644 --- a/client/pdo/client/commands/__init__.py +++ b/client/pdo/client/commands/__init__.py @@ -13,6 +13,7 @@ # limitations under the License. __all__ = [ + 'collection', 'context', 'contract', 'eservice', diff --git a/client/pdo/client/commands/collection.py b/client/pdo/client/commands/collection.py new file mode 100644 index 00000000..cf7771af --- /dev/null +++ b/client/pdo/client/commands/collection.py @@ -0,0 +1,197 @@ +# Copyright 2024 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import copy +import logging +import os +import toml +import typing + +from zipfile import ZipFile + +import pdo.client.builder.shell as pshell +import pdo.client.builder.script as pscript +import pdo.client.builder as pbuilder +from pdo.common.utility import experimental + +logger = logging.getLogger(__name__) + +__all__ = [ + 'export_contract_collection', + 'import_contract_collection', + 'script_command_export', + 'script_command_import', + 'do_collection', + 'load_commands', +] + +# ----------------------------------------------------------------- +# ----------------------------------------------------------------- +def __find_contracts__(context : dict) -> typing.List[str] : + """Find all contract save files in a context dictionary + + @type context : dict + @param context : the export context + """ + save_files = [] + for k, v in context.items() : + if k == 'save_file' : + save_files.append(v) + elif isinstance(v, dict): + save_files.extend(__find_contracts__(v)) + + return save_files + +# ----------------------------------------------------------------- +# ----------------------------------------------------------------- +@experimental +def export_contract_collection(context, context_paths : typing.List[str], contract_cache : str, export_file : str) : + """Export the context and associated contract files to a zip file that + can be shared with others who want to use the contract + + @type context: pdo.client.builder.context.Context + @param context: current context + @param context_paths : list of path expressions to retrieve values from a context + @param contract_cache : name of the directory where contract save files are stored + @param export_file : name of the file where the contract family will be written + """ + + # the context we create is initialized, mark it so + export_context = { + 'contexts' : context_paths, + 'initialized' : True, + } + + # copy the portions of the context specified in the context_paths + + # note: while there are fields in the context that are unnecessary for future use of the + # contract, it is far easier to simply copy them here. at some point, this may be smarter about + # only copying the fields that are necessary. + + for c in context_paths : + # since the incoming contexts are paths, we need to make sure + # we copy the context from/to the right location + (*prefix, key) = c.split('.') + ec = export_context + for p in prefix : + if p not in ec : + ec[p] = {} + ec = ec[p] + ec[key] = copy.deepcopy(context.get(c)) + + # now find all of the contract references in the exported context + save_files = __find_contracts__(export_context) + + # and write the contract collection into the zip file + with ZipFile(export_file, 'w') as zf : + # add the context to the package, this has a canonical name + zf.writestr('context.toml', toml.dumps(export_context)) + + # add the contract save files to the package + for s in save_files : + contract_file_name = os.path.join(contract_cache, s) + zf.write(contract_file_name, arcname=s) + +# ----------------------------------------------------------------- +# ----------------------------------------------------------------- +@experimental +def import_contract_collection(context_file_name : str, contract_cache : str, import_file : str) -> dict : + """Import the context and contract files from a collections zip file + + @param context_file_name : name of the file to save imported context + @param contract_cache : name of the directory where contract save files are stored + @param import_file : name of the contract collection file to import + @rtype: dict + @return: the initialized context + """ + with ZipFile(import_file, 'r') as zf : + # extract the context file from the package and save it + # in the specified file + import_context = toml.loads(zf.read('context.toml').decode()) + with open(context_file_name, 'w') as cf : + toml.dump(import_context, cf) + + # find all of the contract references in the exported context + save_files = __find_contracts__(import_context) + + # extract the contract save files into the standard directory + for save_file in save_files : + zf.extract(save_file, contract_cache) + + return import_context + +## ----------------------------------------------------------------- +## ----------------------------------------------------------------- +class script_command_export(pscript.script_command_base) : + name = "export" + help = "Export a context and associated contract files to a contract collection file" + + @classmethod + def add_arguments(cls, subparser) : + subparser.add_argument('--collection-file', help="file where the collection will be saved", required=True, type=str) + subparser.add_argument('--path', help="path to the context key", required=True, nargs='+', type=str) + subparser.add_argument('--prefix', help="prefix for new contract context", type=str) + + @classmethod + def invoke(cls, state, bindings, path, collection_file, prefix='', **kwargs) : + data_directory = bindings.get('data', state.get(['Contract', 'DataDirectory'])) + contract_cache = bindings.get('save', os.path.join(data_directory, '__contract_cache__')) + export_file = bindings.expand(collection_file) + + context = pbuilder.Context(state, prefix) + export_contract_collection(context, path, contract_cache, export_file) + + return export_file + +## ----------------------------------------------------------------- +## ----------------------------------------------------------------- +class script_command_import(pscript.script_command_base) : + name = "import" + help = "Import a context and associated contract files from a contract collection file" + + @classmethod + def add_arguments(cls, subparser) : + subparser.add_argument('--collection-file', help="file where the collection will be saved", required=True, type=str) + subparser.add_argument('--context-file', help="file where context will be saved", required=True, type=str) + subparser.add_argument('--prefix', help="prefix for new contract context", type=str) + + @classmethod + def invoke(cls, state, bindings, collection_file, context_file, prefix='', **kwargs) : + data_directory = bindings.get('data', state.get(['Contract', 'DataDirectory'])) + contract_cache = bindings.get('save', os.path.join(data_directory, '__contract_cache__')) + import_file = bindings.expand(collection_file) + context_file = bindings.expand(context_file) + + if import_contract_collection(context_file, contract_cache, import_file) : + prefix = prefix or [] + pbuilder.Context.LoadContextFile(state, bindings, context_file, prefix=prefix) + return True + + return False + +## ----------------------------------------------------------------- +## Create the generic, shell independent version of the aggregate command +## ----------------------------------------------------------------- +__subcommands__ = [ + script_command_export, + script_command_import, +] +do_collection = pscript.create_shell_command('collection', __subcommands__) + +## ----------------------------------------------------------------- +## Enable binding of the shell independent version to a pdo-shell command +## ----------------------------------------------------------------- +def load_commands(cmdclass) : + pshell.bind_shell_command(cmdclass, 'collection', do_collection) diff --git a/python/pdo/common/utility.py b/python/pdo/common/utility.py index 3cd371d1..26608b31 100644 --- a/python/pdo/common/utility.py +++ b/python/pdo/common/utility.py @@ -55,6 +55,18 @@ def new_func(*args, **kwargs): return new_func +def experimental(func): + """decorator to mark functions as experimental, logs a warning + with information about the function and the caller + """ + @functools.wraps(func) + def new_func(*args, **kwargs): + stack = inspect.stack() + logger.warn('invocation of experimental function %s by %s in file %s', func.__name__, stack[1][3], stack[1][1]) + return func(*args, **kwargs) + + return new_func + class classproperty(property) : """decorator to mark a class method as a property, used for simplified access to module initiated variables From 903ec7179cf232664c451cec97a0a651df721003 Mon Sep 17 00:00:00 2001 From: Mic Bowman Date: Tue, 5 Mar 2024 17:35:18 -0800 Subject: [PATCH 2/2] Small updates for PR feedback Added documentation (client/docs/collection.md). Added entrypoint for a collection command. Updated types. Signed-off-by: Mic Bowman --- client/docs/collection.md | 141 +++++++++++++++++++++++ client/pdo/client/commands/collection.py | 16 ++- client/pdo/client/scripts/EntryPoints.py | 4 + client/setup.py | 1 + 4 files changed, 158 insertions(+), 4 deletions(-) create mode 100644 client/docs/collection.md diff --git a/client/docs/collection.md b/client/docs/collection.md new file mode 100644 index 00000000..1cf1802e --- /dev/null +++ b/client/docs/collection.md @@ -0,0 +1,141 @@ + + +# Contract Collection Files # + +**THIS FILE DESCRIBES AN EXPERIMENTAL FACILITY FOR PDO** + +Often a contract is meaningful only in the context of other +contracts. For example, an asset issuer contract object may depend +upon an asset type contract object. The type object is necessary to +establish trust in the scope of the asset being issued. + +In order to simplify sharing, a contract collection file packages +information about a collection of contracts and their relationship to +each other into a single bundle. The file that is created includes a +context file that describes the relationships between the contracts +and a list of contract description files that can be used to operate +on the contract objects. + +## What's the Problem ## + +Ultimately, the contract collection file operations are designed to +simplify sharing of complex sets of inter-related contracts. + +Currently, context files are the means through which relationships +between contract objects are captured. For example, the following +context file uses context relative links to describe the relationship +between an asset type, a vetting organization, and an asset issuer. In +this case, the blue marble issuer depends on specific asset type and +vetting contract object. That is, many operations on the blue marble +issuer contract object require operations on the corresponding asset +type and vetting contract objects. + +``` +[marbles.blue.asset_type] +identity = "blue_type" +source = "@{ContractFamily.Exchange.asset_type.source}" +name = "blue marble" +description = "blue marble asset type" +link = "http://" + +[marbles.blue.vetting] +identity = "blue_vetting" +source = "@{ContractFamily.Exchange.vetting.source}" +asset_type_context = "@{..asset_type}" + +[marbles.blue.issuer] +identity = "blue_issuer" +source = "${ContractFamily.Exchange.issuer.source}" +asset_type_context = "@{..asset_type}" +vetting_context = "@{..vetting}" +``` + +When the asset type, vetting organization and issuer contracts have +been created, common PDO tools will add a reference to the contract +description file (the `save_file` attribute). + +``` +[marbles.blue.asset_type] +identity = "blue_type" +source = "@{ContractFamily.Exchange.asset_type.source}" +name = "blue marble" +description = "blue marble asset type" +link = "http://" +save_file = "asset_type_5057b384b77f99cd.pdo" + +[marbles.blue.vetting] +identity = "blue_vetting" +source = "@{ContractFamily.Exchange.vetting.source}" +asset_type_context = "@{..asset_type}" +save_file = "vetting_6e3338599072eecd.pdo" + +[marbles.blue.issuer] +identity = "blue_issuer" +source = "${ContractFamily.Exchange.issuer.source}" +asset_type_context = "@{..asset_type}" +vetting_context = "@{..vetting}" +save_file = "issuer_contract_6926ae75188c1954.pdo" +``` + +Contract collection file operations are intended to provide a simple +way to share complete collections of contract object and the +relationships between them. + +## Format of the Contract Collection File ## + +A contract collection file is a compressed archive that includes a +collection of contract description files and a context file that +describes the relationships between the contract objects in the +description files. + +The context file, `context.toml`, is a TOML formatted file that +captures the relative relationships between the contract objects. All +context relative paths (e.g. `@{..vetting}`) must be resolved within +the context defined in `context.toml`. In addition, the context +contains a list of the high level contexts to simplify enumeration of +the objects in the context file. + +Each of the contract description files is stored separately in the +bundle. While the operations provided by PDO currently attach unique +identifiers to the file names, ultimately the file names should not be +assumed to be globally unique. + +## Export a Contract Collection File ## + +The function `export_contract_collection` creates a contract collection file: +``` + export_contract_collection(context, context_paths, contract_cache, export_file) +``` + + * **context**: current context + * **context_paths** : list of path expressions to retrieve values from a context + * **contract_cache** : name of the directory where contract save files are stored + * **export_file** : name of the file where the contract family will be written + +```python + export_contract_collection( + context.get_context('marbles.blue'), + ['asset_type', 'vetting', 'issuer'], + '__contract_cache__', + 'blue_marble_collection.zip') +``` + +## Import a Contract Collection File ## + +The function `import_contract_collection` creates a context and save +contract save files from a contract collection file: +``` +import_contract_collection(context_file_name, contract_cache, import_file) +``` + + * **context_file_name** : name of the file to save imported context + * **contract_cache** : name of the directory where contract save files are stored + * **import_file** : name of the contract collection file to import + +```python + import_contract_collection('blue_marble.toml', '__contract_cache__', 'blue_marble_collection.zip') + context.LoadContextFile(state, bindings, 'blue_marble.toml', prefix='marbles.blue') +``` diff --git a/client/pdo/client/commands/collection.py b/client/pdo/client/commands/collection.py index cf7771af..272c1d02 100644 --- a/client/pdo/client/commands/collection.py +++ b/client/pdo/client/commands/collection.py @@ -12,6 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. +# collections are a package for sharing multiple, interrelated +# contracts. more information is available in the file +# $PDO_SOURCE_ROOT/client/docs/collection.md + import argparse import copy import logging @@ -29,8 +33,8 @@ logger = logging.getLogger(__name__) __all__ = [ - 'export_contract_collection', - 'import_contract_collection', + 'export', + 'import', 'script_command_export', 'script_command_import', 'do_collection', @@ -57,11 +61,15 @@ def __find_contracts__(context : dict) -> typing.List[str] : # ----------------------------------------------------------------- # ----------------------------------------------------------------- @experimental -def export_contract_collection(context, context_paths : typing.List[str], contract_cache : str, export_file : str) : +def export_contract_collection( + context : pbuilder.context.Context, + context_paths : typing.List[str], + contract_cache : str, + export_file : str) : """Export the context and associated contract files to a zip file that can be shared with others who want to use the contract - @type context: pdo.client.builder.context.Context + @type context: pbuilder.context.Context @param context: current context @param context_paths : list of path expressions to retrieve values from a context @param contract_cache : name of the directory where contract save files are stored diff --git a/client/pdo/client/scripts/EntryPoints.py b/client/pdo/client/scripts/EntryPoints.py index dd8c2461..234fc338 100644 --- a/client/pdo/client/scripts/EntryPoints.py +++ b/client/pdo/client/scripts/EntryPoints.py @@ -22,6 +22,10 @@ warnings.catch_warnings() warnings.simplefilter("ignore") +# ----------------------------------------------------------------- +def run_shell_collection() : + run_shell_command('do_collection', 'pdo.client.commands.collection') + # ----------------------------------------------------------------- def run_shell_context() : run_shell_command('do_context', 'pdo.client.commands.context') diff --git a/client/setup.py b/client/setup.py index 289823dd..0d67b527 100644 --- a/client/setup.py +++ b/client/setup.py @@ -62,6 +62,7 @@ entry_points = { 'console_scripts': [ 'pdo-shell = pdo.client.scripts.ShellCLI:Main', + 'pdo-collection = pdo.client.scripts.EntryPoints:run_shell_collection', 'pdo-contract = pdo.client.scripts.EntryPoints:run_shell_contract', 'pdo-context = pdo.client.scripts.EntryPoints:run_shell_context', 'pdo-ledger = pdo.client.scripts.EntryPoints:run_shell_ledger',