From c458ff037ff60569d2c892b7cf6d214cde6369f2 Mon Sep 17 00:00:00 2001 From: thorbjoernl <51087536+thorbjoernl@users.noreply.github.com> Date: Tue, 3 Sep 2024 13:17:06 +0200 Subject: [PATCH 01/20] docs: Update installation instructions in readme --- README.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 703dc0f..4b78281 100644 --- a/README.md +++ b/README.md @@ -32,8 +32,12 @@ Our documentation can be found [here](https://aerovaldb.readthedocs.io/) 9. To be discussed: do we want to allow the programmer to set open-modes, e.g. 'r', 'rw', 'w'? ## Installation -`python -m pip install 'aerovaldb@git+https://github.com/metno/aerovaldb.git'` +The most recent stable version of aerovaldb can be installed from pypi like this: +- `pip install aerovaldb` + +Development versions can be installed directly from Github like this: +`pip install 'aerovaldb@git+https://github.com/metno/aerovaldb.git'` ## Usage @@ -97,7 +101,7 @@ with aerovaldb.open('json_files:path/to/data/') as db: ## COPYRIGHT -Copyright (C) 2024 Augustin Mortier, Thorbjørn Lunding, Heiko Klein, Norwegian Meteorological Institute +Copyright (C) 2024 Augustin Mortier, Thorbjørn Lundin, Heiko Klein, Norwegian Meteorological Institute This library is free software; you can redistribute it and/or modify it under the terms of the GNU General Public From c6868d95737e4a0d76996beb933170080736db4a Mon Sep 17 00:00:00 2001 From: thorbjoernl <51087536+thorbjoernl@users.noreply.github.com> Date: Tue, 3 Sep 2024 13:21:34 +0200 Subject: [PATCH 02/20] docs: Update installation instructions on readthedocs --- docs/installation.rst | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/docs/installation.rst b/docs/installation.rst index ae2ce33..0e011c7 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -3,21 +3,23 @@ Installation You can install aerovaldb via pip or from source. -Via pip -^^^^^^^ +Latest Stable Release: +^^^^^^^^^^^^^^^^^^^^^^ -This will install the latest aerovaldb and all its dependencies. +This will install the latest stable aerovaldb version from PyPi: :: + pip install aerovaldb - # install aerovaldb - python -m pip install aerovaldb@git+https://github.com/metno/aerovaldb@main +Development releases +^^^^^^^^^^^^^^^^^^^^ +This will install the most recent development version from Github: +:: + pip install git+https://github.com/metno/aerovaldb.git@main From source: -^^^^^^^ +^^^^^^^^^^^^ :: - - # install aerovaldb on machines with numpy git clone https://github.com/metno/aerovaldb.git cd aerovaldb python -m pip install . From e21fa33c18cbfc15f017b32df745231206cdf5d7 Mon Sep 17 00:00:00 2001 From: thorbjoernl <51087536+thorbjoernl@users.noreply.github.com> Date: Tue, 3 Sep 2024 13:38:59 +0200 Subject: [PATCH 03/20] doc: Start documentation for extending aerovaldb --- docs/extending-aerovaldb.rst | 9 +++++++++ docs/index.rst | 1 + docs/installation.rst | 4 ++-- 3 files changed, 12 insertions(+), 2 deletions(-) create mode 100644 docs/extending-aerovaldb.rst diff --git a/docs/extending-aerovaldb.rst b/docs/extending-aerovaldb.rst new file mode 100644 index 0000000..255fc27 --- /dev/null +++ b/docs/extending-aerovaldb.rst @@ -0,0 +1,9 @@ +Extending AerovalDB +=================== + +AerovalDB is designed to allow custom implementations to allow storage in new storage formats. This is done by writing a new class inheriting from :class:`aerovaldb.AerovalDB`. + +Unless overridden, getters and setters will be routed to :meth:`aerovaldb.AerovalDB._get` and :meth:`aerovaldb.AerovalDB._put` respectively, so this is were the main read and write behaviour should be implemented. These functions receive a route, as well as arguments based on which to get/put data and are intended to be used based on route-lookup (for example, :class:`aerovaldb.jsondb.jsonfiledb.AerovalJsonFileDB` translates the route into a file path template, while :class:`aerovaldb.sqlitedb.sqlitedb.AerovalSqliteDB` translates the route into a table name). + +URI Scheme +---------- diff --git a/docs/index.rst b/docs/index.rst index 353db81..ef4043c 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -52,6 +52,7 @@ Writing installation api locking + extending-aerovaldb genindex Indices and tables diff --git a/docs/installation.rst b/docs/installation.rst index 0e011c7..10201e4 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -10,8 +10,8 @@ This will install the latest stable aerovaldb version from PyPi: :: pip install aerovaldb -Development releases -^^^^^^^^^^^^^^^^^^^^ +Development releases: +^^^^^^^^^^^^^^^^^^^^^ This will install the most recent development version from Github: :: From 09ffa050326f45759a3e334a5391a17793181d18 Mon Sep 17 00:00:00 2001 From: thorbjoernl <51087536+thorbjoernl@users.noreply.github.com> Date: Tue, 3 Sep 2024 13:56:53 +0200 Subject: [PATCH 04/20] docs: Fix typos --- src/aerovaldb/aerovaldb.py | 64 ++++++++++++++++++++------------------ src/aerovaldb/plugins.py | 20 +++++++----- src/aerovaldb/types.py | 3 ++ 3 files changed, 48 insertions(+), 39 deletions(-) diff --git a/src/aerovaldb/aerovaldb.py b/src/aerovaldb/aerovaldb.py index 8cd483f..d304298 100644 --- a/src/aerovaldb/aerovaldb.py +++ b/src/aerovaldb/aerovaldb.py @@ -133,7 +133,7 @@ async def get_glob_stats( :param default: Default value that will be returned instead of raising FileNotFoundError if not data was found (Will be returned as is and not converted to match access_type). - :returns The fetched data. + :returns : The fetched data. """ raise NotImplementedError @@ -193,7 +193,7 @@ async def get_heatmap( :param default: Default value that will be returned instead of raising FileNotFoundError if not data was found (Will be returned as is and not converted to match access_type). - :returns The fetched data. + :returns : The fetched data. """ raise NotImplementedError @@ -253,9 +253,9 @@ async def get_contour( :param access_type: How the data is to be retrieved (See AccessType for details) :param cache: Whether to use cache for this read. :param default: Default value that will be returned instead of raising FileNotFoundError - if not data was found (Will be returned as is and not converted to match access_type). + if no data was found (Will be returned as is and not converted to match access_type). - :returns The fetched data. + :returns : The fetched data. """ raise NotImplementedError @@ -311,9 +311,9 @@ async def get_timeseries( :param access_type: How the data is to be retrieved (See AccessType for details) :param cache: Whether to use cache for this read. :param default: Default value that will be returned instead of raising FileNotFoundError - if not data was found (Will be returned as is and not converted to match access_type). + if no data was found (Will be returned as is and not converted to match access_type). - :returns The fetched data. + :returns : The fetched data. """ raise NotImplementedError @@ -391,7 +391,7 @@ async def get_timeseries_weekly( :param default: Default value that will be returned instead of raising FileNotFoundError if not data was found (Will be returned as is and not converted to match access_type). - :returns The fetched data. + :returns : The fetched data. """ raise NotImplementedError @@ -443,7 +443,7 @@ async def get_experiments( :param default: Default value that will be returned instead of raising FileNotFoundError if not data was found (Will be returned as is and not converted to match access_type). - :returns The fetched data. + :returns : The fetched data. """ raise NotImplementedError @@ -492,7 +492,7 @@ async def get_config( :param default: Default value that will be returned instead of raising FileNotFoundError if not data was found (Will be returned as is and not converted to match access_type). - :returns The fetched data. + :returns : The fetched data. """ raise NotImplementedError @@ -531,7 +531,7 @@ async def get_menu( :param default: Default value that will be returned instead of raising FileNotFoundError if not data was found (Will be returned as is and not converted to match access_type). - :returns The fetched data. + :returns : The fetched data. """ raise NotImplementedError @@ -569,7 +569,7 @@ async def get_statistics( :param default: Default value that will be returned instead of raising FileNotFoundError if not data was found (Will be returned as is and not converted to match access_type). - :returns The fetched data. + :returns : The fetched data. """ raise NotImplementedError @@ -609,7 +609,7 @@ async def get_ranges( :param default: Default value that will be returned instead of raising FileNotFoundError if not data was found (Will be returned as is and not converted to match access_type). - :returns The fetched data. + :returns : The fetched data. """ raise NotImplementedError @@ -647,7 +647,7 @@ async def get_regions( :param default: Default value that will be returned instead of raising FileNotFoundError if not data was found (Will be returned as is and not converted to match access_type). - :returns The fetched data. + :returns : The fetched data. """ raise NotImplementedError @@ -685,7 +685,7 @@ async def get_models_style( :param default: Default value that will be returned instead of raising FileNotFoundError if not data was found (Will be returned as is and not converted to match access_type). - :returns The fetched data. + :returns : The fetched data. """ raise NotImplementedError @@ -737,7 +737,7 @@ async def get_map( :param default: Default value that will be returned instead of raising FileNotFoundError if not data was found (Will be returned as is and not converted to match access_type). - :returns The fetched data. + :returns : The fetched data. """ raise NotImplementedError @@ -824,7 +824,7 @@ async def get_scatter( :param default: Default value that will be returned instead of raising FileNotFoundError if not data was found (Will be returned as is and not converted to match access_type). - :returns The fetched data. + :returns : The fetched data. """ raise NotImplementedError @@ -888,7 +888,7 @@ async def get_profiles( :param default: Default value that will be returned instead of raising FileNotFoundError if not data was found (Will be returned as is and not converted to match access_type). - :returns The fetched data. + :returns : The fetched data. """ raise NotImplementedError @@ -948,7 +948,7 @@ async def get_heatmap_timeseries( :param default: Default value that will be returned instead of raising FileNotFoundError if not data was found (Will be returned as is and not converted to match access_type). - :returns The fetched data. + :returns : The fetched data. """ raise NotImplementedError @@ -1010,7 +1010,7 @@ async def get_forecast( :param default: Default value that will be returned instead of raising FileNotFoundError if not data was found (Will be returned as is and not converted to match access_type). - :returns The fetched data. + :returns : The fetched data. """ raise NotImplementedError @@ -1068,7 +1068,7 @@ async def get_gridded_map( :param default: Default value that will be returned instead of raising FileNotFoundError if not data was found (Will be returned as is and not converted to match access_type). - :returns The fetched data. + :returns : The fetched data. """ raise NotImplementedError @@ -1120,7 +1120,7 @@ async def get_report( :param default: Default value that will be returned instead of raising FileNotFoundError if not data was found (Will be returned as is and not converted to match access_type). - :returns The fetched data. + :returns : The fetched data. """ raise NotImplementedError @@ -1158,9 +1158,11 @@ async def get_by_uri( Note: ----- - URI is intended to be consistent between implementations. Using get_by_uri() + URI is intended to be consistent between implementations but not between + versions of aerovaldb. Using :meth:`aerovaldb.AerovalDB.get_by_uri` to fetch an identifier which can then be written to another connector using - its respective put_by_uri() method. + its respective :meth:`aerovaldb.AerovalDB.put_by_uri` method is a supported + use case. """ raise NotImplementedError @@ -1174,9 +1176,11 @@ async def put_by_uri(self, obj, uri: str): Note: ----- - URI is intended to be consistent between implementations. Using get_by_uri() + URI is intended to be consistent between implementations but not between + versions of aerovaldb. Using :meth:`aerovaldb.AerovalDB.get_by_uri` to fetch an identifier which can then be written to another connector using - its respective put_by_uri() method. + its respective :meth:`aerovaldb.AerovalDB.put_by_uri` method is a supported + use case. """ raise NotImplementedError @@ -1185,7 +1189,7 @@ def lock(self): between instances of aerovaldb. Intended to be used as a context manager. - See also: https://aerovaldb.readthedocs.io/en/latest/locking.html + See also: `Locking `_ """ raise NotImplementedError @@ -1217,12 +1221,10 @@ def _normalize_access_type( @async_and_sync async def list_all(self, access_type: str | AccessType = AccessType.URI): - """Iterator to list over the URI of each object - stored in the current aerovaldb connection, returning - the URI of each. + """Returns a list of identifiers for each object stored in the database - :param access_type : What to return (This is implementation specific, but in general - each implementation should support URI). + :param access_type : What to return (This is implementation specific, but + in general each implementation should support URI, some may support FILE_PATH). :raises : UnsupportedOperation For non-supported acces types. """ diff --git a/src/aerovaldb/plugins.py b/src/aerovaldb/plugins.py index e1ca368..9aeeebe 100644 --- a/src/aerovaldb/plugins.py +++ b/src/aerovaldb/plugins.py @@ -49,14 +49,18 @@ def open(resource, /, use_async: bool = False) -> AerovalDB: """open an AerovalDB directly, sending args and kwargs directly to the `AervoalDB()` function - :param resource: the resource-name for the database. The resource can be - - 'entrypoint:path', with path being the location where the database should be generated - (eg. 'json_files:.') - - 'path', with path containing either an aerovaldb.cfg (Not yet implemented) configuration - or path being a json_files dabasase (for example, '.' is equivalent to 'json_files:.') - :param use_async : If true, aiofile will be used to read files, otherwise files will be read - synchronously. - :return: an implementation-object of AerovalDB openend to a location + :param resource: the resource-identifier for the database. The resource can be + - 'entrypoint:path', with entrypoint being the type of database connection + (eg. 'json_files' or 'sqlitedb') being the location where the database is + located (eg. 'json_files:.') + - 'path', a path to a json_files folder or sqlite file. + - ':memory:' an sqlite in-memory database. Contents are not persistently + stored! + + :param use_async : If true, aiofile will be used to read files, otherwise + files will be read synchronously (Currently only used by json_files). + :return : an implementation-object of AerovalDB openend and initialized to a + location. """ if resource == ":memory:": # Special case for sqlite in memory database. diff --git a/src/aerovaldb/types.py b/src/aerovaldb/types.py index 3147bb7..62264a5 100644 --- a/src/aerovaldb/types.py +++ b/src/aerovaldb/types.py @@ -6,9 +6,12 @@ class AccessType(Enum): and returned. JSON_STR: Result will be returned as an unparsed json string. + FILE_PATH: Result will be returned as the file path to the file containing the data. + OBJ: The json will be parsed and returned as a python object. + URI: A string which is a unique identifier of this asset between implementations of Aerovaldb. Can be used with `get_by_uuid()` and `put_by_uuid()` to read or write respectively. From d84f5ee8513390f31e987f142de9a56b907f7db8 Mon Sep 17 00:00:00 2001 From: thorbjoernl <51087536+thorbjoernl@users.noreply.github.com> Date: Tue, 3 Sep 2024 14:07:23 +0200 Subject: [PATCH 05/20] docs: Fix typos --- src/aerovaldb/aerovaldb.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aerovaldb/aerovaldb.py b/src/aerovaldb/aerovaldb.py index d304298..32f2118 100644 --- a/src/aerovaldb/aerovaldb.py +++ b/src/aerovaldb/aerovaldb.py @@ -450,7 +450,7 @@ async def get_experiments( @async_and_sync @put_method(ROUTE_EXPERIMENTS) async def put_experiments(self, obj, project: str, /, *args, **kwargs): - """Stores a list of experiments for a project from the db. + """Stores a list of experiments for a project to the db. :param project: Project ID. """ From e2096e9e65011a5e3b5ea25bcc88eb3e40dc6343 Mon Sep 17 00:00:00 2001 From: thorbjoernl <51087536+thorbjoernl@users.noreply.github.com> Date: Tue, 3 Sep 2024 14:33:06 +0200 Subject: [PATCH 06/20] . --- src/aerovaldb/plugins.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/aerovaldb/plugins.py b/src/aerovaldb/plugins.py index 9aeeebe..57c0661 100644 --- a/src/aerovaldb/plugins.py +++ b/src/aerovaldb/plugins.py @@ -53,7 +53,9 @@ def open(resource, /, use_async: bool = False) -> AerovalDB: - 'entrypoint:path', with entrypoint being the type of database connection (eg. 'json_files' or 'sqlitedb') being the location where the database is located (eg. 'json_files:.') + - 'path', a path to a json_files folder or sqlite file. + - ':memory:' an sqlite in-memory database. Contents are not persistently stored! From d98878fda6a1655d4af4193d2a27663ca3921ef1 Mon Sep 17 00:00:00 2001 From: thorbjoernl <51087536+thorbjoernl@users.noreply.github.com> Date: Tue, 3 Sep 2024 14:38:36 +0200 Subject: [PATCH 07/20] . --- docs/installation.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/installation.rst b/docs/installation.rst index 10201e4..75283c4 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -19,6 +19,7 @@ This will install the most recent development version from Github: From source: ^^^^^^^^^^^^ + :: git clone https://github.com/metno/aerovaldb.git cd aerovaldb From bffd1ca3f17a84e91ba813f6a46a339652753e8e Mon Sep 17 00:00:00 2001 From: thorbjoernl <51087536+thorbjoernl@users.noreply.github.com> Date: Tue, 1 Oct 2024 16:54:59 +0200 Subject: [PATCH 08/20] docs: Hopefully fixes formatting in docstrings --- src/aerovaldb/aerovaldb.py | 121 +++++++++++++++++++------------------ 1 file changed, 61 insertions(+), 60 deletions(-) diff --git a/src/aerovaldb/aerovaldb.py b/src/aerovaldb/aerovaldb.py index 32f2118..0884195 100644 --- a/src/aerovaldb/aerovaldb.py +++ b/src/aerovaldb/aerovaldb.py @@ -133,7 +133,7 @@ async def get_glob_stats( :param default: Default value that will be returned instead of raising FileNotFoundError if not data was found (Will be returned as is and not converted to match access_type). - :returns : The fetched data. + :returns: The fetched data. """ raise NotImplementedError @@ -193,7 +193,7 @@ async def get_heatmap( :param default: Default value that will be returned instead of raising FileNotFoundError if not data was found (Will be returned as is and not converted to match access_type). - :returns : The fetched data. + :returns: The fetched data. """ raise NotImplementedError @@ -224,7 +224,7 @@ async def list_glob_stats( :param project: str :param experiment: str - :return List of URIs. + :return: List of URIs. """ raise NotImplementedError @@ -255,7 +255,7 @@ async def get_contour( :param default: Default value that will be returned instead of raising FileNotFoundError if no data was found (Will be returned as is and not converted to match access_type). - :returns : The fetched data. + :returns: The fetched data. """ raise NotImplementedError @@ -311,9 +311,9 @@ async def get_timeseries( :param access_type: How the data is to be retrieved (See AccessType for details) :param cache: Whether to use cache for this read. :param default: Default value that will be returned instead of raising FileNotFoundError - if no data was found (Will be returned as is and not converted to match access_type). + if no data was found (Will be returned as is and not converted to match access_type). - :returns : The fetched data. + :returns: The fetched data. """ raise NotImplementedError @@ -391,7 +391,7 @@ async def get_timeseries_weekly( :param default: Default value that will be returned instead of raising FileNotFoundError if not data was found (Will be returned as is and not converted to match access_type). - :returns : The fetched data. + :returns: The fetched data. """ raise NotImplementedError @@ -443,7 +443,7 @@ async def get_experiments( :param default: Default value that will be returned instead of raising FileNotFoundError if not data was found (Will be returned as is and not converted to match access_type). - :returns : The fetched data. + :returns: The fetched data. """ raise NotImplementedError @@ -459,8 +459,8 @@ async def put_experiments(self, obj, project: str, /, *args, **kwargs): def rm_experiment_data(self, project: str, experiment: str): """Deletes ALL data associated with an experiment. - :param project : Project ID. - :param experiment : Experiment ID. + :param project: Project ID. + :param experiment: Experiment ID. """ raise NotImplementedError @@ -490,9 +490,9 @@ async def get_config( :param access_type: How the data is to be retrieved (See AccessType for details) :param cache: Whether to use cache for this read. :param default: Default value that will be returned instead of raising FileNotFoundError - if not data was found (Will be returned as is and not converted to match access_type). + if not data was found (Will be returned as is and not converted to match access_type). - :returns : The fetched data. + :returns: The fetched data. """ raise NotImplementedError @@ -531,7 +531,7 @@ async def get_menu( :param default: Default value that will be returned instead of raising FileNotFoundError if not data was found (Will be returned as is and not converted to match access_type). - :returns : The fetched data. + :returns: The fetched data. """ raise NotImplementedError @@ -567,9 +567,9 @@ async def get_statistics( :param access_type: How the data is to be retrieved (See AccessType for details) :param cache: Whether to use cache for this read. :param default: Default value that will be returned instead of raising FileNotFoundError - if not data was found (Will be returned as is and not converted to match access_type). + if not data was found (Will be returned as is and not converted to match access_type). - :returns : The fetched data. + :returns: The fetched data. """ raise NotImplementedError @@ -607,9 +607,9 @@ async def get_ranges( :param access_type: How the data is to be retrieved (See AccessType for details) :param cache: Whether to use cache for this read. :param default: Default value that will be returned instead of raising FileNotFoundError - if not data was found (Will be returned as is and not converted to match access_type). + if not data was found (Will be returned as is and not converted to match access_type). - :returns : The fetched data. + :returns: The fetched data. """ raise NotImplementedError @@ -645,9 +645,9 @@ async def get_regions( :param access_type: How the data is to be retrieved (See AccessType for details) :param cache: Whether to use cache for this read. :param default: Default value that will be returned instead of raising FileNotFoundError - if not data was found (Will be returned as is and not converted to match access_type). + if not data was found (Will be returned as is and not converted to match access_type). - :returns : The fetched data. + :returns: The fetched data. """ raise NotImplementedError @@ -683,9 +683,9 @@ async def get_models_style( :param access_type: How the data is to be retrieved (See AccessType for details) :param cache: Whether to use cache for this read. :param default: Default value that will be returned instead of raising FileNotFoundError - if not data was found (Will be returned as is and not converted to match access_type). + if not data was found (Will be returned as is and not converted to match access_type). - :returns : The fetched data. + :returns: The fetched data. """ raise NotImplementedError @@ -735,9 +735,9 @@ async def get_map( :param access_type: How the data is to be retrieved (See AccessType for details) :param cache: Whether to use cache for this read. :param default: Default value that will be returned instead of raising FileNotFoundError - if not data was found (Will be returned as is and not converted to match access_type). + if not data was found (Will be returned as is and not converted to match access_type). - :returns : The fetched data. + :returns: The fetched data. """ raise NotImplementedError @@ -785,7 +785,7 @@ async def list_map( :param project: The project ID. :param experiment: The experiment ID. - :return List with the URIs. + :returns: List with the URIs. """ raise NotImplementedError @@ -822,9 +822,9 @@ async def get_scatter( :param access_type: How the data is to be retrieved (See AccessType for details) :param cache: Whether to use cache for this read. :param default: Default value that will be returned instead of raising FileNotFoundError - if not data was found (Will be returned as is and not converted to match access_type). + if not data was found (Will be returned as is and not converted to match access_type). - :returns : The fetched data. + :returns: The fetched data. """ raise NotImplementedError @@ -886,9 +886,9 @@ async def get_profiles( :param access_type: How the data is to be retrieved (See AccessType for details) :param cache: Whether to use cache for this read. :param default: Default value that will be returned instead of raising FileNotFoundError - if not data was found (Will be returned as is and not converted to match access_type). + if not data was found (Will be returned as is and not converted to match access_type). - :returns : The fetched data. + :returns: The fetched data. """ raise NotImplementedError @@ -946,9 +946,9 @@ async def get_heatmap_timeseries( :param access_type: How the data is to be retrieved (See AccessType for details) :param cache: Whether to use cache for this read. :param default: Default value that will be returned instead of raising FileNotFoundError - if not data was found (Will be returned as is and not converted to match access_type). + if not data was found (Will be returned as is and not converted to match access_type). - :returns : The fetched data. + :returns: The fetched data. """ raise NotImplementedError @@ -1008,9 +1008,9 @@ async def get_forecast( :param access_type: How the data is to be retrieved (See AccessType for details) :param cache: Whether to use cache for this read. :param default: Default value that will be returned instead of raising FileNotFoundError - if not data was found (Will be returned as is and not converted to match access_type). + if not data was found (Will be returned as is and not converted to match access_type). - :returns : The fetched data. + :returns: The fetched data. """ raise NotImplementedError @@ -1066,9 +1066,9 @@ async def get_gridded_map( :param access_type: How the data is to be retrieved (See AccessType for details) :param cache: Whether to use cache for this read. :param default: Default value that will be returned instead of raising FileNotFoundError - if not data was found (Will be returned as is and not converted to match access_type). + if not data was found (Will be returned as is and not converted to match access_type). - :returns : The fetched data. + :returns: The fetched data. """ raise NotImplementedError @@ -1118,9 +1118,9 @@ async def get_report( :param access_type: How the data is to be retrieved (See AccessType for details) :param cache: Whether to use cache for this read. :param default: Default value that will be returned instead of raising FileNotFoundError - if not data was found (Will be returned as is and not converted to match access_type). + if not data was found (Will be returned as is and not converted to match access_type). - :returns : The fetched data. + :returns: The fetched data. """ raise NotImplementedError @@ -1153,8 +1153,8 @@ async def get_by_uri( :param access_type : See AccessType. :param cache : Whether to use the cache. :param default : If provided, this value will be returned instead of raising - a FileNotFoundError if not file exists. The provided object will be returned - as is, and will not be converted to match access_type. + a FileNotFoundError if not file exists. The provided object will be returned + as is, and will not be converted to match access_type. Note: ----- @@ -1202,7 +1202,8 @@ def _normalize_access_type( :param default: The type to return if access_type is None. Defaults to AccessType.OBJ :raises ValueError: If str access_type can't be converted to AccessType. :raises ValueError: If access_type is not str or AccessType - :return: The normalized AccessType. + + :returns: The normalized AccessType. """ if isinstance(access_type, AccessType): return access_type @@ -1223,9 +1224,9 @@ def _normalize_access_type( async def list_all(self, access_type: str | AccessType = AccessType.URI): """Returns a list of identifiers for each object stored in the database - :param access_type : What to return (This is implementation specific, but + :param access_type: What to return (This is implementation specific, but in general each implementation should support URI, some may support FILE_PATH). - :raises : UnsupportedOperation + :raises UnsupportedOperation: For non-supported acces types. """ raise NotImplementedError @@ -1242,11 +1243,11 @@ async def get_report_image( """ Getter for static images that are referenced from the report json files. - :param project : Project ID. - :param experiment : Experiment ID. - :param access_type : One of AccessType.BLOB, AccessType.FILE_PATH + :param project: Project ID. + :param experiment: Experiment ID. + :param access_type: One of AccessType.BLOB, AccessType.FILE_PATH - :return Either a string (If file path requested) or a bytes object with the + :returns: Either a string (If file path requested) or a bytes object with the image data """ raise NotImplementedError @@ -1257,11 +1258,11 @@ async def put_report_image(self, obj, project: str, experiment: str, path: str): """ Putter for static images that are referenced from the report json files. - :param obj : A bytes object representing the image data to be written. - :param project : Project ID. - :param experiment : Experiment ID. + :param obj: A bytes object representing the image data to be written. + :param project: Project ID. + :param experiment: Experiment ID. - :return Either a string (If file path requested) or a bytes object with the + :returns: Either a string (If file path requested) or a bytes object with the image data """ raise NotImplementedError @@ -1279,11 +1280,11 @@ async def get_map_overlay( ): """Getter for map overlay images. - :param project : Project ID. - :param experiment : Experiment ID. - :param source : Data source. Can be either an observation network or a model ID. - :param variable : Variable name. - :param date : Date. + :param project: Project ID. + :param experiment: Experiment ID. + :param source: Data source. Can be either an observation network or a model ID. + :param variable: Variable name. + :param date: Date. """ raise NotImplementedError @@ -1300,11 +1301,11 @@ async def put_map_overlay( ): """Putter for map overlay images. - :param obj : The object to be stored. - :param project : Project ID. - :param experiment : Experiment ID. - :param source : Data source. Can be either an observation network or a model ID. - :param variable : Variable name. - :param date : Date. + :param obj: The object to be stored. + :param project: Project ID. + :param experiment: Experiment ID. + :param source: Data source. Can be either an observation network or a model ID. + :param variable: Variable name. + :param date: Date. """ raise NotImplementedError From 2ad50abbab55c0199e0b7775473cc69b194eef02 Mon Sep 17 00:00:00 2001 From: thorbjoernl <51087536+thorbjoernl@users.noreply.github.com> Date: Thu, 3 Oct 2024 10:38:01 +0200 Subject: [PATCH 09/20] :returns: -> :return: --- src/aerovaldb/aerovaldb.py | 48 +++++++++++++++++++------------------- src/aerovaldb/plugins.py | 4 ++-- 2 files changed, 26 insertions(+), 26 deletions(-) diff --git a/src/aerovaldb/aerovaldb.py b/src/aerovaldb/aerovaldb.py index 0884195..c3c2cde 100644 --- a/src/aerovaldb/aerovaldb.py +++ b/src/aerovaldb/aerovaldb.py @@ -133,7 +133,7 @@ async def get_glob_stats( :param default: Default value that will be returned instead of raising FileNotFoundError if not data was found (Will be returned as is and not converted to match access_type). - :returns: The fetched data. + :return: The fetched data. """ raise NotImplementedError @@ -193,7 +193,7 @@ async def get_heatmap( :param default: Default value that will be returned instead of raising FileNotFoundError if not data was found (Will be returned as is and not converted to match access_type). - :returns: The fetched data. + :return: The fetched data. """ raise NotImplementedError @@ -255,7 +255,7 @@ async def get_contour( :param default: Default value that will be returned instead of raising FileNotFoundError if no data was found (Will be returned as is and not converted to match access_type). - :returns: The fetched data. + :return: The fetched data. """ raise NotImplementedError @@ -313,7 +313,7 @@ async def get_timeseries( :param default: Default value that will be returned instead of raising FileNotFoundError if no data was found (Will be returned as is and not converted to match access_type). - :returns: The fetched data. + :return: The fetched data. """ raise NotImplementedError @@ -391,7 +391,7 @@ async def get_timeseries_weekly( :param default: Default value that will be returned instead of raising FileNotFoundError if not data was found (Will be returned as is and not converted to match access_type). - :returns: The fetched data. + :return: The fetched data. """ raise NotImplementedError @@ -443,7 +443,7 @@ async def get_experiments( :param default: Default value that will be returned instead of raising FileNotFoundError if not data was found (Will be returned as is and not converted to match access_type). - :returns: The fetched data. + :return: The fetched data. """ raise NotImplementedError @@ -481,7 +481,7 @@ async def get_config( cache: bool = False, default=None, **kwargs, - ): + ) -> int: """Fetches a configuration from the db. :param project: Project ID. @@ -492,7 +492,7 @@ async def get_config( :param default: Default value that will be returned instead of raising FileNotFoundError if not data was found (Will be returned as is and not converted to match access_type). - :returns: The fetched data. + :return: The fetched data. """ raise NotImplementedError @@ -531,7 +531,7 @@ async def get_menu( :param default: Default value that will be returned instead of raising FileNotFoundError if not data was found (Will be returned as is and not converted to match access_type). - :returns: The fetched data. + :return: The fetched data. """ raise NotImplementedError @@ -569,7 +569,7 @@ async def get_statistics( :param default: Default value that will be returned instead of raising FileNotFoundError if not data was found (Will be returned as is and not converted to match access_type). - :returns: The fetched data. + :return: The fetched data. """ raise NotImplementedError @@ -609,7 +609,7 @@ async def get_ranges( :param default: Default value that will be returned instead of raising FileNotFoundError if not data was found (Will be returned as is and not converted to match access_type). - :returns: The fetched data. + :return: The fetched data. """ raise NotImplementedError @@ -647,7 +647,7 @@ async def get_regions( :param default: Default value that will be returned instead of raising FileNotFoundError if not data was found (Will be returned as is and not converted to match access_type). - :returns: The fetched data. + :return: The fetched data. """ raise NotImplementedError @@ -685,7 +685,7 @@ async def get_models_style( :param default: Default value that will be returned instead of raising FileNotFoundError if not data was found (Will be returned as is and not converted to match access_type). - :returns: The fetched data. + :return: The fetched data. """ raise NotImplementedError @@ -737,7 +737,7 @@ async def get_map( :param default: Default value that will be returned instead of raising FileNotFoundError if not data was found (Will be returned as is and not converted to match access_type). - :returns: The fetched data. + :return: The fetched data. """ raise NotImplementedError @@ -785,7 +785,7 @@ async def list_map( :param project: The project ID. :param experiment: The experiment ID. - :returns: List with the URIs. + :return: List with the URIs. """ raise NotImplementedError @@ -824,7 +824,7 @@ async def get_scatter( :param default: Default value that will be returned instead of raising FileNotFoundError if not data was found (Will be returned as is and not converted to match access_type). - :returns: The fetched data. + :return: The fetched data. """ raise NotImplementedError @@ -888,7 +888,7 @@ async def get_profiles( :param default: Default value that will be returned instead of raising FileNotFoundError if not data was found (Will be returned as is and not converted to match access_type). - :returns: The fetched data. + :return: The fetched data. """ raise NotImplementedError @@ -948,7 +948,7 @@ async def get_heatmap_timeseries( :param default: Default value that will be returned instead of raising FileNotFoundError if not data was found (Will be returned as is and not converted to match access_type). - :returns: The fetched data. + :return: The fetched data. """ raise NotImplementedError @@ -1010,7 +1010,7 @@ async def get_forecast( :param default: Default value that will be returned instead of raising FileNotFoundError if not data was found (Will be returned as is and not converted to match access_type). - :returns: The fetched data. + :return: The fetched data. """ raise NotImplementedError @@ -1068,7 +1068,7 @@ async def get_gridded_map( :param default: Default value that will be returned instead of raising FileNotFoundError if not data was found (Will be returned as is and not converted to match access_type). - :returns: The fetched data. + :return: The fetched data. """ raise NotImplementedError @@ -1120,7 +1120,7 @@ async def get_report( :param default: Default value that will be returned instead of raising FileNotFoundError if not data was found (Will be returned as is and not converted to match access_type). - :returns: The fetched data. + :return: The fetched data. """ raise NotImplementedError @@ -1203,7 +1203,7 @@ def _normalize_access_type( :raises ValueError: If str access_type can't be converted to AccessType. :raises ValueError: If access_type is not str or AccessType - :returns: The normalized AccessType. + :return: The normalized AccessType. """ if isinstance(access_type, AccessType): return access_type @@ -1247,7 +1247,7 @@ async def get_report_image( :param experiment: Experiment ID. :param access_type: One of AccessType.BLOB, AccessType.FILE_PATH - :returns: Either a string (If file path requested) or a bytes object with the + :return: Either a string (If file path requested) or a bytes object with the image data """ raise NotImplementedError @@ -1262,7 +1262,7 @@ async def put_report_image(self, obj, project: str, experiment: str, path: str): :param project: Project ID. :param experiment: Experiment ID. - :returns: Either a string (If file path requested) or a bytes object with the + :return: Either a string (If file path requested) or a bytes object with the image data """ raise NotImplementedError diff --git a/src/aerovaldb/plugins.py b/src/aerovaldb/plugins.py index 57c0661..cd6c7c5 100644 --- a/src/aerovaldb/plugins.py +++ b/src/aerovaldb/plugins.py @@ -59,9 +59,9 @@ def open(resource, /, use_async: bool = False) -> AerovalDB: - ':memory:' an sqlite in-memory database. Contents are not persistently stored! - :param use_async : If true, aiofile will be used to read files, otherwise + :param use_async: If true, aiofile will be used to read files, otherwise files will be read synchronously (Currently only used by json_files). - :return : an implementation-object of AerovalDB openend and initialized to a + :return: an implementation-object of AerovalDB openend and initialized to a location. """ if resource == ":memory:": From 9444363131faf7a8ac01d5a2b5415d83fbe5264c Mon Sep 17 00:00:00 2001 From: thorbjoernl <51087536+thorbjoernl@users.noreply.github.com> Date: Fri, 7 Feb 2025 16:31:30 +0100 Subject: [PATCH 10/20] docs: Extending aerovaldb --- .gitignore | 4 +- docs/extending-aerovaldb.rst | 60 ++++++++++++++++++++++++-- setup.cfg | 2 + src/aerovaldb/jsondb/cache.py | 7 +++- src/aerovaldb/jsondb/jsonfiledb.py | 23 ++++++++++ src/aerovaldb/sqlitedb/sqlitedb.py | 20 +++++++++ src/aerovaldb/utils/compat.py | 18 -------- tests/test_aerovaldb.py | 67 +++++------------------------- 8 files changed, 121 insertions(+), 80 deletions(-) delete mode 100644 src/aerovaldb/utils/compat.py diff --git a/.gitignore b/.gitignore index 0ce447a..685ff68 100644 --- a/.gitignore +++ b/.gitignore @@ -162,4 +162,6 @@ cython_debug/ .vscode/ tests/test-db/tmp .mutmut-cache -XX~/ \ No newline at end of file +XX~/ + +playground/ \ No newline at end of file diff --git a/docs/extending-aerovaldb.rst b/docs/extending-aerovaldb.rst index 255fc27..12d30ba 100644 --- a/docs/extending-aerovaldb.rst +++ b/docs/extending-aerovaldb.rst @@ -1,9 +1,63 @@ Extending AerovalDB =================== -AerovalDB is designed to allow custom implementations to allow storage in new storage formats. This is done by writing a new class inheriting from :class:`aerovaldb.AerovalDB`. +AerovalDB is designed to allow custom implementations to allow storage in interchangeable storage backends. This is done by writing a new class inheriting from :class:`aerovaldb.AerovalDB`. Unless overridden, getters and setters will be routed to :meth:`aerovaldb.AerovalDB._get` and :meth:`aerovaldb.AerovalDB._put` respectively, so this is were the main read and write behaviour should be implemented. These functions receive a route, as well as arguments based on which to get/put data and are intended to be used based on route-lookup (for example, :class:`aerovaldb.jsondb.jsonfiledb.AerovalJsonFileDB` translates the route into a file path template, while :class:`aerovaldb.sqlitedb.sqlitedb.AerovalSqliteDB` translates the route into a table name). -URI Scheme ----------- +The following minimal example illustrates the principles of how one would implement these functions. + +.. code-block:: python + + import aerovaldb + from aerovaldb.utils.uri import build_uri, parse_uri + + class InMemoryAerovalDB(aerovaldb.AerovalDB): + def __init__(self): + self._store = {} + + async def _get(self, route: str, route_args: dict, **kwargs): + access_type = self._normalize_access_type(kwargs.pop("access_type", aerovaldb.AccessType.OBJ)) + + if access_type != aerovaldb.AccessType.OBJ: + raise ValueError(f"Unsupported accesstype, '{access_type}'.") + + # Alternatively you can use route as a lookup table. + uri = build_uri(route, route_args, kwargs) + return self._store[uri] + + async def _put(self, obj, route: str, route_args: dict, **kwargs): + uri = build_uri(route, route_args, kwargs) + + self._store[uri] = obj + + + if __name__ == "__main__": + with InMemoryAerovalDB() as db: + db.put_experiments("obj", "test") + print(db.get_experiments("test")) + +This is a minimal implementation and thus lacks certain niceties such as proper exception handling, persistent storage, and support for all access modes, but it serves to illustrate how most functionality in an AerovalDB implementation is implemented. + +In addition some functions must be overridden. This includes all methods that do not access json; that is, map_overlays, report_images; functions that list assets; functions for accessing by uri; as well as functions that require different behaviour for backwards compatibility reasons (eg. contours). To see which functions require to be overriden, I recommend looking a jsonfiledb.py and searching for the '@override' decorator. + +Testing +------- +The tests in tests/test_aerovaldb.py implements tests that all implementations of the AerovalDB interface need to be able to pass. This is to ensure that implementations are in fact interchangeable. It is recommended that you test your implementation against this as soon as possible. + +These tests rely on a test database in the data storage format that is being tested. The canonical version of this database is the one found in tests/test-db/json. This runs into a bootstrapping problem, as the test-db for a new format is most easily created by copying this db, which requires a (somewhat) functional database implementation. + +Here is the recommended way of doing this: + +- Implement get_by_uri and put_by_uri for your new implementation. +- Add your implementation to the tests/utils/test_copy test and verify that it passes. +- Use the built-in copy function to make a version of the test database in your new storage format. +- Add your implementation to the test_aerovaldb tests. For this the following changes need to be made: + - The tmpdb fixture needs to be able to create a guaranteed empty, temporary db instance for your storage format. + - The TESTDB_PARAMETRIZATION needs to be extended with the resource string matching the test-db created above. + - The IMPLEMENTATION_PARAMETRIZATION needs to include the identifier for you implementation, so that it matches the tmpdb identifier. +- `Tweak until all tests are green `` + + + + diff --git a/setup.cfg b/setup.cfg index 3401dde..66faaa1 100644 --- a/setup.cfg +++ b/setup.cfg @@ -28,6 +28,8 @@ install_requires = async_lru packaging filetype + typing-extensions; python_version < 3.12 + package_dir = =src packages = diff --git a/src/aerovaldb/jsondb/cache.py b/src/aerovaldb/jsondb/cache.py index e56d4f2..eb09e33 100644 --- a/src/aerovaldb/jsondb/cache.py +++ b/src/aerovaldb/jsondb/cache.py @@ -1,12 +1,15 @@ import logging import os +import sys from abc import ABC, abstractmethod from collections import defaultdict, deque from pathlib import Path from typing import Hashable, TypedDict -from ..utils.compat import override - +if sys.version_info >= (3, 12): + from typing import override +else: + from typing_extensions import override logger = logging.getLogger(__name__) diff --git a/src/aerovaldb/jsondb/jsonfiledb.py b/src/aerovaldb/jsondb/jsonfiledb.py index dc6b8bb..0f7a085 100644 --- a/src/aerovaldb/jsondb/jsonfiledb.py +++ b/src/aerovaldb/jsondb/jsonfiledb.py @@ -4,6 +4,7 @@ import logging import os import shutil +import sys from hashlib import md5 from pathlib import Path from typing import Any, Awaitable, Callable @@ -33,6 +34,11 @@ from ..utils.string_mapper import StringMapper, VersionConstraintMapper from .cache import CacheMissError, KeyCacheDecorator, LRUFileCache +if sys.version_info >= (3, 12): + from typing import override +else: + from typing_extensions import override + logger = logging.getLogger(__name__) @@ -206,6 +212,7 @@ async def _get_template(self, route: str, substitutions: dict) -> str: """ return await self.PATH_LOOKUP.lookup(route, **substitutions) + @override async def _get( self, route, @@ -282,6 +289,7 @@ async def _get( raise UnsupportedOperation + @override async def _put(self, obj, route, route_args, **kwargs): """Jsondb implemention of database put operation. @@ -306,6 +314,7 @@ async def _put(self, obj, route, route_args, **kwargs): with open(file_path, "w") as f: f.write(json) + @override def rm_experiment_data(self, project: str, experiment: str) -> None: """Deletes ALL data associated with an experiment. @@ -321,6 +330,7 @@ def rm_experiment_data(self, project: str, experiment: str) -> None: shutil.rmtree(exp_dir) @async_and_sync + @override async def get_regional_stats( self, project: str, @@ -352,6 +362,7 @@ async def get_regional_stats( ) @async_and_sync + @override async def get_heatmap( self, project: str, @@ -461,6 +472,7 @@ async def _get_uri_for_file(self, file_path: str) -> str: raise ValueError(f"Unable to build URI for file path {file_path}") @async_and_sync + @override async def list_glob_stats( self, project: str, @@ -497,6 +509,7 @@ async def list_glob_stats( return result @async_and_sync + @override async def list_timeseries( self, project: str, @@ -538,6 +551,7 @@ async def list_timeseries( return result @async_and_sync + @override async def list_map( self, project: str, @@ -581,6 +595,7 @@ async def list_map( return result @async_and_sync + @override async def get_by_uri( self, uri: str, @@ -623,6 +638,7 @@ async def get_by_uri( ) @async_and_sync + @override async def put_by_uri(self, obj, uri: str): route, route_args, kwargs = parse_uri(uri) @@ -660,6 +676,7 @@ def lock(self): return FakeLock() @async_and_sync + @override async def list_all(self, access_type: str | AccessType = AccessType.URI): access_type = self._normalize_access_type(access_type) @@ -685,6 +702,7 @@ async def list_all(self, access_type: str | AccessType = AccessType.URI): return result @async_and_sync + @override async def get_report_image( self, project: str, @@ -732,6 +750,7 @@ async def get_report_image( return f.read() @async_and_sync + @override async def put_report_image(self, obj, project: str, experiment: str, path: str): template = await self._get_template(ROUTE_REPORT_IMAGE, {}) @@ -744,6 +763,7 @@ async def put_report_image(self, obj, project: str, experiment: str, path: str): f.write(obj) @async_and_sync + @override async def get_map_overlay( self, project: str, @@ -801,6 +821,7 @@ async def get_map_overlay( return f.read() @async_and_sync + @override async def put_map_overlay( self, obj, @@ -844,6 +865,7 @@ async def put_map_overlay( f.write(obj) @async_and_sync + @override async def get_contour( self, project: str, @@ -930,6 +952,7 @@ async def get_contour( raise FileNotFoundError @async_and_sync + @override async def put_contour( self, obj, diff --git a/src/aerovaldb/sqlitedb/sqlitedb.py b/src/aerovaldb/sqlitedb/sqlitedb.py index 3446108..43e62dc 100644 --- a/src/aerovaldb/sqlitedb/sqlitedb.py +++ b/src/aerovaldb/sqlitedb/sqlitedb.py @@ -3,6 +3,7 @@ import logging import os import sqlite3 +import sys from hashlib import md5 from typing import Any, Awaitable, Callable @@ -36,6 +37,11 @@ parse_uri, ) +if sys.version_info >= (3, 12): + from typing import override +else: + from typing_extensions import override + logger = logging.getLogger(__name__) @@ -372,6 +378,7 @@ def _get_column_list_and_substitution_list(self, kwargs: dict) -> tuple[str, str return (columnlist, substitutionlist) + @override async def _get(self, route, route_args, **kwargs): cache = kwargs.pop("cache", False) default = kwargs.pop("default", None) @@ -472,6 +479,7 @@ async def _get(self, route, route_args, **kwargs): raise UnsupportedOperation + @override async def _put(self, obj, route, route_args, **kwargs): cur = self._con.cursor() @@ -505,6 +513,7 @@ async def _put(self, obj, route, route_args, **kwargs): self._con.commit() @async_and_sync + @override async def get_by_uri( self, uri: str, @@ -546,6 +555,7 @@ async def get_by_uri( ) @async_and_sync + @override async def put_by_uri(self, obj, uri: str): route, route_args, kwargs = parse_uri(uri) if route == ROUTE_REPORT_IMAGE: @@ -568,6 +578,7 @@ async def put_by_uri(self, obj, uri: str): await self._put(obj, route, route_args, **kwargs) @async_and_sync + @override async def list_all(self): cur = self._con.cursor() result = [] @@ -617,6 +628,7 @@ def lock(self): return FakeLock() + @override def list_glob_stats( self, project: str, @@ -660,6 +672,7 @@ def list_glob_stats( return result @async_and_sync + @override async def list_timeseries( self, project: str, @@ -701,6 +714,7 @@ async def list_timeseries( result.append(uri) return result + @override def rm_experiment_data(self, project: str, experiment: str) -> None: cur = self._con.cursor() for table in [ @@ -735,6 +749,7 @@ def rm_experiment_data(self, project: str, experiment: str) -> None: ) @async_and_sync + @override async def get_report_image( self, project: str, @@ -777,6 +792,7 @@ async def get_report_image( ) @async_and_sync + @override async def put_report_image(self, obj, project: str, experiment: str, path: str): cur = self._con.cursor() @@ -793,6 +809,7 @@ async def put_report_image(self, obj, project: str, experiment: str, path: str): self._con.commit() @async_and_sync + @override async def get_map_overlay( self, project: str, @@ -839,6 +856,7 @@ async def get_map_overlay( ) @async_and_sync + @override async def put_map_overlay( self, obj, @@ -863,6 +881,7 @@ async def put_map_overlay( self._con.commit() @async_and_sync + @override async def get_contour( self, project: str, @@ -920,6 +939,7 @@ async def get_contour( raise FileNotFoundError @async_and_sync + @override async def put_contour( self, obj, diff --git a/src/aerovaldb/utils/compat.py b/src/aerovaldb/utils/compat.py deleted file mode 100644 index cc96c92..0000000 --- a/src/aerovaldb/utils/compat.py +++ /dev/null @@ -1,18 +0,0 @@ -import sys - -if sys.version_info >= (3, 12): - from typing import override - - override = override -else: - - def override(method, /): - """ - A wrapper for the typing.override wrapper, that ensures it does nothing on unsupported - python versions (<3.12). - - Since this wrapper is for the benefit of typecheckers (eg. mypy), we don't want it - to do anything on unsupported versions. - https://docs.python.org/3/library/typing.html#typing.override - """ - return method diff --git a/tests/test_aerovaldb.py b/tests/test_aerovaldb.py index b5fa8f3..0b9ff47 100644 --- a/tests/test_aerovaldb.py +++ b/tests/test_aerovaldb.py @@ -44,6 +44,9 @@ def tmpdb(tmp_path, dbtype: str) -> aerovaldb.AerovalDB: ), ) +IMPLEMENTATION_PARAMETRIZATION = pytest.mark.parametrize( + "dbtype", (pytest.param("json_files"), pytest.param("sqlitedb")) +) GET_PARAMETRIZATION = pytest.mark.parametrize( "fun,args,kwargs,expected", ( @@ -357,9 +360,7 @@ def test_getter_json_str(testdb: str, fun: str, args: list, kwargs: dict, expect @pytest.mark.asyncio -@pytest.mark.parametrize( - "dbtype", (pytest.param("json_files"), pytest.param("sqlitedb")) -) +@IMPLEMENTATION_PARAMETRIZATION @PUT_PARAMETRIZATION async def test_setters(dbtype: str, fun: str, args: list, kwargs: dict, tmpdb): """ @@ -383,9 +384,7 @@ async def test_setters(dbtype: str, fun: str, args: list, kwargs: dict, tmpdb): assert data["data"] == expected -@pytest.mark.parametrize( - "dbtype", (pytest.param("json_files"), pytest.param("sqlitedb")) -) +@IMPLEMENTATION_PARAMETRIZATION @PUT_PARAMETRIZATION def test_setters_sync(fun: str, args: list, kwargs: dict, tmpdb): """ @@ -409,9 +408,7 @@ def test_setters_sync(fun: str, args: list, kwargs: dict, tmpdb): assert data["data"] == expected -@pytest.mark.parametrize( - "dbtype", (pytest.param("json_files"), pytest.param("sqlitedb")) -) +@IMPLEMENTATION_PARAMETRIZATION @PUT_PARAMETRIZATION def test_setters_json_str(fun: str, args: list, kwargs: dict, tmpdb): with tmpdb as db: @@ -431,17 +428,7 @@ def test_setters_json_str(fun: str, args: list, kwargs: dict, tmpdb): assert data["data"] == expected -@pytest.mark.parametrize( - "dbtype", - ( - pytest.param( - "json_files", - ), - pytest.param( - "sqlitedb", - ), - ), -) +@IMPLEMENTATION_PARAMETRIZATION def test_write_and_read_of_nan(tmpdb): with tmpdb as db: data = dict(value=float("nan")) @@ -522,9 +509,7 @@ def test_list_timeseries(testdb): assert len(list(timeseries)) == 1 -@pytest.mark.parametrize( - "dbtype", (pytest.param("json_files"), pytest.param("sqlitedb")) -) +@IMPLEMENTATION_PARAMETRIZATION def test_rm_experiment_data(tmpdb): with aerovaldb.open("json_files:./tests/test-db/json") as db: copy_db_contents(db, tmpdb) @@ -559,17 +544,7 @@ def test_get_report_image(testdb, sub_path: str): assert len(blob) > 0 -@pytest.mark.parametrize( - "dbtype", - ( - pytest.param( - "json_files", - ), - pytest.param( - "sqlitedb", - ), - ), -) +@IMPLEMENTATION_PARAMETRIZATION def test_put_report_image(tmpdb): with open("tests/test-db/json/reports/project/experiment/img/pixel.png", "rb") as f: data = f.read() @@ -583,17 +558,7 @@ def test_put_report_image(tmpdb): assert len(blob) > 0 -@pytest.mark.parametrize( - "dbtype", - ( - pytest.param( - "json_files", - ), - pytest.param( - "sqlitedb", - ), - ), -) +@IMPLEMENTATION_PARAMETRIZATION def test_serialize_set(tmpdb): with tmpdb as db: db.put_config({"set": {"a", "b", "c"}}, "test", "test") @@ -643,17 +608,7 @@ def test_get_map_overlay(testdb): assert filetype.guess_extension(data) == "png" -@pytest.mark.parametrize( - "dbtype", - ( - pytest.param( - "json_files", - ), - pytest.param( - "sqlitedb", - ), - ), -) +@IMPLEMENTATION_PARAMETRIZATION @pytest.mark.parametrize( "expected_extension", ( From 0b47799f1a2f925b36bd5c8cb64541a6314a315c Mon Sep 17 00:00:00 2001 From: thorbjoernl <51087536+thorbjoernl@users.noreply.github.com> Date: Fri, 7 Feb 2025 16:32:43 +0100 Subject: [PATCH 11/20] fix: Quotes --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 66faaa1..0ec209b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -28,7 +28,7 @@ install_requires = async_lru packaging filetype - typing-extensions; python_version < 3.12 + typing-extensions; python_version < "3.12" package_dir = =src From d697db03bedfa4e8af77df72512626d166123676 Mon Sep 17 00:00:00 2001 From: thorbjoernl <51087536+thorbjoernl@users.noreply.github.com> Date: Fri, 7 Feb 2025 16:38:04 +0100 Subject: [PATCH 12/20] Fix --- docs/extending-aerovaldb.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/extending-aerovaldb.rst b/docs/extending-aerovaldb.rst index 12d30ba..3cd0ffe 100644 --- a/docs/extending-aerovaldb.rst +++ b/docs/extending-aerovaldb.rst @@ -56,7 +56,7 @@ Here is the recommended way of doing this: - The tmpdb fixture needs to be able to create a guaranteed empty, temporary db instance for your storage format. - The TESTDB_PARAMETRIZATION needs to be extended with the resource string matching the test-db created above. - The IMPLEMENTATION_PARAMETRIZATION needs to include the identifier for you implementation, so that it matches the tmpdb identifier. -- `Tweak until all tests are green `` +- Tweak until all tests are green. From 507da2ea00a2341c4fddd2ac1aad6d859d968f2b Mon Sep 17 00:00:00 2001 From: thorbjoernl <51087536+thorbjoernl@users.noreply.github.com> Date: Mon, 10 Feb 2025 09:51:10 +0100 Subject: [PATCH 13/20] fix: Formatting --- docs/installation.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/installation.rst b/docs/installation.rst index 75283c4..4f2d7f7 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -20,7 +20,8 @@ This will install the most recent development version from Github: From source: ^^^^^^^^^^^^ -:: +.. code-block:: bash + git clone https://github.com/metno/aerovaldb.git cd aerovaldb python -m pip install . From ca35e732af227bf130deeb58174777d7e0ae2c48 Mon Sep 17 00:00:00 2001 From: thorbjoernl <51087536+thorbjoernl@users.noreply.github.com> Date: Mon, 10 Feb 2025 09:53:43 +0100 Subject: [PATCH 14/20] fix: Locking dir --- docs/locking.rst | 4 ++-- src/aerovaldb/jsondb/jsonfiledb.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/locking.rst b/docs/locking.rst index f52d2e1..a1b0929 100644 --- a/docs/locking.rst +++ b/docs/locking.rst @@ -3,9 +3,9 @@ Locking To ensure consistent writes, aerovaldb provides a locking mechanism which can be used to coordinate writes between multiple instances of aerovaldb. The lock applies on the entire database, not per-file. -For :class:`AerovalJsonFileDB` the locking mechanism uses a folder of lock files (`~/.aerovaldb/` by default) to coordinate the lock. It is important that the file system where the lock files are stored supports `fcntl `. +For :class:`AerovalJsonFileDB` the locking mechanism uses a folder of lock files (:code:`~/.aerovaldb/lock` by default) to coordinate the lock. It is important that the file system where the lock files are stored supports `fcntl `. -By default locking is disabled as it may impact performance. To enable, set the environment variable `AVDB_USE_LOCKING=1`. +By default locking is disabled as it may impact performance. To enable, set the environment variable :code:`AVDB_USE_LOCKING=1`. Overriding the lock-file directory ---------------------------------- diff --git a/src/aerovaldb/jsondb/jsonfiledb.py b/src/aerovaldb/jsondb/jsonfiledb.py index 0f7a085..0e8a7ca 100644 --- a/src/aerovaldb/jsondb/jsonfiledb.py +++ b/src/aerovaldb/jsondb/jsonfiledb.py @@ -662,9 +662,9 @@ async def put_by_uri(self, obj, uri: str): await self._put(obj, route, route_args, **kwargs) def _get_lock_file(self) -> str: - os.makedirs(os.path.expanduser("~/.aerovaldb/.lock/"), exist_ok=True) + os.makedirs(os.path.expanduser("~/.aerovaldb/lock/"), exist_ok=True) lock_file = os.path.join( - os.environ.get("AVDB_LOCK_DIR", os.path.expanduser("~/.aerovaldb/.lock/")), + os.environ.get("AVDB_LOCK_DIR", os.path.expanduser("~/.aerovaldb/lock/")), md5(self._basedir.encode()).hexdigest(), ) return lock_file From af684ae14bef66512ae5d4574ab9cc9ba7a651b6 Mon Sep 17 00:00:00 2001 From: thorbjoernl <51087536+thorbjoernl@users.noreply.github.com> Date: Mon, 10 Feb 2025 09:55:52 +0100 Subject: [PATCH 15/20] fix: Minor formatting issues --- src/aerovaldb/aerovaldb.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/aerovaldb/aerovaldb.py b/src/aerovaldb/aerovaldb.py index fbcbed8..c7992d6 100644 --- a/src/aerovaldb/aerovaldb.py +++ b/src/aerovaldb/aerovaldb.py @@ -496,7 +496,7 @@ async def get_config( :param access_type: How the data is to be retrieved (See AccessType for details) :param cache: Whether to use cache for this read. :param default: Default value that will be returned instead of raising FileNotFoundError - if not data was found (Will be returned as is and not converted to match access_type). + if no data was found (Will be returned as is and not converted to match access_type). :return: The fetched data. """ @@ -1163,10 +1163,10 @@ async def get_by_uri( ): """Gets a stored object by its URI. - :param uri : URI of the item to fetch. - :param access_type : See AccessType. - :param cache : Whether to use the cache. - :param default : If provided, this value will be returned instead of raising + :param uri: URI of the item to fetch. + :param access_type: See AccessType. + :param cache: Whether to use the cache. + :param default: If provided, this value will be returned instead of raising a FileNotFoundError if not file exists. The provided object will be returned as is, and will not be converted to match access_type. From 0a2f12bdc8f103115698ac9917d9be242a05f474 Mon Sep 17 00:00:00 2001 From: thorbjoernl <51087536+thorbjoernl@users.noreply.github.com> Date: Mon, 10 Feb 2025 10:02:30 +0100 Subject: [PATCH 16/20] fix: return->returns --- src/aerovaldb/aerovaldb.py | 56 +++++++++++++++++++------------------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/src/aerovaldb/aerovaldb.py b/src/aerovaldb/aerovaldb.py index c7992d6..92bb2ca 100644 --- a/src/aerovaldb/aerovaldb.py +++ b/src/aerovaldb/aerovaldb.py @@ -13,7 +13,7 @@ def get_method(route): :param route: the route for this method :param wrapped: function template, the wrapper function will never be called - :return: getter function + :returns: getter function """ def wrap(wrapped): @@ -47,7 +47,7 @@ def put_method(route): :param route: the route for this method :param wrapped: function template, the function will never be called - :return: putter function + :returns: putter function """ def wrap(wrapped): @@ -135,7 +135,7 @@ async def get_glob_stats( :param default: Default value that will be returned instead of raising FileNotFoundError if not data was found (Will be returned as is and not converted to match access_type). - :return: The fetched data. + :returns: The fetched data. """ raise NotImplementedError @@ -195,7 +195,7 @@ async def get_heatmap( :param default: Default value that will be returned instead of raising FileNotFoundError if not data was found (Will be returned as is and not converted to match access_type). - :return: The fetched data. + :returns: The fetched data. """ raise NotImplementedError @@ -226,7 +226,7 @@ async def list_glob_stats( :param project: str :param experiment: str - :return: List of URIs. + :returns: List of URIs. """ raise NotImplementedError @@ -259,7 +259,7 @@ async def get_contour( :param default: Default value that will be returned instead of raising FileNotFoundError if no data was found (Will be returned as is and not converted to match access_type). - :return: The fetched data. + :returns: The fetched data. """ raise NotImplementedError @@ -319,7 +319,7 @@ async def get_timeseries( :param default: Default value that will be returned instead of raising FileNotFoundError if no data was found (Will be returned as is and not converted to match access_type). - :return: The fetched data. + :returns: The fetched data. """ raise NotImplementedError @@ -361,8 +361,8 @@ async def list_timeseries( """Returns a list of URIs of all timeseries files for a given project and experiment id. - :param project : Project ID. - :param experiment : Experiment ID. + :param project: Project ID. + :param experiment: Experiment ID. """ raise NotImplementedError @@ -397,7 +397,7 @@ async def get_timeseries_weekly( :param default: Default value that will be returned instead of raising FileNotFoundError if not data was found (Will be returned as is and not converted to match access_type). - :return: The fetched data. + :returns: The fetched data. """ raise NotImplementedError @@ -449,7 +449,7 @@ async def get_experiments( :param default: Default value that will be returned instead of raising FileNotFoundError if not data was found (Will be returned as is and not converted to match access_type). - :return: The fetched data. + :returns: The fetched data. """ raise NotImplementedError @@ -498,7 +498,7 @@ async def get_config( :param default: Default value that will be returned instead of raising FileNotFoundError if no data was found (Will be returned as is and not converted to match access_type). - :return: The fetched data. + :returns: The fetched data. """ raise NotImplementedError @@ -537,7 +537,7 @@ async def get_menu( :param default: Default value that will be returned instead of raising FileNotFoundError if not data was found (Will be returned as is and not converted to match access_type). - :return: The fetched data. + :returns: The fetched data. """ raise NotImplementedError @@ -575,7 +575,7 @@ async def get_statistics( :param default: Default value that will be returned instead of raising FileNotFoundError if not data was found (Will be returned as is and not converted to match access_type). - :return: The fetched data. + :returns: The fetched data. """ raise NotImplementedError @@ -615,7 +615,7 @@ async def get_ranges( :param default: Default value that will be returned instead of raising FileNotFoundError if not data was found (Will be returned as is and not converted to match access_type). - :return: The fetched data. + :returns: The fetched data. """ raise NotImplementedError @@ -653,7 +653,7 @@ async def get_regions( :param default: Default value that will be returned instead of raising FileNotFoundError if not data was found (Will be returned as is and not converted to match access_type). - :return: The fetched data. + :returns: The fetched data. """ raise NotImplementedError @@ -691,7 +691,7 @@ async def get_models_style( :param default: Default value that will be returned instead of raising FileNotFoundError if not data was found (Will be returned as is and not converted to match access_type). - :return: The fetched data. + :returns: The fetched data. """ raise NotImplementedError @@ -747,7 +747,7 @@ async def get_map( :param default: Default value that will be returned instead of raising FileNotFoundError if not data was found (Will be returned as is and not converted to match access_type). - :return: The fetched data. + :returns: The fetched data. Note ---- @@ -799,7 +799,7 @@ async def list_map( :param project: The project ID. :param experiment: The experiment ID. - :return: List with the URIs. + :returns: List with the URIs. """ raise NotImplementedError @@ -838,7 +838,7 @@ async def get_scatter( :param default: Default value that will be returned instead of raising FileNotFoundError if not data was found (Will be returned as is and not converted to match access_type). - :return: The fetched data. + :returns: The fetched data. """ raise NotImplementedError @@ -902,7 +902,7 @@ async def get_profiles( :param default: Default value that will be returned instead of raising FileNotFoundError if not data was found (Will be returned as is and not converted to match access_type). - :return: The fetched data. + :returns: The fetched data. """ raise NotImplementedError @@ -962,7 +962,7 @@ async def get_heatmap_timeseries( :param default: Default value that will be returned instead of raising FileNotFoundError if not data was found (Will be returned as is and not converted to match access_type). - :return: The fetched data. + :returns: The fetched data. """ raise NotImplementedError @@ -1024,7 +1024,7 @@ async def get_forecast( :param default: Default value that will be returned instead of raising FileNotFoundError if not data was found (Will be returned as is and not converted to match access_type). - :return: The fetched data. + :returns: The fetched data. """ raise NotImplementedError @@ -1082,7 +1082,7 @@ async def get_gridded_map( :param default: Default value that will be returned instead of raising FileNotFoundError if not data was found (Will be returned as is and not converted to match access_type). - :return: The fetched data. + :returns: The fetched data. """ raise NotImplementedError @@ -1134,7 +1134,7 @@ async def get_report( :param default: Default value that will be returned instead of raising FileNotFoundError if not data was found (Will be returned as is and not converted to match access_type). - :return: The fetched data. + :returns: The fetched data. """ raise NotImplementedError @@ -1217,7 +1217,7 @@ def _normalize_access_type( :raises ValueError: If str access_type can't be converted to AccessType. :raises ValueError: If access_type is not str or AccessType - :return: The normalized AccessType. + :returns: The normalized AccessType. """ if isinstance(access_type, AccessType): return access_type @@ -1261,7 +1261,7 @@ async def get_report_image( :param experiment: Experiment ID. :param access_type: One of AccessType.BLOB, AccessType.FILE_PATH - :return: Either a string (If file path requested) or a bytes object with the + :returns: Either a string (If file path requested) or a bytes object with the image data """ raise NotImplementedError @@ -1276,7 +1276,7 @@ async def put_report_image(self, obj, project: str, experiment: str, path: str): :param project: Project ID. :param experiment: Experiment ID. - :return: Either a string (If file path requested) or a bytes object with the + :returns: Either a string (If file path requested) or a bytes object with the image data """ raise NotImplementedError From 9bf34746a06bc1c40c251a620efd113e90e8e9dd Mon Sep 17 00:00:00 2001 From: thorbjoernl <51087536+thorbjoernl@users.noreply.github.com> Date: Mon, 10 Feb 2025 10:15:13 +0100 Subject: [PATCH 17/20] fix: Links in extending docs --- docs/extending-aerovaldb.rst | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/docs/extending-aerovaldb.rst b/docs/extending-aerovaldb.rst index 3cd0ffe..02e27c1 100644 --- a/docs/extending-aerovaldb.rst +++ b/docs/extending-aerovaldb.rst @@ -3,7 +3,7 @@ Extending AerovalDB AerovalDB is designed to allow custom implementations to allow storage in interchangeable storage backends. This is done by writing a new class inheriting from :class:`aerovaldb.AerovalDB`. -Unless overridden, getters and setters will be routed to :meth:`aerovaldb.AerovalDB._get` and :meth:`aerovaldb.AerovalDB._put` respectively, so this is were the main read and write behaviour should be implemented. These functions receive a route, as well as arguments based on which to get/put data and are intended to be used based on route-lookup (for example, :class:`aerovaldb.jsondb.jsonfiledb.AerovalJsonFileDB` translates the route into a file path template, while :class:`aerovaldb.sqlitedb.sqlitedb.AerovalSqliteDB` translates the route into a table name). +Unless overridden, getters and setters will be routed to :meth:`aerovaldb.AerovalDB._get` and :meth:`aerovaldb.AerovalDB._put` respectively, so this is were the main read and write behaviour should be implemented. Direct overrides of an endpoint should only be used when non-standard behaviour is needed for a specific endpoint. :meth:`aerovaldb.AerovalDB._get` and :meth:`aerovaldb.AerovalDB._put` receive a route (denoting the asset type), as well as arguments based on which to get/put data and are a convenient way for using route-based lookup (for example, :class:`aerovaldb.jsondb.jsonfiledb.AerovalJsonFileDB` translates the route into a file path template, while :class:`aerovaldb.sqlitedb.sqlitedb.AerovalSqliteDB` translates the route into a SQL table name). The following minimal example illustrates the principles of how one would implement these functions. @@ -13,6 +13,9 @@ The following minimal example illustrates the principles of how one would implem from aerovaldb.utils.uri import build_uri, parse_uri class InMemoryAerovalDB(aerovaldb.AerovalDB): + """ + Minimal example of _get and _put in an aerovaldb implementation. + """ def __init__(self): self._store = {} @@ -36,28 +39,25 @@ The following minimal example illustrates the principles of how one would implem with InMemoryAerovalDB() as db: db.put_experiments("obj", "test") print(db.get_experiments("test")) + # prints 'obj' -This is a minimal implementation and thus lacks certain niceties such as proper exception handling, persistent storage, and support for all access modes, but it serves to illustrate how most functionality in an AerovalDB implementation is implemented. +While lacking in certain niceties such as proper exception handling, persistent storage, and support for all access types, it serves as an illustration of how most functionality in an AerovalDB implementation is implemented. -In addition some functions must be overridden. This includes all methods that do not access json; that is, map_overlays, report_images; functions that list assets; functions for accessing by uri; as well as functions that require different behaviour for backwards compatibility reasons (eg. contours). To see which functions require to be overriden, I recommend looking a jsonfiledb.py and searching for the '@override' decorator. +In addition some functions must be overridden. This includes all methods that do not access json; that is, map_overlays and report_images; functions that list assets; functions for accessing by uri; as well as functions that require different behaviour for backwards compatibility reasons (eg. contours). To see which functions may require to be overriden, I recommend looking a :code:`src/aerovaldb/jsondb/jsonfiledb.py` and searching for the :code:`@override`` decorator. Testing ------- -The tests in tests/test_aerovaldb.py implements tests that all implementations of the AerovalDB interface need to be able to pass. This is to ensure that implementations are in fact interchangeable. It is recommended that you test your implementation against this as soon as possible. +The tests in :code:`tests/test_aerovaldb.py` implements tests that all implementations of the AerovalDB interface need to be able to pass. This is to ensure that implementations are in fact interchangeable. It is recommended that you test your implementation against this test-suite as soon as possible. -These tests rely on a test database in the data storage format that is being tested. The canonical version of this database is the one found in tests/test-db/json. This runs into a bootstrapping problem, as the test-db for a new format is most easily created by copying this db, which requires a (somewhat) functional database implementation. +These tests rely on a test database in the data storage format that is being tested. The canonical version of this database is the one found in :code:`tests/test-db/json`. This runs into a bootstrapping problem, as the test-db for a new format is most easily created by copying the canonical database, which requires a (somewhat) functional database implementation. -Here is the recommended way of doing this: +Here is the recommended way of bootstrapping testing for a new implementation: -- Implement get_by_uri and put_by_uri for your new implementation. -- Add your implementation to the tests/utils/test_copy test and verify that it passes. -- Use the built-in copy function to make a version of the test database in your new storage format. -- Add your implementation to the test_aerovaldb tests. For this the following changes need to be made: - - The tmpdb fixture needs to be able to create a guaranteed empty, temporary db instance for your storage format. - - The TESTDB_PARAMETRIZATION needs to be extended with the resource string matching the test-db created above. - - The IMPLEMENTATION_PARAMETRIZATION needs to include the identifier for you implementation, so that it matches the tmpdb identifier. +- Implement :meth:`aerovaldb.AerovalDB.get_by_uri` and :meth:`aerovaldb.AerovalDB.put_by_uri` for your new implementation. +- Add your implementation to the :code:`tests/utils/test_copy.py`` test and verify that it passes. +- Use :meth:`aerovaldb.utils.copy.copy_db_contents` to make a version of the test database in your new storage format. +- Add your implementation to the :code:`tests/test_aerovaldb.py` tests. To do this, the following changes need to be made: + - The :code:`tmpdb` fixture needs to be able to create a guaranteed empty, temporary db instance for your storage format. + - The :code:`TESTDB_PARAMETRIZATION` needs to be extended with the resource string matching the test-db created above. + - The :code:`IMPLEMENTATION_PARAMETRIZATION`` needs to include the identifier for you implementation, so that it matches the tmpdb identifier. - Tweak until all tests are green. - - - - From 999c47edeec835efad995fccd8d6704a01845830 Mon Sep 17 00:00:00 2001 From: thorbjoernl <51087536+thorbjoernl@users.noreply.github.com> Date: Mon, 10 Feb 2025 10:19:40 +0100 Subject: [PATCH 18/20] fix: List formatting --- docs/extending-aerovaldb.rst | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/docs/extending-aerovaldb.rst b/docs/extending-aerovaldb.rst index 02e27c1..aa47b6f 100644 --- a/docs/extending-aerovaldb.rst +++ b/docs/extending-aerovaldb.rst @@ -53,11 +53,13 @@ These tests rely on a test database in the data storage format that is being tes Here is the recommended way of bootstrapping testing for a new implementation: -- Implement :meth:`aerovaldb.AerovalDB.get_by_uri` and :meth:`aerovaldb.AerovalDB.put_by_uri` for your new implementation. -- Add your implementation to the :code:`tests/utils/test_copy.py`` test and verify that it passes. -- Use :meth:`aerovaldb.utils.copy.copy_db_contents` to make a version of the test database in your new storage format. -- Add your implementation to the :code:`tests/test_aerovaldb.py` tests. To do this, the following changes need to be made: - - The :code:`tmpdb` fixture needs to be able to create a guaranteed empty, temporary db instance for your storage format. - - The :code:`TESTDB_PARAMETRIZATION` needs to be extended with the resource string matching the test-db created above. - - The :code:`IMPLEMENTATION_PARAMETRIZATION`` needs to include the identifier for you implementation, so that it matches the tmpdb identifier. -- Tweak until all tests are green. +* Implement :meth:`aerovaldb.AerovalDB.get_by_uri` and :meth:`aerovaldb.AerovalDB.put_by_uri` for your new implementation. +* Add your implementation to the :code:`tests/utils/test_copy.py`` test and verify that it passes. +* Use :meth:`aerovaldb.utils.copy.copy_db_contents` to make a version of the test database in your new storage format. +* Add your implementation to the :code:`tests/test_aerovaldb.py` tests. To do this, the following changes need to be made: + + * The :code:`tmpdb` fixture needs to be able to create a guaranteed empty, temporary db instance for your storage format. + * The :code:`TESTDB_PARAMETRIZATION` needs to be extended with the resource string matching the test-db created above. + * The :code:`IMPLEMENTATION_PARAMETRIZATION`` needs to include the identifier for you implementation, so that it matches the tmpdb identifier. + +* Tweak until all tests are green. From 4059943653d7152c022030b4b3b498be34e26c5b Mon Sep 17 00:00:00 2001 From: thorbjoernl <51087536+thorbjoernl@users.noreply.github.com> Date: Mon, 10 Feb 2025 10:20:22 +0100 Subject: [PATCH 19/20] fix: List formatting --- docs/extending-aerovaldb.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/extending-aerovaldb.rst b/docs/extending-aerovaldb.rst index aa47b6f..0971a7c 100644 --- a/docs/extending-aerovaldb.rst +++ b/docs/extending-aerovaldb.rst @@ -54,12 +54,17 @@ These tests rely on a test database in the data storage format that is being tes Here is the recommended way of bootstrapping testing for a new implementation: * Implement :meth:`aerovaldb.AerovalDB.get_by_uri` and :meth:`aerovaldb.AerovalDB.put_by_uri` for your new implementation. + * Add your implementation to the :code:`tests/utils/test_copy.py`` test and verify that it passes. + * Use :meth:`aerovaldb.utils.copy.copy_db_contents` to make a version of the test database in your new storage format. + * Add your implementation to the :code:`tests/test_aerovaldb.py` tests. To do this, the following changes need to be made: * The :code:`tmpdb` fixture needs to be able to create a guaranteed empty, temporary db instance for your storage format. + * The :code:`TESTDB_PARAMETRIZATION` needs to be extended with the resource string matching the test-db created above. + * The :code:`IMPLEMENTATION_PARAMETRIZATION`` needs to include the identifier for you implementation, so that it matches the tmpdb identifier. * Tweak until all tests are green. From 06a0a5b1ded4c952a5933e55379402944021bd43 Mon Sep 17 00:00:00 2001 From: thorbjoernl <51087536+thorbjoernl@users.noreply.github.com> Date: Mon, 10 Feb 2025 10:23:28 +0100 Subject: [PATCH 20/20] fix: AccessType list formatting --- src/aerovaldb/types.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/src/aerovaldb/types.py b/src/aerovaldb/types.py index 0a0cc95..77a9858 100644 --- a/src/aerovaldb/types.py +++ b/src/aerovaldb/types.py @@ -5,20 +5,21 @@ class AccessType(Enum): """Enumeration of access types. Specifies how data will be read and returned. - JSON_STR: Result will be returned as an unparsed json string. + * JSON_STR: Result will be returned as an unparsed json string. - FILE_PATH: Result will be returned as the file path to the file + * FILE_PATH: Result will be returned as the file path to the file containing the data. - OBJ: The json will be parsed and returned as a python object. + * OBJ: The json will be parsed and returned as a python object. - URI: A string which is a unique identifier of this asset between - implementations of Aerovaldb. Can be used with `get_by_uuid()` and - `put_by_uuid()` to read or write respectively. - (_ROW_ID: For Internal use) - MTIME: The timestamp for last modification for the resource will be + * URI: A string which is a unique identifier of this asset between + implementations of Aerovaldb. Can be used with :meth:`aerovaldb.AerovalDB.get_by_uuid` + and :meth:`aerovaldb.AerovalDB.put_by_uuid` to read or write respectively. + + * MTIME: The timestamp for last modification for the resource will be returned (as datetime.datetime). - CTIME: The creation timestamp for the resource will be returned (as + + * CTIME: The creation timestamp for the resource will be returned (as datetime.datetime) """