Merged updates from develop into 3.0.1 release branch

.pre-commit-config.yaml

@@ -6,15 +6,15 @@ repos:
       - id: debug-statements
   - repo: https://github.com/astral-sh/ruff-pre-commit
     # Ruff version.
-    rev: v0.5.2
+    rev: v0.6.2
     hooks:
       # Run the linter.
       - id: ruff
         args: [--fix, --exit-non-zero-on-fix]
       # Run the formatter.
       - id: ruff-format
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.10.1
+    rev: v1.11.2
     hooks:
       - id: mypy
         additional_dependencies:

docs/data_managers/command_line_tools/command_line_usage/safedata_zenodo_sync_local_dir.txt

@@ -1,6 +1,6 @@
 cl_prompt $ safedata_zenodo sync_local_dir -h
 usage: safedata_zenodo sync_local_dir [-h] [--not-just-xlsx]
-                                      [--replace-modified]
+                                      [--replace-modified] [--dry-run]
                                       datadir
 
 Synchronize a local data directory
@@ -31,3 +31,5 @@ options:
   --replace-modified
                 Should locally modified files be overwritten with the archive
                 version
+  --dry-run     Run the synchronisation process without altering the local
+                directory.

docs/data_managers/using_safedata/publish_script.py

@@ -1,11 +1,16 @@
-"""Python script to publish a dataset using safedata_validator."""
+"""Python script to publish a dataset using safedata_validator.
+
+Note that this is essentially just a slimmed down version of the publish_dataset
+function underlying the `safedata_zenodo publish_dataset` command line tool.
+"""
 
 from pathlib import Path
 
 import simplejson
 
 from safedata_validator.resources import Resources
 from safedata_validator.zenodo import (
+    ZenodoResources,
     create_deposit,
     generate_inspire_xml,
     publish_deposit,
@@ -19,18 +24,24 @@
 extra_file = "Supplementary_files.zip"
 xml_file = "Example_GEMINI.xml"
 
-# Create a Resources object from a configuration file in a standard location
+# Create a Resources object from a configuration file in a standard location and convert
+# to the Zenodo specific resource class
 resources = Resources()
+zenodo_resources = ZenodoResources(resources)
 
 # Extract the validated dataset metadata
 with open(metadata_path) as md_json:
     data_metadata = simplejson.load(md_json)
 
 # Create the new deposit to publish the dataset
-zenodo_metadata, error = create_deposit(resources=resources)
+create_response = create_deposit(zen_res=zenodo_resources)
+
+# Bail if unsuccessful
+if not create_response.ok:
+    raise RuntimeError(create_response.error_message)
 
-# Monitor the success of individual steps
-all_good = error is None
+# Extract the Zenodo metadata from the response
+zenodo_metadata = create_response.json_data
 
 # Generate XML
 xml_content = generate_inspire_xml(
@@ -41,25 +52,26 @@
 
 # Post the files
 files = [Path(f) for f in (dataset, extra_file, xml_file)]
-if all_good:
-    file_upload_response, error = upload_files(
-        metadata=zenodo_metadata, filepaths=files, resources=resources
-    )
-    all_good = error is None
+file_upload_response = upload_files(
+    zenodo=zenodo_metadata, filepaths=files, zen_res=zenodo_resources
+)
+
+if not file_upload_response.ok:
+    raise RuntimeError(file_upload_response.error_message)
 
 # Post the metadata
-if all_good:
-    md_upload_response, error = upload_metadata(
-        metadata=data_metadata, zenodo=zenodo_metadata, resources=resources
-    )
-    all_good = error is None
+md_upload_response = upload_metadata(
+    metadata=data_metadata, zenodo=zenodo_metadata, zen_res=zenodo_resources
+)
+
+if not md_upload_response.ok:
+    raise RuntimeError(md_upload_response.error_message)
 
 # Publish the deposit
-if all_good:
-    publish_response, error = publish_deposit(
-        zenodo=zenodo_metadata, resources=resources
-    )
-    all_good = error is None
+publish_response = publish_deposit(zenodo=zenodo_metadata, zen_res=zenodo_resources)
+
+if not publish_response.ok:
+    raise RuntimeError(publish_response.error_message)
 
 # Show the new publication
-publish_response["links"]["html"]
+print(publish_response.json_data["links"]["html"])

docs/data_managers/using_safedata/publishing_datasets.md

@@ -16,9 +16,11 @@ optional.
 ## Validating and publishing as a new dataset
 
 The [`safedata_zenodo publish_dataset`
-command](../command_line_tools/safedata_zenodo.md#the-publish_dataset-subcommand) is
-the main function for publishing a dataset. The example below shows it being used to
-publish a dataset and additional external files:
+command](../command_line_tools/safedata_zenodo.md#the-publish_dataset-subcommand) is the
+main function for publishing a dataset. It is important to note that the dataset
+metadata file must be provided first and the dataset file provided second, otherwise the
+publication process will fail. The example below shows it being used to publish a
+dataset and additional external files:
 
 ```sh
 safedata_zenodo publish_dataset Example.json Example.xlsx \

docs/data_managers/using_safedata/update_script.py

@@ -5,8 +5,8 @@
 import simplejson
 
 from safedata_validator.resources import Resources
-from safedata_validator.server import post_metadata
 from safedata_validator.zenodo import (
+    ZenodoResources,
     create_deposit,
     delete_files,
     generate_inspire_xml,
@@ -21,22 +21,25 @@
 extra_file = "Supplementary_files.zip"
 xml_file = "Example_GEMINI.xml"
 
-# Create a Resources object from a configuration file in a standard location
+# Create a Resources object from a configuration file in a standard location and then
+# get the Zenodo specific resources from that
 resources = Resources()
+zenodo_resources = ZenodoResources(resources=resources)
 
 # Extract the validated dataset metadata
 with open(metadata_path) as md_json:
     data_metadata = simplejson.load(md_json)
 
 # Create a new version of an existing dataset using the record ID of the most recent
 # version
-zenodo_metadata, error = create_deposit(
-    new_version=1143714,
-    resources=resources,
-)
+create_response = create_deposit(new_version=1143714, zen_res=zenodo_resources)
+
+# Bail if unsuccessful
+if not create_response.ok:
+    raise RuntimeError(create_response.error_message)
 
-# Monitor the success of individual steps
-all_good = error is None
+# Extract the Zenodo metadata from the response
+zenodo_metadata = create_response.json_data
 
 # Generate XML
 xml_content = generate_inspire_xml(
@@ -49,37 +52,37 @@
 # them all before uploading the provided versions. Note that the publish_dataset
 # function does this in a much more sophisticated way.
 existing_online_files = [p["key"] for p in zenodo_metadata["files"]]
-if all_good:
-    file_delete_response, error = delete_files(
-        metadata=zenodo_metadata, filenames=existing_online_files, resources=resources
-    )
-    all_good = error is None
+file_delete_response = delete_files(
+    metadata=zenodo_metadata, filenames=existing_online_files, zen_res=zenodo_resources
+)
 
+# Bail if unsuccessful
+if not file_delete_response.ok:
+    raise RuntimeError(file_delete_response.error_message)
 
 # Post the files
 files = [Path(f) for f in (dataset, extra_file, xml_file)]
-if all_good:
-    file_upload_response, error = upload_files(
-        metadata=zenodo_metadata, filepaths=files, resources=resources
-    )
-    all_good = error is None
+file_upload_response = upload_files(
+    zenodo=zenodo_metadata, filepaths=files, zen_res=zenodo_resources
+)
+
+# Bail if unsuccessful
+if not file_upload_response.ok:
+    raise RuntimeError(file_upload_response.error_message)
 
 # Post the metadata
-if all_good:
-    md_upload_response, error = upload_metadata(
-        metadata=data_metadata, zenodo=zenodo_metadata, resources=resources
-    )
-    all_good = error is None
+md_upload_response = upload_metadata(
+    metadata=data_metadata, zenodo=zenodo_metadata, zen_res=zenodo_resources
+)
+
+if not md_upload_response.ok:
+    raise RuntimeError(file_upload_response.error_message)
 
 # Publish the deposit
-if all_good:
-    publish_response, error = publish_deposit(
-        zenodo=zenodo_metadata, resources=resources
-    )
-    all_good = error is None
-
-# Post the dataset metadata to the safedata server
-if all_good:
-    response, error = post_metadata(
-        zenodo=zenodo_metadata, metadata=data_metadata, resources=resources
-    )
+publish_response = publish_deposit(zenodo=zenodo_metadata, zen_res=zenodo_resources)
+
+if not publish_response.ok:
+    raise RuntimeError(publish_response.error_message)
+
+# Show the new publication
+print(publish_response.json_data["links"]["html"])

docs/developers/api/overview.md

@@ -29,3 +29,7 @@ components of the validation process.
   taxonomy databases and build usable local SQLite3 databases from them.
 * The [`zenodo` module](./zenodo.md) is used to communicate with the Zenodo in order to
   create and upload Zenodo deposits.
+* The [`server` module](./server.md) is used to communicate with the metadata server in
+  order to upload dataset metadata.
+* The [`utilities` module](./utilities.md) contains helper functions used across the
+  other modules

docs/developers/api/server.md

@@ -7,5 +7,7 @@
         members:
             - post_metadata
             - update_resources
+            - MetadataResponse
+            - MetadataResources
         show_root_heading: false
         show_root_toc_entry: false

docs/developers/api/utilities.md

@@ -0,0 +1,8 @@
+# The `utilities` submodule
+
+::: safedata_validator.utilities
+    options:
+        show_source: true
+        group_by_category: false
+        show_root_heading: false
+        show_root_toc_entry: false

docs/developers/api/zenodo.md

@@ -19,6 +19,7 @@
             - download_ris_data
             - sync_local_dir
             - taxon_index_to_text
-            - ZenodoFunctionResponseType
+            - ZenodoResponse
+            - ZenodoResources
         show_root_heading: false
         show_root_toc_entry: false

mkdocs.yml

@@ -69,6 +69,7 @@ nav:
       - 'Taxa': 'developers/api/taxa.md'
       - 'Zenodo': 'developers/api/zenodo.md'
       - 'Server': 'developers/api/server.md'
+      - 'Utilities': 'developers/api/utilities.md'
     - 'Package development': developers/package_development.md
     - 'Strange problems': developers/strange_problems.md
   - 'safedata at the SAFE project': 'safedata_at_SAFE.md'