Refactor CKAN API scripts: reorganize structure, update logic, and en…

…hance configuration handling

- Add ckanapi documentation as root README.

doc/scripts/ckanapi/.old/.gitignore

@@ -0,0 +1,6 @@
+venv
+env
+sites.yml
+sites.*.yml
+output/*
+!sites.example.yml

doc/scripts/ckanapi/.old/README.md

@@ -0,0 +1,175 @@
+# CKAN API Script
+
+This script allows you to interact with a CKAN instance to perform various actions such as updating packages, exporting groups, and exporting organizations. The configuration for different CKAN instances and actions is specified in a YAML file.
+
+## Prerequisites
+
+- Python 3.x
+
+## Setup
+
+### Creating a Virtual Environment
+
+1. Create a virtual environment:
+
+```sh
+python3 -m venv venv
+```
+
+2. Activate the virtual environment:
+
+- On Linux and macOS:
+
+```sh
+source venv/bin/activate
+```
+
+- On Windows:
+
+```sh
+.\venv\Scripts\activate
+```
+
+### Installing Dependencies
+
+Install the required libraries using pip:
+
+```sh
+pip install -r requirements.txt
+```
+
+## Configuration
+
+The configuration for the CKAN instances and actions is specified in a YAML file located at `./input/sites.yml`. If this file does not exist, you can use the provided `sites.example.yml` as a template.
+
+### Creating `sites.yml`
+
+1. Copy the `sites.example.yml` file to `sites.yml`:
+
+```sh
+cp ./input/sites.example.yml ./input/sites.yml
+```
+
+2. Edit the `sites.yml` file to include your CKAN instance details and the actions you want to perform. Below is an example configuration:
+
+```yaml
+# ./input/sites.yml
+
+default:
+  ckan_site_url: 'https://demo.ckan.dcat-ap-3.es'
+  api_token: 'your_api_token'
+  actions:
+    - update_packages
+    - export_groups
+    - export_organizations
+  override:
+    theme_es: "http://datos.gob.es/kos/sector-publico/sector/medio-ambiente"
+  target_values:
+    publisher_name:
+      condition: "Sample Company"
+      override:
+        publisher_type: "http://purl.org/adms/publishertype/Company"
+
+site_1:
+  ckan_site_url: 'https://site1.ckan.instance'
+  api_token: 'site1_api_token'
+  actions:
+    - update_packages
+    - export_groups
+    - export_organizations
+  override:
+    theme_es: "http://site1.theme.url"
+  target_values:
+    publisher_name:
+      condition: "Site 1 Company"
+      override:
+        publisher_type: "http://site1.publisher.type"
+
+site_2:
+  ckan_site_url: 'https://site2.ckan.instance'
+  api_token: null
+  actions:
+    - export_groups
+    - export_organizations
+
+site_3:
+  ckan_site_url: 'https://site2.ckan.instance'
+  api_token: 'site2_api_token'
+  actions:
+    - export_groups
+    - export_organizations
+    - create_organizations
+    - create_groups
+    - create_users
+  organizations: './input/site_3/organizations.json'
+  groups: './input/site_3/groups.json'
+  users: './input/site_3/users.json'
+
+```
+
+## Usage
+
+To run the script, use the following command:
+
+```sh
+python -m src -s <site>
+```
+
+Replace `<site>` with the site configuration you want to use (e.g., `default`, `site_1`, `site_2`).
+
+### Example
+
+To run the script using the `site_2` configuration:
+
+```sh
+python -m src -s site_2
+```
+
+### Command-line Arguments
+
+- `-s` or `--site`: The site configuration to load (default: `default`).
+- `-c` or `--config`: The path to the YAML configuration file (default: `./input/sites.yml`).
+
+### Notes
+
+- Actions that modify data (e.g., `update_packages`, `create`, `delete`) require a non-null `api_token`.
+- If only `GET` actions are specified (e.g., `export_groups`, `export_organizations`), the `api_token`, `override`, and `target_values` are not required.
+
+## Script Details
+
+The script performs the following actions based on the configuration:
+
+1. **Update Packages**: Updates the packages in the CKAN instance based on the specified overrides and target values.
+2. **Export Groups**: Exports all groups from the CKAN instance to a JSON file (`groups.json`).
+3. **Export Organizations**: Exports all organizations from the CKAN instance to a JSON file (`organizations.json`).
+
+### Functions
+
+#### Connect
+
+- `connect_to_ckan(ckan_site_url, api_token)`: Connects to the CKAN instance.
+
+#### Create
+
+- `create_organizations(rc, json_path)`: Creates new organizations in CKAN from a JSON file.
+- `create_groups(rc, json_path)`: Creates new groups in CKAN from a JSON file.
+- `create_users(rc, json_path)`: Creates new users in CKAN from a JSON file.
+
+#### Update
+
+- `update_package(rc, package_id, theme_es, publisher_name, publisher_type)`: Updates a package given its ID.
+
+#### Get
+
+- `export_groups_to_json(rc, file_path)`: Exports all groups to a JSON file.
+- `export_organizations_to_json(rc, file_path)`: Exports all organizations to a JSON file.
+
+
+### Example Output
+
+- `groups.json`: Contains detailed information about all groups.
+- `organizations.json`: Contains detailed information about all organizations.
+
+## License
+
+This project is licensed under the MIT License.

doc/scripts/ckanapi/src/config.py → doc/scripts/ckanapi/.old/src/config.py

@@ -17,6 +17,7 @@ class SampleConfig:
         groups (str): Path to the JSON file containing group data.
         users (str): Path to the JSON file containing user data.
         override (dict): Dictionary of override values.
+        delete (list): List of keys to delete.
         target_values (dict): Dictionary of target values.
     """
     def __init__(self, config):
@@ -27,7 +28,14 @@ def __init__(self, config):
         self.groups = config.get('groups')
         self.users = config.get('users')
         self.override = config.get('override', {})
+        self.delete = config.get('delete', [])
         self.target_values = config.get('target_values', {})
+        target_site_config = config.get('target_site', {})
+        self.target_ckan_site_url = target_site_config.get('ckan_site_url')
+        self.target_api_token = target_site_config.get('api_token')
+        self.target_organization_id = target_site_config.get('target_organization_id')
+        self.delete_all_before_copy = target_site_config.get('delete_all_before_copy', False)  # Nueva opción
+
 
 def load_config(config_file, site='default'):
     """
@@ -72,6 +80,9 @@ def parse_args():
     print(f"Actions: {config.actions}")
     if config.override:
         print(f"Override: {config.override}")
+    #FIXME: Not working, duplicate datasets.
+    # if config.delete:
+    #     print(f"Delete fields: {config.delete}")
     if config.target_values:
         print(f"Target Values: {config.target_values}")
     if config.organizations:

doc/scripts/ckanapi/src/logic/__init__.py → doc/scripts/ckanapi/.old/src/logic/__init__.py

@@ -39,12 +39,10 @@ def execute_action(action_name, rc, config, output_dir):
 
             # Iterate over each package and update it
             for package_id in package_list:
-                actions[action_name](rc, package_id, config.override, config.target_values)
+                actions[action_name](rc, package_id, config.override, config.delete, config.target_values)
         elif action_name == 'create_organizations':
             actions[action_name](rc, config.organizations)
         elif action_name == 'create_groups':
             actions[action_name](rc, config.groups)
         elif action_name == 'create_users':
-            actions[action_name](rc, config.users)
-        else:
-            actions[action_name](rc, output_dir)
+            actions[action_name](rc, config.users)

doc/scripts/ckanapi/src/logic/create.py → doc/scripts/ckanapi/.old/src/logic/create.py

@@ -1,5 +1,7 @@
 import json
 import os
+from ckanapi import RemoteCKAN, NotFound, ValidationError
+import uuid
 
 def load_json(file_path):
     """

doc/scripts/ckanapi/src/logic/update.py → doc/scripts/ckanapi/.old/src/logic/update.py

@@ -31,16 +31,23 @@ def apply_target_values(package, target_values):
             package = apply_overrides(package, overrides)
     return package
 
-def update_package(rc, package_id, overrides, target_values):
+def update_package(rc, package_id, overrides, deletes, target_values=None):
     """
     Update a package given its ID.
 
     Args:
         rc (RemoteCKAN): The CKAN instance connection.
         package_id (str): The ID of the package.
         overrides (dict): The dictionary of fields to override.
-        target_values (dict): The dictionary of conditions and overrides.
+        deletes (list): The list of keys to delete.
+        target_values (dict, optional): The dictionary of conditions and overrides. Defaults to {}.
+
+    Returns:
+        None
     """
+    if target_values is None:
+        target_values = {}
+
     package = rc.action.package_show(id=package_id)
 
     # Apply overrides

doc/scripts/ckanapi/src/quick_ckanapi.py → doc/scripts/ckanapi/.old/src/quick_ckanapi.py

@@ -18,15 +18,15 @@ def connect_to_ckan(ckan_site_url, api_token):
 
 def is_modifying_action(action_name):
     """
-    Determine if an action modifies data based on its name.
+    Determina si una acción modifica datos basándose en su nombre.
 
     Args:
-        action_name (str): The name of the action.
+        action_name (str): El nombre de la acción.
 
     Returns:
-        bool: True if the action modifies data, False otherwise.
+        bool: True si la acción modifica datos, False de lo contrario.
     """
-    modifying_prefixes = ('update_', 'create_', 'delete_')
+    modifying_prefixes = ('update_', 'create_', 'delete_', 'copy_')
     return action_name.startswith(modifying_prefixes)
 
 def main(config):
@@ -36,9 +36,6 @@ def main(config):
     Args:
         config (SampleConfig): The configuration object.
     """
-    # Connect to the CKAN instance
-    rc = connect_to_ckan(config.ckan_site_url, config.api_token)
-
     # Create output directories if they don't exist
     output_dir = os.path.join('output', config.site)
     os.makedirs(output_dir, exist_ok=True)
@@ -51,7 +48,13 @@ def main(config):
 
     # Execute actions based on the configuration
     for action in config.actions:
-        execute_action(action, rc, config, output_dir)
+        if action == "copy_datasets":
+            # For copy_datasets, connections are handled within the function
+            execute_action(action, None, config, output_dir)
+        else:
+            # Connect to the CKAN instance
+            rc = connect_to_ckan(config.ckan_site_url, config.api_token)
+            execute_action(action, rc, config, output_dir)
 
     print("Actions completed.")