Release version 1.8.8

docs/administration/configuring-yoda.md

@@ -145,11 +145,18 @@ yoda_davrods_anonymous_fqdn  | Yoda Davrods anonymous WebDAV fully qualified dom
 yoda_davrods_logo_path       | Path of the DavRODS logo on the portal. Defaults to the themed logo.
 yoda_davrods_logo_link       | URL that the DavRODS logo is linked to (default:  https://www.uu.nl)
 yoda_enable_httpd            | Whether to enable the httpd service (boolean, default value: true). Set to false if manual actions are needed before starting the web server (e.g. mounting encrypted volumes)
-httpd_log_forwarded_for      | Whether to log X-Forwarded-For headers in Apache logs (boolean, default value: false). This logs source IP addresses of requests if requests to the Yoda web portal and/or WebDAV interface are routed via a load balancer.
-httpd_log_user_agent         | Whether to log the user agent of browsers and WebDAV clients in the Apache logs (boolean, default value: false)
 tcp_keepalive_time           | IPv4 TCP keepalives: time until first keepalive (kernel parameter). Can be useful to tune in order to prevent timeouts on long transfers.
 tcp_keepalive_intvl          | IPv4 TCP keepalives: time between keepalives (kernel parameter). Can be useful to tune in order to prevent timeouts on long transfers.
 
+### Generic logging configuration
+
+Variable                           | Description
+-----------------------------------|---------------------------------------------
+httpd_log_forwarded_for            | Whether to log X-Forwarded-For headers in Apache logs (boolean, default value: false). This logs source IP addresses of requests if requests to the Yoda web portal and/or WebDAV interface are routed via a load balancer.
+httpd_log_user_agent               | Whether to log the user agent of browsers and WebDAV clients in the Apache logs (boolean, default value: false)
+yoda_portal_log_api_call_duration  | Whether to log duration and parameters of all API calls from the Yoda portal. This is mainly useful for performance testing (boolean, default value: false)
+
+
 ### iRODS configuration
 
 Variable                     | Description
@@ -173,6 +180,9 @@ irods_enable_service                 | Whether to enable the iRODS service. Set
 irods_rum_job_enabled                | Whether to enable the daily RUM job for removing unused metadata entries (default: true)
 irods_rum_job_hour                   | Time to run RUM job - hour (default: 20)
 irods_rum_job_minute                 | Time to run RUM job - minute (default: 0)
+irods_enable_gocommands              | Whether to install the GoCommands CLI (disabled by default)
+irods_gocommands_version             | GoCommands version
+irods_gocommands_archive_checksum    | MD5 checksum of the GoCommands archive for the version to be installed
 
 ### Research module configuration
 

docs/release-notes/release-1.8.md

@@ -28,8 +28,9 @@ Released: July 2022
 - Added Mailpit for easier mail testing during [development](../development/development-tips.md)
 - DataCite connection uses REST API instead of legacy MDS
 - Several UX improvements to default theme
-- Upgrade iRODS to v4.2.11
-- Upgrade python-irodsclient to v1.1.3
+- Upgrade iRODS to v4.2.11 (Yoda 1.8.8+ has iRODS 4.2.12)
+- Upgrade python-irodsclient to v1.1.3 (Yoda 1.8.8+ has Python-irodsclient 1.1.8)
+- Support for PostgreSQL 15, as well as connection pooling for the iCAT database (in Yoda 1.8.7 and higher)
 - Removed `legacy_tls` flag (legacy TLS support, TLS 1.0 and 1.1)
 
 ### Known issues

environments/development/allinone/group_vars/allinone.yml

@@ -54,6 +54,7 @@ enable_tape_archive: false
 yoda_theme_path: /var/www/yoda/themes              # Base path holding customised portal themes
 yoda_theme: uu                                     # Yoda theme: uu or vu (default: uu)
 portal_title_text: Yoda - Dev
+yoda_portal_log_api_call_duration: true
 
 # iRODS configuration
 irods_password: rods                       # iRODS admin password
@@ -63,6 +64,7 @@ irods_icat_fqdn: combined.yoda.test        # iRODS iCAT fully qualified domain n
 irods_database_fqdn: combined.yoda.test    # iRODS database fully qualified domain name (FQDN)
 irods_resource_fqdn: combined.yoda.test    # iRODS resource fully qualified domain name (FQDN)
 irods_ssl_verify_server: none              # Verify TLS certificate, use 'cert' for acceptance and production
+irods_enable_gocommands: false
 irods_resources:
   - name: dev001_1
     host: "{{ irods_icat_fqdn }}"

environments/development/full/group_vars/full.yml

@@ -54,6 +54,7 @@ enable_tape_archive: false
 yoda_theme_path: /var/www/yoda/themes              # Base path holding customised portal themes
 yoda_theme: uu                                     # Yoda theme: uu or vu (default: uu)
 portal_title_text: Yoda - Dev
+yoda_portal_log_api_call_duration: true
 
 # iRODS configuration
 irods_password: rods                       # iRODS admin password
@@ -63,6 +64,7 @@ irods_icat_fqdn: icat.yoda.test            # iRODS iCAT fully qualified domain n
 irods_database_fqdn: database.yoda.test    # iRODS database fully qualified domain name (FQDN)
 irods_resource_fqdn: resource.yoda.test    # iRODS resource fully qualified domain name (FQDN)
 irods_ssl_verify_server: none              # Verify TLS certificate, use 'cert' for acceptance and production
+irods_enable_gocommands: false
 irods_resources:
   - name: dev001_1
     host: "{{ irods_icat_fqdn }}"

environments/development/surf/group_vars/surf.yml

@@ -54,6 +54,7 @@ enable_tape_archive: false
 yoda_theme_path: /var/www/yoda/themes              # Base path holding customised portal themes
 yoda_theme: uu                                     # Yoda theme: uu or vu (default: uu)
 portal_title_text: Yoda - Surf Config Test
+yoda_portal_log_api_call_duration: true
 
 # iRODS configuration
 irods_password: rods                       # iRODS admin password
@@ -63,6 +64,7 @@ irods_icat_fqdn: portal.surfyoda.test            # iRODS iCAT fully qualified do
 irods_database_fqdn: portal.surfyoda.test    # iRODS database fully qualified domain name (FQDN)
 #irods_resource_fqdn: resource.surfyoda.test    # iRODS resource fully qualified domain name (FQDN)
 irods_ssl_verify_server: none              # Verify TLS certificate, use 'cert' for acceptance and production
+irods_enable_gocommands: false
 irods_resources:
   - name: dev001_1
     host: "{{ yoda_davrods_fqdn }}"

playbook.yml

@@ -129,6 +129,8 @@
       when: enable_postfix
     - role: mailpit
       when: enable_mailpit
+    - role: irods_gocommands
+      when: irods_enable_gocommands
   tags:
     - icat
 
@@ -141,6 +143,8 @@
     - irods_runtime
     - irods_microservices
     - irods_completion
+    - role: irods_gocommands
+      when: irods_enable_gocommands
   tags:
     - resource
 

roles/composable_resources/tasks/main.yml

@@ -15,12 +15,21 @@
   when: not ansible_check_mode
 
 
+- name: Get iRODS home directory
+  become_user: '{{ irods_service_account }}'
+  become: true
+  ansible.builtin.command: echo ~
+  register: irods_directory
+  changed_when: false
+  check_mode: false
+
+
 - name: Ensure iRODS default resource is set in irods_environment.json
   become_user: '{{ irods_service_account }}'
   become: true
   # noqa fqcn[action]
   irods_config:
-    path: '/var/lib/irods/.irods/irods_environment.json'
+    path: '{{ irods_directory.stdout }}/.irods/irods_environment.json'
     key: 'irods_default_resource'
     value: '{{ irods_default_resc }}'
 

roles/irods_gocommands/defaults/main.yml

@@ -0,0 +1,6 @@
+---
+# copyright Utrecht University
+
+irods_enable_gocommands: false
+irods_gocommands_version: 0.7.4
+irods_gocommands_archive_checksum: "md5:7ce94161cb3b4761018ed1eca8606a2d"

roles/irods_gocommands/meta/main.yml

@@ -0,0 +1,11 @@
+---
+# copyright Utrecht University
+
+galaxy_info:
+  author: Sietse Snel
+  description: Install iRODS GoCommands
+  license: GPLv3
+  min_ansible_version: "2.7"
+  platforms:
+    - name: EL
+      version: 7

roles/irods_gocommands/tasks/main.yml

@@ -0,0 +1,48 @@
+---
+# copyright Utrecht University
+
+- name: Get iRODS home directory
+  become_user: '{{ irods_service_account }}'
+  become: true
+  ansible.builtin.command: echo ~
+  register: irods_directory
+  changed_when: false
+  check_mode: false
+
+
+- name: Create GoCommands directories
+  ansible.builtin.file:
+    path: '{{ item }}'
+    state: directory
+    mode: '0755'
+    owner: '{{ irods_service_account }}'
+    group: '{{ irods_service_account }}'
+  with_items:
+    - '{{ irods_directory.stdout }}/gocommands'
+    - '{{ irods_directory.stdout }}/gocommands/{{ irods_gocommands_version }}'
+
+
+- name: Download GoCommands
+  ansible.builtin.get_url:
+    url: 'https://github.com/cyverse/gocommands/releases/download/v{{ irods_gocommands_version }}/gocmd-v{{ irods_gocommands_version }}-linux-amd64.tar.gz'
+    dest: '{{ irods_directory.stdout }}/gocommands/gocmd-v{{ irods_gocommands_version }}-linux-amd64.tar.gz'
+    checksum: '{{ irods_gocommands_archive_checksum }}'
+    mode: '0644'
+
+
+- name: Extract GoCommands archive
+  ansible.builtin.unarchive:
+    src: '{{ irods_directory.stdout }}/gocommands/gocmd-v{{ irods_gocommands_version }}-linux-amd64.tar.gz'
+    dest: '{{ irods_directory.stdout }}/gocommands/{{ irods_gocommands_version }}'
+    remote_src: true
+    creates: '{{ irods_directory.stdout }}/gocommands/{{ irods_gocommands_version }}/gocmd'
+
+
+- name: Sync GoCommands executable to path
+  ansible.builtin.copy:
+    src: '{{ irods_directory.stdout }}/gocommands/{{ irods_gocommands_version }}/gocmd'
+    remote_src: true
+    dest: '/usr/bin/gocmd'
+    owner: irods
+    group: irods
+    mode: '755'

roles/irods_icat/tasks/main.yml

@@ -62,52 +62,48 @@
           services['irods']['state'] == "running"))
 
 
-- name: Ensure iRODS 4.2.7 packages are absent
+- name: Ensure old iRODS packages are absent
   ansible.builtin.package:
     name:
-      - irods-uu-microservices-4.2.7_0.8.1
-      - irods-sudo-microservices-4.2.7_1.0.0
-      - davrods-4.2.7_1.4.2
-      - irods-runtime-4.2.7
-      - irods-server-4.2.7
-      - irods-rule-engine-plugin-python-4.2.7
-      - irods-icommands-4.2.7
-      - irods-database-plugin-postgres-4.2.7
-      - irods-uu-microservices-4.2.10_0.8.2
-      - irods-sudo-microservices-4.2.10_1.0.0
-      - davrods-4.2.10_1.5.0
-      - irods-runtime-4.2.10-1
-      - irods-server-4.2.10-1
-      - irods-database-plugin-postgres-4.2.10-1
-      - irods-icommands-4.2.10-1
-      - irods-rule-engine-plugin-python-4.2.10.0-1
-      - irods-rule-engine-plugin-indexing-4.2.10.0-1
-      - irods-rule-engine-plugin-elasticsearch-4.2.10.0-1
-      - irods-rule-engine-plugin-document-type-4.2.10.0-1
-      - irods-rule-engine-plugin-indexing-4.2.10.1-1
-      - irods-rule-engine-plugin-elasticsearch-4.2.10.1-1
-      - irods-rule-engine-plugin-document-type-4.2.10.1-1
+      - irods-uu-microservices-4.2.11_0.8.2-1
+      - irods-sudo-microservices-4.2.11_1.0.0-1
+      - davrods-4.2.11_1.5.0-1
+      - irods-runtime-4.2.11-1
+      - irods-server-4.2.11-1
+      - irods-database-plugin-postgres-4.2.11-1
+      - irods-rule-engine-plugin-python-4.2.11.1-1
+      - irods-icommands-4.2.11-1
     state: absent
   notify: Restart iRODS
 
 
 - name: Ensure iRODS iCAT server and plugins are present
   ansible.builtin.package:
     name:
-      - irods-server-4.2.11-1
-      - irods-runtime-4.2.11-1
-      - irods-database-plugin-postgres-4.2.11-1
-      - irods-rule-engine-plugin-python-4.2.11.1-1
+      - irods-server-4.2.12-1
+      - irods-runtime-4.2.12-1
+      - irods-database-plugin-postgres-4.2.12-1
+      - irods-rule-engine-plugin-python-4.2.12.0-1
     state: present
   when: not ansible_check_mode
 
 
-- name: Ensure iRODS indexing plugins are present
+- name: Ensure old iRODS indexing plugins are present
   ansible.builtin.package:
     name:
       - irods-rule-engine-plugin-indexing-4.2.11.0-1
       - irods-rule-engine-plugin-elasticsearch-4.2.11.0-1
       - irods-rule-engine-plugin-document-type-4.2.11.0-1
+    state: absent
+  when: not ansible_check_mode and enable_open_search
+
+
+- name: Ensure iRODS indexing plugins are present
+  ansible.builtin.package:
+    name:
+      - irods-rule-engine-plugin-indexing-4.2.12.0-1
+      - irods-rule-engine-plugin-elasticsearch-4.2.12.0-1
+      - irods-rule-engine-plugin-document-type-4.2.12.0-1
     state: present
   when: not ansible_check_mode and enable_open_search
 
@@ -207,12 +203,21 @@
     value: '{{ irods_icat_fqdn }}'
 
 
+- name: Get iRODS home directory
+  become_user: '{{ irods_service_account }}'
+  become: true
+  ansible.builtin.command: echo ~
+  register: irods_directory
+  changed_when: false
+  check_mode: false
+
+
 - name: Ensure iRODS host is defined
   become_user: '{{ irods_service_account }}'
   become: true
   # noqa fqcn[action]
   irods_config:
-    path: '/var/lib/irods/.irods/irods_environment.json'
+    path: '{{ irods_directory.stdout }}/.irods/irods_environment.json'
     key: 'irods_host'
     value: '{{ irods_icat_fqdn }}'
 
@@ -395,7 +400,7 @@
   become: true
   # noqa fqcn[action]
   irods_config:
-    path: '/var/lib/irods/.irods/irods_environment.json'
+    path: '{{ irods_directory.stdout }}/.irods/irods_environment.json'
     key: '{{ item.key }}'
     value: '{{ item.value }}'
   with_items:
@@ -428,7 +433,7 @@
       value: '{{ irods_database_port }}'
 
 
-- name: Ensure deprecated iRODS environment file for Python client is present
+- name: Ensure deprecated iRODS environment file for Python client is absent
   ansible.builtin.file:
     path: /var/lib/irods/.irods/python_client_environment.json
     state: absent
@@ -471,8 +476,10 @@
 
 
 - name: Ensure iRODS .ssh directory has strict permissions
+  become_user: '{{ irods_service_account }}'
+  become: true
   ansible.builtin.file:
-    path: /var/lib/irods/.ssh
+    path: ~/.ssh
     state: directory
     mode: 0700
     owner: '{{ irods_service_account }}'
@@ -484,7 +491,7 @@
   become: true
   ansible.builtin.template:
     src: id_ed25519.j2
-    dest: '/var/lib/irods/.ssh/id_ed25519'
+    dest: '~/.ssh/id_ed25519'
     owner: '{{ irods_service_account }}'
     mode: 0600
   no_log: true
@@ -496,7 +503,7 @@
   become: true
   ansible.builtin.shell: "ssh-keyscan {{ yoda_public_host }} >> ~/.ssh/known_hosts"
   args:
-    creates: '/var/lib/irods/.ssh/known_hosts'
+    creates: '~/.ssh/known_hosts'
   when: upload_priv_key is defined
 
 

roles/irods_icommands/tasks/main.yml

@@ -7,30 +7,22 @@
   register: irods_server
 
 
-- name: Ensure iRODS 4.2.7 packages are absent
+- name: Ensure old iRODS packages are absent
   ansible.builtin.package:
     name:
-      - irods-uu-microservices-4.2.7_0.8.1
-      - irods-sudo-microservices-4.2.7_1.0.0
-      - davrods-4.2.7_1.4.2
-      - irods-runtime-4.2.7
-      - irods-server-4.2.7
-      - irods-rule-engine-plugin-python-4.2.7
-      - irods-icommands-4.2.7
-      - irods-database-plugin-postgres-4.2.7
-      - irods-uu-microservices-4.2.10_0.8.2
-      - irods-sudo-microservices-4.2.10_1.0.0
-      - davrods-4.2.10_1.5.0
-      - irods-runtime-4.2.10-1
-      - irods-server-4.2.10-1
-      - irods-database-plugin-postgres-4.2.10-1
-      - irods-icommands-4.2.10-1
-      - irods-rule-engine-plugin-python-4.2.10.0-1
+      - irods-uu-microservices-4.2.11_0.8.2-1
+      - irods-sudo-microservices-4.2.11_1.0.0-1
+      - davrods-4.2.11_1.5.0-1
+      - irods-runtime-4.2.11-1
+      - irods-server-4.2.11-1
+      - irods-database-plugin-postgres-4.2.11-1
+      - irods-rule-engine-plugin-python-4.2.11.1-1
+      - irods-icommands-4.2.11-1
     state: absent
 
 
 - name: Ensure iRODS iCommands is installed
   ansible.builtin.yum:
-    name: irods-icommands-4.2.11-1
+    name: irods-icommands-4.2.12-1
     state: present
   when: not irods_server.stat.exists and not ansible_check_mode