Merge pull request #100 from gchq/4.0

4.0

ci_build.sh

@@ -769,6 +769,11 @@ main() {
   # Now build each of the release branches (if they have changed)
   for branch_name in "${release_branches[@]}"; do
 
+    if [[ "${branch_name}" = "3.5" ]]; then
+      # 3.5 is not set up with the hugo docs site so skip it
+      break
+    fi
+
     if ! git ls-remote --exit-code --heads origin "refs/heads/${branch_name}"; then
       echo -e "${RED}ERROR: Branch ${BLUE}${branch_name}${RED}" \
         "does not exist. Check contents of release_branches array.${NC}"

container_build/docker_hugo/Dockerfile

@@ -17,7 +17,9 @@
 FROM klakegg/hugo:0.95.0-ext-alpine
 
 # Run separately so docker can cache the layer to speed up image creation
-RUN apk add --no-cache \
+RUN true \
+    && apk update \
+    && apk add --no-cache \
         bash \
         git
 
@@ -37,7 +39,8 @@ ARG USER_ID
 ARG GROUP_ID    
 
 # Docker can't cache this layer as it has variables in it.
-RUN echo "USER_ID: [$USER_ID]" \
+RUN true \
+    && echo "USER_ID: [$USER_ID]" \
     && echo "GROUP_ID: [$GROUP_ID]" \
     && echo \
     && echo "Ensuring group exists for group id [${GROUP_ID}]" \
@@ -60,9 +63,13 @@ RUN echo "USER_ID: [$USER_ID]" \
     && echo \
     && mkdir -p /builder/shared \
     && mkdir -p /hugo-cache \
-    && chown "${USER_ID}:${GROUP_ID}" /hugo-cache
+    && chown "${USER_ID}:${GROUP_ID}" /hugo-cache \
+    && mkdir -p /npm-cache \
+    && chown "${USER_ID}:${GROUP_ID}" /npm-cache \
+    && npm config set cache /npm-cache --global
 
 USER $USER_ID
 VOLUME /hugo-cache
+VOLUME /npm-cache
 
 # vim: set tabstop=4 shiftwidth=4 expandtab:

container_build/docker_pdf/Dockerfile

@@ -2,8 +2,6 @@
 # https://github.com/puppeteer/puppeteer/blob/main/docs/troubleshooting.md
 FROM node:20.2.0-buster-slim
 
-# Comment to change cache key
-
 WORKDIR /builder
 
 # Install latest chrome dev package and fonts to support major charsets (Chinese, Japanese, Arabic, Hebrew, Thai and a few others)

container_build/generate_buildx_cache_key.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-set -e
+set -eo pipefail
 
 setup_echo_colours() {
   # Exit the script on any error
@@ -47,6 +47,12 @@ main() {
   IS_DEBUG=false
 
   setup_echo_colours
+  local is_mac_os
+  if [[ "$( uname -s )" == "Darwin" ]]; then
+    is_mac_os=true
+  else
+    is_mac_os=false
+  fi
 
   user_id=
   user_id="$(id -u)"
@@ -75,8 +81,13 @@ main() {
 
   # Make a hash of these things and effectively use this as the cache key for
   # buildx so any change makes it ignore a previous cache.
+  if [[ "${is_mac_os}" = true ]]; then
+    shasum -a 256 <<< "${cache_key_source}" \
+      | cut -d" " -f1
+  else
   sha256sum <<< "${cache_key_source}" \
     | cut -d" " -f1
+  fi
 }
 
 main "$@"

container_build/runInHugoDocker.sh

@@ -102,6 +102,7 @@ main() {
         "-c"  \
         "cd docs; hugo server" \
       )
+        #"hugo server --cleanDestinationDir --watch" \
         #"hugo server --baseURL 'localhost:1313/stroom-docs'" \
       if [[ $# -eq 2 ]] && [[ "${2}" = "detach" ]]; then
         extra_docker_args=( "--detach" )
@@ -111,8 +112,9 @@ main() {
         # Build the site and output to ./public
         "bash" \
         "-c"  \
-        "cd docs; hugo --buildDrafts" \
+        "cd docs; hugo --environment production" \
       )
+        #"hugo --environment production --cleanDestinationDir" \
         #"hugo --buildDrafts --baseURL '/stroom-docs'" \
     elif [[ $# -ge 1 ]] && [[ "$1" = "build" ]]; then
       echo "Using baseUrl: $2"
@@ -121,7 +123,7 @@ main() {
         # Build the site and output to ./public
         "bash" \
         "-c"  \
-        "cd docs; hugo --buildDrafts --baseUrl \"$2\"" \
+        "cd docs; hugo --environment production --baseUrl \"$2\"" \
       )
     else
       run_cmd=( \
@@ -132,6 +134,12 @@ main() {
     fi
   fi
 
+  local is_mac_os
+  if [[ "$( uname -s )" == "Darwin" ]]; then
+    is_mac_os=true
+  else
+    is_mac_os=false
+  fi
   user_id=
   user_id="$(id -u)"
 
@@ -150,19 +158,39 @@ main() {
 
   dest_dir="/builder/shared"
 
-  docker_group_id="$(stat -c '%g' /var/run/docker.sock)"
+  if [[ "${is_mac_os}" = true ]]; then
+    # No GNU binutils on macos
+    docker_group_id="$(stat -f '%g' /var/run/docker.sock)"
+  else
+    docker_group_id="$(stat -c '%g' /var/run/docker.sock)"
+  fi
 
   echo -e "${GREEN}HOME ${BLUE}${HOME}${NC}"
   echo -e "${GREEN}User ID ${BLUE}${user_id}${NC}"
   echo -e "${GREEN}Group ID ${BLUE}${group_id}${NC}"
   echo -e "${GREEN}Host repo root dir ${BLUE}${host_abs_repo_dir}${NC}"
   echo -e "${GREEN}Docker group id ${BLUE}${docker_group_id}${NC}"
 
+  if ! docker version >/dev/null 2>&1; then
+    echo -e "${RED}ERROR: Docker is not installed. Please install Docker or Docker Desktop.${NC}"
+    exit 1
+  fi
+
+  if ! docker buildx version >/dev/null 2>&1; then
+    echo -e "${RED}ERROR: Docker buildx is not installed. Please install it.${NC}"
+    exit 1
+  fi
+
+  # Create a persistent vol for the home dir, idempotent
+  docker volume create builder-home-dir-vol
+
   # Create a persistent vol for the hugo cache which contains the downloaded
   # go modules, else they will end up in /tmp and have to be downloaded
   # on each run.
   hugo_cache_vol="builder-hugo-cache-vol"
   docker volume create "${hugo_cache_vol}"
+  npm_cache_vol="builder-npm-cache-vol"
+  docker volume create "${npm_cache_vol}"
 
   # So we are not rate limited, login before doing the build as this
   # will pull images
@@ -206,9 +234,12 @@ main() {
     echo -e "${GREEN}Removing old cache directories${NC}"
     #ls -1trd "${cache_dir_base}/from_"*
 
+    # List all matching dirs
+    # Remove the last item
+    # Delete each item
     ls -1trd "${cache_dir_base}/from_"* \
-      | head -n -1 \
-      | xargs -d '\n' rm -rf --
+      | sed '$d' \
+      | xargs rm -rf --
     echo -e "${GREEN}Remaining cache directories${NC}"
     ls -1trd "${cache_dir_base}/from_"*
   fi
@@ -231,18 +262,6 @@ main() {
 
     #--workdir "${dest_dir}" \
 
-  # The cache gets written into a different dir to where it is read from
-  # so we can clear out anything stale in the old ones and then move
-  # the new one over ready to be read on next run
-  #echo
-  #ls -l "${cache_dir_base}"
-  #rm -rf "${cache_dir_base:?}/from_"*
-  #echo
-  #ls -l "${cache_dir_base}"
-  #mv "${cache_dir_to}" "${cache_dir_from}"
-  #echo
-  #ls -l "${cache_dir_base}"
-  #echo
 
   if [ -t 1 ]; then 
     # In a terminal
@@ -267,8 +286,12 @@ main() {
   echo -e "${GREEN}Hugo cache is in docker volume" \
     "${YELLOW}${hugo_cache_vol}${GREEN}, use" \
     "${BLUE}docker volume rm ${hugo_cache_vol}${GREEN} to clear it.${NC}"
+  echo -e "${GREEN}NPM cache is in docker volume" \
+    "${YELLOW}${npm_cache_vol}${GREEN}, use" \
+    "${BLUE}docker volume rm ${npm_cache_vol}${GREEN} to clear it.${NC}"
 
   hudo_cache_dir="/hugo-cache"
+  npm_cache_dir="/npm-cache"
 
   docker run \
     "${tty_args[@]+"${tty_args[@]}"}" \
@@ -277,13 +300,15 @@ main() {
     --tmpfs /tmp:exec \
     --mount "type=bind,src=${host_abs_repo_dir},dst=${dest_dir}" \
     --volume "${hugo_cache_vol}:${hudo_cache_dir}" \
+    --volume "${npm_cache_vol}:${npm_cache_dir}" \
     --read-only \
     --name "schema-hugo-build-env" \
     --network "hugo-schema" \
     --env "BUILD_VERSION=${BUILD_VERSION:-SNAPSHOT}" \
     --env "DOCKER_USERNAME=${DOCKER_USERNAME}" \
     --env "DOCKER_PASSWORD=${DOCKER_PASSWORD}" \
-    --env "HUGO_CACHEDIR=/${hudo_cache_dir}" \
+    --env "HUGO_CACHEDIR=${hudo_cache_dir}" \
+    --env "npm_config_cache=${npm_cache_dir}" \
     "${extra_docker_args[@]}" \
     "${image_tag}" \
     "${run_cmd[@]}"

container_build/runInPupeteerDocker.sh

@@ -187,10 +187,10 @@ main() {
 
   if [ "${bash_cmd}" = "bash" ]; then
     run_cmd=( "bash" )
-  elif [[ "${bash_cmd}" = "PDF" || "${bash_cmd}" = "pdf" ]]; then
 
-    # Hugo is running in another container so use the service name 'site' as
-    # the host
+  elif [[ "${bash_cmd}" = "PDF" || "${bash_cmd}" = "pdf" ]]; then
+    # Hugo is running in another container so use the service name 
+    # 'schema-hugo-build-env' for the host
     run_cmd=( \
       "node" \
       "../generate-pdf.js" \

event-logging-transformer-main/build.gradle

@@ -73,11 +73,10 @@ task transformersDist(type: Zip) {
     from 'pipelines/transformations'
 }
 
-
 // Run the python script to validate all the version numbers in the schema file
 task validateSchemaVersions(type: Exec) {
     workingDir '../'
-    commandLine "python", "validateSchemaVersions.py", versions.eventLogging
+    commandLine "python3", "validateSchemaVersions.py", versions.eventLogging
 }
 
 // Run a bash script to compare the currennt jaxb source with the latest released version so

event-logging.xsd

@@ -14,7 +14,7 @@
   ~ See the License for the specific language governing permissions and
   ~ limitations under the License.
 -->
-<xs:schema xmlns:evt="event-logging:3" xmlns:xs="http://www.w3.org/2001/XMLSchema" targetNamespace="event-logging:3" elementFormDefault="qualified" attributeFormDefault="unqualified" version="4.0.2" id="event-logging-v4.0.2">
+<xs:schema xmlns:evt="event-logging:3" xmlns:xs="http://www.w3.org/2001/XMLSchema" targetNamespace="event-logging:3" elementFormDefault="qualified" attributeFormDefault="unqualified" version="4.0.3" id="event-logging-v4.0.3">
   <xs:annotation>
     <xs:documentation>This schema describes the allowed element structure for event logging. Please refer to the documentation and examples for a description of how to use this schema in addition to the descriptions given for each element within this schema.</xs:documentation>
   </xs:annotation>
@@ -1229,7 +1229,7 @@
             <xs:documentation>The organisations permitted to access the protectively marked item.</xs:documentation>
           </xs:annotation>
           <xs:sequence>
-            <xs:element name="PermittedOrganisation" type="evt:OrganisationComplexType">
+            <xs:element name="PermittedOrganisation" type="evt:OrganisationComplexType" maxOccurs="unbounded">
               <xs:annotation>
                 <xs:documentation>An organisation permitted to access the protectively marked item. The PermittedOrganisation is described by an Organisation and optionally qualified by the organisation's country code.</xs:documentation>
               </xs:annotation>
@@ -3635,7 +3635,7 @@
       <xs:documentation>Type for specifying the version numbers of XML documents that are supported by this version of the XMLSchema.</xs:documentation>
     </xs:annotation>
     <xs:restriction base="xs:string">
-      <xs:enumeration value="4.0.2"/>
+      <xs:enumeration value="4.0.3"/>
     </xs:restriction>
   </xs:simpleType>
   <xs:simpleType name="VirtualSessionSessionStateSimpleType">

serve_docs_site.sh

@@ -0,0 +1,78 @@
+#!/usr/bin/env bash
+
+# #############################################################################
+# Relies on docker and its buildx plugin to do the svg generation and to run
+# Script to run the documentation site
+# Hugo.
+# #############################################################################
+
+set -e
+
+setup_echo_colours() {
+  # Exit the script on any error
+  set -e
+
+  # shellcheck disable=SC2034
+  if [ "${MONOCHROME}" = true ]; then
+    RED=''
+    GREEN=''
+    YELLOW=''
+    BLUE=''
+    BLUE2=''
+    DGREY=''
+    NC='' # No Colour
+  else 
+    RED='\033[1;31m'
+    GREEN='\033[1;32m'
+    YELLOW='\033[1;33m'
+    BLUE='\033[1;34m'
+    BLUE2='\033[1;34m'
+    DGREY='\e[90m'
+    NC='\033[0m' # No Colour
+  fi
+}
+
+debug_value() {
+  local name="$1"; shift
+  local value="$1"; shift
+
+  if [ "${IS_DEBUG}" = true ]; then
+    echo -e "${DGREY}DEBUG ${name}: ${value}${NC}"
+  fi
+}
+
+debug() {
+  local str="$1"; shift
+
+  if [ "${IS_DEBUG}" = true ]; then
+    echo -e "${DGREY}DEBUG ${str}${NC}"
+  fi
+}
+
+check_prerequisites() {
+  if ! docker version >/dev/null 2>&1; then
+    echo -e "${RED}ERROR: Docker is not installed. Please install Docker or Docker Desktop.${NC}"
+    exit 1
+  fi
+
+  if ! docker buildx version >/dev/null 2>&1; then
+    echo -e "${RED}ERROR: Docker buildx plugin is not installed. Please install it. See https://github.com/docker/buildx#installing${NC}"
+    exit 1
+  fi
+}
+
+main() {
+  IS_DEBUG=false
+  local repo_root
+  repo_root="$(git rev-parse --show-toplevel)"
+  pushd "${repo_root}" > /dev/null
+
+  setup_echo_colours
+  check_prerequisites
+
+  echo -e "${GREEN}Running the Hugo server on localhost:1313${NC}"
+
+  ./container_build/runInHugoDocker.sh server
+}
+
+main "$@"

unreleased_changes/20240524_092711_920__86.md

@@ -0,0 +1,24 @@
+* Issue **#86** : Change `PermittedOrganisation` to be unbounded within `PermittedOrganisations`.
+
+
+```sh
+# ********************************************************************************
+# Issue title: `PermittedOrganisation` is 1-1 rather than 1-*
+# Issue link:  https://github.com/gchq/event-logging-schema/issues/86
+# ********************************************************************************
+
+# ONLY the top line will be included as a change entry in the CHANGELOG.
+# The entry should be in GitHub flavour markdown and should be written on a SINGLE
+# line with no hard breaks. You can have multiple change files for a single GitHub issue.
+# The  entry should be written in the imperative mood, i.e. 'Fix nasty bug' rather than
+# 'Fixed nasty bug'.
+#
+# Examples of acceptable entries are:
+#
+#
+# * Issue **123** : Fix bug with an associated GitHub issue in this repository
+#
+# * Issue **namespace/other-repo#456** : Fix bug with an associated GitHub issue in another repository
+#
+# * Fix bug with no associated GitHub issue.
+```