diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 6fddca0..567c3fd 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,3 +1,4 @@ +--- version: 2 updates: # Maintain dependencies for GitHub Actions diff --git a/.github/workflows/shasum.yml b/.github/workflows/shasum.yml index 9111305..0c3e570 100644 --- a/.github/workflows/shasum.yml +++ b/.github/workflows/shasum.yml @@ -29,14 +29,19 @@ jobs: # Runs the Super-Linter action - name: Calculate the sum for new images run: | - for f in $(git diff --name-only ${{ github.event.pull_request.base.sha }} | grep "appdb.*\.yaml") + for f in $(git diff --name-only \ + ${{ github.event.pull_request.base.sha }} \ + | grep "appdb.*\.yaml") do [ -e "$f" ] || continue echo "Testing $f" url=$(yq eval .appdb.url "$f") curl -I "$url" || exit 1 # now download and shasum - sha_url=$(curl -s "$url" | sha512sum | cut -f1 -d"-" | tr -d "[:space:]") + sha_url=$(curl -s "$url" \ + | sha512sum \ + | cut -f1 -d"-" \ + | tr -d "[:space:]") sha_appdb=$(yq eval .appdb.sha512 "$f") if [ "$sha_url" != "$sha_appdb" ] then diff --git a/README.md b/README.md index 08cd8b4..51f0923 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ The repository has a GitHub action workflow that will try to build images from changes detected in the `*.json` files. This starts a VM at an EGI site (SCAI) that will: -1. get the repo files at the current commit +1. get the repository files at the current commit 1. install packer 1. build the image described in the json 1. upload to another site (IFCA-LCG2) diff --git a/builder/cloud-init.yaml b/builder/cloud-init.yaml index ba11975..e783241 100644 --- a/builder/cloud-init.yaml +++ b/builder/cloud-init.yaml @@ -33,9 +33,7 @@ write_files: cd /var/tmp/egi || exit # Valid GitHub token to access the repo - OAUTH_TOKEN="%TOKEN%" COMMIT_SHA="%REF%" - SHORT_COMMIT_SHA="%SHORT_REF%" IMAGE="%IMAGE%" FEDCLOUD_LOCKER_TOKEN="%FEDCLOUD_LOCKER_TOKEN%" @@ -57,10 +55,22 @@ write_files: cat /var/log/image-build.log >>"/var/tmp/egi/$VM_ID" export PATH="/var/tmp/egi/.venv/bin:$PATH" # try 10 times, otherwise just die +<<<<<<< HEAD /var/tmp/egi/builder/refresh.sh cloud.egi.eu "$(cat /var/tmp/egi/.refresh_token)" backend OS_TOKEN="$(yq -r '.clouds.backend.auth.token' /etc/openstack/clouds.yaml)" retry -t 25 -d 200 -- openstack --os-cloud backend --os-token "$OS_TOKEN" \ object create --name "$VM_ID" fedcloud-vmi "/var/tmp/egi/$VM_ID" +======= + /var/tmp/egi/builder/refresh.sh cloud.egi.eu \ + "$(cat /var/tmp/egi/.refresh_token)" \ + backend + OS_TOKEN="$(yq -r '.clouds.backend.auth.token' \ + /etc/openstack/clouds.yaml)" + retry -t 25 -d 200 -- openstack --os-cloud backend \ + --os-token "$OS_TOKEN" \ + object create --name "$VM_ID" \ + fedcloud-vmi "/var/tmp/egi/$VM_ID" +>>>>>>> 88f47f9 (Mess) path: /usr/local/bin/notify.sh permissions: '0755' - content: | diff --git a/centos/provisioners/config.yaml b/centos/provisioners/config.yaml index bf3027a..c790046 100644 --- a/centos/provisioners/config.yaml +++ b/centos/provisioners/config.yaml @@ -1,8 +1,9 @@ +--- - hosts: all tasks: - - name: Include cloud-init recipe - include_tasks: cloud-init.yaml - - name: Include "{{ansible_distribution_major_version}}" network recipe - include_tasks: "network-centos{{ansible_distribution_major_version}}.yaml" - - name: Include clean-up recipe - include_tasks: clean.yaml + - name: Include cloud-init recipe + include_tasks: cloud-init.yaml + - name: Include "{{ansible_distribution_major_version}}" network recipe + include_tasks: "network-centos{{ansible_distribution_major_version}}.yaml" + - name: Include clean-up recipe + include_tasks: clean.yaml diff --git a/centos/provisioners/init.yaml b/centos/provisioners/init.yaml index 5d4e36f..46d3575 100644 --- a/centos/provisioners/init.yaml +++ b/centos/provisioners/init.yaml @@ -1,20 +1,21 @@ +--- - hosts: all gather_facts: False tasks: - - name: install python - raw: test -e /usr/bin/python || test -e /usr/bin/python3 || (yum install -y python39) + - name: install python + raw: test -e /usr/bin/python || test -e /usr/bin/python3 || (yum install -y python39) - hosts: all tasks: - - name: update packages - yum: - name: '*' - state: latest - - name: get the rpm package facts - package_facts: - manager: "auto" - - name: Rebuilding initramfs for kernel - command: | - dracut -f --no-hostonly /boot/initramfs-{{ item['version'] }}-{{ item['release'] }}.{{ item['arch'] }}.img - {{ item['version'] }}-{{ item['release'] }}.{{ item['arch'] }} - loop: "{{ ansible_facts.packages['kernel'] }}" + - name: update packages + yum: + name: '*' + state: latest + - name: get the rpm package facts + package_facts: + manager: "auto" + - name: Rebuilding initramfs for kernel + command: | + dracut -f --no-hostonly /boot/initramfs-{{ item['version'] }}-{{ item['release'] }}.{{ item['arch'] }}.img + {{ item['version'] }}-{{ item['release'] }}.{{ item['arch'] }} + loop: "{{ ansible_facts.packages['kernel'] }}" diff --git a/ubuntu/provisioners/appliance.yaml b/ubuntu/provisioners/appliance.yaml index 9371c2d..2c778db 100644 --- a/ubuntu/provisioners/appliance.yaml +++ b/ubuntu/provisioners/appliance.yaml @@ -1,10 +1,11 @@ +--- - hosts: all roles: - - cloud-init - - role: grycap.docker - vars: - docker_version: "5:20.10.8~3-0~{{ansible_distribution | lower}}-{{ansible_distribution_release}}" - docker_compose_version: 1.29.2 - docker_install_pip: false - - appliance - - cleanup + - cloud-init + - role: grycap.docker + vars: + docker_version: "5:20.10.8~3-0~{{ansible_distribution | lower}}-{{ansible_distribution_release}}" + docker_compose_version: 1.29.2 + docker_install_pip: false + - appliance + - cleanup diff --git a/ubuntu/provisioners/docker.yaml b/ubuntu/provisioners/docker.yaml index dd6efe8..6222bb7 100644 --- a/ubuntu/provisioners/docker.yaml +++ b/ubuntu/provisioners/docker.yaml @@ -1,6 +1,7 @@ +--- - hosts: all become: true roles: - - cloud-init - - docker - - cleanup + - cloud-init + - docker + - cleanup diff --git a/ubuntu/provisioners/init.yaml b/ubuntu/provisioners/init.yaml index 5ecd987..9dcaebd 100644 --- a/ubuntu/provisioners/init.yaml +++ b/ubuntu/provisioners/init.yaml @@ -1,13 +1,13 @@ +--- - hosts: all become: true gather_facts: False tasks: - - name: install python - raw: test -e /usr/bin/python || (apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y python-minimal || DEBIAN_FRONTEND=noninteractive apt-get install -y python3-minimal) + - name: install python + raw: test -e /usr/bin/python || (apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y python-minimal || DEBIAN_FRONTEND=noninteractive apt-get install -y python3-minimal) - hosts: all become: true tasks: - - name: update packages - apt: upgrade=dist - + - name: update packages + apt: upgrade=dist diff --git a/ubuntu/provisioners/jupyterhub.yaml b/ubuntu/provisioners/jupyterhub.yaml index 9298471..e8e7223 100644 --- a/ubuntu/provisioners/jupyterhub.yaml +++ b/ubuntu/provisioners/jupyterhub.yaml @@ -1,164 +1,156 @@ --- - hosts: all tasks: - - name: Include cloud-init recipe - include_tasks: cloud-init.yaml + - name: Include cloud-init recipe + include_tasks: cloud-init.yaml - hosts: all vars: docker_version: 17.03.2~ce-0~ubuntu-xenial tasks: - - name: install requirements - apt: - name: - - apt-transport-https - - ca-certificates - - curl - - software-properties-common - - "linux-image-extra-{{ ansible_kernel }}" - - apparmor - state: latest - + - name: install requirements + apt: + name: + - apt-transport-https + - ca-certificates + - curl + - software-properties-common + - "linux-image-extra-{{ ansible_kernel }}" + - apparmor + state: latest - name: add Docker repo key apt_key: url="https://download.docker.com/linux/debian/gpg" ignore_errors: yes - - name: add docker repo apt_repository: repo: 'deb [arch=amd64] https://download.docker.com/linux/ubuntu {{ansible_distribution_release}} stable' state: present update_cache: yes - - name: Install docker {{ docker_version }} apt: name: "docker-ce={{ docker_version }}" - state: present + state: present force: yes # notebooks - hosts: all tasks: - - name: add node repo key - apt_key: url="https://deb.nodesource.com/gpgkey/nodesource.gpg.key" - ignore_errors: yes - - name: add node repo - apt_repository: - repo: 'deb [arch=amd64] https://deb.nodesource.com/node_11.x xenial main' - state: present - update_cache: yes - - name: install packages - apt: - name: - - nginx - - python3-pip - - nodejs - state: present - - name: install python packages - pip: - executable: pip3 - name: "jupyterhub dockerspawner jupyter_client git+https://github.com/enolfc/egispawner.git git+https://github.com/enolfc/oauthenticator.git@checkin" - - name: Install "configurable-http-proxy" node.js package globally. - npm: - name: configurable-http-proxy - global: yes - - name: Create dirs - file: - path: "{{ item }}" - state: directory - with_items: [/etc/jupyterhub, /var/lib/jupyterhub] - - name: Create jupyterhub_config.py file - copy: - dest: /etc/jupyterhub/jupyterhub_config.py - content: | - from jupyter_client.localinterfaces import public_ips - c.JupyterHub.hub_ip = public_ips()[0] - - c.JupyterHub.cookie_secret_file = '/var/lib/jupyterhub/jupyterhub_cookie_secret' - c.JupyterHub.db_url = 'sqlite:////var/lib/jupyterhub/jupyterhub.sqlite' - - import os - os.environ['EGICHECKIN_HOST'] = 'aai-dev.egi.eu' - c.JupyterHub.authenticator_class = 'oauthenticator.egicheckin.EGICheckinAuthenticator' - c.EGICheckinAuthenticator.client_id = '' - c.EGICheckinAuthenticator.client_secret = '' - c.EGICheckinAuthenticator.oauth_callback_url = 'https:///hub/oauth_callback' - c.EGICheckinAuthenticator.scope = ['openid', 'profile', 'email', 'offline_access', - 'eduperson_scoped_affiliation', 'eduperson_entitlement'] - c.JupyterHub.spawner_class = 'dockerspawner.DockerSpawner' - notebook_dir = os.environ.get('DOCKER_NOTEBOOK_DIR') or '/home/jovyan/work' - c.DockerSpawner.notebook_dir = notebook_dir - c.DockerSpawner.image = 'jupyter/scipy-notebook:latest' - c.DockerSpawner.volumes = { 'jupyterhub-user-{username}': notebook_dir } - c.DockerSpawner.remove_containers = True - c.Spawner.default_url = '/lab' - - - name: Create nginx connection.conf file - copy: - dest: /etc/nginx/conf.d/connection.conf - content: | - map $http_upgrade $connection_upgrade { - default upgrade; - '' close; - } - - - name: remove default nginx site - file: - path: /etc/nginx/sites-enabled/default - state: absent - - - name: Proxy jupyterhub in nginx conf - copy: - dest: /etc/nginx/sites-enabled/jupyterhub - content: | - server { - listen 80 default_server; - listen [::]:80 default_server; - - root /var/www/html; - - # Add index.php to the list if you are using PHP - index index.html index.htm index.nginx-debian.html; - - server_name _; - - # Managing literal requests to the JupyterHub front end - location / { - proxy_pass http://localhost:8000; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header Host $host; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - - # websocket headers - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection $connection_upgrade; + - name: add node repo key + apt_key: url="https://deb.nodesource.com/gpgkey/nodesource.gpg.key" + ignore_errors: yes + - name: add node repo + apt_repository: + repo: 'deb [arch=amd64] https://deb.nodesource.com/node_11.x xenial main' + state: present + update_cache: yes + - name: install packages + apt: + name: + - nginx + - python3-pip + - nodejs + state: present + - name: install python packages + pip: + executable: pip3 + name: "jupyterhub dockerspawner jupyter_client git+https://github.com/enolfc/egispawner.git git+https://github.com/enolfc/oauthenticator.git@checkin" + - name: Install "configurable-http-proxy" node.js package globally. + npm: + name: configurable-http-proxy + global: yes + - name: Create dirs + file: + path: "{{ item }}" + state: directory + with_items: [/etc/jupyterhub, /var/lib/jupyterhub] + - name: Create jupyterhub_config.py file + copy: + dest: /etc/jupyterhub/jupyterhub_config.py + content: | + from jupyter_client.localinterfaces import public_ips + c.JupyterHub.hub_ip = public_ips()[0] + + c.JupyterHub.cookie_secret_file = '/var/lib/jupyterhub/jupyterhub_cookie_secret' + c.JupyterHub.db_url = 'sqlite:////var/lib/jupyterhub/jupyterhub.sqlite' + + import os + os.environ['EGICHECKIN_HOST'] = 'aai-dev.egi.eu' + c.JupyterHub.authenticator_class = 'oauthenticator.egicheckin.EGICheckinAuthenticator' + c.EGICheckinAuthenticator.client_id = '' + c.EGICheckinAuthenticator.client_secret = '' + c.EGICheckinAuthenticator.oauth_callback_url = 'https:///hub/oauth_callback' + c.EGICheckinAuthenticator.scope = ['openid', 'profile', 'email', 'offline_access', + 'eduperson_scoped_affiliation', 'eduperson_entitlement'] + c.JupyterHub.spawner_class = 'dockerspawner.DockerSpawner' + notebook_dir = os.environ.get('DOCKER_NOTEBOOK_DIR') or '/home/jovyan/work' + c.DockerSpawner.notebook_dir = notebook_dir + c.DockerSpawner.image = 'jupyter/scipy-notebook:latest' + c.DockerSpawner.volumes = { 'jupyterhub-user-{username}': notebook_dir } + c.DockerSpawner.remove_containers = True + c.Spawner.default_url = '/lab' + - name: Create nginx connection.conf file + copy: + dest: /etc/nginx/conf.d/connection.conf + content: | + map $http_upgrade $connection_upgrade { + default upgrade; + '' close; } - } - - - name: Create systemd for jupyterhub - copy: - dest: /etc/systemd/system/jupyterhub.service - content: | - [Unit] - Description=Jupyterhub - After=syslog.target network.target - - [Service] - User=root - ExecStart=/usr/local/bin/jupyterhub -f /etc/jupyterhub/jupyterhub_config.py - - [Install] - WantedBy=multi-user.target - - - name: Enable services - service: - name: "{{ item }}" - enabled: yes - with_items: - - nginx - - jupyterhub + - name: remove default nginx site + file: + path: /etc/nginx/sites-enabled/default + state: absent + - name: Proxy jupyterhub in nginx conf + copy: + dest: /etc/nginx/sites-enabled/jupyterhub + content: | + server { + listen 80 default_server; + listen [::]:80 default_server; + + root /var/www/html; + + # Add index.php to the list if you are using PHP + index index.html index.htm index.nginx-debian.html; + + server_name _; + + # Managing literal requests to the JupyterHub front end + location / { + proxy_pass http://localhost:8000; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + + # websocket headers + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; + } + } + - name: Create systemd for jupyterhub + copy: + dest: /etc/systemd/system/jupyterhub.service + content: | + [Unit] + Description=Jupyterhub + After=syslog.target network.target + + [Service] + User=root + ExecStart=/usr/local/bin/jupyterhub -f /etc/jupyterhub/jupyterhub_config.py + + [Install] + WantedBy=multi-user.target + - name: Enable services + service: + name: "{{ item }}" + enabled: yes + with_items: + - nginx + - jupyterhub - hosts: all tasks: - - name: Include clean-up recipe - include_tasks: clean.yaml + - name: Include clean-up recipe + include_tasks: clean.yaml diff --git a/ubuntu/provisioners/roles/appliance/defaults/main.yaml b/ubuntu/provisioners/roles/appliance/defaults/main.yaml index 0dfa9b7..523c620 100644 --- a/ubuntu/provisioners/roles/appliance/defaults/main.yaml +++ b/ubuntu/provisioners/roles/appliance/defaults/main.yaml @@ -1,3 +1,4 @@ +--- appliance_tag: "v0.16" ssm_tag: "3.2.1-1" base_url: "https://raw.githubusercontent.com/enolfc/fedcloudappliance/{{ appliance_tag }}" diff --git a/ubuntu/provisioners/roles/appliance/tasks/main.yaml b/ubuntu/provisioners/roles/appliance/tasks/main.yaml index 514cfd5..2405471 100644 --- a/ubuntu/provisioners/roles/appliance/tasks/main.yaml +++ b/ubuntu/provisioners/roles/appliance/tasks/main.yaml @@ -1,3 +1,4 @@ +--- - name: Create appliance directories file: path: "{{ item }}" diff --git a/ubuntu/provisioners/roles/cleanup/tasks/main.yaml b/ubuntu/provisioners/roles/cleanup/tasks/main.yaml index c7748db..3cc97b0 100644 --- a/ubuntu/provisioners/roles/cleanup/tasks/main.yaml +++ b/ubuntu/provisioners/roles/cleanup/tasks/main.yaml @@ -71,7 +71,7 @@ command: locale-gen --purge en_GB.utf8 - name: clean log files - command: find /var/log/ -type f -exec cp /dev/null {} \; + command: find /var/log/ -type f -exec cp /dev/null {} \; - name: fill disk with zeros shell: | diff --git a/ubuntu/provisioners/roles/cloud-init/tasks/cloud-init-focal.yaml b/ubuntu/provisioners/roles/cloud-init/tasks/cloud-init-focal.yaml index 9f23a78..7a02c58 100644 --- a/ubuntu/provisioners/roles/cloud-init/tasks/cloud-init-focal.yaml +++ b/ubuntu/provisioners/roles/cloud-init/tasks/cloud-init-focal.yaml @@ -4,6 +4,6 @@ path: "{{ item }}" state: absent with_items: - - "/etc/cloud/cloud.cfg.d/99-installer.cfg" - - "/etc/cloud/cloud.cfg.d/curtin-preserve-sources.cfg" - - "/etc/cloud/cloud.cfg.d/subiquity-disable-cloudinit-networking.cfg" + - "/etc/cloud/cloud.cfg.d/99-installer.cfg" + - "/etc/cloud/cloud.cfg.d/curtin-preserve-sources.cfg" + - "/etc/cloud/cloud.cfg.d/subiquity-disable-cloudinit-networking.cfg" diff --git a/ubuntu/provisioners/roles/docker/tasks/main.yaml b/ubuntu/provisioners/roles/docker/tasks/main.yaml index 356de36..28c213b 100644 --- a/ubuntu/provisioners/roles/docker/tasks/main.yaml +++ b/ubuntu/provisioners/roles/docker/tasks/main.yaml @@ -1,8 +1,9 @@ +--- - name: install requirements apt: name: - # this is to make the nfs-based volumes work - - nfs-common + # this is to make the nfs-based volumes work + - nfs-common state: latest - name: Configure docker @@ -14,7 +15,7 @@ docker_install_pip: false docker_config_values: exec-opts: - - native.cgroupdriver=systemd + - native.cgroupdriver=systemd log-driver: json-file log-opts: max-size: 100m @@ -51,7 +52,7 @@ kube_version: 1.28.7-1.1 apt: name: - - "kubelet={{ kube_version }}" - - "kubeadm={{ kube_version }}" - - "kubectl={{ kube_version }}" + - "kubelet={{ kube_version }}" + - "kubeadm={{ kube_version }}" + - "kubectl={{ kube_version }}" state: present