From a0aacd49ceaac9ec2b4846d96413eb0a9e54a319 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81lvaro=20Arranz?= Date: Fri, 2 Jul 2021 17:39:25 +0200 Subject: [PATCH 1/2] =?UTF-8?q?Update=20psycopg2-binary=20dependency=20to?= =?UTF-8?q?=20avoid=20the=20"AssertionError:=20database=20connection=20isn?= =?UTF-8?q?=E2=80=99t=20set=20to=20UTC"=20error?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dev/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/Dockerfile b/dev/Dockerfile index 4cb1118..7e4e189 100644 --- a/dev/Dockerfile +++ b/dev/Dockerfile @@ -45,7 +45,7 @@ COPY ./manage.py /usr/local/bin/ RUN apt-get update && \ apt-get install -y --no-install-recommends libmemcached-dev libpcre3-dev gosu gcc ca-certificates curl && \ - pip install --no-cache-dir social-auth-app-django "gunicorn==19.3.0" psycopg2-binary pylibmc pysolr "elasticsearch==2.4.1" && \ + pip install --no-cache-dir social-auth-app-django "gunicorn==19.3.0" "psycopg2-binary<2.9" pylibmc pysolr "elasticsearch==2.4.1" && \ pip install --no-cache-dir /dist/*.whl && \ rm -rf /dist && \ adduser --system --group --shell /bin/bash wirecloud && \ From 666ffbffb37bd93dc1698c50d0af50a1da2cfed3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81lvaro=20Arranz?= Date: Thu, 27 May 2021 13:26:26 +0200 Subject: [PATCH 2/2] Initial WireCloud 1.4 docker image --- .github/workflows/ci.yml | 3 +- 1.4/.gitignore | 4 + 1.4/Dockerfile | 41 ++++ 1.4/docker-compose-config-file.yml | 68 ++++++ 1.4/docker-compose-custom-user.yml | 17 ++ 1.4/docker-compose-idm.yml | 83 +++++++ 1.4/docker-compose-simple.yml | 29 +++ 1.4/docker-compose-standalone.yml | 18 ++ 1.4/docker-compose.yml | 75 ++++++ 1.4/docker-entrypoint.sh | 45 ++++ 1.4/manage.py | 7 + 1.4/nginx.conf | 49 ++++ 1.4/settings.py | 366 +++++++++++++++++++++++++++++ 1.4/tests.py | 238 +++++++++++++++++++ 1.4/urls.py | 56 +++++ README.md | 3 +- latest | 2 +- 17 files changed, 1101 insertions(+), 3 deletions(-) create mode 100644 1.4/.gitignore create mode 100644 1.4/Dockerfile create mode 100644 1.4/docker-compose-config-file.yml create mode 100644 1.4/docker-compose-custom-user.yml create mode 100644 1.4/docker-compose-idm.yml create mode 100644 1.4/docker-compose-simple.yml create mode 100644 1.4/docker-compose-standalone.yml create mode 100644 1.4/docker-compose.yml create mode 100755 1.4/docker-entrypoint.sh create mode 100755 1.4/manage.py create mode 100644 1.4/nginx.conf create mode 100644 1.4/settings.py create mode 100644 1.4/tests.py create mode 100644 1.4/urls.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 669e69f..defc10f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,6 +24,7 @@ jobs: wirecloud-version: - "1.2" - "1.3" + - "1.4" - "dev" - "latest" @@ -41,7 +42,7 @@ jobs: VERSION: ${{ matrix.wirecloud-version }} run: | docker build --squash -t fiware/wirecloud:${VERSION} . - test "${VERSION}" = "latest" && sed -ri "s|fiware/wirecloud:1.3|fiware/wirecloud:latest|g" docker-compose*.yml || true + test "${VERSION}" = "latest" && sed -ri "s|fiware/wirecloud:1.4|fiware/wirecloud:latest|g" docker-compose*.yml || true sudo python3 tests.py - name: Push to docker env: diff --git a/1.4/.gitignore b/1.4/.gitignore new file mode 100644 index 0000000..10cdebd --- /dev/null +++ b/1.4/.gitignore @@ -0,0 +1,4 @@ +/elasticsearch-data/ +/postgres-data/ +/wirecloud-data/ +/wirecloud-static/ diff --git a/1.4/Dockerfile b/1.4/Dockerfile new file mode 100644 index 0000000..0f1469b --- /dev/null +++ b/1.4/Dockerfile @@ -0,0 +1,41 @@ +ARG PYTHON_VERSION=3.8 +FROM python:${PYTHON_VERSION}-slim + +ENV GITHUB_ACCOUNT=${GITHUB_ACCOUNT} \ + GITHUB_REPOSITORY=${GITHUB_REPOSITORY} \ + DEFAULT_THEME=wirecloud.defaulttheme \ + FORWARDED_ALLOW_IPS=* \ + DB_PORT=5432 \ + LOGLEVEL=info + +# Install WireCloud & dependencies +COPY ./docker-entrypoint.sh / +COPY ./manage.py /usr/local/bin/ + +RUN apt update && \ + apt install -y --no-install-recommends libmemcached-dev libpcre3-dev gosu gcc ca-certificates curl && \ + pip install --no-cache-dir social-auth-app-django "gunicorn==19.3.0" "psycopg2-binary<2.9" pylibmc pysolr "elasticsearch==2.4.1" && \ + pip install --no-cache-dir "wirecloud<1.5" && \ + adduser --system --group --shell /bin/bash wirecloud && \ + pip install --no-cache-dir "channels<2.3" "channels-redis" "channels-rabbitmq" "wirecloud-keycloak>=0.3.0" && \ + mkdir -p /opt/wirecloud_instance /var/www/static && \ + cd /opt && \ + wirecloud-admin startproject wirecloud_instance wirecloud_instance && \ + chown -R wirecloud:wirecloud wirecloud_instance /var/www/static && \ + chmod a+x wirecloud_instance/manage.py && \ + chmod a+x /docker-entrypoint.sh && \ + apt-get remove -y gcc libmemcached-dev --autoremove && \ + rm -rf /var/lib/apt/lists/* + +COPY ./settings.py ./urls.py /opt/wirecloud_instance/wirecloud_instance/ + +WORKDIR /opt/wirecloud_instance + +VOLUME /var/www/static +VOLUME /opt/wirecloud_instance/data + +HEALTHCHECK --interval=5s \ + --start-period=120s \ + CMD curl --fail http://localhost:8000/api/features || exit 1 + +ENTRYPOINT ["/docker-entrypoint.sh"] diff --git a/1.4/docker-compose-config-file.yml b/1.4/docker-compose-config-file.yml new file mode 100644 index 0000000..b59ba44 --- /dev/null +++ b/1.4/docker-compose-config-file.yml @@ -0,0 +1,68 @@ +version: "3" + +services: + + nginx: + restart: always + image: nginx:latest + ports: + - 80:80 + volumes: + - ./nginx.conf:/etc/nginx/nginx.conf:ro + - wirecloud-static:/var/www/static:ro + depends_on: + - wirecloud + + + postgres: + restart: always + image: postgres:latest + environment: + - POSTGRES_PASSWORD=wirepass # Change this password! + volumes: + - postgres-data:/var/lib/postgresql/data + + + elasticsearch: + restart: always + image: elasticsearch:2.4 + volumes: + - elasticsearch-data:/usr/share/elasticsearch/data + command: elasticsearch -Des.index.max_result_window=50000 + + + memcached: + restart: always + image: memcached:1 + command: memcached -m 2048m + + + wirecloud: + restart: always + image: fiware/wirecloud:dev + depends_on: + - postgres + - elasticsearch + - memcached + environment: + - DEBUG=False + # - DEFAULT_THEME=wirecloud.defaulttheme + - DB_HOST=postgres + - DB_PASSWORD=wirepass # Change this password! + - FORWARDED_ALLOW_IPS=* + - ELASTICSEARCH2_URL=http://elasticsearch:9200/ + - MEMCACHED_LOCATION=memcached:11211 + # Uncomment the following environment variables to enable IDM integration + #- FIWARE_IDM_SERVER=${FIWARE_IDM_SERVER} + #- SOCIAL_AUTH_FIWARE_KEY=${SOCIAL_AUTH_FIWARE_KEY} + #- SOCIAL_AUTH_FIWARE_SECRET=${SOCIAL_AUTH_FIWARE_SECRET} + volumes: + - wirecloud-data:/opt/wirecloud_instance/data + - wirecloud-static:/var/www/static + - ./settings.py:/opt/wirecloud_instance/wirecloud_instance/settings.py:ro + +volumes: + elasticsearch-data: + postgres-data: + wirecloud-data: + wirecloud-static: diff --git a/1.4/docker-compose-custom-user.yml b/1.4/docker-compose-custom-user.yml new file mode 100644 index 0000000..b599599 --- /dev/null +++ b/1.4/docker-compose-custom-user.yml @@ -0,0 +1,17 @@ +version: "3" + +services: + + wirecloud: + restart: always + image: fiware/wirecloud:dev + # If you want to use a user from the host, provide here the id of the + # user, e.g. `export WIRECLOUD_USER=$(id -u username)` + user: ${WIRECLOUD_USER:-0} + ports: + - 80:8000 + environment: + - DEBUG=True + volumes: + - ./wirecloud-data:/opt/wirecloud_instance/data + - ./wirecloud-static:/var/www/static diff --git a/1.4/docker-compose-idm.yml b/1.4/docker-compose-idm.yml new file mode 100644 index 0000000..f7d6499 --- /dev/null +++ b/1.4/docker-compose-idm.yml @@ -0,0 +1,83 @@ +version: "3" + +services: + + nginx: + restart: always + image: nginx:latest + ports: + - 80:80 + volumes: + - ./nginx.conf:/etc/nginx/nginx.conf:ro + - wirecloud-static:/var/www/static:ro + depends_on: + - wirecloud + + + postgres: + restart: always + image: postgres:latest + environment: + - POSTGRES_PASSWORD=wirepass # Change this password! + volumes: + - postgres-data:/var/lib/postgresql/data + + + elasticsearch: + restart: always + image: elasticsearch:2.4 + volumes: + - elasticsearch-data:/usr/share/elasticsearch/data + command: elasticsearch -Des.index.max_result_window=50000 + + + memcached: + restart: always + image: memcached:1 + command: memcached -m 2048m + + + mysql: + restart: always + image: mysql/mysql-server:5.7.21 + environment: + - MYSQL_ROOT_PASSWORD=idm + - MYSQL_ROOT_HOST=% + + + keyrock: + restart: always + image: fiware/idm:7.0.2 + ports: + - 3000:3000 + environment: + - DATABASE_HOST=mysql + + + wirecloud: + restart: always + image: fiware/wirecloud:dev + depends_on: + - postgres + - elasticsearch + - memcached + environment: + - DEBUG=False + # - DEFAULT_THEME=wirecloud.defaulttheme + - DB_HOST=postgres + - DB_PASSWORD=wirepass # Change this password! + - FORWARDED_ALLOW_IPS=* + - ELASTICSEARCH2_URL=http://elasticsearch:9200/ + - MEMCACHED_LOCATION=memcached:11211 + - FIWARE_IDM_SERVER=${FIWARE_IDM_SERVER} + - SOCIAL_AUTH_FIWARE_KEY=${SOCIAL_AUTH_FIWARE_KEY} + - SOCIAL_AUTH_FIWARE_SECRET=${SOCIAL_AUTH_FIWARE_SECRET} + volumes: + - wirecloud-data:/opt/wirecloud_instance/data + - wirecloud-static:/var/www/static + +volumes: + elasticsearch-data: + postgres-data: + wirecloud-data: + wirecloud-static: diff --git a/1.4/docker-compose-simple.yml b/1.4/docker-compose-simple.yml new file mode 100644 index 0000000..4703389 --- /dev/null +++ b/1.4/docker-compose-simple.yml @@ -0,0 +1,29 @@ +version: "3" + +services: + + nginx: + restart: always + image: nginx:latest + ports: + - 80:80 + volumes: + - ./nginx.conf:/etc/nginx/nginx.conf:ro + - wirecloud-static:/var/www/static:ro + depends_on: + - wirecloud + + + wirecloud: + restart: always + image: fiware/wirecloud:dev + environment: + - DEBUG=False + - FORWARDED_ALLOW_IPS=* + volumes: + - wirecloud-data:/opt/wirecloud_instance/data + - wirecloud-static:/var/www/static + +volumes: + wirecloud-data: + wirecloud-static: diff --git a/1.4/docker-compose-standalone.yml b/1.4/docker-compose-standalone.yml new file mode 100644 index 0000000..82c8104 --- /dev/null +++ b/1.4/docker-compose-standalone.yml @@ -0,0 +1,18 @@ +version: "3" + +services: + + wirecloud: + restart: always + image: fiware/wirecloud:dev + ports: + - 80:8000 + environment: + - DEBUG=True + volumes: + - wirecloud-data:/opt/wirecloud_instance/data + - wirecloud-static:/var/www/static + +volumes: + wirecloud-data: + wirecloud-static: diff --git a/1.4/docker-compose.yml b/1.4/docker-compose.yml new file mode 100644 index 0000000..32dfe15 --- /dev/null +++ b/1.4/docker-compose.yml @@ -0,0 +1,75 @@ +version: "3" + +services: + + nginx: + restart: always + image: nginx:latest + ports: + - 80:80 + volumes: + - ./nginx.conf:/etc/nginx/nginx.conf:ro + - wirecloud-static:/var/www/static:ro + depends_on: + - wirecloud + + + postgres: + restart: always + image: postgres:latest + environment: + - POSTGRES_PASSWORD=wirepass # Change this password! + volumes: + - postgres-data:/var/lib/postgresql/data + + + elasticsearch: + restart: always + image: elasticsearch:2.4 + volumes: + - elasticsearch-data:/usr/share/elasticsearch/data + command: elasticsearch -Des.index.max_result_window=50000 + + + memcached: + restart: always + image: memcached:1 + command: memcached -m 2048m + + + wirecloud: + restart: always + image: fiware/wirecloud:dev + depends_on: + - postgres + - elasticsearch + - memcached + environment: + - DEBUG=False + - LOGLEVEL=INFO + # - DEFAULT_THEME=wirecloud.defaulttheme + - DB_HOST=postgres + - DB_PASSWORD=wirepass # Change this password! + - FORWARDED_ALLOW_IPS=* + - ELASTICSEARCH2_URL=http://elasticsearch:9200/ + - MEMCACHED_LOCATION=memcached:11211 + # Uncomment the following environment variables to enable FIWARE IDM integration + #- FIWARE_IDM_SERVER=${FIWARE_IDM_SERVER} + #- SOCIAL_AUTH_FIWARE_KEY=${SOCIAL_AUTH_FIWARE_KEY} + #- SOCIAL_AUTH_FIWARE_SECRET=${SOCIAL_AUTH_FIWARE_SECRET} + # Uncomment the following environment variables to enable Keycloak IDM Integration + #- KEYCLOAK_IDM_SERVER=${KEYCLOAK_IDM_SERVER} + #- KEYCLOAK_REALM=${KEYCLOAK_REALM} + #- KEYCLOAK_KEY=${KEYCLOAK_KEY} + #- KEYCLOAK_GLOBAL_ROLE=${KEYCLOAK_GLOBAL_ROLE} + #- SOCIAL_AUTH_KEYCLOAK_KEY=${SOCIAL_AUTH_KEYCLOAK_KEY} + #- SOCIAL_AUTH_KEYCLOAK_SECRET=${SOCIAL_AUTH_KEYCLOAK_SECRET} + volumes: + - wirecloud-data:/opt/wirecloud_instance/data + - wirecloud-static:/var/www/static + +volumes: + elasticsearch-data: + postgres-data: + wirecloud-data: + wirecloud-static: diff --git a/1.4/docker-entrypoint.sh b/1.4/docker-entrypoint.sh new file mode 100755 index 0000000..4a79d21 --- /dev/null +++ b/1.4/docker-entrypoint.sh @@ -0,0 +1,45 @@ +#!/bin/bash + +set -e + +# allow the container to be started with `--user` +if [ "$(id -u)" = '0' ]; then + chown -R wirecloud data + chown -R wirecloud /var/www/static +fi + +# Real entry point +case "$1" in + initdb) + manage.py migrate --fake-initial + manage.py populate + ;; + createdefaultsuperuser) + echo "from django.contrib.auth.models import User; User.objects.create_superuser('admin', 'admin@example.com', 'admin')" | manage.py shell > /dev/null + ;; + createsuperuser) + manage.py createsuperuser + ;; + *) + manage.py collectstatic --noinput + manage.py migrate --fake-initial + manage.py populate + + # allow the container to be started with `--user` + if [ "$(id -u)" = '0' ]; then + exec gosu wirecloud /usr/local/bin/gunicorn wirecloud_instance.wsgi:application \ + --forwarded-allow-ips "${FORWARDED_ALLOW_IPS}" \ + --workers 2 \ + --bind 0.0.0.0:8000 \ + --log-file - \ + --log-level ${LOGLEVEL} + else + exec /usr/local/bin/gunicorn wirecloud_instance.wsgi:application \ + --forwarded-allow-ips "${FORWARDED_ALLOW_IPS}" \ + --workers 2 \ + --bind 0.0.0.0:8000 \ + --log-file - \ + --log-level ${LOGLEVEL} + fi + ;; +esac diff --git a/1.4/manage.py b/1.4/manage.py new file mode 100755 index 0000000..304a777 --- /dev/null +++ b/1.4/manage.py @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +cd /opt/wirecloud_instance +if [ "$(id -u)" = '0' ]; then + gosu wirecloud python manage.py $@ +else + python manage.py $@ +fi diff --git a/1.4/nginx.conf b/1.4/nginx.conf new file mode 100644 index 0000000..4debf13 --- /dev/null +++ b/1.4/nginx.conf @@ -0,0 +1,49 @@ +user nginx; +worker_processes 1; + +error_log /var/log/nginx/error.log warn; +pid /var/run/nginx.pid; + + +events { + worker_connections 1024; +} + + +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + + access_log /var/log/nginx/access.log main; + + sendfile on; + #tcp_nopush on; + + keepalive_timeout 65; + + #gzip on; + + server { + + listen 80; + server_name example.org; + client_max_body_size 20M; + charset utf-8; + + location /static { + alias /var/www/static; + } + + location / { + proxy_pass http://wirecloud:8000; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + + } +} diff --git a/1.4/settings.py b/1.4/settings.py new file mode 100644 index 0000000..1e97f07 --- /dev/null +++ b/1.4/settings.py @@ -0,0 +1,366 @@ +# -*- coding: utf-8 -*- +# Django settings for wirecloud_instance project. + +import os +from wirecloud.commons.utils.conf import load_default_wirecloud_conf +from django.urls import reverse_lazy + +DEBUG = os.environ.get("DEBUG", "False").strip().lower() == "true" +BASEDIR = os.path.dirname(os.path.abspath(__file__)) +DATADIR = os.path.join(BASEDIR, "..", "data") +load_default_wirecloud_conf(locals()) + +USE_XSENDFILE = False + +ADMINS = ( + # ('Your Name', 'your_email@example.com'), +) + +MANAGERS = ADMINS + +# Default settings +DB_USERNAME = "postgres" +DB_PASSWORD = "postgres" + +if "ELASTICSEARCH2_URL" in os.environ: + HAYSTACK_CONNECTIONS = { + 'default': { + 'ENGINE': 'wirecloud.commons.haystack_backends.elasticsearch2_backend.Elasticsearch2SearchEngine', + 'URL': os.environ['ELASTICSEARCH2_URL'], + 'INDEX_NAME': 'wirecloud', + }, + } +else: + HAYSTACK_CONNECTIONS = { + 'default': { + 'ENGINE': 'wirecloud.commons.haystack_backends.whoosh_backend.WhooshEngine', + 'PATH': os.path.join(DATADIR, 'index'), + }, + } + +# Hosts/domain names that are valid for this site; required if DEBUG is False +# See https://docs.djangoproject.com/en/1.5/ref/settings/#allowed-hosts +ALLOWED_HOSTS = os.environ.get('ALLOWED_HOSTS', '*').split() + +# Local time zone for this installation. Choices can be found here: +# http://en.wikipedia.org/wiki/List_of_tz_zones_by_name +# although not all choices may be available on all operating systems. +# In a Windows environment this must be set to your system time zone. +TIME_ZONE = 'America/Chicago' + +# Language code for this installation. All choices can be found here: +# http://www.i18nguy.com/unicode/language-identifiers.html +LANGUAGE_CODE = os.environ.get('LANGUAGE_CODE', 'en-us') + +# If you set this to False, Django will make some optimizations so as not +# to load the internationalization machinery. +USE_I18N = True + +# If you set this to False, Django will not format dates, numbers and +# calendars according to the current locale. +USE_L10N = True + +# Absolute filesystem path to the directory that will hold user-uploaded files. +# Example: "/var/www/example.com/media/" +MEDIA_ROOT = '' + +# URL that handles the media served from MEDIA_ROOT. Make sure to use a +# trailing slash. +# Examples: "http://example.com/media/", "http://media.example.com/" +MEDIA_URL = '' + +# Absolute path to the directory static files should be collected to. +# Don't put anything in this directory yourself; store your static files +# in apps' "static/" subdirectories and in STATICFILES_DIRS. +# Example: "/var/www/example.com/static/" +STATIC_ROOT = "/var/www/static" + +# Controls the absolute file path that linked static will be read from and +# compressed static will be written to when using the default COMPRESS_STORAGE. +COMPRESS_ROOT = STATIC_ROOT + +# URL prefix for static files. +# Example: "http://example.com/static/", "http://static.example.com/" +STATIC_URL = '/static/' + +# Additional locations of static files +# STATICFILES_DIRS = ( +# # Put strings here, like "/home/html/static" or "C:/www/django/static". +# # Always use forward slashes, even on Windows. +# # Don't forget to use absolute paths, not relative paths. +# ) + +# List of finder classes that know how to find static files in +# various locations. +# STATICFILES_FINDERS += ( +# 'django.contrib.staticfiles.finders.FileSystemFinder', +# 'django.contrib.staticfiles.finders.DefaultStorageFinder', +# ) + +# Default value, this value must be overwritten using one of the following +# environment variables: SECRET_KEY or SECRET_KEY_FILE +SECRET_KEY = '4&0+qo=m4yk!7hohzh&xsw=i&g_7t88*-9_^j(xi!fzm9zz^7l' + +ROOT_URLCONF = 'wirecloud_instance.urls' + +# Python dotted path to the WSGI application used by Django's runserver. +WSGI_APPLICATION = 'wirecloud_instance.wsgi.application' + +# Handle some basic settings + +## String settings +STRING_SETTINGS = ( + "CACHE_MIDDLEWARE_KEY_PREFIX", + "CSRF_COOKIE_NAME", + "DB_PASSWORD", + "DB_USERNAME", + "DEFAULT_FROM_EMAIL", + "EMAIL_HOST", + "EMAIL_HOST_PASSWORD", + "EMAIL_HOST_USER", + "FIWARE_IDM_SERVER", + "FIWARE_IDM_PUBLIC_URL", + "FORCE_SCRIPT_NAME", + "LOGOUT_REDIRECT_URL", + "SECRET_KEY", + "SESSION_COOKIE_NAME", + "SOCIAL_AUTH_FIWARE_KEY", + "SOCIAL_AUTH_FIWARE_SECRET", + "SOCIAL_AUTH_KEYCLOAK_OIDC_KEY", + "SOCIAL_AUTH_KEYCLOAK_OIDC_REALM", + "SOCIAL_AUTH_KEYCLOAK_OIDC_SECRET", + "SOCIAL_AUTH_KEYCLOAK_OIDC_URL", +) +SENSITIVE_SETTINGS = ( + "DB_PASSWORD", + "DB_USERNAME", + "EMAIL_HOST_PASSWORD", + "LOGOUT_REDIRECT_URL", + "SECRET_KEY", + "SOCIAL_AUTH_FIWARE_KEY", + "SOCIAL_AUTH_FIWARE_SECRET", + "SOCIAL_AUTH_KEYCLOAK_OIDC_KEY", + "SOCIAL_AUTH_KEYCLOAK_OIDC_SECRET", +) +for setting in STRING_SETTINGS: + if setting in SENSITIVE_SETTINGS and (setting + '_FILE') in os.environ: + filename = os.environ[setting + '_FILE'] + try: + with open(filename, 'rb') as f: + value = f.read() + except IOError as error: + print("Error reading the file ({}) pointed out by {}: {}".format(setting + '_FILE', filename, error)) + print("Ignoring it") + value = os.environ.get(setting, "").strip() + else: + value = os.environ.get(setting, "").strip() + if value != "": + locals()[setting] = value + +## Number settings +NUMBER_SETTINGS = ( + "CSRF_COOKIE_AGE", + "EMAIL_PORT", + "SESSION_COOKIE_AGE", +) +for setting in NUMBER_SETTINGS: + value = os.environ.get(setting, "").strip() + try: + locals()[setting] = int(value) + except ValueError: + pass + +## Boolean settings +BOOLEAN_SETTINGS = ( + "CSRF_COOKIE_HTTPONLY", + "CSRF_COOKIE_SECURE", + "EMAIL_USE_TLS", + "EMAIL_USE_SSL", + "SESSION_COOKIE_HTTPONLY", + "SESSION_COOKIE_SECURE", + "SOCIAL_AUTH_KEYCLOAK_OIDC_GLOBAL_ROLE", +) +for setting in BOOLEAN_SETTINGS: + value = os.environ.get(setting, "").strip() + if value != "": + locals()[setting] = value.lower() == "true" + +## Verify SSL settings +VERIFY_SETTINGS = ( + "WIRECLOUD_HTTPS_VERIFY", + "SOCIAL_AUTH_FIWARE_VERIFY_SSL", + "SOCIAL_AUTH_VERIFY_SSL", + "SOCIAL_AUTH_KEYCLOAK_OIDC_VERIFY_SSL", +) +for setting in VERIFY_SETTINGS: + value = os.environ.get(setting, "/etc/ssl/certs/ca-certificates.crt").strip() + locals()[setting] = True if value.lower() == "true" else False if value.lower() == "false" else value + +# Database configuration +# We only support postgres and sqlite3 for now +if os.environ.get("DB_HOST", "").strip() != "": + DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.postgresql_psycopg2', + 'NAME': os.environ.get("DB_NAME", "postgres"), + 'USER': DB_USERNAME, + 'PASSWORD': DB_PASSWORD, + 'HOST': os.environ["DB_HOST"], + 'PORT': os.environ.get("DB_PORT", "5432"), + }, + } +else: + DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.sqlite3', + 'NAME': os.path.join(DATADIR, 'wirecloud.db'), + 'USER': '', + 'PASSWORD': '', + 'HOST': '', + 'PORT': '', + }, + } + +# Email configuration +if "DEFAULT_FROM_EMAIL" not in locals(): + DEFAULT_FROM_EMAIL = locals().get("EMAIL_HOST_USER", "webmaster@localhost") + +# FIWARE & Keycloak configuration +IDM_AUTH = 'fiware' if "FIWARE_IDM_SERVER" in locals() and "SOCIAL_AUTH_FIWARE_KEY" in locals() and "SOCIAL_AUTH_FIWARE_SECRET" in locals() else None +IDM_AUTH = 'keycloak' if "SOCIAL_AUTH_KEYCLOAK_OIDC_URL" in locals() and "SOCIAL_AUTH_KEYCLOAK_OIDC_REALM" in locals() and "SOCIAL_AUTH_KEYCLOAK_OIDC_KEY" in locals() and "SOCIAL_AUTH_KEYCLOAK_OIDC_SECRET" in locals() else IDM_AUTH + +if IDM_AUTH == 'fiware': + INSTALLED_APPS += ( + 'wirecloud.fiware', + 'social_django', + 'haystack', + ) +elif IDM_AUTH == 'keycloak': + INSTALLED_APPS += ( + 'wirecloud.fiware', + 'wirecloud.keycloak', + 'social_django', + 'haystack', + ) +else: + INSTALLED_APPS += ( + 'wirecloud.oauth2provider', + 'wirecloud.fiware', + 'haystack', + ) + +# Login/logout URLs +LOGIN_URL = reverse_lazy('login') +LOGOUT_URL = reverse_lazy('wirecloud.root') +LOGIN_REDIRECT_URL = reverse_lazy('wirecloud.root') + +THEME_ACTIVE = os.environ.get("DEFAULT_THEME", "wirecloud.defaulttheme") +DEFAULT_LANGUAGE = os.environ.get("DEFAULT_LANGUAGE", 'browser') +from django.conf.global_settings import LANGUAGES + +# Configure available translations +# Filter using available translations by default +lang_filter = os.environ.get("LANGUAGES", "en es ja").split() +if len(lang_filter) > 0: + LANGUAGES = dict(LANGUAGES) + LANGUAGES = [(lang_code, LANGUAGES[lang_code]) for lang_code in lang_filter if lang_code in LANGUAGES] + +# WGT deployment dirs +CATALOGUE_MEDIA_ROOT = os.path.join(DATADIR, 'catalogue_resources') +GADGETS_DEPLOYMENT_DIR = os.path.join(DATADIR, 'widget_files') + +# Cache settings +CACHES = { + "default": {} +} + +if "MEMCACHED_LOCATION" in os.environ: + CACHES['default'] = { + 'BACKEND': 'django.core.cache.backends.memcached.PyLibMCCache', + 'LOCATION': os.environ['MEMCACHED_LOCATION'], + } +else: + CACHES['default'] = { + 'BACKEND': 'wirecloud.platform.cache.backends.locmem.LocMemCache', + 'OPTIONS': { + 'MAX_ENTRIES': 3000, + }, + } + +NOT_PROXY_FOR = ['localhost', '127.0.0.1'] + +# Allow X-Forwarded-* headers on Django (they are filtered by gunicorn depending on the value of FORWARDED_ALLOW_IPS env var) +SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https') +USE_X_FORWARDED_HOST = True +USE_X_FORWARDED_PORT = True + +# Auth configuration +if IDM_AUTH == 'fiware': + AUTHENTICATION_BACKENDS = ( + 'wirecloud.fiware.social_auth_backend.FIWAREOAuth2', + ) +elif IDM_AUTH == 'keycloak': + AUTHENTICATION_BACKENDS = ( + 'wirecloud.keycloak.social_auth_backend.KeycloakOpenIdConnect', + ) +else: + AUTHENTICATION_BACKENDS = ( + 'django.contrib.auth.backends.ModelBackend', + ) + +SOCIAL_AUTH_NO_DEFAULT_PROTECTED_USER_FIELDS = True +SOCIAL_AUTH_PROTECTED_USER_FIELDS = ('username', 'id', 'pk', 'email', 'password', 'is_active') +SOCIAL_AUTH_PIPELINE = ( + 'social_core.pipeline.social_auth.social_details', + 'social_core.pipeline.social_auth.social_uid', + 'social_core.pipeline.social_auth.auth_allowed', + 'social_core.pipeline.social_auth.social_user', + 'social_core.pipeline.user.get_username', + 'social_core.pipeline.user.create_user', + 'social_core.pipeline.social_auth.associate_user', + 'social_core.pipeline.social_auth.load_extra_data', + 'social_core.pipeline.user.user_details', +) + +if os.environ.get("SOCIAL_AUTH_FIWARE_SYNC_ORGANIZATIONS", "False").strip().lower() == "true": + SOCIAL_AUTH_PIPELINE += ('wirecloud.fiware.social_auth_backend.create_organizations',) + +if os.environ.get("SOCIAL_AUTH_FIWARE_SYNC_ROLE_GROUPS", "False").strip().lower() == "true": + SOCIAL_AUTH_PIPELINE += ('wirecloud.fiware.social_auth_backend.sync_role_groups',) + + +DATA_UPLOAD_MAX_MEMORY_SIZE = 262144000 + +LOGGING = { + 'version': 1, + 'disable_existing_loggers': False, + 'filters': { + 'require_debug_false': { + '()': 'django.utils.log.RequireDebugFalse' + }, + 'skip_unreadable_posts': { + '()': 'wirecloud.commons.utils.log.SkipUnreadablePosts', + } + }, + 'handlers': { + 'console': { + 'level': 'INFO', + 'class': 'logging.StreamHandler', + }, + 'mail_admins': { + 'level': 'ERROR', + 'filters': ['require_debug_false', 'skip_unreadable_posts'], + 'class': 'django.utils.log.AdminEmailHandler' + } + }, + 'loggers': { + '': { + 'handlers': ['console'], + }, + 'django.request': { + 'handlers': ['console', 'mail_admins'], + 'level': 'ERROR', + 'propagate': False, + }, + } +} diff --git a/1.4/tests.py b/1.4/tests.py new file mode 100644 index 0000000..368fcf9 --- /dev/null +++ b/1.4/tests.py @@ -0,0 +1,238 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Copyright (c) 2018-2020 Future Internet Consulting and Development Solutions S.L. + +import grp +import pwd +import os +import shutil +import time +import unittest +from urllib.parse import parse_qs, urlparse + +import requests +import sh + +POLL_FREQUENCY = 0.5 # How long to sleep inbetween calls to the method + + +class TimeoutException(Exception): + """ + Thrown when a command does not complete in enough time. + """ + pass + + +def wait_until_running(timeout=120): + end_time = time.time() + timeout + while True: + try: + response = requests.get("http://localhost/api/version") + if response.status_code != 502: + return + except requests.exceptions.ConnectionError: + pass + time.sleep(POLL_FREQUENCY) + if time.time() > end_time: + break + raise TimeoutException() + + +class WireCloudTests(object): + + def test_version_api_should_be_available(self): + response = requests.get("http://localhost/api/version") + self.assertEqual(response.status_code, 200) + + def test_search_engine_should_work(self): + response = requests.get("http://localhost/api/search?namespace=workspace&maxresults=1") + self.assertEqual(response.status_code, 200) + + def test_root_page_should_work(self): + response = requests.get("http://localhost/") + self.assertEqual(response.status_code, 200) + + def test_home_page_should_work(self): + response = requests.get("http://localhost/wirecloud/home") + self.assertEqual(response.status_code, 200) + + def test_should_serve_static_files(self): + response = requests.get("http://localhost/static/theme/wirecloud.defaulttheme/images/logos/header.png") + self.assertEqual(response.status_code, 200) + + def test_should_serve_translations(self): + response = requests.get("http://localhost/api/i18n/js_catalogue?language=es") + self.assertEqual(response.status_code, 200) + # Look for basic translations + self.assertIn('"Yes"', response.text) + self.assertIn('"No"', response.text) + self.assertIn('"Warning"', response.text) + + +class StandaloneTests(unittest.TestCase, WireCloudTests): + + @classmethod + def setUpClass(cls): + print("\n################################################################################\n") + print("#") + print("# Initializing standalone test case") + print("#\n", flush=True) + sh.docker_compose("-f", "docker-compose-standalone.yml", "up", d=True, remove_orphans=True, _fg=True) + wait_until_running() + print(flush=True) + + @classmethod + def tearDownClass(cls): + print() + print("#") + print("# Removing containers and volumes") + print("#\n", flush=True) + sh.docker_compose.down(remove_orphans=True, v=True, _fg=True) + print(flush=True) + + +class SimpleTests(unittest.TestCase, WireCloudTests): + + @classmethod + def setUpClass(cls): + print("\n################################################################################\n") + print("#") + print("# Initializing simple test case") + print("#\n", flush=True) + sh.docker_compose("-f", "docker-compose-simple.yml", "up", d=True, remove_orphans=True, _fg=True) + wait_until_running() + print(flush=True) + + @classmethod + def tearDownClass(cls): + print() + print("#") + print("# Removing containers and volumes") + print("#\n", flush=True) + sh.docker_compose.down(remove_orphans=True, v=True, _fg=True) + print(flush=True) + + +class ComposedTests(unittest.TestCase, WireCloudTests): + + @classmethod + def setUpClass(cls): + print("\n################################################################################\n") + print("#") + print("# Initializing composed test case") + print("#\n", flush=True) + sh.docker_compose.up(d=True, remove_orphans=True, _fg=True) + wait_until_running() + print(flush=True) + + @classmethod + def tearDownClass(cls): + print() + print("#") + print("# Removing containers and volumes") + print("#\n", flush=True) + sh.docker_compose.down(remove_orphans=True, v=True, _fg=True) + print(flush=True) + + +class ReadOnlyConfigTests(unittest.TestCase, WireCloudTests): + + @classmethod + def setUpClass(cls): + print("\n################################################################################\n") + print("#") + print("# Initializing read-only config test case") + print("#\n", flush=True) + + sh.docker_compose("-f", "docker-compose-config-file.yml", "up", d=True, remove_orphans=True, _fg=True) + wait_until_running() + print(flush=True) + + @classmethod + def tearDownClass(cls): + print() + print("#") + print("# Removing containers and volumes") + print("#\n", flush=True) + sh.docker_compose.down(remove_orphans=True, v=True, _fg=True) + print(flush=True) + + +class IDMTests(unittest.TestCase, WireCloudTests): + + @classmethod + def setUpClass(cls): + print("\n################################################################################\n") + print("#") + print("# Initializing idm test case") + print("#\n", flush=True) + + env = {} + env.update(os.environ) + env["FIWARE_IDM_SERVER"] = "https://accounts.example.com" + env["SOCIAL_AUTH_FIWARE_KEY"] = "wirecloud_test_client_id" + env["SOCIAL_AUTH_FIWARE_SECRET"] = "notused" + sh.docker_compose("-f", "docker-compose-idm.yml", "up", d=True, remove_orphans=True, _env=env, _fg=True) + wait_until_running() + print(flush=True) + + @classmethod + def tearDownClass(cls): + print() + print("#") + print("# Removing containers and volumes") + print("#\n", flush=True) + sh.docker_compose.down(remove_orphans=True, v=True, _fg=True) + print(flush=True) + + def test_login_should_redirect_to_idm(self): + response = requests.get("http://localhost/login/fiware/", allow_redirects=False) + self.assertEqual(response.status_code, 302) + location = urlparse(response.headers['Location']) + self.assertEqual(location.scheme, 'https') + self.assertEqual(location.netloc, 'accounts.example.com') + self.assertEqual(location.path, '/oauth2/authorize') + parameters = parse_qs(location.query) + self.assertEqual(parameters['client_id'], ['wirecloud_test_client_id']) + self.assertEqual(parameters['redirect_uri'], ['http://localhost/complete/fiware/']) + + +class CustomUserTests(unittest.TestCase, WireCloudTests): + + @classmethod + def setUpClass(cls): + print("\n################################################################################\n") + print("#") + print("# Initializing custom user test case") + print("#\n", flush=True) + + sh.adduser("mycustomuser", system=True, group=True, shell="/bin/bash") + uid = pwd.getpwnam("mycustomuser").pw_uid + gid = grp.getgrnam("mycustomuser").gr_gid + os.mkdir('wirecloud-data', 0o700) + os.chown('wirecloud-data', uid, gid) + os.mkdir('wirecloud-static', 0o700) + os.chown('wirecloud-static', uid, gid) + + env = {} + env.update(os.environ) + env["WIRECLOUD_USER"] = "{}".format(uid) + sh.docker_compose("-f", "docker-compose-custom-user.yml", "up", d=True, remove_orphans=True, _env=env, _fg=True) + wait_until_running() + print(flush=True) + + @classmethod + def tearDownClass(cls): + print() + print("#") + print("# Removing containers and volumes") + print("#\n", flush=True) + sh.docker_compose.down(remove_orphans=True, v=True, _fg=True) + shutil.rmtree('wirecloud-data') + shutil.rmtree('wirecloud-static') + print(flush=True) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/1.4/urls.py b/1.4/urls.py new file mode 100644 index 0000000..319cbe4 --- /dev/null +++ b/1.4/urls.py @@ -0,0 +1,56 @@ +# -*- coding: utf-8 -*- + +import os + +from django.conf import settings +from django.conf.urls import include, url +from django.contrib import admin +from django.contrib.auth import views as django_auth +from django.contrib.staticfiles.urls import staticfiles_urlpatterns + +from wirecloud.commons import authentication as wc_auth +from wirecloud.fiware import views as wc_fiware +from wirecloud.keycloak import views as wc_keycloak +import wirecloud.platform.urls + +admin.autodiscover() + +login_method = django_auth.LoginView.as_view() +logout_method = wc_auth.logout + +if settings.IDM_AUTH == 'fiware': + login_method = wc_fiware.login + +if settings.IDM_AUTH == 'keycloak': + login_method = wc_keycloak.login + logout_method = wc_keycloak.logout + +urlpatterns = ( + + # Catalogue + url(r'^catalogue/', include('wirecloud.catalogue.urls')), + + # Proxy + url(r'^cdp/', include('wirecloud.proxy.urls')), + + # Login/logout + url(r'^login/?$', login_method, name="login"), + url(r'^logout/?$', logout_method, name="logout"), + url(r'^admin/logout/?$', wc_auth.logout), + + # Admin interface + url(os.environ.get("ADMIN_URL_PATH", r'^admin/'), admin.site.urls), +) + +if settings.IDM_AUTH: + urlpatterns += (url('', include('social_django.urls', namespace='social')),) + if settings.IDM_AUTH == "keycloak": + urlpatterns += (url('', include('wirecloud.keycloak.urls')),) + +urlpatterns += wirecloud.platform.urls.urlpatterns +urlpatterns += tuple(staticfiles_urlpatterns()) + +handler400 = "wirecloud.commons.views.bad_request" +handler403 = "wirecloud.commons.views.permission_denied" +handler404 = "wirecloud.commons.views.page_not_found" +handler500 = "wirecloud.commons.views.server_error" diff --git a/README.md b/README.md index 27b7d70..1e9f9e1 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,8 @@ # Supported tags and respective `Dockerfile` links # +- [`1.4`, `1.4.0`, `FIWARE_8.0`, `latest`](https://github.com/Wirecloud/docker-wirecloud/blob/master/1.4/Dockerfile) - [`1.3`, `1.3.1`, `FIWARE_7.7.1`](https://github.com/Wirecloud/docker-wirecloud/blob/master/1.3/Dockerfile) -- [`1.2`, `latest`](https://github.com/Wirecloud/docker-wirecloud/blob/master/1.2/Dockerfile) +- [`1.2`](https://github.com/Wirecloud/docker-wirecloud/blob/master/1.2/Dockerfile) - [`dev`](https://github.com/Wirecloud/docker-wirecloud/blob/master/dev/Dockerfile) diff --git a/latest b/latest index a58941b..840ca8c 120000 --- a/latest +++ b/latest @@ -1 +1 @@ -1.3 \ No newline at end of file +1.4 \ No newline at end of file