From 5e0079e1a75aaa21c78373b2eb8b97f728a64165 Mon Sep 17 00:00:00 2001 From: Sylvie Lamy-Thepaut Date: Thu, 28 Nov 2019 14:27:15 +0000 Subject: [PATCH] Building docker image --- Dockerfile | 231 +++++++++++++++++++++++++++++++++++++++++++++++ Makefile | 41 +++++++++ VERSION.cmake | 4 +- hooks/build | 33 +++++++ hooks/env | 32 +++++++ requirements.txt | 4 + 6 files changed, 343 insertions(+), 2 deletions(-) create mode 100644 Dockerfile create mode 100644 Makefile create mode 100755 hooks/build create mode 100755 hooks/env create mode 100644 requirements.txt diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000..1520e884a --- /dev/null +++ b/Dockerfile @@ -0,0 +1,231 @@ +# Slim Docker multi-stage build +# for Magics + +# Build image +FROM python:3.8.0-slim-buster as build + +RUN set -ex \ + && apt-get update + +# Install tools +RUN set -ex \ + && apt-get install --yes --no-install-suggests --no-install-recommends \ + wget \ + git + +# Install build tools. +RUN set -ex \ + && apt-get install --yes --no-install-suggests --no-install-recommends \ + bison \ + bzip2 \ + ca-certificates \ + cmake \ + curl \ + file \ + flex \ + g++-8 \ + gcc-8 \ + gfortran-8 \ + git \ + make \ + patch \ + sudo \ + swig \ + xz-utils + +RUN set -ex \ + && ln -s /usr/bin/g++-8 /usr/bin/g++ \ + && ln -s /usr/bin/gcc-8 /usr/bin/gcc \ + && ln -s /usr/bin/gfortran-8 /usr/bin/gfortran + +# Install build-time dependencies. +RUN set -ex \ + && apt-get install --yes --no-install-suggests --no-install-recommends \ + libarmadillo-dev \ + libatlas-base-dev \ + libboost-dev \ + libbz2-dev \ + libc6-dev \ + libcairo2-dev \ + libcurl4-openssl-dev \ + libeigen3-dev \ + libexpat1-dev \ + libfreetype6-dev \ + libfribidi-dev \ + libgdal-dev \ + libgeos-dev \ + libharfbuzz-dev \ + libhdf5-dev \ + libjpeg-dev \ + liblapack-dev \ + libncurses5-dev \ + libnetcdf-dev \ + libpango1.0-dev \ + libpcre3-dev \ + libpng-dev \ + libproj-dev \ + libreadline6-dev \ + libsqlite3-dev \ + libssl-dev \ + libxml-parser-perl \ + libxml2-dev \ + libxslt1-dev \ + libyaml-dev \ + zlib1g-dev + +# Install Python run-time dependencies. +COPY requirements.txt /root/ + +RUN set -ex \ + && pip install -r /root/requirements.txt + +# Install ecbuild +ARG ECBUILD_VERSION=2019.07.1 +RUN set -eux \ + && mkdir -p /src/ \ + && cd /src \ + && git clone https://github.com/ecmwf/ecbuild.git \ + && cd ecbuild \ + && git checkout ${ECBUILD_VERSION} \ + && mkdir -p /build/ecbuild \ + && cd /build/ecbuild \ + && cmake /src/ecbuild -DCMAKE_BUILD_TYPE=Release \ + && make -j4 \ + && make install + +# Install eccodes +# requires ecbuild +ARG ECCODES_VERSION=2.14.0 +RUN set -eux \ + && wget https://confluence.ecmwf.int/download/attachments/45757960/eccodes-${ECCODES_VERSION}-Source.tar.gz?api=v2 --output-document=eccodes.tar.gz \ + && tar -xf eccodes.tar.gz \ + && mkdir -p /src/ \ + && mv eccodes-${ECCODES_VERSION}-Source /src/eccodes \ + && mkdir -p /build/eccodes \ + && cd /build/eccodes \ + && /usr/local/bin/ecbuild /src/eccodes -DECMWF_GIT=https -DCMAKE_BUILD_TYPE=Release \ + && make -j4 \ + && make install \ + && /sbin/ldconfig + +# Install Magics +# requires ecbuild, eccodes + +COPY . /src/magics + +RUN set -eux \ + && mkdir -p /build/magics/ \ + && cd /build/magics \ + && /usr/local/bin/ecbuild /src/magics -DECMWF_GIT=https -DCMAKE_BUILD_TYPE=Release \ + && make -j4 \ + && make install \ + && /sbin/ldconfig + +# Remove unneeded files. +RUN set -ex \ + && find /usr/local -name 'lib*.so' | xargs -r -- strip --strip-unneeded || true \ + && find /usr/local/bin | xargs -r -- strip --strip-all || true \ + && find /usr/local/lib -name __pycache__ | xargs -r -- rm -rf + +# +# Run-time image. +# +FROM debian:stable-slim + +# Install run-time depencencies. +# Delete resources after installation +RUN set -ex \ + && apt-get update \ + && apt-get install --yes --no-install-suggests --no-install-recommends \ + ca-certificates \ + curl \ + ghostscript \ + imagemagick \ + ksh \ + libarmadillo9 \ + libbz2-1.0 \ + libcairo-gobject2 \ + libcairo-script-interpreter2 \ + libcairo2 \ + libcroco3 \ + libcurl4 \ + libexif12 \ + libexpat1 \ + libfontconfig1 \ + libfreetype6 \ + libfribidi0 \ + libgdal20 \ + libgeoip1 \ + libgeos-c1v5 \ + libgif7 \ + libgomp1 \ + libgssrpc4 \ + libharfbuzz0b \ + libhdf5-103 \ + libicu63 \ + libilmbase23 \ + libjbig0 \ + libjpeg62-turbo \ + libjs-jquery \ + liblcms2-2 \ + liblqr-1-0 \ + libncurses5 \ + libnetcdf13 \ + libopenexr23 \ + libpangocairo-1.0-0 \ + libpangoxft-1.0-0 \ + libpcre3 \ + libpcrecpp0v5 \ + libpng16-16 \ + libproj13 \ + libreadline7 \ + libsqlite3-0 \ + libssl1.1 \ + libtiff5 \ + libtiffxx5 \ + libwebp6 \ + libxft2 \ + libxml2 \ + libxslt1.1 \ + poppler-utils \ + rsync \ + zlib1g \ + && rm -rf /var/lib/apt/lists/* + +# Copy Python run-time and ECMWF softwate. +COPY --from=build /usr/local/share/eccodes/ /usr/local/share/eccodes/ +COPY --from=build /usr/local/share/magics/ /usr/local/share/magics/ +COPY --from=build /usr/local/bin/ /usr/local/bin/ +COPY --from=build /usr/local/lib/ /usr/local/lib/ + +# Ensure shared libs installed by the previous step are available. +RUN set -ex \ + && /sbin/ldconfig + +# Configure Python runtime. +ENV \ + PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 + +# Run selfcheck +CMD python -m Magics selfcheck + +# METADATA +# Build-time metadata as defined at http://label-schema.org +# --build-arg BUILD_DATE=`date -u +"%Y-%m-%dT%H:%M:%SZ"` +ARG BUILD_DATE +# --build-arg VCS_REF=`git rev-parse --short HEAD`, e.g. 'c30d602' +ARG VCS_REF +# --build-arg VCS_URL=`git config --get remote.origin.url`, e.g. 'https://github.com/eduardrosert/docker-magics' +ARG VCS_URL +# --build-arg VERSION=`git tag`, e.g. '0.2.1' +ARG VERSION +LABEL org.label-schema.build-date=$BUILD_DATE \ + org.label-schema.name="Magics" \ + org.label-schema.description="Magics is the latest generation of the ECMWF's meteorological plotting software and can be either accessed directly through its Python or Fortran interfaces or by using Metview." \ + org.label-schema.url="https://confluence.ecmwf.int/display/MAGP/Magics" \ + org.label-schema.vcs-ref=$VCS_REF \ + org.label-schema.vcs-url=$VCS_URL \ + org.label-schema.vendor="ECMWF - European Centre for Medium-Range Weather Forecasts" \ + org.label-schema.version=$VERSION \ + org.label-schema.schema-version="1.0" \ No newline at end of file diff --git a/Makefile b/Makefile new file mode 100644 index 000000000..38a2e6083 --- /dev/null +++ b/Makefile @@ -0,0 +1,41 @@ + + +# test +TEST_CMD := python -m Magics selfcheck + +# load variables from ./hooks/env +DATE := ${shell . ./hooks/env && echo $$DATE} +SOURCE_URL := ${shell . ./hooks/env && echo $$SOURCE_URL} +SOURCE_BRANCH := ${shell . ./hooks/env && echo $$SOURCE_BRANCH} +SOURCE_COMMIT := ${shell . ./hooks/env && echo $$SOURCE_COMMIT} +SOURCE_TAG := ${shell . ./hooks/env && echo $$SOURCE_TAG} +DOCKER_TAG := ${shell . ./hooks/env && echo $$DOCKER_TAG} +IMAGE_NAME := ${shell . ./hooks/env && echo $$IMAGE_NAME} + +all: build + +.PHONY: build run run-interactive test login login-user-pass push clean + +build: + ./hooks/build + +run: + @docker run --rm -i -t ${IMAGE_NAME} + +run-interactive: + @docker run --rm -i -t ${IMAGE_NAME} sh + +test: + @docker run --rm -i -t ${IMAGE_NAME} ${TEST_CMD} + +login: + @docker login + +login-user-pass: + @docker login -u ${DOCKER_USER} -p ${DOCKER_PASS} + +push: login + @docker push ${DOCKER_REPO} + +clean: + @docker rmi ${DOCKER_REPO} \ No newline at end of file diff --git a/VERSION.cmake b/VERSION.cmake index 235545ce4..ef57bda55 100644 --- a/VERSION.cmake +++ b/VERSION.cmake @@ -7,8 +7,8 @@ # does it submit to any jurisdiction. -set ( metabuilder_version 4.2.3 ) -set ( _version 4.2.3 ) +set ( metabuilder_version 4.2.4 ) +set ( _version 4.2.4 ) if ( MAGICS_BUILD ) diff --git a/hooks/build b/hooks/build new file mode 100755 index 000000000..a707f2875 --- /dev/null +++ b/hooks/build @@ -0,0 +1,33 @@ +#!/bin/bash +# This file overrides the default "build" phase of +# docker autobuild. +# +# Advanced options for Autobuild and Autotest +# see https://docs.docker.com/docker-hub/builds/advanced/ + +# Environment variables for building and testing +# Several utility environment variables are set by the build process, and are available during automated builds, automated tests, and while executing hooks. +# +# SOURCE_BRANCH: the name of the branch or the tag that is currently being tested. +# SOURCE_COMMIT: the SHA1 hash of the commit being tested. +# COMMIT_MSG: the message from the commit being tested and built. +# DOCKER_REPO: the name of the Docker repository being built. +# DOCKERFILE_PATH: the dockerfile currently being built. +# DOCKER_TAG: the Docker repository tag being built. +# IMAGE_NAME: the name and tag of the Docker repository being built. (This variable is a combination of DOCKER_REPO:DOCKER_TAG.) + +# load environment variables from env file +. "$(dirname $0)/env" + + + +docker build \ + --build-arg ECBUILD_VERSION=${ECBUILD_VERSION} \ + --build-arg ECCODES_VERSION=${ECCODES_VERSION} \ + --build-arg BUILD_DATE=${DATE} \ + --build-arg VCS_URL=${SOURCE_URL} \ + --build-arg VCS_REF=${SOURCE_COMMIT} \ + --build-arg VERSION=${DOCKER_TAG} \ + --file ${DOCKERFILE_PATH} \ + --tag ${IMAGE_NAME} \ + . diff --git a/hooks/env b/hooks/env new file mode 100755 index 000000000..eafc40f76 --- /dev/null +++ b/hooks/env @@ -0,0 +1,32 @@ +#!/bin/bash +# Sets required environment variables if they are unset +# and/or overwrites predefined environment variables +# that are set by dockerhub automatic image build + +[ -n "$SOURCE_URL" ] || SOURCE_URL=https://github.com/ecmwf/magics +[ -n "$DOCKER_REPO" ] || DOCKER_REPO=ecmwf/magics +[ -n "$ECBUILD_VERSION" ] || ECBUILD_VERSION="2019.07.1" +[ -n "$ECCODES_VERSION" ] || ECCODES_VERSION="2.15.0" + +[ -n "$SOURCE_BRANCH" ] || SOURCE_BRANCH=$(git symbolic-ref -q --short HEAD) +if [[ "${SOURCE_BRANCH/-*/}" =~ ^[0-9][0-9.]*$ ]]; then + VERSION=${SOURCE_BRANCH/-*/} +fi +[ -n "$SOURCE_COMMIT" ] || SOURCE_COMMIT=$(git rev-parse -q HEAD) + +[ -n "$SOURCE_TAG" ] || SOURCE_TAG=$(git tag --sort=taggerdate | tail -1) + +if [ -z "$DOCKER_TAG" ]; then + if [ -z "$SOURCE_TAG" ]; then + # untagged git commits are tagged as 'latest' + DOCKER_TAG=latest + else + # tagged git commits produce docker images with the same tag + DOCKER_TAG="$SOURCE_TAG" + fi +fi + +[ -n "$DOCKERFILE_PATH" ] || DOCKERFILE_PATH=Dockerfile + +[ -n "$IMAGE_NAME" ] || IMAGE_NAME=${DOCKER_REPO}:${DOCKER_TAG} +[ -n "$DATE" ] || DATE=$(date -u +%Y-%m-%dT%H:%M:%SZ) \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 000000000..de3ecf321 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,4 @@ +# pip requirements file +# pip freeze > requirements.txt +Jinja2==2.10.3 +Magics \ No newline at end of file