diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 0000000..98fd284 Binary files /dev/null and b/.DS_Store differ diff --git a/.dockerignore b/.dockerignore index 4e6c8c1..646c288 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,3 +1,11 @@ .git* +.dockerignore +.env +.s3env +.travis.yml +.vscode hooks* Makefile +docker-compose.yml +venv* + diff --git a/.env b/.env new file mode 100644 index 0000000..bf92f13 --- /dev/null +++ b/.env @@ -0,0 +1,28 @@ +# +# local development server configuration +# + +# the path inside the docker container that stores the GRIB and NetCDF data +SKINNYWMS_DATA_PATH=/data/ + +# the host ip the server should listen on +# set to 0.0.0.0 to listen on all ips +SKINNYWMS_HOST=0.0.0.0 + +# the http port the flask server listens on +SKINNYWMS_PORT=5000 + +# enable/disable layer dimension grouping (by elevation) +SKINNYWMS_ENABLE_DIMENSION_GROUPING=0 + +# flask env to enable auto reload on code changes +FLASK_ENV=development + +# # Configure Magics +#MAGPLUS_HOME=/usr/local/lib/python3.6/site-packages/ecmwflibs +#MAGICS_STYLE_PATH=/usr/local/lib/python3.6/site-packages/ecmwflibs/share/magics/styles/ecmwf +MAGICS_STYLES_DEBUG=on + +MAGPLUS_DEBUG=on +MAGPLUS_DEV=on +MAGICS_QUIET= \ No newline at end of file diff --git a/.github/workflows/docker-latest.yml b/.github/workflows/docker-latest.yml new file mode 100644 index 0000000..54adb38 --- /dev/null +++ b/.github/workflows/docker-latest.yml @@ -0,0 +1,49 @@ +# This is a basic workflow to help you get started with Actions + +name: Publish Latest Docker + +# Controls when the workflow will run +on: + # Triggers the workflow on push or pull request events but only for the develop branch + push: + branches: [ develop ] +# pull_request: +# branches: [ develop ] + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +# A workflow run is made up of one or more jobs that can run sequentially or in parallel +jobs: + # This workflow contains a single job called "build" + build: + # The type of runner that the job will run on + runs-on: ubuntu-latest + + # Steps represent a sequence of tasks that will be executed as part of the job + steps: + # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it + - name: Checkout Repo + uses: actions/checkout@v2 + + - name: Login to Docker Hub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKER_HUB_USERNAME }} + password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} + + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v1 + + - name: Build and push + id: docker_build + uses: docker/build-push-action@v2 + with: + context: ./ + file: ./Dockerfile + push: true + tags: ecmwf/skinnywms:latest + + - name: Image digest + run: echo ${{ steps.docker_build.outputs.digest }} diff --git a/.github/workflows/docker-release.yml b/.github/workflows/docker-release.yml new file mode 100644 index 0000000..3a6a996 --- /dev/null +++ b/.github/workflows/docker-release.yml @@ -0,0 +1,54 @@ +# This is a basic workflow to help you get started with Actions + +name: Publish Tagged Images to DockerHub + +# Controls when the workflow will run +on: + # Triggers the workflow on push or pull request events but only for the develop branch + push: + tags: + - '*' +# pull_request: +# branches: [ develop ] + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +# A workflow run is made up of one or more jobs that can run sequentially or in parallel +jobs: + # This workflow contains a single job called "build" + build: + # The type of runner that the job will run on + runs-on: ubuntu-latest + + # Steps represent a sequence of tasks that will be executed as part of the job + steps: + # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it + - name: Checkout Repo + uses: actions/checkout@v2 + + - name: Set tag + id: vars + run: echo ::set-output name=tag::${GITHUB_REF#refs/*/} + + - name: Login to Docker Hub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKER_HUB_USERNAME }} + password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} + + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v1 + + - name: Build and push + id: docker_build + uses: docker/build-push-action@v2 + with: + context: ./ + file: ./Dockerfile + push: true + tags: ecmwf/skinnywms:${{ steps.vars.outputs.tag }} + + - name: Image digest + run: echo ${{ steps.docker_build.outputs.digest }} diff --git a/.gitignore b/.gitignore index c6b6d0c..666d8df 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,6 @@ __pycache__/ meta.xml *.pyc *&* +venv/ +.venv/ +skinnywms/testdata/*/* \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 75cfc18..05f301a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,18 +1,44 @@ # Build image -# Use slim python 3 + Magics image as base -ARG MAGICS_IMAGE=ecmwf/magics:4.2.4 -FROM ${MAGICS_IMAGE} +# Use slim python 3 image as base +ARG PYTHON_IMAGE=python:3.8-slim-buster +FROM ${PYTHON_IMAGE} +# Install UWSGI +RUN set -ex \ + && apt-get update \ + && apt-get install -y --no-install-recommends \ + gcc \ + build-essential \ + libglib2.0 \ + && rm -rf /var/lib/apt/lists/* \ + && pip install uwsgi \ + && apt-get purge -y --auto-remove \ + gcc \ + build-essential + +# Install skinnywms RUN set -eux \ && mkdir -p /app/ COPY . /app/skinnywms -RUN pip install /app/skinnywms --no-dependencies +RUN pip install /app/skinnywms -# Install Python run-time dependencies. -COPY requirements.txt /root/ -RUN set -ex \ - && pip install -r /root/requirements.txt +ENV SKINNYWMS_HOST=0.0.0.0 +ENV SKINNYWMS_PORT=5000 +ENV SKINNYWMS_MOUNT=/ +ENV SKINNYWMS_DATA_PATH= +ENV SKINNYWMS_UWSGI_WORKERS=4 + +#USER nobody + +# UWSGI entrypoint +CMD uwsgi \ + --http ${SKINNYWMS_HOST}:${SKINNYWMS_PORT} \ + --master \ + --process ${SKINNYWMS_UWSGI_WORKERS} \ + --mount ${SKINNYWMS_MOUNT}=skinnywms.wmssvr:application \ + --manage-script-name \ + --uid nobody # demo application will listen at http://0.0.0.0:5000 EXPOSE 5000/tcp @@ -20,7 +46,7 @@ EXPOSE 5000/tcp # start demo # add option --path # to look for grib files in specific directory -CMD python /app/skinnywms/demo.py --host='0.0.0.0' --port=5000 +###CMD python /app/skinnywms/demo.py --host='0.0.0.0' --port=5000 # METADATA # Build-time metadata as defined at http://label-schema.org diff --git a/Makefile b/Makefile deleted file mode 100644 index c5826e1..0000000 --- a/Makefile +++ /dev/null @@ -1,36 +0,0 @@ -# test command -TEST_CMD := python /app/skinnywms/demo.py --host='0.0.0.0' --port=5000 - -# load variables from ./hooks/env -MAGICS_IMAGE := ${shell . ./hooks/env && echo $$MAGICS_IMAGE} -DATE := ${shell . ./hooks/env && echo $$DATE} -SOURCE_URL := ${shell . ./hooks/env && echo $$SOURCE_URL} -SOURCE_BRANCH := ${shell . ./hooks/env && echo $$SOURCE_BRANCH} -SOURCE_COMMIT := ${shell . ./hooks/env && echo $$SOURCE_COMMIT} -SOURCE_TAG := ${shell . ./hooks/env && echo $$SOURCE_TAG} -DOCKER_TAG := ${shell . ./hooks/env && echo $$DOCKER_TAG} -IMAGE_NAME := ${shell . ./hooks/env && echo $$IMAGE_NAME} - -all: deploy - -.PHONY: key deploy login build test push - -key: - python3 -m keyring set https://upload.pypi.org/legacy/ SylvieLamy-Thepaut - -deploy: - rm -fR dist buil skinnywms.egg-info/ - python3 setup.py sdist bdist_wheel - python3 -m twine upload --verbose --repository-url https://upload.pypi.org/legacy/ dist/* -u SylvieLamy-Thepaut - -login: - docker login - -build: - ./hooks/build - -test: - docker run --rm -p 5000:5000 -i -t ${IMAGE_NAME} ${TEST_CMD} - -push: login - @docker push ${DOCKER_REPO} \ No newline at end of file diff --git a/README.md b/README.md index 680f717..10a4df8 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,7 @@ The principle is simple: skinny will browse the directory, or the single file pa [![Docker Build Status](https://img.shields.io/docker/cloud/build/ecmwf/skinnywms.svg)](https://hub.docker.com/r/ecmwf/skinnywms) [![Docker Pulls](https://img.shields.io/docker/pulls/ecmwf/skinnywms)](https://hub.docker.com/r/ecmwf/skinnywms)[![PyPI version](https://badge.fury.io/py/skinnywms.svg)](https://badge.fury.io/py/skinnywms) [![Anaconda-Server Badge](https://anaconda.org/conda-forge/skinnywms/badges/version.svg)](https://anaconda.org/conda-forge/skinnywms) [![Anaconda-Server Badge](https://anaconda.org/conda-forge/skinnywms/badges/downloads.svg)](https://anaconda.org/conda-forge/skinnywms) + Features: --------- SkinnyWMS implements 3 of the WMS endpoints: @@ -36,13 +37,41 @@ skinny-wms --path /path/to/mydata ```bash uwsgi --http localhost:5000 --master --process 20 --mount /=skinnywms.wmssvr:application --env SKINNYWMS_DATA_PATH=/path/to/mydata ``` + + Run using Docker ---------------- +By default the docker image will start the application using uwsgi and will load and display some demo data. + +* Run the demo: +```bash +docker run --rm -p 5000:5000 -it ecmwf/skinnywms +``` +Now you can try the leaflet demo at http://localhost:5000/ + +* Run using data on your machine: +```bash +docker run --rm -p 5000:5000 -it \ + --volume=/path/to/my/data:/path/inside/the/container \ + --env SKINNYWMS_DATA_PATH=/path/inside/the/container \ + ecmwf/skinnywms +``` +Now you can access the leaflet demo with your data at http://localhost:5000/ + +* Configure different options by setting environment variables accordingly: ```bash - docker run --rm -p 5000:5000 -i -t ecmwf/skinnywms - ``` - Now you can try the leaflet demo at http://localhost:5000/ +docker run --rm -p 5000:5000 -it \ + --volume=/path/to/my/data:/path/inside/the/container \ + --env SKINNYWMS_DATA_PATH=/path/inside/the/container \ + --env SKINNYWMS_HOST=0.0.0.0 \ + --env SKINNYWMS_PORT=5000 \ + --env SKINNYWMS_MOUNT=/mymodel/ \ + --env SKINNYWMS_UWSGI_WORKERS=4 \ + --env SKINNYWMS_ENABLE_DIMENSION_GROUPING=1 \ + ecmwf/skinnywms +``` +Now you can access the ```GetCapabilities`` document for your data at http://localhost:5000/mymodel/wms?request=GetCapabilities Installation @@ -67,6 +96,13 @@ Limitations: ------------ - SkinnyWMS will perform better on well formatted and documented NetCDF and GRIB. +- grib fields containing corresponding wind components u,v need to be placed together in a single grib file in order to be displayed as vectors/wind barbs in SkinnyWMS. You can combine multiple grib files into a single file using ecCodes ``grib_copy`` (included in the docker image), e.g.: +```bash +grib_copy input_wind_u_component.grb2 input_wind_v_component.grib2 output_wind_u_v_combined.grb2 +``` + +- The time and elevation dimension implementations follow [OGC Met Ocean DWG WMS 1.3 Best Practice for using Web Map Services (WMS) with Time-Dependent or Elevation-Dependent Data](https://external.ogc.org/twiki_public/MetOceanDWG/MetOceanWMSBPOnGoingDrafts). To enable dimension grouping (disabled by default) set the environment variable ``SKINNYWMS_ENABLE_DIMENSION_GROUPING=1`` + - development stage: **Alpha**, @@ -95,6 +131,14 @@ Magics is available on github https://github.com/ecmwf/magics Note that *Magics* support for the Windows operating system is experimental. +Start up a local development environment (Docker) +----------------------------------------- + +Make sure you have ``Docker`` and ``docker-compose`` installed. Then run: +```bash +docker-compose up +``` +This will build a dev image and start up a local flask development server (with automatic reload on code changes) at http://localhost:5000 based on the configuration stored in [docker-compose.yml](./docker-compose.yml) and [.env](./.env) and by default try to load all GRIB and NetCDF data stored in [skinnywms/testdata](./skinnywms/testdata). Contributing diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..41fb4de --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,31 @@ +version: '3.7' + +services: + skinnywms: + build: + context: ./ + dockerfile: Dockerfile + args: + - PYTHON_IMAGE=python:3.8-slim-buster + - http_proxy + - https_proxy + - no_proxy + volumes: + - '.:/app/skinnywms:ro' + - './skinnywms/testdata:/data:ro' + # override the default command to run flask app without uwgsi + # to make use of automatic reload on code changes for development purposes + command: + - bash + - -c + - 'python /app/skinnywms/demo.py --host=${SKINNYWMS_HOST} --port=${SKINNYWMS_PORT} --path=${SKINNYWMS_DATA_PATH}' + restart: always + ports: + - 5000:5000 + env_file: + - ./.env + # environment: + # - SKINNYWMS_DATA_PATH=${SKINNYWMS_DATA_PATH} + # - SKINNYWMS_HOST=${SKINNYWMS_HOST} + # - SKINNYWMS_PORT=${SKINNYWMS_PORT} + # - FLASK_ENV=${FLASK_ENV} \ No newline at end of file diff --git a/hooks/build b/hooks/build deleted file mode 100755 index fc0c29d..0000000 --- a/hooks/build +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/bash -# This file overrides the default "build" phase of -# docker autobuild. -# -# Advanced options for Autobuild and Autotest -# see https://docs.docker.com/docker-hub/builds/advanced/ - -# Environment variables for building and testing -# Several utility environment variables are set by the build process, and are available during automated builds, automated tests, and while executing hooks. -# -# SOURCE_BRANCH: the name of the branch or the tag that is currently being tested. -# SOURCE_COMMIT: the SHA1 hash of the commit being tested. -# COMMIT_MSG: the message from the commit being tested and built. -# DOCKER_REPO: the name of the Docker repository being built. -# DOCKERFILE_PATH: the dockerfile currently being built. -# DOCKER_TAG: the Docker repository tag being built. -# IMAGE_NAME: the name and tag of the Docker repository being built. (This variable is a combination of DOCKER_REPO:DOCKER_TAG.) - -# Custom environment variables: -# MAGICS_IMAGE="eduardrosert/magics:version-4.2.0" #: base image for this image -# SOURCE_URL="https://github.com/ecmwf/skinnywms" #: (metadata) source url -# DATE=$(date -u +%Y-%m-%dT%H:%M:%SZ) #: (metadata) build time stamp - -# load environment variables from env file -. "$(dirname $0)/env" - -docker build \ - --build-arg MAGICS_IMAGE=${MAGICS_IMAGE} \ - --build-arg BUILD_DATE=${DATE} \ - --build-arg VCS_URL=${SOURCE_URL} \ - --build-arg VCS_REF=${SOURCE_COMMIT} \ - --build-arg VERSION=${DOCKER_TAG} \ - --file ${DOCKERFILE_PATH} \ - --tag ${IMAGE_NAME} \ - . \ No newline at end of file diff --git a/hooks/env b/hooks/env deleted file mode 100755 index 0927164..0000000 --- a/hooks/env +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash -# Sets required environment variables if they are unset -# and/or overwrites predefined environment variables -# that are set by dockerhub automatic image build - -[ -n "$SOURCE_URL" ] || SOURCE_URL=https://github.com/ecmwf/skinnywms -[ -n "$DOCKER_REPO" ] || DOCKER_REPO=ecmwf/skinnywms -[ -n "$MAGICS_IMAGE" ] || MAGICS_IMAGE=ecmwf/magics:4.2.4 - -[ -n "$SOURCE_BRANCH" ] || SOURCE_BRANCH=$(git symbolic-ref -q --short HEAD) -if [[ "${SOURCE_BRANCH/-*/}" =~ ^[0-9][0-9.]*$ ]]; then - VERSION=${SOURCE_BRANCH/-*/} -fi -[ -n "$SOURCE_COMMIT" ] || SOURCE_COMMIT=$(git rev-parse -q HEAD) - -[ -n "$SOURCE_TAG" ] || SOURCE_TAG=$(git tag --sort=taggerdate | tail -1) - -if [ -z "$DOCKER_TAG" ]; then - if [ -z "$SOURCE_TAG" ]; then - # untagged git commits are tagged as 'latest' - DOCKER_TAG=latest - else - # tagged git commits produce docker images with the same tag - DOCKER_TAG="$SOURCE_TAG" - fi -fi - -[ -n "$DOCKERFILE_PATH" ] || DOCKERFILE_PATH=Dockerfile - -[ -n "$IMAGE_NAME" ] || IMAGE_NAME=${DOCKER_REPO}:${DOCKER_TAG} -[ -n "$DATE" ] || DATE=$(date -u +%Y-%m-%dT%H:%M:%SZ) diff --git a/postgres-backup.yaml b/postgres-backup.yaml new file mode 100644 index 0000000..0db7e60 --- /dev/null +++ b/postgres-backup.yaml @@ -0,0 +1,43 @@ +apiVersion: v1 +kind: Pod +metadata: + name: backup-restore + namespace: sbu-accounting-prod +spec: + containers: + - command: + - sleep + - inifinity + env: + - name: PGHOST + value: postgresql-0.postgresql + image: eccr.ecmwf.int/webdev/postgresql13:2021-04-06-001 + imagePullPolicy: IfNotPresent + name: backup + resources: + requests: + cpu: 500m + memory: 1Gi + securityContext: + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /backup + name: backup + dnsPolicy: ClusterFirst + enableServiceLinks: true + imagePullSecrets: + - name: registry-creds + restartPolicy: Never + tolerations: + - effect: NoExecute + key: node.kubernetes.io/not-ready + operator: Exists + tolerationSeconds: 300 + - effect: NoExecute + key: node.kubernetes.io/unreachable + operator: Exists + tolerationSeconds: 300 + volumes: + - name: backup + persistentVolumeClaim: + claimName: postgresql-backups diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 49b2a2c..0000000 --- a/requirements.txt +++ /dev/null @@ -1,4 +0,0 @@ -# pip requirements file -# pip freeze > requirements.txt -Flask>=1.1.1 -xarray==0.14.0 \ No newline at end of file diff --git a/setup.py b/setup.py index 8547087..8b78f81 100644 --- a/setup.py +++ b/setup.py @@ -19,7 +19,6 @@ import setuptools - def read(fname): file_path = os.path.join(os.path.dirname(__file__), fname) return io.open(file_path, encoding="utf-8").read() @@ -48,8 +47,13 @@ def read(fname): packages=setuptools.find_packages(), include_package_data=True, install_requires=[ - "ecmwflibs", + "ecmwflibs>=0.4.17", + "netCDF4", + "dask[array]", + "Magics", "Flask", + "xarray", + "future-annotations", # A backport of __future__ annotations to python<3.7 ], entry_points={ "console_scripts": ["skinny-wms=skinnywms.skinny:main"], diff --git a/skinnywms/__init__.py b/skinnywms/__init__.py index 777f190..3e2f46a 100644 --- a/skinnywms/__init__.py +++ b/skinnywms/__init__.py @@ -1 +1 @@ -__version__ = "0.8.0" +__version__ = "0.9.0" diff --git a/skinnywms/data/fs.py b/skinnywms/data/fs.py index 6240ccd..35cbd76 100644 --- a/skinnywms/data/fs.py +++ b/skinnywms/data/fs.py @@ -4,10 +4,11 @@ import os import traceback import threading +from typing import Dict from skinnywms import datatypes +from skinnywms.server import WMSServer from skinnywms.fields.NetCDFField import NetCDFReader - from skinnywms.fields.GRIBField import GRIBReader __all__ = [ @@ -44,7 +45,7 @@ def load(self): ) self._loaded = True - def add_directory(self, path): + def add_directory(self, path:str): for fname in sorted(os.listdir(path)): fname = os.path.join(path, fname) if os.path.isdir(fname): @@ -54,7 +55,7 @@ def add_directory(self, path): self.add_file(fname) - def add_file(self, path): + def add_file(self, path:str): self.log.info("Scanning %s", path) try: reader = _reader(self.context, path) @@ -76,7 +77,7 @@ def as_dict(self): return d -READERS = { +READERS:Dict[bytes,datatypes.FieldReader] = { b"GRIB": GRIBReader, b"\x89HDF": NetCDFReader, b"CDF\x01": NetCDFReader, @@ -84,7 +85,7 @@ def as_dict(self): } -def _reader(context, path): +def _reader(context:WMSServer, path:str) -> datatypes.FieldReader: # GRIBReader | NetCDFReader with open(path, "rb") as f: header = f.read(4) diff --git a/skinnywms/datatypes.py b/skinnywms/datatypes.py index df6f302..23f8169 100644 --- a/skinnywms/datatypes.py +++ b/skinnywms/datatypes.py @@ -1,3 +1,6 @@ +# -*- coding: future_annotations -*- +# from __future__ import annotations +from abc import ABC, abstractmethod # see https://www.python.org/dev/peps/pep-0563/ # (C) Copyright 2012-2019 ECMWF. # # This software is licensed under the terms of the Apache Licence Version 2.0 @@ -8,7 +11,11 @@ import datetime import logging +from skinnywms.server import WMSServer from skinnywms import errors + +from typing import List, Dict + import weakref __all__ = [ @@ -68,7 +75,7 @@ def adjust_grib_plotting(self, params): class Field: - def style(self, name): + def style(self, name:str) -> str: if name == "": if self.styles: @@ -82,101 +89,318 @@ def style(self, name): raise errors.StyleNotDefined(name) + @property + def name(self) -> str: + if self._name: + return self._name + else: + return "undefined" + + @name.setter + def name(self, value:str) -> None: + self._name = value + + @property + def group_name(self) -> str: + if self._group_name: + return self._group_name + else: + return self.name # fallback to name if unset + + @group_name.setter + def group_name(self, value:str) -> None: + self._group_name = value + + @property + def title(self) -> str: + if self._title: + return self._title + else: + return "undefined" + + @title.setter + def title(self, value:str) -> None: + self._title = value + + @property + def group_title(self) -> str: + if self._group_title: + return self._group_title + else: + return self.title # fallback to title if unset + + @group_title.setter + def group_title(self, value:str) -> None: + self._group_title = value + + @property + def companion(self) -> Field: + if self._companion: + return self._companion + else: + return None + + @companion.setter + def companion(self, value:Field) -> Field: + self._companion = value + + +class FieldReader(ABC): + """Get WMS layers (fields) from a file.""" + + def __init__(self, context:WMSServer, path:str) -> None: + self._context = context + self._path = path + + @property + def context(self) -> WMSServer: + return self._context + + @context.setter + def context(self, context:WMSServer) -> None: + self._context = weakref.ref(context) + + @property + def path(self) -> str: + return self._path + + @path.setter + def path(self, path:str) -> None: + self._path = path + + @abstractmethod + def get_fields(self) -> List[Field]: + """Returns a list of wms layers (fields) + + :raises NotImplementedError: [description] + :return: a list of wms layers (fields) + :rtype: List[Field] + """ + raise NotImplementedError() class Layer: - def __init__(self, name, title, zindex=0, description=None, keywords=[]): + def __init__(self, name:str, title:str, zindex:int=0, description:str=None, keywords:List[str]=[]): self.name = name self.title = title self.legend_title = self.title self.description = description self.zindex = zindex + + def add_field(self, field: Field) -> None: + """Adds a data field to this layer to group together data for the same parameter, + e.g. with different time or elevation dimension. + + :param field: the field to add to this layer + :type field: Field + """ + raise NotImplementedError() class Dimension: - def __init__(self, name, units, default, extent): + def __init__(self, name:str, units:str, default:str, extent:str, unitSymbol:str): self.name = name self.units = units self.default = default self.extent = extent - - -class TimeDimension: - def __init__(self, times, time_unit="hours"): - + self.unitSymbol = unitSymbol + + def add_field(self, field:Field) -> None: + """Adds a data field to this dimension to group together data for the same parameter, + that has the same dimensionality (e.g. time and elevation), but a different extent. + + Example(s): + - pressure at mean sea level at 09:00 UTC and at 10:00 UTC + - temperature at 12:00 UTC at 2m and temperature at 12:00 UTC at 10m + - soil temperature at 5mm and soil temperature at 10mm + + :param field: the field to add to this dimension + :type field: Field + """ + raise NotImplementedError() + + +class TimeDimension(Dimension): + def __init__(self, times:List[datetime.datetime]): + super(TimeDimension, self).__init__( + name = "time", + units = "ISO8601", + default = None, + extent = "", + unitSymbol=None) times = sorted(times) - - self.name = "time" - self.units = "ISO8601" self.default = times[0].isoformat() + "Z" - extent = [] - last_step = None - last_iso = None - - prev = times[0] + self.extent = TimeDimension.format_extent(times) + + def format_extent(times:List[datetime.datetime]) -> str: + """Formats a sorted list of times as WMS time extent string. - def step_diff(date1, date2, seconds): - step = date1 - date2 - return step.days * 24 + step.seconds / seconds - - if time_unit == "hours": - seconds = 3600 - unit = "H" - - if time_unit == "minutes": - seconds = 60 - unit = "M" + :param times: a sorted list of times + :type times: List[datetime.datetime] + :return: the WMS time extent string + :rtype: str + """ + extent = [] + last_delta = None + last_iso_ts = None + prev_time = times[0] + # build the textual representation of the time dimension extent for time in times: - iso = time.isoformat() + "Z" + iso_ts = time.isoformat() + "Z" - step = step_diff(time, prev, seconds) - prev = time - if step == last_step: - extent[-1] = "/".join([last_iso, iso, "PT%d%s" % (step, unit)]) + delta = time - prev_time + prev_time = time + if delta == last_delta and delta != datetime.timedelta(0): + extent[-1] = "/".join([last_iso_ts, iso_ts, TimeDimension.format_iso_8601_duration(delta)]) else: - extent.append(iso) - last_step = step - last_iso = iso - - self.extent = ",".join(extent) + extent.append(iso_ts) + last_delta = delta + last_iso_ts = iso_ts + + return ",".join(extent) + + def format_iso_8601_duration(period:datetime.timedelta) -> str: + """Converts a timedelta object into ISO 8601 duration/period format. + + :param period: the period to be converted + :return: the period in ISO 8601 duration format + :rtype: str + """ + + ret = "P" + if period.days != 0: + ret += "%dD" % period.days + + if period.seconds > 0 or period.microseconds > 0: + ret += "T" + else: + return ret # no seconds or microseconds in this period + + remainder_s = period.seconds + if remainder_s >= 3600: + ret += "%dH" % (remainder_s / 3600) # extract whole hours + remainder_s = remainder_s % 3600 + + if remainder_s >= 60: + ret += "%dM" % (remainder_s / 60) # extract whole minutes + remainder_s = remainder_s % 60 + + if remainder_s > 0 and period.microseconds == 0: + ret += "%dS" % remainder_s # only whole seconds + elif period.microseconds > 0: + ret += "%fS" % (remainder_s + period.microseconds/1000000) # floating point number + return ret + + +class ElevationDimension(Dimension): + """An elevation dimension representing vertical 'levels' as described in + https://external.ogc.org/twiki_public/pub/MetOceanDWG/MetOceanWMSBPOnGoingDrafts/12-111r1_Best_Practices_for_WMS_with_Time_or_Elevation_dependent_data.pdf + + Most common cases: + + 1) Numeric elevation values, e.g isobaric (pressure) levels in [hPa] or isometric levels in [m] + 100,200,500,1000 + + 2) Named surfaces + 1/90/1 + """ + def __init__(self, levels:List[str], default:str, units:str="computed_surface", unitSymbol:str=""): + super(ElevationDimension, self).__init__( + name = "elevation", + units = units, + default = default, + extent = ",".join(levels), + unitSymbol = unitSymbol + ) + if self.default is None and len(levels) > 0: + self.default = levels[0] + + # TODO: process list of levels to fill extent + # ... + + def add_field(self, field: Field) -> None: + pass class DataLayer(Layer): # TODO: check the time-zone of the dates.... - def __init__(self, field): - super(DataLayer, self).__init__(field.name, field.title) + def __init__(self, field:Field, group_dimensions:bool=False) -> None: + self._group_dimensions = group_dimensions + if self._group_dimensions: + super(DataLayer, self).__init__(field.group_name, field.group_title) + else: + super(DataLayer, self).__init__(field.name, field.title) assert field.time is None or isinstance(field.time, datetime.datetime) + assert field.levelist is None or isinstance(field.levelist, int) self._first = field - self._fields = {field.time: field} - - def add_field(self, field): - assert self.name == field.name - - if self.title != field.title: - raise Exception( - "Title redefined for %s [%s] => [%s]" % (self, self.title, field.title) - ) - - # Cannot have a mix of None and Dates - assert field.time is not None - assert isinstance(field.time, datetime.datetime) - - if field.time in self._fields: - LOG.info( - "Duplicate date %s in %s (%s, %s)" - % (field.time, self, field, self._fields[field.time]) - ) - # # Why are we sometimes throwing this exception .. : need to be checked - # raise Exception( - # "Duplicate date %s in %s (%s, %s)" - # % (field.time, self, field, self._fields[field.time]) - # ) + self._fields = {(field.time, field.levelist): field} - self._fields[field.time] = field + @property + def group_dimensions(self) -> bool: + """If set to 'True', fields are grouped together as layers if they differ in more than + the time dimension, e.g. in time and elevation dimension. + + :return: 'True' if dimension grouping is enabled, else 'False' + :rtype: bool + """ + return self._group_dimensions + + def add_field(self, field: Field) -> None: + if self._group_dimensions: + assert self.name == field.group_name + + if self.title != field.group_title: + raise Exception( + "Title redefined for %s [%s] => [%s]" % (self, self.title, field.group_title) + ) + + # Cannot have a mix of None and Dates + assert field.time is not None or isinstance(field.time, datetime.datetime) + assert field.levelist is None or isinstance(field.levelist, int) + + if (field.time, field.levelist) in self._fields: + LOG.info( + "Duplicate field (time: %s, elevation: %s) in %s (%s, %s)" + % (field.time, field.levelist, self, field, self._fields[(field.time, field.levelist)]) + ) + + # # Why are we sometimes throwing this exception .. : need to be checked + # raise Exception( + # "Duplicate date %s in %s (%s, %s)" + # % (field.time, self, field, self._fields[field.time]) + # ) + + self._fields[(field.time, field.levelist)] = field + + else: # don't group levels + assert self.name == field.name + + if self.title != field.title: + raise Exception( + "Title redefined for %s [%s] => [%s]" % (self, self.title, field.title) + ) + + # Cannot have a mix of None and Dates + assert field.time is not None or isinstance(field.time, datetime.datetime) + assert field.levelist is None or isinstance(field.levelist, int) + + if (field.time, field.levelist) in self._fields: + LOG.info( + "Duplicate field (time: %s, elevation: %s) in %s (%s, %s)" + % (field.time, field.levelist, self, field, self._fields[(field.time, field.levelist)]) + ) + + # # Why are we sometimes throwing this exception .. : need to be checked + # raise Exception( + # "Duplicate date %s in %s (%s, %s)" + # % (field.time, self, field, self._fields[field.time]) + # ) + + self._fields[(field.time, field.levelist)] = field @property def fixed_layer(self): @@ -184,10 +408,31 @@ def fixed_layer(self): @property def dimensions(self): - if self.fixed_layer: - return [] - else: - return [TimeDimension(self._fields.keys())] + dims = [] + if not self.fixed_layer: + times = sorted(list({l[0] for l in self._fields.keys()})) + if len(times) > 0: + dims.append(TimeDimension(times)) + elevation = sorted(list({str(l[1]) for l in self._fields.keys() if l[1] is not None})) + if len(elevation) > 0: + elev_units = "computed_surface" + unit_symbol = "" + if self._first.levtype == "pl": + # pressure levels + # TODO: see if you can get the + elev_units = "hectoPascal" + unit_symbol = "hPa" + + dims.append( + ElevationDimension( + # levels = [str(l) for l in range(100,1000,100)], + levels = elevation, + units=elev_units, + default=None, + unitSymbol=unit_symbol + ) + ) + return dims @property def styles(self): @@ -201,14 +446,27 @@ def select(self, dims): # TODO: select on more dimensions if dims is None: return self._first - time = dims.get("time", None) - LOG.info("Look up layer with %s and time %s (%s)" % (self, time, type(time))) + + time = dims.get("time", None) # try get time string + elevation = dims.get("elevation", None) # try get elevation string + LOG.info("Look up layer with %s and time %s (%s) and elevation %s (%s)" % (self, time, type(time), elevation, type(elevation))) + if time is None: - field = self._first + time = self._first.time else: + # parse string date time = datetime.datetime.strptime(time[:19], "%Y-%m-%dT%H:%M:%S") - field = self._fields[time] - return field + + if elevation is None: + elevation = [i[1] for i in self._fields.keys() if i[0] == time].pop(0) + else: + # parse int elevation + elevation = int(elevation) + + if (time,elevation) not in self._fields.keys(): + raise KeyError("(%s,%s) not found. Available combinations: %s" % (time,elevation, self._fields.keys())) + + return self._fields[(time,elevation)] def as_dict(self): return dict( @@ -218,39 +476,74 @@ def as_dict(self): class Availability: - def __init__(self, auto_add_plotter_layers=True): + def __init__(self, auto_add_plotter_layers:bool=True, group_dimensions:bool=False): self._context = None - self._layers = {} + self._layers:Dict[str,DataLayer] = {} self._aliases = {} self._auto_add_plotter_layers = auto_add_plotter_layers + self._group_dimensions=group_dimensions @property - def context(self): + def context(self) -> WMSServer: return self._context() - # @property.setter - def set_context(self, context): + # @context.setter + def set_context(self, context:WMSServer): self._context = weakref.ref(context) @property - def auto_add_plotter_layers(self): + def group_dimensions(self) -> bool: + """If set to 'True', fields are grouped together as layers if they differ in more than + the time dimension, e.g. in time and elevation dimension. + + :return: 'True' if dimension grouping is enabled, else 'False' + :rtype: bool + """ + return self._group_dimensions + + @property + def auto_add_plotter_layers(self) -> bool: return self._auto_add_plotter_layers - def add_field(self, field): + def add_field(self, field:Field) -> None: + """Adds a data field to the list of available layers. + + If a layer with the same name as the field already exists, + the field is added to the existing layer. + + :param field: the field to be added + :type field: Field + """ # TODO: Use config.... - if not self._layers: - self._aliases["default"] = field.name - if field.name in self._layers: - self._layers[field.name].add_field(field) - else: - self._layers[field.name] = DataLayer(field) + if self._group_dimensions: + if not self._layers: + self._aliases["default"] = field.group_name + + if field.group_name in self._layers: + # field with the same name already + # exists, so try to group + self._layers[field.group_name].add_field(field) + else: + self._layers[field.group_name] = DataLayer(field, group_dimensions=self.group_dimensions) + else: # don't group dimensions + if not self._layers: + self._aliases["default"] = field.name + + if field.name in self._layers: + # field with the same name already + # exists, so try to group + self._layers[field.name].add_field(field) + else: + self._layers[field.name] = DataLayer(field, group_dimensions=self.group_dimensions) def layers(self): if not self._layers: self.load() - # TODO: Sort - return [l for l in self._layers.values()] + # return a sorted list + ret = [l for l in self._layers.values()] + ret.sort(key=lambda x: x.name, reverse = False) + return ret def layer(self, name, dims): if not self._layers: @@ -279,11 +572,11 @@ def as_dict(self): class Plotter: @property - def context(self): + def context(self) -> WMSServer: return self._context() - # @property.setter - def set_context(self, context): + # @context.setter + def set_context(self, context:WMSServer): self._context = weakref.ref(context) def layers(self): @@ -320,9 +613,9 @@ def plot( class Styler: @property - def context(self): + def context(self) -> WMSServer: return self._context() - # @property.setter - def set_context(self, context): + # @context.setter + def set_context(self, context:WMSServer): self._context = weakref.ref(context) diff --git a/skinnywms/fields/GRIBField.py b/skinnywms/fields/GRIBField.py index c34313e..8351a76 100644 --- a/skinnywms/fields/GRIBField.py +++ b/skinnywms/fields/GRIBField.py @@ -6,109 +6,212 @@ # granted to it by virtue of its status as an intergovernmental organisation nor # does it submit to any jurisdiction. +from threading import settrace +from typing import Dict +import weakref +from skinnywms.grib_bindings.GribField import GribField +from skinnywms.server import WMSServer from skinnywms import datatypes import logging from skinnywms import grib_bindings +wind_companions = { "10u" : "10v", "u" : "v", "U_10m" : "V_10m", "U" : "V" } +"""A collection of wind u-component/v-component (key/value) grib shortName pairs that may be paired for better visualisation as wind barbs.""" +wind_ucomponents = set(wind_companions.keys()) +"""A collection grib shortNames that represend wind u-components""" +wind_vcomponents = set(wind_companions.values()) -companions = { "10u" : "10v" , "10v" : "10u" } - -ucomponents = ["10u"] -vcomponents = ["10v"] +# add the inverse combination as well to make matching independent of the order in which fields are processed +for key,value in list(wind_companions.items()): + wind_companions[value] = key possible_matches = {} - +"""a collection of imported fields that could have companions +which is filled during init process""" class GRIBField(datatypes.Field): log = logging.getLogger(__name__) - def __init__(self, context, path, grib, index): + def __init__(self, context: WMSServer, path: str, grib:GribField, index:int): + super(datatypes.Field,self).__init__() self.path = path self.index = index self.mars = grib.mars_request self.render = self.render_contour + self.byte_offset = grib.byte_offset + + self.metadata = grib.metadata + + self.context = context self.time = grib.valid_date + self.levtype = grib.levtype + if self.levtype == "150": self.levtype = "ml" # DWD ICON hack + self.shortName = grib.shortName + self.longName = grib.name + self.levelist = grib.levelist if hasattr(grib,"levelist") and grib.levtype != "sfc" else None # None = 2d field - if grib.levtype == "sfc": - self.name = grib.shortName - self.title = grib.name - else: - self.name = "%s_%s" % (grib.shortName, grib.levelist) - self.title = "%s at %s" % (grib.name, grib.levelist) - self.levelist = grib.levelist + self.companion = None - if self.shortName in companions: - companion = companions[self.shortName] - matches = possible_matches.get(companion, []) + # check if this field could have a companion field (e.g. wind components) + if self.shortName in wind_companions: + companion_name = wind_companions[self.shortName] + # get the possible companions that have already been found + # but haven't been matched up with other fields + possible_companions = possible_matches.get(companion_name, []) - found = False - for match in matches: - found = self.match(match) - if found : - break; - if not found: - if self.name not in possible_matches: - possible_matches[self.name] = [self] + for possible_companion in possible_companions: + # check if this companion matches + found = self.matches(possible_companion) + if found: + self.update_companions(companion=possible_companion) + break + + if self.companion is None: + # if we didn't manage to match up this field with another one + # we'll keep it for later + if self.shortName not in possible_matches: + possible_matches[self.shortName] = [self] else: - possible_matches[self.name].append(self) - - key = "style.grib.%s" % (self.name,) + # there could be multiple fields with same name (shortName) + # but with different time or level properties + # for matching, so remember them all + # as possible candidates for matching + possible_matches[self.shortName].append(self) + key = "style.grib.%s" % (self.name,) + # Optimisation self.styles = context.stash.get(key) if self.styles is None: - self.styles = context.stash[key] = context.styler.grib_styles( - self, grib, path, index - ) + self.styles = context.stash[key] = context.styler.grib_styles_from_meta(self) - def match(self, companion): - if self.time != companion.time: + @property + def metadata(self) -> Dict[str,str]: + if self.companion is None: + return self._metadata + else: + joined_meta = {} + common_keys = set(self.ucomponent._metadata.keys()).intersection(set(self.vcomponent._metadata.keys())) + for key in common_keys: + joined_meta[key] = "%s/%s" % (self.ucomponent._metadata[key], self.vcomponent._metadata[key]) + return joined_meta + + @metadata.setter + def metadata(self, metadata:Dict[str,str]): + self._metadata = metadata + + + @property + def context(self) -> WMSServer: + return self._context() + + @property + def magics_metadata(): + return { + "" + } + + @context.setter + def context(self, context:WMSServer): + self._context = weakref.ref(context) + + def matches(self, other) -> bool: + """Check if companion has matching grib properties (filename, time, levtype and levelist). + + :param companion: a grib field that can be used in combination to visualise self + :type companion: GRIBField + :return: True if companion matches the properties of self, else False + :rtype: bool + """ + if self.path != other.path: + # TODO: matching up wind components from two different grib files is not supported in magics yet + return False + if self.time != other.time: + return False + if self.levtype != other.levtype: return False - if self.levtype != companion.levtype: + if self.levelist != other.levelist: return False - if self.levtype != "sfc": - if self.levelist != companion.levelist: - return False - # Found a match WE have a vector + return True + + def update_companions(self, companion): + """Updates/overwrites self.companion with the given companion and vice versa. + Updates render function and ucomponent and vcomponent attributes for + self and companion. + + :param companion: the new companion field + :type companion: GRIBField + """ + # found a match (right now it's always wind components) + self.companion = companion + companion.companion = self + + # remember wind components u,v + self.ucomponent = self if self.shortName in wind_ucomponents else companion + self.vcomponent = companion if self.shortName in wind_ucomponents else self + + companion.ucomponent = self.ucomponent + companion.vcomponent = self.vcomponent + + # render these fields as wind self.render = self.render_wind - if self.name in ucomponents: - self.ucomponent = self.index - self.vcomponent = companion.index - companion.ucomponent = self.index - companion.vcomponent = companion.index - if self.levtype == "sfc": - self.name = "_".format(self.shortName, companion.shortName) - self.title = "/".format(self.name, companion.name) - else: - self.name = "{}_{}_%s" % (self.shortName, companion.shortName, self.levelist) - self.title = "{}/{} at %s" % (self.shortName, companion.shortName, self.levelist) - + companion.render = companion.render_wind + + key = "style.grib.%s" % (self.name,) + self.styles = self.context.stash[key] = self.context.styler.grib_styles_from_meta(self) + companion.styles = self.styles + + @property + def name(self) -> str: + # override getter for name + nameSuffix = "" if self.levelist is None else "@%s_%s" % (self.levtype, self.levelist) + + if self.companion is None: + return "%s%s" % (self.shortName, nameSuffix) else: - self.vcomponent = self.index - self.ucomponent = companion.index - companion.vcomponent = self.index - companion.ucomponent = companion.index - if self.levtype == "sfc": - self.name = "{}/{}".format(companion.shortName, self.shortName) - self.title = "{}/{}".format(companion.shortName, self.shortName) - else: - self.name = "{}_{}_{}".format(companion.shortName, self.shortName, self.levelist) - self.title = "{}/{} at {}".format(companion.shortName, self.shortName, self.levelist) - - - return True - + return "%s/%s%s" % (self.ucomponent.shortName, self.vcomponent.shortName, nameSuffix) + + @property + def group_name(self) -> str: + # override getter for name + nameSuffix = "" if self.levelist is None else "@%s" % (self.levtype) + if self.companion is None: + return "%s%s" % (self.shortName, nameSuffix) + else: + return "%s/%s%s" % (self.ucomponent.shortName, self.vcomponent.shortName, nameSuffix) + + @property + def title(self) -> str: + # override getter for title + titleSuffix = "" if self.levelist is None else " @ %s_%s" % (self.levtype, self.levelist) - def render_contour(self, context, driver, style, legend={}): + if self.companion is None: + return "%s%s" % (self.longName, titleSuffix) + else: + return "%s/%s%s" % (self.ucomponent.longName, self.vcomponent.longName,titleSuffix) + + @property + def group_title(self) -> str: + # override getter for title + titleSuffix = "" if self.levelist is None else " @ %s" % (self.levtype) + + if self.companion is None: + return "%s%s" % (self.longName, titleSuffix) + else: + return "%s/%s%s" % (self.ucomponent.longName, self.vcomponent.longName,titleSuffix) + + def render_contour(self, context, driver, style, legend={}) -> list: data = [] params = dict( - grib_input_file_name=self.path, grib_field_position=self.index + 1 + grib_input_file_name=self.path, + grib_field_position=self.byte_offset, + grib_file_address_mode="byte_offset" ) if style: @@ -119,13 +222,14 @@ def render_contour(self, context, driver, style, legend={}): return data - def render_wind(self, context, driver, style, legend={}): + def render_wind(self, context, driver, style, legend={}) -> list: data = [] - + params = dict( grib_input_file_name = self.path, - grib_wind_position_1 = self.ucomponent+1, - grib_wind_position_2 = self.vcomponent+1 + grib_wind_position_1 = self.ucomponent.byte_offset, + grib_wind_position_2 = self.vcomponent.byte_offset, + grib_file_address_mode="byte_offset" ) if style: @@ -148,29 +252,39 @@ def as_dict(self): time=self.time.isoformat() if self.time is not None else None, ) - def __repr__(self): + def __repr__(self) -> str: return "GRIBField[%r,%r,%r]" % (self.path, self.index, self.mars) + def __eq__(self, other) -> bool: + if isinstance(other, GRIBField): + return self.__hash__() == other.__hash__() + else: + return False + + def __hash__(self) -> int: + return hash(self.__repr__()) -class GRIBReader: + +class GRIBReader(datatypes.FieldReader): """Get WMS layers from a GRIB file.""" log = logging.getLogger(__name__) - def __init__(self, context, path): - self.path = path - self.context = context + def __init__(self, context:WMSServer, path:str): + super(GRIBReader,self).__init__(context=context, path=path) - def get_fields(self): + def get_fields(self) -> list: self.log.info("Scanning file: %s", self.path) - fields = [] + fields = set() for i, m in enumerate(grib_bindings.GribFile(self.path)): - fields.append(GRIBField(self.context, self.path, m, i)) + fields.add(GRIBField(self.context, self.path, m, i)) if not fields: raise Exception("GRIBReader no 2D fields found in %s", self.path) - return fields + # fields that were successfully matched with their companion fields + # carry the same layer name and thus will be removed at a later stage + return list(fields) diff --git a/skinnywms/fields/NetCDFField.py b/skinnywms/fields/NetCDFField.py index 23a67ec..f98409a 100644 --- a/skinnywms/fields/NetCDFField.py +++ b/skinnywms/fields/NetCDFField.py @@ -6,6 +6,7 @@ # granted to it by virtue of its status as an intergovernmental organisation nor # does it submit to any jurisdiction. +from skinnywms.server import WMSServer from skinnywms import datatypes import logging import datetime @@ -120,6 +121,7 @@ def __init__(self, context, path, ds, variable, slices): self.name = self.variable + self.levelist = None # if level: # self.name += '_' + str(level) @@ -147,6 +149,9 @@ def __init__(self, context, path, ds, variable, slices): if isinstance(s, TimeSlice): self.time = s.value + + # if isinstance(s, Slice): + # self.levelist = s.value if s.is_info: self.title += " (" + s.name + "=" + str(s.value) + ")" @@ -203,15 +208,14 @@ def as_dict(self): ) -class NetCDFReader: +class NetCDFReader(datatypes.FieldReader): """Get WMS layers from a NetCDF file.""" log = logging.getLogger(__name__) - def __init__(self, context, path): - self.path = path - self.context = context + def __init__(self, context:WMSServer, path:str): + super(NetCDFReader,self).__init__(context=context, path=path) self.log.info("__init__") def get_fields(self): diff --git a/skinnywms/grib_bindings/GribField.py b/skinnywms/grib_bindings/GribField.py index b571284..914b41d 100644 --- a/skinnywms/grib_bindings/GribField.py +++ b/skinnywms/grib_bindings/GribField.py @@ -8,9 +8,10 @@ # import datetime +from typing import Dict import numpy as np -from .bindings import grib_handle_delete, grib_get, grib_values +from .bindings import grib_get_metadata, grib_handle_delete, grib_get, grib_values from .bindings import ( grib_get_keys_values, grib_get_gaussian_latitudes, @@ -289,6 +290,7 @@ def coordinates(self, grib, coords, combine_order, attributes, dims): 103: SingleLevel(), # 103 sfc Specified height level above ground (m) 106: SingleLevel(), # 106 sfc Depth below land surface (m) 111: ModelLevel(), # 111 ml Eta level + 150: ModelLevel(), # 150 dwd model level } LEVEL_TYPES = {"pl": PressureLevel(), "sfc": SingleLevel(), "ml": ModelLevel()} @@ -327,6 +329,14 @@ def __init__(self, handle, path, offset): "Unsupported level type '{}' in grib {}".format(self.levtype, path) ) + @property + def metadata(self) -> Dict[str,str]: + return grib_get_metadata(self._handle) + + @property + def byte_offset(self): + return self._offset + @property def values(self): if self._values is None: diff --git a/skinnywms/grib_bindings/bindings.py b/skinnywms/grib_bindings/bindings.py index 2c45aaa..9bb1cbd 100644 --- a/skinnywms/grib_bindings/bindings.py +++ b/skinnywms/grib_bindings/bindings.py @@ -509,6 +509,28 @@ def grib_values(handle, name="values"): return array +def grib_get_metadata(handle, names:list=['centre', + 'channel', + 'level', + 'levelist', + 'levtype', + 'long_name', + 'originatingCentre', + 'param', + 'paramId', + 'parameterUnits', + 'shortName', + 'standard_name', + 'type', + 'units']): + ret = {} + for name in names: + # TODO: the names list should be retrieved directly from magics, not hardcoded + try: + ret[name] = grib_get(handle, name) + except: + pass # field name missing in grib file + return ret def grib_pl_array(handle, name="pl"): return grib_get_long_array(handle, name) diff --git a/skinnywms/plot/magics.py b/skinnywms/plot/magics.py index cd2571d..4795a60 100644 --- a/skinnywms/plot/magics.py +++ b/skinnywms/plot/magics.py @@ -13,9 +13,13 @@ import pprint import json +from numpy.lib.utils import deprecate + from Magics import macro from skinnywms import datatypes, errors +from skinnywms.fields.GRIBField import GRIBField +from skinnywms.grib_bindings.GribField import GribField __all__ = [ @@ -243,6 +247,7 @@ def mmap(self, bbox, width, height, crs_name, lon_vertical): "page_frame": "off", "skinny_mode": "on", "page_id_line": "off", + "subpage_gutter_percentage": 20., } # add extra settings for polar stereographic projection when @@ -392,6 +397,7 @@ def legend( subpage_y_length=height_cm, subpage_x_position=0.0, subpage_y_position=0.0, + subpage_gutter_percentage = 20., output_width=width, page_frame="off", page_id_line="off", @@ -485,7 +491,7 @@ class Styler(datatypes.Styler): log = logging.getLogger(__name__) - def __init__(self, user_style=None, driver=macro): + def __init__(self, user_style:str=None, driver=macro): self.user_style = None self.driver = driver if user_style: @@ -516,15 +522,15 @@ def netcdf_styles(self, field, ncvar, path, variable): return [MagicsWebStyle(**s) for s in styles.get("styles", [])] - def grib_styles(self, field, grib, path, index): + def grib_styles_from_meta(self, field:GRIBField): if self.user_style: return [MagicsWebStyle(self.user_style["name"])] with LOCK: try: styles = self.driver.wmsstyles( - self.driver.mgrib( - grib_input_file_name=path, grib_field_position=index + 1 + self.driver.minput( + input_metadata = field.metadata ) ) # Looks like they are provided in reverse order @@ -534,6 +540,40 @@ def grib_styles(self, field, grib, path, index): return [MagicsWebStyle(**s) for s in styles.get("styles", [])] + @deprecate + def grib_styles(self, field:GRIBField, grib:GribField, path:str, byte_offset:int, byte_offset_companion:int=None): + if self.user_style: + return [MagicsWebStyle(self.user_style["name"])] + + with LOCK: + try: + if byte_offset_companion: + + styles = self.driver.wmsstyles( + self.driver.mgrib( + grib_input_file_name=path, + grib_wind_position_1=byte_offset, + grib_wind_position_2=byte_offset_companion, + grib_file_address_mode="byte_offset", + grib_wind_style=True + ) + ) + + else: + styles = self.driver.wmsstyles( + self.driver.mgrib( + grib_input_file_name=path, + grib_field_position=grib.byte_offset, + grib_file_address_mode="byte_offset" + ) + ) + # Looks like they are provided in reverse order + except Exception as e: + self.log.exception("grib_styles: Error: %s", e) + styles = {} + + return [MagicsWebStyle(**s) for s in styles.get("styles", [])] + def contours(self, field, driver, style, legend={}): if self.user_style: @@ -552,10 +592,9 @@ def winds(self, field, driver, style, legend={}): if self.user_style: return driver.mwind(self.user_style) - return driver.mwind(wind_thinning_method = "automatic", wind_thinning_factor = 5) - # TODO : add automatic styling for winds - # return driver.mwinds( - # legend, - # contour_automatic_setting="style_name", - # contour_style_name=style.name, - # ) + # automatic wind styling + return driver.mwind( + legend, + wind_automatic_setting="style_name", + wind_style_name=style.name, + ) diff --git a/skinnywms/server.py b/skinnywms/server.py index 75f81ba..3662577 100644 --- a/skinnywms/server.py +++ b/skinnywms/server.py @@ -1,3 +1,4 @@ +# -*- coding: future_annotations -*- # (C) Copyright 2012-2019 ECMWF. # # This software is licensed under the terms of the Apache Licence Version 2.0 @@ -6,11 +7,15 @@ # granted to it by virtue of its status as an intergovernmental organisation nor # does it submit to any jurisdiction. +# from __future__ import annotations # see PEP 563, python 3.7+ +from typing import TYPE_CHECKING # see PEP 563, python 3.7+ +if TYPE_CHECKING: + from skinnywms.datatypes import Availability, Plotter, Styler + import logging import os import tempfile - from skinnywms import errors, protocol LOG = logging.getLogger(__name__) @@ -57,7 +62,7 @@ def create_output(self): class WMSServer: - def __init__(self, availability, plotter, styler, caching=NoCaching()): + def __init__(self, availability:Availability, plotter:Plotter, styler:Styler, caching=NoCaching()): self.availability = availability self.availability.set_context(self) diff --git a/skinnywms/templates/getcapabilities_1.1.1.xml b/skinnywms/templates/getcapabilities_1.1.1.xml index fb54658..0510a68 100644 --- a/skinnywms/templates/getcapabilities_1.1.1.xml +++ b/skinnywms/templates/getcapabilities_1.1.1.xml @@ -129,9 +129,9 @@ {{ s.name }} {{ s.title }} {{ s.description }} - + image/png - + {% endfor %} diff --git a/skinnywms/templates/getcapabilities_1.3.0.xml b/skinnywms/templates/getcapabilities_1.3.0.xml index 797dc14..a20a084 100644 --- a/skinnywms/templates/getcapabilities_1.3.0.xml +++ b/skinnywms/templates/getcapabilities_1.3.0.xml @@ -125,7 +125,12 @@ {% endif %} {% for v in l.dimensions %} - {{ v.extent }} + {% if v.unitSymbol == None %} + {{ v.extent }} + {% else %} + {{ v.extent }} + + {% endif %} {% endfor %} {% for s in l.styles %} @@ -134,9 +139,9 @@ {{ s.title }} {{ s.description }} - + image/png - + diff --git a/skinnywms/wmssvr.py b/skinnywms/wmssvr.py index 01899ba..c7e3bb5 100644 --- a/skinnywms/wmssvr.py +++ b/skinnywms/wmssvr.py @@ -20,7 +20,12 @@ demo = os.path.join(os.path.dirname(__file__), "testdata", "sfc.grib") -demo = os.environ.get("SKINNYWMS_DATA_PATH", demo) +if os.environ.get("SKINNYWMS_DATA_PATH", "") != "": + demo = os.environ.get("SKINNYWMS_DATA_PATH") + +enable_dimension_grouping = False +if os.environ.get("SKINNYWMS_ENABLE_DIMENSION_GROUPING", "") != "": + enable_dimension_grouping = (os.environ.get("SKINNYWMS_ENABLE_DIMENSION_GROUPING") == "1") parser = argparse.ArgumentParser(description="Simple WMS server") @@ -50,6 +55,11 @@ help="prefix used to pass information to magics", ) +parser.add_argument( + "--enable-dimension-grouping", + action='store_true', + help="Group together layers by more than the time dimension, e.g. by elevation" +) args = parser.parse_args() @@ -59,8 +69,13 @@ if args.user_style != "": os.environ["MAGICS_USER_STYLE_PATH"] = args.user_style + +group_dimensions = args.enable_dimension_grouping or enable_dimension_grouping + server = WMSServer( - Availability(args.path), Plotter(args.baselayer), Styler(args.user_style) + Availability(args.path, group_dimensions=group_dimensions), + Plotter(args.baselayer), + Styler(args.user_style) )