From 2d8f209bf00b3d00e7afdadb476472441c59b299 Mon Sep 17 00:00:00 2001 From: lazaa32 Date: Thu, 20 Dec 2018 16:32:10 +1300 Subject: [PATCH 1/4] Updated configuration --- pywps.cfg | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pywps.cfg b/pywps.cfg index 01be2c2..753bd9d 100644 --- a/pywps.cfg +++ b/pywps.cfg @@ -29,18 +29,18 @@ url=http://localhost:5000/wps outputurl=http://localhost:5000/outputs/ outputpath=outputs workdir=workdir -wd_inp_subdir=inputs -wd_out_subdir=outputs maxprocesses=10 parallelprocesses=2 +allowedinputpaths=/pywps-flask/data [processing] -mode=docker +mode=default port_min=5050 port_max=5070 -docker_img=container +docker_img=pywps dckr_inp_dir=/pywps-flask/data dckr_out_dir=/pywps-flask/outputs +sleep_secs=5 [logging] level=INFO From d189566bb0b9bd3405dbbfe8ea4e7a77cb15e8ab Mon Sep 17 00:00:00 2001 From: lazaa32 Date: Thu, 20 Dec 2018 18:25:14 +1300 Subject: [PATCH 2/4] Docker extension for pywps-flask --- README.rst | 35 +++++++++---- docker/isolation/Dockerfile | 30 +++++------ requirements.txt | 3 +- .../execute_buffer_post_referenceoutput.xml | 2 +- tests/common.py | 6 +-- tests/test_execute.py | 50 +++++++++++++++++-- 6 files changed, 90 insertions(+), 36 deletions(-) diff --git a/README.rst b/README.rst index 1ebd0b0..3a8face 100644 --- a/README.rst +++ b/README.rst @@ -1,24 +1,39 @@ PyWPS demo with docker support ============================== -Clone pywps-flask, install libraries, build docker image (due to GDAL compiling lasts quite long):: +Install Docker. For detailed instruction see Docker `docs `_. +Check if Docker-engine is running.:: - $ pip3 install -r requirements.txt - $ cd pywps-flask/docker/alpine/flask - $ docker build -t pywps_container . + $ sudo apt install docker-ce + $ sudo systemctl status docker + +Clone pywps-flask, switch to docker_extension branch:: + $ git clone https://github.com/lazaa32/pywps-flask.git + $ git checkout docker_extension -Clone pywps and OWSLib:: +Install libraries, build docker image (due to GDAL compiling lasts quite long):: - $ git clone https://github.com/lazaa32/pywps.git - $ git clone https://github.com/lazaa32/OWSLib.git + $ pip3 install -r requirements.txt + $ cd pywps-flask/docker/isolation + $ docker build -t pywps . -Set PYTHONPATH:: +Check ``pywps.cfg``, set mode to ``docker`` and docker image name to ``pywps``:: - $ export PYTHONPATH=$PYTHONPATH:$PWD/OWSLib:$PWD/pywps + mode=docker + docker_img=pywps Run server:: - python3 demo.py + $ cd pywps-flask + $ python3 demo.py + +Send example POST request:: + + $ curl -X POST -d @static/requests/execute_buffer_async_reference.xml http://localhost:5000/wps + +You should get response with ``ProcessAccepted`` status code. During execution check whether a container was created:: + + $ docker ps -a PyWPS example service diff --git a/docker/isolation/Dockerfile b/docker/isolation/Dockerfile index ac97dcb..060844b 100644 --- a/docker/isolation/Dockerfile +++ b/docker/isolation/Dockerfile @@ -17,25 +17,24 @@ RUN apk update && apk add --no-cache \ libxslt-dev \ linux-headers \ expat \ - expat-dev + expat-dev \ + g++ \ + libstdc++ \ + make \ + swig - -RUN apk --update --no-cache add g++ libstdc++ make swig - -# Xerces -RUN wget http://www.apache.org/dist/xerces/c/3/sources/xerces-c-${XERCES_VERSION}.tar.gz -O /tmp/xerces-c-${XERCES_VERSION}.tar.gz && \ - tar xvf /tmp/xerces-c-${XERCES_VERSION}.tar.gz -C /tmp && \ - cd /tmp/xerces-c-${XERCES_VERSION} && \ - ./configure --prefix=/opt/xerces && \ - make -j $PROCESSOR_N && \ - make install - -# Geos RUN apk add --no-cache \ --repository http://dl-cdn.alpinelinux.org/alpine/edge/testing \ geos \ geos-dev +#Compiling Xerces +RUN wget http://www.apache.org/dist/xerces/c/3/sources/xerces-c-${XERCES_VERSION}.tar.gz -O /tmp/xerces-c-${XERCES_VERSION}.tar.gz && \ + tar xvf /tmp/xerces-c-${XERCES_VERSION}.tar.gz -C /tmp && \ + cd /tmp/xerces-c-${XERCES_VERSION} && \ + LDFLAGS="-s" ./configure --prefix=/usr/local/src/xerces && \ + make -j $PROCESSOR_N install + # Install GDAL RUN wget http://download.osgeo.org/gdal/${GDAL_VERSION}/gdal-${GDAL_VERSION}.tar.gz -O /tmp/gdal.tar.gz && \ tar xzf /tmp/gdal.tar.gz -C /tmp && \ @@ -47,12 +46,13 @@ RUN cd /tmp/gdal-${GDAL_VERSION}/swig/python \ && python3 setup.py build \ && python3 setup.py install -RUN git clone https://github.com/geopython/pywps-flask.git +RUN rm -rf /var/cache/apk/* + +RUN git clone https://github.com/lazaa32/pywps-flask.git WORKDIR /pywps-flask RUN pip3 install -r requirements.txt - EXPOSE 5000 ENTRYPOINT ["/usr/bin/python3", "demo.py","-a"] diff --git a/requirements.txt b/requirements.txt index 93b2b3a..6088a05 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,4 +9,5 @@ Shapely Werkzeug SQLAlchemy psutil --e git+https://github.com/geopython/pywps.git@master#egg=pywps-master +docker +-e git+https://github.com/lazaa32/pywps.git@docker_extension#egg=pywps diff --git a/static/requests/execute_buffer_post_referenceoutput.xml b/static/requests/execute_buffer_post_referenceoutput.xml index 8729262..753591c 100644 --- a/static/requests/execute_buffer_post_referenceoutput.xml +++ b/static/requests/execute_buffer_post_referenceoutput.xml @@ -35,7 +35,7 @@ - + buff_out diff --git a/tests/common.py b/tests/common.py index 5b8ac37..f657927 100644 --- a/tests/common.py +++ b/tests/common.py @@ -50,9 +50,5 @@ def validate(url, schema, post_data=None): schema = get_schema(schema) - if schema.validate(body_doc): - return True - else: - print(body) - return False + return schema.validate(body_doc) diff --git a/tests/test_execute.py b/tests/test_execute.py index 1497c02..9e9b935 100644 --- a/tests/test_execute.py +++ b/tests/test_execute.py @@ -43,8 +43,7 @@ def setUp(self): def test_valid(self): "POST Execute request" - validate(self.url, self.schema_url, self.request_data) - + assert validate(self.url, self.schema_url, self.request_data) #def test_valid_lineage(self): @@ -70,7 +69,6 @@ def _get_response(self, request): return response_doc - def test_sync(self): request = self._get_request('http://localhost:5000/static/requests/execute_buffer_post.xml') response = self._get_response(request) @@ -100,7 +98,50 @@ def test_sync(self): 'wps:ComplexData/ogr:FeatureCollection', namespaces=NAMESPACES)) + def test_sync_reference(self): + request = self._get_request('http://localhost:5000/static/requests/execute_buffer_sync_reference.xml') + response = self._get_response(request) + + self.assertEqual( + response.xpath('//wps:ExecuteResponse/wps:Process/ows:Identifier', + namespaces=NAMESPACES)[0].text, 'buffer') + + self.assertEqual( + response.xpath( + '//wps:ExecuteResponse/wps:Status/wps:ProcessSucceeded', + namespaces=NAMESPACES)[0].text, + 'PyWPS Process GDAL Buffer process finished') + + self.assertEqual(len(response.xpath( + '//wps:ExecuteResponse/wps:ProcessOutputs/wps:Output', + namespaces=NAMESPACES)), 1) + + self.assertEqual(response.xpath( + '//wps:ExecuteResponse/wps:ProcessOutputs/' + 'wps:Output/wps:Reference', + namespaces=NAMESPACES)[0].get('mimeType'), + 'application/gml+xml') + + self.assertTrue(response.xpath( + '//wps:ExecuteResponse/wps:ProcessOutputs/wps:Output/' + 'wps:Reference', + namespaces=NAMESPACES)) + + data_href = response.xpath( + '//wps:ExecuteResponse/wps:ProcessOutputs/wps:Output/' + 'wps:Reference', + namespaces=NAMESPACES)[0].get('href') + + data = self._get_request(data_href) + data_doc = etree.fromstring(data) + + self.assertTrue(data_doc.xpath('//ogr:FeatureCollection', + namespaces=NAMESPACES)) + def test_async(self): + """ + This test fails for Docker. Docker supports only referenced output in response. + """ request = self._get_request('http://localhost:5000/static/requests/execute_buffer_async.xml') response = self._get_response(request) @@ -164,7 +205,7 @@ def test_async_reference(self): data_href = status_doc.xpath( '//wps:ExecuteResponse/wps:ProcessOutputs/wps:Output/' 'wps:Reference', - namespaces=NAMESPACES)[0].get('{http://www.w3.org/1999/xlink}href') + namespaces=NAMESPACES)[0].get('href') data = self._get_request(data_href) data_doc = etree.fromstring(data) @@ -172,6 +213,7 @@ def test_async_reference(self): self.assertTrue(data_doc.xpath('//ogr:FeatureCollection', namespaces=NAMESPACES)) + def load_tests(loader=None, tests=None, pattern=None): if not loader: loader = unittest.TestLoader() From 5a7660f7373c0aa4609eab514622f0aee71e5e1b Mon Sep 17 00:00:00 2001 From: lazaa32 Date: Thu, 20 Dec 2018 18:30:42 +1300 Subject: [PATCH 3/4] Requests added for test purposes --- static/requests/execute_buffer_async.xml | 57 +++++++++++++++++++ .../execute_buffer_async_reference.xml | 57 +++++++++++++++++++ .../execute_buffer_sync_reference.xml | 44 ++++++++++++++ 3 files changed, 158 insertions(+) create mode 100644 static/requests/execute_buffer_async.xml create mode 100644 static/requests/execute_buffer_async_reference.xml create mode 100644 static/requests/execute_buffer_sync_reference.xml diff --git a/static/requests/execute_buffer_async.xml b/static/requests/execute_buffer_async.xml new file mode 100644 index 0000000..fb1d0ef --- /dev/null +++ b/static/requests/execute_buffer_async.xml @@ -0,0 +1,57 @@ + + + buffer + + + poly_in + + + + + + + 0.04853540200402788 + 0.01369376726825339 + + + 0.04853540200402788 + 0.01369376726825339 + + + + + + + + 0.048535402004028,0.013693767268253 + + + 2 + + + + + + + + buffer + + 1 + + + + + + + buff_out + + + + diff --git a/static/requests/execute_buffer_async_reference.xml b/static/requests/execute_buffer_async_reference.xml new file mode 100644 index 0000000..a287965 --- /dev/null +++ b/static/requests/execute_buffer_async_reference.xml @@ -0,0 +1,57 @@ + + + buffer + + + poly_in + + + + + + + 0.04853540200402788 + 0.01369376726825339 + + + 0.04853540200402788 + 0.01369376726825339 + + + + + + + + 0.048535402004028,0.013693767268253 + + + 2 + + + + + + + + buffer + + 1 + + + + + + + buff_out + + + + diff --git a/static/requests/execute_buffer_sync_reference.xml b/static/requests/execute_buffer_sync_reference.xml new file mode 100644 index 0000000..58539c4 --- /dev/null +++ b/static/requests/execute_buffer_sync_reference.xml @@ -0,0 +1,44 @@ + + + buffer + + + poly_in + + + + + + 0.048535402004027880.01369376726825339 + 0.048535402004027880.01369376726825339 + + + + + 0.048535402004028,0.013693767268253 + 2 + + + + + + + + buffer + + 1 + + + + + + + buff_out + + + + From 4452f7c49f3b568cc1f09dc749160c1706543e15 Mon Sep 17 00:00:00 2001 From: lazaa32 Date: Wed, 26 Dec 2018 20:57:04 +1300 Subject: [PATCH 4/4] edited README.rst --- README.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index b865e17..cdf26b5 100644 --- a/README.rst +++ b/README.rst @@ -9,12 +9,13 @@ Check if Docker-engine is running.:: Clone pywps-flask, switch to docker_extension branch:: $ git clone https://github.com/lazaa32/pywps-flask.git + $ cd ./pywps-flask $ git checkout docker_extension Install libraries, build docker image (due to GDAL compiling lasts quite long):: $ pip3 install -r requirements.txt - $ cd pywps-flask/docker/isolation + $ cd ./docker/isolation $ docker build -t pywps . Check ``pywps.cfg``, set mode to ``docker`` and docker image name to ``pywps``::