Split httpd deps (mlrun#158)

* split httpd deps

* forgot to add here

* dask requirements in a file

* oops

* missing backslash

* copy *

* oops

Dockerfile.dask

@@ -16,8 +16,7 @@ FROM python:3.6
 WORKDIR /mlrun
 
 COPY requirements.txt ./
-RUN python -m pip install -r requirements.txt
+RUN python -m pip install -r requirements.txt -r dask-requirements.txt
 
 COPY . .
 RUN python setup.py install
-RUN pip install dask==2.9.1 distributed==2.9.1 kubernetes==10.0.0 dask_kubernetes==0.10.0 kubernetes-asyncio==10.0.0 msgpack==0.6.2

Dockerfile.db-gunicorn → Dockerfile.httpd

@@ -16,10 +16,12 @@ FROM python:3.6-slim
 RUN apt-get update && apt-get install -y gcc git-core
 RUN python -m pip install --upgrade --no-cache pip
 WORKDIR /mlrun
-COPY requirements.txt ./
-RUN python -m pip install -r requirements.txt
+COPY *requirements.txt ./
+RUN python -m pip install \
+    -r dask-requirements.txt \
+    -r httpd-requirements.txt \
+    -r requirements.txt
 COPY . .
-RUN pip install dask distributed kubernetes==10.0.0 dask_kubernetes==0.10.0 kubernetes-asyncio==10.0.0
 ENV MLRUN_httpdb__dirpath=/mlrun/db
 ENV MLRUN_httpdb__port=8080
 VOLUME /mlrun/db

Dockerfile.test

@@ -27,6 +27,9 @@ RUN add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/debia
 RUN apt update && apt-get install -y docker-ce-cli
 RUN python -m pip install --upgrade --no-cache pip
 WORKDIR /mlrun
-COPY requirements.txt dev-requirements.txt ./
-RUN pip install -r requirements.txt -r dev-requirements.txt
+COPY *requirements.txt ./
+RUN pip install \
+    -r dev-requirements.txt \
+    -r httpd-requirements.txt \
+    -r requirements.txt
 COPY . .

MANIFEST.in

@@ -15,10 +15,12 @@
 include Dockerfile*
 include Makefile
 include README.md
+include *requirements.txt
 recursive-include examples *
 recursive-include hack *
 recursive-include mlrun/db/sql *.sql
 recursive-include tests *
 
 recursive-exclude tests/test_results *
 recursive-exclude examples/.ipynb_checkpoints *
+recursive-exclude * *.pyc

Makefile

@@ -44,17 +44,10 @@ run-httpd:
 	python -m mlrun db
 
 .PHONY: docker-db
-docker-db:
+docker-httpd:
 	docker build \
-	    -f Dockerfile.db \
-	    --tag mlrun/mlrun-db \
-	    .
-
-.PHONY: docker-db-gunicorn
-docker-db-gunicorn:
-	docker build \
-	    -f Dockerfile.db-gunicorn \
-	    --tag mlrun/mlrun-db-gunicorn \
+	    -f Dockerfile.httpd \
+	    --tag mlrun/mlrun-httpd \
 	    .
 
 .PHONY: circleci

dask-requirements.txt

@@ -0,0 +1,6 @@
+dask==2.9.1
+dask_kubernetes==0.10.0
+distributed==2.9.1
+kubernetes-asyncio==10.0.0
+kubernetes==10.0.0
+msgpack==0.6.2

hack/local/Dockerfile.jupy

@@ -1,6 +1,5 @@
 FROM jupyter/scipy-notebook
-RUN pip install mlrun
-RUN pip install dask==2.9.1 distributed==2.9.1 kubernetes==10.0.0 dask_kubernetes==0.10.0 kubernetes-asyncio==10.0.0 msgpack==0.6.2
+RUN pip install mlrun[dask]
 copy ./README.md /home/jovyan/
 copy ./examples /home/jovyan/examples/
 

httpd-requirements.txt

@@ -0,0 +1,4 @@
+Flask>=1.1.1
+croniter==0.3.31
+gevent==1.4.0
+gunicorn==19.9.0

requirements.txt

@@ -1,11 +1,7 @@
-Flask>=1.1.1
 GitPython>=2.1.0
 aiohttp>=3.5.0
 boto3>=1.9
 click>=7.0
-croniter==0.3.31
-gevent==1.4.0
-gunicorn==19.9.0
 kfp>=0.1.29
 nest-asyncio>=1.0.0
 nuclio-jupyter>=0.8.2

setup.py

@@ -42,6 +42,8 @@ def load_deps(path):
 
 install_requires = list(load_deps('requirements.txt'))
 tests_require = list(load_deps('dev-requirements.txt'))
+httpd_deps = list(load_deps('httpd-requirements.txt'))
+dask_deps = list(load_deps('dask-requirements.txt'))
 
 
 setup(
@@ -57,6 +59,10 @@ def load_deps(path):
     packages=['mlrun', 'mlrun.runtimes', 'mlrun.db',
               'mlrun.platforms', 'mlrun.projects'],
     install_requires=install_requires,
+    extra_requires={
+        'httpd': httpd_deps,
+        'dask': dask_deps,
+    },
     classifiers=[
         'Development Status :: 4 - Beta',
         'Intended Audience :: Developers',

tests/Dockerfile.test-nb

@@ -16,8 +16,11 @@ FROM python:3.6-slim
 RUN apt-get update && apt-get install -y g++ make git-core
 RUN python -m pip install --upgrade --no-cache pip
 WORKDIR /mlrun
-COPY requirements.txt dev-requirements.txt ./
-RUN python -m pip install -r requirements.txt -r dev-requirements.txt
+COPY *requirements.txt ./
+RUN python -m pip install \
+    -r dev-requirements.txt \
+    -r httpd-requirements.txt \
+    -r requirements.txt
 COPY . .
 ENV PYTHONPATH=/mlrun
 {args}

tests/test_docker.py

@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from contextlib import contextmanager
 from os import environ
 from subprocess import PIPE, run
 from uuid import uuid4
@@ -27,39 +28,26 @@
 should_run = (not in_docker) and is_ci
 
 
-@pytest.fixture
-def build_docker():
-    tag = f'mlrun/test-{uuid4().hex}'
-    cid = None
+@contextmanager
+def clean_docker(cmd, tag):
+    yield
+    run(['docker', cmd, '-f', tag])
 
-    def build(dockerfile):
-        nonlocal cid
 
-        cmd = ['docker', 'build', '-f', dockerfile, '-t', tag, '.']
-        run(cmd, cwd=prj_dir, check=True)
+@pytest.mark.skipif(not should_run, reason='in docker container or not CI')
+def test_docker():
+    tag = f'mlrun/test-{uuid4().hex}'
+    cid = None
 
+    cmd = ['docker', 'build', '-f', 'Dockerfile.httpd', '-t', tag, '.']
+    run(cmd, cwd=prj_dir, check=True)
+    with clean_docker('rmi', tag):
         port = 8080
         cmd = ['docker', 'run', '--detach', '-p', f'{port}:{port}', tag]
         out = run(cmd, stdout=PIPE, check=True)
         cid = out.stdout.decode('utf-8').strip()
-
-        url = f'http://localhost:{port}/api/healthz'
-        timeout = 30
-        assert wait_for_server(url, timeout), \
-            f'server failed to start after {timeout} seconds, url={url}'
-        return cid
-
-    yield build
-
-    if cid is not None:
-        run(['docker', 'rm', '-f', cid])
-    run(['docker', 'rmi', '-f', tag])
-
-
-dockerfiles = ['Dockerfile.db', 'Dockerfile.db-gunicorn']
-
-
-@pytest.mark.skipif(not should_run, reason='in docker container or not CI')
-@pytest.mark.parametrize('dockerfile', dockerfiles)
-def test_docker(build_docker, dockerfile):
-    build_docker(dockerfile)
+        with clean_docker('rm', cid):
+            url = f'http://localhost:{port}/api/healthz'
+            timeout = 30
+            assert wait_for_server(url, timeout), \
+                f'server failed to start after {timeout} seconds, url={url}'

tests/test_httpdb.py

@@ -30,7 +30,7 @@
 root = Path(__file__).absolute().parent.parent
 Server = namedtuple('Server', 'url conn')
 
-docker_tag = 'mlrun/test-db-gunicorn'
+docker_tag = 'mlrun/test-httpd-gunicorn'
 
 
 def free_port():
@@ -87,7 +87,7 @@ def create(env_config=None):
         env_config = {} if env_config is None else env_config
         cmd = [
             'docker', 'build',
-            '-f', 'Dockerfile.db-gunicorn',
+            '-f', 'Dockerfile.httpd',
             '--tag', docker_tag,
             '.',
         ]