SQL database (mlrun#70)

SQL database

MANIFEST.in

@@ -0,0 +1,24 @@
+# Copyright 2018 Iguazio
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+include Dockerfile*
+include Makefile
+include README.md
+recursive-include examples *
+recursive-include hack *
+recursive-include mlrun/db/sql *.sql
+recursive-include tests *
+
+recursive-exclude tests/test_results *
+recursive-exclude examples/.ipynb_checkpoints *

mlrun/config.py

@@ -22,7 +22,6 @@
 """
 
 import os
-from os import path
 from collections.abc import Mapping
 from threading import Lock
 import json
@@ -46,7 +45,8 @@
     'log_level': 'ERROR',
     'httpdb': {
         'port': 8080,
-        'dirpath': path.expanduser('~/.mlrun/db'),
+        # 'dirpath': path.expanduser('~/.mlrun/db'),
+        'dsn': 'sqlite:///:memory:',
         'debug': False,
         'user': '',
         'password': '',

mlrun/db/filedb.py

@@ -13,23 +13,24 @@
 # limitations under the License.
 
 import json
-import time
-from os import path, remove, makedirs
-import yaml
 import pathlib
 from datetime import datetime, timedelta
+from os import makedirs, path, remove
+
+import yaml
 
-from ..utils import get_in, match_labels, dict_to_yaml, update_in, dict_to_json
 from ..datastore import StoreManager
+from ..lists import ArtifactList, RunList
+from ..utils import (
+    dict_to_json, dict_to_yaml, get_in, logger, match_labels, match_value,
+    update_in
+)
 from .base import RunDBError, RunDBInterface
-from ..lists import RunList, ArtifactList, FunctionList
-from ..utils import logger
 
 run_logs = 'runs'
 artifacts_dir = 'artifacts'
 functions_dir = 'functions'
 default_project = 'default'
-_missing = object()
 
 
 class FileRunDB(RunDBInterface):
@@ -78,6 +79,7 @@ def store_run(self, struct, uid, project='', iter=0):
 
     def update_run(self, updates: dict, uid, project='', iter=0):
         run = self.read_run(uid, project, iter=iter)
+        # TODO: Should we raise if run not found?
         if run and updates:
             for key, val in updates.items():
                 update_in(run, key, val)
@@ -234,7 +236,7 @@ def list_functions(self, name, project='', tag='', labels=None):
         if name:
             filepath = '{}{}/'.format(filepath, name)
             mask = '*'
-        for func, p in self._load_list(filepath, mask):
+        for func, _ in self._load_list(filepath, mask):
             if match_labels(get_in(func, 'metadata.labels', {}), labels):
                 results.append(func)
 
@@ -285,9 +287,3 @@ def _safe_del(self, filepath):
             remove(filepath)
         else:
             raise RunDBError(f'run file is not found or valid ({filepath})')
-
-
-def match_value(value, obj, key):
-    if not value:
-        return True
-    return get_in(obj, key, _missing) == value

mlrun/db/httpd.py

@@ -17,19 +17,19 @@
 from distutils.util import strtobool
 from functools import wraps
 from http import HTTPStatus
-from os import environ, path
+from os import environ
 
 from flask import Flask, jsonify, request, Response
 
 from mlrun.datastore import get_object, get_object_stat
-from mlrun.db import RunDBError
-from mlrun.db.filedb import FileRunDB
+from mlrun.db import RunDBError, RunDBInterface
+from mlrun.db.sqldb import SQLDB
 from mlrun.utils import logger, parse_function_uri, get_in
 from mlrun.config import config
 from mlrun.runtimes import RunError
 from mlrun.run import new_function, import_function
 
-_file_db: FileRunDB = None
+_db: RunDBInterface
 app = Flask(__name__)
 basic_prefix = 'Basic '
 bearer_prefix = 'Bearer '
@@ -179,7 +179,8 @@ def get_files():
     if not ctype:
         ctype = 'application/octet-stream'
 
-    return Response(body, mimetype=ctype, headers={"x-suggested-filename": filename})
+    return Response(
+        body, mimetype=ctype, headers={"x-suggested-filename": filename})
 
 
 # curl http://localhost:8080/api/filestat?schema=s3&path=mybucket/a.txt
@@ -217,16 +218,16 @@ def get_filestat():
 def store_log(project, uid):
     append = strtobool(request.args.get('append', 'no'))
     body = request.get_data()  # TODO: Check size
-    _file_db.store_log(uid, project, body, append)
+    _db.store_log(uid, project, body, append)
     return jsonify(ok=True)
 
 
 # curl http://localhost:8080/log/prj/7
 @app.route('/api/log/<project>/<uid>', methods=['GET'])
 def get_log(project, uid):
-    data = _file_db.get_log(uid, project)
+    data = _db.get_log(uid, project)
     if data is None:
-        data = _file_db.read_run(uid, project)
+        data = _db.read_run(uid, project)
         if not data:
             return json_error(HTTPStatus.NOT_FOUND,
                               project=project, uid=uid)
@@ -246,7 +247,7 @@ def store_run(project, uid):
 
     logger.debug(data)
     iter = int(request.args.get('iter', '0'))
-    _file_db.store_run(data, uid, project, iter=iter)
+    _db.store_run(data, uid, project, iter=iter)
     app.logger.info('store run: {}'.format(data))
     return jsonify(ok=True)
 
@@ -261,7 +262,7 @@ def update_run(project, uid):
         return json_error(HTTPStatus.BAD_REQUEST, reason='bad JSON body')
 
     iter = int(request.args.get('iter', '0'))
-    _file_db.update_run(data, uid, project, iter=iter)
+    _db.update_run(data, uid, project, iter=iter)
     app.logger.info('update run: {}'.format(data))
     return jsonify(ok=True)
 
@@ -271,7 +272,7 @@ def update_run(project, uid):
 @catch_err
 def read_run(project, uid):
     iter = int(request.args.get('iter', '0'))
-    data = _file_db.read_run(uid, project, iter=iter)
+    data = _db.read_run(uid, project, iter=iter)
     return jsonify(ok=True, data=data)
 
 
@@ -280,7 +281,7 @@ def read_run(project, uid):
 @catch_err
 def del_run(project, uid):
     iter = int(request.args.get('iter', '0'))
-    _file_db.del_run(uid, project, iter=iter)
+    _db.del_run(uid, project, iter=iter)
     return jsonify(ok=True)
 
 
@@ -297,7 +298,7 @@ def list_runs():
     iter = strtobool(request.args.get('iter', 'on'))
     last = int(request.args.get('last', '0'))
 
-    runs = _file_db.list_runs(
+    runs = _db.list_runs(
         name=name,
         uid=uid,
         project=project,
@@ -319,7 +320,7 @@ def del_runs():
     state = request.args.get('state', '')
     days_ago = int(request.args.get('days_ago', '0'))
 
-    _file_db.del_runs(name, project, labels, state, days_ago)
+    _db.del_runs(name, project, labels, state, days_ago)
     return jsonify(ok=True)
 
 
@@ -333,15 +334,15 @@ def store_artifact(project, uid, key):
         return json_error(HTTPStatus.BAD_REQUEST, reason='bad JSON body')
 
     tag = request.args.get('tag', '')
-    _file_db.store_artifact(key, data, uid, tag, project)
+    _db.store_artifact(key, data, uid, tag, project)
     return jsonify(ok=True)
 
 
 # curl http://localhost:8080/artifact/p1/tag/key
 @app.route('/api/artifact/<project>/<tag>/<path:key>', methods=['GET'])
 @catch_err
 def read_artifact(project, tag, key):
-    data = _file_db.read_artifact(key, tag, project)
+    data = _db.read_artifact(key, tag, project)
     return data
 
 # curl -X DELETE http://localhost:8080/artifact/p1&key=k&tag=t
@@ -353,7 +354,7 @@ def del_artifact(project, uid):
         return json_error(HTTPStatus.BAD_REQUEST, reason='missing data')
 
     tag = request.args.get('tag', '')
-    _file_db.del_artifact(key, tag, project)
+    _db.del_artifact(key, tag, project)
     return jsonify(ok=True)
 
 # curl http://localhost:8080/artifacts?project=p1?label=l1
@@ -365,7 +366,7 @@ def list_artifacts():
     tag = request.args.get('tag', '')
     labels = request.args.getlist('label')
 
-    artifacts = _file_db.list_artifacts(name, project, tag, labels)
+    artifacts = _db.list_artifacts(name, project, tag, labels)
     return jsonify(ok=True, artifacts=artifacts)
 
 # curl -X DELETE http://localhost:8080/artifacts?project=p1?label=l1
@@ -377,7 +378,7 @@ def del_artifacts():
     tag = request.args.get('tag', '')
     labels = request.args.getlist('label')
 
-    _file_db.del_artifacts(name, project, tag, labels)
+    _db.del_artifacts(name, project, tag, labels)
     return jsonify(ok=True)
 
 # curl -d@/path/to/func.json http://localhost:8080/func/prj/7?tag=0.3.2
@@ -391,7 +392,7 @@ def store_function(project, name):
 
     tag = request.args.get('tag', '')
 
-    _file_db.store_function(data, name, project, tag)
+    _db.store_function(data, name, project, tag)
     return jsonify(ok=True)
 
 
@@ -400,7 +401,7 @@ def store_function(project, name):
 @catch_err
 def get_function(project, name):
     tag = request.args.get('tag', '')
-    func = _file_db.get_function(name, project, tag)
+    func = _db.get_function(name, project, tag)
     return jsonify(ok=True, func=func)
 
 
@@ -413,7 +414,7 @@ def list_functions():
     tag = request.args.get('tag', '')
     labels = request.args.getlist('label')
 
-    out = _file_db.list_functions(name, project, tag, labels)
+    out = _db.list_functions(name, project, tag, labels)
     return jsonify(
         ok=True,
         funcs=list(out),
@@ -427,13 +428,11 @@ def health():
 
 @app.before_first_request
 def init_app():
-    global _file_db
-
-    from mlrun.config import config
+    global _db
 
     logger.info('configuration dump\n%s', config.dump_yaml())
-    _file_db = FileRunDB(config.httpdb.dirpath, '.yaml')
-    _file_db.connect()
+    _db = SQLDB(config.httpdb.dsn)
+    _db.connect()
 
 
 # Don't remove this function, it's an entry point in setup.py