Repository: mlrun/mlrun

Issue:
Add options to store logs on file system (vs in database)
Logs can get bit and inefficient in databases, have an optioin (environemnt variable?) to store them on disk instead, in the DB just the path probably.

Discussion:

Patch:
diff --git a/dev-requirements.txt b/dev-requirements.txt
index 51c7c92fa..21a105959 100644
--- a/dev-requirements.txt
+++ b/dev-requirements.txt
@@ -1,6 +1,8 @@
 distributed>=2.3.0
 kfp>=0.1.29
+numpydoc==0.9.1
 pytest==5.3.1
 scikit-learn==0.21.3
+sphinx==2.2.2
 twine==3.1.1
 xgboost==0.90
diff --git a/docs/Makefile b/docs/Makefile
new file mode 100644
index 000000000..d4bb2cbb9
--- /dev/null
+++ b/docs/Makefile
@@ -0,0 +1,20 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line, and also
+# from the environment for the first two.
+SPHINXOPTS    ?=
+SPHINXBUILD   ?= sphinx-build
+SOURCEDIR     = .
+BUILDDIR      = _build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
diff --git a/docs/conf.py b/docs/conf.py
new file mode 100644
index 000000000..a4050ce3a
--- /dev/null
+++ b/docs/conf.py
@@ -0,0 +1,56 @@
+# Configuration file for the Sphinx documentation builder.
+#
+# This file only contains a selection of the most common options. For a full
+# list see the documentation:
+# https://www.sphinx-doc.org/en/master/usage/configuration.html
+
+# -- Path setup --------------------------------------------------------------
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+
+import os
+import sys
+sys.path.insert(0, os.path.abspath('..'))
+
+
+# -- Project information -----------------------------------------------------
+
+project = 'mlrun'
+copyright = '2019, Iguazio'
+author = 'Iguazio'
+
+
+# -- General configuration ---------------------------------------------------
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = [
+    'sphinx.ext.autodoc',
+    'sphinx.ext.todo',
+    'sphinx.ext.viewcode',
+    'numpydoc',
+]
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This pattern also affects html_static_path and html_extra_path.
+exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
+
+
+# -- Options for HTML output -------------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+#
+html_theme = 'alabaster'
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['_static']
diff --git a/docs/index.rst b/docs/index.rst
new file mode 100644
index 000000000..41ac8c779
--- /dev/null
+++ b/docs/index.rst
@@ -0,0 +1,30 @@
+Welcome to mlrun's documentation!
+=================================
+
+A generic an easy to use mechanism for data scientists and developers/engineers
+to describe and run machine learning related tasks in various scalable runtime
+environments while automatically tracking code, metadata, inputs, and outputs
+of executions.
+
+MLRun is integrated with `Nuclio serverless project`_
+
+Read more details in `this doc link`_
+
+.. _`Nuclio serverless project`: https://github.com/nuclio/nuclio
+.. _`this doc link`: https://docs.google.com/document/d/1JRoWx4X7ld3fzQtdTGVIbcZx-5HzlYmkFiQz6ei8izE/edit?usp=sharing
+
+.. automodule:: mlrun
+   :members:
+   :undoc-members:
+
+.. toctree::
+   :maxdepth: 2
+   :caption: Contents:
+
+
+Indices and tables
+==================
+
+* :ref:`genindex`
+* :ref:`modindex`
+* :ref:`search`
diff --git a/mlrun/config.py b/mlrun/config.py
index bd0e7a25c..b7f4028e9 100644
--- a/mlrun/config.py
+++ b/mlrun/config.py
@@ -21,11 +21,12 @@ Environment variables are in the format "MLRUN_httpdb__port=8080". This will be
 mapped to config.httpdb.port. Values should be in JSON format.
 """
 
+import json
 import os
 from collections.abc import Mapping
-from os import path
+from distutils.util import strtobool
+from os.path import expanduser
 from threading import Lock
-import json
 from urllib.parse import urlparse
 
 import yaml
@@ -33,6 +34,7 @@ import yaml
 env_prefix = 'MLRUN_'
 env_file_key = f'{env_prefix}CONIFG_FILE'
 _load_lock = Lock()
+_none_type = type(None)
 
 
 default_config = {
@@ -49,12 +51,13 @@ default_config = {
     'log_level': 'ERROR',
     'httpdb': {
         'port': 8080,
-        'dirpath': path.expanduser('~/.mlrun/db'),
+        'dirpath': expanduser('~/.mlrun/db'),
         'dsn': 'sqlite:///:memory:?check_same_thread=false',
         'debug': False,
         'user': '',
         'password': '',
         'token': '',
+        'logs_path': expanduser('~/.mlrun/logs'),
         'files_path': '',
         'db_type': 'filerundb',
     },
@@ -139,6 +142,17 @@ def _do_populate(env=None):
         config.update(data)
 
 
+def _convert_str(value, typ):
+    if typ in (str, _none_type):
+        return value
+
+    if typ is bool:
+        return strtobool(value)
+
+    # e.g. int('8080') → 8080
+    return typ(value)
+
+
 def read_env(env=None, prefix=env_prefix):
     """Read configuration from environment"""
     env = os.environ if env is None else env
diff --git a/mlrun/db/httpd.py b/mlrun/db/httpd.py
index f0c9da829..a2be3de33 100644
--- a/mlrun/db/httpd.py
+++ b/mlrun/db/httpd.py
@@ -20,6 +20,7 @@ from http import HTTPStatus
 from os import environ
 import traceback
 from datetime import datetime
+from pathlib import Path
 
 from flask import Flask, jsonify, request, Response
 
@@ -35,6 +36,7 @@ from mlrun.k8s_utils import k8s_helper
 
 _db: RunDBInterface
 _k8s: k8s_helper = None
+_logs_dir = None
 app = Flask(__name__)
 basic_prefix = 'Basic '
 bearer_prefix = 'Bearer '
@@ -319,25 +321,36 @@ def get_filestat():
                    mimetype=ctype)
 
 
+def log_path(project, uid) -> Path:
+    return _logs_dir / project / uid
+
 # curl -d@/path/to/log http://localhost:8080/log/prj/7?append=true
 @app.route('/api/log/<project>/<uid>', methods=['POST'])
 @catch_err
 def store_log(project, uid):
     append = strtobool(request.args.get('append', 'no'))
+    log_file = log_path(project, uid)
+    log_file.parent.mkdir(parents=True, exist_ok=True)
     body = request.get_data()  # TODO: Check size
-    _db.store_log(uid, project, body, append)
+    mode = 'ab' if append else 'wb'
+    with log_file.open(mode) as fp:
+        fp.write(body)
     return jsonify(ok=True)
 
 
 # curl http://localhost:8080/log/prj/7
 @app.route('/api/log/<project>/<uid>', methods=['GET'])
 def get_log(project, uid):
-    size = int(request.args.get('size', '0'))
+    size = int(request.args.get('size', '-1'))
     offset = int(request.args.get('offset', '0'))
 
-    out = b''
-    status, resp = _db.get_log(uid, project, offset=offset, size=size)
-    if resp is None:
+    log_file = log_path(project, uid)
+    if log_file.exists():
+        with log_file.open('rb') as fp:
+            fp.seek(offset)
+            out = fp.read(size)
+        status = ''
+    else:
         data = _db.read_run(uid, project)
         if not data:
             return json_error(HTTPStatus.NOT_FOUND,
@@ -368,9 +381,6 @@ def get_log(project, uid):
                             _db.store_run(data, uid, project)
                 status = new_status
 
-    else:
-        out = resp or out
-
     return Response(out, mimetype='text/plain',
                     headers={"pod_status": status})
 
@@ -567,8 +577,7 @@ def health():
 
 @app.before_first_request
 def init_app():
-    global _db
-    global _k8s
+    global _db, _logs_dir, _k8s
 
     logger.info('configuration dump\n%s', config.dump_yaml())
     if config.httpdb.db_type == 'sqldb':
@@ -578,6 +587,8 @@ def init_app():
         logger.info('using FileRunDB')
         _db = FileRunDB(config.httpdb.dirpath)
     _db.connect()
+    _logs_dir = Path(config.httpdb.logs_path)
+
     try:
         _k8s = k8s_helper()
     except Exception:
diff --git a/mlrun/db/httpdb.py b/mlrun/db/httpdb.py
index dfbb8ffde..8b185b5ba 100644
--- a/mlrun/db/httpdb.py
+++ b/mlrun/db/httpdb.py
@@ -39,7 +39,7 @@ def bool2str(val):
 
 class HTTPRunDB(RunDBInterface):
     kind = 'http'
-    
+
     def __init__(self, base_url, user='', password='', token=''):
         self.base_url = base_url
         self.user = user
@@ -55,7 +55,7 @@ class HTTPRunDB(RunDBInterface):
         url = f'{self.base_url}/api/{path}'
         kw = {
             key: value
-            for key, value in (('params', params), ('data', body), 
+            for key, value in (('params', params), ('data', body),
                                ('json', json))
             if value is not None
         }
@@ -90,7 +90,7 @@ class HTTPRunDB(RunDBInterface):
         error = f'store log {project}/{uid}'
         self.api_call('POST', path, error, params, body)
 
-    def get_log(self, uid, project='', offset=0, size=0):
+    def get_log(self, uid, project='', offset=0, size=-1):
         params = {'offset': offset, 'size': size}
         path = self._path_of('log', project, uid)
         error = f'get log {project}/{uid}'
diff --git a/mlrun/db/sqldb.py b/mlrun/db/sqldb.py
index e6d43b3dd..712b970d3 100644
--- a/mlrun/db/sqldb.py
+++ b/mlrun/db/sqldb.py
@@ -14,6 +14,7 @@
 
 import pickle
 from datetime import datetime, timedelta
+import warnings
 
 from sqlalchemy import (
     BLOB, TIMESTAMP, Column, ForeignKey, Integer, String, UniqueConstraint,
@@ -54,65 +55,66 @@ def make_label(table):
     return Label
 
 
-class Artifact(Base, HasStruct):
-    __tablename__ = 'artifacts'
-    __table_args__ = (
-        UniqueConstraint('uid', 'project', 'key', name='_artifacts_uc'),
-    )
+# quell SQLAlchemy warnings on duplicate class name (Label)
+with warnings.catch_warnings():
+    warnings.simplefilter("ignore")
 
-    Label = make_label(__tablename__)
-
-    id = Column(Integer, primary_key=True)
-    key = Column(String)
-    project = Column(String)
-    tag = Column(String)
-    uid = Column(String)
-    updated = Column(TIMESTAMP)
-    body = Column(BLOB)
-    labels = relationship(Label)
-
-
-class Function(Base, HasStruct):
-    __tablename__ = 'functions'
-    __table_args__ = (
-        UniqueConstraint('name', 'project', 'tag', name='_functions_uc'),
-    )
-
-    Label = make_label(__tablename__)
+    class Artifact(Base, HasStruct):
+        __tablename__ = 'artifacts'
+        __table_args__ = (
+            UniqueConstraint('uid', 'project', 'key', name='_artifacts_uc'),
+        )
 
-    id = Column(Integer, primary_key=True)
-    name = Column(String)
-    project = Column(String)
-    tag = Column(String)
-    body = Column(BLOB)
-    labels = relationship(Label)
+        Label = make_label(__tablename__)
 
+        id = Column(Integer, primary_key=True)
+        key = Column(String)
+        project = Column(String)
+        tag = Column(String)
+        uid = Column(String)
+        updated = Column(TIMESTAMP)
+        body = Column(BLOB)
+        labels = relationship(Label)
+
+    class Function(Base, HasStruct):
+        __tablename__ = 'functions'
+        __table_args__ = (
+            UniqueConstraint('name', 'project', 'tag', name='_functions_uc'),
+        )
 
-class Log(Base):
-    __tablename__ = 'logs'
+        Label = make_label(__tablename__)
 
-    id = Column(Integer, primary_key=True)
-    uid = Column(String)
-    project = Column(String)
-    body = Column(BLOB)
+        id = Column(Integer, primary_key=True)
+        name = Column(String)
+        project = Column(String)
+        tag = Column(String)
+        body = Column(BLOB)
+        labels = relationship(Label)
 
+    class Log(Base):
+        __tablename__ = 'logs'
 
-class Run(Base, HasStruct):
-    __tablename__ = 'runs'
-    __table_args__ = (
-        UniqueConstraint('uid', 'project', 'iteration', name='_runs_uc'),
-    )
+        id = Column(Integer, primary_key=True)
+        uid = Column(String)
+        project = Column(String)
+        body = Column(BLOB)
+
+    class Run(Base, HasStruct):
+        __tablename__ = 'runs'
+        __table_args__ = (
+            UniqueConstraint('uid', 'project', 'iteration', name='_runs_uc'),
+        )
 
-    Label = make_label(__tablename__)
+        Label = make_label(__tablename__)
 
-    id = Column(Integer, primary_key=True)
-    uid = Column(String)
-    project = Column(String)
-    iteration = Column(Integer)
-    state = Column(String)
-    body = Column(BLOB)
-    start_time = Column(TIMESTAMP)
-    labels = relationship(Label)
+        id = Column(Integer, primary_key=True)
+        uid = Column(String)
+        project = Column(String)
+        iteration = Column(Integer)
+        state = Column(String)
+        body = Column(BLOB)
+        start_time = Column(TIMESTAMP)
+        labels = relationship(Label)
 
 
 class SQLDB(RunDBInterface):
@@ -201,6 +203,8 @@ class SQLDB(RunDBInterface):
             query = query.order_by(Run.start_time.desc())
         if last:
             query = query.limit(last)
+        if not iter:
+            query = query.filter(Run.iteration == 0)
 
         runs = RunList()
         for run in query:
diff --git a/mlrun/runtimes/serving.py b/mlrun/runtimes/serving.py
index 2a9eb93c4..2fba03a2a 100644
--- a/mlrun/runtimes/serving.py
+++ b/mlrun/runtimes/serving.py
@@ -1,4 +1,3 @@
-import json
 # Copyright 2018 Iguazio
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,6 +12,7 @@ import json
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import json
 import os
 from io import BytesIO
 from typing import Dict
@@ -90,41 +90,34 @@ class HTTPHandler:
         return model
 
     def parse_event(self, event):
-        body = {"instances": []}
+        parsed_event = {'instances': []}
         try:
             body = json.loads(event.body)
+            self.context.logger.info(f'event.body: {event.body}')
             if 'data_url' in body:
                 # Get data from URL
                 url = body['data_url']
                 self.context.logger.debug_with('downloading data', url=url)
                 data = urlopen(url).read()
                 sample = BytesIO(data)
-                body['instances'].append(sample)
+                parsed_event['instances'].append(sample)
 
-        except json.decoder.JSONDecodeError as e:
+        except Exception as e:
             if event.content_type.startswith('image/'):
                 sample = BytesIO(event.body)
-                body['instances'].append(sample)
-                body['content_type'] = event.content_type
+                parsed_event['instances'].append(sample)
+                parsed_event['content_type'] = event.content_type
             else:
-                return self.context.Response(
-                    body="Unrecognized request format: %s" % e,
-                    content_type='text/plain',
-                    status_code=400)
-        return body
+                raise Exception("Unrecognized request format: %s" % e)
+                
+        return parsed_event
 
     def validate(self, request):
         if "instances" not in request:
-            return self.context.Response(
-                body="Expected key \"instances\" in request body",
-                content_type='text/plain',
-                status_code=400)
+            raise Exception("Expected key \"instances\" in request body")
 
         if not isinstance(request["instances"], list):
-            return self.context.Response(
-                body="Expected \"instances\" to be a list",
-                content_type='text/plain',
-                status_code=400)
+            raise Exception("Expected \"instances\" to be a list")
 
         return request
 


Test patch:
diff --git a/tests/test_httpdb.py b/tests/test_httpdb.py
index 5ca30ddda..8a5259912 100644
--- a/tests/test_httpdb.py
+++ b/tests/test_httpdb.py
@@ -111,7 +111,7 @@ def server_fixture():
 
     def create(env=None):
         nonlocal proc, log_fp
-        root = mkdtemp(prefix='mlrun-test')
+        root = mkdtemp(prefix='mlrun-test-')
         print(f'root={root!r}')
         db_path = f'{root}/mlrun.sqlite3?check_same_thread=false'
         log_fp = open(f'{root}/httpd.log', 'w+')
@@ -147,7 +147,11 @@ def create_server(request):
 
 
 def test_log(create_server):
-    server: Server = create_server()
+    logs_path = mkdtemp()
+    env = {
+        'MLRUN_httpdb__logs_path': logs_path,
+    }
+    server: Server = create_server(env)
     db = server.conn
     prj, uid, body = 'p19', '3920', b'log data'
     db.store_log(uid, prj, body)
diff --git a/tests/test_sqldb.py b/tests/test_sqldb.py
index 06c3eff1c..e35ee8770 100644
--- a/tests/test_sqldb.py
+++ b/tests/test_sqldb.py
@@ -158,3 +158,17 @@ def test_artifacts(db: sqldb.SQLDB):
     db.del_artifact(key=k1)
     with pytest.raises(sqldb.RunDBError):
         db.read_artifact(k1)
+
+
+def test_list_runs(db: sqldb.SQLDB):
+    uid = 'u183'
+    run = new_run('s1', ['l1', 'l2'], x=1)
+    count = 5
+    for iter in range(count):
+        db.store_run(run, uid, iter=iter)
+
+    runs = list(db.list_runs(uid=uid))
+    assert 1 == len(runs), 'iter=False'
+
+    runs = list(db.list_runs(uid=uid, iter=True))
+    assert 5 == len(runs), 'iter=True'