Repository: mlrun/mlrun

Issue:
More HTTP API endpoints
@yaronha said:
> expose list_projects, list_artifact_tags, list_schedules in httpd/web-api (so the UI can access those)


Discussion:

Patch:
diff --git a/mlrun/db/httpd.py b/mlrun/db/httpd.py
index 2c89f1eeb..f447d4eb0 100644
--- a/mlrun/db/httpd.py
+++ b/mlrun/db/httpd.py
@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 """mlrun database HTTP server"""
+from argparse import ArgumentParser
 import mimetypes
 from base64 import b64decode
 from distutils.util import strtobool
@@ -554,21 +555,21 @@ def del_run(project, uid):
 @app.route('/api/runs', methods=['GET'])
 @catch_err
 def list_runs():
-    name = request.args.get('name', '')
-    uid = request.args.get('uid', '')
-    project = request.args.get('project', 'default')
+    name = request.args.get('name')
+    uid = request.args.get('uid')
+    project = request.args.get('project')
     labels = request.args.getlist('label')
-    state = request.args.get('state', '')
+    state = request.args.get('state')
     sort = strtobool(request.args.get('sort', 'on'))
     iter = strtobool(request.args.get('iter', 'on'))
     last = int(request.args.get('last', '0'))
 
     runs = _db.list_runs(
-        name=name,
-        uid=uid,
-        project=project,
+        name=name or None,
+        uid=uid or None,
+        project=project or None,
         labels=labels,
-        state=state,
+        state=state or None,
         sort=sort,
         last=last,
         iter=iter,
@@ -603,6 +604,17 @@ def store_artifact(project, uid, key):
     return jsonify(ok=True)
 
 
+# curl http://localhost:8080/artifact/p1/tags
+@app.route('/api/artifact/<project>/tags', methods=['GET'])
+@catch_err
+def list_artifact_tags(project):
+    return jsonify(
+        ok=True,
+        project=project,
+        tags=_db.list_artifact_tags(project),
+    )
+
+
 # curl http://localhost:8080/artifact/p1/tag/key
 @app.route('/api/artifact/<project>/<tag>/<path:key>', methods=['GET'])
 @catch_err
@@ -626,9 +638,9 @@ def del_artifact(project, uid):
 @app.route('/api/artifacts', methods=['GET'])
 @catch_err
 def list_artifacts():
-    name = request.args.get('name', '')
-    project = request.args.get('project', 'default')
-    tag = request.args.get('tag', '')
+    name = request.args.get('name') or None
+    project = request.args.get('project', config.default_project)
+    tag = request.args.get('tag') or None
     labels = request.args.getlist('label')
 
     artifacts = _db.list_artifacts(name, project, tag, labels)
@@ -674,9 +686,9 @@ def get_function(project, name):
 @app.route('/api/funcs', methods=['GET'])
 @catch_err
 def list_functions():
-    name = request.args.get('name', '')
-    project = request.args.get('project', 'default')
-    tag = request.args.get('tag', '')
+    name = request.args.get('name') or None
+    project = request.args.get('project', config.default_project)
+    tag = request.args.get('tag') or None
     labels = request.args.getlist('label')
 
     out = _db.list_functions(name, project, tag, labels)
@@ -686,6 +698,26 @@ def list_functions():
     )
 
 
+# curl http://localhost:8080/projects
+@app.route('/api/projects', methods=['GET'])
+@catch_err
+def list_projects():
+    return jsonify(
+        ok=True,
+        project=_db.list_projects()
+    )
+
+
+# curl http://localhost:8080/schedules
+@app.route('/api/schedules', methods=['GET'])
+@catch_err
+def list_schedules():
+    return jsonify(
+        ok=True,
+        schedules=_db.list_schedules()
+    )
+
+
 @app.route('/api/healthz', methods=['GET'])
 def health():
     return 'OK\n'
@@ -724,6 +756,8 @@ def init_app():
 
 # Don't remove this function, it's an entry point in setup.py
 def main():
+    parser = ArgumentParser(description=__doc__)
+    parser.parse_args()
     app.run(
         host='0.0.0.0',
         port=config.httpdb.port,
diff --git a/mlrun/db/sqldb.py b/mlrun/db/sqldb.py
index f364742ab..fd5677ce3 100644
--- a/mlrun/db/sqldb.py
+++ b/mlrun/db/sqldb.py
@@ -13,8 +13,8 @@
 # limitations under the License.
 
 import pickle
-from datetime import datetime, timedelta
 import warnings
+from datetime import datetime, timedelta
 
 from sqlalchemy import (
     BLOB, TIMESTAMP, Column, ForeignKey, Integer, String, UniqueConstraint,
@@ -193,7 +193,7 @@ class SQLDB(RunDBInterface):
         self.session.commit()
         self._delete_empty_labels(Run.Label)
 
-    def read_run(self, uid, project='', iter=0):
+    def read_run(self, uid, project=None, iter=None):
         project = project or config.default_project
         run = self._get_run(uid, project, iter)
         if not run:
@@ -201,10 +201,11 @@ class SQLDB(RunDBInterface):
         return run.struct
 
     def list_runs(
-            self, name='', uid=None, project='', labels=None,
-            state='', sort=True, last=0, iter=False):
+            self, name=None, uid=None, project=None, labels=None,
+            state=None, sort=True, last=0, iter=None):
+        # FIXME: Run has no "name"
         project = project or config.default_project
-        query = self._find_runs(name, uid, project, labels, state)
+        query = self._find_runs(uid, project, labels, state)
         if sort:
             query = query.order_by(Run.start_time.desc())
         if last:
@@ -218,16 +219,17 @@ class SQLDB(RunDBInterface):
 
         return runs
 
-    def del_run(self, uid, project='', iter=0):
+    def del_run(self, uid, project=None, iter=None):
         project = project or config.default_project
         # We currently delete *all* iterations
         self._delete(Run, uid=uid, project=project)
 
     def del_runs(
-        self, name='', project='', labels=None,
-            state='', days_ago=0):
+        self, name=None, project=None, labels=None,
+            state=None, days_ago=0):
+        # FIXME: Run has no `name`
         project = project or config.default_project
-        query = self._find_runs(name, '', project, labels, state)
+        query = self._find_runs(None, project, labels, state)
         if days_ago:
             since = datetime.now() - timedelta(days=days_ago)
             query = query.filter(Run.start_time >= since)
@@ -272,13 +274,12 @@ class SQLDB(RunDBInterface):
             raise RunDBError(f'Artifact {key}:{tag}:{project} not found')
         return art.struct
 
-    def list_artifacts(
-            self, name='', project='', tag='', labels=None):
+    def list_artifacts(self, name=None, project=None, tag=None, labels=None):
         project = project or config.default_project
         arts = ArtifactList()
         arts.extend(
             obj.struct
-            for obj in self._find_artifacts(name, project, tag, labels)
+            for obj in self._find_artifacts(project, tag, labels)
         )
         return arts
 
@@ -290,7 +291,7 @@ class SQLDB(RunDBInterface):
     def del_artifacts(
             self, name='', project='', tag='', labels=None):
         project = project or config.default_project
-        for obj in self._find_artifacts(name, project, tag, labels):
+        for obj in self._find_artifacts(project, tag, labels):
             self.session.delete(obj)
         self.session.commit()
 
@@ -317,7 +318,7 @@ class SQLDB(RunDBInterface):
             return obj.struct
 
     def list_functions(
-            self, name, project='', tag='', labels=None):
+            self, name, project=None, tag=None, labels=None):
         project = project or config.default_project
         funcs = FunctionList()
         funcs.extend(
@@ -343,7 +344,7 @@ class SQLDB(RunDBInterface):
         return [s.struct for s in self.session.query(Schedule)]
 
     def _query(self, cls, **kw):
-        kw = {k: v for k, v in kw.items() if v}
+        kw = {k: v for k, v in kw.items() if v is not None}
         return self.session.query(cls).filter_by(**kw)
 
     def _get_function(self, name, project, tag):
@@ -371,18 +372,17 @@ class SQLDB(RunDBInterface):
             cls = obj.__class__.__name__
             raise RunDBError(f'duplicate {cls} - {err}')
 
-    def _find_runs(self, name, uid, project, labels, state):
+    def _find_runs(self, uid, project, labels, state):
         labels = label_set(labels)
-        query = self._query(
-            Run, name=name, uid=uid, project=project, state=state)
+        query = self._query(Run, uid=uid, project=project, state=state)
         if labels:
             query = query.join(Run.Label).filter(Run.Label.name.in_(labels))
         return query
 
-    def _find_artifacts(self, name, project, tag, labels):
+    def _find_artifacts(self, project, tag, labels):
         # FIXME tag = tag or 'latest'
         labels = label_set(labels)
-        query = self._query(Artifact, name=name, project=project, tag=tag)
+        query = self._query(Artifact, project=project, tag=tag)
         if labels:
             query = query.join(Run.Label).filter(Run.Label.name.in_(labels))
         return query


Test patch:
diff --git a/tests/test_httpd.py b/tests/test_httpd.py
new file mode 100644
index 000000000..3efac28ac
--- /dev/null
+++ b/tests/test_httpd.py
@@ -0,0 +1,34 @@
+from mlrun.db import httpd
+import pytest
+from http import HTTPStatus
+
+
+@pytest.fixture
+def client():
+    old_testing = httpd.app.config['TESTING']
+    httpd.app.config['TESTING'] = True
+    with httpd.app.test_client() as client:
+        yield client
+
+    httpd.app.config['TESTING'] = old_testing
+
+
+def test_list_projects(client):
+    resp = client.get('/api/projects')
+    assert resp.status_code == HTTPStatus.OK, 'status'
+    assert resp.json['ok'], 'not ok'
+
+
+def test_list_artifact_tags(client):
+    project = 'p11'
+    resp = client.get(f'/api/artifact/{project}/tags')
+    assert resp.status_code == HTTPStatus.OK, 'status'
+    assert resp.json['ok'], 'not ok'
+    assert resp.json['project'] == project, 'project'
+
+
+def test_list_schedules(client):
+    resp = client.get(f'/api/schedules')
+    assert resp.status_code == HTTPStatus.OK, 'status'
+    assert resp.json['ok'], 'not ok'
+    assert 'schedules' in resp.json, 'no schedules'
diff --git a/tests/test_sqldb.py b/tests/test_sqldb.py
index 6639a92c4..bc93b5817 100644
--- a/tests/test_sqldb.py
+++ b/tests/test_sqldb.py
@@ -200,3 +200,11 @@ def test_schedules(db: sqldb.SQLDB):
     scheds = list(db.list_schedules())
     assert count == len(scheds), 'wrong number of schedules'
     assert set(range(count)) == set(s['i'] for s in scheds), 'bad scheds'
+
+
+def test_run_iter0(db: sqldb.SQLDB):
+    uid, prj = 'uid39', 'lemon'
+    run = new_run('s1', ['l1', 'l2'], x=1)
+    for i in range(7):
+        db.store_run(run, uid, prj, i)
+    db._get_run(uid, prj, 0)  # See issue 140