Repository: cylc/cylc-uiserver (language: python)

Issue:
Settings for hub IP and Port must be dynamic
As in the case of a Notebook, the UI server that we have now assumes the hub is always running at 0.0.0.0:8000, but that may have different values in different sites.

The spawner has the hub information. So we just need to send that information to the UI server, store it, and re-use later.


Discussion / hints:
Done. In the Tornado app, we just need to bind the routes to whatever value is in `JUPYTERHUB_SERVICE_PREFIX`. That gives us the `/user/kinow` part. And in JavaScript, we must always address the Hub as a relative path.

So if we are in the URL `http://abc.def:19198/hub/user/kinow`, we must not try to guess the Hub IP and Port.

We must instead simply call `/service`/ or `/hub/service/`, etc. As our Tornado app is behind the reverse proxy, it is transparent to us where the hub is actually running.

Fix patch:
diff --git a/.gitignore b/.gitignore
index cf2d6aaf..cc317f42 100644
--- a/.gitignore
+++ b/.gitignore
@@ -129,3 +129,6 @@ dmypy.json
 jupyterhub-proxy.pid
 jupyterhub.sqlite
 jupyterhub_cookie_secret
+
+# IDE files
+.idea/
diff --git a/.mailmap b/.mailmap
index 88872240..9e42d07f 100644
--- a/.mailmap
+++ b/.mailmap
@@ -6,5 +6,6 @@ Sadie Bartholomew      <sadie.bartholomew@metoffice.gov.uk>   Sadie L. Bartholom
 Ronnie Dutta           <ronnie.dutta@metoffice.gov.uk>                                <61982285+MetRonnie@users.noreply.github.com>
 Mel Hall               <mel.hall@metoffice.gov.uk>                                    <37735232+datamel@users.noreply.github.com>
 Tim Pillinger           <tim.pillinger@metoffice.gov.uk>                              <26465611+wxtim@users.noreply.github.com>
-Stephen McVeigh        <smcveigh941@gmail.com>                                        <stephenmcveigh@users-MacBook-Pro.local>                  
+Stephen McVeigh        <smcveigh941@gmail.com>                                        <stephenmcveigh@users-MacBook-Pro.local>
+Christopher Bennett    <christopher.bennett@metoffice.gov.uk>
 github-actions[bot]    <github-actions@noreply.github.com>                            <41898282+github-actions[bot]@users.noreply.github.com> 
diff --git a/CHANGES.md b/CHANGES.md
index d8edcc8c..b79fd551 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -15,6 +15,11 @@ ones in. -->
 
 <!-- [Updated cylc-ui to x.y.z](https://github.com/cylc/cylc-ui/blob/master/CHANGES.md) -->
 
+### Enhancements
+
+[#434](https://github.com/cylc/cylc-uiserver/pull/434)- Added the server side
+code for the analysis view in the UI
+
 ### Fixes
 
 [#431](https://github.com/cylc/cylc-uiserver/pull/431)- Adds an additional
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index abe07fa2..41ef50ae 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -51,6 +51,7 @@ below.
  - Ronnie Dutta
  - David Matthews
  - Mel Hall
+ - Christopher Bennett
  <!-- end-shortlog -->
 
 (All contributors are identifiable with email addresses in the git version
diff --git a/cylc/uiserver/schema.py b/cylc/uiserver/schema.py
index 02552734..01cdcee6 100644
--- a/cylc/uiserver/schema.py
+++ b/cylc/uiserver/schema.py
@@ -30,12 +30,17 @@
     CyclePoint,
     GenericResponse,
     ID,
+    SortArgs,
+    Task,
     Mutations,
     Queries,
+    process_resolver_info,
+    STRIP_NULL_DEFAULT,
     Subscriptions,
     WorkflowID,
     _mut_field,
-    sstrip
+    sstrip,
+    get_nodes_all
 )
 from cylc.uiserver.resolvers import Resolvers
 
@@ -254,6 +259,227 @@ class Arguments:
     result = GenericScalar()
 
 
+async def get_jobs(root, info, **kwargs):
+    if kwargs['live']:
+        return await get_nodes_all(root, info, **kwargs)
+
+    _, field_ids = process_resolver_info(root, info, kwargs)
+
+    if hasattr(kwargs, 'id'):
+        kwargs['ids'] = [kwargs.get('id')]
+    if field_ids:
+        if isinstance(field_ids, str):
+            field_ids = [field_ids]
+        elif isinstance(field_ids, dict):
+            field_ids = list(field_ids)
+        kwargs['ids'] = field_ids
+    elif field_ids == []:
+        return []
+
+    for arg in ('ids', 'exids'):
+        # live objects can be represented by a universal ID
+        kwargs[arg] = [Tokens(n_id, relative=True) for n_id in kwargs[arg]]
+    kwargs['workflows'] = [
+        Tokens(w_id) for w_id in kwargs['workflows']]
+    kwargs['exworkflows'] = [
+        Tokens(w_id) for w_id in kwargs['exworkflows']]
+
+    return await list_jobs(kwargs)
+
+
+async def list_jobs(args):
+    if not args['workflows']:
+        raise Exception('At least one workflow must be provided.')
+    from cylc.flow.rundb import CylcWorkflowDAO
+    from cylc.flow.pathutil import get_workflow_run_dir
+    from cylc.flow.workflow_files import WorkflowFiles
+    jobs = []
+    for workflow in args['workflows']:
+        db_file = get_workflow_run_dir(
+            workflow['workflow'],
+            WorkflowFiles.LogDir.DIRNAME,
+            "db"
+        )
+        with CylcWorkflowDAO(db_file, is_public=True) as dao:
+            conn = dao.connect()
+            jobs.extend(make_query(conn, workflow))
+    return jobs
+
+
+def make_query(conn, workflow):
+
+    # TODO: support all arguments including states
+    # https://github.com/cylc/cylc-uiserver/issues/440
+    tasks = []
+    for row in conn.execute('''
+SELECT
+    name,
+    cycle,
+    submit_num,
+    submit_status,
+    time_run,
+    time_run_exit,
+    job_id,
+    platform_name,
+    time_submit,
+
+    -- Calculate Queue time stats
+    MIN(queue_time) AS min_queue_time,
+    AVG(queue_time) AS mean_queue_time,
+    MAX(queue_time) AS max_queue_time,
+    AVG(queue_time * queue_time) AS mean_squares_queue_time,
+    MAX(CASE WHEN queue_time_quartile = 1 THEN queue_time END)
+     queue_quartile_1,
+    MAX(CASE WHEN queue_time_quartile = 2 THEN queue_time END)
+    queue_quartile_2,
+    MAX(CASE WHEN queue_time_quartile = 3 THEN queue_time END)
+    queue_quartile_3,
+
+    -- Calculate Run time stats
+    MIN(run_time) AS min_run_time,
+    AVG(run_time) AS mean_run_time,
+    MAX(run_time) AS max_run_time,
+    AVG(run_time * run_time) AS mean_squares_run_time,
+    MAX(CASE WHEN run_time_quartile = 1 THEN run_time END) run_quartile_1,
+    MAX(CASE WHEN run_time_quartile = 2 THEN run_time END) run_quartile_2,
+    MAX(CASE WHEN run_time_quartile = 3 THEN run_time END) run_quartile_3,
+
+    -- Calculate Total time stats
+    MIN(total_time) AS min_total_time,
+    AVG(total_time) AS mean_total_time,
+    MAX(total_time) AS max_total_time,
+    AVG(total_time * total_time) AS mean_squares_total_time,
+    MAX(CASE WHEN total_time_quartile = 1 THEN total_time END)
+    total_quartile_1,
+    MAX(CASE WHEN total_time_quartile = 2 THEN total_time END)
+    total_quartile_2,
+    MAX(CASE WHEN total_time_quartile = 3 THEN total_time END)
+    total_quartile_3,
+
+    COUNT(*) AS n
+
+FROM
+    (SELECT
+        *,
+        NTILE (4) OVER (PARTITION BY name ORDER BY queue_time)
+        queue_time_quartile,
+        NTILE (4) OVER (PARTITION BY name ORDER BY run_time)
+        run_time_quartile,
+        NTILE (4) OVER (PARTITION BY name ORDER BY total_time)
+        total_time_quartile
+    FROM
+        (SELECT
+            *,
+            STRFTIME('%s', time_run_exit) -
+            STRFTIME('%s', time_submit) AS total_time,
+            STRFTIME('%s', time_run_exit) -
+            STRFTIME('%s', time_run) AS run_time,
+            STRFTIME('%s', time_run) -
+            STRFTIME('%s', time_submit) AS queue_time
+        FROM
+            task_jobs))
+WHERE
+    run_status = 0
+GROUP BY
+    name;
+'''):
+        tasks.append({
+            'id': workflow.duplicate(
+                cycle=row[1],
+                task=row[0],
+                job=row[2]
+            ),
+            'name': row[0],
+            'cycle_point': row[1],
+            'submit_num': row[2],
+            'state': row[3],
+            'started_time': row[4],
+            'finished_time': row[5],
+            'job_ID': row[6],
+            'platform': row[7],
+            'submitted_time': row[8],
+            # Queue time stats
+            'min_queue_time': row[9],
+            'mean_queue_time': row[10],
+            'max_queue_time': row[11],
+            'std_dev_queue_time': (row[12] - row[10]**2)**0.5,
+            'queue_quartiles': [row[13],
+                                row[13] if row[14] is None else row[14],
+                                row[13] if row[15] is None else row[15]],
+            # Run time stats
+            'min_run_time': row[16],
+            'mean_run_time': row[17],
+            'max_run_time': row[18],
+            'std_dev_run_time': (row[19] - row[17]**2)**0.5,
+            'run_quartiles': [row[20],
+                              row[20] if row[21] is None else row[21],
+                              row[20] if row[22] is None else row[22]],
+            # Total
+            'min_total_time': row[23],
+            'mean_total_time': row[24],
+            'max_total_time': row[25],
+            'std_dev_total_time': (row[26] - row[24] ** 2) ** 0.5,
+            'total_quartiles': [row[27],
+                                row[27] if row[28] is None else row[28],
+                                row[27] if row[29] is None else row[29]],
+
+            'count': row[30]
+        })
+
+    return tasks
+
+
+class UISTask(Task):
+
+    platform = graphene.String()
+    min_total_time = graphene.Int()
+    mean_total_time = graphene.Int()
+    max_total_time = graphene.Int()
+    std_dev_total_time = graphene.Int()
+    queue_quartiles = graphene.List(
+        graphene.Int,
+        description=sstrip('''
+            List containing the first, second,
+            third and forth quartile queue times.'''))
+    min_queue_time = graphene.Int()
+    mean_queue_time = graphene.Int()
+    max_queue_time = graphene.Int()
+    std_dev_queue_time = graphene.Int()
+    run_quartiles = graphene.List(
+        graphene.Int,
+        description=sstrip('''
+            List containing the first, second,
+            third and forth quartile run times.'''))
+    min_run_time = graphene.Int()
+    mean_run_time = graphene.Int()
+    max_run_time = graphene.Int()
+    std_dev_run_time = graphene.Int()
+    total_quartiles = graphene.List(
+        graphene.Int,
+        description=sstrip('''
+            List containing the first, second,
+            third and forth quartile total times.'''))
+    count = graphene.Int()
+
+
+class UISQueries(Queries):
+
+    tasks = graphene.List(
+        UISTask,
+        description=Task._meta.description,
+        live=graphene.Boolean(default_value=True),
+        strip_null=STRIP_NULL_DEFAULT,
+        resolver=get_jobs,
+        workflows=graphene.List(ID, default_value=[]),
+        exworkflows=graphene.List(ID, default_value=[]),
+        ids=graphene.List(ID, default_value=[]),
+        exids=graphene.List(ID, default_value=[]),
+        mindepth=graphene.Int(default_value=-1),
+        maxdepth=graphene.Int(default_value=-1),
+        sort=SortArgs(default_value=None),
+    )
+
+
 class UISSubscriptions(Subscriptions):
     # Example graphiql workflow log subscription:
     # subscription {
@@ -320,7 +546,7 @@ class UISMutations(Mutations):
 
 
 schema = graphene.Schema(
-    query=Queries,
+    query=UISQueries,
     subscription=UISSubscriptions,
     mutation=UISMutations
 )


Test patch:
diff --git a/cylc/uiserver/tests/test_workflow_retrieval.py b/cylc/uiserver/tests/test_workflow_retrieval.py
new file mode 100644
index 00000000..0689896f
--- /dev/null
+++ b/cylc/uiserver/tests/test_workflow_retrieval.py
@@ -0,0 +1,203 @@
+# Copyright (C) NIWA & British Crown (Met Office) & Contributors.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import pytest
+import sqlite3
+
+from cylc.flow.id import Tokens
+from cylc.uiserver.schema import make_query
+
+'''This file tests the ability for the cylc UI to retrieve workflow information
+and perform simple statistical calculations for the analysis tab'''
+
+
+def test_make_query_1():
+    conn = sqlite3.connect(':memory:')
+    conn.execute('''CREATE TABLE task_jobs(cycle TEXT, name TEXT,
+    submit_num INTEGER, flow_nums TEXT, is_manual_submit INTEGER,
+    try_num INTEGER, time_submit TEXT, time_submit_exit TEXT,
+    submit_status INTEGER, time_run TEXT, time_run_exit TEXT,
+    run_signal TEXT, run_status INTEGER, platform_name TEXT,
+    job_runner_name TEXT, job_id TEXT,
+    PRIMARY KEY(cycle, name, submit_num));''')
+
+    conn.executemany(
+        'INSERT into task_jobs VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)',
+        [('1', 'Task_1', '01', '{1}', 0, 1,
+          '2022-12-14T15:00:00Z', '2022-12-14T15:01:00Z', 0,
+          '2022-12-14T15:01:00Z', '2022-12-14T15:10:00Z', None, 0,
+          'MyPlatform', 'User', 'UsersJob')
+         ])
+    conn.commit()
+    workflow = Tokens('~user/workflow')
+
+    return_value = make_query(conn, workflow)
+
+    assert return_value[0]['count'] == 1
+    assert return_value[0]['cycle_point'] == '1'
+    assert return_value[0]['finished_time'] == '2022-12-14T15:10:00Z'
+    assert return_value[0]['queue_quartiles'][0] == 60
+    assert return_value[0]['run_quartiles'][0] == 540
+    assert return_value[0]['total_quartiles'][0] == 600
+    assert return_value[0]['id'].id == '~user/workflow//1/Task_1/01'
+    assert return_value[0]['job_ID'] == 'UsersJob'
+    assert return_value[0]['max_queue_time'] == 60
+    assert return_value[0]['max_run_time'] == 540
+    assert return_value[0]['max_total_time'] == 600
+    assert return_value[0]['mean_queue_time'] == pytest.approx(60.0, 0.01)
+    assert return_value[0]['mean_run_time'] == pytest.approx(540.0, 0.01)
+    assert return_value[0]['mean_total_time'] == pytest.approx(600.0, 0.01)
+    assert return_value[0]['min_queue_time'] == 60
+    assert return_value[0]['min_run_time'] == 540
+    assert return_value[0]['min_total_time'] == 600
+    assert return_value[0]['name'] == 'Task_1'
+    assert return_value[0]['platform'] == 'MyPlatform'
+    assert return_value[0]['queue_quartiles'][1] == 60
+    assert return_value[0]['run_quartiles'][1] == 540
+    assert return_value[0]['total_quartiles'][1] == 600
+    assert return_value[0]['started_time'] == '2022-12-14T15:01:00Z'
+    assert return_value[0]['state'] == 0
+    assert return_value[0]['std_dev_queue_time'] == pytest.approx(0.0, 0.01)
+    assert return_value[0]['std_dev_run_time'] == pytest.approx(0.0, 0.01)
+    assert return_value[0]['std_dev_total_time'] == pytest.approx(0.0, 0.01)
+    assert return_value[0]['submit_num'] == 1
+    assert return_value[0]['submitted_time'] == '2022-12-14T15:00:00Z'
+    assert return_value[0]['queue_quartiles'][2] == 60
+    assert return_value[0]['run_quartiles'][2] == 540
+    assert return_value[0]['total_quartiles'][2] == 600
+
+
+def test_make_query_2():
+    conn = sqlite3.connect(':memory:')
+    conn.execute('''CREATE TABLE task_jobs(cycle TEXT, name TEXT,
+    submit_num INTEGER, flow_nums TEXT, is_manual_submit INTEGER,
+    try_num INTEGER, time_submit TEXT, time_submit_exit TEXT,
+    submit_status INTEGER, time_run TEXT, time_run_exit TEXT,
+    run_signal TEXT, run_status INTEGER, platform_name TEXT,
+    job_runner_name TEXT, job_id TEXT,
+    PRIMARY KEY(cycle, name, submit_num));''')
+
+    conn.executemany(
+        'INSERT into task_jobs VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)',
+        [('1', 'Task_1', '01', '{1}', 0, 1,
+          '2022-12-14T15:00:00Z', '2022-12-14T15:01:00Z', 0,
+          '2022-12-14T15:01:00Z', '2022-12-14T15:10:00Z', None, 0,
+          'MyPlatform', 'User', 'UsersJob'),
+         ('2', 'Task_1', '01', '{1}', 0, 1,
+          '2022-12-15T15:00:00Z', '2022-12-15T15:01:15Z', 0,
+          '2022-12-15T15:01:16Z', '2022-12-15T15:12:00Z', None, 0,
+          'MyPlatform', 'User', 'UsersJob')
+         ])
+    conn.commit()
+    workflow = Tokens('~user/workflow')
+
+    return_value = make_query(conn, workflow)
+
+    assert return_value[0]['count'] == 2
+    assert return_value[0]['cycle_point'] == '2'
+    assert return_value[0]['finished_time'] == '2022-12-15T15:12:00Z'
+    assert return_value[0]['queue_quartiles'][0] == 60
+    assert return_value[0]['run_quartiles'][0] == 540
+    assert return_value[0]['total_quartiles'][0] == 600
+    assert return_value[0]['id'].id == '~user/workflow//2/Task_1/01'
+    assert return_value[0]['job_ID'] == 'UsersJob'
+    assert return_value[0]['max_queue_time'] == 76
+    assert return_value[0]['max_run_time'] == 644
+    assert return_value[0]['max_total_time'] == 720
+    assert return_value[0]['mean_queue_time'] == pytest.approx(68.0, 0.01)
+    assert return_value[0]['mean_run_time'] == pytest.approx(592.0, 0.01)
+    assert return_value[0]['mean_total_time'] == pytest.approx(660.0, 0.01)
+    assert return_value[0]['min_queue_time'] == 60
+    assert return_value[0]['min_run_time'] == 540
+    assert return_value[0]['min_total_time'] == 600
+    assert return_value[0]['name'] == 'Task_1'
+    assert return_value[0]['platform'] == 'MyPlatform'
+    assert return_value[0]['queue_quartiles'][1] == 76
+    assert return_value[0]['run_quartiles'][1] == 644
+    assert return_value[0]['total_quartiles'][1] == 720
+    assert return_value[0]['started_time'] == '2022-12-15T15:01:16Z'
+    assert return_value[0]['state'] == 0
+    assert return_value[0]['std_dev_queue_time'] == pytest.approx(8.00, 0.01)
+    assert return_value[0]['std_dev_run_time'] == pytest.approx(52.0, 0.01)
+    assert return_value[0]['std_dev_total_time'] == pytest.approx(60.0, 0.01)
+    assert return_value[0]['submit_num'] == 1
+    assert return_value[0]['submitted_time'] == '2022-12-15T15:00:00Z'
+    assert return_value[0]['queue_quartiles'][2] == 60
+    assert return_value[0]['run_quartiles'][2] == 540
+    assert return_value[0]['total_quartiles'][2] == 600
+
+
+def test_make_query_3():
+    conn = sqlite3.connect(':memory:')
+    conn.execute('''CREATE TABLE task_jobs(cycle TEXT, name TEXT,
+    submit_num INTEGER, flow_nums TEXT, is_manual_submit INTEGER,
+    try_num INTEGER, time_submit TEXT, time_submit_exit TEXT,
+    submit_status INTEGER, time_run TEXT, time_run_exit TEXT,
+    run_signal TEXT, run_status INTEGER, platform_name TEXT,
+    job_runner_name TEXT, job_id TEXT,
+    PRIMARY KEY(cycle, name, submit_num));''')
+
+    conn.executemany(
+        'INSERT into task_jobs VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)',
+        [('1', 'Task_1', '01', '{1}', 0, 1,
+          '2022-12-14T15:00:00Z', '2022-12-14T15:01:00Z', 0,
+          '2022-12-14T15:01:00Z', '2022-12-14T15:10:00Z', None, 0,
+          'MyPlatform', 'User', 'UsersJob'),
+         ('2', 'Task_1', '01', '{1}', 0, 1,
+          '2022-12-15T15:00:00Z', '2022-12-15T15:01:15Z', 0,
+          '2022-12-15T15:01:16Z', '2022-12-15T15:12:00Z', None, 0,
+          'MyPlatform', 'User', 'UsersJob'),
+         ('3', 'Task_1', '01', '{1}', 0, 1,
+          '2022-12-16T15:00:00Z', '2022-12-16T15:01:15Z', 0,
+          '2022-12-16T15:01:16Z', '2022-12-16T15:12:00Z', None, 0,
+          'MyPlatform', 'User', 'UsersJob')
+         ])
+    conn.commit()
+    workflow = Tokens('~user/workflow')
+
+    return_value = make_query(conn, workflow)
+
+    assert return_value[0]['count'] == 3
+    assert return_value[0]['cycle_point'] == '3'
+    assert return_value[0]['finished_time'] == '2022-12-16T15:12:00Z'
+    assert return_value[0]['queue_quartiles'][0] == 60
+    assert return_value[0]['run_quartiles'][0] == 540
+    assert return_value[0]['total_quartiles'][0] == 600
+    assert return_value[0]['id'].id == '~user/workflow//3/Task_1/01'
+    assert return_value[0]['job_ID'] == 'UsersJob'
+    assert return_value[0]['max_queue_time'] == 76
+    assert return_value[0]['max_run_time'] == 644
+    assert return_value[0]['max_total_time'] == 720
+    assert return_value[0]['mean_queue_time'] == pytest.approx(70.66, 0.01)
+    assert return_value[0]['mean_run_time'] == pytest.approx(609.33, 0.01)
+    assert return_value[0]['mean_total_time'] == pytest.approx(680.0, 0.01)
+    assert return_value[0]['min_queue_time'] == 60
+    assert return_value[0]['min_run_time'] == 540
+    assert return_value[0]['min_total_time'] == 600
+    assert return_value[0]['name'] == 'Task_1'
+    assert return_value[0]['platform'] == 'MyPlatform'
+    assert return_value[0]['queue_quartiles'][1] == 76
+    assert return_value[0]['run_quartiles'][1] == 644
+    assert return_value[0]['total_quartiles'][1] == 720
+    assert return_value[0]['started_time'] == '2022-12-16T15:01:16Z'
+    assert return_value[0]['state'] == 0
+    assert return_value[0]['std_dev_queue_time'] == pytest.approx(7.54, 0.01)
+    assert return_value[0]['std_dev_run_time'] == pytest.approx(49.02, 0.01)
+    assert return_value[0]['std_dev_total_time'] == pytest.approx(56.56, 0.01)
+    assert return_value[0]['submit_num'] == 1
+    assert return_value[0]['submitted_time'] == '2022-12-16T15:00:00Z'
+    assert return_value[0]['queue_quartiles'][2] == 76
+    assert return_value[0]['run_quartiles'][2] == 644
+    assert return_value[0]['total_quartiles'][2] == 720