Repository: flux-framework/flux-restful-api (language: python)

Issue:
Add support for logs endpoint
From the wise @grondo

> From the commandline you can use flux job attach --read-only JOBID to dump the output
> From Python, you can fetch or "watch" those eventlogs. There are examples in flux-mini.py of how to do that

We need this now for the operator since submitting a job doesn't actually let you see what's going on anymore! This would be a good api endpoint to also put in the web UI for getting logs with a click (maybe nicer than command line?)


Discussion / hints:

Fix patch:
diff --git a/app/core/config.py b/app/core/config.py
index 0ac4ef4..6b8a856 100644
--- a/app/core/config.py
+++ b/app/core/config.py
@@ -3,13 +3,43 @@
 from pydantic import BaseSettings
 
 
+def get_int_envar(key, default=None):
+    """
+    Get (and parse) an integer environment variable
+    """
+    value = os.environ.get(key)
+    if not value:
+        value = default
+    try:
+        value = int(value)
+        return value
+    except Exception:
+        return default
+
+
+def get_bool_envar(key, default=False):
+    """
+    Get a boolean from the environment, meaning the value is set.
+    """
+    return default if not os.environ.get(key) else not default
+
+
 class Settings(BaseSettings):
+    """
+    Basic settings and defaults for the Flux RESTFul API
+    """
+
     app_name: str = "Flux RESTFul API"
 
     # These map to envars, e.g., FLUX_USER
+    has_gpus: bool = get_bool_envar("FLUX_HAS_GPUS")
+
+    # Assume there is at least one node!
+    flux_nodes: int = get_int_envar("FLUX_NUMBER_NODES", 1)
+
     flux_user: str = os.environ.get("FLUX_USER")
     flux_token: str = os.environ.get("FLUX_TOKEN")
-    require_auth: bool = False if not os.environ.get("FLUX_REQUIRE_AUTH") else True
+    require_auth: bool = get_bool_envar("FLUX_REQUIRE_AUTH")
 
 
 settings = Settings()
diff --git a/app/forms.py b/app/forms.py
new file mode 100644
index 0000000..b2a60cc
--- /dev/null
+++ b/app/forms.py
@@ -0,0 +1,58 @@
+from typing import List, Optional
+
+from fastapi import Request
+
+import app.library.flux as flux_cli
+
+
+class SubmitForm:
+    def __init__(self, request: Request):
+        self.request: Request = request
+        self.errors: List = []
+        self.command: str
+        self.workdir: Optional[str] = None
+        self.num_tasks: Optional[int] = None
+        self.num_nodes: Optional[int] = None
+        self.runtime: Optional[int] = None
+        self.cores_per_task: Optional[int] = None
+        self.gpus_per_task: Optional[int] = None
+        self.exclusive: Optional[bool] = False
+
+    # STOPPED HERE - serialize in jquery from form, submit as application/json.
+    async def load_data(self):
+        form = await self.request.form()
+        self.command = form.get("command")
+        self.workdir = form.get("workdir") or None
+        self.num_tasks = form.get("num_tasks") or 1
+        self.num_nodes = form.get("num_nodes") or 1
+        self.runtime = form.get("runtime") or 0
+        self.cores_per_task = form.get("cores_per_task") or None
+        self.gpus_per_task = form.get("gpus_per_task") or None
+        self.exclusive = form.get("exclusive") or False
+
+    @property
+    def kwargs(self):
+        """
+        Prepared key value dictionary of items.
+        """
+        kwargs = {}
+        for key in [
+            "command",
+            "num_tasks",
+            "num_nodes",
+            "cores_per_task",
+            "gpys_per_task",
+            "exclusive",
+        ]:
+            if getattr(self, key, None) is not None:
+                kwargs[key] = getattr(self, key)
+        return kwargs
+
+    def is_valid(self):
+        """
+        Determine if the form is valid (devoid of errors)
+        """
+        self.errors = flux_cli.validate_submit_kwargs(self.kwargs, runtime=self.runtime)
+        if not self.errors:
+            return True
+        return False
diff --git a/app/library/flux.py b/app/library/flux.py
new file mode 100644
index 0000000..b343fa2
--- /dev/null
+++ b/app/library/flux.py
@@ -0,0 +1,190 @@
+import json
+import os
+import re
+import shlex
+
+import flux
+import flux.job
+
+from app.core.config import settings
+
+
+def validate_submit_kwargs(kwargs, envars=None, runtime=None):
+    """
+    Shared function to validate submit, from API or web UI.
+
+    Kwargs are expected to be given to JobspecV1, and
+    everything else is added to the fluxjob.
+    """
+    errors = []
+    if "command" not in kwargs or not kwargs["command"]:
+        errors.append("'command' is required.")
+
+    # We can't ask for more nodes than available!
+    num_nodes = kwargs.get("num_nodes")
+    if num_nodes and num_nodes > settings.flux_nodes:
+        errors.append(
+            f"The server only has {settings.flux_nodes} nodes, you requested {num_nodes}"
+        )
+
+    # If the user asks for gpus and we don't have any, no go
+    if "gpus_per_task" in kwargs and not settings.has_gpus:
+        errors.append("This server does not support gpus: gpus_per_task cannot be set.")
+
+    # Minimum value of zero
+    if runtime and runtime < 0:
+        errors.append(f"Runtime must be >= 0, found {runtime}")
+
+    # Minimum values of 1
+    for key in ["cpus_per_task", "gpus_per_task"]:
+        if key in kwargs and kwargs[key] < 1:
+            errors.append(f"Parameter {key} must be >= 1")
+
+    if envars and not isinstance(envars, dict):
+        errors.append("Environment variables must be key/value pairs (dict)")
+    return errors
+
+
+def prepare_job(kwargs, runtime=0, workdir=None, envars=None):
+    """
+    After validation, prepare the job (shared function).
+    """
+    envars = envars or {}
+
+    # Generate the flux job
+    command = kwargs["command"]
+    if isinstance(command, str):
+        command = shlex.split(command)
+
+    # Delete command from the kwargs (we added because is required and validated that way)
+    del kwargs["command"]
+    fluxjob = flux.job.JobspecV1.from_command(command, **kwargs)
+
+    if workdir is not None:
+        fluxjob.workdir = workdir
+
+    # A duration of zero (the default) means unlimited
+    fluxjob.duration = runtime
+
+    # Additional envars in the payload?
+    environment = dict(os.environ)
+    environment.update(envars)
+    fluxjob.environment = environment
+    return fluxjob
+
+
+def query_job(jobinfo, query):
+    """
+    This would be better suited for a database, but should work for small numbers.
+    """
+    searchstr = "".join([str(x) for x in list(jobinfo.values())])
+    return re.search(query, searchstr)
+
+
+def query_jobs(contenders, query):
+    """
+    Wrapper to query more than one job.
+    """
+    jobs = []
+    for contender in contenders:
+        if not query_job(contender, query):
+            continue
+        jobs.append(contender)
+    return jobs
+
+
+def get_job_output(jobid):
+    """
+    Given a jobid, get the output.
+    """
+    lines = []
+    from app.main import app
+
+    # If the submit is too close to the log reqest, it cannot find the file handle
+    # It could be also the jobid cannot be found.
+    try:
+        for line in flux.job.event_watch(app.handle, jobid, "guest.output"):
+            if "data" in line.context:
+                lines.append(line.context["data"])
+    except Exception:
+        pass
+    return lines
+
+
+def list_jobs_detailed(limit=None, query=None):
+    """
+    Get a detailed listing of jobs.
+    """
+    listing = list_jobs()
+    ids = listing.get()["jobs"]
+    jobs = {}
+    for job in ids:
+
+        # Stop if a limit is defined and we have hit it!
+        if limit is not None and len(jobs) >= limit:
+            break
+
+        try:
+            jobinfo = get_job(job["id"])
+
+            # Best effort hack to do a query
+            if query and not query_job(jobinfo, query):
+                continue
+            jobs[job["id"]] = jobinfo
+        except Exception:
+            pass
+    return jobs
+
+
+def list_jobs():
+    """
+    Get a simple listing of jobs (just the ids)
+    """
+    from app.main import app
+
+    return flux.job.job_list(app.handle)
+
+
+def get_simple_job(jobid):
+    """
+    Not used - an original (simpler) implementation.
+    """
+    from app.main import app
+
+    info = flux.job.job_list_id(app.handle, jobid, attrs=["all"])
+    return json.loads(info.get_str())["job"]
+
+
+def get_job(jobid):
+    """
+    Get details for a job
+    """
+    from app.main import app
+
+    payload = {"id": int(jobid), "attrs": ["all"]}
+    rpc = flux.job.list.JobListIdRPC(app.handle, "job-list.list-id", payload)
+    try:
+        jobinfo = rpc.get()
+
+    # The job does not exist!
+    except FileNotFoundError:
+        return None
+
+    jobinfo = jobinfo["job"]
+
+    # User friendly string from integer
+    state = jobinfo["state"]
+    jobinfo["state"] = flux.job.info.statetostr(state)
+
+    # Get job info to add to result
+    info = rpc.get_jobinfo()
+    jobinfo["nnodes"] = info._nnodes
+    jobinfo["result"] = info.result
+    jobinfo["returncode"] = info.returncode
+    jobinfo["runtime"] = info.runtime
+    jobinfo["priority"] = info._priority
+    jobinfo["waitstatus"] = info._waitstatus
+    jobinfo["nodelist"] = info._nodelist
+    jobinfo["nodelist"] = info._nodelist
+    jobinfo["exception"] = info._exception.__dict__
+    return jobinfo
diff --git a/app/library/helpers.py b/app/library/helpers.py
index 3565abb..b3f6afc 100644
--- a/app/library/helpers.py
+++ b/app/library/helpers.py
@@ -4,6 +4,26 @@
 import markdown
 
 
+def has_boolean_arg(payload, key):
+    """
+    A helper to determine if a payload has a key, and it's in some derivation of True
+    """
+    return key in payload and payload.get(key) in [True, "true"]
+
+
+def get_int_arg(payload, key):
+    """
+    Attempt to get (and parse) and integer argument. Fallback to None.
+    """
+    arg = payload.get(key)
+    if arg:
+        try:
+            arg = int(arg)
+        except ValueError:
+            arg = None
+    return arg
+
+
 def read_json(filename):
     with open(filename, "r") as fd:
         content = json.loads(fd.read())
diff --git a/app/main.py b/app/main.py
index 8c1950c..f3b3782 100644
--- a/app/main.py
+++ b/app/main.py
@@ -3,15 +3,12 @@
 import sys
 
 from fastapi import FastAPI, Request
-from fastapi.responses import HTMLResponse
 from fastapi.staticfiles import StaticFiles
 from fastapi.templating import Jinja2Templates
 
 from app.core.logging import init_loggers
 from app.routers import api, views
 
-from .library.helpers import openfile
-
 init_loggers()
 log = logging.getLogger("flux-restful")
 
@@ -28,6 +25,7 @@
 app.mount("/data", StaticFiles(directory=data_root), name="data")
 
 app.include_router(views.router)
+app.include_router(views.auth_views_router)
 app.include_router(api.router)
 
 try:
@@ -53,21 +51,3 @@ async def load_app_data(request: Request, call_next):
             "Cannot find flux instance! Ensure you have run flux start or similar."
         )
     return await call_next(request)
-
-
-@app.get("/", response_class=HTMLResponse)
-async def home(request: Request):
-    data = openfile("index.md")
-    return templates.TemplateResponse(
-        "index.html",
-        {
-            "request": request,
-            "data": data,
-        },
-    )
-
-
-@app.get("/page/{page_name}", response_class=HTMLResponse)
-async def show_page(request: Request, page_name: str):
-    data = openfile(page_name + ".md")
-    return templates.TemplateResponse("page.html", {"request": request, "data": data})
diff --git a/app/routers/api.py b/app/routers/api.py
index 27b2991..6499671 100644
--- a/app/routers/api.py
+++ b/app/routers/api.py
@@ -1,6 +1,4 @@
-import json
 import os
-import shlex
 
 import flux.job
 import flux.resource
@@ -8,14 +6,16 @@
 from fastapi.encoders import jsonable_encoder
 from fastapi.responses import JSONResponse
 from fastapi.templating import Jinja2Templates
-from flux.job import JobspecV1
 
+import app.library.flux as flux_cli
+import app.library.helpers as helpers
 from app.core.config import settings
 from app.library.auth import alert_auth, check_auth
 
 # Print (hidden message) to give status of auth
 alert_auth()
 router = APIRouter(
+    prefix="/v1",
     tags=["jobs"],
     dependencies=[Depends(check_auth)] if settings.require_auth else [],
     responses={404: {"description": "Not found"}},
@@ -34,15 +34,68 @@ async def service_stop():
     os.system("flux shutdown")
 
 
+@router.get("/jobs/search")
+async def jobs_listing(request: Request):
+    """
+    Jobslist is intended to be used by the server to render data tables
+
+    Since this is specific to jquery datables, we don't document.
+    """
+    start = request.query_params.get("start")
+    length = request.query_params.get("length")
+    draw = request.query_params.get("draw") or 1
+    query = request.query_params.get("search[value]") or request.query_params.get(
+        "search"
+    )
+
+    # If we have a query, filter to those that have in the name
+    jobs = list(flux_cli.list_jobs_detailed().values())
+    total = len(jobs)
+
+    # Now filter
+    if query:
+        jobs = flux_cli.query_jobs(jobs, query)
+
+    # If we are filtering to a range
+    if start and int(start) < len(jobs):
+        jobs = jobs[int(start) :]
+
+    # Do we have a length?
+    if length and int(length) < len(jobs):
+        jobs = jobs[0 : int(length)]
+    return JSONResponse(
+        content={
+            "data": jobs,
+            "draw": draw,
+            "recordsTotal": total,
+            "recordsFiltered": len(jobs),
+        },
+        status_code=200,
+    )
+
+
 @router.get("/jobs")
-async def list_jobs():
+async def list_jobs(request: Request):
     """
     List flux jobs associated with the handle.
     """
-    from app.main import app
-
-    listing = flux.job.job_list(app.handle)
-    jobs = jsonable_encoder({"jobs": listing.get_jobs()})
+    try:
+        payload = await request.json()
+    except Exception:
+        payload = {}
+
+    # Does the requester want details - in dict or listing form?
+    if helpers.has_boolean_arg(payload, "details"):
+
+        # Job limit (only relevant for details)
+        limit = helpers.get_int_arg(payload, "limit")
+
+        jobs = flux_cli.list_jobs_detailed(limit)
+        if helpers.has_boolean_arg(payload, "listing"):
+            jobs = list(jobs.values())
+    else:
+        listing = flux_cli.list_jobs()
+        jobs = jsonable_encoder({"jobs": listing.get_jobs()})
     return JSONResponse(content=jobs, status_code=200)
 
 
@@ -92,19 +145,21 @@ async def submit_job(request: Request):
     """
     from app.main import app
 
-    payload = await request.json()
-    if "command" not in payload or not payload["command"]:
+    # This can bork if no payload is provided
+    try:
+        payload = await request.json()
+    except Exception:
         return JSONResponse(
-            content={"Message": "'command' is required."}, status_code=400
+            content={"Message": "A 'command' is minimally required."}, status_code=400
         )
 
-    # Generate the flux job
-    command = payload["command"]
-    if isinstance(command, str):
-        command = shlex.split(command)
+    kwargs = {}
+
+    # Required arguments
+    for required in ["command"]:
+        kwargs[required] = payload.get(required)
 
-    # Optional defaults
-    kwargs = {"command": command}
+    # Optional arguments
     for optional in [
         "num_tasks",
         "cores_per_task",
@@ -114,21 +169,26 @@ async def submit_job(request: Request):
     ]:
         if optional in payload and payload[optional]:
             kwargs[optional] = payload[optional]
-    fluxjob = JobspecV1.from_command(**kwargs)
 
-    # Does the user want a working directory?
+    # One off args not provided to JobspecV1
+    envars = payload.get("envars", {})
     workdir = payload.get("workdir")
-    if workdir is not None:
-        fluxjob.workdir = workdir
+    runtime = payload.get("runtime", 0) or 0
 
-    # A duration of zero (the default) means unlimited
-    fluxjob.duration = payload.get("runtime", 0) or 0
+    # Validate the payload, return meaningful message if something is off
+    invalid_messages = flux_cli.validate_submit_kwargs(
+        kwargs, envars=envars, runtime=runtime
+    )
+    if invalid_messages:
+        return JSONResponse(
+            content={"Message": "Invalid submit request", "Errors": invalid_messages},
+            status_code=400,
+        )
 
-    # Additional envars in the payload?
-    environment = dict(os.environ)
-    extra_envars = payload.get("envars", {})
-    environment.update(extra_envars)
-    fluxjob.environment = environment
+    # Prepare the flux job!
+    fluxjob = flux_cli.prepare_job(
+        kwargs, runtime=runtime, workdir=workdir, envars=envars
+    )
 
     # Submit the job and return the ID, but allow for error
     try:
@@ -140,8 +200,6 @@ async def submit_job(request: Request):
         return JSONResponse(content=result, status_code=400)
 
     jobid = flux_future.get_id()
-
-    # TODO should we write jobid and other metadata to a database?
     result = jsonable_encoder({"Message": "Job submit.", "id": jobid})
     return JSONResponse(content=result, status_code=200)
 
@@ -151,8 +209,24 @@ async def get_job(jobid):
     """
     Get job info based on id.
     """
-    from app.main import app
+    info = flux_cli.get_job(jobid)
+    info = jsonable_encoder(info)
+    return JSONResponse(content=info, status_code=200)
+
 
-    info = flux.job.job_list_id(app.handle, jobid, attrs=["all"])
-    info = jsonable_encoder(json.loads(info.get_str()))
+@router.get("/jobs/{jobid}/output")
+async def get_job_output(jobid):
+    """
+    Get job output based on id.
+    """
+    lines = flux_cli.get_job_output(jobid)
+
+    # We have output
+    if lines:
+        output = jsonable_encoder({"Output": lines})
+        return JSONResponse(content=output, status_code=200)
+
+    info = jsonable_encoder(
+        {"Message": "The output does not exist yet, or the jobid is incorrect."}
+    )
     return JSONResponse(content=info, status_code=200)
diff --git a/app/routers/views.py b/app/routers/views.py
index 22887b8..c2ce80e 100644
--- a/app/routers/views.py
+++ b/app/routers/views.py
@@ -1,7 +1,128 @@
-from fastapi import APIRouter
+import flux
+import flux.job
+from fastapi import APIRouter, Depends, Request
+from fastapi.responses import HTMLResponse
 from fastapi.templating import Jinja2Templates
 
-router = APIRouter()
+import app.library.flux as flux_cli
+from app.core.config import settings
+from app.forms import SubmitForm
+from app.library.auth import check_auth
+from app.library.helpers import openfile
+
+# These views never have auth!
+router = APIRouter(tags=["views"])
+
+# These views do :)
+auth_views_router = APIRouter(
+    tags=["auth-views"],
+    dependencies=[Depends(check_auth)] if settings.require_auth else [],
+    responses={404: {"description": "Not found"}},
+)
 templates = Jinja2Templates(directory="templates/")
 
-# TODO front end interfaces
+
+@router.get("/", response_class=HTMLResponse)
+async def home(request: Request):
+    """
+    Home page to show welcome, etc.
+    """
+    data = openfile("index.md")
+    return templates.TemplateResponse(
+        "index.html",
+        {
+            "request": request,
+            "data": data,
+        },
+    )
+
+
+# List jobs
+@auth_views_router.get("/jobs", response_class=HTMLResponse)
+async def jobs_table(request: Request):
+    jobs = list(flux_cli.list_jobs_detailed().values())
+    print(jobs)
+    return templates.TemplateResponse(
+        "jobs/jobs.html",
+        {
+            "request": request,
+            "jobs": jobs,
+        },
+    )
+
+
+# View job detail (and log)
+@auth_views_router.get("/job/{jobid}", response_class=HTMLResponse)
+async def job_info(request: Request, jobid):
+    job = flux_cli.get_job(jobid)
+    info = flux_cli.get_job_output(jobid)
+    return templates.TemplateResponse(
+        "jobs/job.html",
+        {
+            "title": f"Job {jobid}",
+            "request": request,
+            "job": job,
+            "info": info,
+        },
+    )
+
+
+# Submit a job via a form
+@auth_views_router.get("/jobs/submit", response_class=HTMLResponse)
+async def submit_job(request: Request):
+    form = SubmitForm(request)
+    return templates.TemplateResponse(
+        "jobs/submit.html",
+        {"request": request, "has_gpus": settings.has_gpus, "form": form},
+    )
+
+
+@auth_views_router.post("/jobs/submit")
+async def submit_job_post(request: Request):
+    """
+    Receive data posted (submit) to the form.
+    """
+    from app.main import app
+
+    form = SubmitForm(request)
+    await form.load_data()
+    if form.is_valid():
+        print("🍦 Submit form is valid!")
+
+        # Prepare the flux job! We don't support envars here yet
+        fluxjob = flux_cli.prepare_job(
+            form.kwargs, runtime=form.runtime, workdir=form.workdir
+        )
+
+        # Submit the job and return the ID, but allow for error
+        try:
+            flux_future = flux.job.submit_async(app.handle, fluxjob)
+            jobid = flux_future.get_id()
+            return templates.TemplateResponse(
+                "jobs/submit.html",
+                context={
+                    "request": request,
+                    "form": form,
+                    "messages": [f"Your job was successfully submit! 🦊 {jobid}"],
+                },
+            )
+        except Exception as e:
+            form.errors.append("There was an issue submitting that job: %s" % str(e))
+    else:
+        print("🍒 Submit form is NOT valid!")
+    return templates.TemplateResponse(
+        "jobs/submit.html",
+        context={
+            "request": request,
+            "form": form,
+            "has_gpus": settings.has_gpus,
+            **form.__dict__,
+        },
+    )
+
+
+# These are generic informational pages
+@auth_views_router.get("/page/{page_name}", response_class=HTMLResponse)
+async def show_page(request: Request, page_name: str):
+    data = openfile(page_name + ".md")
+    return templates.TemplateResponse("page.html", {"request": request, "data": data})
diff --git a/clients/python/CHANGELOG.md b/clients/python/CHANGELOG.md
index af3c300..591ea82 100644
--- a/clients/python/CHANGELOG.md
+++ b/clients/python/CHANGELOG.md
@@ -14,5 +14,6 @@ and **Merged pull requests**. Critical items to know are:
 The versions coincide with releases on pip. Only major versions will be released as tags on Github.
 
 ## [0.0.x](https://github.com/flux-framework/flux-restful-api/tree/main) (0.0.x)
+ - support for job info and logs (0.0.11)
  - Parse envars from the command line, add descriptions for submit (0.0.1)
  - Project skeleton release (0.0.0)
diff --git a/clients/python/flux_restful_client/client/__init__.py b/clients/python/flux_restful_client/client/__init__.py
index ea7fa1d..3a4f650 100644
--- a/clients/python/flux_restful_client/client/__init__.py
+++ b/clients/python/flux_restful_client/client/__init__.py
@@ -173,6 +173,11 @@ def get_parser():
         description="get information for a job",
         formatter_class=argparse.RawTextHelpFormatter,
     )
+    logs = subparsers.add_parser(
+        "logs",
+        description="get log output for a job",
+        formatter_class=argparse.RawTextHelpFormatter,
+    )
     subparsers.add_parser(
         "list-nodes",
         description="list node that the Flux RESTful API knows about",
@@ -188,7 +193,7 @@ def get_parser():
         description="Kill your Flux RESTFul API server (cannot be undone)",
         formatter_class=argparse.RawTextHelpFormatter,
     )
-    for command in cancel, info:
+    for command in cancel, info, logs:
         command.add_argument("jobid", help="The jobid to interact with.")
 
     return parser
@@ -247,6 +252,8 @@ def help(return_code=0):
         from .cancel import main
     elif args.command == "config":
         from .config import main
+    elif args.command == "logs":
+        from .logs import main
     elif args.command == "stop-service":
         from .stop import stop_service as main
     elif args.command == "list-jobs":
diff --git a/clients/python/flux_restful_client/client/logs.py b/clients/python/flux_restful_client/client/logs.py
new file mode 100644
index 0000000..bf982aa
--- /dev/null
+++ b/clients/python/flux_restful_client/client/logs.py
@@ -0,0 +1,16 @@
+import json
+
+from flux_restful_client.main import get_client
+
+
+def main(args, parser, extra, subparser):
+    cli = get_client(quiet=args.quiet, settings=args.settings_file)
+
+    # Update config settings on the fly
+    cli.settings.update_params(args.config_params)
+    logs = cli.output(jobid=args.jobid)
+    if "Output" in logs:
+        for line in logs["Output"]:
+            print(line)
+    else:
+        print(json.dumps(logs, indent=4))
diff --git a/clients/python/flux_restful_client/main/client.py b/clients/python/flux_restful_client/main/client.py
index 31d62fc..35800dd 100644
--- a/clients/python/flux_restful_client/main/client.py
+++ b/clients/python/flux_restful_client/main/client.py
@@ -22,6 +22,7 @@ def __init__(
         token=None,
         quiet=False,
         settings_file=None,
+        prefix="v1",
         **kwargs,
     ):
 
@@ -33,6 +34,8 @@ def __init__(
         self.user = user or os.environ.get("FLUX_USER") or self.settings.flux_user
         self.token = token or os.environ.get("FLUX_TOKEN") or self.settings.flux_token
         self.headers = {}
+        self.quiet = quiet
+        self.prefix = prefix
         if self.user and self.token:
             self.set_basic_auth(self.user, self.token)
         self.session = requests.Session()
@@ -58,7 +61,7 @@ def reset(self):
         else:
             self.headers = {}
 
-    def do_request(self, endpoint, method="GET", data=None, headers=None):
+    def do_request(self, endpoint, method="GET", data=None, headers=None, params=None):
         """
         Do a request. This is a wrapper around requests.
         """
@@ -66,10 +69,12 @@ def do_request(self, endpoint, method="GET", data=None, headers=None):
         self.reset()
 
         headers = headers or self.headers
-        url = "%s/%s" % (self.host, endpoint)
+        url = "%s/%s/%s" % (self.host, self.prefix, endpoint)
 
         # Make the request and return to calling function, unless requires auth
-        response = self.session.request(method, url, json=data, headers=headers)
+        response = self.session.request(
+            method, url, params=params, json=data, headers=headers
+        )
 
         # A 401 response is a request for authentication
         if response.status_code != 401:
@@ -141,25 +146,52 @@ def cancel(self, jobid):
         """
         return self.do_request(f"jobs/{jobid}/cancel", "POST").json()
 
+    def output(self, jobid):
+        """
+        Request for a job to be cancelled based on identifier.
+        """
+        return self.do_request(f"jobs/{jobid}/output", "GET").json()
+
     def stop_service(self):
         """
         Stop the server running.
         """
         return self.do_request("service/stop", "POST").json()
 
-    def jobs(self, jobid=None):
+    def jobs(self, jobid=None, detail=False, listing=False):
         """
         Get a listing of jobs that the Flux RESTful API knows about!
         """
         endpoint = "jobs"
+        params = {}
         if jobid:
             endpoint += "/" + str(jobid)
-        result = self.do_request(endpoint, "GET")
+
+        # This indicates a jobs listing (not a single job)
+        else:
+            if detail:
+                params["detail"] = "true"
+            if listing:
+                params["listing"] = "true"
+        result = self.do_request(endpoint, "GET", params=params)
         if result.status_code == 404:
             print("There is no job for that identifier.")
             return
         return result.json()
 
+    def search(self, query=None, start=None, length=None):
+        """
+        Search endpoint for jobs.
+        """
+        params = {}
+        if query:
+            params["query"] = str(query)
+        if start is not None:
+            params["start"] = start
+        if length is not None:
+            params["length"] = length
+        return self.do_request("jobs/search", "GET", params=params).json()
+
     def submit(self, command, **kwargs):
         """
         Submit a job to the Flux RESTful API
diff --git a/clients/python/flux_restful_client/version.py b/clients/python/flux_restful_client/version.py
index 5551a33..940eef3 100644
--- a/clients/python/flux_restful_client/version.py
+++ b/clients/python/flux_restful_client/version.py
@@ -1,4 +1,4 @@
-__version__ = "0.0.1"
+__version__ = "0.0.11"
 AUTHOR = "Vanessa Sochat"
 EMAIL = "vsoch@users.noreply.github.com"
 NAME = "flux-restful-client"
diff --git a/docs/auto_examples/api_tutorial.ipynb b/docs/auto_examples/api_tutorial.ipynb
index b59e110..07871b2 100644
--- a/docs/auto_examples/api_tutorial.ipynb
+++ b/docs/auto_examples/api_tutorial.ipynb
@@ -108,6 +108,24 @@
         "print(\"\ud83c\udf53 Getting job info...\")\nres = cli.jobs(res[\"id\"])\nif res:\n    print(json.dumps(res, indent=4))"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "And job logs\nThis will be added to the client\n\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "collapsed": false
+      },
+      "outputs": [],
+      "source": [
+        "print(\"\ud83d\ude34 Submitting job to echo pancakes \ud83e\udd5e\ud83e\udd5e\ud83e\udd5e\")\nres = cli.submit(command=\"echo pancakes are really just morning cakes.\")\nres = cli.output(res[\"id\"])\nif res:\n    print(json.dumps(res, indent=4))"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {},
@@ -126,6 +144,24 @@
         "print(\"Submitting 3 jobs to sleep!\")\nfor time in [10, 20, 30]:\n    cli.submit(command=[\"sleep\", time])\nres = cli.jobs()\nif res:\n    print(json.dumps(res, indent=4))"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "And this is how to search (with a start, length, or query)\n\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "collapsed": false
+      },
+      "outputs": [],
+      "source": [
+        "print(\"\ud83c\udf13 Querying jobs!\") \nres = cli.search(\"sleep\", start=1, length=2)\nif res:\n    print(json.dumps(res, indent=4))"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {},
diff --git a/docs/auto_examples/api_tutorial.py b/docs/auto_examples/api_tutorial.py
index 153be2a..094a6d2 100644
--- a/docs/auto_examples/api_tutorial.py
+++ b/docs/auto_examples/api_tutorial.py
@@ -65,6 +65,17 @@
 if res:
     print(json.dumps(res, indent=4))
 
+
+#%%
+# And job logs
+# This will be added to the client
+print("😴 Submitting job to echo pancakes 🥞🥞🥞")
+res = cli.submit(command="echo pancakes are really just morning cakes.")
+res = cli.output(res["id"])
+if res:
+    print(json.dumps(res, indent=4))
+
+
 #%%
 # Now let's submit three jobs in unison so we can list them back!
 # Submit the job to flux
@@ -75,6 +86,14 @@
 if res:
     print(json.dumps(res, indent=4))
 
+#%%
+# And this is how to search (with a start, length, or query)
+print("🌓 Querying jobs!")
+res = cli.search("sleep", start=1, length=2)
+if res:
+    print(json.dumps(res, indent=4))
+
+
 #%%
 # Finally, let's submit and cancel a job
 print("Submitting job sleep 60 intending to cancel..")
diff --git a/docs/auto_examples/api_tutorial.py.md5 b/docs/auto_examples/api_tutorial.py.md5
index 219bb34..78b7422 100644
--- a/docs/auto_examples/api_tutorial.py.md5
+++ b/docs/auto_examples/api_tutorial.py.md5
@@ -1,1 +1,1 @@
-d8f70026580a06a451b0e658b603ae3b
+0d4a42194df06e3f4f8a7dd9a5828e3f
diff --git a/docs/auto_examples/api_tutorial.rst b/docs/auto_examples/api_tutorial.rst
index 7d3cb59..b8a6aad 100644
--- a/docs/auto_examples/api_tutorial.rst
+++ b/docs/auto_examples/api_tutorial.rst
@@ -103,7 +103,7 @@ Let's list the nodes in our cluster!
     Listing nodes
     {
         "nodes": [
-            "3a2a7a9428e6"
+            "57d0f60fce2e"
         ]
     }
 
@@ -153,7 +153,7 @@ FLUX_REQUIRE_AUTH=true
 
 And finally, let's get job info.
 
-.. GENERATED FROM PYTHON SOURCE LINES 63-68
+.. GENERATED FROM PYTHON SOURCE LINES 63-69
 
 .. code-block:: default
 
@@ -166,41 +166,82 @@ And finally, let's get job info.
 
 
 
+
 .. rst-class:: sphx-glr-script-out
 
  .. code-block:: none
 
     🍓 Getting job info...
     {
-        "job": {
-            "id": 13428886339584,
-            "userid": 0,
-            "urgency": 16,
-            "priority": 16,
-            "t_submit": 1667955436.1993887,
-            "t_depend": 1667955436.1993887,
-            "t_run": 1667955436.2116482,
-            "state": 16,
-            "name": "sleep",
-            "ntasks": 1,
-            "ncores": 1,
-            "duration": 0.0,
-            "nnodes": 1,
-            "ranks": "0",
-            "nodelist": "3a2a7a9428e6",
-            "expiration": 4821555436.0
+        "id": 323743595364352,
+        "userid": 0,
+        "urgency": 16,
+        "priority": 16,
+        "t_submit": 1668303686.305028,
+        "t_depend": 1668303686.305028,
+        "t_run": 1668303686.3178189,
+        "state": "RUN",
+        "name": "sleep",
+        "ntasks": 1,
+        "ncores": 1,
+        "duration": 0.0,
+        "nnodes": 1,
+        "ranks": "0",
+        "nodelist": "57d0f60fce2e",
+        "expiration": 4821903686.0,
+        "result": "",
+        "returncode": "",
+        "runtime": 0.47638893127441406,
+        "waitstatus": "",
+        "exception": {
+            "occurred": "",
+            "severity": "",
+            "type": "",
+            "note": ""
         }
     }
 
 
 
 
-.. GENERATED FROM PYTHON SOURCE LINES 69-71
+.. GENERATED FROM PYTHON SOURCE LINES 70-72
+
+And job logs
+This will be added to the client
+
+.. GENERATED FROM PYTHON SOURCE LINES 72-79
+
+.. code-block:: default
+
+    print("😴 Submitting job to echo pancakes 🥞🥞🥞")
+    res = cli.submit(command="echo pancakes are really just morning cakes.")
+    res = cli.output(res["id"])
+    if res:
+        print(json.dumps(res, indent=4))
+
+
+
+
+
+
+.. rst-class:: sphx-glr-script-out
+
+ .. code-block:: none
+
+    😴 Submitting job to echo pancakes 🥞🥞🥞
+    {
+        "Message": "The output does not exist yet, or the jobid is incorrect."
+    }
+
+
+
+
+.. GENERATED FROM PYTHON SOURCE LINES 80-82
 
 Now let's submit three jobs in unison so we can list them back!
 Submit the job to flux
 
-.. GENERATED FROM PYTHON SOURCE LINES 71-78
+.. GENERATED FROM PYTHON SOURCE LINES 82-89
 
 .. code-block:: default
 
@@ -223,31 +264,109 @@ Submit the job to flux
     {
         "jobs": [
             {
-                "id": 13430681501696
+                "id": 323754819321856
+            },
+            {
+                "id": 323754148233216
+            },
+            {
+                "id": 323752906719232
+            },
+            {
+                "id": 323743595364352
+            },
+            {
+                "id": 323752151744512
+            },
+            {
+                "id": 322373450465280
+            },
+            {
+                "id": 322377242116096
+            },
+            {
+                "id": 322376034156544
+            },
+            {
+                "id": 322375480508416
+            },
+            {
+                "id": 322379221827584
+            },
+            {
+                "id": 322374071222272
+            },
+            {
+                "id": 314610330632192
+            },
+            {
+                "id": 314609810538496
+            },
+            {
+                "id": 313911777689600
+            },
+            {
+                "id": 314609307222016
             },
             {
-                "id": 13430010413056
+                "id": 314380113674240
             },
             {
-                "id": 13429473542144
+                "id": 310431579111424
             },
             {
-                "id": 13428886339584
+                "id": 310434615787520
             },
             {
-                "id": 24310185984
+                "id": 310434011807744
             },
             {
-                "id": 25987907584
+                "id": 310433458159616
             },
             {
-                "id": 25467813888
+                "id": 310443658706944
             },
             {
-                "id": 24914165760
+                "id": 310432183091200
             },
             {
-                "id": 26625441792
+                "id": 306433887305728
+            },
+            {
+                "id": 306436286447616
+            },
+            {
+                "id": 305828548575232
+            },
+            {
+                "id": 306435061710848
+            },
+            {
+                "id": 306434508062720
+            },
+            {
+                "id": 306436873650176
+            },
+            {
+                "id": 244509770252288
+            },
+            {
+                "id": 116794505297920
+            },
+            {
+                "id": 39064489164800
+            },
+            {
+                "id": 38133571780608
+            },
+            {
+                "id": 36512691388416
+            },
+            {
+                "id": 30820953751552
+            },
+            {
+                "id": 26132493631488
             }
         ]
     }
@@ -255,11 +374,101 @@ Submit the job to flux
 
 
 
-.. GENERATED FROM PYTHON SOURCE LINES 79-80
+.. GENERATED FROM PYTHON SOURCE LINES 90-91
+
+And this is how to search (with a start, length, or query)
+
+.. GENERATED FROM PYTHON SOURCE LINES 91-97
+
+.. code-block:: default
+
+    print("🌓 Querying jobs!")
+    res = cli.search("sleep", start=1, length=2)
+    if res:
+        print(json.dumps(res, indent=4))
+
+
+
+
+
+
+.. rst-class:: sphx-glr-script-out
+
+ .. code-block:: none
+
+    🌓 Querying jobs!
+    {
+        "data": [
+            {
+                "id": 323754148233216,
+                "userid": 0,
+                "urgency": 16,
+                "priority": 16,
+                "t_submit": 1668303686.9337833,
+                "t_depend": 1668303686.9337833,
+                "t_run": 1668303686.9520447,
+                "state": "RUN",
+                "name": "sleep",
+                "ntasks": 1,
+                "ncores": 1,
+                "duration": 0.0,
+                "nnodes": 1,
+                "ranks": "0",
+                "nodelist": "57d0f60fce2e",
+                "expiration": 4821903686.0,
+                "result": "",
+                "returncode": "",
+                "runtime": 0.08960390090942383,
+                "waitstatus": "",
+                "exception": {
+                    "occurred": "",
+                    "severity": "",
+                    "type": "",
+                    "note": ""
+                }
+            },
+            {
+                "id": 323752906719232,
+                "userid": 0,
+                "urgency": 16,
+                "priority": 16,
+                "t_submit": 1668303686.8595636,
+                "t_depend": 1668303686.8595636,
+                "t_run": 1668303686.872508,
+                "state": "RUN",
+                "name": "sleep",
+                "ntasks": 1,
+                "ncores": 1,
+                "duration": 0.0,
+                "nnodes": 1,
+                "ranks": "0",
+                "nodelist": "57d0f60fce2e",
+                "expiration": 4821903686.0,
+                "result": "",
+                "returncode": "",
+                "runtime": 0.16966652870178223,
+                "waitstatus": "",
+                "exception": {
+                    "occurred": "",
+                    "severity": "",
+                    "type": "",
+                    "note": ""
+                }
+            }
+        ],
+        "draw": 1,
+        "recordsTotal": 35,
+        "recordsFiltered": 2
+    }
+
+
+
+
+.. GENERATED FROM PYTHON SOURCE LINES 98-99
 
 Finally, let's submit and cancel a job
 
-.. GENERATED FROM PYTHON SOURCE LINES 80-88
+.. GENERATED FROM PYTHON SOURCE LINES 99-107
 
 .. code-block:: default
 
@@ -282,7 +491,7 @@ Finally, let's submit and cancel a job
     Submitting job sleep 60 intending to cancel..
     {
         "Message": "Job submit.",
-        "id": 13431470030848
+        "id": 323756681592832
     }
     Requesting job cancel..
     {
@@ -292,11 +501,11 @@ Finally, let's submit and cancel a job
 
 
 
-.. GENERATED FROM PYTHON SOURCE LINES 89-90
+.. GENERATED FROM PYTHON SOURCE LINES 108-109
 
 And this would be how you stop your cluster service
 
-.. GENERATED FROM PYTHON SOURCE LINES 90-92
+.. GENERATED FROM PYTHON SOURCE LINES 109-111
 
 .. code-block:: default
 
@@ -318,7 +527,7 @@ And this would be how you stop your cluster service
 
 .. rst-class:: sphx-glr-timing
 
-   **Total running time of the script:** ( 0 minutes  0.508 seconds)
+   **Total running time of the script:** ( 0 minutes  1.091 seconds)
 
 
 .. _sphx_glr_download_auto_examples_api_tutorial.py:
diff --git a/docs/auto_examples/api_tutorial_codeobj.pickle b/docs/auto_examples/api_tutorial_codeobj.pickle
index a709c2d..903ee1c 100644
Binary files a/docs/auto_examples/api_tutorial_codeobj.pickle and b/docs/auto_examples/api_tutorial_codeobj.pickle differ
diff --git a/docs/auto_examples/auto_examples_jupyter.zip b/docs/auto_examples/auto_examples_jupyter.zip
index d45605d..9c703a0 100644
Binary files a/docs/auto_examples/auto_examples_jupyter.zip and b/docs/auto_examples/auto_examples_jupyter.zip differ
diff --git a/docs/auto_examples/auto_examples_python.zip b/docs/auto_examples/auto_examples_python.zip
index 6208666..b21185b 100644
Binary files a/docs/auto_examples/auto_examples_python.zip and b/docs/auto_examples/auto_examples_python.zip differ
diff --git a/docs/auto_examples/sg_execution_times.rst b/docs/auto_examples/sg_execution_times.rst
index 9c912a8..9ee74e1 100644
--- a/docs/auto_examples/sg_execution_times.rst
+++ b/docs/auto_examples/sg_execution_times.rst
@@ -5,8 +5,8 @@
 
 Computation times
 =================
-**00:00.508** total execution time for **auto_examples** files:
+**00:01.091** total execution time for **auto_examples** files:
 
 +---------------------------------------------------------------------+-----------+--------+
-| :ref:`sphx_glr_auto_examples_api_tutorial.py` (``api_tutorial.py``) | 00:00.508 | 0.0 MB |
+| :ref:`sphx_glr_auto_examples_api_tutorial.py` (``api_tutorial.py``) | 00:01.091 | 0.0 MB |
 +---------------------------------------------------------------------+-----------+--------+
diff --git a/docs/examples/api_tutorial.py b/docs/examples/api_tutorial.py
index 153be2a..094a6d2 100644
--- a/docs/examples/api_tutorial.py
+++ b/docs/examples/api_tutorial.py
@@ -65,6 +65,17 @@
 if res:
     print(json.dumps(res, indent=4))
 
+
+#%%
+# And job logs
+# This will be added to the client
+print("😴 Submitting job to echo pancakes 🥞🥞🥞")
+res = cli.submit(command="echo pancakes are really just morning cakes.")
+res = cli.output(res["id"])
+if res:
+    print(json.dumps(res, indent=4))
+
+
 #%%
 # Now let's submit three jobs in unison so we can list them back!
 # Submit the job to flux
@@ -75,6 +86,14 @@
 if res:
     print(json.dumps(res, indent=4))
 
+#%%
+# And this is how to search (with a start, length, or query)
+print("🌓 Querying jobs!")
+res = cli.search("sleep", start=1, length=2)
+if res:
+    print(json.dumps(res, indent=4))
+
+
 #%%
 # Finally, let's submit and cancel a job
 print("Submitting job sleep 60 intending to cancel..")
diff --git a/docs/getting_started/api.md b/docs/getting_started/api.md
index e2233fc..223ecd3 100644
--- a/docs/getting_started/api.md
+++ b/docs/getting_started/api.md
@@ -1,10 +1,12 @@
 # API
 
-The following is the spec for the Flux Python RESTful API provided here:
+The following is the spec for the Flux Python RESTful API provided here.
+Generally, API endpoints are versioned (e.g., start with v1) and views
+do not.
 
 ## Service
 
-### POST `/service/stop`
+### POST `/v1/service/stop`
 
 Request to stop the server from running. This is intended for applications like
 the Flux Operator that need programmatic ability to end the flux start command
@@ -12,19 +14,38 @@ and thus exit the job and bring down the mini-cluster.
 
 ## Jobs
 
-### GET `/jobs`
+### GET `/v1/jobs`
 
 List jobs owned by the flux executor (the current user).
+By default, this returns a simple listing with "jobs" -> a list of dict with ids.
 
-### GET `/jobs/{uid}`
+#### Payload
 
-Get a job with a specific identifier.
+**Optional** parameters in the get are:
 
-### POST `/jobs/{uid}/cancel`
+- details (bool): provide details as True if you want to get complete metadata for the job
+- listing (bool): provide listing as True if you want to get details in a list
+- limit (int): provide a maximum number of jobs to retrieve (defaults to all if not provided)
 
-Request for a job cancellation based on identifier.
+### GET '/v1/jobs/search'
+
+A custom search endpoint to query for jobs. It is used internally by the site to
+render data tables and similar.
+
+**Optional** parameters:
+
+- start: a number to start at, must be <= the length of total jobs
+- length: a length to stop at, after the start is applied (if applicable)
+- query: a string to search all attributes for
+
+**Returns** parameters:
+
+ - data: the list of jobs
+ - recordsTotal: the total number of jobs available
+ - recordsFiltered: the total after fitlering by the query, length, start
+ - draw: an integer used by Jquery Datatables
 
-### POST `/jobs/submit`
+### POST `/v1/jobs/submit`
 
 Submit a new job
 
@@ -45,9 +66,20 @@ Submit a new job
 - num_nodes (int): Number of nodes (defaults to None)
 - exclusive (bool): is the job exclusive? (defaults to False)
 
+### GET `/v1/jobs/{uid}`
+
+Get a job with a specific identifier.
+
+### POST `/v1/jobs/{uid}/cancel`
+
+Request for a job cancellation based on identifier.
+
+### GET `/v1/jobs/{uid}/output`
+
+Get lines of job output.
 
 ## Nodes
 
-### GET `/nodes`
+### GET `/v1/nodes`
 
 List cluster nodes.
diff --git a/docs/getting_started/developer-guide.md b/docs/getting_started/developer-guide.md
index c79d2db..6591ddd 100644
--- a/docs/getting_started/developer-guide.md
+++ b/docs/getting_started/developer-guide.md
@@ -125,7 +125,18 @@ and we will soon add a table of jobs.
 Once you have the server running, you can use an example client to interact
 with the server. See our [User Guide](https://flux-framework.org/flux-restful-api/getting_started/user-guide.html) for these instructions.
 
+## Environment
 
+Wherever you run the app, you can control variables (settings) via the environment.
+The following variables are available (with their defaults):
+
+| Name | Description | Default |
+|------|-------------|---------|
+|FLUX_REQUIRE_AUTH| The server should require basic auth for API and authenticated endpoints | False (unset) |
+|FLUX_TOKEN| The token password to require for Basic Auth (if `FLUX_REQUIRE_AUTH` is set) | unset |
+|FLUX_USER| The username to require for Basic Auth (if `FLUX_REQUIRE_AUTH` is set) | unset |
+|FLUX_HAS_GPU | GPUs are available for the user to request | unset |
+|FLUX_NUMBER_NODES| The number of nodes available in the cluster | 1 |
 
 ## Code Linting
 
@@ -189,7 +200,7 @@ dependencies (done in devcontainer):
 
 ```console
 cd docs
-pip install -r requrements.txt
+pip install -r requirements.txt
 
 # Build the docs into _build/html
 make html
@@ -206,6 +217,16 @@ $ python -m http.server 9999
 
 And open your browser to `localhost:9999`
 
+
+### Run Tests
+
+To run tests, from within the devcontainers environment (or with a local install)
+of Flux alongside the app) you can do:
+
+```bash
+$ flux start pytest -xs tests/test_api.py
+```
+
 ### Docstrings
 
 To render our Python API into the docs, we keep an updated restructured
diff --git a/docs/getting_started/img/api.png b/docs/getting_started/img/api.png
new file mode 100644
index 0000000..42c5a4d
Binary files /dev/null and b/docs/getting_started/img/api.png differ
diff --git a/docs/getting_started/img/home.png b/docs/getting_started/img/home.png
new file mode 100644
index 0000000..17e7ebd
Binary files /dev/null and b/docs/getting_started/img/home.png differ
diff --git a/docs/getting_started/img/job-info.png b/docs/getting_started/img/job-info.png
new file mode 100644
index 0000000..781e9f6
Binary files /dev/null and b/docs/getting_started/img/job-info.png differ
diff --git a/docs/getting_started/img/jobs-table.png b/docs/getting_started/img/jobs-table.png
new file mode 100644
index 0000000..5ef1b2a
Binary files /dev/null and b/docs/getting_started/img/jobs-table.png differ
diff --git a/docs/getting_started/img/submit.png b/docs/getting_started/img/submit.png
new file mode 100644
index 0000000..ddc2275
Binary files /dev/null and b/docs/getting_started/img/submit.png differ
diff --git a/docs/getting_started/index.md b/docs/getting_started/index.md
index 8f8cd2a..4f0eb20 100644
--- a/docs/getting_started/index.md
+++ b/docs/getting_started/index.md
@@ -12,5 +12,6 @@ any questions or issues, please [let us know](https://github.com/flux-framework/
 :maxdepth: 3
 user-guide
 developer-guide
+interface
 api
 ```
diff --git a/docs/getting_started/interface.md b/docs/getting_started/interface.md
new file mode 100644
index 0000000..cb9116f
--- /dev/null
+++ b/docs/getting_started/interface.md
@@ -0,0 +1,49 @@
+# Interface
+
+The following interfaces are provided by the Flux Python RESTful API.
+If you deploy a server with authentication, all views except for the home page
+will be behind a basic authentication (asking for a user name and password).
+For the API, you'll be able to see endpoints but not interact with them.
+Note that these endpoints are distinct from the [API](https://flux-framework.org/flux-restful-api/getting_started/api.html) in that
+they are not prefixed with "v1."
+
+## Home
+
+### `GET /`
+
+The home screen (portal) is the only page that is always available without authentication.
+It's quite useless, really, aside to say hello 👋! Yes. you've found it. Good job.
+It's so boring I'm not even going to show it.
+
+## Jobs
+
+### `GET /jobs/submit`
+
+You likely want to submit jobs, and then see them progress. We first do this via the Submit portal:
+
+![img/submit.png](img/submit.png)
+
+
+### `GET /jobs`
+
+And then the jobs table to view all jobs:
+
+![img/jobs-table.png](img/jobs-table.png)
+
+### `GET /jobs/<jobid>`
+
+And then clicking a job id will show you job info, and the latest log.
+
+![img/job-info.png](img/job-info.png)
+
+You can refresh the page
+to re-request the logs, and maybe someday we will have a fancy-shmancy thing that automatically
+does this. For now I'd rather not make longer or additional requests to Flux if it's not necessary.
+
+## API
+
+### `GET /docs`
+
+To interact with the API (or see available endpoints) you can go to this view.
+
+![img/api.png](img/api.png)
diff --git a/docs/getting_started/user-guide.md b/docs/getting_started/user-guide.md
index 3defb3a..62a6fe3 100644
--- a/docs/getting_started/user-guide.md
+++ b/docs/getting_started/user-guide.md
@@ -4,7 +4,8 @@ Welcome to the Flux RESTful API user guide! If you come here, we are assuming
 that you have a deployment available to use to interact with. If you need
 to deploy your own API, see the [Developer Documentation](https://flux-framework.org/flux-restful-api/getting_started/developer-guide.html).
 We provide [clients](https://github.com/flux-framework/flux-restful-api/tree/main/clients) to interact with
-a running server, each described below. The Python client is also included in our [Tutorials](https://flux-framework.org/flux-restful-api/auto_examples/index.html).
+a running server, each described below, along with a quick container tutorial (coming soon).
+The Python client is also included in our [Tutorials](https://flux-framework.org/flux-restful-api/auto_examples/index.html).
 
 ## Environment
 
@@ -168,6 +169,21 @@ $ flux-restful-cli info 12402053611520
 }
 ```
 
+### Logs
+
+To see output for a job, just do:
+
+```bash
+$ flux-restful-cli logs 12402053611520
+```
+```console
+# flux-restful-cli logs 244509770252288
+pancakes 🥞🥞🥞🥞🥞
+```
+
+If you get a message that the output doesn't exist, it usually means your job didn't have output (e.g., sleep),
+or the job was not found.
+
 ### List Jobs
 
 You can list jobs:
diff --git a/img/flux-restful.png b/img/flux-restful.png
index 47b2f46..42c5a4d 100644
Binary files a/img/flux-restful.png and b/img/flux-restful.png differ
diff --git a/templates/base.html b/templates/base.html
index 116fadf..ca95056 100644
--- a/templates/base.html
+++ b/templates/base.html
@@ -36,6 +36,7 @@
         <!-- Page Content  -->
         <div id="content">
             {% include 'include/topnav.html' %}
+            {% include 'include/messages.html' %}
             {% block page_content %}
             {% endblock %}
         </div>
diff --git a/templates/include/jobs/result.html b/templates/include/jobs/result.html
new file mode 100644
index 0000000..e8e5dfe
--- /dev/null
+++ b/templates/include/jobs/result.html
@@ -0,0 +1,3 @@
+{% if job.result == "COMPLETED "%}<button class="btn" style="background-color:darkseagreen">{{ job.result }}</button>{% else %}
+<button class="btn" style="background-color:skyblue">{{ job.result }}</button>
+{% endif %}
diff --git a/templates/include/jobs/state.html b/templates/include/jobs/state.html
new file mode 100644
index 0000000..f1fc165
--- /dev/null
+++ b/templates/include/jobs/state.html
@@ -0,0 +1,6 @@
+{% if job.state == "INACTIVE" %}<button class="btn" style="background-color:grey;color:white">{{ job.state }} 😴 </button>{% elif job.state == "RUN" %}
+<button class="btn" style="background-color:grey">{{ job.state }} 🏃 </button>{% elif job.state == "CANCELLED" %}
+<button class="btn" style="background-color:grey">{{ job.state }} 🎉  </button>{% elif job.state == "COMPLETED" %}
+<button class="btn" style="background-color:grey">{{ job.state }} 🚧  </button>{% else %}
+<button class="btn" style="background-color:skyblue">{{ job.state }}</button>
+{% endif %}
diff --git a/templates/include/messages.html b/templates/include/messages.html
new file mode 100644
index 0000000..8c25d49
--- /dev/null
+++ b/templates/include/messages.html
@@ -0,0 +1,10 @@
+{% if messages %}
+<div class="container">
+   <div class="row">
+      <div class="col">
+        <div class="mb-3">{% for message in messages %}
+            <p class="alert alert-info alert-dismissible">{{ message }}</p>{% endfor %}
+        </div>
+      </div>
+   </div>
+</div>{% endif %}
diff --git a/templates/include/sidebar.html b/templates/include/sidebar.html
index f3acef0..fb02703 100644
--- a/templates/include/sidebar.html
+++ b/templates/include/sidebar.html
@@ -9,8 +9,15 @@ <h3>Flux RESTFul Interface</h3>
     </div>
 
     <ul class="list-unstyled components">
-        <li class="active">
-            <a href="/">Home</a>
+        <li><a href="/">Home</a></li>
+        <li>
+            <a href="/jobs">Jobs</a>
+        </li>
+        <li>
+            <a href="/jobs/submit">Submit</a>
+        </li>
+        <li>
+            <a href="/docs">Docs</a>
         </li>
         <li class="dropdown">
             <a href="#"class="dropdown-toggle" id="dropdownMenuButton1"  data-bs-toggle="dropdown" aria-expanded="false">Pages</a>
diff --git a/templates/include/topnav.html b/templates/include/topnav.html
index e43b427..92235ca 100644
--- a/templates/include/topnav.html
+++ b/templates/include/topnav.html
@@ -3,13 +3,17 @@
         <button type="button" id="sidebarCollapse" class="btn btn-info">
             <i class="fas fa-align-left"></i>
         </button>
+        {% if title %}<h3 style="margin-left:20px">{{ title }}</h3>{% endif %}
+
         <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbarSupportedContent" aria-controls="navbarSupportedContent" aria-expanded="false" aria-label="Toggle navigation">
             <span class="navbar-toggler-icon"></span>
         </button>
-
         <div class="collapse navbar-collapse" id="navbarSupportedContent">
             <ul class="navbar-nav me-auto mb-2 mb-lg-0">
-                <li class="nav-item {{'active' if active_page == 'api' }}">
+                <li class="nav-item {{'active' if active_page == 'submit_job' }}">
+                    <a class="nav-link" href="/jobs/submit">Submit</a>
+                </li>
+                <li class="nav-item {{'active' if active_page == 'jobs' }}">
                     <a class="nav-link" href="/jobs">Jobs</a>
                 </li>
                 <li class="nav-item {{'active' if active_page == 'api' }}">
diff --git a/templates/jobs/job.html b/templates/jobs/job.html
new file mode 100644
index 0000000..00e097a
--- /dev/null
+++ b/templates/jobs/job.html
@@ -0,0 +1,88 @@
+{% extends "base.html" %}
+
+{% block title %}Flux RESTFul Interface{% endblock %}
+{% block head %}
+{{ super() }}
+
+
+<link href="https://cdnjs.cloudflare.com/ajax/libs/prism/1.23.0/themes/prism-coy.min.css" rel="stylesheet" type="text/css">
+{% endblock %}
+
+{% block page_content %}
+{% include 'include/sidebar.html' %}
+<main role="main" class="container-wide card" style="padding:30px">
+    <div class="row">
+        <div class="col-md-3">
+            <table class="table table-hover">
+                <thead>
+                  <tr>
+                    <th scope="col">Attribute</th>
+                    <th scope="col">Value</th>
+                  </tr>
+                </thead>
+                <tbody>
+                  <tr>
+                    <td scope="row">ID</td>
+                    <td>{{ job.id }}</td>
+                  </tr>
+                  <tr>
+                    <td scope="row">Name</td>
+                    <td>{{ job.name }}</td>
+                  </tr>
+                  <tr>
+                    <td scope="row">State</td>
+                    <td>{% include 'include/jobs/state.html' %}</td>
+                  </tr>
+                  <tr>
+                    <td scope="row">Result</td>
+                    <td>{% include 'include/jobs/result.html' %}</td>
+                  </tr>
+                  <tr>
+                    <td scope="row">Urgency</td>
+                    <td>{{ job.urgency }}</td>
+                  </tr>
+                  <tr>
+                    <td scope="row">Priority</td>
+                    <td>{{ job.priority }}</td>
+                  </tr>
+                  <tr>
+                    <td scope="row">Number Tasks</td>
+                    <td>{{ job.ntasks }}</td>
+                  </tr>
+                  <tr>
+                    <td scope="row">Cores</td>
+                    <td>{{ job.ncores }}</td>
+                  </tr>
+                  <tr>
+                    <td scope="row">Nodes</td>
+                    <td>{{ job.nnodes }} ({{ job.nodelist}})</td>
+                  </tr>
+                  <tr>
+                    <td scope="row">Ranks</td>
+                    <td>{{ job.ranks }}</td>
+                  </tr>
+                  <tr>
+                    <td scope="row">Runtime</td>
+                    <td>{{ job.runtime }}</td>
+                  </tr>
+                  <tr>
+                    <td scope="row">Return Code</td>
+                    <td>{{ job.returncode }}</td>
+                  </tr>
+                </tbody>
+              </table>
+        </div>
+        <div class="col-md-9">
+            <pre><code class="language-bash">{% if info %}{% for line in info %}{{ line }}{% endfor %}{% else %}
+            This job did not produce any output.{% endif %}
+           </code></pre>
+        </div>
+    </div>
+</main><!-- /.container -->
+
+{% endblock %}
+
+{% block scripts %}
+{{ super() }}
+<script src="https://cdnjs.cloudflare.com/ajax/libs/prism/1.23.0/prism.min.js"></script>
+{% endblock %}
diff --git a/templates/jobs/jobs.html b/templates/jobs/jobs.html
new file mode 100644
index 0000000..c5f71cd
--- /dev/null
+++ b/templates/jobs/jobs.html
@@ -0,0 +1,161 @@
+{% extends "base.html" %}
+
+{% block title %}Flux RESTFul Interface{% endblock %}
+{% block head %}
+{{ super() }}
+
+<!-- Custom styles for this template -->
+<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/normalize/8.0.1/normalize.min.css">
+<link rel="stylesheet" href="//fonts.googleapis.com/css?family=Source+Sans+Pro:300,300i,600">
+<link rel='stylesheet' href='https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.1.1/css/all.min.css'>
+
+<!-- Custom CSS -->
+<link href="https://fonts.googleapis.com/css?family=Nunito+Sans" rel="stylesheet">
+
+<link rel='stylesheet' href='https://cdn.datatables.net/1.12.1/css/dataTables.bootstrap4.min.css'>
+<link rel="stylesheet" href="https://cdn.datatables.net/1.13.1/css/jquery.dataTables.min.css">
+<style>
+.dataTables_wrapper .dataTables_paginate .paginate_button {
+  padding: 0px !important
+}
+.dataTables_wrapper .dataTables_paginate .paginate_button:hover {
+  color: white !important;
+  border: 1px solid white;
+  background-color: white;
+}
+</style>
+
+<style>
+#software_filter, #software_length, #software_info {
+ color: #333;
+}
+td, th {
+  color: #333;
+}
+.tag {
+  margin-right: 2px !important;
+}
+</style>
+{% endblock %}
+
+{% block page_content %}
+{% include 'include/sidebar.html' %}
+
+<div class="container-wide">
+    <a type="button" class="btn btn-info" style="margin-bottom:10px">Submit Another Job</button>
+    <a type="button" class="btn btn-info filter-reset" onclick="$('#jobs').DataTable().search('').draw()" style='float:right;padding-bottom:5px' href="#">reset</a></li>
+    <table id="jobs"  class="table table-bordered" cellspacing="0" width="100%">
+        <thead>
+            <tr>
+                <th>id</th>
+                <th>returncode</th>
+                <th>runtime</th>
+                <th>result</th>
+                <th>urgency</th>
+                <th>priority</th>
+                <th>state</th>
+                <th>name</th>
+                <th>ntasks</th>
+                <th>ncores</th>
+                <th>duration</th>
+                <th>nnodes</th>
+                <th>ranks</th>
+                <th>nodelist</th>
+                <th>expiration</th>
+                <th>waitstatus</th>
+                <th>exception</th>
+            </tr>
+            <tbody></tbody>
+        </thead>
+    </table>
+</div>
+
+{% endblock %}
+
+{% block scripts %}
+{{ super() }}
+
+<script src="https://cdn.datatables.net/1.13.1/js/jquery.dataTables.min.js"></script>
+<script src="https://cdn.datatables.net/1.13.1/js/dataTables.bootstrap5.min.js"></script>
+<script>
+
+$(document).ready(function() {
+
+    $('#jobs').DataTable({
+    "processing": true,
+    "pageLength": 100,
+    "serverSide": true,
+    "sPaginationType": "full_numbers",
+    "order": [],
+    "ajax": {
+        "url": "/v1/jobs/search",
+        "type": 'get',
+        "dataType": 'json'
+    },
+    columnDefs: [
+             { // id
+                render: function (data, type, row) {
+                    return "<a style='color:darkmagenta' href='/job/" + data +"'>" + data + "</a>"
+                },
+                targets: 0,
+            },
+             { // exception
+                render: function (data, type, row) {
+                    if (data['occurred'] == true) {
+                        return "<span class='badge badge-warning'>" + data['note'] + "</span>"
+                    }
+                    return "no exceptions"
+                },
+                targets: 16,
+            },
+            { // result
+                createdCell: function (td, data, rowData, row, col) {
+                    color = "skyblue"
+                    if (data == "INACTIVE") {
+                        color = "grey"
+                    }
+                    if (data == "CANCELLED") {
+                        color = "yellow"
+                    }
+                    if (data == "COMPLETED") {
+                        color = "darkseagreen"
+                    }
+                    $(td).css('background-color', color);
+                },
+                targets: 3
+            },
+            { // status
+                createdCell: function (td, data, rowData, row, col) {
+                    color = "skyblue"
+                    if (data == "INACTIVE") {
+                        color = "grey"
+                    }
+                    $(td).css('background-color', color);
+                },
+                targets: 6
+            },
+        ],
+        "columns": [
+        { "data": "id" },
+        { "data": "returncode" },
+        { "data": "runtime" },
+        { "data": "result" },
+        { "data": "urgency" },
+        { "data": "priority" },
+        { "data": "state" },
+        { "data": "name" },
+        { "data": "ntasks" },
+        { "data": "ncores" },
+        { "data": "duration" },
+        { "data": "nnodes" },
+        { "data": "ranks" },
+        { "data": "nodelist" },
+        { "data": "expiration" },
+        { "data": "waitstatus" },
+        { "data": "exception" },
+    ]
+  });
+  $('#jobs_filter').parent().attr("class", "col-md-12")
+});
+</script>
+{% endblock %}
diff --git a/templates/jobs/submit.html b/templates/jobs/submit.html
new file mode 100644
index 0000000..93bff1f
--- /dev/null
+++ b/templates/jobs/submit.html
@@ -0,0 +1,75 @@
+{% extends "base.html" %}
+
+{% block title %}Flux RESTFul Interface{% endblock %}
+{% block head %}
+{{ super() }}
+
+{% endblock %}
+
+{% block page_content %}
+{% include 'include/sidebar.html' %}
+<main role="main" class="container card" style="padding:30px">
+    <div class="row">
+        <div class="col">
+            <h3>Submit a Job</h3>
+            <form id="submitform" method="POST">
+                <button type="submit" class="btn btn-info" style="float:right; margin-bottom:5px">Submit</button>
+                <a href="/jobs" type="button" class="btn btn-info" style="float:right; margin-bottom:5px; margin-right:5px">View Jobs</a>
+                {% if form.errors %}<div class="mb-3">{% for error in form.errors %}
+                    <p class="alert alert-danger alert-dismissible">{{ error }}</p>{% endfor %}
+                </div>{% endif %}
+                <div class="mb-3">
+                  <label for="command" class="form-label">Command</label>
+                  <input class="form-control" name="command" id="command" aria-describedby="commandHelp" {% if form.command %}value="{{ form.command }}"{% endif %} required>
+                  <div id="commandHelp" class="form-text">The full command to provide to flux (required).</div>
+                </div>
+                <div class="mb-3">
+                    <label for="workdir" class="form-label">Working Directory</label>
+                    <input class="form-control" {% if form.workdir %}value="{{ form.workdir }}"{% endif %} name="workdir" id="workdir" aria-describedby="workdirHelp">
+                    <div id="workdirHelp" class="form-text">A custom working directory in the job container (optional).</div>
+                </div>
+                <div class="mb-3">
+                    <label for="runtime" class="form-label">Maximum Runtime</label>
+                    <input type="number" min="0" {% if form.runtime %}value="{{ form.runtime }}"{% endif %} class="form-control" name="runtime" id="runtime" aria-describedby="runtimeHelp">
+                    <div id="runtimeHelp" class="form-text">The maximum runtime in minutes, 0 means no limit (optional).</div>
+                </div>
+                <div class="mb-3">
+                    <label for="num_tasks" class="form-label">Number Tasks</label>
+                    <input type="number" min="1" class="form-control" name="num_tasks" id="num_tasks" {% if form.num_tasks %}value="{{ form.num_tasks }}"{% endif %} aria-describedby="num_tasksHelp">
+                    <div id="num_tasksHelp" class="form-text">Number of tasks (optional).</div>
+                </div>
+                <div class="mb-3">
+                    <label for="cores_per_task" class="form-label">Cores Per Task</label>
+                    <input type="number" min="1" class="form-control" name="cores_per_task" {% if form.cores_per_task %}value="{{ form.cores_per_task }}"{% endif %} id="cores_per_task" aria-describedby="cores_per_taskHelp">
+                    <div id="cores_per_tasksHelp" class="form-text">Cores per task, defaults to 1 (optional).</div>
+                </div>
+                <div class="mb-3">
+                    <label for="gpus_per_task" class="form-label">GPUs Per Task</label>
+                    <input type="number" min="1" class="form-control" name="gpus_per_task" {% if form.gpus_per_task %}value="{{ form.gpus_per_task }}"{% endif %} id="gpus_per_task" aria-describedby="gpus_per_taskHelp" {% if has_gpus %}disabled{% endif %}>
+                    <div id="gpus_per_tasksHelp" class="form-text">GPUs per task, defaults to 1 (optional).</div>
+                </div>
+                <div class="mb-3">
+                    <label for="num_nodes" class="form-label">Number Nodes</label>
+                    <input type="number" min="1" max="{{num_nodes}}" name="num_nodes" class="form-control" id="num_nodes" aria-describedby="num_nodesHelp">
+                    <div id="num_nodesHelp" class="form-text">Number of nodes to request for the job, defaults to 1 (optional).</div>
+                </div>
+                <div class="mb-3 form-check">
+                  <input type="checkbox" class="form-check-input" id="exclusive" aria-describedby="exclusiveHelp">
+                  <label class="form-check-label" for="exclusive" name="exlusive" {% if form.exclusive %}checked{% endif %}>Exclusive</label>
+                  <div id="exclusiveHelp" class="form-text">Ask for exclusive nodes (only used by this job).</div>
+                </div>
+                <div class="mb-3">
+                  <p class="alert alert-info">Additional environment variables are not yet supported through the user interface! If you need this, please use a command line client.</p>
+                </div>
+                <button type="submit" class="btn btn-info">Submit</button>
+              </form>
+        </div>
+    </div>
+</main><!-- /.container -->
+
+{% endblock %}
+
+{% block scripts %}
+{{ super() }}
+
+{% endblock %}


Test patch:
diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml
index 3f03bbe..6a62cc3 100644
--- a/.github/workflows/tests.yaml
+++ b/.github/workflows/tests.yaml
@@ -17,4 +17,5 @@ jobs:
     - name: Run tests
       run: |
         flux start pytest -xs tests/test_api.py
-        flux start pytest -xs tests/test_api_auth.py
+        export TEST_AUTH=true
+        flux start pytest -xs tests/test_api.py
diff --git a/clients/python/flux_restful_client/tests/test_api.py b/clients/python/flux_restful_client/tests/test_api.py
index a04dff8..eb1bb70 100644
--- a/clients/python/flux_restful_client/tests/test_api.py
+++ b/clients/python/flux_restful_client/tests/test_api.py
@@ -53,3 +53,53 @@ def test_cancel_job():
     response = client.cancel(jobid)
     assert "Message" in response
     assert "cannot be cancelled" in response["Message"]
+
+
+def test_job_output():
+    """
+    Test endpoint to retrieve list of job output
+    """
+    client = get_client()
+
+    # Now submit a job, ensure we get one job back
+    result = client.submit("echo pancakes 🥞️🥞️🥞️")
+    jobid = result["id"]
+    lines = client.output(jobid)
+
+    # First try, often we won't have output yet
+    if "Message" in lines:
+        assert "not exist yet" in lines["Message"]
+    time.sleep(3)
+
+    # Try again - we should have it after a sleep
+    lines = client.output(jobid)
+    assert "Output" in lines
+    assert "pancakes 🥞️🥞️🥞️\n" in lines["Output"]
+
+
+def test_job_query():
+    """
+    Test endpoint to query jobs
+    """
+    client = get_client()
+
+    # Submit 5 jobs
+    for _ in range(5):
+        client.submit("sleep 1")
+    result = client.search()
+
+    for key in ["recordsTotal", "recordsFiltered", "draw", "data"]:
+        assert key in result
+
+    total = result["recordsTotal"]
+    assert len(result["data"]) == total
+
+    # Ask to start at 2 (should be one less record)
+    result = client.search(start=1)
+    assert result["recordsFiltered"] == total - 1
+    assert result["recordsTotal"] == total
+
+    # Ask for specific length
+    result = client.search(start=1, length=3)
+    assert result["recordsFiltered"] == 3
+    assert result["recordsTotal"] == total
diff --git a/tests/test_api.py b/tests/test_api.py
index 17556ac..0f387dd 100644
--- a/tests/test_api.py
+++ b/tests/test_api.py
@@ -1,5 +1,7 @@
+import base64
 import os
 import sys
+import time
 
 from fastapi.testclient import TestClient
 
@@ -12,24 +14,59 @@
 
 client = TestClient(app)
 
+here = os.path.abspath(os.path.dirname(__file__))
+root = os.path.dirname(here)
+
+sys.path.insert(0, root)
+
+
+def get_basic_auth(username, password):
+    auth_str = "%s:%s" % (username, password)
+    return base64.b64encode(auth_str.encode("utf-8")).decode("utf-8")
+
+
+def get_headers():
+    auth_header = get_basic_auth(flux_user, flux_token)
+    if isinstance(auth_header, bytes):
+        auth_header = auth_header.decode("utf-8")
+    return {"Authorization": "Basic %s" % auth_header}
+
+
+# Do we want auth?
+test_auth = False
+headers = {}
+if os.environ.get("TEST_AUTH"):
+
+    # Define authentication in environment for server
+    flux_user = "fluxuser"
+    flux_token = "12345"
+    os.environ["FLUX_USER"] = flux_user
+    os.environ["FLUX_TOKEN"] = flux_token
+    os.environ["FLUX_REQUIRE_AUTH"] = "true"
+    test_auth = True
+    headers = get_headers()
+
 
 def test_submit_list_job():
     """
     Test a manual submission
     """
     # No jobs are submit!
-    response = client.get("/jobs")
+    response = client.get("/v1/jobs", headers=headers)
     assert response.status_code == 200
     assert not response.json()["jobs"]
 
     # Now submit a job, ensure we get one job back
-    response = client.post("/jobs/submit", json={"command": "sleep 5"})
+    response = client.post(
+        "/v1/jobs/submit", json={"command": "sleep 5"}, headers=headers
+    )
     assert response.status_code == 200
     result = response.json()
     assert "id" in result
     jobid = result["id"]
 
-    response = client.get("/jobs")
+    # List jobs with details
+    response = client.get("/v1/jobs", headers=headers)
     assert response.status_code == 200
     result = response.json()
     assert "jobs" in result
@@ -37,14 +74,107 @@ def test_submit_list_job():
     assert len(result) == 1
     assert result[0]["id"] == jobid
 
+    # No details
+    assert len(result[0].keys()) == 1
+
+    response = client.get("/v1/jobs", headers=headers, json={"details": True})
+    assert response.status_code == 200
+    result = response.json()
+    assert str(jobid) in result
+    result = result[str(jobid)]
+
+    # There are 25 total
+    assert len(result) > 20
+    assert result["id"] == jobid
+
+    # Testing asking for a listing
+    response = client.get(
+        "/v1/jobs", headers=headers, json={"details": True, "listing": True}
+    )
+    assert response.status_code == 200
+    result = response.json()
+    assert isinstance(result, list)
+    assert result[0]["id"] == jobid
+
+    # And setting an integer limit (only relevant for details)
+    response = client.get(
+        "/v1/jobs", headers=headers, json={"limit": 0, "details": True}
+    )
+    assert response.status_code == 200
+    result = response.json()
+    assert not result
+
 
 def test_cancel_job():
 
     # Now submit a job, ensure we get one job back
-    response = client.post("/jobs/submit", json={"command": "sleep 10"})
+    response = client.post(
+        "/v1/jobs/submit", json={"command": "sleep 10"}, headers=headers
+    )
     assert response.status_code == 200
     result = response.json()
     jobid = result["id"]
-    response = client.post(f"/jobs/{jobid}/cancel")
+    response = client.post(f"/v1/jobs/{jobid}/cancel", headers=headers)
     assert response.status_code == 200
-    # TODO we don't have way to actually verify this!
+    # TODO we don't have way to actually verify that cancel happened
+
+
+def test_job_output():
+    """
+    Test endpoint to retrieve list of job output
+    """
+
+    # Now submit a job, ensure we get one job back
+    response = client.post(
+        "/v1/jobs/submit", json={"command": "echo pancakes 🥞️🥞️🥞️"}, headers=headers
+    )
+    assert response.status_code == 200
+    result = response.json()
+    jobid = result["id"]
+    res = client.get(f"/v1/jobs/{jobid}/output", headers=headers)
+    assert response.status_code == 200
+    lines = res.json()
+
+    # First try, often we won't have output yet
+    if "Message" in lines:
+        assert "not exist yet" in lines["Message"]
+    time.sleep(3)
+
+    # Try again - we should have it after a sleep
+    res = client.get(f"/v1/jobs/{jobid}/output", headers=headers)
+    assert response.status_code == 200
+    lines = res.json()
+    assert "Output" in lines
+    assert "pancakes 🥞️🥞️🥞️\n" in lines["Output"]
+
+
+def test_job_query():
+    """
+    Test endpoint to query jobs
+    """
+    # Submit 5 jobs
+    for _ in range(5):
+        client.post("/v1/jobs/submit", json={"command": "sleep 1"}, headers=headers)
+
+    response = client.get("/v1/jobs/search", headers=headers)
+    assert response.status_code == 200
+    result = response.json()
+
+    for key in ["recordsTotal", "recordsFiltered", "draw", "data"]:
+        assert key in result
+    total = result["recordsTotal"]
+    assert len(result["data"]) == total
+
+    # Ask to start at 2 (should be one less record)
+    response = client.get("/v1/jobs/search", params={"start": 1}, headers=headers)
+    assert response.status_code == 200
+    result = response.json()
+    assert result["recordsFiltered"] == total - 1
+    assert result["recordsTotal"] == total
+
+    # Ask for specific length
+    response = client.get("/v1/jobs/search", params={"length": 3}, headers=headers)
+    assert response.status_code == 200
+    result = response.json()
+    assert result["recordsFiltered"] == 3
+    assert result["recordsTotal"] == total
diff --git a/tests/test_api_auth.py b/tests/test_api_auth.py
deleted file mode 100644
index 3f4f37d..0000000
--- a/tests/test_api_auth.py
+++ /dev/null
@@ -1,77 +0,0 @@
-import base64
-import os
-import sys
-
-from fastapi.testclient import TestClient
-
-here = os.path.abspath(os.path.dirname(__file__))
-root = os.path.dirname(here)
-
-sys.path.insert(0, root)
-
-# Define authentication in environment for server
-flux_user = "fluxuser"
-flux_token = "12345"
-os.environ["FLUX_USER"] = flux_user
-os.environ["FLUX_TOKEN"] = flux_token
-os.environ["FLUX_REQUIRE_AUTH"] = "true"
-
-
-from app.main import app  # noqa
-
-client = TestClient(app)
-
-
-def get_basic_auth(username, password):
-    auth_str = "%s:%s" % (username, password)
-    return base64.b64encode(auth_str.encode("utf-8")).decode("utf-8")
-
-
-def get_headers():
-    auth_header = get_basic_auth(flux_user, flux_token)
-    if isinstance(auth_header, bytes):
-        auth_header = auth_header.decode("utf-8")
-    return {"Authorization": "Basic %s" % auth_header}
-
-
-def test_submit_list_job():
-
-    # No jobs are submit!
-    response = client.get("/jobs")
-    assert response.status_code == 401
-    assert "www-authenticate" in response.headers
-
-    headers = get_headers()
-    response = client.get("/jobs", headers=headers)
-    assert response.status_code == 200
-
-    # Now submit a job, ensure we get one job back
-    response = client.post("/jobs/submit", json={"command": "sleep 5"}, headers=headers)
-    assert response.status_code == 200
-    result = response.json()
-    assert "id" in result
-    jobid = result["id"]
-
-    response = client.get("/jobs", headers=headers)
-    assert response.status_code == 200
-    result = response.json()
-    assert "jobs" in result
-    result = result["jobs"]
-    assert len(result) == 1
-    assert result[0]["id"] == jobid
-
-
-def test_cancel_job():
-
-    headers = get_headers()
-
-    # Now submit a job, ensure we get one job back
-    response = client.post(
-        "/jobs/submit", json={"command": "sleep 10"}, headers=headers
-    )
-    assert response.status_code == 200
-    result = response.json()
-    jobid = result["id"]
-    response = client.post(f"/jobs/{jobid}/cancel", headers=headers)
-    assert response.status_code == 200
-    # TODO we don't have way to actually verify this!