Repository: Dallinger/Dallinger
Pull request #7498: Logger dashboard
Author: login: polvanrijn
Created: 2025-03-13T20:43:09Z | Merged: 2025-03-25T12:50:09Z

## Description
I provide a dashboard panel showing all logging messages of the experiment.

## Motivation and Context
I implemented simple search functionalities (which allow to quickly find errors) and line jumping across the log.

## How Has This Been Tested?
- [x] Tested in psynet locally
- [x] Tested in dallinger locally
- [x] Tested on Docker, logger now syncs across workers

## Screenshots (if appropriate):

https://github.com/user-attachments/assets/baa5e406-4071-4f26-9957-1ad968753680


**Added:**
- Dump logs in `jsonl` format and stream it to a front-end dashboard tab, also supporting searching the logs from the front-end



Diff:
diff --git a/dallinger/bots.py b/dallinger/bots.py
index 17845792b8..674a970c54 100644
--- a/dallinger/bots.py
+++ b/dallinger/bots.py
@@ -16,7 +16,7 @@
 from selenium.webdriver.support.ui import WebDriverWait
 from six.moves import urllib
 
-logger = logging.getLogger(__file__)
+logger = logging.getLogger(__name__)
 
 DRIVER_MAP = {
     "firefox": webdriver.Firefox,
diff --git a/dallinger/command_line/docker_ssh.py b/dallinger/command_line/docker_ssh.py
index 3790422e0b..cd41c244cd 100644
--- a/dallinger/command_line/docker_ssh.py
+++ b/dallinger/command_line/docker_ssh.py
@@ -35,7 +35,7 @@
 from dallinger.data import bootstrap_db_from_zip, export_db_uri
 from dallinger.db import create_db_engine
 from dallinger.deployment import handle_launch_data, setup_experiment
-from dallinger.utils import abspath_from_egg, check_output
+from dallinger.utils import JSON_LOGFILE, abspath_from_egg, check_output
 
 # A couple of constants to colour console output
 RED = "\033[31m"
@@ -525,7 +525,7 @@ def _deploy_in_mode(
     sftp.putfo(
         BytesIO(
             get_docker_compose_yml(
-                cfg, experiment_id, image_name, postgresql_password
+                cfg, experiment_id, image_name, postgresql_password, executor
             ).encode()
         ),
         f"dallinger/{experiment_id}/docker-compose.yml",
@@ -778,48 +778,6 @@ def destroy(server, app):
     print(f"App {app} removed")
 
 
-def get_docker_compose_yml(
-    config: Dict[str, str],
-    experiment_id: str,
-    experiment_image: str,
-    postgresql_password: str,
-) -> str:
-    """Generate a docker-compose.yml file based on the given"""
-    docker_volumes = config.get("docker_volumes", "")
-    config_str = {key: re.sub("\\$", "$$", str(value)) for key, value in config.items()}
-
-    return DOCKER_COMPOSE_EXP_TPL.render(
-        experiment_id=experiment_id,
-        experiment_image=experiment_image,
-        config=config_str,
-        docker_volumes=docker_volumes,
-        postgresql_password=postgresql_password,
-    )
-
-
-def get_retrying_http_client():
-    parameter_name = "method_whitelist"
-    if hasattr(Retry.DEFAULT, "allowed_methods"):
-        parameter_name = "allowed_methods"
-
-    retry_strategy = Retry(
-        total=30,
-        backoff_factor=0.2,
-        status_forcelist=[429, 500, 502, 503, 504],
-        **{parameter_name: ["POST"]},
-    )
-    adapter = HTTPAdapter(max_retries=retry_strategy)
-    http = requests.Session()
-    http.mount("https://", adapter)
-    http.mount("http://", adapter)
-    return http
-
-
-def get_dns_host(ssh_host):
-    ip_addr = gethostbyname_ex(ssh_host)[2][0]
-    return f"{ip_addr}.nip.io"
-
-
 class Executor:
     """Execute remote commands using paramiko"""
 
@@ -927,6 +885,55 @@ def run_and_echo(self, cmd):  # pragma: no cover
                     break
 
 
+def get_docker_compose_yml(
+    config: Dict[str, str],
+    experiment_id: str,
+    experiment_image: str,
+    postgresql_password: str,
+    executor: Executor = None,
+) -> str:
+    """Generate a docker-compose.yml file based on the given"""
+    docker_volumes = config.get("docker_volumes", "")
+    logger_filename = JSON_LOGFILE
+    # touch the logger file so that it exists when the container starts
+    if executor:
+        executor.run(f"touch {logger_filename}")
+    if logger_filename:
+        docker_volumes += f",./{logger_filename}:/experiment/{logger_filename}"
+    config_str = {key: re.sub("\\$", "$$", str(value)) for key, value in config.items()}
+
+    return DOCKER_COMPOSE_EXP_TPL.render(
+        experiment_id=experiment_id,
+        experiment_image=experiment_image,
+        config=config_str,
+        docker_volumes=docker_volumes,
+        postgresql_password=postgresql_password,
+    )
+
+
+def get_retrying_http_client():
+    parameter_name = "method_whitelist"
+    if hasattr(Retry.DEFAULT, "allowed_methods"):
+        parameter_name = "allowed_methods"
+
+    retry_strategy = Retry(
+        total=30,
+        backoff_factor=0.2,
+        status_forcelist=[429, 500, 502, 503, 504],
+        **{parameter_name: ["POST"]},
+    )
+    adapter = HTTPAdapter(max_retries=retry_strategy)
+    http = requests.Session()
+    http.mount("https://", adapter)
+    http.mount("http://", adapter)
+    return http
+
+
+def get_dns_host(ssh_host):
+    ip_addr = gethostbyname_ex(ssh_host)[2][0]
+    return f"{ip_addr}.nip.io"
+
+
 class ExecuteException(Exception):
     pass
 
diff --git a/dallinger/config.py b/dallinger/config.py
index cb3eff5a78..8ee0360205 100644
--- a/dallinger/config.py
+++ b/dallinger/config.py
@@ -13,7 +13,7 @@
 from setuptools.dist import strtobool
 from six.moves import configparser
 
-logger = logging.getLogger(__file__)
+logger = logging.getLogger(__name__)
 
 marker = object()
 
diff --git a/dallinger/db.py b/dallinger/db.py
index b4b9549eb1..80f0d134fa 100644
--- a/dallinger/db.py
+++ b/dallinger/db.py
@@ -22,7 +22,7 @@
 from dallinger.config import initialize_experiment_package
 from dallinger.redis_utils import connect_to_redis
 
-logger = logging.getLogger("dallinger.db")
+logger = logging.getLogger(__name__)
 
 
 def corrected_db_url(db_url):
diff --git a/dallinger/docker/tools.py b/dallinger/docker/tools.py
index 59985b72a5..7bb0d394f1 100644
--- a/dallinger/docker/tools.py
+++ b/dallinger/docker/tools.py
@@ -12,7 +12,11 @@
 from pip._internal.req import parse_requirements
 
 from dallinger.docker.wheel_filename import parse_wheel_filename
-from dallinger.utils import abspath_from_egg, get_editable_dallinger_path
+from dallinger.utils import (
+    JSON_LOGFILE,
+    abspath_from_egg,
+    get_editable_dallinger_path,
+)
 
 docker_compose_template = Template(
     abspath_from_egg("dallinger", "dallinger/docker/docker-compose.yml.j2").read_text()
@@ -59,10 +63,15 @@ def __init__(
 
     def copy_docker_compose_files(self):
         """Prepare a docker-compose.yml file and place it in the experiment tmp dir"""
+        logger_filename = JSON_LOGFILE
         volumes = [
             f"{self.original_dir}:{self.original_dir}",
             f"{self.tmp_dir}:/experiment",
+            f"./{logger_filename}:/experiment/{logger_filename}",
         ]
+        with open(f"{self.tmp_dir}/{logger_filename}", "w") as f:  # noqa: F841
+            pass
+
         editable_dallinger_path = get_editable_dallinger_path()
         if editable_dallinger_path:
             volumes.append(f"{editable_dallinger_path}/dallinger:/dallinger/dallinger")
diff --git a/dallinger/experiment.py b/dallinger/experiment.py
index 0eb9945ae0..25cc5c3ceb 100644
--- a/dallinger/experiment.py
+++ b/dallinger/experiment.py
@@ -51,7 +51,7 @@
 from dallinger.transformations import Compression, Mutation, Replication, Response
 from dallinger.utils import classproperty, deferred_route_decorator, struct_to_html
 
-logger = logging.getLogger(__file__)
+logger = logging.getLogger(__name__)
 
 
 def exp_class_working_dir(meth):
diff --git a/dallinger/experiment_server/dashboard.py b/dallinger/experiment_server/dashboard.py
index 4a7bf6823e..d1f6442980 100644
--- a/dallinger/experiment_server/dashboard.py
+++ b/dallinger/experiment_server/dashboard.py
@@ -2,10 +2,13 @@
 import logging
 from copy import deepcopy
 from datetime import datetime, timedelta
+from typing import Optional, Union
 from xml.sax.saxutils import escape
 
+import bs4
 import six
 import timeago
+from ansi2html import Ansi2HTMLConverter
 from flask import (
     Blueprint,
     abort,
@@ -20,6 +23,7 @@
 from flask_login import UserMixin, current_user, login_required, login_user, logout_user
 from flask_login.utils import login_url as make_login_url
 from flask_wtf import FlaskForm
+from pygtail import Pygtail
 from six.moves.urllib.parse import urlencode
 from tzlocal import get_localzone
 from wtforms import BooleanField, HiddenField, PasswordField, StringField, SubmitField
@@ -30,7 +34,10 @@
 from dallinger.config import get_config
 from dallinger.db import get_all_mapped_classes
 from dallinger.heroku.tools import HerokuApp
-from dallinger.utils import deferred_route_decorator
+from dallinger.utils import (
+    JSON_LOGFILE,
+    deferred_route_decorator,
+)
 
 from .utils import date_handler, error_response, success_response
 
@@ -225,6 +232,7 @@ def database_children():
         DashboardTab("Monitoring", "dashboard.dashboard_monitoring"),
         DashboardTab("Lifecycle", "dashboard.dashboard_lifecycle"),
         DashboardTab("Database", "dashboard.dashboard_database", database_children),
+        DashboardTab("Logger", "dashboard.dashboard_logger"),
         DashboardTab("Development", "dashboard.dashboard_develop"),
     ]
 )
@@ -652,6 +660,301 @@ def dashboard_lifecycle():
     )
 
 
+def clean_http_request(msg: str) -> str:
+    """Remove redundant prefix from HTTP request log messages.
+
+    Parameters
+    ----------
+    msg : str
+        The log message to clean
+
+    Returns
+    -------
+    str
+        The cleaned message with redundant prefix removed
+
+    Examples
+    --------
+    >>> msg = '127.0.0.1 - - [20/Mar/2024:10:00:00] "GET /dashboard HTTP/1.1" 200 -'
+    >>> clean_http_request(msg)
+    'GET /dashboard HTTP/1.1" 200 -'
+
+    >>> msg = 'Regular log message without HTTP request'
+    >>> clean_http_request(msg)
+    'Regular log message without HTTP request'
+    """
+    has_http_method = any(f" {method} " in msg for method in ("GET", "POST"))
+    if has_http_method and msg.endswith(("-", '"')):
+        return '"'.join(msg.split('"')[1:])
+    return msg
+
+
+def convert_ansi_to_html(msg: str) -> str:
+    """Convert ANSI terminal colors to HTML formatted text.
+
+    Parameters
+    ----------
+    msg : str
+        Message potentially containing ANSI color codes
+
+    Returns
+    -------
+    str
+        HTML-formatted message with ANSI codes converted to CSS
+
+    Examples
+    --------
+    >>> msg = '\033[31mError:\033[0m Database connection failed'
+    >>> convert_ansi_to_html(msg)
+    '<span style="color: red">Error:</span> Database connection failed'
+
+    >>> msg = 'Plain text without ANSI codes'
+    >>> convert_ansi_to_html(msg)
+    'Plain text without ANSI codes'
+    """
+    return Ansi2HTMLConverter().convert(msg)
+
+
+def ensure_html_structure(html: str) -> str:
+    """Ensure HTML has proper structure with head and body tags.
+
+    Parameters
+    ----------
+    html : str
+        HTML content to structure
+
+    Returns
+    -------
+    str
+        Properly structured HTML document
+
+    Examples
+    --------
+    >>> html = '<span style="color: red">Error</span>'
+    >>> ensure_html_structure(html)
+    '<html><head></head><body><span style="color: red">Error</span></body></html>'
+
+    >>> html = '<head><style>...</style></head><body>Content</body>'
+    >>> ensure_html_structure(html)
+    '<html><head><style>...</style></head><body>Content</body></html>'
+    """
+    soup = bs4.BeautifulSoup(html, "html.parser")
+    return f"<html>{soup.head or ''}{soup.body or ''}</html>"
+
+
+def clean_line_info(line_info: dict, log_line_number: Optional[int] = None) -> dict:
+    """Clean and format a log message dictionary for dashboard display.
+
+    Parameters
+    ----------
+    line_info : dict
+        Dictionary containing log message data with at least a 'message' key
+    log_line_number : int, optional
+        Line number in the log file
+
+    Returns
+    -------
+    dict
+        Cleaned dictionary with HTML-formatted message
+
+    Examples
+    --------
+    >>> line_info = {
+    ...     'message': '\033[31mError:\033[0m GET /dashboard failed',
+    ...     'level': 'ERROR'
+    ... }
+    >>> cleaned = clean_line_info(line_info, log_line_number=42)
+    >>> cleaned
+    {
+        'message': '<html><head>...</head><body><span style="color: red">Error:</span> GET /dashboard failed</body></html>',
+        'level': 'ERROR',
+        'log_line_number': 42
+    }
+    """
+    msg = line_info["message"]
+
+    # Clean and format the message
+    msg = clean_http_request(msg)
+    msg = convert_ansi_to_html(msg)
+    msg = ensure_html_structure(msg)
+
+    # Finalize the dictionary
+    line_info["message"] = msg
+    if log_line_number is not None:
+        line_info["log_line_number"] = log_line_number
+
+    return line_info
+
+
+def log_read_lines(line_start: int, line_end: int) -> tuple[list[dict], bool, int]:
+    """
+    Read the log file and return the lines in the specified range.
+
+    :param line_start: The line number to start reading from
+    :type line_start: int
+
+    :param line_end: The line number to stop reading at
+    :type line_end: int
+
+    :return: A tuple containing the lines, a boolean indicating if the end of the file was reached, and the last line number
+
+    :Note: The line numbers are 1-based
+
+    """
+    lines = []
+    line_range = range(line_start, line_end)
+    early_stop = False
+    with open(JSON_LOGFILE) as f:
+        for i, line in enumerate(f):
+            number = i + 1
+            if number in line_range:
+                line_info = clean_line_info(json.loads(line), number)
+                lines.append(line_info)
+            if number > line_end:
+                early_stop = True
+                break
+    return lines, early_stop, i
+
+
+def find_log_line_number(substring) -> Union[int, None]:
+    """
+    Find the line number in the log file that contains a substring.
+
+    :param substring: The substring to search for
+    :type substring: str
+
+    :return: The line number (1-based) or None if the substring was not found
+    """
+    with open(JSON_LOGFILE) as f:
+        for number, line in enumerate(f):
+            if substring in line:
+                return number + 1
+    return None
+
+
+def log_search_substring(substring: str):
+    """
+    Search the log file for a substring and return the matching lines.
+
+    :param substring: The substring to search for
+    :type substring: str
+
+    :return: A generator that yields the matching lines in the right format (i.e. f"data:{json.dumps(obj)}\n\n")
+
+    :Note: The line numbers are 1-based
+    :Note: The generator will yield a 'stop' message when the end of the file is reached
+    """
+    with open(JSON_LOGFILE) as f:
+        for number, line in enumerate(f):
+            if substring in line:
+                line_info = clean_line_info(json.loads(line), number)
+                yield f"data:{json.dumps(line_info)}\n\n"
+    yield f"data:{json.dumps({'stop': True})}\n\n"
+
+
+@dashboard.route("/logs/live", methods=["GET"])
+@login_required
+def logs_live():
+    """Stream log file updates to the dashboard using Server-Sent Events (SSE).
+
+    Creates a streaming response that pushes new log entries to connected clients
+    in real-time. Each log entry is:
+    - Read from the log file using Pygtail
+    - Parsed from JSON
+    - Cleaned and formatted for display
+    - Sent as an SSE message
+
+    Returns
+    -------
+    Response
+        A Flask response object with mimetype 'text/event-stream' containing
+        the event stream of log updates. Each event contains:
+        - message : str
+            The formatted log message with ANSI codes converted to HTML
+        - original_line : str
+            The raw log line before processing
+        - log_line_number : int, optional
+            The line number in the log file
+
+    Notes
+    -----
+
+    The client should connect to this endpoint using an EventSource object
+    to receive the updates.
+    """
+
+    def generate():
+        for line in Pygtail(JSON_LOGFILE):
+            try:
+                line_info = clean_line_info(json.loads(line))
+                line_info["original_line"] = line
+                yield f"data:{json.dumps(line_info)}\n\n"
+            except json.decoder.JSONDecodeError:
+                yield f"data:{json.dumps({'message': line})}\n\n"
+
+    return Response(generate(), mimetype="text/event-stream")
+
+
+@dashboard.route("/logs/range", methods=["GET"])
+@login_required
+def logs_range():
+    """
+    Return a range of log lines from the log file.
+    """
+    params = request.args
+    start, end = params.get("start", None), params.get("end", None)
+    if start is None or end is None:
+        return json_error_response("Both 'start' and 'end' parameters are required.")
+    start, end = int(start), int(end)
+    if start < 1:
+        return json_error_response("'start' must be > 0.")
+
+    if start > end:
+        return json_error_response("'start' must be <= 'end'.")
+
+    lines, early_stop, last_line = log_read_lines(start, end + 1)
+    if not early_stop and start > last_line:
+        return json_error_response(
+            f"'start' must be less than the current line number {last_line}."
+        )
+
+    return lines
+
+
+def json_error_response(message, status_code=400):
+    return {"msg": message}, status_code
+
+
+@dashboard.route("/logs/find_lines", methods=["GET"])
+@login_required
+def logs_find_lines():
+    """
+    Find lines in the log file that contain a substring (GET parameter 'query').
+    """
+    params = request.args
+    query = params.get("query", None)
+    if query is None:
+        return json_error_response("No query provided.")
+    return Response(log_search_substring(query), mimetype="text/event-stream")
+
+
+@dashboard.route("/logs/find_line_number", methods=["POST"])
+@login_required
+def logs_find_line_number():
+    """
+    Find the line number in the log file that contains a substring (POST parameter 'query').
+    """
+    params = request.json
+    query = params.get("query", None)
+    if query is None:
+        return json_error_response("No query provided.")
+
+    line_number = find_log_line_number(query)
+    if line_number is not None:
+        return {"line_number": line_number}
+    return json_error_response("No line found.", 404)
+
+
 TABLE_DEFAULTS = {
     "dom": "frtilBpP",
     "ordering": True,
@@ -795,6 +1098,15 @@ def dashboard_database():
     )
 
 
+@dashboard.route("/logger")
+@login_required
+def dashboard_logger():
+    """
+    Streams the tail of the log file to the dashboard or shows selected lines
+    """
+    return render_template("dashboard_logger.html")
+
+
 @dashboard.route("/develop", methods=["GET", "POST"])
 @login_required
 def dashboard_develop():
diff --git a/dallinger/experiment_server/experiment_server.py b/dallinger/experiment_server/experiment_server.py
index 98893c6bb8..835f1e73fa 100644
--- a/dallinger/experiment_server/experiment_server.py
+++ b/dallinger/experiment_server/experiment_server.py
@@ -1,5 +1,6 @@
 """This module provides the backend Flask server that serves an experiment."""
 
+import logging
 import os
 import re
 from datetime import datetime
@@ -28,7 +29,7 @@
 from dallinger.config import get_config
 from dallinger.notifications import MessengerError, admin_notifier
 from dallinger.recruiters import ProlificRecruiter
-from dallinger.utils import generate_random_id, get_from_config
+from dallinger.utils import attach_json_logger, generate_random_id, get_from_config
 
 from . import dashboard
 from .replay import ReplayBackend
@@ -96,6 +97,14 @@ def Experiment(args):
     return klass(args)
 
 
+log = logging.getLogger()
+DATE_FORMAT = "%Y-%m-%d %H:%M:%S"
+logging.basicConfig(
+    level=logging.INFO,
+    datefmt=DATE_FORMAT,
+)
+attach_json_logger(log)
+
 # Load the experiment's extra routes, if any.
 try:
     from dallinger_experiment.experiment import extra_routes
@@ -248,7 +257,6 @@ def handle_exp_error(exception):
 def shutdown_session(_=None):
     """Rollback and close session at end of a request."""
     session.remove()
-    db.logger.debug("Closing Dallinger DB session at flask request end")
 
 
 @app.context_processor
diff --git a/dallinger/experiment_server/gunicorn.py b/dallinger/experiment_server/gunicorn.py
index 258e59c44f..c762ec03a1 100644
--- a/dallinger/experiment_server/gunicorn.py
+++ b/dallinger/experiment_server/gunicorn.py
@@ -9,8 +9,9 @@
 from werkzeug.middleware.proxy_fix import ProxyFix
 
 from dallinger.config import get_config
+from dallinger.utils import attach_json_logger
 
-logger = logging.getLogger(__file__)
+logger = logging.getLogger(__name__)
 
 WORKER_CLASS = "gevent"
 
@@ -99,12 +100,13 @@ def launch():
     ]
     LOG_LEVEL = LOG_LEVELS[config.get("loglevel")]
     logging.basicConfig(format="%(asctime)s %(message)s", level=LOG_LEVEL)
-
     # Avoid duplicate logging to stderr
     error_logger = logging.getLogger("gunicorn.error")
     error_logger.propagate = False
     access_logger = logging.getLogger("gunicorn.access")
     access_logger.propagate = False
+    attach_json_logger(error_logger)
+    attach_json_logger(access_logger)
 
     # Set up logging to file
     # (We're not using gunicorn's errorlog and accesslog settings
diff --git a/dallinger/experiment_server/replay.py b/dallinger/experiment_server/replay.py
index 9e06394b63..360405f5d3 100644
--- a/dallinger/experiment_server/replay.py
+++ b/dallinger/experiment_server/replay.py
@@ -5,7 +5,7 @@
 
 from dallinger.utils import get_base_url
 
-logger = logging.getLogger(__file__)
+logger = logging.getLogger(__name__)
 
 
 class ReplayBackend(object):
diff --git a/dallinger/frontend/templates/dashboard_logger.html b/dallinger/frontend/templates/dashboard_logger.html
new file mode 100644
index 0000000000..730fc2b262
--- /dev/null
+++ b/dallinger/frontend/templates/dashboard_logger.html
@@ -0,0 +1,299 @@
+{% extends "base/dashboard.html" %}
+
+{% block body %}
+    <h1>Logger</h1>
+    <span id="info-text">
+    Below are the logs for the experiment. You can filter the logs by log level. By clicking on the line number, you can
+    get a link to that specific line.
+    </span>
+    <form id="search-form" class="mb-3">
+        <div class="input-group">
+            <input type="text" class="form-control" name="query"
+                   placeholder="Search the logs for a particular substring (case sensitive)...">
+            <button class="btn btn-primary" type="submit">Search</button>
+        </div>
+    </form>
+
+    <div id="filters"></div>
+    <div>
+    </div>
+    <div id="log-container">
+        <table id="log" class="monospace">
+            <tbody>
+            </tbody>
+        </table>
+    </div>
+
+    <style>
+        #log-container {
+            overflow: auto;
+            height: 80vh;
+            font-size: .875em;
+            background: #f6f8fa;
+            padding: 16px;
+            color: #1f2328;
+        }
+
+        #log-container pre {
+            font-size: 1em;
+        }
+
+        .monospace {
+            font-family: ui-monospace, SFMono-Regular, SF Mono, Menlo, Consolas, Liberation Mono, monospace;
+        }
+
+        #log {
+            caption-side: bottom;
+        }
+
+        #log a {
+            color: inherit;
+        }
+
+        #log tr {
+            vertical-align: top;
+        }
+
+        #log tr.highlight {
+            background: lightyellow;
+        }
+
+        .level-checkbox {
+            margin-right: 10px;
+            font-weight: bold;
+        }
+
+    </style>
+
+    <script>
+        // Constants
+        const logLevels = ["DEBUG", "INFO", "SUCCESS", "WARNING", "ERROR", "CRITICAL"];
+        const colors = {
+            "DEBUG": "#4d98cc",
+            "SUCCESS": "#79b25f",
+            "WARNING": "#ddaa2d",
+            "ERROR": "#cf3f61",
+            "CRITICAL": "#cf3f61",
+            "DEFAULT": "#6c757d"
+        };
+        const infoText = document.getElementById('info-text');
+        const highlightLine = new URLSearchParams(window.location.search).get('highlight');
+
+        // Variables
+        let lineOffset, firstLine
+        let nPrintedRows = 0;
+
+        /**
+         * Function to create checkboxes dynamically
+         * @param {string} level - The log level for the checkbox
+         * @returns {string} - The HTML string for the checkbox
+         */
+        function createCheckbox(level) {
+            const color = colors[level] || colors.DEFAULT;
+            return `<label class="level-checkbox" style="color: ${color};">
+                <input type="checkbox" value="${level}" checked> ${level}
+            </label>`;
+        }
+
+        // Append checkboxes to the filter div
+        document.addEventListener("DOMContentLoaded", () => {
+            const filtersDiv = document.getElementById("filters");
+            logLevels.forEach(level => {
+                filtersDiv.innerHTML += createCheckbox(level);
+            });
+
+            document.querySelectorAll('.level-checkbox input').forEach(checkbox => {
+                checkbox.addEventListener('change', function () {
+                    document.querySelectorAll(`.status-${this.value.toLowerCase()}`)
+                        .forEach(row => row.style.display = this.checked ? "table-row" : "none");
+                });
+            });
+
+            activateTooltips();
+            setupSearchReset();
+            fetchLogData();
+        });
+
+        /**
+         * Function to activate Bootstrap tooltips
+         */
+        function activateTooltips() {
+            const tooltipTriggerList = document.querySelectorAll('[data-bs-toggle="tooltip"]');
+            [...tooltipTriggerList].map(tooltipTriggerEl => new bootstrap.Tooltip(tooltipTriggerEl, {
+                boundary: 'viewport',
+                sanitize: false,
+                html: true
+            }));
+        }
+
+        /**
+         * Function to scroll to the highlighted line
+         */
+        function moveToHighlight() {
+            if (highlightLine) {
+                const line = document.getElementsByClassName('highlight')[0];
+                if (line) {
+                    line.scrollIntoView();
+                }
+            }
+        }
+
+        /**
+         * Function to add a reset button to the info text
+         */
+        function addResetButton() {
+            const resetButton = document.createElement('button');
+            resetButton.innerHTML = "Go back to the live logs";
+            resetButton.className = "btn btn-secondary btn-small my-3";
+            resetButton.onclick = () => window.location.href = window.location.pathname;
+            infoText.innerHTML += "<br>";
+            infoText.appendChild(resetButton);
+        }
+
+        /**
+         * Function to reset search query
+         */
+        function setupSearchReset() {
+            if (window.location.search.includes('query')) {
+                infoText.innerHTML += ` The logs have been filtered by the search query "<strong>${new URLSearchParams(window.location.search).get('query')}</strong>".`;
+                addResetButton();
+            } else if (window.location.search.includes('highlight')) {
+                infoText.innerHTML += ` Line <strong>${highlightLine}</strong> is highlighted. Click on another line to view this line in context.`;
+                addResetButton();
+            } else {
+                infoText.innerHTML += " The logs are live and will update automatically.";
+            }
+        }
+
+        /**
+         * Function to remove credentials from the URL
+         */
+        function removeCredsFromUrl() {
+            let url = new URL(document.URL);
+            if (url.username !== "" && url.password !== "") {
+                url = window.location.href;
+                console.log("Redirecting to: " + url);
+                window.location.replace(url);
+            }
+        }
+
+        removeCredsFromUrl();
+
+        /**
+         * Function to fetch logs and append them dynamically
+         */
+        function fetchLogData() {
+            if (window.location.search.includes('start') || window.location.search.includes('end')) {
+
+                let badResponse = false;
+                const logContainer = document.querySelector("#log tbody");
+                fetch("/dashboard/logs/range" + window.location.search)
+                    .then(response => {
+                        if (!response.ok) {
+                            badResponse = true;
+                        }
+                        return response.json();
+                    })
+                    .then(data => {
+                        if (badResponse) {
+                            throw new Error(data.msg);
+                        }
+                        logContainer.innerHTML = "";
+                        data.forEach(line => logContainer.appendChild(printLogLine(line)));
+                        activateTooltips();
+                        moveToHighlight();
+                    })
+                    .catch(error => {
+                        console.error('There has been a problem with your fetch operation:', error);
+                        $("#log tbody").append(`<tr><td colspan="3" style="color: ${colors.red};">Error: ${error.message}</td></tr>`);
+                    });
+
+            } else {
+                var source;
+                if (window.location.search.includes('query')) {
+                    source = new EventSource("/dashboard/logs/find_lines" + window.location.search);
+                } else {
+                    source = new EventSource("/dashboard/logs/live");
+                }
+                source.onmessage = function (event) {
+                    let data = JSON.parse(event.data);
+                    if (data.stop) {
+                        moveToHighlight();
+                        source.close();
+                    } else {
+                        $("#log tbody").prepend(printLogLine(data));
+                        activateTooltips()
+                    }
+                }
+            }
+        }
+
+        /**
+         * Function to check if the logs are live
+         * @returns {boolean} - True if the logs are live, false otherwise
+         */
+        function isLive() {
+            return !(window.location.search.includes('query') || window.location.search.includes('highlight'));
+        }
+
+        /**
+         * Function to generate log row HTML
+         * @param {Object} data - The log data
+         * @returns {HTMLElement} - The log row element
+         */
+        function printLogLine(data) {
+            nPrintedRows += 1;
+            const color = colors[data.levelname] || "inherit";
+            const source = data.name === "werkzeug" ? "werkzeug" : `${data.filename}:${data.name || ""}:${data.lineno}`;
+            const tooltipText = `<span style='font-weight: bold'>Time</span> <span class='monospace'>${data.asctime}</span><br>
+                         <span style='font-weight: bold'>Source</span> <span class='monospace'>${source}</span>`;
+            let lineNumber = data.log_line_number;
+
+            const row = document.createElement("tr");
+            row.classList.add(`status-${data.levelname.toLowerCase()}`);
+            if (parseInt(new URLSearchParams(window.location.search).get('highlight')) === parseInt(lineNumber)) {
+                row.classList.add("highlight");
+            }
+            function makeUrl(lineNumber) {
+                let url = `?highlight=${lineNumber}&start=${lineNumber - 10}&end=${lineNumber + 10}`
+                return `<a href="${url}">${lineNumber}</a>`
+            }
+            if (isLive() && !firstLine) {
+                firstLine = data.original_line;
+                fetch("/dashboard/logs/find_line_number", {
+                    method: 'POST',
+                    headers: {
+                        'Content-Type': 'application/json'
+                    },
+                    body: JSON.stringify({ query: firstLine, type: 'line_number' })
+                    })
+                .then(response => response.json())
+                    .then(data => {
+                        lineOffset = data.line_number;
+                        console.log(lineOffset, firstLine, nPrintedRows);
+                        $('.line_number.pending').each(function (index) {
+                            $(this).html(makeUrl(lineOffset + nPrintedRows - index));
+                        });
+                    })
+            }
+            if (lineOffset) {
+                lineNumber = lineOffset + nPrintedRows;
+            }
+            if (lineNumber) {
+                row.innerHTML = `<td class="line_number">${makeUrl(lineNumber)}</td>`
+            } else {
+                row.innerHTML = `<td class="line_number pending">PENDING</td>`
+            }
+            row.innerHTML += `<td data-bs-toggle="tooltip" data-bs-html="true" data-bs-placement="left" title="${tooltipText}" style="color: ${color}; font-weight: bold">${data.levelname}</td>`
+            let message = data.message.replace(/(https?:\/\/[^\s]+)/g, '<a href="$1" target="_blank">$1</a>');
+            message = `<span style="color: ${color}">${message}</span>`
+            if (data.exc_info) {
+                message += `<pre>${data.exc_info}</pre>`
+            }
+            row.innerHTML += `<td>${message}</td>`;
+
+            return row;
+        }
+    </script>
+
+{% endblock %}
\ No newline at end of file
diff --git a/dallinger/mturk.py b/dallinger/mturk.py
index 7e70ad27cd..37ae30084b 100644
--- a/dallinger/mturk.py
+++ b/dallinger/mturk.py
@@ -6,7 +6,7 @@
 from botocore.exceptions import ClientError, NoCredentialsError
 from cached_property import cached_property
 
-logger = logging.getLogger(__file__)
+logger = logging.getLogger(__name__)
 PERCENTAGE_APPROVED_REQUIREMENT_ID = "000000000000000000L0"
 LOCALE_REQUIREMENT_ID = "00000000000000000071"
 MAX_SUPPORTED_BATCH_SIZE = 100
diff --git a/dallinger/notifications.py b/dallinger/notifications.py
index 31c46d2aa3..15c99b3f17 100644
--- a/dallinger/notifications.py
+++ b/dallinger/notifications.py
@@ -5,7 +5,7 @@
 import six
 from cached_property import cached_property
 
-logger = logging.getLogger(__file__)
+logger = logging.getLogger(__name__)
 CONFIG_PLACEHOLDER = "???"
 
 
diff --git a/dallinger/prolific.py b/dallinger/prolific.py
index dbdb9f8862..dd16bee871 100644
--- a/dallinger/prolific.py
+++ b/dallinger/prolific.py
@@ -10,7 +10,7 @@
 
 from dallinger.version import __version__
 
-logger = logging.getLogger(__file__)
+logger = logging.getLogger(__name__)
 
 
 #####################
diff --git a/dallinger/recruiters.py b/dallinger/recruiters.py
index 39e9b72432..7e89b79d42 100644
--- a/dallinger/recruiters.py
+++ b/dallinger/recruiters.py
@@ -39,7 +39,7 @@
 )
 from dallinger.utils import ParticipationTime, generate_random_id, get_base_url
 
-logger = logging.getLogger(__file__)
+logger = logging.getLogger(__name__)
 
 
 # These are constants because other components may listen for these
diff --git a/dallinger/registration.py b/dallinger/registration.py
index e65a34767e..3bdba9850e 100644
--- a/dallinger/registration.py
+++ b/dallinger/registration.py
@@ -7,7 +7,7 @@
 
 from dallinger.config import get_config
 
-logger = logging.getLogger(__file__)
+logger = logging.getLogger(__name__)
 root = "https://api.osf.io/v2"
 
 
diff --git a/dallinger/utils.py b/dallinger/utils.py
index 213bfac865..2d8f1417ab 100644
--- a/dallinger/utils.py
+++ b/dallinger/utils.py
@@ -3,6 +3,7 @@
 import functools
 import io
 import locale
+import logging
 import os
 import random
 import re
@@ -22,6 +23,7 @@
 import requests
 from faker import Faker
 from flask import request
+from pythonjsonlogger import jsonlogger
 
 from dallinger import db
 from dallinger.compat import is_command
@@ -35,6 +37,17 @@
 
 fake = Faker()
 
+JSON_LOGFILE = "logs.jsonl"
+
+
+def attach_json_logger(log):
+    fmt = jsonlogger.JsonFormatter(
+        "%(name)s %(asctime)s %(levelname)s %(filename)s %(lineno)s %(message)s"
+    )
+    handler = logging.FileHandler(JSON_LOGFILE)
+    handler.setFormatter(fmt)
+    log.addHandler(handler)
+
 
 def get_base_url():
     """Returns the base url for the experiment.
diff --git a/dallinger_scripts/worker.py b/dallinger_scripts/worker.py
index 3ac98c36f0..5506d8d9cf 100644
--- a/dallinger_scripts/worker.py
+++ b/dallinger_scripts/worker.py
@@ -1,3 +1,5 @@
+from dallinger.utils import attach_json_logger
+
 listen = ["high", "default", "low"]
 
 
@@ -39,6 +41,7 @@ def main():
         format="%(asctime)s %(message)s",
         level=LOG_LEVEL,
     )
+    attach_json_logger(logging.getLogger())
     redis_url = os.getenv("REDIS_URL", "redis://localhost:6379")
     # Specify queue class for improved performance with gevent.
     # see http://carsonip.me/posts/10x-faster-python-gevent-redis-connection-pool/
diff --git a/demos/dlgr/demos/bartlett1932/experiment.py b/demos/dlgr/demos/bartlett1932/experiment.py
index 922ce16089..bb9699ca1c 100644
--- a/demos/dlgr/demos/bartlett1932/experiment.py
+++ b/demos/dlgr/demos/bartlett1932/experiment.py
@@ -12,7 +12,7 @@
 from dallinger.experiment import Experiment
 from dallinger.networks import Chain
 
-logger = logging.getLogger(__file__)
+logger = logging.getLogger(__name__)
 
 
 class Bartlett1932(Experiment):
diff --git a/demos/dlgr/demos/chatroom/bots.py b/demos/dlgr/demos/chatroom/bots.py
index 985d027d51..d8afef4daf 100644
--- a/demos/dlgr/demos/chatroom/bots.py
+++ b/demos/dlgr/demos/chatroom/bots.py
@@ -15,7 +15,7 @@
 
 from dallinger.bots import BotBase
 
-logger = logging.getLogger(__file__)
+logger = logging.getLogger("bot")
 
 
 class Bot(BotBase):
diff --git a/demos/dlgr/demos/chatroom/experiment.py b/demos/dlgr/demos/chatroom/experiment.py
index b4a27ebbd4..0bc1d30c82 100644
--- a/demos/dlgr/demos/chatroom/experiment.py
+++ b/demos/dlgr/demos/chatroom/experiment.py
@@ -15,7 +15,7 @@
 except ImportError:
     pass
 
-logger = logging.getLogger(__file__)
+logger = logging.getLogger("experiment")
 
 
 def extra_parameters():
diff --git a/demos/dlgr/demos/chatroom_ws/bots.py b/demos/dlgr/demos/chatroom_ws/bots.py
index 985d027d51..d8afef4daf 100644
--- a/demos/dlgr/demos/chatroom_ws/bots.py
+++ b/demos/dlgr/demos/chatroom_ws/bots.py
@@ -15,7 +15,7 @@
 
 from dallinger.bots import BotBase
 
-logger = logging.getLogger(__file__)
+logger = logging.getLogger("bot")
 
 
 class Bot(BotBase):
diff --git a/demos/dlgr/demos/chatroom_ws/experiment.py b/demos/dlgr/demos/chatroom_ws/experiment.py
index 22630372b9..6333f63899 100644
--- a/demos/dlgr/demos/chatroom_ws/experiment.py
+++ b/demos/dlgr/demos/chatroom_ws/experiment.py
@@ -17,7 +17,7 @@
 except ImportError:
     pass
 
-logger = logging.getLogger(__file__)
+logger = logging.getLogger("experiment")
 
 
 def extra_parameters():
diff --git a/docs/source/creating_an_experiment.rst b/docs/source/creating_an_experiment.rst
index f5a2550025..6c483cee68 100644
--- a/docs/source/creating_an_experiment.rst
+++ b/docs/source/creating_an_experiment.rst
@@ -586,7 +586,7 @@ interacts directly with the server.
 
 	from dallinger.bots import BotBase, HighPerformanceBotBase
 
-	logger = logging.getLogger(__file__)
+	logger = logging.getLogger("bot")
 
 The bot code first imports the bot base classes, along with some
 webdriver code for the regular bot, and the `requests` library, for
diff --git a/pyproject.toml b/pyproject.toml
index 87b0ab0ec4..776ee1292a 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -29,7 +29,9 @@ classifiers = [
 ]
 
 dependencies = [
+    "ansi2html",
     "apscheduler",
+    "beautifulsoup4",
     "boto3",
     "build",
     "cached-property",
@@ -53,7 +55,9 @@ dependencies = [
     "pip-tools",
     "psycopg2",
     "psutil",
+    "pygtail",
     "pyopenssl",
+    "python-json-logger",
     "redis",
     "requests",
     "rq",


Reviews:
user: login: pmcharrison
body: Thanks very much for this exciting contribution @polvanrijn!
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-21T12:28:39Z
user: login: polvanrijn
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-24T08:50:35Z
user: login: polvanrijn
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-24T08:50:46Z
user: login: polvanrijn
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-24T08:54:00Z
user: login: polvanrijn
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-24T09:39:42Z
user: login: polvanrijn
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-24T09:39:51Z
user: login: polvanrijn
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-24T09:42:13Z
user: login: pmcharrison
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-24T09:49:09Z
user: login: polvanrijn
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-24T09:57:50Z
user: login: polvanrijn
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-24T09:58:57Z
user: login: pmcharrison
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-24T10:19:15Z
user: login: pmcharrison
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-24T10:20:15Z
user: login: polvanrijn
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-24T10:36:06Z
user: login: polvanrijn
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-24T10:44:57Z
user: login: polvanrijn
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-24T10:45:04Z
user: login: polvanrijn
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-24T10:45:08Z
user: login: polvanrijn
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-24T10:49:34Z
user: login: polvanrijn
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-24T10:50:02Z
user: login: polvanrijn
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-24T11:25:23Z
user: login: pmcharrison
body: Not sure if you pushed all the changes yet (see other comment) so am submitting this now and waiting for confirmation!
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-24T13:30:46Z
user: login: polvanrijn
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-24T13:37:00Z
user: login: pmcharrison
body: Looking great, almost there!
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-24T14:25:54Z
user: login: polvanrijn
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-24T15:25:09Z
user: login: polvanrijn
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-24T15:25:22Z
user: login: polvanrijn
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-24T15:35:00Z
user: login: polvanrijn
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-24T15:35:05Z
user: login: polvanrijn
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-24T15:35:08Z
user: login: polvanrijn
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-24T15:35:43Z
user: login: polvanrijn
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-24T15:36:57Z
user: login: polvanrijn
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-24T15:39:31Z
user: login: polvanrijn
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-24T15:41:36Z
user: login: pmcharrison
state: APPROVED
author_association: COLLABORATOR
submitted_at: 2025-03-24T18:56:49Z

Review comments:
diff_hunk: @@ -20,7 +20,7 @@
 )
 from .patches import patch
 
-logger = logging.getLogger(__name__)
+logger = logging.getLogger("dallinger")
path: dallinger/__init__.py
user: login: pmcharrison
body: How come it's necessary to change all of these `__name__` values to these hardcoded values?
created_at: 2025-03-21T11:33:03Z
author_association: COLLABORATOR
diff_hunk: @@ -20,7 +20,7 @@
 )
 from .patches import patch
 
-logger = logging.getLogger(__name__)
+logger = logging.getLogger("dallinger")
path: dallinger/__init__.py
user: login: pmcharrison
body: How come it's necessary to change all of these `__name__` values to these hardcoded values?
created_at: 2025-03-21T11:33:04Z
author_association: COLLABORATOR
diff_hunk: @@ -99,12 +100,13 @@ def launch():
     ]
     LOG_LEVEL = LOG_LEVELS[config.get("loglevel")]
     logging.basicConfig(format="%(asctime)s %(message)s", level=LOG_LEVEL)
-
     # Avoid duplicate logging to stderr
     error_logger = logging.getLogger("gunicorn.error")
     error_logger.propagate = False
     access_logger = logging.getLogger("gunicorn.access")
     access_logger.propagate = False
+    attach_json_logger(error_logger)
+    attach_json_logger(access_logger)
path: dallinger/experiment_server/gunicorn.py
user: login: pmcharrison
body: I think it's messy now that we have the JSON logger which is hard-coded as dallinger.log and then we have an ordinary logger corresponding to config.logfile (which defaults to server.log). I think we should consolidate this logic a bit better. 

One option is to hijack server.log and change the format to JSON. However, this breaks back-compatibility.

Another option is to call our new file `server.log.json`. That makes it clear that the files are the same but just have a different format.

I think you should also update the organisation of `launch` to clarify that the `attach_json_logger` steps are analogous to the below code which attaches a non-JSON logger:

```py
    logfile = config.get("logfile")
    if config.get("logfile") != "-":
        handler = logging.FileHandler(logfile)
        error_logger.addHandler(handler)
        access_logger.addHandler(handler)
```
created_at: 2025-03-21T11:52:03Z
author_association: COLLABORATOR
diff_hunk: @@ -652,6 +660,141 @@ def dashboard_lifecycle():
     )
 
 
+def clean_line_dict(line_dict, log_line_number=None):
path: dallinger/experiment_server/dashboard.py
user: login: pmcharrison
body: I don't think it's Dallinger style (or good Python style) to put the object type in the object name. I would just call this `clean_line` and then add a type hint for the line dictionary.
created_at: 2025-03-21T12:07:07Z
author_association: COLLABORATOR
diff_hunk: @@ -652,6 +660,141 @@ def dashboard_lifecycle():
     )
 
 
+def clean_line_dict(line_dict, log_line_number=None):
+    msg = line_dict["message"]
path: dallinger/experiment_server/dashboard.py
user: login: pmcharrison
body: This function is a bit messy. It would be good to add some documentation, explaining what the input and output is, and perhaps some comments explaining what each step is doing (you could give examples even), or alternatively refactoring the different steps into named functions.
created_at: 2025-03-21T12:08:11Z
author_association: COLLABORATOR
diff_hunk: @@ -652,6 +660,141 @@ def dashboard_lifecycle():
     )
 
 
+def clean_line_dict(line_dict, log_line_number=None):
+    msg = line_dict["message"]
+    if msg.endswith(("-", '"')) and ("GET " in msg or " POST" in msg):
+        msg = '"'.join(msg.split('"')[1:])
+    msg = Ansi2HTMLConverter().convert(msg)
+    parsed_msg = bs4.BeautifulSoup(msg, "html.parser")
+    msg = f"<html>{parsed_msg.head}{parsed_msg.body}</html>"
+    line_dict["message"] = msg
+    if log_line_number is not None:
+        line_dict["log_line_number"] = log_line_number
+    return line_dict
+
+
+LOG_FILE = get_logger_filename()
+
+
+def live_log():
+    """
+    Route that streams log file updates using SSE
+    """
+
+    def generate():
+        for line in Pygtail(LOG_FILE):
+            try:
+                line_dict = clean_line_dict(json.loads(line))
+                line_dict["original_line"] = line
+                yield f"data:{json.dumps(line_dict)}\n\n"
+            except json.decoder.JSONDecodeError:
+                continue
path: dallinger/experiment_server/dashboard.py
user: login: pmcharrison
body: Where are you expecting this `JSONDecodeError` to happen? In the `clean_line_dict`? Normally it's a bad idea to silently suppress error messages like this. Would it perhaps be better to find a way to relay the uncleaned version?
created_at: 2025-03-21T12:10:50Z
author_association: COLLABORATOR
diff_hunk: @@ -652,6 +660,141 @@ def dashboard_lifecycle():
     )
 
 
+def clean_line_dict(line_dict, log_line_number=None):
+    msg = line_dict["message"]
+    if msg.endswith(("-", '"')) and ("GET " in msg or " POST" in msg):
+        msg = '"'.join(msg.split('"')[1:])
+    msg = Ansi2HTMLConverter().convert(msg)
+    parsed_msg = bs4.BeautifulSoup(msg, "html.parser")
+    msg = f"<html>{parsed_msg.head}{parsed_msg.body}</html>"
+    line_dict["message"] = msg
+    if log_line_number is not None:
+        line_dict["log_line_number"] = log_line_number
+    return line_dict
+
+
+LOG_FILE = get_logger_filename()
+
+
+def live_log():
+    """
+    Route that streams log file updates using SSE
path: dallinger/experiment_server/dashboard.py
user: login: pmcharrison
body: I think it's normally useful to have more helpful docstrings. I find Cursor very good at automatically generating docstrings. In this case it gave me this:

```py
    """Stream log file updates to the dashboard using Server-Sent Events (SSE).
    
    Creates a streaming response that pushes new log entries to connected clients 
    in real-time. Each log entry is:
    - Read from the log file using Pygtail
    - Parsed from JSON
    - Cleaned and formatted for display
    - Sent as an SSE message
    
    Returns
    -------
    Response
        A Flask response object with mimetype 'text/event-stream' containing
        the event stream of log updates. Each event contains:
        - message : str
            The formatted log message with ANSI codes converted to HTML
        - original_line : str
            The raw log line before processing
        - log_line_number : int, optional
            The line number in the log file
    
    Notes
    -----
    Invalid JSON entries in the log file are silently skipped to maintain
    the stream connection.
    
    The client should connect to this endpoint using an EventSource object
    to receive the updates.
    """
```
created_at: 2025-03-21T12:12:23Z
author_association: COLLABORATOR
diff_hunk: @@ -652,6 +660,141 @@ def dashboard_lifecycle():
     )
 
 
+def clean_line_dict(line_dict, log_line_number=None):
+    msg = line_dict["message"]
+    if msg.endswith(("-", '"')) and ("GET " in msg or " POST" in msg):
+        msg = '"'.join(msg.split('"')[1:])
+    msg = Ansi2HTMLConverter().convert(msg)
+    parsed_msg = bs4.BeautifulSoup(msg, "html.parser")
+    msg = f"<html>{parsed_msg.head}{parsed_msg.body}</html>"
+    line_dict["message"] = msg
+    if log_line_number is not None:
+        line_dict["log_line_number"] = log_line_number
+    return line_dict
+
+
+LOG_FILE = get_logger_filename()
+
+
+def live_log():
+    """
+    Route that streams log file updates using SSE
+    """
+
+    def generate():
+        for line in Pygtail(LOG_FILE):
+            try:
+                line_dict = clean_line_dict(json.loads(line))
+                line_dict["original_line"] = line
+                yield f"data:{json.dumps(line_dict)}\n\n"
+            except json.decoder.JSONDecodeError:
+                continue
+
+    return Response(generate(), mimetype="text/event-stream")
+
+
+def log_read_lines(line_range):
path: dallinger/experiment_server/dashboard.py
user: login: pmcharrison
body: Again, I think it would be good to add docstrings to these functions, I don't find them completely self-explanatory from their names.
created_at: 2025-03-21T12:13:02Z
author_association: COLLABORATOR
diff_hunk: @@ -652,6 +660,141 @@ def dashboard_lifecycle():
     )
 
 
+def clean_line_dict(line_dict, log_line_number=None):
+    msg = line_dict["message"]
+    if msg.endswith(("-", '"')) and ("GET " in msg or " POST" in msg):
+        msg = '"'.join(msg.split('"')[1:])
+    msg = Ansi2HTMLConverter().convert(msg)
+    parsed_msg = bs4.BeautifulSoup(msg, "html.parser")
+    msg = f"<html>{parsed_msg.head}{parsed_msg.body}</html>"
+    line_dict["message"] = msg
+    if log_line_number is not None:
+        line_dict["log_line_number"] = log_line_number
+    return line_dict
+
+
+LOG_FILE = get_logger_filename()
+
+
+def live_log():
+    """
+    Route that streams log file updates using SSE
+    """
+
+    def generate():
+        for line in Pygtail(LOG_FILE):
+            try:
+                line_dict = clean_line_dict(json.loads(line))
+                line_dict["original_line"] = line
+                yield f"data:{json.dumps(line_dict)}\n\n"
+            except json.decoder.JSONDecodeError:
+                continue
+
+    return Response(generate(), mimetype="text/event-stream")
+
+
+def log_read_lines(line_range):
+    lines = []
+    max_line = max(line_range)
+    early_stop = False
+    with open(LOG_FILE) as f:
+        for i, line in enumerate(f):
+            number = i + 1
+            if number in line_range:
+                line_dict = clean_line_dict(json.loads(line), number)
+                lines.append(line_dict)
+            if number > max_line:
+                early_stop = True
+                break
+    return lines, early_stop, i
+
+
+def find_log_line_number(substring):
+    with open(get_logger_filename()) as f:
+        for number, line in enumerate(f):
+            if substring in line:
+                return number + 1
+    return None
+
+
+def log_search_substring(substring):
+    with open(LOG_FILE) as f:
+        for number, line in enumerate(f):
+            if substring in line:
+                line_dict = clean_line_dict(json.loads(line), number)
+                yield f"data:{json.dumps(line_dict)}\n\n"
+    yield f"data:{json.dumps({'stop': True})}\n\n"
+
+
+@dashboard.route("/logs", methods=["GET", "POST"])
path: dallinger/experiment_server/dashboard.py
user: login: pmcharrison
body: Why do we support multiple methods?
created_at: 2025-03-21T12:13:36Z
author_association: COLLABORATOR
diff_hunk: @@ -652,6 +660,141 @@ def dashboard_lifecycle():
     )
 
 
+def clean_line_dict(line_dict, log_line_number=None):
+    msg = line_dict["message"]
+    if msg.endswith(("-", '"')) and ("GET " in msg or " POST" in msg):
+        msg = '"'.join(msg.split('"')[1:])
+    msg = Ansi2HTMLConverter().convert(msg)
+    parsed_msg = bs4.BeautifulSoup(msg, "html.parser")
+    msg = f"<html>{parsed_msg.head}{parsed_msg.body}</html>"
+    line_dict["message"] = msg
+    if log_line_number is not None:
+        line_dict["log_line_number"] = log_line_number
+    return line_dict
+
+
+LOG_FILE = get_logger_filename()
+
+
+def live_log():
+    """
+    Route that streams log file updates using SSE
+    """
+
+    def generate():
+        for line in Pygtail(LOG_FILE):
+            try:
+                line_dict = clean_line_dict(json.loads(line))
+                line_dict["original_line"] = line
+                yield f"data:{json.dumps(line_dict)}\n\n"
+            except json.decoder.JSONDecodeError:
+                continue
+
+    return Response(generate(), mimetype="text/event-stream")
+
+
+def log_read_lines(line_range):
+    lines = []
+    max_line = max(line_range)
+    early_stop = False
+    with open(LOG_FILE) as f:
+        for i, line in enumerate(f):
+            number = i + 1
+            if number in line_range:
+                line_dict = clean_line_dict(json.loads(line), number)
+                lines.append(line_dict)
+            if number > max_line:
+                early_stop = True
+                break
+    return lines, early_stop, i
+
+
+def find_log_line_number(substring):
+    with open(get_logger_filename()) as f:
+        for number, line in enumerate(f):
+            if substring in line:
+                return number + 1
+    return None
+
+
+def log_search_substring(substring):
+    with open(LOG_FILE) as f:
+        for number, line in enumerate(f):
+            if substring in line:
+                line_dict = clean_line_dict(json.loads(line), number)
+                yield f"data:{json.dumps(line_dict)}\n\n"
+    yield f"data:{json.dumps({'stop': True})}\n\n"
+
+
+@dashboard.route("/logs", methods=["GET", "POST"])
+@login_required
+def progress_log():
+    if request.method == "POST":
+        params = request.json
+    else:
+        params = request.args
+    start, end = params.get("start", None), params.get("end", None)
+    n_nulled_params = sum(param is None for param in [start, end])
+    query = params.get("query", None)
+
+    if query:
+        query_type = params.get("type", "lines")
+        if query_type == "lines":
+            return Response(log_search_substring(query), mimetype="text/event-stream")
+        elif query_type == "line_number":
+            line_number = find_log_line_number(query)
+            if line_number is not None:
+                return jsonify({"line_number": line_number})
+            return jsonify({"msg": "No line found."}), 404
+        else:
+            return (
+                jsonify(
+                    {
+                        "msg": f"Invalid query type '{query_type}'. Must be 'line' or 'line_number'."
+                    }
+                ),
+                400,
+            )
+    if n_nulled_params == 2:
path: dallinger/experiment_server/dashboard.py
user: login: pmcharrison
body: I found this `n_nulled_params` logic unwieldy. More readable to write 

```py
if start is None and end is None:
...
if (start is None) != (end is None)
...
```
created_at: 2025-03-21T12:15:23Z
author_association: COLLABORATOR
diff_hunk: @@ -652,6 +660,141 @@ def dashboard_lifecycle():
     )
 
 
+def clean_line_dict(line_dict, log_line_number=None):
+    msg = line_dict["message"]
+    if msg.endswith(("-", '"')) and ("GET " in msg or " POST" in msg):
+        msg = '"'.join(msg.split('"')[1:])
+    msg = Ansi2HTMLConverter().convert(msg)
+    parsed_msg = bs4.BeautifulSoup(msg, "html.parser")
+    msg = f"<html>{parsed_msg.head}{parsed_msg.body}</html>"
+    line_dict["message"] = msg
+    if log_line_number is not None:
+        line_dict["log_line_number"] = log_line_number
+    return line_dict
+
+
+LOG_FILE = get_logger_filename()
+
+
+def live_log():
+    """
+    Route that streams log file updates using SSE
+    """
+
+    def generate():
+        for line in Pygtail(LOG_FILE):
+            try:
+                line_dict = clean_line_dict(json.loads(line))
+                line_dict["original_line"] = line
+                yield f"data:{json.dumps(line_dict)}\n\n"
+            except json.decoder.JSONDecodeError:
+                continue
+
+    return Response(generate(), mimetype="text/event-stream")
+
+
+def log_read_lines(line_range):
+    lines = []
+    max_line = max(line_range)
+    early_stop = False
+    with open(LOG_FILE) as f:
+        for i, line in enumerate(f):
+            number = i + 1
+            if number in line_range:
+                line_dict = clean_line_dict(json.loads(line), number)
+                lines.append(line_dict)
+            if number > max_line:
+                early_stop = True
+                break
+    return lines, early_stop, i
+
+
+def find_log_line_number(substring):
+    with open(get_logger_filename()) as f:
+        for number, line in enumerate(f):
+            if substring in line:
+                return number + 1
+    return None
+
+
+def log_search_substring(substring):
+    with open(LOG_FILE) as f:
+        for number, line in enumerate(f):
+            if substring in line:
+                line_dict = clean_line_dict(json.loads(line), number)
+                yield f"data:{json.dumps(line_dict)}\n\n"
+    yield f"data:{json.dumps({'stop': True})}\n\n"
+
+
+@dashboard.route("/logs", methods=["GET", "POST"])
+@login_required
path: dallinger/experiment_server/dashboard.py
user: login: pmcharrison
body: There should be a detailed docstring here explaining the different query types and the structure of the returned data
created_at: 2025-03-21T12:19:18Z
author_association: COLLABORATOR
diff_hunk: @@ -652,6 +660,141 @@ def dashboard_lifecycle():
     )
 
 
+def clean_line_dict(line_dict, log_line_number=None):
+    msg = line_dict["message"]
+    if msg.endswith(("-", '"')) and ("GET " in msg or " POST" in msg):
+        msg = '"'.join(msg.split('"')[1:])
+    msg = Ansi2HTMLConverter().convert(msg)
+    parsed_msg = bs4.BeautifulSoup(msg, "html.parser")
+    msg = f"<html>{parsed_msg.head}{parsed_msg.body}</html>"
+    line_dict["message"] = msg
+    if log_line_number is not None:
+        line_dict["log_line_number"] = log_line_number
+    return line_dict
+
+
+LOG_FILE = get_logger_filename()
+
+
+def live_log():
+    """
+    Route that streams log file updates using SSE
+    """
+
+    def generate():
+        for line in Pygtail(LOG_FILE):
+            try:
+                line_dict = clean_line_dict(json.loads(line))
+                line_dict["original_line"] = line
+                yield f"data:{json.dumps(line_dict)}\n\n"
+            except json.decoder.JSONDecodeError:
+                continue
+
+    return Response(generate(), mimetype="text/event-stream")
+
+
+def log_read_lines(line_range):
+    lines = []
+    max_line = max(line_range)
+    early_stop = False
+    with open(LOG_FILE) as f:
+        for i, line in enumerate(f):
+            number = i + 1
+            if number in line_range:
+                line_dict = clean_line_dict(json.loads(line), number)
+                lines.append(line_dict)
+            if number > max_line:
+                early_stop = True
+                break
+    return lines, early_stop, i
+
+
+def find_log_line_number(substring):
+    with open(get_logger_filename()) as f:
+        for number, line in enumerate(f):
+            if substring in line:
+                return number + 1
+    return None
+
+
+def log_search_substring(substring):
+    with open(LOG_FILE) as f:
+        for number, line in enumerate(f):
+            if substring in line:
+                line_dict = clean_line_dict(json.loads(line), number)
+                yield f"data:{json.dumps(line_dict)}\n\n"
+    yield f"data:{json.dumps({'stop': True})}\n\n"
+
+
+@dashboard.route("/logs", methods=["GET", "POST"])
+@login_required
+def progress_log():
+    if request.method == "POST":
+        params = request.json
+    else:
+        params = request.args
+    start, end = params.get("start", None), params.get("end", None)
+    n_nulled_params = sum(param is None for param in [start, end])
+    query = params.get("query", None)
+
+    if query:
+        query_type = params.get("type", "lines")
path: dallinger/experiment_server/dashboard.py
user: login: pmcharrison
body: I'm not sure I would want to combine so many different commands within the same route. My understanding is that with APIs the best practice is to have a standard output format and not let this vary so much depending on the options provided.

What about

- `/logs/live`
- `/logs/find_lines`
- `/logs/find_line_number`

which you then separate into different functions?
created_at: 2025-03-21T12:21:31Z
author_association: COLLABORATOR
diff_hunk: @@ -652,6 +660,141 @@ def dashboard_lifecycle():
     )
 
 
+def clean_line_dict(line_dict, log_line_number=None):
+    msg = line_dict["message"]
+    if msg.endswith(("-", '"')) and ("GET " in msg or " POST" in msg):
+        msg = '"'.join(msg.split('"')[1:])
+    msg = Ansi2HTMLConverter().convert(msg)
+    parsed_msg = bs4.BeautifulSoup(msg, "html.parser")
+    msg = f"<html>{parsed_msg.head}{parsed_msg.body}</html>"
+    line_dict["message"] = msg
+    if log_line_number is not None:
+        line_dict["log_line_number"] = log_line_number
+    return line_dict
+
+
+LOG_FILE = get_logger_filename()
+
+
+def live_log():
+    """
+    Route that streams log file updates using SSE
+    """
+
+    def generate():
+        for line in Pygtail(LOG_FILE):
+            try:
+                line_dict = clean_line_dict(json.loads(line))
+                line_dict["original_line"] = line
+                yield f"data:{json.dumps(line_dict)}\n\n"
+            except json.decoder.JSONDecodeError:
+                continue
+
+    return Response(generate(), mimetype="text/event-stream")
+
+
+def log_read_lines(line_range):
+    lines = []
+    max_line = max(line_range)
+    early_stop = False
+    with open(LOG_FILE) as f:
+        for i, line in enumerate(f):
+            number = i + 1
+            if number in line_range:
+                line_dict = clean_line_dict(json.loads(line), number)
+                lines.append(line_dict)
+            if number > max_line:
+                early_stop = True
+                break
+    return lines, early_stop, i
+
+
+def find_log_line_number(substring):
+    with open(get_logger_filename()) as f:
+        for number, line in enumerate(f):
+            if substring in line:
+                return number + 1
+    return None
+
+
+def log_search_substring(substring):
+    with open(LOG_FILE) as f:
+        for number, line in enumerate(f):
+            if substring in line:
+                line_dict = clean_line_dict(json.loads(line), number)
+                yield f"data:{json.dumps(line_dict)}\n\n"
+    yield f"data:{json.dumps({'stop': True})}\n\n"
+
+
+@dashboard.route("/logs", methods=["GET", "POST"])
+@login_required
+def progress_log():
+    if request.method == "POST":
+        params = request.json
+    else:
+        params = request.args
+    start, end = params.get("start", None), params.get("end", None)
+    n_nulled_params = sum(param is None for param in [start, end])
+    query = params.get("query", None)
+
+    if query:
+        query_type = params.get("type", "lines")
+        if query_type == "lines":
+            return Response(log_search_substring(query), mimetype="text/event-stream")
+        elif query_type == "line_number":
+            line_number = find_log_line_number(query)
+            if line_number is not None:
+                return jsonify({"line_number": line_number})
+            return jsonify({"msg": "No line found."}), 404
+        else:
+            return (
+                jsonify(
+                    {
+                        "msg": f"Invalid query type '{query_type}'. Must be 'line' or 'line_number'."
+                    }
+                ),
+                400,
+            )
+    if n_nulled_params == 2:
+        return live_log()
+    if n_nulled_params == 1:
+        return (
+            jsonify(
+                {
+                    "msg": "You should either provide both 'start' and 'end' or none of them."
+                }
+            ),
+            400,
+        )
+
+    start, end = int(start), int(end)
+    if start < 1:
+        return jsonify({"msg": "'start' must be greater than 0."}), 400
+
+    if start > end:
+        return (
+            jsonify(
+                {
+                    "msg": "The 'end' parameter should be less than the 'start' parameter."
+                }
+            ),
+            400,
+        )
+
+    lines, early_stop, last_line = log_read_lines(range(start, end + 1))
+    if not early_stop and start > last_line:
+        return (
+            jsonify(
path: dallinger/experiment_server/dashboard.py
user: login: pmcharrison
body: There's quite a lot of duplication here, it might be good to refactor the error logic into a helper function.
created_at: 2025-03-21T12:22:18Z
author_association: COLLABORATOR
diff_hunk: @@ -0,0 +1,254 @@
+{% extends "base/dashboard.html" %}
+
+{% block body %}
+    <h1>Logger</h1>
+    <span id="info-text">
+    Below are the logs for the experiment. You can filter the logs by log level. By clicking on the line number, you can
+    get a link to that specific line.
+    </span>
+    <form id="search-form" class="mb-3">
+        <div class="input-group">
+            <input type="text" class="form-control" name="query"
+                   placeholder="Search the logs for a particular substring (case sensitive)...">
+            <button class="btn btn-primary" type="submit">Search</button>
+        </div>
+    </form>
+
+    <div id="filters"></div>
+    <div>
+    </div>
+    <div id="log-container">
+        <table id="log" class="monospace">
+            <tbody>
+            </tbody>
+        </table>
+    </div>
+
+    <style>
+        #log-container {
+            overflow: auto;
+            height: 80vh;
+            font-size: .875em;
+            background: #f6f8fa;
+            padding: 16px;
+            color: #1f2328;
+        }
+
+        #log-container pre {
+            font-size: 1em;
+        }
+
+        .monospace {
+            font-family: ui-monospace, SFMono-Regular, SF Mono, Menlo, Consolas, Liberation Mono, monospace;
+        }
+
+        #log {
+            caption-side: bottom;
+        }
+
+        #log a {
+            color: inherit;
+        }
+
+        #log tr {
+            vertical-align: top;
+        }
+
+        #log tr.highlight {
+            background: lightyellow;
+        }
+
+        .level-checkbox {
+            margin-right: 10px;
+            font-weight: bold;
+        }
+
+    </style>
+
+    <script>
+        const logLevels = ["DEBUG", "INFO", "SUCCESS", "WARNING", "ERROR", "CRITICAL"];
+        const colors = {
+            "DEBUG": "#4d98cc",
+            "SUCCESS": "#79b25f",
+            "WARNING": "#ddaa2d",
+            "ERROR": "#cf3f61",
+            "CRITICAL": "#cf3f61",
+            "DEFAULT": "#6c757d"
+        };
+
+        // Function to create checkboxes dynamically
+        function createCheckbox(level) {
+            const color = colors[level] || colors.DEFAULT;
+            return `<label class="level-checkbox" style="color: ${color};">
+                <input type="checkbox" value="${level}" checked> ${level}
+            </label>`;
+        }
+
+        // Append checkboxes to the filter div
+        document.addEventListener("DOMContentLoaded", () => {
+            const filtersDiv = document.getElementById("filters");
+            logLevels.forEach(level => {
+                filtersDiv.innerHTML += createCheckbox(level);
+            });
+
+            document.querySelectorAll('.level-checkbox input').forEach(checkbox => {
+                checkbox.addEventListener('change', function () {
+                    document.querySelectorAll(`.status-${this.value.toLowerCase()}`)
+                        .forEach(row => row.style.display = this.checked ? "table-row" : "none");
+                });
+            });
+
+            activateTooltips();
+            setupSearchReset();
+            fetchLogData();
+        });
+
+        // Function to activate Bootstrap tooltips
+        function activateTooltips() {
path: dallinger/frontend/templates/dashboard_logger.html
user: login: pmcharrison
body: I think all the functions here could benefit from documentation
created_at: 2025-03-21T12:22:38Z
author_association: COLLABORATOR
diff_hunk: @@ -652,6 +660,141 @@ def dashboard_lifecycle():
     )
 
 
+def clean_line_dict(line_dict, log_line_number=None):
+    msg = line_dict["message"]
+    if msg.endswith(("-", '"')) and ("GET " in msg or " POST" in msg):
+        msg = '"'.join(msg.split('"')[1:])
+    msg = Ansi2HTMLConverter().convert(msg)
+    parsed_msg = bs4.BeautifulSoup(msg, "html.parser")
+    msg = f"<html>{parsed_msg.head}{parsed_msg.body}</html>"
+    line_dict["message"] = msg
+    if log_line_number is not None:
+        line_dict["log_line_number"] = log_line_number
+    return line_dict
+
+
+LOG_FILE = get_logger_filename()
+
+
+def live_log():
path: dallinger/experiment_server/dashboard.py
user: login: pmcharrison
body: The current implementation only shows logs that are new since the page was opened. This is good for managing performance, but won't users sometimes want to read previous logs without having to search?
created_at: 2025-03-21T12:25:46Z
author_association: COLLABORATOR
diff_hunk: @@ -652,6 +660,141 @@ def dashboard_lifecycle():
     )
 
 
+def clean_line_dict(line_dict, log_line_number=None):
+    msg = line_dict["message"]
+    if msg.endswith(("-", '"')) and ("GET " in msg or " POST" in msg):
+        msg = '"'.join(msg.split('"')[1:])
+    msg = Ansi2HTMLConverter().convert(msg)
+    parsed_msg = bs4.BeautifulSoup(msg, "html.parser")
+    msg = f"<html>{parsed_msg.head}{parsed_msg.body}</html>"
+    line_dict["message"] = msg
+    if log_line_number is not None:
+        line_dict["log_line_number"] = log_line_number
+    return line_dict
+
+
+LOG_FILE = get_logger_filename()
+
+
+def live_log():
path: dallinger/experiment_server/dashboard.py
user: login: pmcharrison
body: I suppose you could just explain in the user message on the logger page that this is a limitation of the current interface.
created_at: 2025-03-21T12:28:16Z
author_association: COLLABORATOR
diff_hunk: @@ -20,7 +20,7 @@
 )
 from .patches import patch
 
-logger = logging.getLogger(__name__)
+logger = logging.getLogger("dallinger")
path: dallinger/__init__.py
user: login: polvanrijn
body: Because it then logs all of those messages under `dallinger` and the logger will tell which file (e.g. `recruiters.py`), which function and which line. Setting it to `__name__` it is kinda messy, especially if `dallinger` is installed in editable mode and `__name__` is the full absolute path.

I was thinking as a compromise one can add a function:
``` python
def get_logger(name:str="dallinger")
    import logging
    return logger = logging.getLogger(name)
created_at: 2025-03-24T08:50:35Z
author_association: COLLABORATOR
diff_hunk: @@ -20,7 +20,7 @@
 )
 from .patches import patch
 
-logger = logging.getLogger(__name__)
+logger = logging.getLogger("dallinger")
path: dallinger/__init__.py
user: login: polvanrijn
body: See proposal above
created_at: 2025-03-24T08:50:46Z
author_association: COLLABORATOR
diff_hunk: @@ -99,12 +100,13 @@ def launch():
     ]
     LOG_LEVEL = LOG_LEVELS[config.get("loglevel")]
     logging.basicConfig(format="%(asctime)s %(message)s", level=LOG_LEVEL)
-
     # Avoid duplicate logging to stderr
     error_logger = logging.getLogger("gunicorn.error")
     error_logger.propagate = False
     access_logger = logging.getLogger("gunicorn.access")
     access_logger.propagate = False
+    attach_json_logger(error_logger)
+    attach_json_logger(access_logger)
path: dallinger/experiment_server/gunicorn.py
user: login: polvanrijn
body: Yes, it's a difficult one isn't it. `dallinger.log` is a different log output than `server.log`. Also the default config is to set `server.log` to `-` which will stop the logging, so I did not want to touch this.

Actually, the right extension would be [`.jsonl`](https://jsonlines.org/).
So what about hardcoding it to `dallinger.jsonl`? I don't want to mess with the config variable `logfile` because it's overridden in most demos and is error-prone.
created_at: 2025-03-24T08:53:59Z
author_association: COLLABORATOR
diff_hunk: @@ -652,6 +660,141 @@ def dashboard_lifecycle():
     )
 
 
+def clean_line_dict(line_dict, log_line_number=None):
path: dallinger/experiment_server/dashboard.py
user: login: polvanrijn
body: Yes, I agree. I renamed it to `line_info` to clarify it's not a string, but a dictionary with all information on the line.
created_at: 2025-03-24T09:39:41Z
author_association: COLLABORATOR
diff_hunk: @@ -652,6 +660,141 @@ def dashboard_lifecycle():
     )
 
 
+def clean_line_dict(line_dict, log_line_number=None):
+    msg = line_dict["message"]
path: dallinger/experiment_server/dashboard.py
user: login: polvanrijn
body: Done
created_at: 2025-03-24T09:39:51Z
author_association: COLLABORATOR
diff_hunk: @@ -652,6 +660,141 @@ def dashboard_lifecycle():
     )
 
 
+def clean_line_dict(line_dict, log_line_number=None):
+    msg = line_dict["message"]
+    if msg.endswith(("-", '"')) and ("GET " in msg or " POST" in msg):
+        msg = '"'.join(msg.split('"')[1:])
+    msg = Ansi2HTMLConverter().convert(msg)
+    parsed_msg = bs4.BeautifulSoup(msg, "html.parser")
+    msg = f"<html>{parsed_msg.head}{parsed_msg.body}</html>"
+    line_dict["message"] = msg
+    if log_line_number is not None:
+        line_dict["log_line_number"] = log_line_number
+    return line_dict
+
+
+LOG_FILE = get_logger_filename()
+
+
+def live_log():
+    """
+    Route that streams log file updates using SSE
+    """
+
+    def generate():
+        for line in Pygtail(LOG_FILE):
+            try:
+                line_dict = clean_line_dict(json.loads(line))
+                line_dict["original_line"] = line
+                yield f"data:{json.dumps(line_dict)}\n\n"
+            except json.decoder.JSONDecodeError:
+                continue
path: dallinger/experiment_server/dashboard.py
user: login: polvanrijn
body: Done
created_at: 2025-03-24T09:42:13Z
author_association: COLLABORATOR
diff_hunk: @@ -99,12 +100,13 @@ def launch():
     ]
     LOG_LEVEL = LOG_LEVELS[config.get("loglevel")]
     logging.basicConfig(format="%(asctime)s %(message)s", level=LOG_LEVEL)
-
     # Avoid duplicate logging to stderr
     error_logger = logging.getLogger("gunicorn.error")
     error_logger.propagate = False
     access_logger = logging.getLogger("gunicorn.access")
     access_logger.propagate = False
+    attach_json_logger(error_logger)
+    attach_json_logger(access_logger)
path: dallinger/experiment_server/gunicorn.py
user: login: pmcharrison
body: The `dallinger` filename seems pretty vague to me. Is a good compromise `logs.jsonl`?
created_at: 2025-03-24T09:49:09Z
author_association: COLLABORATOR
diff_hunk: @@ -652,6 +660,141 @@ def dashboard_lifecycle():
     )
 
 
+def clean_line_dict(line_dict, log_line_number=None):
+    msg = line_dict["message"]
+    if msg.endswith(("-", '"')) and ("GET " in msg or " POST" in msg):
+        msg = '"'.join(msg.split('"')[1:])
+    msg = Ansi2HTMLConverter().convert(msg)
+    parsed_msg = bs4.BeautifulSoup(msg, "html.parser")
+    msg = f"<html>{parsed_msg.head}{parsed_msg.body}</html>"
+    line_dict["message"] = msg
+    if log_line_number is not None:
+        line_dict["log_line_number"] = log_line_number
+    return line_dict
+
+
+LOG_FILE = get_logger_filename()
+
+
+def live_log():
+    """
+    Route that streams log file updates using SSE
path: dallinger/experiment_server/dashboard.py
user: login: polvanrijn
body: Done
created_at: 2025-03-24T09:57:50Z
author_association: COLLABORATOR
diff_hunk: @@ -20,7 +20,7 @@
 )
 from .patches import patch
 
-logger = logging.getLogger(__name__)
+logger = logging.getLogger("dallinger")
path: dallinger/__init__.py
user: login: polvanrijn
body: @pmcharrison, what do you think?
created_at: 2025-03-24T09:58:57Z
author_association: COLLABORATOR
diff_hunk: @@ -20,7 +20,7 @@
 )
 from .patches import patch
 
-logger = logging.getLogger(__name__)
+logger = logging.getLogger("dallinger")
path: dallinger/__init__.py
user: login: pmcharrison
body: I just tried out various options myself, with editable mode:

```
>>> logging.getLogger(__name__)
INFO:dallinger.recruiters:Initializing HotAirRecruiter
```

```
>>> logging.getLogger(__file__)
INFO:Users/peter/Dallinger/dallinger/recruiters.py:Initializing HotAirRecruiter
```

```
>>> logging.getLogger("dallinger")
INFO:dallinger:Initializing HotAirRecruiter
```

The full path problem only happens with `__file__`, but I think `__name__` is fine, no? I think it's quite handy actually how it points you to the right file.
created_at: 2025-03-24T10:19:15Z
author_association: COLLABORATOR
diff_hunk: @@ -20,7 +20,7 @@
 )
 from .patches import patch
 
-logger = logging.getLogger(__name__)
+logger = logging.getLogger("dallinger")
path: dallinger/__init__.py
user: login: pmcharrison
body: With these kinds of things I tend to default to not changing pre-existing Dallinger behaviour unless there's a strong reason to
created_at: 2025-03-24T10:20:14Z
author_association: COLLABORATOR
diff_hunk: @@ -652,6 +660,141 @@ def dashboard_lifecycle():
     )
 
 
+def clean_line_dict(line_dict, log_line_number=None):
+    msg = line_dict["message"]
+    if msg.endswith(("-", '"')) and ("GET " in msg or " POST" in msg):
+        msg = '"'.join(msg.split('"')[1:])
+    msg = Ansi2HTMLConverter().convert(msg)
+    parsed_msg = bs4.BeautifulSoup(msg, "html.parser")
+    msg = f"<html>{parsed_msg.head}{parsed_msg.body}</html>"
+    line_dict["message"] = msg
+    if log_line_number is not None:
+        line_dict["log_line_number"] = log_line_number
+    return line_dict
+
+
+LOG_FILE = get_logger_filename()
+
+
+def live_log():
+    """
+    Route that streams log file updates using SSE
+    """
+
+    def generate():
+        for line in Pygtail(LOG_FILE):
+            try:
+                line_dict = clean_line_dict(json.loads(line))
+                line_dict["original_line"] = line
+                yield f"data:{json.dumps(line_dict)}\n\n"
+            except json.decoder.JSONDecodeError:
+                continue
+
+    return Response(generate(), mimetype="text/event-stream")
+
+
+def log_read_lines(line_range):
+    lines = []
+    max_line = max(line_range)
+    early_stop = False
+    with open(LOG_FILE) as f:
+        for i, line in enumerate(f):
+            number = i + 1
+            if number in line_range:
+                line_dict = clean_line_dict(json.loads(line), number)
+                lines.append(line_dict)
+            if number > max_line:
+                early_stop = True
+                break
+    return lines, early_stop, i
+
+
+def find_log_line_number(substring):
+    with open(get_logger_filename()) as f:
+        for number, line in enumerate(f):
+            if substring in line:
+                return number + 1
+    return None
+
+
+def log_search_substring(substring):
+    with open(LOG_FILE) as f:
+        for number, line in enumerate(f):
+            if substring in line:
+                line_dict = clean_line_dict(json.loads(line), number)
+                yield f"data:{json.dumps(line_dict)}\n\n"
+    yield f"data:{json.dumps({'stop': True})}\n\n"
+
+
+@dashboard.route("/logs", methods=["GET", "POST"])
+@login_required
+def progress_log():
+    if request.method == "POST":
+        params = request.json
+    else:
+        params = request.args
+    start, end = params.get("start", None), params.get("end", None)
+    n_nulled_params = sum(param is None for param in [start, end])
+    query = params.get("query", None)
+
+    if query:
+        query_type = params.get("type", "lines")
path: dallinger/experiment_server/dashboard.py
user: login: polvanrijn
body: Great suggestion. Much cleaner now!!
created_at: 2025-03-24T10:36:06Z
author_association: COLLABORATOR
diff_hunk: @@ -652,6 +660,141 @@ def dashboard_lifecycle():
     )
 
 
+def clean_line_dict(line_dict, log_line_number=None):
+    msg = line_dict["message"]
+    if msg.endswith(("-", '"')) and ("GET " in msg or " POST" in msg):
+        msg = '"'.join(msg.split('"')[1:])
+    msg = Ansi2HTMLConverter().convert(msg)
+    parsed_msg = bs4.BeautifulSoup(msg, "html.parser")
+    msg = f"<html>{parsed_msg.head}{parsed_msg.body}</html>"
+    line_dict["message"] = msg
+    if log_line_number is not None:
+        line_dict["log_line_number"] = log_line_number
+    return line_dict
+
+
+LOG_FILE = get_logger_filename()
+
+
+def live_log():
+    """
+    Route that streams log file updates using SSE
+    """
+
+    def generate():
+        for line in Pygtail(LOG_FILE):
+            try:
+                line_dict = clean_line_dict(json.loads(line))
+                line_dict["original_line"] = line
+                yield f"data:{json.dumps(line_dict)}\n\n"
+            except json.decoder.JSONDecodeError:
+                continue
+
+    return Response(generate(), mimetype="text/event-stream")
+
+
+def log_read_lines(line_range):
+    lines = []
+    max_line = max(line_range)
+    early_stop = False
+    with open(LOG_FILE) as f:
+        for i, line in enumerate(f):
+            number = i + 1
+            if number in line_range:
+                line_dict = clean_line_dict(json.loads(line), number)
+                lines.append(line_dict)
+            if number > max_line:
+                early_stop = True
+                break
+    return lines, early_stop, i
+
+
+def find_log_line_number(substring):
+    with open(get_logger_filename()) as f:
+        for number, line in enumerate(f):
+            if substring in line:
+                return number + 1
+    return None
+
+
+def log_search_substring(substring):
+    with open(LOG_FILE) as f:
+        for number, line in enumerate(f):
+            if substring in line:
+                line_dict = clean_line_dict(json.loads(line), number)
+                yield f"data:{json.dumps(line_dict)}\n\n"
+    yield f"data:{json.dumps({'stop': True})}\n\n"
+
+
+@dashboard.route("/logs", methods=["GET", "POST"])
+@login_required
path: dallinger/experiment_server/dashboard.py
user: login: polvanrijn
body: Yep
created_at: 2025-03-24T10:44:57Z
author_association: COLLABORATOR
diff_hunk: @@ -652,6 +660,141 @@ def dashboard_lifecycle():
     )
 
 
+def clean_line_dict(line_dict, log_line_number=None):
+    msg = line_dict["message"]
+    if msg.endswith(("-", '"')) and ("GET " in msg or " POST" in msg):
+        msg = '"'.join(msg.split('"')[1:])
+    msg = Ansi2HTMLConverter().convert(msg)
+    parsed_msg = bs4.BeautifulSoup(msg, "html.parser")
+    msg = f"<html>{parsed_msg.head}{parsed_msg.body}</html>"
+    line_dict["message"] = msg
+    if log_line_number is not None:
+        line_dict["log_line_number"] = log_line_number
+    return line_dict
+
+
+LOG_FILE = get_logger_filename()
+
+
+def live_log():
+    """
+    Route that streams log file updates using SSE
+    """
+
+    def generate():
+        for line in Pygtail(LOG_FILE):
+            try:
+                line_dict = clean_line_dict(json.loads(line))
+                line_dict["original_line"] = line
+                yield f"data:{json.dumps(line_dict)}\n\n"
+            except json.decoder.JSONDecodeError:
+                continue
+
+    return Response(generate(), mimetype="text/event-stream")
+
+
+def log_read_lines(line_range):
+    lines = []
+    max_line = max(line_range)
+    early_stop = False
+    with open(LOG_FILE) as f:
+        for i, line in enumerate(f):
+            number = i + 1
+            if number in line_range:
+                line_dict = clean_line_dict(json.loads(line), number)
+                lines.append(line_dict)
+            if number > max_line:
+                early_stop = True
+                break
+    return lines, early_stop, i
+
+
+def find_log_line_number(substring):
+    with open(get_logger_filename()) as f:
+        for number, line in enumerate(f):
+            if substring in line:
+                return number + 1
+    return None
+
+
+def log_search_substring(substring):
+    with open(LOG_FILE) as f:
+        for number, line in enumerate(f):
+            if substring in line:
+                line_dict = clean_line_dict(json.loads(line), number)
+                yield f"data:{json.dumps(line_dict)}\n\n"
+    yield f"data:{json.dumps({'stop': True})}\n\n"
+
+
+@dashboard.route("/logs", methods=["GET", "POST"])
+@login_required
+def progress_log():
+    if request.method == "POST":
+        params = request.json
+    else:
+        params = request.args
+    start, end = params.get("start", None), params.get("end", None)
+    n_nulled_params = sum(param is None for param in [start, end])
+    query = params.get("query", None)
+
+    if query:
+        query_type = params.get("type", "lines")
+        if query_type == "lines":
+            return Response(log_search_substring(query), mimetype="text/event-stream")
+        elif query_type == "line_number":
+            line_number = find_log_line_number(query)
+            if line_number is not None:
+                return jsonify({"line_number": line_number})
+            return jsonify({"msg": "No line found."}), 404
+        else:
+            return (
+                jsonify(
+                    {
+                        "msg": f"Invalid query type '{query_type}'. Must be 'line' or 'line_number'."
+                    }
+                ),
+                400,
+            )
+    if n_nulled_params == 2:
+        return live_log()
+    if n_nulled_params == 1:
+        return (
+            jsonify(
+                {
+                    "msg": "You should either provide both 'start' and 'end' or none of them."
+                }
+            ),
+            400,
+        )
+
+    start, end = int(start), int(end)
+    if start < 1:
+        return jsonify({"msg": "'start' must be greater than 0."}), 400
+
+    if start > end:
+        return (
+            jsonify(
+                {
+                    "msg": "The 'end' parameter should be less than the 'start' parameter."
+                }
+            ),
+            400,
+        )
+
+    lines, early_stop, last_line = log_read_lines(range(start, end + 1))
+    if not early_stop and start > last_line:
+        return (
+            jsonify(
path: dallinger/experiment_server/dashboard.py
user: login: polvanrijn
body: Done
created_at: 2025-03-24T10:45:03Z
author_association: COLLABORATOR
diff_hunk: @@ -0,0 +1,254 @@
+{% extends "base/dashboard.html" %}
+
+{% block body %}
+    <h1>Logger</h1>
+    <span id="info-text">
+    Below are the logs for the experiment. You can filter the logs by log level. By clicking on the line number, you can
+    get a link to that specific line.
+    </span>
+    <form id="search-form" class="mb-3">
+        <div class="input-group">
+            <input type="text" class="form-control" name="query"
+                   placeholder="Search the logs for a particular substring (case sensitive)...">
+            <button class="btn btn-primary" type="submit">Search</button>
+        </div>
+    </form>
+
+    <div id="filters"></div>
+    <div>
+    </div>
+    <div id="log-container">
+        <table id="log" class="monospace">
+            <tbody>
+            </tbody>
+        </table>
+    </div>
+
+    <style>
+        #log-container {
+            overflow: auto;
+            height: 80vh;
+            font-size: .875em;
+            background: #f6f8fa;
+            padding: 16px;
+            color: #1f2328;
+        }
+
+        #log-container pre {
+            font-size: 1em;
+        }
+
+        .monospace {
+            font-family: ui-monospace, SFMono-Regular, SF Mono, Menlo, Consolas, Liberation Mono, monospace;
+        }
+
+        #log {
+            caption-side: bottom;
+        }
+
+        #log a {
+            color: inherit;
+        }
+
+        #log tr {
+            vertical-align: top;
+        }
+
+        #log tr.highlight {
+            background: lightyellow;
+        }
+
+        .level-checkbox {
+            margin-right: 10px;
+            font-weight: bold;
+        }
+
+    </style>
+
+    <script>
+        const logLevels = ["DEBUG", "INFO", "SUCCESS", "WARNING", "ERROR", "CRITICAL"];
+        const colors = {
+            "DEBUG": "#4d98cc",
+            "SUCCESS": "#79b25f",
+            "WARNING": "#ddaa2d",
+            "ERROR": "#cf3f61",
+            "CRITICAL": "#cf3f61",
+            "DEFAULT": "#6c757d"
+        };
+
+        // Function to create checkboxes dynamically
+        function createCheckbox(level) {
+            const color = colors[level] || colors.DEFAULT;
+            return `<label class="level-checkbox" style="color: ${color};">
+                <input type="checkbox" value="${level}" checked> ${level}
+            </label>`;
+        }
+
+        // Append checkboxes to the filter div
+        document.addEventListener("DOMContentLoaded", () => {
+            const filtersDiv = document.getElementById("filters");
+            logLevels.forEach(level => {
+                filtersDiv.innerHTML += createCheckbox(level);
+            });
+
+            document.querySelectorAll('.level-checkbox input').forEach(checkbox => {
+                checkbox.addEventListener('change', function () {
+                    document.querySelectorAll(`.status-${this.value.toLowerCase()}`)
+                        .forEach(row => row.style.display = this.checked ? "table-row" : "none");
+                });
+            });
+
+            activateTooltips();
+            setupSearchReset();
+            fetchLogData();
+        });
+
+        // Function to activate Bootstrap tooltips
+        function activateTooltips() {
path: dallinger/frontend/templates/dashboard_logger.html
user: login: polvanrijn
body: Done
created_at: 2025-03-24T10:45:08Z
author_association: COLLABORATOR
diff_hunk: @@ -20,7 +20,7 @@
 )
 from .patches import patch
 
-logger = logging.getLogger(__name__)
+logger = logging.getLogger("dallinger")
path: dallinger/__init__.py
user: login: polvanrijn
body: Ok, I reverted back to:
```python
logger = logging.getLogger(__name__)
```
created_at: 2025-03-24T10:49:34Z
author_association: COLLABORATOR
diff_hunk: @@ -652,6 +660,141 @@ def dashboard_lifecycle():
     )
 
 
+def clean_line_dict(line_dict, log_line_number=None):
+    msg = line_dict["message"]
+    if msg.endswith(("-", '"')) and ("GET " in msg or " POST" in msg):
+        msg = '"'.join(msg.split('"')[1:])
+    msg = Ansi2HTMLConverter().convert(msg)
+    parsed_msg = bs4.BeautifulSoup(msg, "html.parser")
+    msg = f"<html>{parsed_msg.head}{parsed_msg.body}</html>"
+    line_dict["message"] = msg
+    if log_line_number is not None:
+        line_dict["log_line_number"] = log_line_number
+    return line_dict
+
+
+LOG_FILE = get_logger_filename()
+
+
+def live_log():
path: dallinger/experiment_server/dashboard.py
user: login: polvanrijn
body: Done
created_at: 2025-03-24T10:50:01Z
author_association: COLLABORATOR
diff_hunk: @@ -652,6 +660,141 @@ def dashboard_lifecycle():
     )
 
 
+def clean_line_dict(line_dict, log_line_number=None):
+    msg = line_dict["message"]
+    if msg.endswith(("-", '"')) and ("GET " in msg or " POST" in msg):
+        msg = '"'.join(msg.split('"')[1:])
+    msg = Ansi2HTMLConverter().convert(msg)
+    parsed_msg = bs4.BeautifulSoup(msg, "html.parser")
+    msg = f"<html>{parsed_msg.head}{parsed_msg.body}</html>"
+    line_dict["message"] = msg
+    if log_line_number is not None:
+        line_dict["log_line_number"] = log_line_number
+    return line_dict
+
+
+LOG_FILE = get_logger_filename()
+
+
+def live_log():
path: dallinger/experiment_server/dashboard.py
user: login: polvanrijn
body: Done
created_at: 2025-03-24T11:25:23Z
author_association: COLLABORATOR
diff_hunk: @@ -652,6 +661,203 @@ def dashboard_lifecycle():
     )
 
 
+def clean_line_info(line_info: dict, log_line_number: int = None) -> dict:
+    """
+    Clean the line info for the log viewer. Convert ANSI to HTML (e.g. ANSI bold to HTML bold) and remove the
+    leading timestamp and log level (because it is already displayed in the table).
+
path: dallinger/experiment_server/dashboard.py
user: login: pmcharrison
body: I propose adding the following:

> Add the log's line number if provided.
created_at: 2025-03-24T13:21:19Z
author_association: COLLABORATOR
diff_hunk: @@ -652,6 +660,141 @@ def dashboard_lifecycle():
     )
 
 
+def clean_line_dict(line_dict, log_line_number=None):
+    msg = line_dict["message"]
path: dallinger/experiment_server/dashboard.py
user: login: pmcharrison
body: Did you push the changes here? This is the kind of cleaning I am thinking:

Instead of 

```py
    if msg.endswith(("-", '"')) and ("GET " in msg or " POST" in msg):
        msg = '"'.join(msg.split('"')[1:])
```

you write

```py
msg = clean_http_request_log(msg)

def clean_http_request_log(msg: str) -> str:
    """Remove redundant prefix from HTTP request log messages.
    
    Parameters
    ----------
    msg : str
        The log message to clean
        
    Returns
    -------
    str
        The cleaned message with redundant prefix removed
        
    Examples
    --------
    >>> msg = '127.0.0.1 - - [20/Mar/2024:10:00:00] "GET /dashboard HTTP/1.1" 200 -'
    >>> clean_http_request_log(msg)
    'GET /dashboard HTTP/1.1" 200 -'
    """
    is_http_request = "GET " in msg or " POST" in msg
    has_expected_ending = msg.endswith(("-", '"'))
    
    if is_http_request and has_expected_ending:
        return '"'.join(msg.split('"')[1:])
    return msg
```

This kind of refactoring is really quick with LLMs. You can also really easy use the LLM to write some unit tests for it.
created_at: 2025-03-24T13:29:53Z
author_association: COLLABORATOR
diff_hunk: @@ -652,6 +660,141 @@ def dashboard_lifecycle():
     )
 
 
+def clean_line_dict(line_dict, log_line_number=None):
+    msg = line_dict["message"]
path: dallinger/experiment_server/dashboard.py
user: login: pmcharrison
body: I will continue the review once you confirm there are not pending changes that you are waiting to push
created_at: 2025-03-24T13:30:13Z
author_association: COLLABORATOR
diff_hunk: @@ -652,6 +661,203 @@ def dashboard_lifecycle():
     )
 
 
+def clean_line_info(line_info: dict, log_line_number: int = None) -> dict:
+    """
+    Clean the line info for the log viewer. Convert ANSI to HTML (e.g. ANSI bold to HTML bold) and remove the
+    leading timestamp and log level (because it is already displayed in the table).
+
path: dallinger/experiment_server/dashboard.py
user: login: polvanrijn
body: Done
created_at: 2025-03-24T13:37:00Z
author_association: COLLABORATOR
diff_hunk: @@ -652,6 +661,229 @@ def dashboard_lifecycle():
     )
 
 
+def clean_http_request_log(msg: str) -> str:
+    """Remove redundant prefix from HTTP request log messages.
+
+    Parameters
+    ----------
+    msg : str
+        The log message to clean
+
+    Returns
+    -------
+    str
+        The cleaned message with redundant prefix removed
+
+    Examples
+    --------
+    >>> msg = '127.0.0.1 - - [20/Mar/2024:10:00:00] "GET /dashboard HTTP/1.1" 200 -'
+    >>> clean_http_request_log(msg)
+    'GET /dashboard HTTP/1.1" 200 -'
+    """
+    is_http_request = "GET " in msg or " POST" in msg
+    has_expected_ending = msg.endswith(("-", '"'))
+
+    if is_http_request and has_expected_ending:
+        return '"'.join(msg.split('"')[1:])
+    return msg
+
+
+def clean_line_info(line_info: dict, log_line_number: int = None) -> dict:
+    """
+    Clean the line info for the log viewer. Convert ANSI to HTML (e.g. ANSI bold to HTML bold) and remove the leading
+    timestamp and log level (because it is already displayed in the table) and add the log's line number if provided.
+
+    :param line_info: The line info to clean
+    :type line_info: dict
+
+    :param log_line_number: The line number in the log file (optional)
+    :type log_line_number: int
+
+    :return: The cleaned line info
+    """
+    msg = line_info["message"]
+    msg = clean_http_request_log(msg)
+    msg = Ansi2HTMLConverter().convert(msg)
+    parsed_msg = bs4.BeautifulSoup(msg, "html.parser")
+    msg = f"<html>{parsed_msg.head}{parsed_msg.body}</html>"
+    line_info["message"] = msg
path: dallinger/experiment_server/dashboard.py
user: login: pmcharrison
body: OK, but I also meant that the rest of the function needed refactoring too for readability. So something like

```py
from typing import Optional
import logging
import bs4
from ansi2html import Ansi2HTMLConverter

logger = logging.getLogger(__name__)

def clean_http_request(msg: str) -> str:
    """Remove redundant prefix from HTTP request log messages.
    
    Parameters
    ----------
    msg : str
        The log message to clean
        
    Returns
    -------
    str
        The cleaned message with redundant prefix removed
    
    Examples
    --------
    >>> msg = '127.0.0.1 - - [20/Mar/2024:10:00:00] "GET /dashboard HTTP/1.1" 200 -'
    >>> clean_http_request(msg)
    'GET /dashboard HTTP/1.1" 200 -'
    
    >>> msg = 'Regular log message without HTTP request'
    >>> clean_http_request(msg)
    'Regular log message without HTTP request'
    """
    has_http_method = any(f" {method} " in msg for method in ("GET", "POST"))
    if has_http_method and msg.endswith(("-", '"')):
        return '"'.join(msg.split('"')[1:])
    return msg

def convert_ansi_to_html(msg: str) -> str:
    """Convert ANSI terminal colors to HTML formatted text.
    
    Parameters
    ----------
    msg : str
        Message potentially containing ANSI color codes
        
    Returns
    -------
    str
        HTML-formatted message with ANSI codes converted to CSS
    
    Examples
    --------
    >>> msg = '\033[31mError:\033[0m Database connection failed'
    >>> convert_ansi_to_html(msg)
    '<span style="color: red">Error:</span> Database connection failed'
    
    >>> msg = 'Plain text without ANSI codes'
    >>> convert_ansi_to_html(msg)
    'Plain text without ANSI codes'
    """
    return Ansi2HTMLConverter().convert(msg)

def ensure_html_structure(html: str) -> str:
    """Ensure HTML has proper structure with head and body tags.
    
    Parameters
    ----------
    html : str
        HTML content to structure
        
    Returns
    -------
    str
        Properly structured HTML document
    
    Examples
    --------
    >>> html = '<span style="color: red">Error</span>'
    >>> ensure_html_structure(html)
    '<html><head></head><body><span style="color: red">Error</span></body></html>'
    
    >>> html = '<head><style>...</style></head><body>Content</body>'
    >>> ensure_html_structure(html)
    '<html><head><style>...</style></head><body>Content</body></html>'
    """
    soup = bs4.BeautifulSoup(html, "html.parser")
    return f"<html>{soup.head or ''}{soup.body or ''}</html>"

def clean_line_dict(line_dict: dict, log_line_number: Optional[int] = None) -> dict:
    """Clean and format a log message dictionary for dashboard display.
    
    Parameters
    ----------
    line_dict : dict
        Dictionary containing log message data with at least a 'message' key
    log_line_number : int, optional
        Line number in the log file
        
    Returns
    -------
    dict
        Cleaned dictionary with HTML-formatted message
    
    Examples
    --------
    >>> line_dict = {
    ...     'message': '\033[31mError:\033[0m GET /dashboard failed',
    ...     'level': 'ERROR'
    ... }
    >>> cleaned = clean_line_dict(line_dict, log_line_number=42)
    >>> cleaned
    {
        'message': '<html><head>...</head><body><span style="color: red">Error:</span> GET /dashboard failed</body></html>',
        'level': 'ERROR',
        'log_line_number': 42
    }
    """
    msg = line_dict["message"]
    
    # Clean and format the message
    msg = clean_http_request(msg)
    msg = convert_ansi_to_html(msg)
    msg = ensure_html_structure(msg)
    
    # Finalize the dictionary
    line_dict["message"] = msg
    if log_line_number is not None:
        line_dict["log_line_number"] = log_line_number
    
    return line_dict
```
created_at: 2025-03-24T13:49:05Z
author_association: COLLABORATOR
diff_hunk: @@ -652,6 +661,229 @@ def dashboard_lifecycle():
     )
 
 
+def clean_http_request_log(msg: str) -> str:
+    """Remove redundant prefix from HTTP request log messages.
+
+    Parameters
+    ----------
+    msg : str
+        The log message to clean
+
+    Returns
+    -------
+    str
+        The cleaned message with redundant prefix removed
+
+    Examples
+    --------
+    >>> msg = '127.0.0.1 - - [20/Mar/2024:10:00:00] "GET /dashboard HTTP/1.1" 200 -'
+    >>> clean_http_request_log(msg)
+    'GET /dashboard HTTP/1.1" 200 -'
+    """
+    is_http_request = "GET " in msg or " POST" in msg
+    has_expected_ending = msg.endswith(("-", '"'))
+
+    if is_http_request and has_expected_ending:
+        return '"'.join(msg.split('"')[1:])
+    return msg
+
+
+def clean_line_info(line_info: dict, log_line_number: int = None) -> dict:
+    """
+    Clean the line info for the log viewer. Convert ANSI to HTML (e.g. ANSI bold to HTML bold) and remove the leading
+    timestamp and log level (because it is already displayed in the table) and add the log's line number if provided.
+
+    :param line_info: The line info to clean
+    :type line_info: dict
+
+    :param log_line_number: The line number in the log file (optional)
+    :type log_line_number: int
+
+    :return: The cleaned line info
+    """
+    msg = line_info["message"]
+    msg = clean_http_request_log(msg)
+    msg = Ansi2HTMLConverter().convert(msg)
+    parsed_msg = bs4.BeautifulSoup(msg, "html.parser")
+    msg = f"<html>{parsed_msg.head}{parsed_msg.body}</html>"
+    line_info["message"] = msg
path: dallinger/experiment_server/dashboard.py
user: login: pmcharrison
body: I am a little bit confused why the `ensure_html_structure` bit is necessary. Why would the parsed message end up with a head or a body? 
created_at: 2025-03-24T13:51:07Z
author_association: COLLABORATOR
diff_hunk: @@ -36,6 +38,19 @@
 fake = Faker()
 
 
+def get_logger_filename():
path: dallinger/utils.py
user: login: pmcharrison
body: I'm happy with the new file name, but it is still a bit confusing that now we have two logfiles (server.log and logs.jsonl) that are referred to with very similar names in the Python code (`LOG_FILE` and `logfile`). This will confuse people in the future. I think you either have to come up with a clear way in which these log files are actually differentiated, and name them accordingly, or you have to merge the two into the same file.

I think the easiest would be just to name it `JSON_LOGFILE` instead of `LOG_FILE`, and the same for related functions, e.g. `get_logger_filename` -> `get_json_logfile_name`.
created_at: 2025-03-24T13:56:59Z
author_association: COLLABORATOR
diff_hunk: @@ -652,6 +661,229 @@ def dashboard_lifecycle():
     )
 
 
+def clean_http_request_log(msg: str) -> str:
+    """Remove redundant prefix from HTTP request log messages.
+
+    Parameters
+    ----------
+    msg : str
+        The log message to clean
+
+    Returns
+    -------
+    str
+        The cleaned message with redundant prefix removed
+
+    Examples
+    --------
+    >>> msg = '127.0.0.1 - - [20/Mar/2024:10:00:00] "GET /dashboard HTTP/1.1" 200 -'
+    >>> clean_http_request_log(msg)
+    'GET /dashboard HTTP/1.1" 200 -'
+    """
+    is_http_request = "GET " in msg or " POST" in msg
+    has_expected_ending = msg.endswith(("-", '"'))
+
+    if is_http_request and has_expected_ending:
+        return '"'.join(msg.split('"')[1:])
+    return msg
+
+
+def clean_line_info(line_info: dict, log_line_number: int = None) -> dict:
+    """
+    Clean the line info for the log viewer. Convert ANSI to HTML (e.g. ANSI bold to HTML bold) and remove the leading
+    timestamp and log level (because it is already displayed in the table) and add the log's line number if provided.
+
+    :param line_info: The line info to clean
+    :type line_info: dict
+
+    :param log_line_number: The line number in the log file (optional)
+    :type log_line_number: int
+
+    :return: The cleaned line info
+    """
+    msg = line_info["message"]
+    msg = clean_http_request_log(msg)
+    msg = Ansi2HTMLConverter().convert(msg)
+    parsed_msg = bs4.BeautifulSoup(msg, "html.parser")
+    msg = f"<html>{parsed_msg.head}{parsed_msg.body}</html>"
+    line_info["message"] = msg
+    if log_line_number is not None:
+        line_info["log_line_number"] = log_line_number
+    return line_info
+
+
+LOG_FILE = get_logger_filename()
path: dallinger/experiment_server/dashboard.py
user: login: pmcharrison
body: I'm not sure what we gain from having both `get_logger_filename` and `LOG_FILE`. Why not replace the original definition of `get_logger_filename` with just `JSON_LOG_FILE = "logs.jsonl"`?
created_at: 2025-03-24T13:59:28Z
author_association: COLLABORATOR
diff_hunk: @@ -652,6 +661,229 @@ def dashboard_lifecycle():
     )
 
 
+def clean_http_request_log(msg: str) -> str:
+    """Remove redundant prefix from HTTP request log messages.
+
+    Parameters
+    ----------
+    msg : str
+        The log message to clean
+
+    Returns
+    -------
+    str
+        The cleaned message with redundant prefix removed
+
+    Examples
+    --------
+    >>> msg = '127.0.0.1 - - [20/Mar/2024:10:00:00] "GET /dashboard HTTP/1.1" 200 -'
+    >>> clean_http_request_log(msg)
+    'GET /dashboard HTTP/1.1" 200 -'
+    """
+    is_http_request = "GET " in msg or " POST" in msg
+    has_expected_ending = msg.endswith(("-", '"'))
+
+    if is_http_request and has_expected_ending:
+        return '"'.join(msg.split('"')[1:])
+    return msg
+
+
+def clean_line_info(line_info: dict, log_line_number: int = None) -> dict:
+    """
+    Clean the line info for the log viewer. Convert ANSI to HTML (e.g. ANSI bold to HTML bold) and remove the leading
+    timestamp and log level (because it is already displayed in the table) and add the log's line number if provided.
+
+    :param line_info: The line info to clean
+    :type line_info: dict
+
+    :param log_line_number: The line number in the log file (optional)
+    :type log_line_number: int
+
+    :return: The cleaned line info
+    """
+    msg = line_info["message"]
+    msg = clean_http_request_log(msg)
+    msg = Ansi2HTMLConverter().convert(msg)
+    parsed_msg = bs4.BeautifulSoup(msg, "html.parser")
+    msg = f"<html>{parsed_msg.head}{parsed_msg.body}</html>"
+    line_info["message"] = msg
+    if log_line_number is not None:
+        line_info["log_line_number"] = log_line_number
+    return line_info
+
+
+LOG_FILE = get_logger_filename()
+
+
+def log_read_lines(line_start: int, line_end: int) -> tuple[list[dict], bool, int]:
+    """
+    Read the log file and return the lines in the specified range.
+
+    :param line_start: The line number to start reading from
+    :type line_start: int
+
+    :param line_end: The line number to stop reading at
+    :type line_end: int
+
+    :return: A tuple containing the lines, a boolean indicating if the end of the file was reached, and the last line number
+
+    :Note: The line numbers are 1-based
+
+    """
+    lines = []
+    line_range = range(line_start, line_end)
+    early_stop = False
+    with open(LOG_FILE) as f:
+        for i, line in enumerate(f):
+            number = i + 1
+            if number in line_range:
+                line_dict = clean_line_info(json.loads(line), number)
path: dallinger/experiment_server/dashboard.py
user: login: pmcharrison
body: Consider renaming `line_dict` as `line_info` and `clean_line_info` as `get_line_info` and moving `json.loads` into `get_line_info` (but I don't really mind)
created_at: 2025-03-24T14:16:10Z
author_association: COLLABORATOR
diff_hunk: @@ -652,6 +661,229 @@ def dashboard_lifecycle():
     )
 
 
+def clean_http_request_log(msg: str) -> str:
+    """Remove redundant prefix from HTTP request log messages.
+
+    Parameters
+    ----------
+    msg : str
+        The log message to clean
+
+    Returns
+    -------
+    str
+        The cleaned message with redundant prefix removed
+
+    Examples
+    --------
+    >>> msg = '127.0.0.1 - - [20/Mar/2024:10:00:00] "GET /dashboard HTTP/1.1" 200 -'
+    >>> clean_http_request_log(msg)
+    'GET /dashboard HTTP/1.1" 200 -'
+    """
+    is_http_request = "GET " in msg or " POST" in msg
+    has_expected_ending = msg.endswith(("-", '"'))
+
+    if is_http_request and has_expected_ending:
+        return '"'.join(msg.split('"')[1:])
+    return msg
+
+
+def clean_line_info(line_info: dict, log_line_number: int = None) -> dict:
+    """
+    Clean the line info for the log viewer. Convert ANSI to HTML (e.g. ANSI bold to HTML bold) and remove the leading
+    timestamp and log level (because it is already displayed in the table) and add the log's line number if provided.
+
+    :param line_info: The line info to clean
+    :type line_info: dict
+
+    :param log_line_number: The line number in the log file (optional)
+    :type log_line_number: int
+
+    :return: The cleaned line info
+    """
+    msg = line_info["message"]
+    msg = clean_http_request_log(msg)
+    msg = Ansi2HTMLConverter().convert(msg)
+    parsed_msg = bs4.BeautifulSoup(msg, "html.parser")
+    msg = f"<html>{parsed_msg.head}{parsed_msg.body}</html>"
+    line_info["message"] = msg
+    if log_line_number is not None:
+        line_info["log_line_number"] = log_line_number
+    return line_info
+
+
+LOG_FILE = get_logger_filename()
+
+
+def log_read_lines(line_start: int, line_end: int) -> tuple[list[dict], bool, int]:
+    """
+    Read the log file and return the lines in the specified range.
+
+    :param line_start: The line number to start reading from
+    :type line_start: int
+
+    :param line_end: The line number to stop reading at
+    :type line_end: int
+
+    :return: A tuple containing the lines, a boolean indicating if the end of the file was reached, and the last line number
+
+    :Note: The line numbers are 1-based
+
+    """
+    lines = []
+    line_range = range(line_start, line_end)
+    early_stop = False
+    with open(LOG_FILE) as f:
+        for i, line in enumerate(f):
+            number = i + 1
+            if number in line_range:
+                line_dict = clean_line_info(json.loads(line), number)
+                lines.append(line_dict)
+            if number > line_end:
+                early_stop = True
+                break
+    return lines, early_stop, i
+
+
+def find_log_line_number(substring) -> Union[int, None]:
+    """
+    Find the line number in the log file that contains a substring.
+
+    :param substring: The substring to search for
+    :type substring: str
+
+    :return: The line number (1-based) or None if the substring was not found
+    """
+    with open(get_logger_filename()) as f:
+        for number, line in enumerate(f):
+            if substring in line:
+                return number + 1
+    return None
+
+
+def log_search_substring(substring: str):
+    """
+    Search the log file for a substring and return the matching lines.
+
+    :param substring: The substring to search for
+    :type substring: str
+
+    :return: A generator that yields the matching lines
+
+    :Note: The line numbers are 1-based
+    :Note: The generator will yield a 'stop' message when the end of the file is reached
+    """
+    with open(LOG_FILE) as f:
+        for number, line in enumerate(f):
+            if substring in line:
+                line_dict = clean_line_info(json.loads(line), number)
+                yield f"data:{json.dumps(line_dict)}\n\n"
+    yield f"data:{json.dumps({'stop': True})}\n\n"
path: dallinger/experiment_server/dashboard.py
user: login: pmcharrison
body: Seems a bit unnecessary to have `json.dumps({'stop': True})` instead of just `'{"stop": true}'`
created_at: 2025-03-24T14:19:51Z
author_association: COLLABORATOR
diff_hunk: @@ -652,6 +661,229 @@ def dashboard_lifecycle():
     )
 
 
+def clean_http_request_log(msg: str) -> str:
+    """Remove redundant prefix from HTTP request log messages.
+
+    Parameters
+    ----------
+    msg : str
+        The log message to clean
+
+    Returns
+    -------
+    str
+        The cleaned message with redundant prefix removed
+
+    Examples
+    --------
+    >>> msg = '127.0.0.1 - - [20/Mar/2024:10:00:00] "GET /dashboard HTTP/1.1" 200 -'
+    >>> clean_http_request_log(msg)
+    'GET /dashboard HTTP/1.1" 200 -'
+    """
+    is_http_request = "GET " in msg or " POST" in msg
+    has_expected_ending = msg.endswith(("-", '"'))
+
+    if is_http_request and has_expected_ending:
+        return '"'.join(msg.split('"')[1:])
+    return msg
+
+
+def clean_line_info(line_info: dict, log_line_number: int = None) -> dict:
+    """
+    Clean the line info for the log viewer. Convert ANSI to HTML (e.g. ANSI bold to HTML bold) and remove the leading
+    timestamp and log level (because it is already displayed in the table) and add the log's line number if provided.
+
+    :param line_info: The line info to clean
+    :type line_info: dict
+
+    :param log_line_number: The line number in the log file (optional)
+    :type log_line_number: int
+
+    :return: The cleaned line info
+    """
+    msg = line_info["message"]
+    msg = clean_http_request_log(msg)
+    msg = Ansi2HTMLConverter().convert(msg)
+    parsed_msg = bs4.BeautifulSoup(msg, "html.parser")
+    msg = f"<html>{parsed_msg.head}{parsed_msg.body}</html>"
+    line_info["message"] = msg
+    if log_line_number is not None:
+        line_info["log_line_number"] = log_line_number
+    return line_info
+
+
+LOG_FILE = get_logger_filename()
+
+
+def log_read_lines(line_start: int, line_end: int) -> tuple[list[dict], bool, int]:
+    """
+    Read the log file and return the lines in the specified range.
+
+    :param line_start: The line number to start reading from
+    :type line_start: int
+
+    :param line_end: The line number to stop reading at
+    :type line_end: int
+
+    :return: A tuple containing the lines, a boolean indicating if the end of the file was reached, and the last line number
+
+    :Note: The line numbers are 1-based
+
+    """
+    lines = []
+    line_range = range(line_start, line_end)
+    early_stop = False
+    with open(LOG_FILE) as f:
+        for i, line in enumerate(f):
+            number = i + 1
+            if number in line_range:
+                line_dict = clean_line_info(json.loads(line), number)
+                lines.append(line_dict)
+            if number > line_end:
+                early_stop = True
+                break
+    return lines, early_stop, i
+
+
+def find_log_line_number(substring) -> Union[int, None]:
+    """
+    Find the line number in the log file that contains a substring.
+
+    :param substring: The substring to search for
+    :type substring: str
+
+    :return: The line number (1-based) or None if the substring was not found
+    """
+    with open(get_logger_filename()) as f:
+        for number, line in enumerate(f):
+            if substring in line:
+                return number + 1
+    return None
+
+
+def log_search_substring(substring: str):
+    """
+    Search the log file for a substring and return the matching lines.
+
+    :param substring: The substring to search for
+    :type substring: str
+
+    :return: A generator that yields the matching lines
+
+    :Note: The line numbers are 1-based
+    :Note: The generator will yield a 'stop' message when the end of the file is reached
+    """
+    with open(LOG_FILE) as f:
+        for number, line in enumerate(f):
+            if substring in line:
+                line_dict = clean_line_info(json.loads(line), number)
+                yield f"data:{json.dumps(line_dict)}\n\n"
+    yield f"data:{json.dumps({'stop': True})}\n\n"
+
+
+@dashboard.route("/logs/live", methods=["GET"])
+@login_required
+def logs_live():
+    """Stream log file updates to the dashboard using Server-Sent Events (SSE).
+
+    Creates a streaming response that pushes new log entries to connected clients
+    in real-time. Each log entry is:
+    - Read from the log file using Pygtail
+    - Parsed from JSON
+    - Cleaned and formatted for display
+    - Sent as an SSE message
+
+    Returns
+    -------
+    Response
+        A Flask response object with mimetype 'text/event-stream' containing
+        the event stream of log updates. Each event contains:
+        - message : str
+            The formatted log message with ANSI codes converted to HTML
+        - original_line : str
+            The raw log line before processing
+        - log_line_number : int, optional
+            The line number in the log file
+
+    Notes
+    -----
+
+    The client should connect to this endpoint using an EventSource object
+    to receive the updates.
+    """
+
+    def generate():
+        for line in Pygtail(LOG_FILE):
+            try:
+                line_dict = clean_line_info(json.loads(line))
path: dallinger/experiment_server/dashboard.py
user: login: pmcharrison
body: as before, `line_dict` -> `line_info`
created_at: 2025-03-24T14:20:42Z
author_association: COLLABORATOR
diff_hunk: @@ -652,6 +661,229 @@ def dashboard_lifecycle():
     )
 
 
+def clean_http_request_log(msg: str) -> str:
+    """Remove redundant prefix from HTTP request log messages.
+
+    Parameters
+    ----------
+    msg : str
+        The log message to clean
+
+    Returns
+    -------
+    str
+        The cleaned message with redundant prefix removed
+
+    Examples
+    --------
+    >>> msg = '127.0.0.1 - - [20/Mar/2024:10:00:00] "GET /dashboard HTTP/1.1" 200 -'
+    >>> clean_http_request_log(msg)
+    'GET /dashboard HTTP/1.1" 200 -'
+    """
+    is_http_request = "GET " in msg or " POST" in msg
+    has_expected_ending = msg.endswith(("-", '"'))
+
+    if is_http_request and has_expected_ending:
+        return '"'.join(msg.split('"')[1:])
+    return msg
+
+
+def clean_line_info(line_info: dict, log_line_number: int = None) -> dict:
+    """
+    Clean the line info for the log viewer. Convert ANSI to HTML (e.g. ANSI bold to HTML bold) and remove the leading
+    timestamp and log level (because it is already displayed in the table) and add the log's line number if provided.
+
+    :param line_info: The line info to clean
+    :type line_info: dict
+
+    :param log_line_number: The line number in the log file (optional)
+    :type log_line_number: int
+
+    :return: The cleaned line info
+    """
+    msg = line_info["message"]
+    msg = clean_http_request_log(msg)
+    msg = Ansi2HTMLConverter().convert(msg)
+    parsed_msg = bs4.BeautifulSoup(msg, "html.parser")
+    msg = f"<html>{parsed_msg.head}{parsed_msg.body}</html>"
+    line_info["message"] = msg
+    if log_line_number is not None:
+        line_info["log_line_number"] = log_line_number
+    return line_info
+
+
+LOG_FILE = get_logger_filename()
+
+
+def log_read_lines(line_start: int, line_end: int) -> tuple[list[dict], bool, int]:
+    """
+    Read the log file and return the lines in the specified range.
+
+    :param line_start: The line number to start reading from
+    :type line_start: int
+
+    :param line_end: The line number to stop reading at
+    :type line_end: int
+
+    :return: A tuple containing the lines, a boolean indicating if the end of the file was reached, and the last line number
+
+    :Note: The line numbers are 1-based
+
+    """
+    lines = []
+    line_range = range(line_start, line_end)
+    early_stop = False
+    with open(LOG_FILE) as f:
+        for i, line in enumerate(f):
+            number = i + 1
+            if number in line_range:
+                line_dict = clean_line_info(json.loads(line), number)
+                lines.append(line_dict)
+            if number > line_end:
+                early_stop = True
+                break
+    return lines, early_stop, i
+
+
+def find_log_line_number(substring) -> Union[int, None]:
+    """
+    Find the line number in the log file that contains a substring.
+
+    :param substring: The substring to search for
+    :type substring: str
+
+    :return: The line number (1-based) or None if the substring was not found
+    """
+    with open(get_logger_filename()) as f:
+        for number, line in enumerate(f):
+            if substring in line:
+                return number + 1
+    return None
+
+
+def log_search_substring(substring: str):
+    """
+    Search the log file for a substring and return the matching lines.
+
+    :param substring: The substring to search for
+    :type substring: str
+
+    :return: A generator that yields the matching lines
+
+    :Note: The line numbers are 1-based
+    :Note: The generator will yield a 'stop' message when the end of the file is reached
+    """
+    with open(LOG_FILE) as f:
+        for number, line in enumerate(f):
+            if substring in line:
+                line_dict = clean_line_info(json.loads(line), number)
+                yield f"data:{json.dumps(line_dict)}\n\n"
+    yield f"data:{json.dumps({'stop': True})}\n\n"
path: dallinger/experiment_server/dashboard.py
user: login: pmcharrison
body: I like what you do in `generate` where the formatting with `data:` and equivalently `\n\n` is saved for the top-level route function rather than the helper function. It seems a bit out-of-place for the helper function to be doing this (if you were reading the docstring, you would not expect the helper function to do it)
created_at: 2025-03-24T14:22:24Z
author_association: COLLABORATOR
diff_hunk: @@ -652,6 +661,229 @@ def dashboard_lifecycle():
     )
 
 
+def clean_http_request_log(msg: str) -> str:
+    """Remove redundant prefix from HTTP request log messages.
+
+    Parameters
+    ----------
+    msg : str
+        The log message to clean
+
+    Returns
+    -------
+    str
+        The cleaned message with redundant prefix removed
+
+    Examples
+    --------
+    >>> msg = '127.0.0.1 - - [20/Mar/2024:10:00:00] "GET /dashboard HTTP/1.1" 200 -'
+    >>> clean_http_request_log(msg)
+    'GET /dashboard HTTP/1.1" 200 -'
+    """
+    is_http_request = "GET " in msg or " POST" in msg
+    has_expected_ending = msg.endswith(("-", '"'))
+
+    if is_http_request and has_expected_ending:
+        return '"'.join(msg.split('"')[1:])
+    return msg
+
+
+def clean_line_info(line_info: dict, log_line_number: int = None) -> dict:
+    """
+    Clean the line info for the log viewer. Convert ANSI to HTML (e.g. ANSI bold to HTML bold) and remove the leading
+    timestamp and log level (because it is already displayed in the table) and add the log's line number if provided.
+
+    :param line_info: The line info to clean
+    :type line_info: dict
+
+    :param log_line_number: The line number in the log file (optional)
+    :type log_line_number: int
+
+    :return: The cleaned line info
+    """
+    msg = line_info["message"]
+    msg = clean_http_request_log(msg)
+    msg = Ansi2HTMLConverter().convert(msg)
+    parsed_msg = bs4.BeautifulSoup(msg, "html.parser")
+    msg = f"<html>{parsed_msg.head}{parsed_msg.body}</html>"
+    line_info["message"] = msg
+    if log_line_number is not None:
+        line_info["log_line_number"] = log_line_number
+    return line_info
+
+
+LOG_FILE = get_logger_filename()
+
+
+def log_read_lines(line_start: int, line_end: int) -> tuple[list[dict], bool, int]:
+    """
+    Read the log file and return the lines in the specified range.
+
+    :param line_start: The line number to start reading from
+    :type line_start: int
+
+    :param line_end: The line number to stop reading at
+    :type line_end: int
+
+    :return: A tuple containing the lines, a boolean indicating if the end of the file was reached, and the last line number
+
+    :Note: The line numbers are 1-based
+
+    """
+    lines = []
+    line_range = range(line_start, line_end)
+    early_stop = False
+    with open(LOG_FILE) as f:
+        for i, line in enumerate(f):
+            number = i + 1
+            if number in line_range:
+                line_dict = clean_line_info(json.loads(line), number)
+                lines.append(line_dict)
+            if number > line_end:
+                early_stop = True
+                break
+    return lines, early_stop, i
+
+
+def find_log_line_number(substring) -> Union[int, None]:
+    """
+    Find the line number in the log file that contains a substring.
+
+    :param substring: The substring to search for
+    :type substring: str
+
+    :return: The line number (1-based) or None if the substring was not found
+    """
+    with open(get_logger_filename()) as f:
+        for number, line in enumerate(f):
+            if substring in line:
+                return number + 1
+    return None
+
+
+def log_search_substring(substring: str):
+    """
+    Search the log file for a substring and return the matching lines.
+
+    :param substring: The substring to search for
+    :type substring: str
+
+    :return: A generator that yields the matching lines
+
+    :Note: The line numbers are 1-based
+    :Note: The generator will yield a 'stop' message when the end of the file is reached
+    """
+    with open(LOG_FILE) as f:
+        for number, line in enumerate(f):
+            if substring in line:
+                line_dict = clean_line_info(json.loads(line), number)
+                yield f"data:{json.dumps(line_dict)}\n\n"
+    yield f"data:{json.dumps({'stop': True})}\n\n"
+
+
+@dashboard.route("/logs/live", methods=["GET"])
+@login_required
+def logs_live():
+    """Stream log file updates to the dashboard using Server-Sent Events (SSE).
+
+    Creates a streaming response that pushes new log entries to connected clients
+    in real-time. Each log entry is:
+    - Read from the log file using Pygtail
+    - Parsed from JSON
+    - Cleaned and formatted for display
+    - Sent as an SSE message
+
+    Returns
+    -------
+    Response
+        A Flask response object with mimetype 'text/event-stream' containing
+        the event stream of log updates. Each event contains:
+        - message : str
+            The formatted log message with ANSI codes converted to HTML
+        - original_line : str
+            The raw log line before processing
+        - log_line_number : int, optional
+            The line number in the log file
+
+    Notes
+    -----
+
+    The client should connect to this endpoint using an EventSource object
+    to receive the updates.
+    """
+
+    def generate():
+        for line in Pygtail(LOG_FILE):
+            try:
+                line_dict = clean_line_info(json.loads(line))
+                line_dict["original_line"] = line
+                yield f"data:{json.dumps(line_dict)}\n\n"
+            except json.decoder.JSONDecodeError:
+                yield f"data:{json.dumps({'message': line})}\n\n"
+
+    return Response(generate(), mimetype="text/event-stream")
+
+
+@dashboard.route("/logs/range", methods=["GET"])
+@login_required
+def logs_range():
+    """
+    Return a range of log lines from the log file.
+    """
+    params = request.args
+    start, end = params.get("start", None), params.get("end", None)
+    if start is None or end is None:
+        return json_error_response("Both 'start' and 'end' parameters are required.")
+    start, end = int(start), int(end)
+    if start < 1:
+        return json_error_response("'start' must be > 0.")
+
+    if start > end:
+        return json_error_response("'start' must be <= 'end'.")
+
+    lines, early_stop, last_line = log_read_lines(start, end + 1)
+    if not early_stop and start > last_line:
+        return json_error_response(
+            f"'start' must be less than the current line number {last_line}."
+        )
+
+    return lines
+
+
+def json_error_response(message, status_code=400):
+    return jsonify({"msg": message}), status_code
+
+
+@dashboard.route("/logs/find_lines", methods=["GET"])
+@login_required
+def logs_find_lines():
+    """
+    Find lines in the log file that contain a substring (GET parameter 'query').
+    """
+    params = request.args
+    query = params.get("query", None)
+    if query is None:
+        return json_error_response("No query provided.")
+    return Response(log_search_substring(query), mimetype="text/event-stream")
+
+
+@dashboard.route("/logs/find_line_number", methods=["POST"])
+@login_required
+def logs_find_line_number():
+    """
+    Find the line number in the log file that contains a substring (POST parameter 'query').
+    """
+    params = request.json
+    query = params.get("query", None)
+    if query is None:
+        return json_error_response("No query provided.")
+
+    line_number = find_log_line_number(query)
+    if line_number is not None:
+        return jsonify({"line_number": line_number})
path: dallinger/experiment_server/dashboard.py
user: login: pmcharrison
body: Are you sure you need `jsonify` here? I thought this is done automatically by Flask.
created_at: 2025-03-24T14:23:40Z
author_association: COLLABORATOR
diff_hunk: @@ -652,6 +661,229 @@ def dashboard_lifecycle():
     )
 
 
+def clean_http_request_log(msg: str) -> str:
+    """Remove redundant prefix from HTTP request log messages.
+
+    Parameters
+    ----------
+    msg : str
+        The log message to clean
+
+    Returns
+    -------
+    str
+        The cleaned message with redundant prefix removed
+
+    Examples
+    --------
+    >>> msg = '127.0.0.1 - - [20/Mar/2024:10:00:00] "GET /dashboard HTTP/1.1" 200 -'
+    >>> clean_http_request_log(msg)
+    'GET /dashboard HTTP/1.1" 200 -'
+    """
+    is_http_request = "GET " in msg or " POST" in msg
+    has_expected_ending = msg.endswith(("-", '"'))
+
+    if is_http_request and has_expected_ending:
+        return '"'.join(msg.split('"')[1:])
+    return msg
+
+
+def clean_line_info(line_info: dict, log_line_number: int = None) -> dict:
+    """
+    Clean the line info for the log viewer. Convert ANSI to HTML (e.g. ANSI bold to HTML bold) and remove the leading
+    timestamp and log level (because it is already displayed in the table) and add the log's line number if provided.
+
+    :param line_info: The line info to clean
+    :type line_info: dict
+
+    :param log_line_number: The line number in the log file (optional)
+    :type log_line_number: int
+
+    :return: The cleaned line info
+    """
+    msg = line_info["message"]
+    msg = clean_http_request_log(msg)
+    msg = Ansi2HTMLConverter().convert(msg)
+    parsed_msg = bs4.BeautifulSoup(msg, "html.parser")
+    msg = f"<html>{parsed_msg.head}{parsed_msg.body}</html>"
+    line_info["message"] = msg
path: dallinger/experiment_server/dashboard.py
user: login: polvanrijn
body: Thanks! I should really learn cursor better 😅
Because `Ansi2HTMLConverter` wraps it awkwardly and we have to unpack it…
created_at: 2025-03-24T15:25:09Z
author_association: COLLABORATOR
diff_hunk: @@ -652,6 +661,229 @@ def dashboard_lifecycle():
     )
 
 
+def clean_http_request_log(msg: str) -> str:
+    """Remove redundant prefix from HTTP request log messages.
+
+    Parameters
+    ----------
+    msg : str
+        The log message to clean
+
+    Returns
+    -------
+    str
+        The cleaned message with redundant prefix removed
+
+    Examples
+    --------
+    >>> msg = '127.0.0.1 - - [20/Mar/2024:10:00:00] "GET /dashboard HTTP/1.1" 200 -'
+    >>> clean_http_request_log(msg)
+    'GET /dashboard HTTP/1.1" 200 -'
+    """
+    is_http_request = "GET " in msg or " POST" in msg
+    has_expected_ending = msg.endswith(("-", '"'))
+
+    if is_http_request and has_expected_ending:
+        return '"'.join(msg.split('"')[1:])
+    return msg
+
+
+def clean_line_info(line_info: dict, log_line_number: int = None) -> dict:
+    """
+    Clean the line info for the log viewer. Convert ANSI to HTML (e.g. ANSI bold to HTML bold) and remove the leading
+    timestamp and log level (because it is already displayed in the table) and add the log's line number if provided.
+
+    :param line_info: The line info to clean
+    :type line_info: dict
+
+    :param log_line_number: The line number in the log file (optional)
+    :type log_line_number: int
+
+    :return: The cleaned line info
+    """
+    msg = line_info["message"]
+    msg = clean_http_request_log(msg)
+    msg = Ansi2HTMLConverter().convert(msg)
+    parsed_msg = bs4.BeautifulSoup(msg, "html.parser")
+    msg = f"<html>{parsed_msg.head}{parsed_msg.body}</html>"
+    line_info["message"] = msg
path: dallinger/experiment_server/dashboard.py
user: login: polvanrijn
body: I implemented your suggestion.
created_at: 2025-03-24T15:25:22Z
author_association: COLLABORATOR
diff_hunk: @@ -36,6 +38,19 @@
 fake = Faker()
 
 
+def get_logger_filename():
path: dallinger/utils.py
user: login: polvanrijn
body: Done
created_at: 2025-03-24T15:35:00Z
author_association: COLLABORATOR
diff_hunk: @@ -652,6 +661,229 @@ def dashboard_lifecycle():
     )
 
 
+def clean_http_request_log(msg: str) -> str:
+    """Remove redundant prefix from HTTP request log messages.
+
+    Parameters
+    ----------
+    msg : str
+        The log message to clean
+
+    Returns
+    -------
+    str
+        The cleaned message with redundant prefix removed
+
+    Examples
+    --------
+    >>> msg = '127.0.0.1 - - [20/Mar/2024:10:00:00] "GET /dashboard HTTP/1.1" 200 -'
+    >>> clean_http_request_log(msg)
+    'GET /dashboard HTTP/1.1" 200 -'
+    """
+    is_http_request = "GET " in msg or " POST" in msg
+    has_expected_ending = msg.endswith(("-", '"'))
+
+    if is_http_request and has_expected_ending:
+        return '"'.join(msg.split('"')[1:])
+    return msg
+
+
+def clean_line_info(line_info: dict, log_line_number: int = None) -> dict:
+    """
+    Clean the line info for the log viewer. Convert ANSI to HTML (e.g. ANSI bold to HTML bold) and remove the leading
+    timestamp and log level (because it is already displayed in the table) and add the log's line number if provided.
+
+    :param line_info: The line info to clean
+    :type line_info: dict
+
+    :param log_line_number: The line number in the log file (optional)
+    :type log_line_number: int
+
+    :return: The cleaned line info
+    """
+    msg = line_info["message"]
+    msg = clean_http_request_log(msg)
+    msg = Ansi2HTMLConverter().convert(msg)
+    parsed_msg = bs4.BeautifulSoup(msg, "html.parser")
+    msg = f"<html>{parsed_msg.head}{parsed_msg.body}</html>"
+    line_info["message"] = msg
+    if log_line_number is not None:
+        line_info["log_line_number"] = log_line_number
+    return line_info
+
+
+LOG_FILE = get_logger_filename()
path: dallinger/experiment_server/dashboard.py
user: login: polvanrijn
body: Done
created_at: 2025-03-24T15:35:04Z
author_association: COLLABORATOR
diff_hunk: @@ -652,6 +661,229 @@ def dashboard_lifecycle():
     )
 
 
+def clean_http_request_log(msg: str) -> str:
+    """Remove redundant prefix from HTTP request log messages.
+
+    Parameters
+    ----------
+    msg : str
+        The log message to clean
+
+    Returns
+    -------
+    str
+        The cleaned message with redundant prefix removed
+
+    Examples
+    --------
+    >>> msg = '127.0.0.1 - - [20/Mar/2024:10:00:00] "GET /dashboard HTTP/1.1" 200 -'
+    >>> clean_http_request_log(msg)
+    'GET /dashboard HTTP/1.1" 200 -'
+    """
+    is_http_request = "GET " in msg or " POST" in msg
+    has_expected_ending = msg.endswith(("-", '"'))
+
+    if is_http_request and has_expected_ending:
+        return '"'.join(msg.split('"')[1:])
+    return msg
+
+
+def clean_line_info(line_info: dict, log_line_number: int = None) -> dict:
+    """
+    Clean the line info for the log viewer. Convert ANSI to HTML (e.g. ANSI bold to HTML bold) and remove the leading
+    timestamp and log level (because it is already displayed in the table) and add the log's line number if provided.
+
+    :param line_info: The line info to clean
+    :type line_info: dict
+
+    :param log_line_number: The line number in the log file (optional)
+    :type log_line_number: int
+
+    :return: The cleaned line info
+    """
+    msg = line_info["message"]
+    msg = clean_http_request_log(msg)
+    msg = Ansi2HTMLConverter().convert(msg)
+    parsed_msg = bs4.BeautifulSoup(msg, "html.parser")
+    msg = f"<html>{parsed_msg.head}{parsed_msg.body}</html>"
+    line_info["message"] = msg
+    if log_line_number is not None:
+        line_info["log_line_number"] = log_line_number
+    return line_info
+
+
+LOG_FILE = get_logger_filename()
+
+
+def log_read_lines(line_start: int, line_end: int) -> tuple[list[dict], bool, int]:
+    """
+    Read the log file and return the lines in the specified range.
+
+    :param line_start: The line number to start reading from
+    :type line_start: int
+
+    :param line_end: The line number to stop reading at
+    :type line_end: int
+
+    :return: A tuple containing the lines, a boolean indicating if the end of the file was reached, and the last line number
+
+    :Note: The line numbers are 1-based
+
+    """
+    lines = []
+    line_range = range(line_start, line_end)
+    early_stop = False
+    with open(LOG_FILE) as f:
+        for i, line in enumerate(f):
+            number = i + 1
+            if number in line_range:
+                line_dict = clean_line_info(json.loads(line), number)
path: dallinger/experiment_server/dashboard.py
user: login: polvanrijn
body: Done
created_at: 2025-03-24T15:35:08Z
author_association: COLLABORATOR
diff_hunk: @@ -652,6 +661,229 @@ def dashboard_lifecycle():
     )
 
 
+def clean_http_request_log(msg: str) -> str:
+    """Remove redundant prefix from HTTP request log messages.
+
+    Parameters
+    ----------
+    msg : str
+        The log message to clean
+
+    Returns
+    -------
+    str
+        The cleaned message with redundant prefix removed
+
+    Examples
+    --------
+    >>> msg = '127.0.0.1 - - [20/Mar/2024:10:00:00] "GET /dashboard HTTP/1.1" 200 -'
+    >>> clean_http_request_log(msg)
+    'GET /dashboard HTTP/1.1" 200 -'
+    """
+    is_http_request = "GET " in msg or " POST" in msg
+    has_expected_ending = msg.endswith(("-", '"'))
+
+    if is_http_request and has_expected_ending:
+        return '"'.join(msg.split('"')[1:])
+    return msg
+
+
+def clean_line_info(line_info: dict, log_line_number: int = None) -> dict:
+    """
+    Clean the line info for the log viewer. Convert ANSI to HTML (e.g. ANSI bold to HTML bold) and remove the leading
+    timestamp and log level (because it is already displayed in the table) and add the log's line number if provided.
+
+    :param line_info: The line info to clean
+    :type line_info: dict
+
+    :param log_line_number: The line number in the log file (optional)
+    :type log_line_number: int
+
+    :return: The cleaned line info
+    """
+    msg = line_info["message"]
+    msg = clean_http_request_log(msg)
+    msg = Ansi2HTMLConverter().convert(msg)
+    parsed_msg = bs4.BeautifulSoup(msg, "html.parser")
+    msg = f"<html>{parsed_msg.head}{parsed_msg.body}</html>"
+    line_info["message"] = msg
+    if log_line_number is not None:
+        line_info["log_line_number"] = log_line_number
+    return line_info
+
+
+LOG_FILE = get_logger_filename()
+
+
+def log_read_lines(line_start: int, line_end: int) -> tuple[list[dict], bool, int]:
+    """
+    Read the log file and return the lines in the specified range.
+
+    :param line_start: The line number to start reading from
+    :type line_start: int
+
+    :param line_end: The line number to stop reading at
+    :type line_end: int
+
+    :return: A tuple containing the lines, a boolean indicating if the end of the file was reached, and the last line number
+
+    :Note: The line numbers are 1-based
+
+    """
+    lines = []
+    line_range = range(line_start, line_end)
+    early_stop = False
+    with open(LOG_FILE) as f:
+        for i, line in enumerate(f):
+            number = i + 1
+            if number in line_range:
+                line_dict = clean_line_info(json.loads(line), number)
+                lines.append(line_dict)
+            if number > line_end:
+                early_stop = True
+                break
+    return lines, early_stop, i
+
+
+def find_log_line_number(substring) -> Union[int, None]:
+    """
+    Find the line number in the log file that contains a substring.
+
+    :param substring: The substring to search for
+    :type substring: str
+
+    :return: The line number (1-based) or None if the substring was not found
+    """
+    with open(get_logger_filename()) as f:
+        for number, line in enumerate(f):
+            if substring in line:
+                return number + 1
+    return None
+
+
+def log_search_substring(substring: str):
+    """
+    Search the log file for a substring and return the matching lines.
+
+    :param substring: The substring to search for
+    :type substring: str
+
+    :return: A generator that yields the matching lines
+
+    :Note: The line numbers are 1-based
+    :Note: The generator will yield a 'stop' message when the end of the file is reached
+    """
+    with open(LOG_FILE) as f:
+        for number, line in enumerate(f):
+            if substring in line:
+                line_dict = clean_line_info(json.loads(line), number)
+                yield f"data:{json.dumps(line_dict)}\n\n"
+    yield f"data:{json.dumps({'stop': True})}\n\n"
+
+
+@dashboard.route("/logs/live", methods=["GET"])
+@login_required
+def logs_live():
+    """Stream log file updates to the dashboard using Server-Sent Events (SSE).
+
+    Creates a streaming response that pushes new log entries to connected clients
+    in real-time. Each log entry is:
+    - Read from the log file using Pygtail
+    - Parsed from JSON
+    - Cleaned and formatted for display
+    - Sent as an SSE message
+
+    Returns
+    -------
+    Response
+        A Flask response object with mimetype 'text/event-stream' containing
+        the event stream of log updates. Each event contains:
+        - message : str
+            The formatted log message with ANSI codes converted to HTML
+        - original_line : str
+            The raw log line before processing
+        - log_line_number : int, optional
+            The line number in the log file
+
+    Notes
+    -----
+
+    The client should connect to this endpoint using an EventSource object
+    to receive the updates.
+    """
+
+    def generate():
+        for line in Pygtail(LOG_FILE):
+            try:
+                line_dict = clean_line_info(json.loads(line))
path: dallinger/experiment_server/dashboard.py
user: login: polvanrijn
body: Done
created_at: 2025-03-24T15:35:43Z
author_association: COLLABORATOR
diff_hunk: @@ -652,6 +661,229 @@ def dashboard_lifecycle():
     )
 
 
+def clean_http_request_log(msg: str) -> str:
+    """Remove redundant prefix from HTTP request log messages.
+
+    Parameters
+    ----------
+    msg : str
+        The log message to clean
+
+    Returns
+    -------
+    str
+        The cleaned message with redundant prefix removed
+
+    Examples
+    --------
+    >>> msg = '127.0.0.1 - - [20/Mar/2024:10:00:00] "GET /dashboard HTTP/1.1" 200 -'
+    >>> clean_http_request_log(msg)
+    'GET /dashboard HTTP/1.1" 200 -'
+    """
+    is_http_request = "GET " in msg or " POST" in msg
+    has_expected_ending = msg.endswith(("-", '"'))
+
+    if is_http_request and has_expected_ending:
+        return '"'.join(msg.split('"')[1:])
+    return msg
+
+
+def clean_line_info(line_info: dict, log_line_number: int = None) -> dict:
+    """
+    Clean the line info for the log viewer. Convert ANSI to HTML (e.g. ANSI bold to HTML bold) and remove the leading
+    timestamp and log level (because it is already displayed in the table) and add the log's line number if provided.
+
+    :param line_info: The line info to clean
+    :type line_info: dict
+
+    :param log_line_number: The line number in the log file (optional)
+    :type log_line_number: int
+
+    :return: The cleaned line info
+    """
+    msg = line_info["message"]
+    msg = clean_http_request_log(msg)
+    msg = Ansi2HTMLConverter().convert(msg)
+    parsed_msg = bs4.BeautifulSoup(msg, "html.parser")
+    msg = f"<html>{parsed_msg.head}{parsed_msg.body}</html>"
+    line_info["message"] = msg
+    if log_line_number is not None:
+        line_info["log_line_number"] = log_line_number
+    return line_info
+
+
+LOG_FILE = get_logger_filename()
+
+
+def log_read_lines(line_start: int, line_end: int) -> tuple[list[dict], bool, int]:
+    """
+    Read the log file and return the lines in the specified range.
+
+    :param line_start: The line number to start reading from
+    :type line_start: int
+
+    :param line_end: The line number to stop reading at
+    :type line_end: int
+
+    :return: A tuple containing the lines, a boolean indicating if the end of the file was reached, and the last line number
+
+    :Note: The line numbers are 1-based
+
+    """
+    lines = []
+    line_range = range(line_start, line_end)
+    early_stop = False
+    with open(LOG_FILE) as f:
+        for i, line in enumerate(f):
+            number = i + 1
+            if number in line_range:
+                line_dict = clean_line_info(json.loads(line), number)
+                lines.append(line_dict)
+            if number > line_end:
+                early_stop = True
+                break
+    return lines, early_stop, i
+
+
+def find_log_line_number(substring) -> Union[int, None]:
+    """
+    Find the line number in the log file that contains a substring.
+
+    :param substring: The substring to search for
+    :type substring: str
+
+    :return: The line number (1-based) or None if the substring was not found
+    """
+    with open(get_logger_filename()) as f:
+        for number, line in enumerate(f):
+            if substring in line:
+                return number + 1
+    return None
+
+
+def log_search_substring(substring: str):
+    """
+    Search the log file for a substring and return the matching lines.
+
+    :param substring: The substring to search for
+    :type substring: str
+
+    :return: A generator that yields the matching lines
+
+    :Note: The line numbers are 1-based
+    :Note: The generator will yield a 'stop' message when the end of the file is reached
+    """
+    with open(LOG_FILE) as f:
+        for number, line in enumerate(f):
+            if substring in line:
+                line_dict = clean_line_info(json.loads(line), number)
+                yield f"data:{json.dumps(line_dict)}\n\n"
+    yield f"data:{json.dumps({'stop': True})}\n\n"
path: dallinger/experiment_server/dashboard.py
user: login: polvanrijn
body: Yeah, I think it's good for readability. If you put it in a string you can easily inject rubbish JSON and your IDE will not complain. So I will keep it like this.
created_at: 2025-03-24T15:36:57Z
author_association: COLLABORATOR
diff_hunk: @@ -652,6 +661,229 @@ def dashboard_lifecycle():
     )
 
 
+def clean_http_request_log(msg: str) -> str:
+    """Remove redundant prefix from HTTP request log messages.
+
+    Parameters
+    ----------
+    msg : str
+        The log message to clean
+
+    Returns
+    -------
+    str
+        The cleaned message with redundant prefix removed
+
+    Examples
+    --------
+    >>> msg = '127.0.0.1 - - [20/Mar/2024:10:00:00] "GET /dashboard HTTP/1.1" 200 -'
+    >>> clean_http_request_log(msg)
+    'GET /dashboard HTTP/1.1" 200 -'
+    """
+    is_http_request = "GET " in msg or " POST" in msg
+    has_expected_ending = msg.endswith(("-", '"'))
+
+    if is_http_request and has_expected_ending:
+        return '"'.join(msg.split('"')[1:])
+    return msg
+
+
+def clean_line_info(line_info: dict, log_line_number: int = None) -> dict:
+    """
+    Clean the line info for the log viewer. Convert ANSI to HTML (e.g. ANSI bold to HTML bold) and remove the leading
+    timestamp and log level (because it is already displayed in the table) and add the log's line number if provided.
+
+    :param line_info: The line info to clean
+    :type line_info: dict
+
+    :param log_line_number: The line number in the log file (optional)
+    :type log_line_number: int
+
+    :return: The cleaned line info
+    """
+    msg = line_info["message"]
+    msg = clean_http_request_log(msg)
+    msg = Ansi2HTMLConverter().convert(msg)
+    parsed_msg = bs4.BeautifulSoup(msg, "html.parser")
+    msg = f"<html>{parsed_msg.head}{parsed_msg.body}</html>"
+    line_info["message"] = msg
+    if log_line_number is not None:
+        line_info["log_line_number"] = log_line_number
+    return line_info
+
+
+LOG_FILE = get_logger_filename()
+
+
+def log_read_lines(line_start: int, line_end: int) -> tuple[list[dict], bool, int]:
+    """
+    Read the log file and return the lines in the specified range.
+
+    :param line_start: The line number to start reading from
+    :type line_start: int
+
+    :param line_end: The line number to stop reading at
+    :type line_end: int
+
+    :return: A tuple containing the lines, a boolean indicating if the end of the file was reached, and the last line number
+
+    :Note: The line numbers are 1-based
+
+    """
+    lines = []
+    line_range = range(line_start, line_end)
+    early_stop = False
+    with open(LOG_FILE) as f:
+        for i, line in enumerate(f):
+            number = i + 1
+            if number in line_range:
+                line_dict = clean_line_info(json.loads(line), number)
+                lines.append(line_dict)
+            if number > line_end:
+                early_stop = True
+                break
+    return lines, early_stop, i
+
+
+def find_log_line_number(substring) -> Union[int, None]:
+    """
+    Find the line number in the log file that contains a substring.
+
+    :param substring: The substring to search for
+    :type substring: str
+
+    :return: The line number (1-based) or None if the substring was not found
+    """
+    with open(get_logger_filename()) as f:
+        for number, line in enumerate(f):
+            if substring in line:
+                return number + 1
+    return None
+
+
+def log_search_substring(substring: str):
+    """
+    Search the log file for a substring and return the matching lines.
+
+    :param substring: The substring to search for
+    :type substring: str
+
+    :return: A generator that yields the matching lines
+
+    :Note: The line numbers are 1-based
+    :Note: The generator will yield a 'stop' message when the end of the file is reached
+    """
+    with open(LOG_FILE) as f:
+        for number, line in enumerate(f):
+            if substring in line:
+                line_dict = clean_line_info(json.loads(line), number)
+                yield f"data:{json.dumps(line_dict)}\n\n"
+    yield f"data:{json.dumps({'stop': True})}\n\n"
path: dallinger/experiment_server/dashboard.py
user: login: polvanrijn
body: Ok, I updated the docstring.
created_at: 2025-03-24T15:39:31Z
author_association: COLLABORATOR
diff_hunk: @@ -652,6 +661,229 @@ def dashboard_lifecycle():
     )
 
 
+def clean_http_request_log(msg: str) -> str:
+    """Remove redundant prefix from HTTP request log messages.
+
+    Parameters
+    ----------
+    msg : str
+        The log message to clean
+
+    Returns
+    -------
+    str
+        The cleaned message with redundant prefix removed
+
+    Examples
+    --------
+    >>> msg = '127.0.0.1 - - [20/Mar/2024:10:00:00] "GET /dashboard HTTP/1.1" 200 -'
+    >>> clean_http_request_log(msg)
+    'GET /dashboard HTTP/1.1" 200 -'
+    """
+    is_http_request = "GET " in msg or " POST" in msg
+    has_expected_ending = msg.endswith(("-", '"'))
+
+    if is_http_request and has_expected_ending:
+        return '"'.join(msg.split('"')[1:])
+    return msg
+
+
+def clean_line_info(line_info: dict, log_line_number: int = None) -> dict:
+    """
+    Clean the line info for the log viewer. Convert ANSI to HTML (e.g. ANSI bold to HTML bold) and remove the leading
+    timestamp and log level (because it is already displayed in the table) and add the log's line number if provided.
+
+    :param line_info: The line info to clean
+    :type line_info: dict
+
+    :param log_line_number: The line number in the log file (optional)
+    :type log_line_number: int
+
+    :return: The cleaned line info
+    """
+    msg = line_info["message"]
+    msg = clean_http_request_log(msg)
+    msg = Ansi2HTMLConverter().convert(msg)
+    parsed_msg = bs4.BeautifulSoup(msg, "html.parser")
+    msg = f"<html>{parsed_msg.head}{parsed_msg.body}</html>"
+    line_info["message"] = msg
+    if log_line_number is not None:
+        line_info["log_line_number"] = log_line_number
+    return line_info
+
+
+LOG_FILE = get_logger_filename()
+
+
+def log_read_lines(line_start: int, line_end: int) -> tuple[list[dict], bool, int]:
+    """
+    Read the log file and return the lines in the specified range.
+
+    :param line_start: The line number to start reading from
+    :type line_start: int
+
+    :param line_end: The line number to stop reading at
+    :type line_end: int
+
+    :return: A tuple containing the lines, a boolean indicating if the end of the file was reached, and the last line number
+
+    :Note: The line numbers are 1-based
+
+    """
+    lines = []
+    line_range = range(line_start, line_end)
+    early_stop = False
+    with open(LOG_FILE) as f:
+        for i, line in enumerate(f):
+            number = i + 1
+            if number in line_range:
+                line_dict = clean_line_info(json.loads(line), number)
+                lines.append(line_dict)
+            if number > line_end:
+                early_stop = True
+                break
+    return lines, early_stop, i
+
+
+def find_log_line_number(substring) -> Union[int, None]:
+    """
+    Find the line number in the log file that contains a substring.
+
+    :param substring: The substring to search for
+    :type substring: str
+
+    :return: The line number (1-based) or None if the substring was not found
+    """
+    with open(get_logger_filename()) as f:
+        for number, line in enumerate(f):
+            if substring in line:
+                return number + 1
+    return None
+
+
+def log_search_substring(substring: str):
+    """
+    Search the log file for a substring and return the matching lines.
+
+    :param substring: The substring to search for
+    :type substring: str
+
+    :return: A generator that yields the matching lines
+
+    :Note: The line numbers are 1-based
+    :Note: The generator will yield a 'stop' message when the end of the file is reached
+    """
+    with open(LOG_FILE) as f:
+        for number, line in enumerate(f):
+            if substring in line:
+                line_dict = clean_line_info(json.loads(line), number)
+                yield f"data:{json.dumps(line_dict)}\n\n"
+    yield f"data:{json.dumps({'stop': True})}\n\n"
+
+
+@dashboard.route("/logs/live", methods=["GET"])
+@login_required
+def logs_live():
+    """Stream log file updates to the dashboard using Server-Sent Events (SSE).
+
+    Creates a streaming response that pushes new log entries to connected clients
+    in real-time. Each log entry is:
+    - Read from the log file using Pygtail
+    - Parsed from JSON
+    - Cleaned and formatted for display
+    - Sent as an SSE message
+
+    Returns
+    -------
+    Response
+        A Flask response object with mimetype 'text/event-stream' containing
+        the event stream of log updates. Each event contains:
+        - message : str
+            The formatted log message with ANSI codes converted to HTML
+        - original_line : str
+            The raw log line before processing
+        - log_line_number : int, optional
+            The line number in the log file
+
+    Notes
+    -----
+
+    The client should connect to this endpoint using an EventSource object
+    to receive the updates.
+    """
+
+    def generate():
+        for line in Pygtail(LOG_FILE):
+            try:
+                line_dict = clean_line_info(json.loads(line))
+                line_dict["original_line"] = line
+                yield f"data:{json.dumps(line_dict)}\n\n"
+            except json.decoder.JSONDecodeError:
+                yield f"data:{json.dumps({'message': line})}\n\n"
+
+    return Response(generate(), mimetype="text/event-stream")
+
+
+@dashboard.route("/logs/range", methods=["GET"])
+@login_required
+def logs_range():
+    """
+    Return a range of log lines from the log file.
+    """
+    params = request.args
+    start, end = params.get("start", None), params.get("end", None)
+    if start is None or end is None:
+        return json_error_response("Both 'start' and 'end' parameters are required.")
+    start, end = int(start), int(end)
+    if start < 1:
+        return json_error_response("'start' must be > 0.")
+
+    if start > end:
+        return json_error_response("'start' must be <= 'end'.")
+
+    lines, early_stop, last_line = log_read_lines(start, end + 1)
+    if not early_stop and start > last_line:
+        return json_error_response(
+            f"'start' must be less than the current line number {last_line}."
+        )
+
+    return lines
+
+
+def json_error_response(message, status_code=400):
+    return jsonify({"msg": message}), status_code
+
+
+@dashboard.route("/logs/find_lines", methods=["GET"])
+@login_required
+def logs_find_lines():
+    """
+    Find lines in the log file that contain a substring (GET parameter 'query').
+    """
+    params = request.args
+    query = params.get("query", None)
+    if query is None:
+        return json_error_response("No query provided.")
+    return Response(log_search_substring(query), mimetype="text/event-stream")
+
+
+@dashboard.route("/logs/find_line_number", methods=["POST"])
+@login_required
+def logs_find_line_number():
+    """
+    Find the line number in the log file that contains a substring (POST parameter 'query').
+    """
+    params = request.json
+    query = params.get("query", None)
+    if query is None:
+        return json_error_response("No query provided.")
+
+    line_number = find_log_line_number(query)
+    if line_number is not None:
+        return jsonify({"line_number": line_number})
path: dallinger/experiment_server/dashboard.py
user: login: polvanrijn
body: Done
created_at: 2025-03-24T15:41:36Z
author_association: COLLABORATOR

PR comments:
user: login: pmcharrison
created_at: 2025-03-14T11:16:48Z
body: Hi @polvanrijn, you have requested the review, but this is still WIP, right? Because you need to sort the Docker issue?

A quick note in the meantime is that Dallinger already has a logfile config parameter, defaulting to server.log. I think you would need to check what the existing log file logic is and make sure you interact with that appropriately.
author_association: COLLABORATOR
user: login: polvanrijn
created_at: 2025-03-17T13:14:13Z
body: > Hi @polvanrijn, you have requested the review, but this is still WIP, right? Because you need to sort the Docker issue?
> 
> A quick note in the meantime is that Dallinger already has a logfile config parameter, defaulting to server.log. I think you would need to check what the existing log file logic is and make sure you interact with that appropriately.

I have now implemented and tested it. Ready for review.
author_association: COLLABORATOR
user: login: polvanrijn
created_at: 2025-03-24T10:49:51Z
body: @pmcharrison, ready for another pass
author_association: COLLABORATOR
user: login: pmcharrison
created_at: 2025-03-24T13:17:31Z
body: @polvanrijn I assume you're also working on the broken tests?
author_association: COLLABORATOR
user: login: polvanrijn
created_at: 2025-03-24T13:38:58Z
body: > I will continue the review once you confirm there are not pending changes that you are waiting to push
@pmcharrison, I have pushed the latest changes and I hope to have fixed CI.
author_association: COLLABORATOR