Repository: GEMDAT-repos/GEMDAT (language: python)

Issue:
Consider moving the dashboard to its own repo
The dashboard is not a priority for this project. Instead focus on jupyter notebooks. The dashboard can move to its own repository so that we can pin a gemdat version and develop/test it in isolation.


Discussion / hints:

Fix patch:
diff --git a/docs/dashboard.md b/docs/dashboard.md
index 604e6641..0b5814b9 100644
--- a/docs/dashboard.md
+++ b/docs/dashboard.md
@@ -1,17 +1,25 @@
-Gemdash includes a dashboard to interactively select your input data and experimental parameters. This makes it straightforward to explore your data without any coding experience required.
+GEMDAT includes a dashboard to interactively select your input data and experimental parameters. This makes it straightforward to explore your data without any coding experience required.
+
+The dashboard is available from: https://github.com/GEMDAT-repos/gemdat-dashboard
 
 To use the dashboard, first make sure you install all the dependencies:
 
 ```
-pip install gemdat[dashboard]
+pip install -r requirements.txt
 ```
 
-The dashboard is available via the command
+The dashboard is available via the command:
 
 ```
-gemdash
+python gemdash.py
 ```
 
+This will start a server with a URL you can paste in your browser to visit the dashboard.
+
+!!! note
+
+    The dashboard runs in the web browser and needs a somewhat recent version of [firefox or one of the other supported browsers here](https://docs.streamlit.io/knowledge-base/using-streamlit/supported-browsers).
+
 To pre-set your data, you can set the `VASP_XML` environment variable (useful for testing):
 
 ```
@@ -20,10 +28,11 @@ VASP_XML=./path/to/vasprun.xml gemdash
 
 It is also possible to specify the file that you want to load via the commandline:
 ```
-gemdash --file newrun/vasprun.xml
+python gemdash.py --file newrun/vasprun.xml
 ```
 
-The options of gemdat can be listed with `gemdash --help`:
+The options of gemdat can be listed with `python gemdash.py --help`:
+
 ```
 usage: gemdash [--file [FILE]] [--help]
 
@@ -34,7 +43,3 @@ options:
                         File to load in gemdash
   --help                specify twice to print streamlit help
 ```
-
-!!! note
-
-    The dashboard runs in the web browser and needs a somewhat recent version of [firefox or one of the other supported browsers here](https://docs.streamlit.io/knowledge-base/using-streamlit/supported-browsers).
diff --git a/pyproject.toml b/pyproject.toml
index 10337140..97cb37f5 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -51,10 +51,8 @@ changelog = "https://github.com/GEMDAT-repos/GEMDAT/releases"
 develop = [
     "coverage[toml]",
     "mypy",
-    "playwright",
     "pre-commit",
     "pytest",
-    "pytest-playwright",
 ]
 
 docs = [
@@ -71,14 +69,6 @@ publishing = [
     "build",
 ]
 
-gemdash = [
-    "uncertainties",
-    "streamlit",
-]
-
-[project.scripts]
-gemdash= "gemdat.dashboard:gemdash"
-
 [tool.setuptools]
 package-dir = {"gemdat" = "src/gemdat" }
 include-package-data = true
@@ -86,7 +76,6 @@ include-package-data = true
 [tool.coverage.run]
 branch = true
 source = ["gemdat"]
-omit = ["*/dashboard/*"]
 
 
 [tool.pytest.ini_options]
diff --git a/src/gemdat/dashboard/__init__.py b/src/gemdat/dashboard/__init__.py
deleted file mode 100644
index 666f481b..00000000
--- a/src/gemdat/dashboard/__init__.py
+++ /dev/null
@@ -1,5 +0,0 @@
-from __future__ import annotations
-
-from .gemdash import gemdash
-
-__all__ = ['gemdash']
diff --git a/src/gemdat/dashboard/_shared.py b/src/gemdat/dashboard/_shared.py
deleted file mode 100644
index c99d5bcb..00000000
--- a/src/gemdat/dashboard/_shared.py
+++ /dev/null
@@ -1,85 +0,0 @@
-from __future__ import annotations
-
-import base64
-import os
-from importlib.resources import files
-from pathlib import Path
-from tkinter import filedialog
-
-import streamlit as st
-
-data_directory = Path(files('gemdat') / 'data')  # type: ignore
-
-
-def get_trajectory_location(filename='vasprun.xml'):
-    if trajectory_location := os.environ.get('VASP_XML'):
-        st.info(f'Got `{trajectory_location}` via environment variable.')
-        return Path(trajectory_location)
-
-    trajectory_location = st.session_state.get('trajectory_location',
-                                               default=filename)
-
-    st.markdown('Select input trajectory')
-    col1, col2 = st.columns([0.7, 0.3])
-    with col1:
-        trajectory_location = st.text_input('filename',
-                                            trajectory_location,
-                                            label_visibility='collapsed')
-    with col2:
-        if st.button('Browse'):
-            trajectory_location = filedialog.askopenfilename()
-            st.session_state.trajectory_location = trajectory_location
-            st.experimental_rerun()
-
-    if not trajectory_location:
-        st.info(f'Select `{filename}` to continue')
-        st.stop()
-
-    trajectory_location = Path(trajectory_location).expanduser()
-
-    if not trajectory_location.exists():
-        st.info(
-            f'Could not find `{trajectory_location}`, select `{filename}` to continue'
-        )
-        st.stop()
-
-    return trajectory_location
-
-
-@st.cache_data
-def get_base64_of_bin_file(png_file):
-    with open(png_file, 'rb') as f:
-        data = f.read()
-    return base64.b64encode(data).decode()
-
-
-def build_markup_for_logo(png_file,
-                          background_position='50% 2%',
-                          margin_top='0%',
-                          image_width='60%',
-                          image_height='',
-                          anchor='stSidebar'
-                          # anchor='stSidebarNav'  # Anchor to navigation
-                          ):
-    binary_string = get_base64_of_bin_file(png_file)
-    return f"""
-            <style>
-                [data-testid="{anchor}"] {{
-                    background-image: url("data:image/png;base64,{binary_string}");
-                    background-repeat: no-repeat;
-                    background-position: {background_position};
-                    margin-top: {margin_top};
-                    background-size: {image_width} {image_height};
-                }}
-            </style>
-            """
-
-
-def add_sidebar_logo():
-    """Based on: https://stackoverflow.com/a/73278825."""
-    png_file = data_directory / 'logo.png'
-    logo_markup = build_markup_for_logo(png_file)
-    st.markdown(
-        logo_markup,
-        unsafe_allow_html=True,
-    )
diff --git a/src/gemdat/dashboard/gemdash.py b/src/gemdat/dashboard/gemdash.py
deleted file mode 100644
index 7aaaca8c..00000000
--- a/src/gemdat/dashboard/gemdash.py
+++ /dev/null
@@ -1,48 +0,0 @@
-from __future__ import annotations
-
-import argparse
-import sys
-from importlib.resources import files
-from pathlib import Path
-
-from streamlit.web.cli import main
-
-dashboard_directory = Path(files('gemdat') / 'dashboard')  # type: ignore
-
-
-def gemdash():
-    parser = argparse.ArgumentParser(
-        prog='gemdash',
-        add_help=False,
-        description='Streamlit dashboard for easily visualizing gemdat data')
-    parser.add_argument('--file',
-                        '-f',
-                        nargs='?',
-                        default='vasprun.xml',
-                        help='File to load in gemdash')
-    parser.add_argument('--help',
-                        action='count',
-                        default=0,
-                        help='specify twice to print streamlit help')
-
-    arguments, unknown = parser.parse_known_args(sys.argv[1:])
-    if arguments.help == 1:
-        parser.print_help()
-        return
-    if arguments.help == 2:
-        unknown.append('--help')
-
-    run_file = str(dashboard_directory / 'run.py')
-
-    sys.argv = [
-        *('streamlit', 'run', run_file),
-        *('--theme.base', 'light'),
-        *('--theme.primaryColor', 'e03c31'),
-        *('--theme.secondaryBackgroundColor', 'bcd9ec'),
-        *('--browser.gatherUsageStats', 'false'),
-        *unknown,
-        '--',
-        *('--file', arguments.file),
-    ]
-
-    main()
diff --git a/src/gemdat/dashboard/run.py b/src/gemdat/dashboard/run.py
deleted file mode 100644
index 69772666..00000000
--- a/src/gemdat/dashboard/run.py
+++ /dev/null
@@ -1,331 +0,0 @@
-from __future__ import annotations
-
-import argparse
-from pathlib import Path
-from sys import argv
-from typing import Any
-
-import numpy as np
-import plotly.graph_objects as go
-import streamlit as st
-from _shared import add_sidebar_logo, get_trajectory_location
-from pymatgen.core import Structure
-
-from gemdat import SitesData, __version__, plots
-from gemdat.io import get_list_of_known_materials, load_known_material
-from gemdat.rdf import radial_distribution
-from gemdat.simulation_metrics import SimulationMetrics, SimulationMetricsStd
-from gemdat.trajectory import Trajectory
-from gemdat.utils import is_lattice_similar
-from gemdat.volume import Volume
-
-st.set_page_config(
-    page_title='Gemdat dashboard',
-    layout='wide',
-    page_icon='💎',
-    menu_items={
-        'Get Help':
-        'https://gemdat.readthedocs.io',
-        'Report a bug':
-        'https://github.com/gemdat-repos/gemdat/issues',
-        'About':
-        (f'GEMDASH: a dashboard for GEMDAT ({__version__}). '
-         '\n\nGEMDAT is Python toolkit for molecular dynamics analysis. '
-         '\n\nFor more information, see: https://github.com/GEMDAT-repos/GEMDAT'
-         )
-    })
-
-add_sidebar_logo()
-
-# Check gemdash.py for the more complete parser
-parser = argparse.ArgumentParser()
-parser.add_argument('--file', nargs='?')
-arguments, _ = parser.parse_known_args(argv[1:])
-
-KNOWN_MATERIALS = get_list_of_known_materials()
-
-with st.sidebar:
-    if arguments.file is not None:
-        trajectory_location = get_trajectory_location(arguments.file)
-    else:
-        trajectory_location = get_trajectory_location()
-
-
-@st.cache_data
-def _load_trajectory(trajectory_location):
-    return Trajectory.from_vasprun(trajectory_location)
-
-
-with st.spinner('Loading your trajectory data, this might take a while'):
-    trajectory = _load_trajectory(trajectory_location)
-
-with st.sidebar:
-    # Get list of present elements as tuple of strings
-    elements = tuple(set([str(s) for s in trajectory.species]))
-
-    # Set prefered element to lithium if available
-    try:
-        index = elements.index('Li')
-    except ValueError:
-        index = 0
-
-    diffusing_element = st.selectbox('Diffusive Element',
-                                     elements,
-                                     index=index)
-    equilibration_steps = int(
-        st.number_input('Equilibration Steps',
-                        min_value=0,
-                        max_value=len(trajectory) - 1,
-                        value=1250))
-
-    sites_filename = str(
-        st.selectbox('Load sites from known material', KNOWN_MATERIALS))
-
-    manual_supercell = st.checkbox('Manual Supercell')
-    if manual_supercell:
-        st.markdown('Supercell (x,y,z)')
-        col1, col2, col3 = st.columns(3)
-        supercell = (int(
-            col1.number_input('supercell x',
-                              min_value=1,
-                              value=1,
-                              label_visibility='collapsed',
-                              help=None)),
-                     int(
-                         col2.number_input('supercell y',
-                                           min_value=1,
-                                           value=1,
-                                           label_visibility='collapsed')),
-                     int(
-                         col3.number_input('supercell z',
-                                           min_value=1,
-                                           value=1,
-                                           label_visibility='collapsed')))
-
-    st.markdown('## Enable Error Analysis')
-    do_error = st.checkbox('Error Analysis')
-    n_parts: int = 1
-    if do_error:
-        with st.sidebar:
-            n_parts = int(
-                st.number_input('Number of parts to divide trajectory in',
-                                value=10))
-
-    st.markdown('## Radial distribution function')
-    do_rdf = st.checkbox('Plot RDFs')
-    if do_rdf:
-        with st.sidebar:
-            max_dist_rdf = st.number_input(
-                'Maximum distance for RDF calculation', value=10.0)
-            resolution_rdf = st.number_input('Resolution (width of bins)',
-                                             value=0.1)
-
-number_of_cols = 3  # Number of figure columns
-
-trajectory = trajectory[equilibration_steps:]
-diff_trajectory = trajectory.filter('Li')
-metrics = SimulationMetrics(diff_trajectory)
-metrics_std = SimulationMetricsStd(
-    diff_trajectory.split(n_parts, equal_parts=True))
-
-col1, col2, col3 = st.columns(3)
-
-with col1:
-    import uncertainties as u
-    attempt_freq = u.ufloat(*metrics.attempt_frequency())
-    st.metric('Attempt frequency ($\\mathrm{s^{-1}}$)',
-              value=f'{attempt_freq:g}')
-    st.metric('Vibration amplitude ($\\mathrm{Å}$)',
-              value=f'{metrics_std.vibration_amplitude():g}')
-with col2:
-    st.metric('Particle density ($\\mathrm{m^{-3}}$)',
-              value=f'{metrics.particle_density():g}')
-    st.metric('Mol per liter ($\\mathrm{mol/l}$)',
-              value=f'{metrics.mol_per_liter():g}')
-with col3:
-    st.metric('Tracer diffusivity ($\\mathrm{m^2/s}$)',
-              value=f'{metrics_std.tracer_diffusivity(dimensions=3):g}')
-    st.metric(
-        'Tracer conductivity ($\\mathrm{S/m}$)',
-        value=f'{metrics_std.tracer_conductivity(z_ion=1, dimensions=3):g}')
-
-tab1, tab2, tab3 = st.tabs(['Default plots', 'RDF plots', 'Density plots'])
-
-if manual_supercell:
-    sites_structure = load_known_material(sites_filename, supercell=supercell)
-else:
-    sites_structure = load_known_material(sites_filename)
-    zipped_parameters = zip(trajectory.get_lattice().abc,
-                            sites_structure.lattice.parameters)
-    supercell = [round(a / b) for a, b in zipped_parameters]  # type: ignore
-    sites_structure.make_supercell(supercell)
-
-if not is_lattice_similar(trajectory.get_lattice(), sites_structure):
-    st.error('Lattices are not similar!')
-    st.text(f'{sites_filename}: {sites_structure.lattice.parameters}')
-    st.text(
-        f'{trajectory_location.name}: {trajectory.get_lattice().parameters}')
-    st.stop()
-
-
-def _structure_hash_func(obj: Structure) -> dict:
-    return obj.as_dict()
-
-
-def _trajectory_hash_func(obj: Trajectory) -> np.ndarray:
-    return obj.positions
-
-
-def _volume_hash_func(obj: Volume) -> np.ndarray:
-    return obj.data
-
-
-@st.cache_data(hash_funcs={
-    Structure: _structure_hash_func,
-    Trajectory: _trajectory_hash_func
-})
-def _SitesData(**kwargs):
-    return SitesData(**kwargs)
-
-
-with tab1:
-    st.title('Trajectory and jumps')
-
-    with st.spinner('Calculating jumps...'):
-        sites = _SitesData(
-            structure=sites_structure,
-            trajectory=trajectory,
-            floating_specie=diffusing_element,
-            n_parts=n_parts,
-        )
-
-    diff_trajectory = trajectory.filter(diffusing_element)
-
-    figures = (
-        plots.displacement_per_element(trajectory=trajectory),
-        plots.msd_per_element(trajectory=trajectory),
-        plots.displacement_per_site(trajectory=diff_trajectory),
-        plots.displacement_histogram(trajectory=trajectory, n_parts=n_parts),
-        plots.frequency_vs_occurence(trajectory=diff_trajectory),
-        plots.vibrational_amplitudes(trajectory=diff_trajectory,
-                                     n_parts=n_parts),
-        plots.jumps_vs_distance(sites=sites, n_parts=n_parts),
-        plots.jumps_vs_time(sites=sites, n_parts=n_parts),
-        plots.collective_jumps(sites=sites),
-        plots.jumps_3d(sites=sites),
-    )
-
-    # automagically divide the plots over the number of columns
-    for num, col in enumerate(st.columns(number_of_cols)):
-        for figure in figures[num::number_of_cols]:
-            if isinstance(figure, go.Figure):
-                col.plotly_chart(figure, use_container_width=True)
-            else:
-                col.pyplot(figure)
-
-
-def _sites_hash_func(obj: SitesData) -> tuple[Any, Any, Any]:
-    return obj.structure.frac_coords, obj.trajectory.positions, obj.floating_specie
-
-
-@st.cache_data(hash_funcs={SitesData: _sites_hash_func})
-def _radial_distribution(**kwargs):
-    return radial_distribution(**kwargs)
-
-
-if do_rdf:
-    with tab2:
-        st.title('Radial distribution function')
-
-        with st.spinner('Calculating RDFs...'):
-            rdf_data = _radial_distribution(
-                sites=sites,
-                max_dist=max_dist_rdf,
-                resolution=resolution_rdf,
-            )
-
-        rdf_figures = [
-            plots.radial_distribution(rdfs) for rdfs in rdf_data.values()
-        ]
-
-        # automagically divide the plots over the number of columns
-        for num, col in enumerate(st.columns(number_of_cols)):
-            for figure in rdf_figures[num::number_of_cols]:
-                col.pyplot(figure)
-
-with tab3:
-    from gemdat.volume import trajectory_to_volume
-
-    st.title('Density plots')
-
-    col1, col2 = st.columns(2)
-
-    col1.write('Density')
-    density_resolution = col1.number_input(
-        'Resolution (Å)',
-        min_value=0.1,
-        value=0.3,
-        step=0.05,
-        help='Minimum resolution for the voxels in Ångstrom')
-
-    col2.write('Peak finding')
-    background_level = col2.number_input(
-        'Background level',
-        min_value=0.0,
-        max_value=1.0,
-        value=0.1,
-        step=0.05,
-        help=
-        'Fraction of the maximum volume value to set as the minimum value for peak segmentation.'
-    )
-
-    pad = col2.number_input(
-        'Padding',
-        min_value=0,
-        value=3,
-        step=1,
-        help=
-        ('Extend the volume by this number of voxels by wrapping around. This helps finding '
-         'maxima for blobs sitting at the edge of the unit cell.'))
-
-    @st.cache_data(hash_funcs={Trajectory: _trajectory_hash_func})
-    def _generate_density(**kwargs):
-        return trajectory_to_volume(**kwargs)
-
-    @st.cache_data(hash_funcs={Volume: _volume_hash_func})
-    def _generate_structure(vol, **kwargs):
-        return vol.to_structure(**kwargs)
-
-    @st.cache_data(hash_funcs={
-        Volume: _volume_hash_func,
-        Structure: _structure_hash_func
-    })
-    def _density_plot(**kwargs):
-        return plots.density(**kwargs)
-
-    if st.checkbox('Generate density'):
-        vol = _generate_density(
-            trajectory=diff_trajectory,
-            resolution=density_resolution,
-        )
-
-        structure = _generate_structure(
-            vol,
-            background_level=background_level,
-            pad=pad,
-        )
-
-        chart = _density_plot(vol=vol, structure=structure)
-
-        st.plotly_chart(chart)
-
-        col1, col2 = st.columns((8, 2))
-
-        outputfile = col1.text_input(
-            'Filename',
-            trajectory_location.with_name('volume.vasp'),
-            label_visibility='collapsed')
-
-        if col2.button('Save volume', type='primary'):
-            vol.to_vasp_volume(structure=structure, filename=outputfile)
-            st.success(f'Saved to {Path(outputfile).absolute()}')


Test patch:
diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml
index 82306cdd..71439dc6 100644
--- a/.github/workflows/tests.yaml
+++ b/.github/workflows/tests.yaml
@@ -46,14 +46,11 @@ jobs:
       - name: Install dependencies
         if: steps.cache-virtualenv.outputs.cache-hit != 'true'
         run: |
-          python -m pip install -e .[develop,gemdash]
-
-      - name: Ensure browser is installed
-        run: python -m playwright install chromium
+          python -m pip install -e .[develop]
 
       - name: Test with pytest
         run: |
-          coverage run -p -m pytest --dashboard
+          coverage run -p -m pytest
 
       - name: Combine
         run: |
diff --git a/tests/conftest.py b/tests/conftest.py
index 634b86ec..1a546246 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -7,18 +7,6 @@
 from gemdat.trajectory import Trajectory
 
 
-def pytest_addoption(parser):
-    """Add options to pytest."""
-    parser.addoption('--dashboard',
-                     action='store_true',
-                     default=False,
-                     help='Run dashboard workflow tests')
-
-
-def pytest_configure(config):
-    pytest.skip_dashboard = not config.getoption('--dashboard')
-
-
 @pytest.fixture()
 def trajectory():
     coords = np.array([
diff --git a/tests/playwright_test.py b/tests/playwright_test.py
deleted file mode 100644
index d8c189e3..00000000
--- a/tests/playwright_test.py
+++ /dev/null
@@ -1,53 +0,0 @@
-from __future__ import annotations
-
-import time
-
-import pytest
-from playwright.sync_api import Page, expect
-
-PORT = '8501'
-BASE_URL = f'http://localhost:{PORT}'
-
-if pytest.skip_dashboard:
-    pytestmark = pytest.mark.skip(
-        reason='Use `--dashboard` to test dashboard workflow.')
-
-
-@pytest.fixture(scope='module', autouse=True)
-def before_module():
-    """Run dashboard in module scope."""
-    import subprocess
-
-    p = subprocess.Popen([
-        'gemdash',
-        '--server.port',
-        PORT,
-        '--server.headless',
-        'true',
-    ])
-    time.sleep(5)
-    yield
-    p.kill()
-
-
-def test_gemdash(page: Page):
-    # Goto web page
-    page.goto(BASE_URL)
-
-    page.get_by_label('filename').click()
-    page.get_by_label('filename').fill(
-        'tests/data/short_simulation/vasprun.xml')
-    page.get_by_label('filename').press('Enter')
-
-    # check values
-    expect(page.get_by_text('(8.5+/-0.9)e+12')).to_be_visible(timeout=60_000)
-    expect(page.get_by_text('0.520307+/-0')).to_be_visible()
-    expect(page.get_by_text('2.45567e+28')).to_be_visible()
-    expect(page.get_by_text('40.7774')).to_be_visible()
-    expect(page.get_by_text('(1.566+/-0)e-09')).to_be_visible()
-    expect(page.get_by_text('109.999+/-0')).to_be_visible()
-
-    page.get_by_role('tab', name='RDF plots').click()
-    page.locator('label').filter(has_text='Plot RDFs').locator('span').click()
-
-    page.get_by_role('tab', name='Density plots').click()