Repository: OpenEnergyPlatform/oeplatform

Issue:
Refactor ontology django views and improve page load behavior & UX
## Description of the issue

The ontology oeo and the textual oeo viewer pages are all served in one djnago view. It works but is hard to maintian and extend. To solve this the view must be refactored into smaller parts. The chellange is to keep the current functionality.

While doing this it is a good idea to improve the page loading as some pages can take a few secons to be availabe because the oeo must be parsed on the server first. 

## Ideas of solution

- Restructur the main view of the ontology pages into several views for the oeo class viewer and the overview pages https://github.com/OpenEnergyPlatform/oeplatform/blob/5802cf7c3cb651f6d293de25e0f9929832219196/ontology/views.py#L99 
- Modularization of the code by introducing helper functions, e.g. for parsing the oeo content in Python
- Provision of multi-pe URLs that enable access to new views

- introduce htmx in templates to load additional page content after a first page is already available and display a load spinner to better represent the active page loading process.

## Workflow checklist
- [ ] I am aware of the workflow in [CONTRIBUTING.md](https://github.com/OpenEnergyPlatform/oeplatform/blob/develop/CONTRIBUTING.md)



Discussion:

Resolving patch:
diff --git a/ontology/urls.py b/ontology/urls.py
--- a/ontology/urls.py
+++ b/ontology/urls.py
@@ -1,10 +1,21 @@
 from django.conf.urls import url
 from django.views.generic import TemplateView
+from django.urls import path
 
 from ontology import views
 
 urlpatterns = [
     url(r"^$", views.OntologyVersion.as_view()),
+    url(
+        "partial/page-content/",
+        views.PartialOntologyOverviewContent.as_view(),
+        name="partial-page-content",
+    ),
+    url(
+        "partial/page-sidebar-content/",
+        views.PartialOntologyOverviewSidebarContent.as_view(),
+        name="partial-page-sidebar-content",
+    ),
     url(r"^ontology/$", views.OntologyVersion.as_view()),
     url(
         r"^oeo-steering-committee/$",
@@ -33,6 +44,10 @@
         views.OntologyOverview.as_view(),
         {"imports": True},
     ),
+    url(
+        r"^(?P<ontology>[\w_-]+)?/(?P<module_or_id>[\w\d_-]+)?",
+        views.OntologyViewClasses.as_view(),
+    ),
     url(
         r"^(?P<ontology>[\w_-]+)(/(?P<module_or_id>[\w\d_-]+))?",
         views.OntologyOverview.as_view(),
diff --git a/ontology/views.py b/ontology/views.py
--- a/ontology/views.py
+++ b/ontology/views.py
@@ -9,13 +9,10 @@
 from rdflib import Graph
 
 from oeplatform.settings import (
-    ONTOLOGY_FOLDER,
     ONTOLOGY_ROOT,
     OPEN_ENERGY_ONTOLOGY_NAME,
 )
 
-from rdflib import Graph
-
 
 def collect_modules(path):
     modules = dict()
@@ -30,9 +27,6 @@ def collect_modules(path):
                 g = Graph()
                 g.parse(os.path.join(path, file))
 
-                # Get the root namespace
-                root_namespace = next(iter(g.namespaces()))
-
                 # Set the namespaces in the graph
                 for prefix, uri in g.namespaces():
                     g.bind(prefix, uri)
@@ -74,6 +68,109 @@ def collect_modules(path):
     return modules
 
 
+def read_oeo_context_information(path, file):
+    Ontology_URI = path / file
+    g = Graph()
+    g.parse(Ontology_URI.as_posix())
+
+    q_global = g.query(
+        """
+        SELECT DISTINCT ?s ?o
+        WHERE { ?s rdfs:subClassOf ?o
+        filter(!isBlank(?o))
+        }
+        """
+    )
+
+    q_label = g.query(
+        """
+        SELECT DISTINCT ?s ?o
+        WHERE { ?s rdfs:label ?o }
+        """
+    )
+
+    q_definition = g.query(
+        """
+        SELECT DISTINCT ?s ?o
+        WHERE { ?s obo:IAO_0000115 ?o }
+        """
+    )
+
+    q_note = g.query(
+        """
+        SELECT DISTINCT ?s ?o
+        WHERE { ?s obo:IAO_0000116 ?o }
+        """
+    )
+
+    q_main_description = g.query(
+        """
+        SELECT ?s ?o
+        WHERE { ?s dc:description ?o }
+        """
+    )
+
+    classes_name = {}
+    for row in q_label:
+        class_name = row.s.split("/")[-1]
+        classes_name[class_name] = row.o
+
+    classes_definitions = defaultdict(list)
+    for row in q_definition:
+        class_name = row.s.split("/")[-1]
+        classes_definitions[class_name].append(row.o)
+
+    classes_notes = defaultdict(list)
+    for row in q_note:
+        class_name = row.s.split("/")[-1]
+        classes_notes[class_name].append(row.o)
+
+    ontology_description = ""
+    for row in q_main_description:
+        if row.s.split("/")[-1] == "":
+            ontology_description = row.o
+
+    result = {
+        "q_global": q_global,
+        "classes_name": classes_name,
+        "classes_definitions": dict(classes_definitions),
+        "classes_notes": dict(classes_notes),
+        "ontology_description": ontology_description,
+    }
+
+    return result
+
+
+def get_ontology_version(path, version=None):
+    if not path.exists():
+        raise Http404
+
+    versions = os.listdir(path)
+    if not version:
+        version = max(
+            (d for d in versions), key=lambda d: [int(x) for x in d.split(".")]
+        )
+
+    return version
+
+
+def get_common_data(ontology, version=None, path=None):
+    onto_base_path = Path(ONTOLOGY_ROOT, ontology)
+
+    version = get_ontology_version(onto_base_path, version=version)
+    file = "oeo-full.owl"
+
+    path = onto_base_path / version
+    oeo_context_data = read_oeo_context_information(path=path, file=file)
+
+    return {
+        "ontology": ontology,
+        "version": version,
+        "path": path,
+        "oeo_context_data": oeo_context_data,
+    }
+
+
 class OntologyVersion(View):
     def get(self, request, ontology="oeo", version=None):
         onto_base_path = Path(ONTOLOGY_ROOT, ontology)
@@ -96,9 +193,78 @@ def get(self, request, ontology="oeo", version=None):
         )
 
 
+class PartialOntologyOverviewContent(View):
+    def get(self, request):
+        if request.headers.get("HX-Request") == "true":
+            if request.method == "GET":
+                onto_base_path = Path(ONTOLOGY_ROOT, OPEN_ENERGY_ONTOLOGY_NAME)
+
+                version = get_ontology_version(onto_base_path)
+                path = onto_base_path / version
+                ontology_data = get_common_data(OPEN_ENERGY_ONTOLOGY_NAME)
+
+                submodules = collect_modules(path / "modules")
+                # Collect all file names
+                imports = collect_modules(path / "imports")
+
+                partial = render(
+                    request,
+                    "ontology/partial_ontology_content.html",
+                    dict(
+                        ontology=OPEN_ENERGY_ONTOLOGY_NAME,
+                        version=ontology_data["version"],
+                        submodules=submodules.items(),
+                        imports=imports.items(),
+                        ontology_description=ontology_data["oeo_context_data"][
+                            "ontology_description"
+                        ],
+                    ),
+                ).content.decode("utf-8")
+
+                return HttpResponse(partial)
+
+
+class PartialOntologyOverviewSidebarContent(View):
+    def get(self, request):
+        onto_base_path = Path(ONTOLOGY_ROOT, OPEN_ENERGY_ONTOLOGY_NAME)
+
+        version = get_ontology_version(onto_base_path)
+        path = onto_base_path / version
+        main_module = collect_modules(
+            path
+        )  # TODO fix varname - not clear what path this is
+        if OPEN_ENERGY_ONTOLOGY_NAME in main_module.keys():
+            main_module_name = OPEN_ENERGY_ONTOLOGY_NAME
+        else:
+            raise Exception(
+                f"The main module '{OPEN_ENERGY_ONTOLOGY_NAME}' is not available in {path}."
+            )
+
+        main_module = main_module[main_module_name]
+        main_module["name"] = main_module_name
+        partial = render(
+            request,
+            "ontology/partial_ontology_sidebar_content.html",
+            dict(
+                ontology=OPEN_ENERGY_ONTOLOGY_NAME,
+                main_module=main_module,
+            ),
+        ).content.decode("utf-8")
+
+        return HttpResponse(partial)
+
+
+def initial_for_pageload(request):
+    if request.headers.get("HTTP_HX_REQUEST") == "true":
+        if request.method == "GET":
+            return render(request, "ontology/initial_response_htmx.html")
+
+
 class OntologyOverview(View):
     def get(self, request, ontology, module_or_id=None, version=None, imports=False):
-        onto_base_path = Path(ONTOLOGY_ROOT, ontology)
+        # ignore whatever is in ontology
+        ontology = OPEN_ENERGY_ONTOLOGY_NAME
+        onto_base_path = Path(ONTOLOGY_ROOT, OPEN_ENERGY_ONTOLOGY_NAME)
 
         if not onto_base_path.exists():
             raise Http404
@@ -220,139 +386,154 @@ def get(self, request, ontology, module_or_id=None, version=None, imports=False)
         # End prepare data for oeo-viewer
 
         if "text/html" in request.headers.get("accept", "").split(","):
-            if module_or_id:
-                sub_classes = []
-                super_classes = []
-
-                for row in q_global:
-                    if module_or_id in row.o:
-                        sub_class_ID = row.s.split("/")[-1]
-                        sub_class_name = ""
-                        sub_class_definition = ""
-                        sub_class_note = ""
-                        if sub_class_ID in classes_name.keys():
-                            sub_class_name = classes_name[sub_class_ID]
-                            if sub_class_ID in classes_definitions.keys():
-                                sub_class_definition = classes_definitions[sub_class_ID]
-                            if sub_class_ID in classes_notes.keys():
-                                sub_class_note = classes_notes[sub_class_ID]
-                            sub_classes.append(
-                                {
-                                    "URI": row.s,
-                                    "ID": sub_class_ID,
-                                    "name": sub_class_name,
-                                    "definitions": sub_class_definition,
-                                    "notes": sub_class_note,
-                                }
-                            )
-                    if module_or_id in row.s:
-                        super_class_ID = row.o.split("/")[-1]
-                        super_class_name = ""
-                        super_class_definition = ""
-                        super_class_note = ""
-                        if super_class_ID in classes_name.keys():
-                            super_class_name = classes_name[super_class_ID]
-                            if super_class_ID in classes_definitions.keys():
-                                super_class_definition = classes_definitions[
-                                    super_class_ID
-                                ]
-                            if super_class_ID in classes_notes.keys():
-                                super_class_note = classes_notes[super_class_ID]
-                            super_classes.append(
-                                {
-                                    "URI": row.o,
-                                    "ID": super_class_ID,
-                                    "name": super_class_name,
-                                    "definitions": super_class_definition,
-                                    "notes": super_class_note,
-                                }
-                            )
-
-                class_name = ""
-                if module_or_id in classes_name.keys():
-                    class_name = classes_name[module_or_id]
-
-                class_definitions = ""
-                if module_or_id in classes_definitions.keys():
-                    class_definitions = classes_definitions[module_or_id]
-
-                class_notes = ""
-                if module_or_id in classes_notes.keys():
-                    class_notes = classes_notes[module_or_id]
-
-                return render(
-                    request,
-                    "ontology/class.html",
-                    dict(
-                        class_id=module_or_id,
-                        class_name=class_name,
-                        sub_classes=sub_classes,
-                        super_classes=super_classes,
-                        class_definitions=class_definitions,
-                        class_notes=class_notes,
-                    ),
-                )
+            if module_or_id and "oeo" in module_or_id:
+                pass
             else:
-                main_module = collect_modules(
-                    path
-                )  # TODO fix varname - not clear what path this is
-                if OPEN_ENERGY_ONTOLOGY_NAME in main_module.keys():
-                    main_module_name = OPEN_ENERGY_ONTOLOGY_NAME
-                else:
-                    raise Exception(
-                        f"The main module '{OPEN_ENERGY_ONTOLOGY_NAME}' is not available in {path}."
-                    )
-
-                main_module = main_module[main_module_name]
-                main_module["name"] = main_module_name
-                submodules = collect_modules((path / "modules"))
-                # Collect all file names
-                imports = collect_modules(path / "imports")
-
                 return render(
                     request,
                     "ontology/oeo.html",
-                    dict(
-                        ontology=ontology,
-                        version=version,
-                        main_module=main_module,
-                        submodules=submodules.items(),
-                        imports=imports.items(),
-                        ontology_description=ontology_description,
-                    ),
+                    {"ontology": OPEN_ENERGY_ONTOLOGY_NAME},
                 )
         else:
             module_name = None
             if module_or_id:
                 if imports:
-                    submodules = collect_modules(
-                        f"{ONTOLOGY_FOLDER}/{ontology}/{version}/imports"
-                    )
+                    submodules = collect_modules(path / "imports")
                 else:
-                    submodules = collect_modules(
-                        f"{ONTOLOGY_FOLDER}/{ontology}/{version}/modules"
-                    )
+                    submodules = collect_modules(f"{path}/modules")
                 # If module_or_id is the name of a valid submodule, use this module
                 if module_or_id in submodules:
                     module_name = module_or_id
                 if imports:
-                    return redirect(
-                        f"/ontology/{ontology}/releases/{version}/imports/{module_name}.owl"  # noqa
-                    )
+                    return redirect(f"{path}/imports/{module_name}.owl")  # noqa
                 else:
-                    return redirect(
-                        f"/ontology/{ontology}/releases/{version}/{module_name}.owl"
-                    )
+                    return redirect(f"{path}/{module_name}.owl")
             # If no module was requested or the requested id was not a module,
             # serve main ontology
             if module_name is None:
-                main_module = collect_modules(f"{ONTOLOGY_FOLDER}/{ontology}/{version}")
+                main_module = collect_modules(path)
                 module_name = list(main_module.keys())[0]
             return redirect(
                 f"/ontology/{ontology}/releases/{version}/{module_name}.owl"
             )
 
 
+class OntologyViewClasses(View):
+    def get(self, request, ontology, module_or_id=None, version=None, imports=False):
+        ontology_data = get_common_data(ontology=OPEN_ENERGY_ONTOLOGY_NAME)
+
+        sub_classes = []
+        super_classes = []
+        if module_or_id:
+            for row in ontology_data["oeo_context_data"]["q_global"]:
+                if module_or_id in row.o:
+                    sub_class_ID = row.s.split("/")[-1]
+                    sub_class_name = ""
+                    sub_class_definition = ""
+                    sub_class_note = ""
+                    if (
+                        sub_class_ID
+                        in ontology_data["oeo_context_data"]["classes_name"].keys()
+                    ):
+                        sub_class_name = ontology_data["oeo_context_data"][
+                            "classes_name"
+                        ][sub_class_ID]
+                        if (
+                            sub_class_ID
+                            in ontology_data["oeo_context_data"][
+                                "classes_definitions"
+                            ].keys()
+                        ):
+                            sub_class_definition = ontology_data["oeo_context_data"][
+                                "classes_definitions"
+                            ][sub_class_ID]
+                        if (
+                            sub_class_ID
+                            in ontology_data["oeo_context_data"]["classes_notes"].keys()
+                        ):
+                            sub_class_note = ontology_data["oeo_context_data"][
+                                "classes_notes"
+                            ][sub_class_ID]
+                        sub_classes.append(
+                            {
+                                "URI": row.s,
+                                "ID": sub_class_ID,
+                                "name": sub_class_name,
+                                "definitions": sub_class_definition,
+                                "notes": sub_class_note,
+                            }
+                        )
+                if module_or_id in row.s:
+                    super_class_ID = row.o.split("/")[-1]
+                    super_class_name = ""
+                    super_class_definition = ""
+                    super_class_note = ""
+                    if (
+                        super_class_ID
+                        in ontology_data["oeo_context_data"]["classes_name"].keys()
+                    ):
+                        super_class_name = ontology_data["oeo_context_data"][
+                            "classes_name"
+                        ][super_class_ID]
+                        if (
+                            super_class_ID
+                            in ontology_data["oeo_context_data"][
+                                "classes_definitions"
+                            ].keys()
+                        ):
+                            super_class_definition = ontology_data["oeo_context_data"][
+                                "classes_definitions"
+                            ][super_class_ID]
+                        if (
+                            super_class_ID
+                            in ontology_data["oeo_context_data"]["classes_notes"].keys()
+                        ):
+                            super_class_note = ontology_data["oeo_context_data"][
+                                "classes_notes"
+                            ][super_class_ID]
+                        super_classes.append(
+                            {
+                                "URI": row.o,
+                                "ID": super_class_ID,
+                                "name": super_class_name,
+                                "definitions": super_class_definition,
+                                "notes": super_class_note,
+                            }
+                        )
+
+        class_name = ""
+        if module_or_id in ontology_data["oeo_context_data"]["classes_name"].keys():
+            class_name = ontology_data["oeo_context_data"]["classes_name"][module_or_id]
+
+        class_definitions = ""
+        if (
+            module_or_id
+            in ontology_data["oeo_context_data"]["classes_definitions"].keys()
+        ):
+            class_definitions = ontology_data["oeo_context_data"][
+                "classes_definitions"
+            ][module_or_id]
+
+        class_notes = ""
+        if module_or_id in ontology_data["oeo_context_data"]["classes_notes"].keys():
+            class_notes = ontology_data["oeo_context_data"]["classes_notes"][
+                module_or_id
+            ]
+
+        return render(
+            request,
+            "ontology/class.html",
+            dict(
+                class_id=module_or_id,
+                class_name=class_name,
+                sub_classes=sub_classes,
+                super_classes=super_classes,
+                class_definitions=class_definitions,
+                class_notes=class_notes,
+            ),
+        )
+
+
 class OntologyStatics(View):
     def get(self, request, ontology, file, version=None, extension=None, imports=False):
         """


Test patch: