Repository: OpenEnergyPlatform/oeplatform

Issue:
Automatic validation of the open data licence before data publish
## Description of the issue

We want to introduce a community-driven open peer review. It takes time to build the community and the knowledge for people to review data and metadata. For example, a high level of data literacy is needed to check whether an open data licence can be granted for a particular dataset. 

We need to require some simple form of validation of the licence before the user can publish data on the OEP. We dont want to hinder the user to publish data. Uploading to modeldraft will continue to be possible without restrictions. 

## Ideas of solution

To minimise the potential of uploading non-open data to the OEP, we want to introduce an automatic check of the license for open data. The license is part of the metadata.

It can be checked 1. whether a license is available at all 2. whether the licence name is an open licence 3. whether the licence name is unknown 4. whether the license is not an open license

## Workflow checklist
- [ ] I am aware of the workflow in [CONTRIBUTING.md](https://github.com/OpenEnergyPlatform/oeplatform/blob/develop/CONTRIBUTING.md)



Discussion:

Resolving patch:
diff --git a/login/utilities.py b/login/utilities.py
new file mode 100644
--- /dev/null
+++ b/login/utilities.py
@@ -0,0 +1,78 @@
+import json
+from pathlib import Path
+import logging
+from functools import lru_cache
+import re
+
+from oeplatform.settings import STATIC_ROOT
+
+
+def normalize_license_name(name):
+    # Replace whitespaces with hyphens and convert to uppercase
+    return re.sub(r"\s", "-", name).upper()
+
+
+@lru_cache(maxsize=None)
+def read_spdx_licenses_from_static():
+    # Specify the path to your JSON file
+    file = "data_licenses/licenses.json"
+    json_file_path = Path(STATIC_ROOT, file)
+
+    try:
+        # Open the file in read mode
+        if json_file_path:
+            with open(json_file_path, "r") as file:
+                # Load the JSON data into a Python dictionary
+                data_dict = json.load(file)
+
+        return data_dict
+    except FileNotFoundError as e:
+        logging.error(f"The licenses files was not found in {json_file_path}")
+        raise e
+
+
+@lru_cache(maxsize=None)
+def create_license_id_set():
+    licenses = read_spdx_licenses_from_static()
+    # Check if the "licenses" key exists in the dictionary
+    if "licenses" in licenses:
+        # Create a set of unique licenseId values
+        return {license_info.get("licenseId").upper() for license_info in licenses["licenses"]}
+
+    else:
+        return set()
+
+
+LICENSES = create_license_id_set()
+
+
+def search_oem_license_in_spdx_list(input_license_id, license_set=LICENSES):
+    processed_input = normalize_license_name(input_license_id)
+    return processed_input in license_set
+
+
+def validate_open_data_license(django_table_obj):
+    metadata = django_table_obj.oemetadata
+    if metadata is None:
+        return False, "Metadata is empty!"
+
+    licenses = metadata.get("licenses", [])
+
+    if not licenses:
+        return False, "No license information available in the metadata."
+
+    first_license = licenses[0]
+    if not first_license.get("name"):
+        return (
+            False,
+            "The license name is missing (only checked the first license element in the oemetadata).",
+        )
+
+    identifier = first_license["name"]
+    if not search_oem_license_in_spdx_list(input_license_id=identifier):
+        return (
+            False,
+            "The license name was not found in the SPDX licenses list. (See https://github.com/spdx/license-list-data/blob/main/json/licenses.json)",
+        )
+
+    return True, None
diff --git a/login/views.py b/login/views.py
--- a/login/views.py
+++ b/login/views.py
@@ -13,6 +13,8 @@
 from dataedit.models import PeerReviewManager, Table, PeerReview
 from dataedit.views import schema_whitelist
 
+from oeplatform.settings import UNVERSIONED_SCHEMAS
+
 from .forms import (
     ChangeEmailForm,
     CreateUserForm,
@@ -24,6 +26,8 @@
 from .models import ADMIN_PERM, GroupMembership, UserGroup
 from .models import myuser as OepUser
 
+from login.utilities import validate_open_data_license
+
 
 class TablesView(View):
     def get(self, request, user_id):
@@ -31,22 +35,26 @@ def get(self, request, user_id):
         tables = Table.objects.all().select_related()
         draft_tables = []
         published_tables = []
+        published_but_license_issue = []
 
         for table in tables:
-            if user.get_table_permission_level(table) >= models.WRITE_PERM:
-                table_data = {
-                    "name": table.name,
-                    "schema": table.schema.name,
-                    "is_publish": table.is_publish,
-                    "is_reviewed": table.is_reviewed
-                }
+            permission_level = user.get_table_permission_level(table)
+            license_status = validate_open_data_license(django_table_obj=table)
+
+            table_data = {
+                "name": table.name,
+                "schema": table.schema.name,
+                "is_publish": table.is_publish,
+                "is_reviewed": table.is_reviewed,
+                "license_status": {
+                    "status": license_status[0],
+                    "error": license_status[1],
+                },
+            }
 
-                # Определение категории таблицы
-                if table.is_reviewed:
-                    if table.is_publish:
-                        published_tables.append(table_data)
-                    else:
-                        draft_tables.append(table_data)
+            if permission_level >= models.WRITE_PERM:
+                if table.is_publish or table.schema.name not in UNVERSIONED_SCHEMAS:
+                    published_tables.append(table_data)
                 else:
                     draft_tables.append(table_data)
 
@@ -54,19 +62,14 @@ def get(self, request, user_id):
             "profile_user": user,
             "draft_tables": draft_tables,
             "published_tables": published_tables,
-            "schema_whitelist": schema_whitelist
+            "schema_whitelist": schema_whitelist,
         }
 
-        # TODO: Fix this is_ajax as it is outdated according to django documentation ... provide better api endpoint for http requests via HTMX
         if request.is_ajax():
             return render(request, "login/user_tables.html", context)
         return render(request, "login/user_tables.html", context)
 
 
-
-
-
-
 class ReviewsView(View):
     def get(self, request, user_id):
         """
diff --git a/oeplatform/settings.py b/oeplatform/settings.py
--- a/oeplatform/settings.py
+++ b/oeplatform/settings.py
@@ -87,6 +87,7 @@
     "readthedocs": "https://oeplatform.readthedocs.io/en/latest/?badge=latest",
     "mkdocs": "https://openenergyplatform.github.io/oeplatform/",
     "compendium": "https://openenergyplatform.github.io/organisation/",
+    "spdx_licenses": "https://spdx.github.io/license-list-data/",
 }
 
 


Test patch: