Repository: inveniosoftware/invenio-rdm-records (language: python)

Issue:
Implement Software Heritage (Code Meta) fields as custom fields
Implement the following fields from [Code Meta:](https://codemeta.github.io/terms/)

- [x] codeRepository
- [x] programmingLanguage
- [x] runtimePlatform
- [x] developmentStatus
- [x] operatingSystem


Discussion / hints:

Fix patch:
diff --git a/invenio_rdm_records/contrib/__init__.py b/invenio_rdm_records/contrib/__init__.py
new file mode 100644
index 000000000..b5ee244e3
--- /dev/null
+++ b/invenio_rdm_records/contrib/__init__.py
@@ -0,0 +1,8 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2023 CERN.
+#
+# Invenio-RDM-Records is free software; you can redistribute it and/or modify
+# it under the terms of the MIT License; see LICENSE file for more details.
+
+"""Contrib module."""
diff --git a/invenio_rdm_records/contrib/codemeta/__init__.py b/invenio_rdm_records/contrib/codemeta/__init__.py
new file mode 100644
index 000000000..4dfdc9664
--- /dev/null
+++ b/invenio_rdm_records/contrib/codemeta/__init__.py
@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2023 CERN.
+#
+# Invenio-RDM-Records is free software; you can redistribute it and/or modify
+# it under the terms of the MIT License; see LICENSE file for more details.
+"""CodeMeta contrib module."""
+from .custom_fields import (
+    CODEMETA_CUSTOM_FIELDS,
+    CODEMETA_CUSTOM_FIELDS_UI,
+    CODEMETA_FACETS,
+    CODEMETA_NAMESPACE,
+)
+
+__all__ = (
+    CODEMETA_FACETS,
+    CODEMETA_CUSTOM_FIELDS,
+    CODEMETA_CUSTOM_FIELDS_UI,
+    CODEMETA_NAMESPACE,
+)
diff --git a/invenio_rdm_records/contrib/codemeta/custom_fields.py b/invenio_rdm_records/contrib/codemeta/custom_fields.py
new file mode 100644
index 000000000..213849b17
--- /dev/null
+++ b/invenio_rdm_records/contrib/codemeta/custom_fields.py
@@ -0,0 +1,127 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2023 CERN.
+#
+# Invenio-RDM-Records is free software; you can redistribute it and/or modify
+# it under the terms of the MIT License; see LICENSE file for more details.
+"""CodeMeta specific custom fields.
+
+Implements the following fields:
+- Repository URL
+- Programming language
+- Runtime platform
+- Operating system
+- Development status
+"""
+
+from flask_babelex import _
+from invenio_records_resources.services.custom_fields import KeywordCF
+from invenio_records_resources.services.records.facets import CFTermsFacet
+from invenio_vocabularies.services.custom_fields import VocabularyCF
+from invenio_vocabularies.services.facets import VocabularyLabels
+from marshmallow import validate
+
+CODEMETA_NAMESPACE = {
+    # CodeMeta
+    "code": "https://codemeta.github.io",
+}
+
+
+CODEMETA_CUSTOM_FIELDS = [
+    KeywordCF(
+        name="code:codeRepository",
+        field_args={
+            "validate": validate.URL(),
+            "error_messages": {"validate": "You must provide a valid URL."},
+        },
+    ),
+    KeywordCF(name="code:programmingLanguage"),
+    KeywordCF(name="code:runtimePlatform"),
+    KeywordCF(name="code:operatingSystem"),
+    VocabularyCF(
+        name="code:developmentStatus",
+        vocabulary_id="code:developmentStatus",
+        dump_options=True,
+        multiple=False,
+    ),
+]
+
+CODEMETA_CUSTOM_FIELDS_UI = {
+    "section": _("CodeMeta"),
+    "fields": [
+        dict(
+            field="code:codeRepository",
+            ui_widget="Input",
+            props=dict(
+                label="Repository URL",
+                icon="linkify",
+                description="URL or link where the code repository is hosted.",
+            ),
+        ),
+        dict(
+            field="code:programmingLanguage",
+            ui_widget="Input",
+            props=dict(
+                label="Programming language",
+                icon="code",
+                description="Repository's programming language.",
+                placeholder="Python ...",
+            ),
+        ),
+        dict(
+            field="code:runtimePlatform",
+            ui_widget="Input",
+            props=dict(
+                label="Runtime platform",
+                icon="cog",
+                description="Repository runtime platform or script interpreter dependencies.",
+                placeholder="Java v1, Python2.3, .Net Framework 3.0 ...",
+            ),
+        ),
+        dict(
+            field="code:operatingSystem",
+            ui_widget="Input",
+            props=dict(
+                label="Supported operating system",
+                icon="desktop",
+                description="Supported operating systems.",
+                placeholder="Windows 7, OSX 10.6, Android 1.6 ...",
+            ),
+        ),
+        dict(
+            field="code:developmentStatus",
+            ui_widget="Dropdown",
+            props=dict(
+                label="Development Status",
+                placeholder="Repository status",
+                icon="heartbeat",
+                description="Repository current status.",
+                search=False,
+                multiple=False,
+                clearable=True,
+            ),
+        ),
+    ],
+}
+
+CODEMETA_FACETS = {
+    "developmentStatus": {
+        "facet": CFTermsFacet(
+            field="code:developmentStatus.id",
+            label=_("Development status"),
+            value_labels=VocabularyLabels("code:developmentStatus"),
+        ),
+        "ui": {  # ui display
+            "field": CFTermsFacet.field("code:developmentStatus.id"),
+        },
+    },
+    "programmingLanguage": {
+        "facet": CFTermsFacet(
+            field="code:programmingLanguage",
+            label=_("Programming language"),
+        ),
+        "ui": {  # ui display
+            "field": CFTermsFacet.field("code:programmingLanguage"),
+        },
+    },
+}
diff --git a/invenio_rdm_records/fixtures/data/vocabularies.yaml b/invenio_rdm_records/fixtures/data/vocabularies.yaml
index c82896210..0a7596b12 100644
--- a/invenio_rdm_records/fixtures/data/vocabularies.yaml
+++ b/invenio_rdm_records/fixtures/data/vocabularies.yaml
@@ -49,3 +49,7 @@ funders:
 awards:
   pid-type: awa
   data-file: vocabularies/awards.yaml
+# code meta controlled vocabularies
+code:developmentStatus:
+  pid-type: ds
+  data-file: vocabularies/contrib/codemeta/development_status.yaml
diff --git a/invenio_rdm_records/fixtures/data/vocabularies/contrib/codemeta/development_status.yaml b/invenio_rdm_records/fixtures/data/vocabularies/contrib/codemeta/development_status.yaml
new file mode 100644
index 000000000..616594480
--- /dev/null
+++ b/invenio_rdm_records/fixtures/data/vocabularies/contrib/codemeta/development_status.yaml
@@ -0,0 +1,42 @@
+# The following statuses were retrieved from www.repostatus.org, as pointed by code meta (https://codemeta.github.io/terms/).
+
+- id: "concept"
+  title:
+    en: "Concept"
+  description:
+    en: "Minimal or no implementation has been done yet, or the repository is only intended to be a limited example, demo, or proof-of-concept."
+- id: "wip"
+  title:
+    en: "Wip"
+  description:
+    en: "Initial development is in progress, but there has not yet been a stable, usable release suitable for the public."
+- id: "suspended"
+  title:
+    en: "Suspended"
+  description:
+    en: "Initial development has started, but there has not yet been a stable, usable release; work has been stopped for the time being but the author(s) intend on resuming work."
+- id: "abandoned"
+  title:
+    en: "Abandoned"
+  description:
+    en: "Initial development has started, but there has not yet been a stable, usable release; the project has been abandoned and the author(s) do not intend on continuing development."
+- id: "active"
+  title:
+    en: "Active"
+  description:
+    en: "The project has reached a stable, usable state and is being actively developed."
+- id: "inactive"
+  title:
+    en: "Inactive"
+  description:
+    en: "The project has reacaed a stable, usable state but is no longer being actively developed; support/maintenance will be provided as time allows."
+- id: "unsupported"
+  title:
+    en: "Unsupported"
+  description:
+    en: "The project has reached a stable, usable state but the author(s) have ceased all work on it. A new maintainer may be desired."
+- id: "moved"
+  title:
+    en: "Moved"
+  description:
+    en: "The project has been moved to a new location, and the version at that location should be considered authoritative. This status should be accompanied by a new URL."


Test patch:
diff --git a/tests/conftest.py b/tests/conftest.py
index 6c9100158..8468f2a7a 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -1431,3 +1431,13 @@ def curator(UserFixture, community, app, db):
         system_identity, community.id, invitation_data
     )
     return curator
+
+
+@pytest.fixture(scope="module")
+def cli_runner(base_app):
+    """Create a CLI runner for testing a CLI command."""
+
+    def cli_invoke(command, *args, input=None):
+        return base_app.test_cli_runner().invoke(command, args, input=input)
+
+    return cli_invoke
diff --git a/tests/contrib/codemeta/conftest.py b/tests/contrib/codemeta/conftest.py
new file mode 100644
index 000000000..34a92b2ba
--- /dev/null
+++ b/tests/contrib/codemeta/conftest.py
@@ -0,0 +1,159 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2023 CERN.
+#
+# Invenio-RDM-Records is free software; you can redistribute it and/or modify
+# it under the terms of the MIT License; see LICENSE file for more details.
+"""CodeMeta specific test configs."""
+import pytest
+from invenio_access.permissions import system_identity
+from invenio_vocabularies.proxies import current_service as vocabulary_service
+from invenio_vocabularies.records.api import Vocabulary
+
+from invenio_rdm_records.cli import (
+    create_records_custom_field,
+    custom_field_exists_in_records,
+)
+from invenio_rdm_records.contrib.codemeta.custom_fields import (
+    CODEMETA_CUSTOM_FIELDS,
+    CODEMETA_FACETS,
+    CODEMETA_NAMESPACE,
+)
+from invenio_rdm_records.proxies import current_rdm_records_service
+from invenio_rdm_records.records.api import RDMDraft, RDMRecord
+
+
+@pytest.fixture(scope="module")
+def custom_field_exists(cli_runner):
+    """Factory fixture, tests whether a given custom field exists."""
+
+    def _custom_field_exists(cf_name):
+        return cli_runner(custom_field_exists_in_records, "-f", cf_name)
+
+    return _custom_field_exists
+
+
+@pytest.fixture(scope="function")
+def initialise_custom_fields(app, location, db, search_clear, cli_runner):
+    """Fixture initialises custom fields."""
+    return cli_runner(create_records_custom_field)
+
+
+@pytest.fixture(scope="function")
+def initialise_codemeta_custom_fields(
+    cli_runner, app_codemeta_config, search_clear, db, location, running_app
+):
+    """Initialised codemeta custom fields."""
+    app = running_app.app
+    original_config = app.config.copy()
+    app.config = {**app.config, **app_codemeta_config}
+    yield cli_runner(create_records_custom_field)
+    app.config = original_config
+
+
+@pytest.fixture(scope="session")
+def app_codemeta_config(
+    codemeta_cf_facetted_term,
+):
+    """Yields the app config fixture with codemeta CF specific configs injected."""
+    code_meta_configs = {
+        "RDM_CUSTOM_FIELDS": CODEMETA_CUSTOM_FIELDS,
+        "RDM_NAMESPACES": CODEMETA_NAMESPACE,
+        "RDM_FACETS": CODEMETA_FACETS,
+        "RDM_SEARCH": {
+            "facets": [codemeta_cf_facetted_term],
+        },
+    }
+    return code_meta_configs
+
+
+@pytest.fixture(scope="module")
+def service():
+    """Rdm records service instance."""
+    return current_rdm_records_service
+
+
+@pytest.fixture(scope="function")
+def codemeta_development_status(running_app):
+    """Creates and retrieves a vocabulary type."""
+    v = vocabulary_service.create_type(system_identity, "code:developmentStatus", "ds")
+    return v
+
+
+@pytest.fixture(scope="session")
+def codemeta_development_status_vocabulary_data():
+    """Fixture returns 'codemeta:developmentStatus' vocabulary data."""
+    return {
+        "id": "concept",
+        "title": {
+            "en": "Concept",
+        },
+        "description": {
+            "en": "Minimal or no implementation has been done yet, or the repository is only intended to be a limited example, demo, or proof-of-concept."
+        },
+        "type": "code:developmentStatus",
+    }
+
+
+@pytest.fixture(scope="function")
+def codemeta_development_status_vocabulary(
+    codemeta_development_status,
+    codemeta_development_status_vocabulary_data,
+):
+    """Creates and retrieves controlled vocabulary 'development status'."""
+    record = vocabulary_service.create(
+        identity=system_identity,
+        data=codemeta_development_status_vocabulary_data,
+    )
+    Vocabulary.index.refresh()  # Refresh the index
+
+    return record
+
+
+@pytest.fixture(scope="session")
+def codemeta_cf_name():
+    """Example of a codemeta custom field name."""
+    return "code:developmentStatus"
+
+
+@pytest.fixture(scope="session")
+def codemeta_cf_value():
+    """Example of a codemeta custom field value (from controlled vocabulary)."""
+    return "concept"
+
+
+@pytest.fixture(scope="session")
+def codemeta_cf_facetted_term():
+    """Example of a codemeta custom field term that is used as a facet term."""
+    return "developmentStatus"
+
+
+@pytest.fixture(scope="function")
+def minimal_codemeta_record(minimal_record, codemeta_cf_name, codemeta_cf_value):
+    """Represents a record containing codemeta fields (custom fields)."""
+    return {
+        **minimal_record,
+        "custom_fields": {codemeta_cf_name: {"id": codemeta_cf_value}},
+    }
+
+
+@pytest.fixture(scope="function")
+def codemeta_record(
+    codemeta_development_status_vocabulary,
+    db,
+    minimal_codemeta_record,
+    service,
+    search_clear,
+    superuser_identity,
+):
+    """Creates a record with codemeta custom fields added.
+
+    The record is teared down with the db fixture, when the session is destroyed.
+    Record's indexed document is deleted by the fixture "search_clear".
+    """
+    draft = service.create(superuser_identity, minimal_codemeta_record)
+    record = service.publish(id_=draft.id, identity=superuser_identity)
+    RDMRecord.index.refresh()
+    return record
+    # Search teardown is taken care by 'search_clear' fixture
+    # DB teardown is taken care by 'db' fixture
diff --git a/tests/contrib/codemeta/test_codemeta_custom_fields.py b/tests/contrib/codemeta/test_codemeta_custom_fields.py
new file mode 100644
index 000000000..180840515
--- /dev/null
+++ b/tests/contrib/codemeta/test_codemeta_custom_fields.py
@@ -0,0 +1,108 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2023 CERN.
+#
+# Invenio-RDM-Records is free software; you can redistribute it and/or modify
+# it under the terms of the MIT License; see LICENSE file for more details.
+"""CodeMeta custom fields tests."""
+
+from invenio_access.permissions import system_identity
+
+from invenio_rdm_records.records.api import RDMDraft, RDMRecord
+
+
+def test_not_loaded_by_default(
+    initialise_custom_fields, custom_field_exists, codemeta_cf_name, running_app
+):
+    """Tests that codemeta custom fields are not loaded by default."""
+    # By default, CFs are disabled. They must be manually added (e.g. per instance) and initialised.
+    result = custom_field_exists(codemeta_cf_name)
+    assert result.exit_code == 0
+    assert f"Field {codemeta_cf_name} does not exist" in result.output
+
+
+def test_load_on_demand(
+    custom_field_exists,
+    codemeta_cf_name,
+    initialise_codemeta_custom_fields,
+    running_app,
+):
+    """Tests whether codemeta CF are correctly loaded on demand.
+    This will test the validity of codemeta custom fields (e.g. whether they load without any errors).
+    It is not meant to test the custom fields loading mechanism."""
+    result = custom_field_exists(codemeta_cf_name)
+    assert result.exit_code == 0
+    assert f"Field {codemeta_cf_name} exists" in result.output
+
+
+def test_mappings(initialise_codemeta_custom_fields, codemeta_cf_name, running_app):
+    """Tests codemeta CF mappings after loading."""
+    # Validate custom fiels are mapped
+    record_mapping_field = list(RDMRecord.index.get_mapping().values())[0]["mappings"][
+        "properties"
+    ]["custom_fields"]
+
+    draft_mapping_field = list(RDMDraft.index.get_mapping().values())[0]["mappings"][
+        "properties"
+    ]["custom_fields"]
+
+    assert record_mapping_field["properties"].get(codemeta_cf_name)
+    assert draft_mapping_field["properties"].get(codemeta_cf_name)
+
+
+def test_record_creation_with_codemeta_fields(
+    initialise_codemeta_custom_fields,
+    codemeta_cf_name,
+    codemeta_cf_value,
+    codemeta_record,
+    db,
+    running_app,
+    service,
+):
+    """Tests the creation of a CodeMeta draft and record. Validates record indexing."""
+    # Test record
+    assert not codemeta_record._errors
+    assert codemeta_record.data.get("custom_fields") is not None
+    assert (
+        codemeta_record.data.get("custom_fields")[codemeta_cf_name]["id"]
+        == codemeta_cf_value
+    )
+
+    # Validate record was indexed with custom field
+    search_obj = service.search(system_identity)
+    assert search_obj is not None
+    assert search_obj.total > 0
+
+    document = list(search_obj.hits)[0]
+    assert document is not None
+    assert document.get("custom_fields") is not None
+    assert document.get("custom_fields")[codemeta_cf_name]["id"] == codemeta_cf_value
+
+
+def test_codemeta_facets(
+    initialise_codemeta_custom_fields,
+    codemeta_record,
+    service,
+    codemeta_cf_facetted_term,
+):
+    """Tests CodeMeta custom fields facets."""
+    # Intialize search.
+    search_obj = service.search(system_identity)
+
+    # Validate that search aggregations contain cf name.
+    assert search_obj.aggregations[codemeta_cf_facetted_term]
+
+
+def test_codemeta_fields_search(
+    initialise_codemeta_custom_fields,
+    codemeta_record,
+    codemeta_cf_facetted_term,
+    codemeta_cf_value,
+    service,
+):
+    """Tests a record search by a CodeMeta term."""
+    # Intialize search, pass a CodeMeta term as search parameter.
+    search_obj = service.search(
+        system_identity, params={codemeta_cf_facetted_term: codemeta_cf_value}
+    )
+    assert search_obj.total > 0
diff --git a/tests/fixtures/conftest.py b/tests/fixtures/conftest.py
index fa1be910b..77ddea767 100644
--- a/tests/fixtures/conftest.py
+++ b/tests/fixtures/conftest.py
@@ -47,16 +47,6 @@ def create_app(instance_path, entry_points):
     return _create_app
 
 
-@pytest.fixture(scope="module")
-def cli_runner(base_app):
-    """Create a CLI runner for testing a CLI command."""
-
-    def cli_invoke(command, *args, input=None):
-        return base_app.test_cli_runner().invoke(command, args, input=input)
-
-    return cli_invoke
-
-
 @pytest.fixture
 def vocabularies():
     """Load vocabularies."""