Repository: ROVI-org/battery-data-toolkit (language: python)

Issue:
Cross-Reference Metadata Terms
There are at least 4 sources of metadata terms in the ROVI project:

1. Here
2. batterydata.energy.gov
3. BatteryArchive
4. The new documents being created by the data team

We should start cross-referencing these terms first against [BattINFO](https://big-map.github.io/BattINFO/index.html) and then against each other

- [ ] Create a spreadsheet defining the terms
- [ ] Gather group consensus on which terms to select
- [ ] Replace the terms in batdata's schema
Metadata fields related to synthetic data
Track which simulation code was used, etc


Discussion / hints:

Fix patch:
diff --git a/batdata/schemas/__init__.py b/batdata/schemas/__init__.py
index eb34ccd..cc3288d 100644
--- a/batdata/schemas/__init__.py
+++ b/batdata/schemas/__init__.py
@@ -2,13 +2,14 @@
 from datetime import date
 from typing import List, Tuple, Optional, Dict
 
-from pydantic import BaseModel, Field, AnyUrl, Extra
+from pydantic import BaseModel, Field, AnyUrl
 
+from batdata.schemas.modeling import ModelMetadata
 from batdata.schemas.battery import BatteryDescription
 from batdata.version import __version__
 
 
-class BatteryMetadata(BaseModel, extra=Extra.allow):
+class BatteryMetadata(BaseModel, extra='allow'):
     """Representation for the metadata about a battery
 
     The metadata captures the information about what experiment was run
@@ -20,7 +21,9 @@ class BatteryMetadata(BaseModel, extra=Extra.allow):
     name: Optional[str] = Field(None, description="Name of the cell. Any format for the name is acceptable,"
                                                   " as it is intended to be used by the battery data provider.")
     comments: Optional[str] = Field(None, description="Long form comments describing the test")
-    version: str = Field(__version__, description="Version of this metadata")
+    version: str = Field(__version__, description="Version of this metadata. Set by the battery-data-toolkit")
+    is_measurement: bool = Field(True, description="Whether the data was created observationally as opposed to a computer simulation",
+                                 iri="https://w3id.org/emmo#EMMO_463bcfda_867b_41d9_a967_211d4d437cfb")
 
     # Fields that describe the test protocol
     cycler: Optional[str] = Field(None, description='Name of the cycling machine')
@@ -31,6 +34,9 @@ class BatteryMetadata(BaseModel, extra=Extra.allow):
     # Field that describe the battery assembly
     battery: Optional[BatteryDescription] = Field(None, description="Description of the battery being tested")
 
+    # Fields that describe source of synthetic data
+    modeling: Optional[ModelMetadata] = Field(None, description="Description of simulation approach")
+
     # Fields that describe the source of data
     source: Optional[str] = Field(None, description="Organization who created this data")
     dataset_name: Optional[str] = Field(None, description="Name of a larger dataset this data is associated with")
diff --git a/batdata/schemas/battery.py b/batdata/schemas/battery.py
index 399b3a7..febc090 100644
--- a/batdata/schemas/battery.py
+++ b/batdata/schemas/battery.py
@@ -1,10 +1,10 @@
 """Schemas associated with the components of a battery"""
 from typing import Optional, List
 
-from pydantic import BaseModel, Field, Extra
+from pydantic import BaseModel, Field
 
 
-class ElectrodeDescription(BaseModel, extra=Extra.allow):
+class ElectrodeDescription(BaseModel, extra='allow'):
     """Description of an electrode"""
 
     name: str = Field(..., description='Short description of the electrolyte type')
@@ -20,7 +20,7 @@ class ElectrodeDescription(BaseModel, extra=Extra.allow):
     porosity: Optional[float] = Field(None, description='Relative volume of the electrode occupied by gas (units: %)', ge=0, le=100)
 
 
-class ElectrolyteAdditive(BaseModel, extra=Extra.allow):
+class ElectrolyteAdditive(BaseModel, extra='allow'):
     """Additive to the electrolyte"""
 
     name: str = Field(..., description='Name of the additive')
@@ -28,14 +28,14 @@ class ElectrolyteAdditive(BaseModel, extra=Extra.allow):
     units: Optional[float] = Field(None, description='Units of the amount')
 
 
-class ElectrolyteDescription(BaseModel, extra=Extra.allow):
+class ElectrolyteDescription(BaseModel, extra='allow'):
     """Description of the electrolyte"""
 
     name: str = Field(..., description='Short description of the electrolyte types')
     additives: List[ElectrolyteAdditive] = Field(default_factory=list, help='Any additives present in the electrolyte')
 
 
-class BatteryDescription(BaseModel, extra=Extra.allow):
+class BatteryDescription(BaseModel, extra='allow'):
     """Description of the entire battery"""
 
     # Overall design information
@@ -48,7 +48,8 @@ class BatteryDescription(BaseModel, extra=Extra.allow):
     # Materials description
     anode: Optional[ElectrodeDescription] = Field(None, description="Name of the anode material")
     cathode: Optional[ElectrodeDescription] = Field(None, description="Name of the cathode material")
-    electrolyte: ElectrolyteDescription = Field(None, description="Name of the electrolyte material")
+    electrolyte: Optional[ElectrolyteDescription] = Field(None, description="Name of the electrolyte material")
 
     # Performance information
-    nominal_capacity: float = Field(None, description="Rated capacity of the battery. Units: A-hr")
+    nominal_capacity: float = Field(None, description="Rated capacity of the battery. Units: A-hr",
+                                    iri="https://w3id.org/emmo/domain/electrochemistry#electrochemistry_9b3b4668_0795_4a35_9965_2af383497a26")
diff --git a/batdata/schemas/modeling.py b/batdata/schemas/modeling.py
new file mode 100644
index 0000000..c10da28
--- /dev/null
+++ b/batdata/schemas/modeling.py
@@ -0,0 +1,44 @@
+"""Metadata which describes how data produced by models were generated"""
+from typing import Optional, List
+from enum import Enum
+
+from pydantic import BaseModel, Field, AnyUrl
+
+
+class ModelTypes(str, Enum):
+    """Type of computational method"""
+
+    physics = 'physics'
+    """A computational application that uses a physical model to predict the behaviour of a system,
+    providing a identifiable analogy with the original object.
+
+    IRI: https://w3id.org/emmo#EMMO_8d4962d7_9608_44f7_a2f1_82a4bb173f4a"""
+    data = 'data'
+    """A computational application that uses existing data to predict the behaviour of a system
+    without providing a identifiable analogy with the original object.
+
+    IRI: https://w3id.org/emmo#EMMO_a4b14b83_9392_4a5f_a2e8_b2b58793f59b"""
+
+    empirical = 'empirical'
+    """A computational application that uses an empiric equation to predict the behaviour of a system
+    without relying on the knowledge of the actual physical phenomena occurring in the object.
+
+    IRI: https://w3id.org/emmo#EMMO_67c70dcd_2adf_4e6c_b3f8_f33dd1512487"""
+
+
+class ModelMetadata(BaseModel, extra='allow'):
+    """Describe the type and version of a computational tool used to generate battery data"""
+
+    # High-level information about the code
+    name: str = Field(..., description='Name of the software')
+    version: Optional[str] = Field(..., description='Version of the software if known')
+    type: Optional[ModelTypes] = Field(None, description='Type of the computational method it implements.')
+    references: Optional[List[AnyUrl]] = Field(None, description='List of references associated with the software')
+
+    # Details for physics based simulation
+    models: Optional[List[str]] = Field(None, description='Type of mathematical model(s) being used in physics simulation.'
+                                                          'Use terms defined in BattINFO, such as "BatteryEquivalentCircuitModel".',
+                                        root_iri='https://w3id.org/emmo#EMMO_f7ed665b_c2e1_42bc_889b_6b42ed3a36f0')
+    simulation_type: Optional[str] = Field(None, description='Type of simulation being performed. '
+                                                             'Use terms defined in BattINFO, such as "TightlyCoupledModelsSimulation"',
+                                           root_iri='https://w3id.org/emmo#EMMO_e97af6ec_4371_4bbc_8936_34b76e33302f')
diff --git a/batdata/schemas/ontology.py b/batdata/schemas/ontology.py
new file mode 100644
index 0000000..fa097b1
--- /dev/null
+++ b/batdata/schemas/ontology.py
@@ -0,0 +1,102 @@
+"""Tools used for linking terms in our data format to the BattINFO ontology"""
+from dataclasses import dataclass, field
+from functools import cache
+from typing import Type, List, Optional, Union
+
+from ontopy import World
+from owlready2 import Thing
+from pydantic import BaseModel
+
+_battinfo_url = 'https://raw.githubusercontent.com/emmo-repo/domain-battery/master/battery-inferred.ttl'
+
+
+@cache
+def load_battinfo():
+    return World().get_ontology(_battinfo_url).load()
+
+
+@dataclass
+class TermInfo:
+    """Information about a term as referenced from the BattINFO ontology"""
+
+    name: str
+    """Name of the matching term"""
+    iri: str = field(repr=False)
+    """IRI of the term"""
+    elucidation: Optional[str] = field(repr=False)
+    """Explanation of the term"""
+
+    @classmethod
+    def from_thing(cls, thing: Thing):
+        # Retrieve the description, as provided by EMMO
+        eluc = thing.get_annotations().get('elucidation')
+        if eluc is not None:
+            eluc = str(eluc[0])
+        return TermInfo(name=str(thing), iri=thing.iri, elucidation=eluc)
+
+
+def cross_reference_terms(model: Type[BaseModel]) -> dict[str, TermInfo]:
+    """Gather the descriptions of fields from our schema which
+    are cross-referenced to a term within the BattINFO/EMMO ontologies
+
+    Args:
+        model: Schema object to be cross-referenced
+    Returns:
+        Mapping between metadata fields in elucidation field from the ontology
+    """
+
+    # Load the BattINFO ontology
+    battinfo = load_battinfo()
+
+    # Loop over each field in the schema
+    terms = {}
+    for name, attr in model.model_fields.items():
+        # Map to the term in the ontology if known
+        if attr.json_schema_extra is not None and (iri := attr.json_schema_extra.get('iri')) is not None:
+            term = battinfo.search_one(iri=iri)
+            if term is None:
+                raise ValueError(f'Count not find matching term for {name} with iri={iri}')
+            terms[name] = TermInfo.from_thing(term)
+
+    return terms
+
+
+def resolve_term(name_or_iri: str) -> Thing:
+    """Resolve the Term object associated with a string
+
+    Args:
+        name_or_iri: The preferred label or the IRI of a term in the ontology
+    Returns:
+        Thing matching the term
+    """
+
+    # Attempt to find it
+    bi = load_battinfo()
+    if name_or_iri.startswith('https://'):
+        term = bi.search_one(iri=name_or_iri)
+        t = 'IRI'
+    else:
+        term = bi.search_one(prefLabel=name_or_iri)
+        t = 'name'
+
+    if term is None:
+        raise ValueError(f'Could not find the {t}={name_or_iri}')
+    return term
+
+
+def gather_descendants(term: Union[Type[Thing], str]) -> List[TermInfo]:
+    """Get descriptions of the descendants of a certain base type
+
+    Args:
+        term: Term for which to gather all descendants. Either the class object itself or its preferred label or IRI
+    Returns:
+        List of descriptions of the descendants
+    """
+
+    # Resolve the term object, if needed
+    if isinstance(term, str):
+        term = resolve_term(term)
+
+    return [
+        TermInfo.from_thing(d) for d in term.descendants(include_self=False)
+    ]
diff --git a/docs/schemas.md b/docs/schemas.md
new file mode 100644
index 0000000..3896e53
--- /dev/null
+++ b/docs/schemas.md
@@ -0,0 +1,87 @@
+# Battery Data Schemas
+
+The metadata schemas used by `batdata` standardize how we describe the source of battery datasets.
+Metadata are held as part of the `BatteryDataset` object and saved within the file formats
+produced by `batdata` to ensure that the provenance of a dataset is kept alongside the actual data.
+
+
+## Understanding the Metadata
+
+The metadata we employ in `batdata` follows the style of the JSON or XML data structures which are ubiquitous 
+in scientific computation and data infrastructure.
+
+We recommend creating the metadata for a battery through the Python interface. 
+Start by creating a `BatteryMetadata` object. There are no required fields, but you should always give your data a name.
+
+```python
+from batdata.schemas import BatteryMetadata
+
+metadata = BatteryMetadata(
+    name='test-cell',
+)
+```
+
+The metadata is a nested document where different types of information are grouped together into sub objects.
+For example, the details about the battery being tested are in `BatteryDescription`
+
+```python
+from batdata.schemas.battery import BatteryDescription
+from batdata.schemas import BatteryMetadata
+
+metadata = BatteryMetadata(
+    name='test-cell',
+    battery=BatteryDescription(
+        manufacturer='famous',
+        nominal_capacity=1.,
+    )
+)
+```
+
+See the [schemas module](https://github.com/ROVI-org/battery-data-toolkit/tree/main/batdata/schemas)
+for a full accounting of the available fields in our schema.
+
+> TODO: Render the schemas into an easier-to-read format
+
+Feel free to add your own fields to any part fo the schema. 
+The schema is a continual work in progress and the battery-data-toolkit will 
+store your new fields.
+Consider adding [an Issue](https://github.com/ROVI-org/battery-data-toolkit/issues) to our GitHub 
+if you find you use a term enough that we should add it to the schema.
+
+### Source of Terminology
+
+We use terms from [BattINFO ontology](https://big-map.github.io/BattINFO/index.html) wherever possible.
+
+Fields in the schema whose names correspond to a BattINFO term are marked
+with the "IRI" of the field, which points to a website containing the description.
+
+Fields whose values should be terms from the BattINFO ontology are marked with the root of the terms.
+For example, the `model_type` field of `ModelMetadata` can be any type of
+[MathematicalModel](https://emmo-repo.github.io/emmo.html#EMMO_f7ed665b_c2e1_42bc_889b_6b42ed3a36f0).
+Look them up using some utilities in `batdata`.
+
+```python
+from batdata.schemas.ontology import gather_descendants
+
+print(gather_descendants('MathematicalModel'))
+```
+
+> TODO: Render the options in web-hosted documentation as well
+
+## Column Datasets
+
+The columns of datasets are described in the [cycling module](https://github.com/ROVI-org/battery-data-toolkit/blob/main/batdata/schemas/cycling.py).
+
+Use the descriptions here when formatting your dataset, playing attention to the sign conventions and units for each column.
+
+Record columns that are not defined in our schema in the `*_columns` fields 
+of the `BatteryMetadata`.
+
+```python
+from batdata.schemas import BatteryMetadata
+
+metadata = BatteryMetadata(
+    name='test_cell',
+    raw_data_columns={'new_signal': 'A column not yet defined in our schemas.'}
+)
+```
diff --git a/pyproject.toml b/pyproject.toml
index 4fd9bf2..42e7656 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -19,6 +19,7 @@ dependencies = [
     "h5py == 3.*",
     "scythe-extractors >= 0.1",
     "pyarrow >= 15",
+    "EMMOntoPy",
     "xlrd"
 ]
 


Test patch:
diff --git a/tests/schemas/test_ontology.py b/tests/schemas/test_ontology.py
new file mode 100644
index 0000000..35aeaae
--- /dev/null
+++ b/tests/schemas/test_ontology.py
@@ -0,0 +1,26 @@
+"""Test the ability to resolve cross-references from the ontology"""
+
+from batdata.schemas import BatteryMetadata
+from batdata.schemas.ontology import cross_reference_terms, gather_descendants, load_battinfo, resolve_term
+
+
+def test_crossref():
+    terms = cross_reference_terms(BatteryMetadata)
+    assert 'is_measurement' in terms
+    assert terms['is_measurement'].name == 'emmo.Measurement'
+    assert 'EMMO' in terms['is_measurement'].iri
+    assert 'well defined mesurement procedure.' in terms['is_measurement'].elucidation
+
+
+def test_resolve():
+    assert resolve_term('PhysicsBasedSimulation') is not None
+    assert resolve_term('https://w3id.org/emmo#EMMO_f7ed665b_c2e1_42bc_889b_6b42ed3a36f0') is not None
+
+
+def test_descendants():
+    bi = load_battinfo()
+    desc = [t.name for t in gather_descendants(bi.PhysicsBasedSimulation)]
+    assert 'emmo.StandaloneModelSimulation' in desc
+
+    desc = [t.name for t in gather_descendants('PhysicsBasedSimulation')]
+    assert 'emmo.StandaloneModelSimulation' in desc