Repository: ROVI-org/battery-data-toolkit (language: python)

Issue:
Write Datasets to Battery Archive Format
After discussing with @ypreger , the next step in integration between this toolkit and the data infrastructure will be the ability to read/write data into Battery Archive format.

 That format is described in an [sql schema file](https://github.com/battery-lcf/batteryarchive-agent/blob/main/data/ba_data_schema.sql) and the importer tends to read files. 

The general theme of the output will be to:
- write different CSV files for each type of data (metadata, timeseries, cycle stats)
- convert the names of columns (though I hope that will not be necessary, and that we can adapt our schema)
- chunk larger files into small enough to ingest into the system without much issue


Discussion / hints:

Fix patch:
diff --git a/batdata/exporters/__init__.py b/batdata/exporters/__init__.py
new file mode 100644
index 0000000..f908c70
--- /dev/null
+++ b/batdata/exporters/__init__.py
@@ -0,0 +1,1 @@
+"""Tools for exporting battery data into a format usable by other tools"""
diff --git a/batdata/exporters/ba.py b/batdata/exporters/ba.py
new file mode 100644
index 0000000..66833d9
--- /dev/null
+++ b/batdata/exporters/ba.py
@@ -0,0 +1,125 @@
+"""Tools for streamlining upload to `Battery Archive <https://batteryarchive.org/>`_"""
+
+from typing import Callable, Any, Optional
+from dataclasses import dataclass
+from datetime import datetime
+from pathlib import Path
+from uuid import uuid4
+import logging
+import json
+
+import numpy as np
+import pandas as pd
+
+from .base import DatasetExporter
+from ..data import BatteryDataset
+from ..schemas import BatteryMetadata
+
+logger = logging.getLogger(__name__)
+
+# Mappings between our column names and theirs, with an optional function to perform conversion
+# TODO (wardlt): Standardize fields for the cumulative charge and discharge for each cycle separately (#75)
+# TODO (wardlt): Differentiate the cell temperature from the environment temperature (#76)
+# TODO (wardlt): Compute more derived fields from BatteryArchive (#77)
+_timeseries_reference: dict[str, tuple[str, Optional[Callable[[Any], Any]]]] = {
+    'current': ('i', None),
+    'voltage': ('v', None),
+    'temperature': ('env_temperature', None),  # TODO (wardlt): @ypreger, would you prefer unknown temps as env or cell?
+    'time': ('date_time', lambda x: datetime.fromtimestamp(x).strftime('%m/%d/%Y %H:%M:%S.%f')),
+    'cycle_number': ('cycle_index', lambda x: x + 1),  # BA starts indices from 1
+    'test_time': ('test_time', None),
+}
+
+_battery_metadata_reference: dict[str, str] = {
+    'nominal_capacity': 'ah',  # TODO (wardlt): Why is ah an integer?
+    'form_factor': 'form_factor',
+    'mass': 'weight',  # TODO (wardlt): What units does batteryachive use?
+    'dimensions': 'dimensions',  # TODO (wardlt): How do you express shapes for different form factors
+}
+
+_metadata_reference: dict[str, str] = {
+    'source': 'source',
+}
+
+
+# TODO (wardlt): Reconsider saving in CSV. Parquet would preserve data types
+
+@dataclass
+class BatteryArchiveExporter(DatasetExporter):
+    """Export data into CSV files that follow the format definitions used in BatteryArchive
+
+    The exporter writes files for each table in the
+    `Battery Archive SQL schema <https://github.com/battery-lcf/batteryarchive-agent/blob/main/data/ba_data_schema.sql>`_
+    with column names matches to their definitions.
+    """
+
+    chunk_size: int = 100000
+    """Maximum number of rows to write to disk in a single CSV file"""
+
+    def write_timeseries(self, cell_id: str, data: pd.DataFrame, path: Path):
+        """Write the time series dataset
+
+        Args:
+            cell_id: Name for the cell, used as a foreign key to map between tables
+            data: Time series data to write to disk
+            path: Root path for writing cycling data
+        """
+
+        num_chunks = len(data) // self.chunk_size + 1
+        logger.info(f'Writing time series data to disk in {num_chunks} chunks')
+        for i, chunk in enumerate(np.array_split(data, num_chunks)):
+            # Convert all of our columns
+            out_chunk = pd.DataFrame()
+            for my_col, (out_col, out_fun) in _timeseries_reference.items():
+                if my_col in chunk:
+                    out_chunk[out_col] = chunk[my_col]
+                    if out_fun is not None:
+                        out_chunk[out_col] = out_chunk[out_col].apply(out_fun)
+
+            # Add a cell id to the frame
+            out_chunk['cell_id'] = cell_id
+
+            # Save to disk
+            chunk_path = path / f'cycle-timeseries-{i}.csv'
+            out_chunk.to_csv(chunk_path, index=False, encoding='utf-8')
+            logger.debug(f'Wrote {len(out_chunk)} rows to {chunk_path}')
+
+    def write_metadata(self, cell_id: str, metadata: BatteryMetadata, path: Path):
+        """Write the metadata into a JSON file
+
+        Args:
+            cell_id: ID for the cell
+            metadata: Metadata to be written
+            path: Path in which to write the data
+        """
+
+        output = {'cell_id': cell_id}
+
+        # Write the materials for the anode and cathode as dictionaries
+        for terminal in ['anode', 'cathode']:
+            attr = getattr(metadata.battery, terminal, None)
+            if attr is not None:
+                output[terminal] = attr.model_dump_json(exclude_unset=True)
+
+        # Write the simple fields about the batteries and tester
+        for my_field, ba_field in _battery_metadata_reference.items():
+            attr = getattr(metadata.battery, my_field, None)
+            if attr is not None:
+                output[ba_field] = attr
+
+        for my_field, ba_field in _metadata_reference.items():
+            attr = getattr(metadata, my_field, None)
+            if attr is not None:
+                output[ba_field] = attr
+
+        with open(path / 'metadata.json', 'w') as fp:
+            json.dump(output, fp)
+
+    def export(self, dataset: BatteryDataset, path: Path):
+        cell_name = dataset.metadata.name or str(uuid4())  # Default to UUID if none provided
+
+        if dataset.raw_data is not None:
+            self.write_timeseries(cell_name, dataset.raw_data, path)
+
+        if dataset.metadata is not None:
+            self.write_metadata(cell_name, dataset.metadata, path)
diff --git a/batdata/exporters/base.py b/batdata/exporters/base.py
new file mode 100644
index 0000000..0a12f6f
--- /dev/null
+++ b/batdata/exporters/base.py
@@ -0,0 +1,23 @@
+"""Interface definitions"""
+from pathlib import Path
+
+from batdata.data import BatteryDataset
+
+
+class DatasetExporter:
+    """Tool which exports data from a :class:`~batdata.data.BatteryDataset` to disk in a specific format
+
+    Implementing an Exporter
+    ------------------------
+    """
+
+    def export(self, dataset: BatteryDataset, path: Path):
+        """Write the dataset to disk in a specific path
+
+        All files from the dataset must be placed in the provided directory
+
+        Args:
+            dataset: Dataset to be exported
+            path: Output path
+        """
+        raise NotImplementedError()
diff --git a/batdata/schemas/battery.py b/batdata/schemas/battery.py
index 13d5775..320c8fc 100644
--- a/batdata/schemas/battery.py
+++ b/batdata/schemas/battery.py
@@ -17,7 +17,8 @@ class ElectrodeDescription(BaseModel, extra='allow'):
     thickness: Optional[float] = Field(None, description='Thickness of the material (units: μm)', ge=0)
     area: Optional[float] = Field(None, description='Total area of the electrode (units: cm2)', ge=0)
     loading: Optional[float] = Field(None, description='Amount of active material per area (units: mg/cm^2)', ge=0)
-    porosity: Optional[float] = Field(None, description='Relative volume of the electrode occupied by gas (units: %)', ge=0, le=100)
+    porosity: Optional[float] = Field(None, description='Relative volume of the electrode occupied by gas (units: %)',
+                                      ge=0, le=100)
 
 
 class ElectrolyteAdditive(BaseModel, extra='allow'):
@@ -32,7 +33,8 @@ class ElectrolyteDescription(BaseModel, extra='allow'):
     """Description of the electrolyte"""
 
     name: str = Field(..., description='Short description of the electrolyte types')
-    additives: List[ElectrolyteAdditive] = Field(default_factory=list, help='Any additives present in the electrolyte')
+    additives: List[ElectrolyteAdditive] = Field(default_factory=list,
+                                                 description='Any additives present in the electrolyte')
 
 
 class BatteryDescription(BaseModel, extra='allow'):
@@ -44,11 +46,19 @@ class BatteryDescription(BaseModel, extra='allow'):
 
     # Geometry information
     layer_count: Optional[int] = Field(None, description="Number of layers within the battery", gt=1)
+    form_factor: Optional[str] = Field(None, description="The general shape of the battery",
+                                       iri="https://w3id.org/emmo/domain/electrochemistry#electrochemistry_1586ef26_6d30_49e3_ae32_b4c9fc181941")
+    mass: Optional[float] = Field(None, description="Mass of the entire battery. Units: kg")
+    dimensions: Optional[List[float]] = Field(None, description='Dimensions of the battery in plain text.')
 
     # Materials description
-    anode: Optional[ElectrodeDescription] = Field(None, description="Name of the anode material")
-    cathode: Optional[ElectrodeDescription] = Field(None, description="Name of the cathode material")
-    electrolyte: Optional[ElectrolyteDescription] = Field(None, description="Name of the electrolyte material")
+    anode: Optional[ElectrodeDescription] = Field(None, description="Name of the anode material",
+                                                  iri="https://w3id.org/emmo/domain/electrochemistry#electrochemistry_b6319c74_d2ce_48c0_a75a_63156776b302")
+    cathode: Optional[ElectrodeDescription] = Field(None, description="Name of the cathode material",
+                                                    iri="https://w3id.org/emmo/domain/electrochemistry#electrochemistry_35c650ab_3b23_4938_b312_1b0dede2e6d5")
+    electrolyte: Optional[ElectrolyteDescription] = Field(None, description="Name of the electrolyte material",
+                                                          iri=("https://w3id.org/emmo/domain/electrochemistry"
+                                                               "#electrochemistry_fb0d9eef_92af_4628_8814_e065ca255d59"))
 
     # Performance information
     nominal_capacity: float = Field(None, description="Rated capacity of the battery. Units: A-hr",
diff --git a/batdata/schemas/cycling.py b/batdata/schemas/cycling.py
index e0e035b..593fe7c 100644
--- a/batdata/schemas/cycling.py
+++ b/batdata/schemas/cycling.py
@@ -127,7 +127,7 @@ class RawData(ColumnSchema):
     a common definition.
     """
 
-    cycle_number: List[int] = Field(None, description="Index of the testing cycle. All indices should be"
+    cycle_number: List[int] = Field(None, description="Index of the testing cycle, starting at 0. All indices should be"
                                                       " nonnegative and be monotonically increasing", monotonic=True)
     step_index: List[int] = Field(None, description="Index of the step number within a testing cycle. A step change"
                                                     " is defined by a change states between charging, discharging,"
diff --git a/docs/export.md b/docs/export.md
new file mode 100644
index 0000000..0ca4aa7
--- /dev/null
+++ b/docs/export.md
@@ -0,0 +1,29 @@
+# Exporting Data to Other Tools
+
+The `batdata.exporters` modules provides tools for writing the data from battery-data-toolkit's internal format
+into files suitable for use in other tools.
+
+All exporter interfaces provide a `export` function which takes the dataset to be written
+and a path to the directory in which to store the files.
+The exporter objects themselves take arguments which control how all dataset
+being exported are written, such as the number of rows per file.
+
+```python
+
+from batdata.exporters.ba import BatteryArchiveExporter
+
+exporter = BatteryArchiveExporter(chunk_size=1000000)
+exporter.export(dataset, 'output_dir')
+
+```
+
+> NOTE: We assume each dataset will be stored in its own directory, but that's not a decision we won't revisit.
+
+## BatteryArchive
+
+The [BatteryArchive](https://batteryarchive.org/) project stores battery data in an SQL database.
+Our exporter does not connect to this database and, instead, writes each battery dataset into 
+files with column names that match [BatteryArchive's SQL schema](https://github.com/battery-lcf/batteryarchive-agent/blob/main/data/ba_data_schema.sql)
+so that adding them to the archive is simple.
+
+> TODO: Write how we convert metadata 
\ No newline at end of file


Test patch:
diff --git a/tests/conftest.py b/tests/conftest.py
index 7530368..61b0f55 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -2,8 +2,28 @@
 
 from pytest import fixture
 
+from batdata.data import BatteryDataset, BatteryMetadata
+from batdata.postprocess.cycle_stats import CycleTimes
+
 
 @fixture()
 def file_path() -> Path:
     """Path to test-related files"""
     return Path(__file__).parent / 'files'
+
+
+@fixture()
+def example_data(file_path) -> BatteryDataset:
+    """An example dataset which contains metadata and a few cycles of data"""
+
+    # Load the simple cycling
+    path = file_path / 'example-data' / 'single-resistor-constant-charge_from-discharged.hdf'
+    data = BatteryDataset.from_batdata_hdf(path)
+
+    # Compute basic cycling states
+    for stats in [CycleTimes()]:
+        stats.compute_features(data)
+
+    # Give the cell a name, at least
+    data.metadata = BatteryMetadata(name='test')
+    return data
diff --git a/tests/exporters/test_ba.py b/tests/exporters/test_ba.py
new file mode 100644
index 0000000..791207a
--- /dev/null
+++ b/tests/exporters/test_ba.py
@@ -0,0 +1,40 @@
+from pathlib import Path
+from datetime import datetime
+import json
+
+import pandas as pd
+
+from batdata.exporters.ba import BatteryArchiveExporter
+from batdata.schemas import BatteryMetadata, BatteryDescription
+from batdata.schemas.battery import ElectrodeDescription
+
+
+def test_export(example_data):
+    # Add a datetime
+    example_data.raw_data['time'] = example_data.raw_data['test_time'] + datetime(year=2024, month=7, day=1).timestamp()
+
+    # Add some metadata to the file
+    example_data.metadata = BatteryMetadata(
+        battery=BatteryDescription(
+            anode=ElectrodeDescription(name='graphite', supplier='big-one'),
+            cathode=ElectrodeDescription(name='nmc')
+        )
+    )
+
+    tmpdir = Path('test')
+    tmpdir.mkdir(exist_ok=True)
+    exporter = BatteryArchiveExporter()
+    exporter.export(example_data, tmpdir)
+
+    # Make sure the time series loaded correctly
+    timeseries_path = tmpdir.joinpath('cycle-timeseries-0.csv')
+    assert timeseries_path.is_file()
+    timeseries = pd.read_csv(timeseries_path)
+    assert 'v' in timeseries  # Make sure a conversion occurred correctly
+    assert 'cell_id' in timeseries
+    assert timeseries['date_time'].iloc[0] == '07/01/2024 00:00:00.000000'
+    assert timeseries['cycle_index'].iloc[1] == 1
+
+    # Check that metadata was written
+    metadata = json.loads(tmpdir.joinpath('metadata.json').read_text())
+    assert metadata['cathode'] == '{"name":"nmc"}'