Repository: microsoft/Qcodes

Issue:
Sqlite file gets too large to work with, limiting portability
### Issue description 

While `sqlite` has some advantages for certain use cases, it is not easy to work with when updating data continuously and accessing data from a remote analysis computer; especially when synchronizing the file with a remote server or cloud blob storage. The `sqlite` file is great for transactional updates, but when long measurements take place the file can get quite large (multiple GB) and makes it difficult to synchronize as it is not possible to do so incrementally (afaik). It would be great to have an alternative option available to store data that is more portable.

### Feature suggestion

I'd like to add a feature that allows the user to export the data as a static file on disk at the end of a measurement and, for example, lets the user decide what storage format to use by setting an "export type" variable in the qcodes configuration. This way, I can synchronize the files with a cloud storage, and still have my sqlite database available locally to facilitate transactional updates during the measurements.

I've discussed this feature with @jenshnielsen who is planning a refactor of the `DataSet` class to allow for multilpe data back-ends as an alternative to sqlite; this feature would need to be aligned with or at least not interfere with that work. Since this is solely for static exports at the end of a measurement, not for updating or appending data to an existing file, we think it can live in parallel for the time being.

@jenshnielsen @astafan8 suggested to use netcdf as many QCoDeS users use xarray to store their data.


Discussion:

Resolving patch:
diff --git a/qcodes/dataset/data_set.py b/qcodes/dataset/data_set.py
--- a/qcodes/dataset/data_set.py
+++ b/qcodes/dataset/data_set.py
@@ -6,6 +6,7 @@
 import os
 import time
 import uuid
+
 from dataclasses import dataclass
 from queue import Empty, Queue
 from threading import Thread
@@ -14,6 +15,7 @@
                     Sized, Tuple, Union, cast)
 
 import numpy
+import pandas as pd
 
 import qcodes
 from qcodes.dataset.descriptions.dependencies import InterDependencies_
@@ -47,6 +49,7 @@
                                                  select_one_where)
 from qcodes.instrument.parameter import _BaseParameter
 from qcodes.utils.deprecate import deprecate
+from qcodes.dataset.export_config import DataExportType, get_data_export_type, get_data_export_path, get_data_export_prefix, get_data_export_automatic
 
 from .data_set_cache import DataSetCache
 from .descriptions.versioning import serialization as serial
@@ -336,6 +339,7 @@ def __init__(self, path_to_db: Optional[str] = None,
         self.cache: DataSetCache = DataSetCache(self)
         self._results: List[Dict[str, VALUE]] = []
         self._in_memory_cache = in_memory_cache
+        self._export_path: Optional[str] = None
 
         if run_id is not None:
             if not run_exists(self.conn, run_id):
@@ -1238,8 +1242,9 @@ def _load_to_xarray_dataset(self, data: ParameterData) -> "xr.Dataset":
             cast(Dict[Hashable, xr.DataArray], data_xrdarray_dict))
 
         for dim in xrdataset.dims:
-            paramspec_dict = self.paramspecs[str(dim)]._to_dict()
-            xrdataset.coords[str(dim)].attrs.update(paramspec_dict.items())
+            if "index" != dim:
+                paramspec_dict = self.paramspecs[str(dim)]._to_dict()
+                xrdataset.coords[str(dim)].attrs.update(paramspec_dict.items())
 
         xrdataset.attrs["sample_name"] = self.sample_name
         xrdataset.attrs["exp_name"] = self.exp_name
@@ -1360,11 +1365,13 @@ def write_data_to_text_file(self, path: str,
                 raise DataLengthException("You cannot concatenate data " +
                                           "with different length to a " +
                                           "single file.")
-            if single_file_name == None:
+            if single_file_name is None:
                 raise DataPathException("Please provide the desired file name " +
                                         "for the concatenated data.")
             else:
-                dst = os.path.join(path, f'{single_file_name}.dat')
+                if not single_file_name.lower().endswith(('.dat', '.csv', '.txt')):
+                    single_file_name = f'{single_file_name}.dat'
+                dst = os.path.join(path, single_file_name)
                 df_to_save = pd.concat(dfs_to_save, axis=1)
                 df_to_save.to_csv(path_or_buf=dst, header=False, sep='\t')
 
@@ -1693,6 +1700,94 @@ def _flush_data_to_database(self, block: bool = False) -> None:
             log.debug(f"Waiting for write queue to empty.")
             writer_status.data_write_queue.join()
 
+    def _export_file_name(self, prefix: str, export_type: DataExportType) -> str:
+        """Get export file name"""
+        extension = export_type.value
+        return f"{prefix}{self.run_id}.{extension}"
+
+    def _export_as_netcdf(self, path: str, file_name: str) -> str:
+        """Export data as netcdf to a given path with file prefix"""
+        file_path = os.path.join(path, file_name)
+        xarr_dataset = self.to_xarray_dataset()
+        xarr_dataset.to_netcdf(path=file_path)
+        return path
+
+    def _export_as_csv(self, path: str, file_name: str) -> str:
+        """Export data as csv to a given path with file prefix"""
+        self.write_data_to_text_file(path=path, single_file=True, single_file_name=file_name)
+        return os.path.join(path, file_name)
+
+    def _export_data(self,
+                     export_type: DataExportType,
+                     path: Optional[str] = None,
+                     prefix: Optional[str] = None
+                     ) -> Optional[str]:
+        """Export data to disk with file name {prefix}{run_id}.{ext}.
+        Values for the export type, path and prefix can also be set in the qcodes
+        "dataset" config.
+
+        Args:
+            export_type: Data export type, e.g. DataExportType.NETCDF
+            path: Export path, defaults to value set in config
+            prefix: File prefix, e.g. "qcodes_", defaults to value set in config.
+
+        Returns:
+            str: Path file was saved to, returns None if no file was saved.
+        """
+        # Set defaults to values in config if the value was not set
+        # (defaults to None)
+        path = path if path is not None else get_data_export_path()
+        prefix = prefix if prefix is not None else get_data_export_prefix()
+
+        if DataExportType.NETCDF == export_type:
+            file_name = self._export_file_name(
+                prefix=prefix, export_type=DataExportType.NETCDF)
+            return self._export_as_netcdf(path=path, file_name=file_name)
+
+        elif DataExportType.CSV == export_type:
+            file_name = self._export_file_name(
+                prefix=prefix, export_type=DataExportType.CSV)
+            return self._export_as_csv(path=path, file_name=file_name)
+
+        else:
+            return None
+
+    def export(self,
+               export_type: Optional[Union[DataExportType, str]] = None,
+               path: Optional[str] = None,
+               prefix: Optional[str] = None) -> None:
+        """Export data to disk with file name {prefix}{run_id}.{ext}.
+        Values for the export type, path and prefix can also be set in the "dataset"
+        section of qcodes config.
+
+        Args:
+            export_type: Data export type, e.g. "netcdf" or ``DataExportType.NETCDF``,
+                defaults to a value set in qcodes config
+            path: Export path, defaults to value set in config
+            prefix: File prefix, e.g. ``qcodes_``, defaults to value set in config.
+
+        Raises:
+            ValueError: If the export data type is not specified, raise an error
+        """
+        export_type = get_data_export_type(export_type)
+
+        if export_type is None:
+            raise ValueError(
+                "No data export type specified. Please set the export data type "
+                "by using ``qcodes.dataset.export_config.set_data_export_type`` or "
+                "give an explicit export_type when calling ``dataset.export`` manually."
+            )
+
+        self._export_path = self._export_data(
+            export_type=export_type,
+            path=path,
+            prefix=prefix
+        )
+
+    @property
+    def export_path(self) -> Optional[str]:
+        return self._export_path
+
 
 # public api
 def load_by_id(run_id: int, conn: Optional[ConnectionPlus] = None) -> DataSet:
diff --git a/qcodes/dataset/export_config.py b/qcodes/dataset/export_config.py
new file mode 100644
--- /dev/null
+++ b/qcodes/dataset/export_config.py
@@ -0,0 +1,114 @@
+import enum
+import logging
+
+from os.path import normpath, expanduser, exists
+from typing import Union, Optional
+
+from qcodes import config
+
+_log = logging.getLogger(__name__)
+
+
+DATASET_CONFIG_SECTION = "dataset"
+EXPORT_AUTOMATIC = "export_automatic"
+EXPORT_TYPE = "export_type"
+EXPORT_PATH = "export_path"
+EXPORT_PREFIX = "export_prefix"
+
+
+class DataExportType(enum.Enum):
+    """File extensions for supported data types to export data"""
+    NETCDF = "nc"
+    CSV = "csv"
+
+
+def set_data_export_type(export_type: str) -> None:
+    """Set data export type
+
+    Args:
+        export_type: Export type to use.
+            Currently supported values: netcdf, csv.
+    """
+    # disable file export
+    if export_type is None:
+        config[DATASET_CONFIG_SECTION][EXPORT_TYPE] = None
+
+    elif hasattr(DataExportType, export_type.upper()):
+        config[DATASET_CONFIG_SECTION][EXPORT_TYPE] = export_type.upper()
+
+    else:
+        _log.warning(
+            "Could not set export type to '%s' because it is not supported." \
+            % export_type)
+
+
+def set_data_export_path(export_path: str) -> None:
+    """Set path to export data to at the end of a measurement
+
+    Args:
+        export_path: An existing file path on disk
+
+    Raises:
+        ValueError: If the path does not exist, this raises an error
+    """
+    if not exists(export_path):
+        raise ValueError(f"Cannot set export path to '{export_path}' \
+        because it does not exist.")
+    config[DATASET_CONFIG_SECTION][EXPORT_AUTOMATIC] = export_path
+
+
+def get_data_export_type(
+    export_type: Optional[Union[DataExportType, str]] = None) -> Optional[DataExportType]:
+    """Get the file type for exporting data to disk at the end of
+    a measurement from config
+
+    Args:
+        export_type: Export type string format to convert to DataExportType.
+
+    Returns:
+        Data export type
+    """
+    # If export_type is None, get value from config
+    export_type = export_type or config[DATASET_CONFIG_SECTION][EXPORT_TYPE]
+
+    if isinstance(export_type, DataExportType):
+        return export_type
+    elif export_type:
+        if hasattr(DataExportType, export_type.upper()):
+            return getattr(DataExportType, export_type.upper())
+    return None
+
+
+def get_data_export_automatic() -> bool:
+    """Should the data be exported automatically?"""
+    export_automatic = config[DATASET_CONFIG_SECTION][EXPORT_AUTOMATIC]
+    return export_automatic
+
+
+def get_data_export_path() -> str:
+    """Get the path to export data to at the end of a measurement from config
+
+    Returns:
+        Path
+    """
+    return normpath(expanduser(config[DATASET_CONFIG_SECTION][EXPORT_PATH]))
+
+
+def set_data_export_prefix(export_prefix: str) -> None:
+    """Set the data export file name prefix to export data to at the end of
+    a measurement
+
+    Args:
+        export_prefix: Prefix, e.g. "qcodes_"
+    """
+    config[DATASET_CONFIG_SECTION][EXPORT_PREFIX] = export_prefix
+
+
+def get_data_export_prefix() -> str:
+    """Get the data export file name prefix to export data to at the end of
+    a measurement from config
+
+    Returns:
+        Prefix, e.g. "qcodes_"
+    """
+    return config[DATASET_CONFIG_SECTION][EXPORT_PREFIX]
diff --git a/qcodes/dataset/measurements.py b/qcodes/dataset/measurements.py
--- a/qcodes/dataset/measurements.py
+++ b/qcodes/dataset/measurements.py
@@ -40,6 +40,7 @@
                                          expand_setpoints_helper)
 from qcodes.utils.delaykeyboardinterrupt import DelayedKeyboardInterrupt
 from qcodes.utils.helpers import NumpyJSONEncoder
+from qcodes.dataset.export_config import get_data_export_automatic
 
 log = logging.getLogger(__name__)
 
@@ -412,6 +413,12 @@ def flush_data_to_database(self, block: bool = False) -> None:
         """
         self.dataset._flush_data_to_database(block=block)
 
+    def export_data(self) -> None:
+        """Export data at end of measurement as per export_type
+        specification in "dataset" section of qcodes config
+        """
+        self.dataset.export()
+
     @property
     def run_id(self) -> int:
         return self._dataset.run_id
@@ -583,6 +590,8 @@ def __exit__(self,
             # Note that the completion of a dataset entails waiting for the
             # write thread to terminate (iff the write thread has been started)
             self.ds.mark_completed()
+            if get_data_export_automatic():
+                self.datasaver.export_data()
             log.info(f'Finished measurement with guid: {self.ds.guid}. '
                      f'{self._extra_log_info}')
             self.ds.unsubscribe_all()


Test patch:
diff --git a/qcodes/tests/dataset/measurement/test_measurement_context_manager.py b/qcodes/tests/dataset/measurement/test_measurement_context_manager.py
--- a/qcodes/tests/dataset/measurement/test_measurement_context_manager.py
+++ b/qcodes/tests/dataset/measurement/test_measurement_context_manager.py
@@ -11,6 +11,7 @@
 import pytest
 from hypothesis import HealthCheck, given, settings
 from numpy.testing import assert_allclose, assert_array_equal
+from unittest.mock import patch
 
 import qcodes as qc
 from qcodes.dataset.data_export import get_data_by_id
@@ -26,6 +27,7 @@
 # pylint: disable=unused-import
 from qcodes.tests.test_station import set_default_station_to_none
 from qcodes.utils.validators import Arrays
+from qcodes.dataset.export_config import DataExportType
 
 
 def test_log_messages(caplog, meas_with_registered_param):
@@ -1678,6 +1680,52 @@ def test_datasaver_multidim_array(experiment, bg_writing):  # noqa: F811
             assert datadict['data'].shape == (size1 * size2,)
 
 
+@pytest.mark.parametrize("bg_writing", [True, False])
+@pytest.mark.parametrize("export", [True, False])
+def test_datasaver_export(experiment, bg_writing, tmp_path_factory,
+                          export):
+    """
+    Test export data to csv after measurement ends
+    """
+    meas = Measurement(experiment)
+    size1 = 10
+    size2 = 15
+
+    x1 = qc.ManualParameter('x1')
+    x2 = qc.ManualParameter('x2')
+    y1 = qc.ManualParameter('y1')
+    y2 = qc.ManualParameter('y2')
+
+    meas.register_parameter(x1, paramtype='array')
+    meas.register_parameter(x2, paramtype='array')
+    meas.register_parameter(y1, setpoints=[x1, x2], paramtype='array')
+    meas.register_parameter(y2, setpoints=[x1, x2], paramtype='array')
+    data = np.random.rand(4, size1, size2)
+    expected = {'x1': data[0, :, :],
+                'x2': data[1, :, :],
+                'y1': data[2, :, :],
+                'y2': data[3, :, :]}
+
+    tmp_path = tmp_path_factory.mktemp("export_from_config")
+    path = str(tmp_path)
+
+    with patch("qcodes.dataset.data_set.get_data_export_type") as mock_type, \
+    patch("qcodes.dataset.data_set.get_data_export_path") as mock_path, \
+    patch("qcodes.dataset.measurements.get_data_export_automatic") as mock_automatic:
+        mock_type.return_value = DataExportType.CSV
+        mock_path.return_value = path
+        mock_automatic.return_value = export
+        with meas.run(write_in_background=bg_writing) as datasaver:
+            datasaver.add_result((str(x1), expected['x1']),
+                                 (str(x2), expected['x2']),
+                                 (str(y1), expected['y1']),
+                                 (str(y2), expected['y2']))
+    if export:
+        assert os.listdir(path) == [f"qcodes_{datasaver.dataset.run_id}.csv"]
+    else:
+        assert os.listdir(path) == []
+
+
 @pytest.mark.parametrize("bg_writing", [True, False])
 def test_datasaver_multidim_numeric(experiment, bg_writing):
     """
diff --git a/qcodes/tests/dataset/test_dataset_basic.py b/qcodes/tests/dataset/test_dataset_basic.py
--- a/qcodes/tests/dataset/test_dataset_basic.py
+++ b/qcodes/tests/dataset/test_dataset_basic.py
@@ -1,4 +1,3 @@
-import itertools
 import os
 import random
 import re
@@ -8,6 +7,7 @@
 
 import hypothesis.strategies as hst
 import numpy as np
+import xarray as xr
 import pytest
 from hypothesis import HealthCheck, given, settings
 
@@ -24,8 +24,16 @@
 from qcodes.dataset.sqlite.queries import _unicode_categories
 from qcodes.tests.common import error_caused_by
 from qcodes.tests.dataset.test_links import generate_some_links
-from qcodes.utils.deprecate import QCoDeSDeprecationWarning
 from qcodes.utils.types import numpy_ints, numpy_floats
+from qcodes.dataset.export_config import (
+    set_data_export_path,
+    set_data_export_prefix,
+    set_data_export_type,
+    get_data_export_path,
+    get_data_export_prefix,
+    get_data_export_type,
+    DataExportType
+)
 
 from qcodes.tests.dataset.helper_functions import verify_data_dict
 
@@ -1260,6 +1268,24 @@ def limit_data_to_start_end(start, end, input_names, expected_names,
     return start, end
 
 
+@pytest.mark.usefixtures('experiment')
+@pytest.fixture(name="mock_dataset")
+def _make_mock_dataset():
+    dataset = new_data_set("dataset")
+    xparam = ParamSpecBase("x", 'numeric')
+    yparam = ParamSpecBase("y", 'numeric')
+    zparam = ParamSpecBase("z", 'numeric')
+    idps = InterDependencies_(
+        dependencies={yparam: (xparam,), zparam: (xparam,)})
+    dataset.set_interdependencies(idps)
+
+    dataset.mark_started()
+    results = [{'x': 0, 'y': 1, 'z': 2}]
+    dataset.add_results(results)
+    dataset.mark_completed()
+    return dataset
+
+
 @pytest.mark.usefixtures('experiment')
 def test_write_data_to_text_file_save(tmp_path_factory):
     dataset = new_data_set("dataset")
@@ -1281,22 +1307,10 @@ def test_write_data_to_text_file_save(tmp_path_factory):
 
 
 @pytest.mark.usefixtures('experiment')
-def test_write_data_to_text_file_save_multi_keys(tmp_path_factory):
-    dataset = new_data_set("dataset")
-    xparam = ParamSpecBase("x", 'numeric')
-    yparam = ParamSpecBase("y", 'numeric')
-    zparam = ParamSpecBase("z", 'numeric')
-    idps = InterDependencies_(
-        dependencies={yparam: (xparam,), zparam: (xparam,)})
-    dataset.set_interdependencies(idps)
-
-    dataset.mark_started()
-    results = [{'x': 0, 'y': 1, 'z': 2}]
-    dataset.add_results(results)
-    dataset.mark_completed()
+def test_write_data_to_text_file_save_multi_keys(tmp_path_factory, mock_dataset):
     tmp_path = tmp_path_factory.mktemp("data_to_text_file_save_multi_keys")
     path = str(tmp_path)
-    dataset.write_data_to_text_file(path=path)
+    mock_dataset.write_data_to_text_file(path=path)
     assert sorted(os.listdir(path)) == ['y.dat', 'z.dat']
     with open(os.path.join(path, "y.dat")) as f:
         assert f.readlines() == ['0.0\t1.0\n']
@@ -1305,22 +1319,10 @@ def test_write_data_to_text_file_save_multi_keys(tmp_path_factory):
 
 
 @pytest.mark.usefixtures('experiment')
-def test_write_data_to_text_file_save_single_file(tmp_path_factory):
-    dataset = new_data_set("dataset")
-    xparam = ParamSpecBase("x", 'numeric')
-    yparam = ParamSpecBase("y", 'numeric')
-    zparam = ParamSpecBase("z", 'numeric')
-    idps = InterDependencies_(
-        dependencies={yparam: (xparam,), zparam: (xparam,)})
-    dataset.set_interdependencies(idps)
-
-    dataset.mark_started()
-    results = [{'x': 0, 'y': 1, 'z': 2}]
-    dataset.add_results(results)
-    dataset.mark_completed()
+def test_write_data_to_text_file_save_single_file(tmp_path_factory, mock_dataset):
     tmp_path = tmp_path_factory.mktemp("to_text_file_save_single_file")
     path = str(tmp_path)
-    dataset.write_data_to_text_file(path=path, single_file=True,
+    mock_dataset.write_data_to_text_file(path=path, single_file=True,
                                     single_file_name='yz')
     assert os.listdir(path) == ['yz.dat']
     with open(os.path.join(path, "yz.dat")) as f:
@@ -1353,26 +1355,59 @@ def test_write_data_to_text_file_length_exception(tmp_path):
 
 
 @pytest.mark.usefixtures('experiment')
-def test_write_data_to_text_file_name_exception(tmp_path):
-    dataset = new_data_set("dataset")
-    xparam = ParamSpecBase("x", 'numeric')
-    yparam = ParamSpecBase("y", 'numeric')
-    zparam = ParamSpecBase("z", 'numeric')
-    idps = InterDependencies_(
-        dependencies={yparam: (xparam,), zparam: (xparam,)})
-    dataset.set_interdependencies(idps)
-
-    dataset.mark_started()
-    results = [{'x': 0, 'y': 1, 'z': 2}]
-    dataset.add_results(results)
-    dataset.mark_completed()
-
+def test_write_data_to_text_file_name_exception(tmp_path, mock_dataset):
     temp_dir = str(tmp_path)
     with pytest.raises(Exception, match='desired file name'):
-        dataset.write_data_to_text_file(path=temp_dir, single_file=True,
+        mock_dataset.write_data_to_text_file(path=temp_dir, single_file=True,
                                         single_file_name=None)
 
 
+@pytest.mark.usefixtures('experiment')
+def test_export_csv(tmp_path_factory, mock_dataset):
+    tmp_path = tmp_path_factory.mktemp("export_csv")
+    path = str(tmp_path)
+    mock_dataset.export(export_type="csv", path=path, prefix="qcodes_")
+    assert os.listdir(path) == [f"qcodes_{mock_dataset.run_id}.csv"]
+    with open(os.path.join(path, f"qcodes_{mock_dataset.run_id}.csv")) as f:
+        assert f.readlines() == ['0.0\t1.0\t2.0\n']
+
+
+@pytest.mark.usefixtures('experiment')
+def test_export_netcdf(tmp_path_factory, mock_dataset):
+    tmp_path = tmp_path_factory.mktemp("export_netcdf")
+    path = str(tmp_path)
+    mock_dataset.export(export_type="netcdf", path=path, prefix="qcodes_")
+    assert os.listdir(path) == [f"qcodes_{mock_dataset.run_id}.nc"]
+    file_path = os.path.join(path, f"qcodes_{mock_dataset.run_id}.nc")
+    ds = xr.open_dataset(file_path)
+    df = ds.to_dataframe()
+    assert df.index.name == "x"
+    assert df.index.values.tolist() == [0.]
+    assert df.y.values.tolist() == [1.0]
+    assert df.z.values.tolist() == [2.0]
+
+
+@pytest.mark.usefixtures('experiment')
+def test_export_no_or_nonexistent_type_specified(tmp_path_factory, mock_dataset):
+    with pytest.raises(ValueError):
+        mock_dataset.export()
+
+    with pytest.raises(ValueError):
+        mock_dataset.export(export_type="foo")
+
+
+@pytest.mark.usefixtures('experiment')
+def test_export_from_config(tmp_path_factory, mock_dataset):
+    tmp_path = tmp_path_factory.mktemp("export_from_config")
+    path = str(tmp_path)
+    with patch("qcodes.dataset.data_set.get_data_export_type") as mock_type, \
+    patch("qcodes.dataset.data_set.get_data_export_path") as mock_path:
+        mock_type.return_value = DataExportType.CSV
+        mock_path.return_value = path
+        mock_dataset.export()
+    assert os.listdir(path) == [f"qcodes_{mock_dataset.run_id}.csv"]
+
+
 def test_same_setpoint_warning_for_df_and_xarray(different_setpoint_dataset):
 
     warning_message = (