Repository: microsoft/Qcodes

Issue:
Nonetype not accepted as valid type by data saver
Let's consider the situation where we interleave a 1D and 2D sweep. This will look as follows.  

```python
import numpy as np

from qcodes.dataset.measurements import Measurement
from qcodes import Parameter

x = Parameter("x", set_cmd=None, get_cmd=None)
y = Parameter("y", set_cmd=None, get_cmd=None)

m = Parameter("m")
m.get = lambda:np.sin(x())

n = Parameter("n")
n.get = lambda:np.sin(x()**2 + y())

meas = Measurement()
meas.register_parameter(x)
meas.register_parameter(y)
meas.register_parameter(m, setpoints=[x])
meas.register_parameter(n, setpoints=[x, y])

with meas.run() as datasaver: 
    for ix in np.linspace(0, 6, 100): 
        x.set(ix)
        datasaver.add_result(("x", ix), ("y", None), ("m", m.get()), ("n", None))

        for iy in np.linspace(0, 6, 100): 
            y.set(iy)
            datasaver.add_result(("x", ix), ("y", iy), ("m", None), ("n", n.get()))
```

Currently, this fails because of `ValueError: Wrong value type received. Got <class 'NoneType'>, but only int, float, str, tuple, list, and np.ndarray is allowed.`.  This is unfortuante as the underlying infrastructure has no problems with None values 


Discussion:

Resolving patch:
diff --git a/qcodes/dataset/data_set.py b/qcodes/dataset/data_set.py
--- a/qcodes/dataset/data_set.py
+++ b/qcodes/dataset/data_set.py
@@ -377,8 +377,9 @@ def add_results(self, results: List[Dict[str, VALUE]]) -> int:
 
         Args:
             - list of name, value dictionaries  where each
-              dictionary provides the values for all of the parameters in
-              that result.
+              dictionary provides the values for the parameters in
+              that result. If some parameters are missing the corresponding
+              values are assumed to be None
 
         Returns:
             - the index in the DataSet that the **first** result was stored at
@@ -387,9 +388,11 @@ def add_results(self, results: List[Dict[str, VALUE]]) -> int:
         the name of a parameter in this DataSet.
         It is an error to add results to a completed DataSet.
         """
-        values = [list(val.values()) for val in results]
+        expected_keys = frozenset.union(*[frozenset(d) for d in results])
+        values = [[d.get(k, None) for k in expected_keys] for d in results]
+
         len_before_add = length(self.conn, self.table_name)
-        insert_many_values(self.conn, self.table_name, list(results[0].keys()),
+        insert_many_values(self.conn, self.table_name, list(expected_keys),
                            values)
         # TODO: should this not be made atomic?
         self.conn.commit()


Test patch:
diff --git a/qcodes/tests/dataset/test_dataset_basic.py b/qcodes/tests/dataset/test_dataset_basic.py
--- a/qcodes/tests/dataset/test_dataset_basic.py
+++ b/qcodes/tests/dataset/test_dataset_basic.py
@@ -1,6 +1,7 @@
 from hypothesis import given, settings
 import hypothesis.strategies as hst
 import numpy as np
+import itertools
 
 import qcodes as qc
 from qcodes import ParamSpec, new_data_set, new_experiment, experiments
@@ -372,3 +373,49 @@ def test_numpy_nan(dataset):
     dataset.add_results(data_dict)
     retrieved = dataset.get_data("m")
     assert np.isnan(retrieved[1])
+
+
+def test_missing_keys(dataset):
+    """
+    Test that we can now have partial results with keys missing. This is for
+    example handy when having an interleaved 1D and 2D sweep.
+    """
+
+    x = ParamSpec("x", paramtype='numeric')
+    y = ParamSpec("y", paramtype='numeric')
+    a = ParamSpec("a", paramtype='numeric', depends_on=[x])
+    b = ParamSpec("b", paramtype='numeric', depends_on=[x, y])
+
+    dataset.add_parameter(x)
+    dataset.add_parameter(y)
+    dataset.add_parameter(a)
+    dataset.add_parameter(b)
+
+    def fa(xv):
+        return xv + 1
+
+    def fb(xv, yv):
+        return xv + 2 - yv * 3
+
+    results = []
+    xvals = [1, 2, 3]
+    yvals = [2, 3, 4]
+
+    for xv in xvals:
+        results.append({"x": xv, "a": fa(xv)})
+        for yv in yvals:
+            results.append({"x": xv, "y": yv, "b": fb(xv, yv)})
+
+    dataset.add_results(results)
+
+    assert dataset.get_values("x") == [[r["x"]] for r in results]
+    assert dataset.get_values("y") == [[r["y"]] for r in results if "y" in r]
+    assert dataset.get_values("a") == [[r["a"]] for r in results if "a" in r]
+    assert dataset.get_values("b") == [[r["b"]] for r in results if "b" in r]
+
+    assert dataset.get_setpoints("a") == [[[xv] for xv in xvals]]
+
+    tmp = [list(t) for t in zip(*(itertools.product(xvals, yvals)))]
+    expected_setpoints = [[[v] for v in vals] for vals in tmp]
+
+    assert dataset.get_setpoints("b") == expected_setpoints