Repository: microsoft/Qcodes

Issue:
DataSet.get_data: retrieving parts of dataset - actual start value depends on end
i encountered the following just now when trying to load only part of a dataset using get_data: the start index that's actually returned depends on whether the end index has been supplied or left as `None`.

example screenshot:

![image](https://user-images.githubusercontent.com/892193/48586699-eedd9080-e930-11e8-920c-7da93570edae.png)

my expected behavior would be that the first element returned should not depend on the value of `end`.




Discussion:

Resolving patch:
diff --git a/qcodes/dataset/sqlite_base.py b/qcodes/dataset/sqlite_base.py
--- a/qcodes/dataset/sqlite_base.py
+++ b/qcodes/dataset/sqlite_base.py
@@ -1060,52 +1060,41 @@ def length(conn: SomeConnection,
 def get_data(conn: SomeConnection,
              table_name: str,
              columns: List[str],
-             start: int = None,
-             end: int = None,
+             start: Optional[int] = None,
+             end: Optional[int] = None,
              ) -> List[List[Any]]:
     """
     Get data from the columns of a table.
-    Allows to specfiy a range.
+    Allows to specify a range of rows (1-based indexing, both ends are
+    included).
 
     Args:
         conn: database connection
         table_name: name of the table
         columns: list of columns
-        start: start of range (1 indedex)
-        end: start of range (1 indedex)
+        start: start of range; if None, then starts from the top of the table
+        end: end of range; if None, then ends at the bottom of the table
 
     Returns:
-        the data requested
+        the data requested in the format of list of rows of values
     """
     _columns = ",".join(columns)
-    if start and end:
-        query = f"""
-        SELECT {_columns}
-        FROM "{table_name}"
-        WHERE rowid
-            > {start} and
-              rowid
-            <= {end}
-        """
-    elif start:
-        query = f"""
-        SELECT {_columns}
-        FROM "{table_name}"
-        WHERE rowid
-            >= {start}
-        """
-    elif end:
-        query = f"""
-        SELECT {_columns}
-        FROM "{table_name}"
-        WHERE rowid
-            <= {end}
-        """
-    else:
-        query = f"""
-        SELECT {_columns}
-        FROM "{table_name}"
-        """
+
+    query = f"""
+            SELECT {_columns}
+            FROM "{table_name}"
+            """
+
+    start_specified = start is not None
+    end_specified = end is not None
+
+    where = ' WHERE' if start_specified or end_specified else ''
+    start_condition = f' rowid >= {start}' if start_specified else ''
+    end_condition = f' rowid <= {end}' if end_specified else ''
+    and_ = ' AND' if start_specified and end_specified else ''
+
+    query += where + start_condition + and_ + end_condition
+
     c = atomic_transaction(conn, query)
     res = many_many(c, *columns)
 


Test patch:
diff --git a/qcodes/tests/dataset/test_dataset_basic.py b/qcodes/tests/dataset/test_dataset_basic.py
--- a/qcodes/tests/dataset/test_dataset_basic.py
+++ b/qcodes/tests/dataset/test_dataset_basic.py
@@ -10,7 +10,6 @@
 from qcodes import load_by_id, load_by_counter
 from qcodes.dataset.descriptions import RunDescriber
 from qcodes.dataset.dependencies import InterDependencies
-from qcodes.tests.dataset.test_descriptions import some_paramspecs
 from qcodes.dataset.sqlite_base import _unicode_categories
 from qcodes.dataset.database import get_DB_location
 from qcodes.dataset.data_set import CompletedError, DataSet
@@ -18,10 +17,11 @@
 # pylint: disable=unused-import
 from qcodes.tests.dataset.temporary_databases import (empty_temp_db,
                                                       experiment, dataset)
+# pylint: disable=unused-import
+from qcodes.tests.dataset.test_descriptions import some_paramspecs
 
 n_experiments = 0
 
-
 @pytest.mark.usefixtures("experiment")
 def test_has_attributes_after_init():
     """
@@ -556,3 +556,61 @@ def test_metadata():
     with pytest.raises(ValueError, match=match):
         for tag, value in sorry_metadata.items():
             ds1.add_metadata(tag, value)
+
+
+class TestGetData:
+    x = ParamSpec("x", paramtype='numeric')
+    n_vals = 5
+    xvals = list(range(n_vals))
+    # this is the format of how data is returned by DataSet.get_data
+    # which means "a list of table rows"
+    xdata = [[x] for x in xvals]
+
+    @pytest.fixture(autouse=True)
+    def ds_with_vals(self, dataset):
+        """
+        This fixture creates a DataSet with values that is to be used by all
+        the tests in this class
+        """
+        dataset.add_parameter(self.x)
+        for xv in self.xvals:
+            dataset.add_result({self.x.name: xv})
+
+        return dataset
+
+    @pytest.mark.parametrize(
+        ("start", "end", "expected"),
+        [
+            # test without start and end
+            (None, None, xdata),
+
+            # test for start only
+            (0, None, xdata),
+            (2, None, xdata[(2-1):]),
+            (-2, None, xdata),
+            (n_vals, None, xdata[(n_vals-1):]),
+            (n_vals + 1, None, []),
+            (n_vals + 2, None, []),
+
+            # test for end only
+            (None, 0, []),
+            (None, 2, xdata[:2]),
+            (None, -2, []),
+            (None, n_vals, xdata),
+            (None, n_vals + 1, xdata),
+            (None, n_vals + 2, xdata),
+
+            # test for start and end
+            (0, 0, []),
+            (1, 1, [xdata[1-1]]),
+            (2, 1, []),
+            (2, 0, []),
+            (1, 0, []),
+            (n_vals, n_vals, [xdata[n_vals-1]]),
+            (n_vals, n_vals - 1, []),
+            (2, 4, xdata[(2-1):4]),
+        ],
+    )
+    def test_get_data_with_start_and_end_args(self, ds_with_vals,
+                                              start, end, expected):
+        assert expected == ds_with_vals.get_data(self.x, start=start, end=end)