Repository: microsoft/Qcodes

Issue:
dataset.completed does not work
### Steps to reproduce

```
dataset = load_by_id(1)
dataset.completed
```

Output:

> Traceback (most recent call last):
> 
>   File "<ipython-input-178-f869a978e653>", line 1, in <module>
>     dataset.completed
> 
>   File "c:\users\triton5acq\qcodes\qcodes\dataset\data_set.py", line 306, in completed
>     return self._completed
> 
> AttributeError: 'DataSet' object has no attribute '_completed'

### System
**Windows 7**

**qcodes/master**

**19a2016f9c2d999027b3949a646200d2d5b41e7e**



Discussion:

Resolving patch:
diff --git a/qcodes/dataset/data_set.py b/qcodes/dataset/data_set.py
--- a/qcodes/dataset/data_set.py
+++ b/qcodes/dataset/data_set.py
@@ -21,11 +21,12 @@
 from qcodes.instrument.parameter import _BaseParameter
 from qcodes.dataset.sqlite_base import (atomic, atomic_transaction,
                                         transaction, add_parameter,
-                                        connect, create_run, get_parameters,
+                                        connect, create_run, completed,
+                                        get_parameters,
                                         get_experiments,
                                         get_last_experiment, select_one_where,
                                         length, modify_values,
-                                        add_meta_data, mark_run,
+                                        add_meta_data, mark_run_complete,
                                         modify_many_values, insert_values,
                                         insert_many_values,
                                         VALUE, VALUES, get_data,
@@ -78,7 +79,7 @@ def __init__(self, dataSet, sub_id: str,
         # whether or not this is actually thread safe I am not sure :P
         self.dataSet = dataSet
         self.table_name = dataSet.table_name
-        self.conn = dataSet.conn
+        conn = dataSet.conn
         self.log = logging.getLogger(f"Subscriber {self.sub_id}")
 
         self.state = state
@@ -95,14 +96,14 @@ def __init__(self, dataSet, sub_id: str,
         param_sql = ",".join([f"NEW.{p.name}" for p in parameters])
         self.callbackid = f"callback{self.sub_id}"
 
-        self.conn.create_function(self.callbackid, -1, self.cache)
+        conn.create_function(self.callbackid, -1, self.cache)
         sql = f"""
         CREATE TRIGGER sub{self.sub_id}
             AFTER INSERT ON '{self.table_name}'
         BEGIN
             SELECT {self.callbackid}({param_sql});
         END;"""
-        atomic_transaction(self.conn, sql)
+        atomic_transaction(conn, sql)
         self.data: Queue = Queue()
         self._data_set_len = len(dataSet)
         super().__init__()
@@ -165,7 +166,18 @@ def _clean_up(self) -> None:
 
 
 class DataSet(Sized):
-    def __init__(self, path_to_db: str, conn=None) -> None:
+    def __init__(self, path_to_db: str, run_id: Optional[int]=None,
+                 conn=None) -> None:
+        """
+        Create a new DataSet object. The object can either be intended
+        to hold a new run or and old run.
+
+        Args:
+            path_to_db: path to the sqlite file on disk
+            run_id: provide this when loading an existing run, leave it
+              as None when creating a new run
+            conn: connection to the DB
+        """
         # TODO: handle fail here by defaulting to
         # a standard db
         self.path_to_db = path_to_db
@@ -174,7 +186,11 @@ def __init__(self, path_to_db: str, conn=None) -> None:
         else:
             self.conn = conn
 
+        self.run_id = run_id
         self._debug = False
+        self.subscribers: Dict[str, Subscriber] = {}
+        if run_id:
+            self._completed = completed(self.conn, self.run_id)
 
     def _new(self, name, exp_id, specs: SPECS = None, values=None,
              metadata=None) -> None:
@@ -187,7 +203,6 @@ def _new(self, name, exp_id, specs: SPECS = None, values=None,
 
         # this is really the UUID (an ever increasing count in the db)
         self.run_id = run_id
-        self.subscribers: Dict[str, Subscriber] = {}
         self._completed = False
 
     @property
@@ -229,6 +244,7 @@ def exp_id(self):
         return select_one_where(self.conn, "runs",
                                 "exp_id", "run_id", self.run_id)
 
+
     def toggle_debug(self):
         """
         Toggle debug mode, if debug mode is on
@@ -308,7 +324,8 @@ def completed(self) -> bool:
     @completed.setter
     def completed(self, value):
         self._completed = value
-        mark_run(self.conn, self.run_id, value)
+        if value:
+            mark_run_complete(self.conn, self.run_id)
 
     def mark_complete(self) -> None:
         """Mark dataset as complete and thus read only and notify the
@@ -614,8 +631,7 @@ def load_by_id(run_id)->DataSet:
         the datasets
 
     """
-    d = DataSet(get_DB_location())
-    d.run_id = run_id
+    d = DataSet(get_DB_location(), run_id=run_id)
     return d
 
 
@@ -629,7 +645,7 @@ def load_by_counter(counter, exp_id):
     Returns:
         the dataset
     """
-    d = DataSet(get_DB_location())
+    conn = connect(get_DB_location())
     sql = """
     SELECT run_id
     FROM
@@ -638,8 +654,11 @@ def load_by_counter(counter, exp_id):
       result_counter= ? AND
       exp_id = ?
     """
-    c = transaction(d.conn, sql, counter, exp_id)
-    d.run_id = one(c, 'run_id')
+    c = transaction(conn, sql, counter, exp_id)
+    run_id = one(c, 'run_id')
+    conn.close()
+    d = DataSet(get_DB_location(), run_id=run_id)
+
     return d
 
 
@@ -658,16 +677,27 @@ def new_data_set(name, exp_id: Optional[int] = None,
         metadata:  the values to associate with the dataset
     """
     path_to_db = get_DB_location()
-    d = DataSet(path_to_db, conn=conn)
+    if conn is None:
+        tempcon = True
+        conn = connect(get_DB_location())
+    else:
+        tempcon = False
 
     if exp_id is None:
-        if len(get_experiments(d.conn)) > 0:
-            exp_id = get_last_experiment(d.conn)
+        if len(get_experiments(conn)) > 0:
+            exp_id = get_last_experiment(conn)
         else:
             raise ValueError("No experiments found."
                              "You can start a new one with:"
                              " new_experiment(name, sample_name)")
+    # This is admittedly a bit weird. We create a dataset, link it to some
+    # run in the DB and then (using _new) change what it's linked to
+    if tempcon:
+        conn.close()
+        conn = None
+    d = DataSet(path_to_db, run_id=None, conn=conn)
     d._new(name, exp_id, specs, values, metadata)
+
     return d
 
 
diff --git a/qcodes/dataset/sqlite_base.py b/qcodes/dataset/sqlite_base.py
--- a/qcodes/dataset/sqlite_base.py
+++ b/qcodes/dataset/sqlite_base.py
@@ -805,7 +805,9 @@ def new_experiment(conn: sqlite3.Connection,
     return curr.lastrowid
 
 
-def mark_run(conn: sqlite3.Connection, run_id: int, complete: bool):
+# TODO(WilliamHPNielsen): we should remove the redundant
+# is_completed
+def mark_run_complete(conn: sqlite3.Connection, run_id: int):
     """ Mark run complete
 
     Args:
@@ -821,7 +823,7 @@ def mark_run(conn: sqlite3.Connection, run_id: int, complete: bool):
         is_completed=?
     WHERE run_id=?;
     """
-    atomic_transaction(conn, query, time.time(), complete, run_id)
+    atomic_transaction(conn, query, time.time(), True, run_id)
 
 
 def completed(conn: sqlite3.Connection, run_id)->bool:


Test patch:
diff --git a/qcodes/tests/dataset/test_dataset_loading.py b/qcodes/tests/dataset/test_dataset_loading.py
new file mode 100644
--- /dev/null
+++ b/qcodes/tests/dataset/test_dataset_loading.py
@@ -0,0 +1,57 @@
+import os
+
+import pytest
+import tempfile
+
+import qcodes as qc
+from qcodes.dataset.database import initialise_database
+from qcodes.dataset.experiment_container import new_experiment
+from qcodes.dataset.data_set import new_data_set, load_by_id, load_by_counter
+
+
+@pytest.fixture(scope="function")
+def empty_temp_db():
+    # create a temp database for testing
+    with tempfile.TemporaryDirectory() as tmpdirname:
+        qc.config["core"]["db_location"] = os.path.join(tmpdirname, 'temp.db')
+        qc.config["core"]["db_debug"] = True
+        initialise_database()
+        yield
+
+
+@pytest.fixture(scope='function')
+def experiment(empty_temp_db):
+    e = new_experiment("test-experiment", sample_name="test-sample")
+    yield e
+    e.conn.close()
+
+
+def test_load_by_id(experiment):
+    ds = new_data_set("test-dataset")
+    run_id = ds.run_id
+    ds.mark_complete()
+
+    loaded_ds = load_by_id(run_id)
+    assert loaded_ds.completed is True
+    assert loaded_ds.exp_id == 1
+
+    ds = new_data_set("test-dataset-unfinished")
+    run_id = ds.run_id
+
+    loaded_ds = load_by_id(run_id)
+    assert loaded_ds.completed is False
+    assert loaded_ds.exp_id == 1
+
+
+def test_load_by_counter(empty_temp_db):
+    exp = new_experiment(name="for_loading", sample_name="no_sample")
+    ds = new_data_set("my_first_ds")
+
+    loaded_ds = load_by_counter(exp.exp_id, 1)
+
+    assert loaded_ds.completed is False
+
+    ds.mark_complete()
+    loaded_ds = load_by_counter(exp.exp_id, 1)
+
+    assert loaded_ds.completed is True