Repository: microsoft/Qcodes

Issue:
Captured_run_id is incorrect if run has been inserted
Currently a new run gets a captured_run_id inserted based on the following logic

```
    if captured_run_id is None:
        with atomic(conn) as conn:
            query = """
            SELECT
                max(captured_run_id)
            FROM
                runs"""
            curr = transaction(conn, query)
            existing_captured_run_id = one(curr, 0)
        if existing_captured_run_id is not None:
            captured_run_id = existing_captured_run_id + 1
        else:
            captured_run_id = 1
```

Consider a database with 10 runs where a dataset is added via ds = load_from_netcdf(...) and ds.write_metadata_to_db() 
If that dataset has captured_run_id==100 the next dataset created as part of a measurement would have
run_id ==12 and captured_run_id==101 breaking the expectation that the run_id == captured_run_id for any new dataset. 


Suggested fix replace max(captured_run_id) with max(run_id) in the above code and check once the row has been written that run_id == captured_run_id for new  runs.




Discussion:
> Suggested fix replace max(captured_run_id) with max(run_id) in the above code

oh totally! Good catch!

Resolving patch:
diff --git a/qcodes/dataset/sqlite/queries.py b/qcodes/dataset/sqlite/queries.py
--- a/qcodes/dataset/sqlite/queries.py
+++ b/qcodes/dataset/sqlite/queries.py
@@ -1338,12 +1338,22 @@ def _insert_run(
                                                    where_value=exp_id)
     assert isinstance(run_counter, int)
     assert isinstance(format_string, str)
+    newly_created_run = False
+
+    if (captured_run_id is not None and captured_counter is None) or (
+        captured_counter is not None and captured_run_id is None
+    ):
+        raise ValueError(
+            "Either both of captured_counter and captured_run_id should be set or none of them."
+        )
+
     run_counter += 1
     if captured_counter is None:
+        newly_created_run = True
         with atomic(conn) as conn:
             query = """
             SELECT
-                max(captured_counter)
+                max(result_counter)
             FROM
                 runs
             WHERE
@@ -1354,24 +1364,12 @@ def _insert_run(
                 captured_counter = existing_captured_counter + 1
             else:
                 captured_counter = run_counter
-    formatted_name = format_table_name(format_string, "results", exp_id, run_counter)
-    table = "runs"
-
-    if description is None:
-        description = RunDescriber(InterDependencies_())
-    desc_str = serial.to_json_for_storage(description)
-    legacy_param_specs = new_to_old(description.interdeps).paramspecs
-    # we need to use legacy_param_specs here not because we need the features
-    # of legacy param specs but because they should be inserted into the layouts
-    # table and parameter column in the same order as data_set.add_parameter does
-    # specifically dependencies should come before the dependent parameters for links
-    # in the layout table to work correctly
 
     if captured_run_id is None:
         with atomic(conn) as conn:
             query = """
             SELECT
-                max(captured_run_id)
+                max(run_id)
             FROM
                 runs"""
             curr = transaction(conn, query)
@@ -1381,6 +1379,19 @@ def _insert_run(
         else:
             captured_run_id = 1
 
+    formatted_name = format_table_name(format_string, "results", exp_id, run_counter)
+    table = "runs"
+
+    if description is None:
+        description = RunDescriber(InterDependencies_())
+    desc_str = serial.to_json_for_storage(description)
+    legacy_param_specs = new_to_old(description.interdeps).paramspecs
+    # we need to use legacy_param_specs here not because we need the features
+    # of legacy param specs but because they should be inserted into the layouts
+    # table and parameter column in the same order as data_set.add_parameter does
+    # specifically dependencies should come before the dependent parameters for links
+    # in the layout table to work correctly
+
     with atomic(conn) as conn:
 
         if legacy_param_specs:
@@ -1456,6 +1467,15 @@ def _insert_run(
     run_id = curr.lastrowid
     if run_id is None:
         raise RuntimeError(f"Creation of run with guid: {guid} failed")
+    if newly_created_run:
+        if captured_run_id != run_id:
+            warnings.warn(
+                "captured_run_id was not equal to run_id, this is unexpected and will be an error in the future."
+            )
+        if captured_counter != run_counter:
+            warnings.warn(
+                "captured_counter was not equal to result_counter, this is unexpected and will be an error in the future."
+            )
     return run_counter, formatted_name, run_id
 
 


Test patch:
diff --git a/qcodes/tests/dataset/test_database_extract_runs.py b/qcodes/tests/dataset/test_database_extract_runs.py
--- a/qcodes/tests/dataset/test_database_extract_runs.py
+++ b/qcodes/tests/dataset/test_database_extract_runs.py
@@ -704,9 +704,9 @@ def test_copy_datasets_and_add_new(
         ds.mark_completed()
 
     expected_run_ids = [4, 5, 6]
-    expected_captured_run_ids = [11, 12, 13]
+    expected_captured_run_ids = expected_run_ids
     expected_counter = [4, 5, 6]
-    expected_captured_counter = [6, 7, 8]
+    expected_captured_counter = expected_counter
 
     for ds, eri, ecri, ec, ecc in zip(new_datasets,
                                       expected_run_ids,