Adding the sqlite db where I will archive the test results and added the archiving code

data/sqlite/test_records.db

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d94143eec37c91e71cc3d39507a8c4741b2922848b7cfd33f48b1b5d4b7aebd0
+size 53248

src/testing.py

@@ -2,7 +2,11 @@ from __future__ import annotations  # For self-referencing annotations
 
 import json
 import os
+import shutil
+import sqlite3
+import sys
 
+from huggingface_hub import Repository
 from random import choices
 from typing import List, Dict, Optional
 
@@ -104,7 +108,7 @@ class ArchitectureRequestRecord:
 
 
 class TestGroup:
-    all: Dict[str, List[ArchitectureRequestRecord]] = None
+    all: Dict[str, TestGroup] = None
 
     def __init__(self, test_group:str):
         self.arch_request_records: List[ArchitectureRequestRecord] = []
@@ -183,3 +187,75 @@ class TestGroup:
     def for_test_group_tag(cls, test_group_tag: str) -> TestGroup:
         cls.load_all()
         return cls.all[test_group_tag]
+
+
+def move_test_records_to_db(hf_hub_token: str) -> None:
+    """
+    This is an offline utility to move the transaction logs from
+    a flat file to a database. To keep things simpler transaction logs
+    are initially stored into a json file as a Hugging Face dataset, but
+    this can get cumbersome, so this utility will move the records
+    into an sqlite database.  It can be run periodically just to move
+    them across.
+    """
+    def download_latest_json_file(hf_hub_token: str) -> Repository:
+        if os.path.exists(Architecture.trace_dir):
+            shutil.rmtree(Architecture.trace_dir)
+        return Repository(local_dir=Architecture.trace_dir, clone_from=Architecture.save_repo_url, token=hf_hub_token)
+
+    def create_local_db():
+        db_file = os.path.join(data_dir, 'sqlite', 'test_records.db')
+        con = sqlite3.connect(db_file)
+
+        sql = "CREATE TABLE test_groups (id INTEGER PRIMARY KEY AUTOINCREMENT, test_group TEXT NOT NULL, start INTEGER NOT NULL, end INTEGER NOT NULL);"
+        con.execute(sql)
+
+        sql = "CREATE TABLE arch_requests (id INTEGER PRIMARY KEY AUTOINCREMENT, arch_name TEXT NOT NULL, response_len INTEGER NOT NULL, start INTEGER NOT NULL, end INTEGER NOT NULL, comment TEXT NOT NULL, test_group_id INTEGER NOT NULL, FOREIGN KEY (test_group_id) REFERENCES test_groups (id))"
+        con.execute(sql)
+
+        sql = "CREATE TABLE arch_req_steps (id INTEGER PRIMARY KEY AUTOINCREMENT, name TEXT NOT NULL, start INTEGER NOT NULL, end INTEGER NOT NULL, arch_req_id INTEGER NOT NULL, FOREIGN KEY (arch_req_id) REFERENCES arch_requests(id))"
+        con.execute(sql)
+
+    def get_local_db() -> sqlite3.Connection:
+        db_file = os.path.join(data_dir, 'sqlite', 'test_records.db')
+        if not os.path.exists(db_file):
+            create_local_db()
+        return sqlite3.connect(db_file)
+
+    def load_test_group_to_db(test_group: TestGroup, con: sqlite3.Connection) -> None:
+        cur = con.cursor()
+        sql = f'SELECT count(*) from test_groups where test_group ="{test_group.test_group}"'
+        cur.execute(sql)
+        tg_not_in_db = cur.fetchall()[0][0] == 0
+        if tg_not_in_db:
+            sql = f'INSERT into test_groups (test_group, start, end) VALUES ("{test_group.test_group}", {test_group.start}, {test_group.end})'
+            tg_id = con.execute(sql).lastrowid
+
+            for arr in test_group.arch_request_records:
+                sql = f'INSERT INTO arch_requests (arch_name, response_len, start, end, comment, test_group_id) VALUES ("{arr.arch}", {arr.response_len}, {arr.start}, {arr.end}, "{arr.comment}", {tg_id})'
+                arr_id = con.execute(sql).lastrowid
+
+                for s in arr.steps:
+                    sql= f'INSERT INTO arch_req_steps (name, start, end, arch_req_id) VALUES ("{s.name}", {s.start}, {s.end}, {arr_id})'
+                    con.execute(sql)
+            con.commit()
+        else:
+            print(f"Warning TestGroup {tg_not_in_db} was not added to the DB as it already existed there")
+
+    def load_all_test_groups_to_db(con: sqlite3.Connection) -> None:
+        TestGroup.load_all()
+        for tg in TestGroup.all.values():
+            load_test_group_to_db(tg, con)
+
+    # Main control flow using utility local functions above for better structure
+    download_latest_json_file(hf_hub_token)
+    conn = get_local_db()
+    load_all_test_groups_to_db(conn)
+    Architecture.wipe_trace(hf_hub_token)
+    print("REMINDER: need to commit the local sqlite file to make it available to the server")
+
+
+if __name__ == "__main__":
+    # Expected to only be directly called for the json to db transfer - arg should be the HF token
+    hf_token = sys.argv[1]
+    move_test_records_to_db(hf_token)