Repository: SymbiFlow/FPGA-Tool-Performance-Visualization-Library

Issue:
Legacy Icestorm Processor
As of August 1st, [fpga-tool-perf issue #190](https://github.com/SymbiFlow/fpga-tool-perf/issues/190) to correct the reported frequency of icestorm boards has been resolved. Now, all boards correctly report frequency in hertz.

However, this does not retroactively change fpga-tool-perf evaluations that were run before the fix was merged. Therefore, we need to create a processor that allows the user to explicitly fix the icestorm data to ensure that we can compare icestorm frequency data before and after the fix.

Comments:

Patch:
diff --git a/ftpvl/fetchers.py b/ftpvl/fetchers.py
index e70bc9d..1e9559b 100644
--- a/ftpvl/fetchers.py
+++ b/ftpvl/fetchers.py
@@ -1,4 +1,5 @@
 """ Fetchers are responsible for ingesting and standardizing data for future processing. """
+from datetime import datetime
 import json
 from typing import Any, Dict, List
 
@@ -228,6 +229,37 @@ class HydraFetcher(Fetcher):
 
         return data
 
+    def _check_legacy_icebreaker(self, row):
+        """
+        Returns True if row is from a test on an Icebreaker board before Jul 31,
+        2020.
+
+        This is useful because these legacy tests recorded frequency in MHz
+        instead of Hz, while all other boards record in Hz. This flag can be
+        used to check if the units need to be changed.
+
+        Parameters
+        ----------
+        row : dict
+            a dictionary that is the result of decoding a meta.json file
+
+        Returns
+        -------
+        bool
+            Returns true if row is icebreaker board before Aug 1, 2020. False
+            otherwise.
+        """
+        date = None
+        board = None
+        try:
+            date = row["date"] # format: 2020-07-17T22:12:41
+            board = row["board"]
+            timestamp = datetime.strptime(date, "%Y-%m-%dT%H:%M:%S")
+            return timestamp < datetime(2020, 7, 31) and board == "icebreaker"
+        except KeyError:
+            print("Warning: Unable to find date and board in meta.json, required for supporting legacy Icebreaker.")
+            return False # Assume not legacy icebreaker
+
     def _preprocess(self, data: List[Dict]) -> pd.DataFrame:
         """
         Using data from _download(), processes and standardizes the data and
@@ -237,13 +269,21 @@ class HydraFetcher(Fetcher):
 
         processed_data = []
         for row in flattened_data:
+            legacy_icestorm = self._check_legacy_icebreaker(row)
             processed_row = {}
             if self.mapping is None:
                 processed_row = row
             else:
                 for in_col_name, out_col_name in self.mapping.items():
                     processed_row[out_col_name] = row[in_col_name]
-            processed_row["freq"] = Helpers.get_actual_freq(row, self.hydra_clock_names)
+
+            actual_freq = Helpers.get_actual_freq(row, self.hydra_clock_names)
+            if actual_freq:
+                if legacy_icestorm:
+                    # freq in MHz, no change needed
+                    processed_row["freq"] = actual_freq
+                else:
+                    processed_row["freq"] = actual_freq / 1_000_000 # convert hz to mhz
             processed_row.update(Helpers.get_versions(row))
             processed_data.append(processed_row)
 
diff --git a/ftpvl/helpers.py b/ftpvl/helpers.py
index d326384..54a0121 100644
--- a/ftpvl/helpers.py
+++ b/ftpvl/helpers.py
@@ -62,39 +62,19 @@ def get_versions(obj: dict) -> dict:
     return {k: v for k, v in obj.items() if k.startswith("versions.")}
 
 
-def rescale_actual_freq(freq: Union[int, float]) -> Union[int, float]:
-    """
-    Given a frequency with an unspecified unit, returns frequency in megahertz
-    by assuming the original unit is hertz if input frequency is greater than 1
-    million.
-
-    Parameters
-    ----------
-    freq : Union[int, float]
-        a number that is a frequency
-
-    Returns
-    -------
-    Union[int, float]
-        the input frequency in megahertz
-    """
-    one_mhz = 1_000_000
-    if freq > one_mhz:
-        return freq / one_mhz
-    else:
-        return freq
-
-
 def get_actual_freq(obj: dict, hydra_clock_names: list = None) -> Union[int, float]:
     """
     Given a flattened object decoded from meta.json, return the actual frequency
-    as a number in megahertz.
+    as a number.
 
     Since a meta.json object might contain multiple frequencies, we look through
     all clock names specified in hydra_clock_names and use the first one in the
     list. If none of the specified clock names exists in the object, we use
     the shortest clock name to find the frequency.
 
+    The output of this function returns a number with undetermined units. 
+    External logic should handle converting units if necessary.
+
     Parameters
     ----------
     obj : dict
@@ -114,13 +94,13 @@ def get_actual_freq(obj: dict, hydra_clock_names: list = None) -> Union[int, flo
 
     # if max_freq is unnested
     if "max_freq" in obj:
-        return rescale_actual_freq(obj["max_freq"])
+        return obj["max_freq"]
     else:
         # check if max_freq contains clock_name in HYDRA_CLOCK_NAMES
         for clock_name in hydra_clock_names:
             key = f"max_freq.{clock_name}.actual"
             if key in obj:
-                return rescale_actual_freq(obj[key])
+                return obj[key]
 
         # if none of those exist, choose the shortest one or return None
         max_freq_keys = [
@@ -128,7 +108,7 @@ def get_actual_freq(obj: dict, hydra_clock_names: list = None) -> Union[int, flo
         ]
         if len(max_freq_keys) > 0:
             shortest_clock_name = min(max_freq_keys, key=len)
-            return rescale_actual_freq(obj[shortest_clock_name])
+            return obj[shortest_clock_name]
         else:
             return None
 


Test patch:
diff --git a/tests/test_fetcher_small.py b/tests/test_fetcher_small.py
index 80d8fdc..84bde5e 100644
--- a/tests/test_fetcher_small.py
+++ b/tests/test_fetcher_small.py
@@ -22,6 +22,7 @@ class TestHydraFetcherSmall(unittest.TestCase):
             different mapping
                 exclusion, renaming
             pagination
+            with/without board and date information
     """
 
     def test_hydrafetcher_init(self):
@@ -75,7 +76,11 @@ class TestHydraFetcherSmall(unittest.TestCase):
                     
                     # /meta.json
                     meta_url = f'https://hydra.vtr.tools/build/{build_num}/download/5/meta.json'
-                    payload = {"build_num": build_num}
+                    payload = {
+                        "build_num": build_num,
+                        "date": "2020-07-17T22:12:40",
+                        "board": "arty"
+                    }
                     m.get(meta_url, json=payload) # setup /meta.json request mock
 
             # run tests on different eval_num
@@ -88,8 +93,161 @@ class TestHydraFetcherSmall(unittest.TestCase):
                         eval_num=eval_num)
                     result = hf.get_evaluation().get_df()
 
-                    col = [x for x in range(eval_num * 4, eval_num * 4 + 4)]
-                    expected = pd.DataFrame({"build_num": col})
+                    expected = pd.DataFrame({
+                        "build_num": [x for x in range(eval_num * 4, eval_num * 4 + 4)],
+                        "date": ["2020-07-17T22:12:40" for _ in range(4)],
+                        "board": ["arty" for _ in range(4)]
+                        })
+                    assert_frame_equal(result, expected)
+
+                    eval_id = hf.get_evaluation().get_eval_id()
+                    assert eval_id == eval_num + 1
+
+    def test_hydrafetcher_get_evaluation_icebreaker_old(self):
+        """
+        get_evaluation() should return an Evaluation corresponding to the small
+        dataset and the specified eval_num.
+
+        Tests legacy icebreaker support, where icebreaker boards report
+        max frequency in MHz instead of Hz.
+        """
+        with requests_mock.Mocker() as m:
+            evals_fn = 'tests/sample_data/evals.small.json'
+            evals_url = 'https://hydra.vtr.tools/jobset/dusty/fpga-tool-perf/evals'
+
+            build_fn = 'tests/sample_data/build.small.json'
+
+            # setup /evals request mock
+            with open(evals_fn, "r") as f:
+                json_data = f.read()
+                m.get(evals_url, text=json_data) 
+
+            # setup /build and /meta.json request mock
+            with open(build_fn, "r") as f:
+                json_data = f.read()
+
+                for build_num in range(12):
+                    # /build/:buildid
+                    build_url = f'https://hydra.vtr.tools/build/{build_num}'
+                    m.get(build_url, text=json_data)
+                    
+                    # /meta.json
+                    meta_url = f'https://hydra.vtr.tools/build/{build_num}/download/5/meta.json'
+                    payload = {
+                        "build_num": build_num,
+                        "date": "2020-07-30T22:12:40",
+                        "board": "icebreaker",
+                        "max_freq": 81.05
+                    }
+                    m.get(meta_url, json=payload) # setup /meta.json request mock
+
+            # run tests on different eval_num
+            for eval_num in range(0, 3):
+                with self.subTest(eval_num=eval_num):
+                    # if mapping is not defined, should not remap
+                    hf = HydraFetcher(
+                        project="dusty",
+                        jobset="fpga-tool-perf",
+                        eval_num=eval_num,
+                        mapping={"build_num": "build_num"})
+                    result = hf.get_evaluation().get_df()
+
+                    expected = pd.DataFrame({
+                        "build_num": [x for x in range(eval_num * 4, eval_num * 4 + 4)],
+                        "freq": [81.05 for _ in range(4)]
+                        })
+                    print(result.columns)
+                    assert_frame_equal(result, expected)
+
+                    eval_id = hf.get_evaluation().get_eval_id()
+                    assert eval_id == eval_num + 1
+
+    def test_hydrafetcher_get_evaluation_icebreaker_new(self):
+        """
+        get_evaluation() should return an Evaluation corresponding to the small
+        dataset and the specified eval_num.
+
+        Tests legacy icebreaker support does not affect an eval where the date
+        is not before 7-31-2020.
+        """
+        with requests_mock.Mocker() as m:
+            evals_fn = 'tests/sample_data/evals.small.json'
+            evals_url = 'https://hydra.vtr.tools/jobset/dusty/fpga-tool-perf/evals'
+
+            build_fn = 'tests/sample_data/build.small.json'
+
+            # setup /evals request mock
+            with open(evals_fn, "r") as f:
+                json_data = f.read()
+                m.get(evals_url, text=json_data) 
+
+            # setup /build and /meta.json request mock
+            with open(build_fn, "r") as f:
+                json_data = f.read()
+
+                # test if date is not before 7-31-2020
+                for build_num in range(4):
+                    # /build/:buildid
+                    build_url = f'https://hydra.vtr.tools/build/{build_num}'
+                    m.get(build_url, text=json_data)
+                    
+                    # /meta.json
+                    meta_url = f'https://hydra.vtr.tools/build/{build_num}/download/5/meta.json'
+                    payload = {
+                        "build_num": build_num,
+                        "date": "2020-07-31T22:12:40",
+                        "board": "icebreaker",
+                        "max_freq": 81050000
+                    }
+                    m.get(meta_url, json=payload) # setup /meta.json request mock
+
+                # test if board is not icebreaker
+                for build_num in range(4, 8):
+                    # /build/:buildid
+                    build_url = f'https://hydra.vtr.tools/build/{build_num}'
+                    m.get(build_url, text=json_data)
+                    
+                    # /meta.json
+                    meta_url = f'https://hydra.vtr.tools/build/{build_num}/download/5/meta.json'
+                    payload = {
+                        "build_num": build_num,
+                        "date": "2020-07-30T22:12:40",
+                        "board": "arty",
+                        "max_freq": 81050000
+                    }
+                    m.get(meta_url, json=payload) # setup /meta.json request mock
+
+                # test if date and board are not included
+                for build_num in range(8, 12):
+                    # /build/:buildid
+                    build_url = f'https://hydra.vtr.tools/build/{build_num}'
+                    m.get(build_url, text=json_data)
+                    
+                    # /meta.json
+                    meta_url = f'https://hydra.vtr.tools/build/{build_num}/download/5/meta.json'
+                    payload = {
+                        "build_num": build_num,
+                        # date and build intentionally removed
+                        "max_freq": 81050000
+                    }
+                    m.get(meta_url, json=payload) # setup /meta.json request mock
+
+            # run tests on different eval_num
+            for eval_num in range(0, 3):
+                with self.subTest(eval_num=eval_num):
+                    # if mapping is not defined, should not remap
+                    hf = HydraFetcher(
+                        project="dusty",
+                        jobset="fpga-tool-perf",
+                        eval_num=eval_num,
+                        mapping={"build_num": "build_num"})
+                    result = hf.get_evaluation().get_df()
+
+                    # all results should have frequency converted from hz to mhz (no legacy icebreaker)
+                    expected = pd.DataFrame({
+                        "build_num": [x for x in range(eval_num * 4, eval_num * 4 + 4)],
+                        "freq": [81.05 for _ in range(4)]
+                        })
                     assert_frame_equal(result, expected)
 
                     eval_id = hf.get_evaluation().get_eval_id()
@@ -168,7 +326,7 @@ class TestHydraFetcherSmall(unittest.TestCase):
             # setup /evals request mock
             with open(evals_fn, "r") as f:
                 json_data = f.read()
-                m.get(evals_url, text=json_data) 
+                m.get(evals_url, text=json_data)
 
             # setup /build and /meta.json request mock
             with open(build_fn, "r") as f:
@@ -181,7 +339,11 @@ class TestHydraFetcherSmall(unittest.TestCase):
                     
                     # /meta.json
                     meta_url = f'https://hydra.vtr.tools/build/{build_num}/download/5/meta.json'
-                    payload = {"build_num": build_num}
+                    payload = {
+                        "build_num": build_num,
+                        "date": "2020-07-17T22:12:40",
+                        "board": "arty"
+                    }
                     m.get(meta_url, json=payload) # setup /meta.json request mock
 
             # test exclusion
@@ -243,15 +405,17 @@ class TestHydraFetcherSmall(unittest.TestCase):
                     payload = {
                         "max_freq": {
                             "clk": {
-                                "actual": 100
+                                "actual": 1000000
                             },
                             "clk_i": {
-                                "actual": 200
+                                "actual": 2000000
                             },
                             "sys_clk": {
-                                "actual": 300
+                                "actual": 3000000
                             }
-                        }
+                        },
+                        "date": "2020-07-17T22:12:40",
+                        "board": "arty"
                     }
                     m.get(url, json=payload) # setup /meta.json request mock
 
@@ -260,9 +424,9 @@ class TestHydraFetcherSmall(unittest.TestCase):
             # format test cases as tuples:
             # ({hydra_clock_names}, {expected_clock})
             test_cases = [
-                (["clk", "clk_i", "sys_clk"], 100),
-                (["clk_i", "sys_clk", "clk"], 200),
-                (["sys_clk", "clk", "clk_i"], 300),
+                (["clk", "clk_i", "sys_clk"], 1.0),
+                (["clk_i", "sys_clk", "clk"], 2.0),
+                (["sys_clk", "clk", "clk_i"], 3.0),
             ]
 
             for hydra_clock_name, expected_clock in test_cases:
@@ -286,7 +450,7 @@ class TestHydraFetcherSmall(unittest.TestCase):
                 hydra_clock_names=[])
             result = hf.get_evaluation().get_df()
 
-            expected_col = [100 for _ in range(4)]
+            expected_col = [1.0 for _ in range(4)]
             expected_series = pd.Series(expected_col, name="freq")
             assert_series_equal(result["freq"], expected_series)
 
diff --git a/tests/test_helpers.py b/tests/test_helpers.py
index fed1e6f..e343be5 100644
--- a/tests/test_helpers.py
+++ b/tests/test_helpers.py
@@ -39,30 +39,13 @@ class TestHelpers():
         result = get_versions({})
         assert result == {}
 
-    def test_rescale_actual_freq(self):
-        """ Test if rescaling works correctly """
-        result = rescale_actual_freq(1_000_000)
-        assert result == 1_000_000
-
-        result = rescale_actual_freq(1_000_001)
-        assert result == 1.000001
-
-        result = rescale_actual_freq(1000)
-        assert result == 1000
-
-        result = rescale_actual_freq(32_000_000)
-        assert result == 32
-
-        result = rescale_actual_freq(5_000_000)
-        assert result == 5
-
     def test_get_actual_freq(self):
         """ Test if get_actual_freq correctly extracts max frequency """
         obj = {
-            "max_freq": 500
+            "max_freq": 5_000_000
         }
         result = get_actual_freq(flatten(obj))
-        assert result == 500
+        assert result == 5_000_000
 
         obj = {
             "max_freq": {
@@ -72,7 +55,7 @@ class TestHelpers():
             }
         }
         result = get_actual_freq(flatten(obj))
-        assert result == 12
+        assert result == 12_000_000
 
     def test_get_actual_freq_hydra_clock_names(self):
         """ Test if get_actual_freq correctly selects the most important clock """
@@ -89,7 +72,7 @@ class TestHelpers():
             }
         }
         result = get_actual_freq(flatten(obj), ["clk", "sys_clk", "clk_i"])
-        assert result == 12
+        assert result == 12_000_000
 
         # should select sys_clk since it is higher priority
         obj = {
@@ -103,7 +86,7 @@ class TestHelpers():
             }
         }
         result = get_actual_freq(flatten(obj), ["clk", "sys_clk", "clk_i"])
-        assert result == 24
+        assert result == 24_000_000
 
         # should select clk_i since it is higher priority 
         obj = {
@@ -117,7 +100,7 @@ class TestHelpers():
             }
         }
         result = get_actual_freq(flatten(obj), ["clk_i", "sys_clk", "clk"])
-        assert result == 12
+        assert result == 12_000_000
 
         # should select shortest clock name since none are specified
         obj = {
@@ -131,7 +114,7 @@ class TestHelpers():
             }
         }
         result = get_actual_freq(flatten(obj), ["clk", "sys_clk", "clk_i"])
-        assert result == 12
+        assert result == 12_000_000
     
     def test_get_styling(self):
         """ Test if get_styling correctly queries colormap and returns correct