Repository: microsoft/MLOS

Issue:
Support log scale and quantization of numeric range of the Tunables



Discussion:

Resolving patch:
diff --git a/mlos_bench/mlos_bench/optimizers/convert_configspace.py b/mlos_bench/mlos_bench/optimizers/convert_configspace.py
--- a/mlos_bench/mlos_bench/optimizers/convert_configspace.py
+++ b/mlos_bench/mlos_bench/optimizers/convert_configspace.py
@@ -89,6 +89,8 @@ def _tunable_to_configspace(
                 name=tunable.name,
                 lower=tunable.range[0],
                 upper=tunable.range[1],
+                log=tunable.is_log,
+                q=tunable.quantization,
                 default_value=tunable.default if tunable.in_range(tunable.default) else None,
                 meta=meta)
         })
@@ -108,6 +110,8 @@ def _tunable_to_configspace(
             name=tunable.name,
             lower=tunable.range[0],
             upper=tunable.range[1],
+            log=tunable.is_log,
+            q=tunable.quantization,
             default_value=tunable.default if tunable.in_range(tunable.default) else None,
             meta=meta
         ),
diff --git a/mlos_bench/mlos_bench/tunables/tunable.py b/mlos_bench/mlos_bench/tunables/tunable.py
--- a/mlos_bench/mlos_bench/tunables/tunable.py
+++ b/mlos_bench/mlos_bench/tunables/tunable.py
@@ -32,6 +32,8 @@ class TunableDict(TypedDict, total=False):
     default: TunableValue
     values: Optional[List[Optional[str]]]
     range: Optional[Union[Sequence[int], Sequence[float]]]
+    quantization: Optional[Union[int, float]]
+    log: Optional[bool]
     special: Optional[Union[List[int], List[float]]]
     values_weights: Optional[List[float]]
     special_weights: Optional[List[float]]
@@ -76,6 +78,8 @@ def __init__(self, name: str, config: TunableDict):
             self._values = [str(v) if v is not None else v for v in self._values]
         self._meta: Dict[str, Any] = config.get("meta", {})
         self._range: Optional[Union[Tuple[int, int], Tuple[float, float]]] = None
+        self._quantization: Optional[Union[int, float]] = config.get("quantization")
+        self._log: Optional[bool] = config.get("log")
         config_range = config.get("range")
         if config_range is not None:
             assert len(config_range) == 2, f"Invalid range: {config_range}"
@@ -91,6 +95,7 @@ def __init__(self, name: str, config: TunableDict):
         self.value = self._default
 
     def _sanity_check(self) -> None:
+        # pylint: disable=too-complex,too-many-branches
         """
         Check if the status of the Tunable is valid, and throw ValueError if it is not.
         """
@@ -105,6 +110,10 @@ def _sanity_check(self) -> None:
                 raise ValueError(f"Categorical tunable cannot have special values: {self}")
             if self._range_weight is not None:
                 raise ValueError(f"Categorical tunable cannot have range_weight: {self}")
+            if self._log is not None:
+                raise ValueError(f"Categorical tunable cannot have log parameter: {self}")
+            if self._quantization is not None:
+                raise ValueError(f"Categorical tunable cannot have quantization parameter: {self}")
             if self._weights:
                 if len(self._weights) != len(self._values):
                     raise ValueError(f"Must specify weights for all values: {self}")
@@ -115,6 +124,17 @@ def _sanity_check(self) -> None:
                 raise ValueError(f"Values must be None for the numerical type tunable {self}")
             if not self._range or len(self._range) != 2 or self._range[0] >= self._range[1]:
                 raise ValueError(f"Invalid range for tunable {self}: {self._range}")
+            if self._quantization is not None:
+                if self.dtype == int:
+                    if not isinstance(self._quantization, int):
+                        raise ValueError(f"Quantization of a int param should be an int: {self}")
+                    if self._quantization <= 1:
+                        raise ValueError(f"Number of quantization points is <= 1: {self}")
+                if self.dtype == float:
+                    if not isinstance(self._quantization, (float, int)):
+                        raise ValueError(f"Quantization of a float param should be a float or int: {self}")
+                    if self._quantization <= 0:
+                        raise ValueError(f"Number of quantization points is <= 0: {self}")
             if self._weights:
                 if self._range_weight is None:
                     raise ValueError(f"Must specify weight for the range: {self}")
@@ -138,6 +158,7 @@ def __repr__(self) -> str:
         string : str
             A human-readable version of the Tunable.
         """
+        # TODO? Add weights, specials, quantization, distribution?
         if self.is_categorical:
             return f"{self._name}[{self._type}]({self._values}:{self._default})={self._current_value}"
         return f"{self._name}[{self._type}]({self._range}:{self._default})={self._current_value}"
@@ -416,6 +437,9 @@ def range_weight(self) -> Optional[float]:
         weight : float
             Weight of the range or None.
         """
+        assert self.is_numerical
+        assert self._special
+        assert self._weights
         return self._range_weight
 
     @property
@@ -483,6 +507,32 @@ def range(self) -> Union[Tuple[int, int], Tuple[float, float]]:
         assert self._range is not None
         return self._range
 
+    @property
+    def quantization(self) -> Optional[Union[int, float]]:
+        """
+        Get the number of quantization points, if specified.
+
+        Returns
+        -------
+        quantization : int, float, None
+            Number of quantization points or None.
+        """
+        assert self.is_numerical
+        return self._quantization
+
+    @property
+    def is_log(self) -> Optional[bool]:
+        """
+        Check if numeric tunable is log scale.
+
+        Returns
+        -------
+        log : bool
+            True if numeric tunable is log scale, False if linear.
+        """
+        assert self.is_numerical
+        return self._log
+
     @property
     def categories(self) -> List[Optional[str]]:
         """


Test patch:
diff --git a/mlos_bench/mlos_bench/tests/config/schemas/tunable-params/test-cases/bad/invalid/tunable-params-float-bad-log.jsonc b/mlos_bench/mlos_bench/tests/config/schemas/tunable-params/test-cases/bad/invalid/tunable-params-float-bad-log.jsonc
new file mode 100644
--- /dev/null
+++ b/mlos_bench/mlos_bench/tests/config/schemas/tunable-params/test-cases/bad/invalid/tunable-params-float-bad-log.jsonc
@@ -0,0 +1,13 @@
+{
+    "covariant_group_name-1": {
+        "cost": 1,
+        "params": {
+            "float": {
+                "type": "float",
+                "default": 10,
+                "range": [0, 10],
+                "log": "yes"  // <-- this is invalid
+            }
+        }
+    }
+}
diff --git a/mlos_bench/mlos_bench/tests/config/schemas/tunable-params/test-cases/bad/invalid/tunable-params-float-bad-quantization-type.jsonc b/mlos_bench/mlos_bench/tests/config/schemas/tunable-params/test-cases/bad/invalid/tunable-params-float-bad-quantization-type.jsonc
new file mode 100644
--- /dev/null
+++ b/mlos_bench/mlos_bench/tests/config/schemas/tunable-params/test-cases/bad/invalid/tunable-params-float-bad-quantization-type.jsonc
@@ -0,0 +1,13 @@
+{
+    "covariant_group_name-1": {
+        "cost": 1,
+        "params": {
+            "float": {
+                "type": "float",
+                "default": 10,
+                "range": [0, 10],
+                "quantization": true  // <-- this is invalid
+            }
+        }
+    }
+}
diff --git a/mlos_bench/mlos_bench/tests/config/schemas/tunable-params/test-cases/bad/invalid/tunable-params-int-bad-float-quantization.jsonc b/mlos_bench/mlos_bench/tests/config/schemas/tunable-params/test-cases/bad/invalid/tunable-params-int-bad-float-quantization.jsonc
new file mode 100644
--- /dev/null
+++ b/mlos_bench/mlos_bench/tests/config/schemas/tunable-params/test-cases/bad/invalid/tunable-params-int-bad-float-quantization.jsonc
@@ -0,0 +1,13 @@
+{
+    "covariant_group_name-1": {
+        "cost": 1,
+        "params": {
+            "float": {
+                "type": "float",
+                "default": 10,
+                "range": [1, 500],
+                "quantization": 0   // <-- should be greater than 0
+            }
+        }
+    }
+}
diff --git a/mlos_bench/mlos_bench/tests/config/schemas/tunable-params/test-cases/bad/invalid/tunable-params-int-bad-int-quantization.jsonc b/mlos_bench/mlos_bench/tests/config/schemas/tunable-params/test-cases/bad/invalid/tunable-params-int-bad-int-quantization.jsonc
new file mode 100644
--- /dev/null
+++ b/mlos_bench/mlos_bench/tests/config/schemas/tunable-params/test-cases/bad/invalid/tunable-params-int-bad-int-quantization.jsonc
@@ -0,0 +1,13 @@
+{
+    "covariant_group_name-1": {
+        "cost": 1,
+        "params": {
+            "int": {
+                "type": "int",
+                "default": 10,
+                "range": [1, 500],
+                "quantization": 1   // <-- should be greater than 1
+            }
+        }
+    }
+}
diff --git a/mlos_bench/mlos_bench/tests/config/schemas/tunable-params/test-cases/bad/invalid/tunable-params-int-bad-log.jsonc b/mlos_bench/mlos_bench/tests/config/schemas/tunable-params/test-cases/bad/invalid/tunable-params-int-bad-log.jsonc
new file mode 100644
--- /dev/null
+++ b/mlos_bench/mlos_bench/tests/config/schemas/tunable-params/test-cases/bad/invalid/tunable-params-int-bad-log.jsonc
@@ -0,0 +1,13 @@
+{
+    "covariant_group_name-1": {
+        "cost": 1,
+        "params": {
+            "int": {
+                "type": "int",
+                "default": 10,
+                "range": [1, 500],
+                "log": 1 // <-- this is invalid
+            }
+        }
+    }
+}
diff --git a/mlos_bench/mlos_bench/tests/config/schemas/tunable-params/test-cases/bad/invalid/tunable-params-int-wrong-quantization-type.jsonc b/mlos_bench/mlos_bench/tests/config/schemas/tunable-params/test-cases/bad/invalid/tunable-params-int-wrong-quantization-type.jsonc
new file mode 100644
--- /dev/null
+++ b/mlos_bench/mlos_bench/tests/config/schemas/tunable-params/test-cases/bad/invalid/tunable-params-int-wrong-quantization-type.jsonc
@@ -0,0 +1,13 @@
+{
+    "covariant_group_name-1": {
+        "cost": 1,
+        "params": {
+            "int": {
+                "type": "int",
+                "default": 10,
+                "range": [1, 500],
+                "quantization": "yes"  // <-- this is invalid
+            }
+        }
+    }
+}
diff --git a/mlos_bench/mlos_bench/tests/config/schemas/tunable-params/test-cases/good/full/full-tunable-params-test.jsonc b/mlos_bench/mlos_bench/tests/config/schemas/tunable-params/test-cases/good/full/full-tunable-params-test.jsonc
--- a/mlos_bench/mlos_bench/tests/config/schemas/tunable-params/test-cases/good/full/full-tunable-params-test.jsonc
+++ b/mlos_bench/mlos_bench/tests/config/schemas/tunable-params/test-cases/good/full/full-tunable-params-test.jsonc
@@ -11,7 +11,9 @@
                 "meta": {"suffix": "MB"},
                 "special": [-1],
                 "special_weights": [0.1],
-                "range_weight": 0.9
+                "range_weight": 0.9,
+                "quantization": 50,
+                "log": true
             },
             "float": {
                 "description": "Float",
@@ -19,7 +21,9 @@
                 "default": 10.1,
                 "meta": {"scale": 1000, "prefix": "/proc/var/random/", "base": 2.71828},
                 "range": [1.1, 111.1],
-                "special": [-1.1]
+                "special": [-1.1],
+                "quantization": 10,
+                "log": false
             },
             "cat": {
                 "description": "Cat",
diff --git a/mlos_bench/mlos_bench/tests/tunable_groups_fixtures.py b/mlos_bench/mlos_bench/tests/tunable_groups_fixtures.py
--- a/mlos_bench/mlos_bench/tests/tunable_groups_fixtures.py
+++ b/mlos_bench/mlos_bench/tests/tunable_groups_fixtures.py
@@ -55,13 +55,15 @@
                 // FLAML requires uniform weights, separately for
                 // specials and switching between specials and range.
                 "special_weights": [0.25, 0.25],
-                "range_weight": 0.5
+                "range_weight": 0.5,
+                "log": false
             },
             "kernel_sched_latency_ns": {
                 "description": "Initial value for the scheduler period",
                 "type": "int",
                 "default": 2000000,
-                "range": [0, 1000000000]
+                "range": [0, 1000000000],
+                "log": false
             }
         }
     }
diff --git a/mlos_bench/mlos_bench/tests/tunables/tunable_definition_test.py b/mlos_bench/mlos_bench/tests/tunables/tunable_definition_test.py
--- a/mlos_bench/mlos_bench/tests/tunables/tunable_definition_test.py
+++ b/mlos_bench/mlos_bench/tests/tunables/tunable_definition_test.py
@@ -247,6 +247,43 @@ def test_numerical_weights(tunable_type: str) -> None:
     assert tunable.range_weight == 0.9
 
 
+@pytest.mark.parametrize("tunable_type", ["int", "float"])
+def test_numerical_quantization(tunable_type: str) -> None:
+    """
+    Instantiate a numerical tunable with quantization.
+    """
+    json_config = f"""
+    {{
+        "type": "{tunable_type}",
+        "range": [0, 100],
+        "quantization": 10,
+        "default": 0
+    }}
+    """
+    config = json.loads(json_config)
+    tunable = Tunable(name='test', config=config)
+    assert tunable.quantization == 10
+    assert not tunable.is_log
+
+
+@pytest.mark.parametrize("tunable_type", ["int", "float"])
+def test_numerical_log(tunable_type: str) -> None:
+    """
+    Instantiate a numerical tunable with log scale.
+    """
+    json_config = f"""
+    {{
+        "type": "{tunable_type}",
+        "range": [0, 100],
+        "log": true,
+        "default": 0
+    }}
+    """
+    config = json.loads(json_config)
+    tunable = Tunable(name='test', config=config)
+    assert tunable.is_log
+
+
 @pytest.mark.parametrize("tunable_type", ["int", "float"])
 def test_numerical_weights_no_specials(tunable_type: str) -> None:
     """
@@ -384,6 +421,24 @@ def test_numerical_weights_wrong_values(tunable_type: str) -> None:
         Tunable(name='test', config=config)
 
 
+@pytest.mark.parametrize("tunable_type", ["int", "float"])
+def test_numerical_quantization_wrong(tunable_type: str) -> None:
+    """
+    Instantiate a numerical tunable with invalid number of quantization points.
+    """
+    json_config = f"""
+    {{
+        "type": "{tunable_type}",
+        "range": [0, 100],
+        "quantization": 0,
+        "default": 0
+    }}
+    """
+    config = json.loads(json_config)
+    with pytest.raises(ValueError):
+        Tunable(name='test', config=config)
+
+
 def test_bad_type() -> None:
     """
     Disallow bad types.