Repository: microsoft/hi-ml

Issue:
Submitting a job to a compute instance fails because the number of nodes can't be read
compute instances don't have a flag "scale_settings".


Discussion:

Resolving patch:
diff --git a/hi-ml-azure/src/health_azure/himl.py b/hi-ml-azure/src/health_azure/himl.py
--- a/hi-ml-azure/src/health_azure/himl.py
+++ b/hi-ml-azure/src/health_azure/himl.py
@@ -128,53 +128,21 @@ class AzureRunInfo:
     folder will be uploaded to blob storage regularly during the script run."""
 
 
-def validate_num_nodes(compute_cluster: ComputeTarget, num_nodes: int) -> None:
+def validate_compute_cluster(workspace: Workspace, compute_cluster_name: str) -> None:
     """
-    Check that the user hasn't requested more nodes than the maximum number of nodes allowed by
-    their compute cluster
+    Check that the specified compute cluster exists in the given Workspace
 
-    :param compute_cluster: An AML ComputeTarget representing the cluster whose upper node limit
-        should be checked
-    :param num_nodes: The number of nodes that the user has requested
-    """
-    max_cluster_nodes: int = compute_cluster.scale_settings.maximum_node_count
-    if num_nodes > max_cluster_nodes:
-        raise ValueError(
-            f"You have requested {num_nodes} nodes, which is more than your compute cluster "
-            f"({compute_cluster.name})'s maximum of {max_cluster_nodes} nodes."
-        )
-
-
-def validate_compute_name(existing_compute_targets: Dict[str, ComputeTarget], compute_target_name: str) -> None:
-    """
-    Check that a specified compute target is one of the available existing compute targets in a Workspace
-
-    :param existing_compute_targets: A list of AML ComputeTarget objects available to a given AML Workspace
-    :param compute_cluster_name: The name of the specific compute target whose name to look up in existing
-        compute targets
+    :param workspace: The AML Workspace to check
+    :param compute_cluster_name: The name of the specific compute cluster whose properties should be checked
     """
-    if compute_target_name not in existing_compute_targets:
+    existing_compute_targets: Dict[str, ComputeTarget] = workspace.compute_targets
+    if compute_cluster_name not in existing_compute_targets:
         raise ValueError(
-            f"Could not find the compute target {compute_target_name} in the AzureML workspace. ",
-            f"Existing compute targets: {list(existing_compute_targets)}",
+            f"Could not find compute target '{compute_cluster_name}' in the AzureML workspace. ",
+            f"Existing compute targets: {list(existing_compute_targets.keys())}",
         )
 
 
-def validate_compute_cluster(workspace: Workspace, compute_cluster_name: str, num_nodes: int) -> None:
-    """
-    Check that both the specified compute cluster exists in the given Workspace, and that it has enough
-    nodes to spin up the requested number of nodes
-
-    :param existing_compute_clusters: A list of AML ComputeTarget objects in a given AML Workspace
-    :param compute_cluster_name: The name of the specific compute cluster whose properties should be checked
-    :param num_nodes: The number of nodes that the user has requested
-    """
-    existing_compute_clusters: Dict[str, ComputeTarget] = workspace.compute_targets
-    validate_compute_name(existing_compute_clusters, compute_cluster_name)
-    compute_cluster = existing_compute_clusters[compute_cluster_name]
-    validate_num_nodes(compute_cluster, num_nodes)
-
-
 def create_run_configuration(
     workspace: Workspace,
     compute_cluster_name: str,
@@ -247,7 +215,7 @@ def create_run_configuration(
     if docker_shm_size:
         run_config.docker = DockerConfiguration(use_docker=True, shm_size=docker_shm_size)
 
-    validate_compute_cluster(workspace, compute_cluster_name, num_nodes)
+    validate_compute_cluster(workspace, compute_cluster_name)
 
     run_config.target = compute_cluster_name
 


Test patch:
diff --git a/hi-ml-azure/testazure/testazure/test_himl.py b/hi-ml-azure/testazure/testazure/test_himl.py
--- a/hi-ml-azure/testazure/testazure/test_himl.py
+++ b/hi-ml-azure/testazure/testazure/test_himl.py
@@ -73,6 +73,7 @@
     current_test_name,
     get_shared_config_json,
     repository_root,
+    DEFAULT_WORKSPACE,
 )
 
 INEXPENSIVE_TESTING_CLUSTER_NAME = "lite-testing-ds2"
@@ -80,8 +81,6 @@
 GITHUB_SHIBBOLETH = "GITHUB_RUN_ID"  # https://docs.github.com/en/actions/reference/environment-variables
 AZUREML_FLAG = himl.AZUREML_FLAG
 
-TEST_ML_CLIENT = get_ml_client()
-
 logger = logging.getLogger('test.health_azure')
 logger.setLevel(logging.DEBUG)
 
@@ -172,94 +171,18 @@ def mock_workspace(mock_compute_cluster: MagicMock, dummy_compute_cluster_name:
     return MagicMock(compute_targets={dummy_compute_cluster_name: mock_compute_cluster})
 
 
-@pytest.mark.fast
-def test_validate_num_nodes(
-    dummy_max_num_nodes_available: int, mock_compute_cluster: MagicMock, dummy_compute_cluster_name: str
-) -> None:
-    # If number of requested nodes <= max available nodes, nothing should happen
-    num_nodes_requested = dummy_max_num_nodes_available // 2
-    himl.validate_num_nodes(mock_compute_cluster, num_nodes_requested)
-    num_nodes_requested = dummy_max_num_nodes_available
-    himl.validate_num_nodes(mock_compute_cluster, num_nodes_requested)
-
-    # But if number of nodes requested > max available, a ValueError should be raised
-    num_nodes_requested = randint(dummy_max_num_nodes_available + 1, 5000)
-    expected_error_msg = f"You have requested {num_nodes_requested} nodes, which is more than your compute "
-    f"cluster {dummy_compute_cluster_name}'s maximum of {dummy_max_num_nodes_available} nodes "
-    with pytest.raises(ValueError, match=expected_error_msg):
-        himl.validate_num_nodes(mock_compute_cluster, num_nodes_requested)
-
-
-@pytest.mark.fast
-def test_validate_compute_name(mock_workspace: MagicMock, dummy_compute_cluster_name: str) -> None:
-    existing_compute_clusters = mock_workspace.compute_targets
-    existent_compute_name = dummy_compute_cluster_name
-    himl.validate_compute_name(existing_compute_clusters, existent_compute_name)
-
-    nonexistent_compute_name = 'idontexist'
-    assert nonexistent_compute_name not in existing_compute_clusters
-    expected_error_msg = f"Could not find the compute target {nonexistent_compute_name} in the AzureML workspace."
-    with pytest.raises(ValueError, match=expected_error_msg):
-        himl.validate_compute_name(existing_compute_clusters, nonexistent_compute_name)
-
-
-@pytest.mark.fast
-@patch("health_azure.himl.validate_compute_name")
-@patch("health_azure.himl.validate_num_nodes")
-def test_validate_compute(
-    mock_validate_num_nodes: MagicMock,
-    mock_validate_compute_name: MagicMock,
-    mock_workspace: MagicMock,
-    dummy_compute_cluster_name: str,
-) -> None:
-    def _raise_value_error(*args: Any) -> None:
-        raise ValueError("A ValueError has been raised")
-
-    def _raise_assertion_error(*args: Any) -> None:
-        raise AssertionError("An AssertionError has been raised")
-
-    # first mock the case where validate_num_nodes and validate_compute_name both return None
-    mock_validate_compute_name.return_value = None
-    mock_validate_num_nodes.return_value = None
-    mock_num_available_nodes = 0
-    himl.validate_compute_cluster(mock_workspace, dummy_compute_cluster_name, mock_num_available_nodes)
-    assert mock_validate_num_nodes.call_count == 1
-    assert mock_validate_compute_name.call_count == 1
-
-    # now have validate_num_nodes raise an Assertionerror and check that calling validate_compute_cluster
-    # raises the error
-    mock_validate_num_nodes.side_effect = _raise_assertion_error
-    with pytest.raises(AssertionError, match="An AssertionError has been raised"):
-        himl.validate_compute_cluster(mock_workspace, dummy_compute_cluster_name, mock_num_available_nodes)
-    assert mock_validate_num_nodes.call_count == 2
-    assert mock_validate_compute_name.call_count == 2
-
-    # now have validate_compute_name raise a ValueError and check that calling validate_compute_cluster raises the error
-    mock_validate_compute_name.side_effect = _raise_value_error
-    with pytest.raises(ValueError, match="A ValueError has been raised"):
-        himl.validate_compute_cluster(mock_workspace, dummy_compute_cluster_name, mock_num_available_nodes)
-    assert mock_validate_num_nodes.call_count == 2
-    assert mock_validate_compute_name.call_count == 3
-
-
-def test_validate_compute_real(tmp_path: Path) -> None:
+def test_validate_compute() -> None:
     """
-    Get a real Workspace object and attempt to validate a compute cluster from it, if any exist.
-    This checks that the properties/ methods in the Azure ML SDK are consistent with those used in our
-    codebase
+    Get a real Workspace object and attempt to validate a compute cluster from it.
     """
-    with check_config_json(tmp_path, shared_config_json=get_shared_config_json()):
-        workspace = get_workspace(aml_workspace=None, workspace_config_path=tmp_path / WORKSPACE_CONFIG_JSON)
+    workspace = DEFAULT_WORKSPACE.workspace
     existing_compute_targets: Dict[str, ComputeTarget] = workspace.compute_targets
-    if len(existing_compute_targets) == 0:
-        return
+    assert len(existing_compute_targets) > 0, "Expecting at least one compute target for this test"
     compute_target_name: str = list(existing_compute_targets)[0]
-    compute_target: ComputeTarget = existing_compute_targets[compute_target_name]
-    max_num_nodes = compute_target.scale_settings.maximum_node_count
-    request_num_nodes = max_num_nodes - 1
-    assert isinstance(compute_target_name, str)
-    assert isinstance(compute_target, ComputeTarget)
-    himl.validate_compute_cluster(workspace, compute_target_name, request_num_nodes)
+    himl.validate_compute_cluster(workspace, compute_target_name)
+    does_not_exist = uuid4().hex
+    with pytest.raises(ValueError, match="Could not find compute target"):
+        himl.validate_compute_cluster(workspace, does_not_exist)
 
 
 @pytest.mark.fast
@@ -332,7 +255,7 @@ def test_create_run_configuration_fails(
         himl.create_run_configuration(
             conda_environment_file=Path(__file__), compute_cluster_name="b", workspace=mock_workspace
         )
-    assert "Could not find the compute target b in the AzureML workspace" in str(e.value)
+    assert "Could not find compute target 'b' in the AzureML workspace" in str(e.value)
     assert existing_compute_target in str(e.value)
 
 
@@ -1754,7 +1677,7 @@ def test_get_data_asset_from_config() -> None:
     ]
 
     test_assets = [
-        himl.get_data_asset_from_config(TEST_ML_CLIENT, test_dataset_config)
+        himl.get_data_asset_from_config(DEFAULT_WORKSPACE.ml_client, test_dataset_config)
         for test_dataset_config in test_dataset_configs
     ]
     assert len(test_assets) == n_configs
@@ -1766,7 +1689,7 @@ def test_get_data_asset_from_config() -> None:
         datastore=TEST_DATASTORE_NAME,
         version=test_version,
     )
-    test_versioned_asset = himl.get_data_asset_from_config(TEST_ML_CLIENT, test_versioned_config)
+    test_versioned_asset = himl.get_data_asset_from_config(DEFAULT_WORKSPACE.ml_client, test_versioned_config)
     assert test_versioned_asset.version == str(test_version)
 
 
@@ -1841,7 +1764,7 @@ def test_create_v2_outputs() -> None:
         name=TEST_DATA_ASSET_NAME,
         datastore=TEST_DATASTORE_NAME,
     )
-    outputs = himl.create_v2_outputs(TEST_ML_CLIENT, [test_dataset_config])
+    outputs = himl.create_v2_outputs(DEFAULT_WORKSPACE.ml_client, [test_dataset_config])
 
     output_key = "OUTPUT_0"
     assert output_key in outputs