Repository: microsoft/ray-on-aml (language: python)

Issue:
Update telemetry and fix installation bugs



Discussion / hints:

Fix patch:
diff --git a/examples/quick_start_examples.ipynb b/examples/quick_start_examples.ipynb
index 4122b37..1f07969 100644
--- a/examples/quick_start_examples.ipynb
+++ b/examples/quick_start_examples.ipynb
@@ -1,11 +1,12 @@
 {
  "cells": [
   {
+   "attachments": {},
    "cell_type": "markdown",
    "metadata": {},
    "source": [
     "\n",
-    "## Quick examples to demostrate AML Ray/Dask cluster usage"
+    "### Quick examples to demostrate AML Ray/Dask cluster usage"
    ]
   },
   {
@@ -15,13 +16,29 @@
     "### Interactive use cases"
    ]
   },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Install at this your conda notebook environment to run interactive examples"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
-    "pip install --upgrade ray-on-aml pandas pyarrow xgboost xgboost_ray==0.1.6"
+    "pip install --upgrade ray==2.2.0 ray[air]==2.2.0 ray[data]==2.2.0 azure-ai-ml ray-on-aml"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Start ML client "
    ]
   },
   {
@@ -34,15 +51,17 @@
    },
    "outputs": [],
    "source": [
-    "from azureml.core import Workspace, Experiment, Environment,ScriptRunConfig\n",
-    "# from azureml.widgets import RunDetails\n",
-    "from azureml.core.compute import ComputeTarget, AmlCompute\n",
-    "from azureml.core.compute_target import ComputeTargetException\n",
-    "from azureml.core.environment import Environment\n",
-    "from IPython.display import clear_output\n",
-    "import time\n",
-    "\n",
-    "\n"
+    "from azure.ai.ml import MLClient\n",
+    "from azure.ai.ml import command, Input\n",
+    "from azure.identity import DefaultAzureCredential\n",
+    "# Enter details of your AML workspace\n",
+    "subscription_id = \"840b5c5c-3f4a-459a-94fc-6bad2a969f9d\"\n",
+    "resource_group = \"ml\"\n",
+    "workspace = \"ws02ent\"\n",
+    "# get a handle to the workspace\n",
+    "ml_client = MLClient(\n",
+    "    DefaultAzureCredential(), subscription_id, resource_group, workspace\n",
+    ")\n"
    ]
   },
   {
@@ -54,7 +73,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": 7,
    "metadata": {
     "gather": {
      "logged": 1639185778952
@@ -62,45 +81,75 @@
    },
    "outputs": [
     {
-     "ename": "UserErrorException",
-     "evalue": "UserErrorException:\n\tMessage: You are currently logged-in to 0fbe7234-45ea-498b-b7e4-1a8b2d3be4d9 tenant. You don't have access to 0e9bace8-7a81-4922-83b5-d995ff706507 subscription, please check if it is in this tenant. All the subscriptions that you have access to in this tenant are = \n [SubscriptionInfo(subscription_name=\"James Nguyen's Research and Demo Subscription\", subscription_id='840b5c5c-3f4a-459a-94fc-6bad2a969f9d')]. \n Please refer to aka.ms/aml-notebook-auth for different authentication mechanisms in azureml-sdk.\n\tInnerException None\n\tErrorResponse \n{\n    \"error\": {\n        \"code\": \"UserError\",\n        \"message\": \"You are currently logged-in to 0fbe7234-45ea-498b-b7e4-1a8b2d3be4d9 tenant. You don't have access to 0e9bace8-7a81-4922-83b5-d995ff706507 subscription, please check if it is in this tenant. All the subscriptions that you have access to in this tenant are = \\n [SubscriptionInfo(subscription_name=\\\"James Nguyen's Research and Demo Subscription\\\", subscription_id='840b5c5c-3f4a-459a-94fc-6bad2a969f9d')]. \\n Please refer to aka.ms/aml-notebook-auth for different authentication mechanisms in azureml-sdk.\"\n    }\n}",
-     "output_type": "error",
-     "traceback": [
-      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
-      "\u001b[0;31mUserErrorException\u001b[0m                        Traceback (most recent call last)",
-      "Cell \u001b[0;32mIn [3], line 3\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mray_on_aml\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mcore\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m Ray_On_AML\n\u001b[0;32m----> 3\u001b[0m ws \u001b[38;5;241m=\u001b[39m \u001b[43mWorkspace\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfrom_config\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m      4\u001b[0m \u001b[38;5;66;03m# ray_on_aml =Ray_On_AML(ws=ws, compute_cluster =\"d15-v2\",additional_pip_packages=['torch==1.10.0', 'torchvision', 'sklearn','xgboost_ray==0.1.6'], maxnode=2,exp_name='ray-on-aml-test')\u001b[39;00m\n\u001b[1;32m      6\u001b[0m ray_on_aml \u001b[38;5;241m=\u001b[39mRay_On_AML(ws\u001b[38;5;241m=\u001b[39mws, compute_cluster \u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124md13\u001b[39m\u001b[38;5;124m\"\u001b[39m, maxnode\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m2\u001b[39m)\n",
-      "File \u001b[0;32m/anaconda/envs/azureml_py38/lib/python3.8/site-packages/azureml/core/workspace.py:292\u001b[0m, in \u001b[0;36mWorkspace.from_config\u001b[0;34m(path, auth, _logger, _file_name)\u001b[0m\n\u001b[1;32m    288\u001b[0m subscription_id, resource_group, workspace_name \u001b[38;5;241m=\u001b[39m project_info\u001b[38;5;241m.\u001b[39mget_workspace_info(\n\u001b[1;32m    289\u001b[0m     found_path)\n\u001b[1;32m    291\u001b[0m _logger\u001b[38;5;241m.\u001b[39minfo(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mFound the config file in: \u001b[39m\u001b[38;5;132;01m%s\u001b[39;00m\u001b[38;5;124m'\u001b[39m, found_path)\n\u001b[0;32m--> 292\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mWorkspace\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    293\u001b[0m \u001b[43m    \u001b[49m\u001b[43mworkspace_name\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    294\u001b[0m \u001b[43m    \u001b[49m\u001b[43mauth\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mauth\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    295\u001b[0m \u001b[43m    \u001b[49m\u001b[43msubscription_id\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43msubscription_id\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    296\u001b[0m \u001b[43m    \u001b[49m\u001b[43mresource_group\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mresource_group\u001b[49m\u001b[43m)\u001b[49m\n",
-      "File \u001b[0;32m/anaconda/envs/azureml_py38/lib/python3.8/site-packages/azureml/core/workspace.py:607\u001b[0m, in \u001b[0;36mWorkspace.get\u001b[0;34m(name, auth, subscription_id, resource_group, location, cloud, id)\u001b[0m\n\u001b[1;32m    604\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m auth:\n\u001b[1;32m    605\u001b[0m     auth \u001b[38;5;241m=\u001b[39m InteractiveLoginAuthentication()\n\u001b[0;32m--> 607\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mWorkspace\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    608\u001b[0m \u001b[43m    \u001b[49m\u001b[43msubscription_id\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    609\u001b[0m \u001b[43m    \u001b[49m\u001b[43mresource_group\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    610\u001b[0m \u001b[43m    \u001b[49m\u001b[43mname\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    611\u001b[0m \u001b[43m    \u001b[49m\u001b[43mauth\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mauth\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    612\u001b[0m \u001b[43m    \u001b[49m\u001b[43m_location\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mlocation\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    613\u001b[0m \u001b[43m    \u001b[49m\u001b[43m_cloud\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcloud\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    614\u001b[0m \u001b[43m    \u001b[49m\u001b[43m_workspace_id\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mid\u001b[39;49m\u001b[43m)\u001b[49m\n",
-      "File \u001b[0;32m/anaconda/envs/azureml_py38/lib/python3.8/site-packages/azureml/core/workspace.py:205\u001b[0m, in \u001b[0;36mWorkspace.__init__\u001b[0;34m(self, subscription_id, resource_group, workspace_name, auth, _location, _disable_service_check, _workspace_id, sku, tags, _cloud)\u001b[0m\n\u001b[1;32m    202\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_workspace_autorest_object \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[1;32m    204\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m _disable_service_check:\n\u001b[0;32m--> 205\u001b[0m     auto_rest_workspace \u001b[38;5;241m=\u001b[39m \u001b[43m_commands\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget_workspace\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    206\u001b[0m \u001b[43m        \u001b[49m\u001b[43mauth\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43msubscription_id\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mresource_group\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mworkspace_name\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m_location\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m_cloud\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m_workspace_id\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    207\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_workspace_autorest_object \u001b[38;5;241m=\u001b[39m auto_rest_workspace\n\u001b[1;32m    209\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_service_context \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n",
-      "File \u001b[0;32m/anaconda/envs/azureml_py38/lib/python3.8/site-packages/azureml/_project/_commands.py:466\u001b[0m, in \u001b[0;36mget_workspace\u001b[0;34m(auth, subscription_id, resource_group_name, workspace_name, location, cloud, workspace_id)\u001b[0m\n\u001b[1;32m    464\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m    465\u001b[0m     \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 466\u001b[0m         workspaces \u001b[38;5;241m=\u001b[39m \u001b[43mauth\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_get_service_client\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    467\u001b[0m \u001b[43m            \u001b[49m\u001b[43mAzureMachineLearningWorkspaces\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    468\u001b[0m \u001b[43m            \u001b[49m\u001b[43msubscription_id\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241m.\u001b[39mworkspaces\n\u001b[1;32m    469\u001b[0m         \u001b[38;5;28;01mreturn\u001b[39;00m WorkspacesOperations\u001b[38;5;241m.\u001b[39mget(\n\u001b[1;32m    470\u001b[0m             workspaces,\n\u001b[1;32m    471\u001b[0m             resource_group_name,\n\u001b[1;32m    472\u001b[0m             workspace_name)\n\u001b[1;32m    473\u001b[0m     \u001b[38;5;28;01mexcept\u001b[39;00m ClientRequestError:\n\u001b[1;32m    474\u001b[0m         \u001b[38;5;66;03m# this most likely is because of failure to talk to arm.\u001b[39;00m\n\u001b[1;32m    475\u001b[0m         \u001b[38;5;66;03m# Provide error message to user if they are in synapse enviroment.\u001b[39;00m\n\u001b[1;32m    476\u001b[0m         \u001b[38;5;66;03m# Additional params will enable aml dataplane get workspace call. \u001b[39;00m\n",
-      "File \u001b[0;32m/anaconda/envs/azureml_py38/lib/python3.8/site-packages/azureml/core/authentication.py:230\u001b[0m, in \u001b[0;36mAbstractAuthentication._get_service_client\u001b[0;34m(self, client_class, subscription_id, subscription_bound, base_url, is_check_subscription)\u001b[0m\n\u001b[1;32m    228\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m subscription_id \u001b[38;5;129;01mand\u001b[39;00m is_check_subscription:\n\u001b[1;32m    229\u001b[0m     all_subscription_list, tenant_id \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_get_all_subscription_ids()\n\u001b[0;32m--> 230\u001b[0m     \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_check_if_subscription_exists\u001b[49m\u001b[43m(\u001b[49m\u001b[43msubscription_id\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mall_subscription_list\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtenant_id\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    232\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m base_url:\n\u001b[1;32m    233\u001b[0m     base_url \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_cloud_type\u001b[38;5;241m.\u001b[39mendpoints\u001b[38;5;241m.\u001b[39mresource_manager\n",
-      "File \u001b[0;32m/anaconda/envs/azureml_py38/lib/python3.8/site-packages/azureml/core/authentication.py:681\u001b[0m, in \u001b[0;36mInteractiveLoginAuthentication._check_if_subscription_exists\u001b[0;34m(self, subscription_id, subscription_id_list, tenant_id)\u001b[0m\n\u001b[1;32m    680\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_check_if_subscription_exists\u001b[39m(\u001b[38;5;28mself\u001b[39m, subscription_id, subscription_id_list, tenant_id):\n\u001b[0;32m--> 681\u001b[0m     \u001b[38;5;28;43msuper\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mInteractiveLoginAuthentication\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_check_if_subscription_exists\u001b[49m\u001b[43m(\u001b[49m\u001b[43msubscription_id\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    682\u001b[0m \u001b[43m                                                                              \u001b[49m\u001b[43msubscription_id_list\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtenant_id\u001b[49m\u001b[43m)\u001b[49m\n",
-      "File \u001b[0;32m/anaconda/envs/azureml_py38/lib/python3.8/site-packages/azureml/core/authentication.py:338\u001b[0m, in \u001b[0;36mAbstractAuthentication._check_if_subscription_exists\u001b[0;34m(self, subscription_id, subscription_id_list, tenant_id)\u001b[0m\n\u001b[1;32m    333\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m UserErrorException(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mIt looks like you have specified subscription name, \u001b[39m\u001b[38;5;132;01m{}\u001b[39;00m\u001b[38;5;124m, instead of \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    334\u001b[0m                              \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124msubscription id. Subscription names may not be unique, please specify \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    335\u001b[0m                              \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124msubscription id from this list \u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m \u001b[39m\u001b[38;5;132;01m{}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;241m.\u001b[39mformat(subscription_id,\n\u001b[1;32m    336\u001b[0m                                                                            subscription_id_list))\n\u001b[1;32m    337\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m--> 338\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m UserErrorException(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mYou are currently logged-in to \u001b[39m\u001b[38;5;132;01m{}\u001b[39;00m\u001b[38;5;124m tenant. You don\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mt have access \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    339\u001b[0m                              \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mto \u001b[39m\u001b[38;5;132;01m{}\u001b[39;00m\u001b[38;5;124m subscription, please check if it is in this tenant. \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    340\u001b[0m                              \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mAll the subscriptions that you have access to in this tenant are = \u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    341\u001b[0m                              \u001b[38;5;124m\"\u001b[39m\u001b[38;5;132;01m{}\u001b[39;00m\u001b[38;5;124m. \u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m Please refer to aka.ms/aml-notebook-auth for different \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    342\u001b[0m                              \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mauthentication mechanisms in azureml-sdk.\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;241m.\u001b[39mformat(tenant_id,\n\u001b[1;32m    343\u001b[0m                                                                                 subscription_id,\n\u001b[1;32m    344\u001b[0m                                                                                 subscription_id_list))\n",
-      "\u001b[0;31mUserErrorException\u001b[0m: UserErrorException:\n\tMessage: You are currently logged-in to 0fbe7234-45ea-498b-b7e4-1a8b2d3be4d9 tenant. You don't have access to 0e9bace8-7a81-4922-83b5-d995ff706507 subscription, please check if it is in this tenant. All the subscriptions that you have access to in this tenant are = \n [SubscriptionInfo(subscription_name=\"James Nguyen's Research and Demo Subscription\", subscription_id='840b5c5c-3f4a-459a-94fc-6bad2a969f9d')]. \n Please refer to aka.ms/aml-notebook-auth for different authentication mechanisms in azureml-sdk.\n\tInnerException None\n\tErrorResponse \n{\n    \"error\": {\n        \"code\": \"UserError\",\n        \"message\": \"You are currently logged-in to 0fbe7234-45ea-498b-b7e4-1a8b2d3be4d9 tenant. You don't have access to 0e9bace8-7a81-4922-83b5-d995ff706507 subscription, please check if it is in this tenant. All the subscriptions that you have access to in this tenant are = \\n [SubscriptionInfo(subscription_name=\\\"James Nguyen's Research and Demo Subscription\\\", subscription_id='840b5c5c-3f4a-459a-94fc-6bad2a969f9d')]. \\n Please refer to aka.ms/aml-notebook-auth for different authentication mechanisms in azureml-sdk.\"\n    }\n}"
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Warning: the provided asset name 'ray-on-aml-3490264930' will not be used for anonymous registration\n",
+      "Warning: the provided asset name 'ray-on-aml-3490264930' will not be used for anonymous registration\n",
+      "\u001b[32mUploading .tmp (0.01 MBs): 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 6679/6679 [00:00<00:00, 97620.47it/s]\u001b[0m\n",
+      "\u001b[39m\n",
+      "\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Waiting cluster to start and return head node's ip\n",
+      "............................................................................................................................................\n",
+      " cluster is ready, head node ip  10.0.0.6\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "\u001b[2m\u001b[33m(raylet, ip=10.0.0.8)\u001b[0m /azureml-envs/azureml_c0eb32c1b90d5f184f6500831f6bfab1/lib/python3.10/site-packages/ray/dashboard/agent.py:51: DeprecationWarning: There is no current event loop\n",
+      "\u001b[2m\u001b[33m(raylet, ip=10.0.0.8)\u001b[0m   aiogrpc.init_grpc_aio()\n"
      ]
     }
    ],
    "source": [
     "from ray_on_aml.core import Ray_On_AML\n",
     "\n",
-    "ws = Workspace.from_config()\n",
-    "# ray_on_aml =Ray_On_AML(ws=ws, compute_cluster =\"d15-v2\",additional_pip_packages=['torch==1.10.0', 'torchvision', 'sklearn','xgboost_ray==0.1.6'], maxnode=2,exp_name='ray-on-aml-test')\n",
+    "# from src.ray_on_aml.core import Ray_On_AML\n",
     "\n",
-    "ray_on_aml =Ray_On_AML(ws=ws, compute_cluster =\"d13\", maxnode=2)\n",
-    "ray = ray_on_aml.getRay()\n",
-    "# Note that by default, ci_is_head=True which means  compute instance as head node and all nodes in the remote compute cluster as workers \n",
-    "# But if you want to use one of the nodes in the remote AML compute cluster is used as head node and the remaining are worker nodes.\n",
-    "# then simply specify ray = ray_on_aml.getRay(ci_is_head=False)\n",
-    "# To install additional library, use additional_pip_packages and additional_conda_packages parameters.\n",
-    "time.sleep(50)\n",
-    "ray.cluster_resources()"
+    "import logging\n",
+    "ray_on_aml =Ray_On_AML(ml_client=ml_client, compute_cluster =\"d13\")\n",
+    "#Note that if you need to customize the pip installation of the cluster, you also needs to support the ray package e.g. ray[data] which \n",
+    "#match the version of the ray package(s) in your compute instance. If you don't specify pip_packages then ray[default] is inserted \n",
+    "#automatically\n",
+    "\n",
+    "# For use as client mode, uncomment these lines\n",
+    "ray = ray_on_aml.getRay(num_node=2,pip_packages=[\"ray[air]==2.2.0\",\"ray[data]==2.2.0\",\"torch==1.13.0\",\"fastparquet==2022.12.0\", \n",
+    "\"azureml-mlflow==1.48.0\", \"pyarrow==6.0.1\", \"dask==2022.12.0\", \"adlfs==2022.11.2\", \"fsspec==2022.11.0\"])\n",
+    "client = ray.init(f\"ray://{ray_on_aml.headnode_private_ip}:10001\")\n",
+    "\n",
+    "#use CI as head node\n",
+    "\n",
+    "# ray = ray_on_aml.getRay(ci_is_head=True, num_node=2)"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 8,
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "{'CPU': 16.0,\n",
+       " 'node:10.0.0.6': 1.0,\n",
+       " 'node:10.0.0.8': 1.0,\n",
+       " 'memory': 87326674944.0,\n",
+       " 'object_store_memory': 24481313586.0}"
+      ]
+     },
+     "execution_count": 8,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
    "source": [
     "ray.cluster_resources()"
    ]
@@ -126,19 +175,55 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 9,
    "metadata": {
     "gather": {
      "logged": 1639183635290
     }
    },
-   "outputs": [],
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "vendorID                19233765\n",
+       "lpepPickupDatetime      19233765\n",
+       "lpepDropoffDatetime     19233765\n",
+       "passengerCount          19233765\n",
+       "tripDistance            19233765\n",
+       "puLocationId                   0\n",
+       "doLocationId                   0\n",
+       "pickupLongitude         19233765\n",
+       "pickupLatitude          19233765\n",
+       "dropoffLongitude        19233765\n",
+       "dropoffLatitude         19233765\n",
+       "rateCodeID              19233765\n",
+       "storeAndFwdFlag         19233765\n",
+       "paymentType             19233765\n",
+       "fareAmount              19233765\n",
+       "extra                   19233765\n",
+       "mtaTax                  19233765\n",
+       "improvementSurcharge    19233765\n",
+       "tipAmount               19233765\n",
+       "tollsAmount             19233765\n",
+       "ehailFee                       0\n",
+       "totalAmount             19233765\n",
+       "tripType                19233693\n",
+       "puYear                  19233765\n",
+       "puMonth                 19233765\n",
+       "dtype: int64"
+      ]
+     },
+     "execution_count": 9,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
    "source": [
     "#Scaling up date with Dask dataframe API.\n",
     "#Please make sure you have pandas version 1.4+ and restart to run this successfully.\n",
     "# pip install pandas==1.4.2\n",
     "import dask\n",
-    "from ray.util.dask import ray_dask_get,enable_dask_on_ray\n",
+    "from ray.util.dask import enable_dask_on_ray\n",
     "enable_dask_on_ray()\n",
     "\n",
     "import dask.dataframe as dd\n",
@@ -157,13 +242,31 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 10,
    "metadata": {
     "gather": {
      "logged": 1639186778862
     }
    },
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\u001b[2m\u001b[36m(_get_read_tasks pid=589)\u001b[0m [dataset]: Run `pip install tqdm` to enable progress reporting.\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "116341246"
+      ]
+     },
+     "execution_count": 10,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
    "source": [
     "#Ray also support Ray dataset. You can read into ray dataset then convert to Dask or other ML format which is convenient for ML training.https://docs.ray.io/en/latest/data/dataset.html\n",
     "# you may need to upgrade pyarrow to run this successfully\n",
@@ -172,7 +275,27 @@
     "abfs = AzureBlobFileSystem(account_name=\"azureopendatastorage\",  container_name=\"isdweatherdatacontainer\")\n",
     "#if read all years and months\n",
     "# data = ray.data.read_parquet(\"az://isdweatherdatacontainer/ISDWeather//\", filesystem=abfs)\n",
-    "data =ray.data.read_parquet([\"az://isdweatherdatacontainer/ISDWeather/year=2015/\"], filesystem=abfs)\n"
+    "data =ray.data.read_parquet([\"az://isdweatherdatacontainer/ISDWeather/year=2015/\"], filesystem=abfs)\n",
+    "data.count()\n",
+    "# 1,584,481,119 is the count for all data \n"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Mount remote data to your ray cluster"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "client.disconnect()\n",
+    "ray_on_aml.shutdown()"
    ]
   },
   {
@@ -181,8 +304,35 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "data.count()\n",
-    "# 1,584,481,119 is the count for all data "
+    "#Example of using input and output for interactive job. You need to define adlsstore0001 in your Azure ML workspace first\n",
+    "from azure.ai.ml import command, Input, Output\n",
+    "from ray_on_aml.core import Ray_On_AML\n",
+    "import logging\n",
+    "ray_on_aml =Ray_On_AML(ml_client=ml_client, compute_cluster =\"ds11\", verbosity=logging.INFO )\n",
+    "\n",
+    "inputs={\n",
+    "\n",
+    "    \"ISDWeather\": Input(\n",
+    "        type=\"uri_folder\",\n",
+    "        path=\"azureml://datastores/adlsstore0001/paths/ISDWeather/year=2008\",\n",
+    "    )\n",
+    "}\n",
+    "\n",
+    "outputs={\n",
+    "    \"output1\": Output(\n",
+    "        type=\"uri_folder\",\n",
+    "        path=\"azureml://datastores/adlsstore0001/paths/dev\",\n",
+    "    ),\n",
+    "    \"output2\": Output(\n",
+    "        type=\"uri_folder\",\n",
+    "        path=\"azureml://datastores/adlsstore0001/paths/dev\",\n",
+    "    )\n",
+    "}\n",
+    "\n",
+    "ray = ray_on_aml.getRay(inputs = inputs,outputs=outputs, num_node=2,\n",
+    "pip_packages=[\"ray[air]==2.2.0\",\"ray[data]==2.2.0\",\"torch==1.13.0\",\"fastparquet==2022.12.0\", \n",
+    "\"azureml-mlflow==1.48.0\", \"pyarrow==6.0.1\", \"dask==2022.2.0\", \"adlfs==2022.11.2\", \"fsspec==2022.11.0\"])\n",
+    "client = ray.init(f\"ray://{ray_on_aml.headnode_private_ip}:10001\")\n"
    ]
   },
   {
@@ -191,14 +341,8 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "start = time.time()\n",
-    "#convert Ray dataset to Dask dataframe \n",
-    "result = data.to_dask().describe().compute()\n",
-    "print(result)\n",
-    "stop = time.time()\n",
-    "print(\"duration \", (stop-start))\n",
-    "#717s for single machine nc6\n",
-    "# duration  307.69699811935425s for CI as head and 4 workers of DS14_v2\n"
+    "data = ray.data.read_parquet(ray_on_aml.mount_points['ISDWeather'])\n",
+    "data.count()"
    ]
   },
   {
@@ -510,16 +654,13 @@
   }
  ],
  "metadata": {
-  "interpreter": {
-   "hash": "8858a4df92b06e9052bc306608e3218c33233584bc6448961c72d65ba55843de"
-  },
   "kernel_info": {
    "name": "python38-azureml"
   },
   "kernelspec": {
-   "display_name": "Python 3.8 - AzureML",
+   "display_name": "azureml_py310_sdkv2",
    "language": "python",
-   "name": "python38-azureml"
+   "name": "python3"
   },
   "language_info": {
    "codemirror_mode": {
@@ -531,7 +672,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.8.5"
+   "version": "3.10.6"
   },
   "microsoft": {
    "host": {
@@ -542,6 +683,11 @@
   },
   "nteract": {
    "version": "nteract-front-end@1.0.0"
+  },
+  "vscode": {
+   "interpreter": {
+    "hash": "2139c70ac98f3202d028164a545621647e07f47fd6f5d8ac55cf952bf7c15ed1"
+   }
   }
  },
  "nbformat": 4,


Test patch:
diff --git a/tests/automated_tests/test_ray_data/conda.yml b/tests/automated_tests/test_ray_data/conda.yml
index 5e85a93..840b63b 100644
--- a/tests/automated_tests/test_ray_data/conda.yml
+++ b/tests/automated_tests/test_ray_data/conda.yml
@@ -9,7 +9,7 @@ dependencies:
       - ray[data]
       - git+https://github.com/microsoft/ray-on-aml.git@james-dev
       - adlfs
-      - dask==2022.2.0
+      - dask==2022.3.0
       - pyarrow==6.0.1
       - fsspec
       - fastparquet
diff --git a/tests/misc_test.ipynb b/tests/misc_test.ipynb
index 9c8b103..f753c7f 100644
--- a/tests/misc_test.ipynb
+++ b/tests/misc_test.ipynb
@@ -9,7 +9,7 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 1,
+      "execution_count": 3,
       "metadata": {},
       "outputs": [],
       "source": [
@@ -35,58 +35,77 @@
       "cell_type": "code",
       "execution_count": 1,
       "metadata": {},
+      "outputs": [],
+      "source": [
+        "from ray_on_aml.core import Ray_On_AML\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 2,
+      "metadata": {},
       "outputs": [
         {
-          "ename": "ModuleNotFoundError",
-          "evalue": "No module named 'ray_on_aml._telemetry'",
-          "output_type": "error",
-          "traceback": [
-            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
-            "\u001b[0;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)",
-            "\u001b[1;32m/home/azureuser/cloudfiles/code/Users/janguy/ray-on-aml/tests/misc_test.ipynb Cell 3\u001b[0m in \u001b[0;36m<cell line: 1>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> <a href='vscode-notebook-cell://amlext%2B2f737562736372697074696f6e732f38343062356335632d336634612d343539612d393466632d3662616432613936396639642f7265736f7572636547726f7570732f6d6c2f70726f7669646572732f4d6963726f736f66742e4d616368696e654c6561726e696e6753657276696365732f776f726b7370616365732f77733032656e742f636f6d70757465732f647331316369/home/azureuser/cloudfiles/code/Users/janguy/ray-on-aml/tests/misc_test.ipynb#X66sdnNjb2RlLXJlbW90ZQ%3D%3D?line=0'>1</a>\u001b[0m \u001b[39mfrom\u001b[39;00m \u001b[39mray_on_aml\u001b[39;00m\u001b[39m.\u001b[39;00m\u001b[39mcore\u001b[39;00m \u001b[39mimport\u001b[39;00m Ray_On_AML\n",
-            "File \u001b[0;32m/anaconda/envs/azureml_py310_sdkv2/lib/python3.10/site-packages/ray_on_aml/core.py:13\u001b[0m, in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     11\u001b[0m \u001b[39mimport\u001b[39;00m \u001b[39mlogging\u001b[39;00m\n\u001b[1;32m     12\u001b[0m \u001b[39mimport\u001b[39;00m \u001b[39mzlib\u001b[39;00m\n\u001b[0;32m---> 13\u001b[0m \u001b[39mfrom\u001b[39;00m \u001b[39m.\u001b[39;00m\u001b[39m_telemetry\u001b[39;00m\u001b[39m.\u001b[39;00m\u001b[39m_loggerfactory\u001b[39;00m \u001b[39mimport\u001b[39;00m _LoggerFactory\n\u001b[1;32m     14\u001b[0m \u001b[39m# from warnings import warn\u001b[39;00m\n\u001b[1;32m     16\u001b[0m __version__\u001b[39m=\u001b[39m\u001b[39m'\u001b[39m\u001b[39m0.2.3\u001b[39m\u001b[39m'\u001b[39m\n",
-            "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'ray_on_aml._telemetry'"
+          "name": "stdout",
+          "output_type": "stream",
+          "text": [
+            "Found existing installation: ray-on-aml 0.2.4\n",
+            "Uninstalling ray-on-aml-0.2.4:\n",
+            "  Successfully uninstalled ray-on-aml-0.2.4\n",
+            "Note: you may need to restart the kernel to use updated packages.\n"
           ]
         }
       ],
       "source": [
-        "from ray_on_aml.core import Ray_On_AML\n"
+        "pip uninstall ray_on_aml --y"
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 3,
+      "execution_count": 2,
       "metadata": {},
       "outputs": [
         {
           "name": "stdout",
           "output_type": "stream",
           "text": [
-            "\n",
-            "Usage:   \n",
-            "  /anaconda/envs/azureml_py310_sdkv2/bin/python -m pip <command> [options]\n",
-            "\n",
-            "no such option: --force-reinstall\n",
+            "Collecting git+https://github.com/microsoft/ray-on-aml.git@james-dev\n",
+            "  Cloning https://github.com/microsoft/ray-on-aml.git (to revision james-dev) to /tmp/pip-req-build-r8le2gs2\n",
+            "  Running command git clone --filter=blob:none --quiet https://github.com/microsoft/ray-on-aml.git /tmp/pip-req-build-r8le2gs2\n",
+            "  Running command git checkout -b james-dev --track origin/james-dev\n",
+            "  Switched to a new branch 'james-dev'\n",
+            "  branch 'james-dev' set up to track 'origin/james-dev'.\n",
+            "  Resolved https://github.com/microsoft/ray-on-aml.git to commit 1c66fc88196bd418edb73db39cfbc011404494fc\n",
+            "  Installing build dependencies ... \u001b[?25ldone\n",
+            "\u001b[?25h  Getting requirements to build wheel ... \u001b[?25ldone\n",
+            "\u001b[?25h  Preparing metadata (pyproject.toml) ... \u001b[?25ldone\n",
+            "\u001b[?25hBuilding wheels for collected packages: ray-on-aml\n",
+            "  Building wheel for ray-on-aml (pyproject.toml) ... \u001b[?25ldone\n",
+            "\u001b[?25h  Created wheel for ray-on-aml: filename=ray_on_aml-0.2.4-py3-none-any.whl size=15129 sha256=380b06244b665dc8035fb89505941d1a2eeab49face62d3786c517bda3983bb2\n",
+            "  Stored in directory: /tmp/pip-ephem-wheel-cache-8u911d96/wheels/cb/fb/f3/385e24a3acb7fc0e6abd70405ce482c4aa747f31b2e148601a\n",
+            "Successfully built ray-on-aml\n",
+            "Installing collected packages: ray-on-aml\n",
+            "Successfully installed ray-on-aml-0.2.4\n",
             "Note: you may need to restart the kernel to use updated packages.\n"
           ]
         }
       ],
       "source": [
-        "pip install--upgrade git+https://github.com/microsoft/ray-on-aml.git@james-dev"
+        "pip install git+https://github.com/microsoft/ray-on-aml.git@james-dev"
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 2,
+      "execution_count": 4,
       "metadata": {},
       "outputs": [
         {
           "name": "stderr",
           "output_type": "stream",
           "text": [
-            "Warning: the provided asset name 'ray-on-aml-2580231983' will not be used for anonymous registration\n",
-            "Warning: the provided asset name 'ray-on-aml-2580231983' will not be used for anonymous registration\n",
-            "\u001b[32mUploading .tmp (0.01 MBs): 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 6678/6678 [00:00<00:00, 151621.87it/s]\u001b[0m\n",
+            "Warning: the provided asset name 'ray-on-aml-1991063301' will not be used for anonymous registration\n",
+            "Warning: the provided asset name 'ray-on-aml-1991063301' will not be used for anonymous registration\n",
+            "\u001b[32mUploading .tmp (0.01 MBs): 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 6677/6677 [00:00<00:00, 118500.61it/s]\u001b[0m\n",
             "\u001b[39m\n",
             "\n"
           ]
@@ -96,23 +115,13 @@
           "output_type": "stream",
           "text": [
             "Waiting cluster to start and return head node's ip\n",
-            "...\n",
+            ".......................................................................................................................................\n",
             " cluster is ready, head node ip  10.0.0.8\n"
           ]
-        },
-        {
-          "name": "stderr",
-          "output_type": "stream",
-          "text": [
-            "\u001b[2m\u001b[33m(raylet, ip=10.0.0.9)\u001b[0m /azureml-envs/azureml_e6becbd955b6439d8f9fcc3781a8262b/lib/python3.10/site-packages/ray/dashboard/agent.py:50: DeprecationWarning: There is no current event loop\n",
-            "\u001b[2m\u001b[33m(raylet, ip=10.0.0.9)\u001b[0m   aiogrpc.init_grpc_aio()\n",
-            "\u001b[2m\u001b[33m(raylet, ip=10.0.0.9)\u001b[0m /azureml-envs/azureml_e6becbd955b6439d8f9fcc3781a8262b/lib/python3.10/site-packages/ray/dashboard/agent.py:468: DeprecationWarning: There is no current event loop\n",
-            "\u001b[2m\u001b[33m(raylet, ip=10.0.0.9)\u001b[0m   loop = asyncio.get_event_loop()\n"
-          ]
         }
       ],
       "source": [
-        "from src.ray_on_aml.core import Ray_On_AML\n",
+        "# from src.ray_on_aml.core import Ray_On_AML\n",
         "\n",
         "import logging\n",
         "ray_on_aml =Ray_On_AML(ml_client=ml_client, compute_cluster =\"d13\")\n",
@@ -184,9 +193,21 @@
     },
     {
       "cell_type": "code",
-      "execution_count": null,
+      "execution_count": 5,
       "metadata": {},
-      "outputs": [],
+      "outputs": [
+        {
+          "ename": "ModuleNotFoundError",
+          "evalue": "No module named 'adlfs'",
+          "output_type": "error",
+          "traceback": [
+            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+            "\u001b[0;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)",
+            "\u001b[1;32m/home/azureuser/cloudfiles/code/Users/janguy/ray-on-aml/tests/misc_test.ipynb Cell 10\u001b[0m in \u001b[0;36m<cell line: 1>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> <a href='vscode-notebook-cell://amlext%2B2f737562736372697074696f6e732f38343062356335632d336634612d343539612d393466632d3662616432613936396639642f7265736f7572636547726f7570732f6d6c2f70726f7669646572732f4d6963726f736f66742e4d616368696e654c6561726e696e6753657276696365732f776f726b7370616365732f77733032656e742f636f6d70757465732f647331316369/home/azureuser/cloudfiles/code/Users/janguy/ray-on-aml/tests/misc_test.ipynb#Y110sdnNjb2RlLXJlbW90ZQ%3D%3D?line=0'>1</a>\u001b[0m \u001b[39mfrom\u001b[39;00m \u001b[39madlfs\u001b[39;00m \u001b[39mimport\u001b[39;00m AzureBlobFileSystem\n\u001b[1;32m      <a href='vscode-notebook-cell://amlext%2B2f737562736372697074696f6e732f38343062356335632d336634612d343539612d393466632d3662616432613936396639642f7265736f7572636547726f7570732f6d6c2f70726f7669646572732f4d6963726f736f66742e4d616368696e654c6561726e696e6753657276696365732f776f726b7370616365732f77733032656e742f636f6d70757465732f647331316369/home/azureuser/cloudfiles/code/Users/janguy/ray-on-aml/tests/misc_test.ipynb#Y110sdnNjb2RlLXJlbW90ZQ%3D%3D?line=1'>2</a>\u001b[0m \u001b[39m#install dask, adlfs \u001b[39;00m\n\u001b[1;32m      <a href='vscode-notebook-cell://amlext%2B2f737562736372697074696f6e732f38343062356335632d336634612d343539612d393466632d3662616432613936396639642f7265736f7572636547726f7570732f6d6c2f70726f7669646572732f4d6963726f736f66742e4d616368696e654c6561726e696e6753657276696365732f776f726b7370616365732f77733032656e742f636f6d70757465732f647331316369/home/azureuser/cloudfiles/code/Users/janguy/ray-on-aml/tests/misc_test.ipynb#Y110sdnNjb2RlLXJlbW90ZQ%3D%3D?line=2'>3</a>\u001b[0m \u001b[39mimport\u001b[39;00m \u001b[39mdask\u001b[39;00m\u001b[39m.\u001b[39;00m\u001b[39mdataframe\u001b[39;00m \u001b[39mas\u001b[39;00m \u001b[39mdd\u001b[39;00m\n",
+            "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'adlfs'"
+          ]
+        }
+      ],
       "source": [
         "from adlfs import AzureBlobFileSystem\n",
         "#install dask, adlfs \n",
@@ -249,18 +270,72 @@
     },
     {
       "cell_type": "code",
-      "execution_count": null,
+      "execution_count": 6,
       "metadata": {},
-      "outputs": [],
+      "outputs": [
+        {
+          "data": {
+            "text/plain": [
+              "{'object_store_memory': 24481313586.0,\n",
+              " 'node:10.0.0.9': 1.0,\n",
+              " 'memory': 87228204646.0,\n",
+              " 'CPU': 16.0,\n",
+              " 'node:10.0.0.8': 1.0}"
+            ]
+          },
+          "execution_count": 6,
+          "metadata": {},
+          "output_type": "execute_result"
+        }
+      ],
       "source": [
         "ray.cluster_resources()"
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": null,
+      "execution_count": 8,
       "metadata": {},
-      "outputs": [],
+      "outputs": [
+        {
+          "name": "stdout",
+          "output_type": "stream",
+          "text": [
+            "Collecting aiohttp\n",
+            "  Using cached aiohttp-3.8.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.0 MB)\n",
+            "Requirement already satisfied: aiosignal>=1.1.2 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from aiohttp) (1.3.1)\n",
+            "Requirement already satisfied: yarl<2.0,>=1.0 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from aiohttp) (1.8.2)\n",
+            "Requirement already satisfied: frozenlist>=1.1.1 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from aiohttp) (1.3.3)\n",
+            "Requirement already satisfied: multidict<7.0,>=4.5 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from aiohttp) (6.0.3)\n",
+            "Requirement already satisfied: charset-normalizer<3.0,>=2.0 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from aiohttp) (2.1.1)\n",
+            "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from aiohttp) (4.0.2)\n",
+            "Requirement already satisfied: attrs>=17.3.0 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from aiohttp) (22.1.0)\n",
+            "Requirement already satisfied: idna>=2.0 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from yarl<2.0,>=1.0->aiohttp) (3.4)\n",
+            "Installing collected packages: aiohttp\n",
+            "Successfully installed aiohttp-3.8.3\n",
+            "Note: you may need to restart the kernel to use updated packages.\n"
+          ]
+        }
+      ],
+      "source": [
+        "pip install aiohttp"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 9,
+      "metadata": {},
+      "outputs": [
+        {
+          "name": "stderr",
+          "output_type": "stream",
+          "text": [
+            "2022-12-15 02:01:09,395\tWARNING read_api.py:296 -- ⚠️  The number of blocks in this dataset (1) limits its parallelism to 1 concurrent tasks. This is much less than the number of available CPU slots in the cluster. Use `.repartition(n)` to increase the number of dataset blocks.\n",
+            "Read: 100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 1/1 [00:01<00:00,  1.68s/it]\n",
+            "Repartition: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 4/4 [00:00<00:00, 12.89it/s]\n"
+          ]
+        }
+      ],
       "source": [
         "\n",
         "# Load data.\n",
@@ -478,7 +553,7 @@
       "name": "python3"
     },
     "kernelspec": {
-      "display_name": "azureml_py310_sdkv2",
+      "display_name": "azureml_py38_PT_TF",
       "language": "python",
       "name": "python3"
     },
@@ -492,7 +567,7 @@
       "name": "python",
       "nbconvert_exporter": "python",
       "pygments_lexer": "ipython3",
-      "version": "3.10.6"
+      "version": "3.8.5"
     },
     "nteract": {
       "version": "nteract-front-end@1.0.0"
@@ -500,7 +575,7 @@
     "orig_nbformat": 4,
     "vscode": {
       "interpreter": {
-        "hash": "2139c70ac98f3202d028164a545621647e07f47fd6f5d8ac55cf952bf7c15ed1"
+        "hash": "9169f1d4e16acc976bbb73e323b0dbdf23f1c55e833fb2befffc4fb50ac2de2f"
       }
     }
   },