Repository: activeloopai/deeplake

Issue:
Paths joined as strings
Looking through the code surrounding dataset downloads, I noticed that in [`get_local_storage_path`](https://github.com/activeloopai/deeplake/blob/392f8e92235b29ffa8bd59ee7d0969c7d4bbc2ee/deeplake/util/storage.py#L166) two path-representing strings are joined via an f string:

https://github.com/activeloopai/deeplake/blob/392f8e92235b29ffa8bd59ee7d0969c7d4bbc2ee/deeplake/util/storage.py#L170

Are there reasons why these are not `Path`s? I would have expected the following:

```python
from pathlib import Path

def get_local_storage_path(path: str, prefix: Path) -> Path:
    local_cache_name = path.replace("://", "_")
    local_cache_name = local_cache_name.replace("/", "_")
    local_cache_name = local_cache_name.replace("\\", "_")
    return prefix / local_cache_name
```

Or at least:
```python
import os

def get_local_storage_path(path: str, prefix: str) -> str:
    local_cache_name = path.replace("://", "_")
    local_cache_name = local_cache_name.replace("/", "_")
    local_cache_name = local_cache_name.replace("\\", "_")
    return os.path.join(prefix, local_cache_name)
```

By the way, I noticed this because I got a string/path with two slashes (`//`) from [`get_local_dataset`](https://github.com/activeloopai/deeplake/blob/392f8e92235b29ffa8bd59ee7d0969c7d4bbc2ee/deeplake/util/access_method.py#L84):

`deeplake.util.exceptions.DatasetHandlerError: A dataset does not exist at the download location /tmp/deeplake//hub_activeloop_fashionpedia-train. Cannot use access method 'local'. Use access method 'download' to first download the dataset and then use access method 'local' in subsequent runs.`



Discussion:
@jangop Hey there, thanks for raising the issue. There's no reason for joining the paths as strings and we should do it the way you suggested.
@jangop were you still able to use 'local' access method after performing the download?
> @jangop were you still able to use 'local' access method after performing the download?

Yea. I was testing the different options. Still seems odd to me that one has to manually switch `download` to `local`. I implemented `auto`, which goes something like this:

```python
def handle_access_method(access_method: str) -> str:
    if access_method in ["stream", "download", "local"]:
        return access_method
    elif access_method == "auto":
        [...]
            if dataset_path.exists():
                return "local"
            else:
                return "download"
```
This is because we wanted to have a distinct option to download / re-download the remote dataset if it has any changes. We'll change these options soon to avoid the switch.

Resolving patch:
diff --git a/deeplake/api/dataset.py b/deeplake/api/dataset.py
--- a/deeplake/api/dataset.py
+++ b/deeplake/api/dataset.py
@@ -103,15 +103,19 @@ def init(
                     - 'download'
 
                         - Downloads the data to the local filesystem to the path specified in environment variable ``DEEPLAKE_DOWNLOAD_PATH``.
-                        - Raises an exception if the environment variable is not set, or if the path is not empty.
-                        - Will also raise an exception if the dataset does not exist. The 'download' access method can also be modified to specify num_workers and/or scheduler.
-                        - For example: 'download:2:processed', will use 2 workers and use processed scheduler, while 'download:3' will use 3 workers and default scheduler (threaded), and 'download:processed' will use a single worker and use processed scheduler.
+                          This will overwrite ``DEEPLAKE_DOWNLOAD_PATH``.
+                        - Raises an exception if ``DEEPLAKE_DOWNLOAD_PATH`` environment variable is not set or if the dataset does not exist.
+                        - The 'download' access method can be modified to specify num_workers and/or scheduler.
+                          For example: 'download:2:processed' will use 2 workers and use processed scheduler, while 'download:3' will use 3 workers and
+                          default scheduler (threaded), and 'download:processed' will use a single worker and use processed scheduler.
 
                     - 'local'
 
-                        - Used when download was already done in a previous run.
-                        - Doesn't download the data again.
+                        - Downloads dataset if it doesn't already exist, otherwise loads from local storage.
                         - Raises an exception if ``DEEPLAKE_DOWNLOAD_PATH`` environment variable is not set or the dataset is not found in ``DEEPLAKE_DOWNLOAD_PATH``.
+                        - The 'local' access method can be modified to specify num_workers and/or scheduler to be used in case dataset needs to be downloaded.
+                          If dataset needs to be downloaded, 'local:2:processed' will use 2 workers and use processed scheduler, while 'local:3' will use 3 workers
+                          and default scheduler (threaded), and 'local:processed' will use a single worker and use processed scheduler.
 
         Returns:
             Dataset: Dataset created using the arguments provided.
@@ -125,6 +129,9 @@ def init(
         Danger:
             Setting ``overwrite`` to ``True`` will delete all of your data if it exists! Be very careful when setting this parameter.
 
+        Warning:
+            Setting ``access_method`` to download will overwrite the local copy of the dataset if it was previously downloaded.
+
         Note:
             Any changes made to the dataset in download / local mode will only be made to the local copy and will not be reflected in the original dataset.
         """
@@ -356,16 +363,20 @@ def load(
 
                     - 'download'
 
-                        - Downloads the data to the local filesystem to the path specified in environment variable `DEEPLAKE_DOWNLOAD_PATH`.
-                        - Raises an exception if the environment variable is not set, or if the path is not empty.
-                        - Will also raise an exception if the dataset does not exist. The 'download' access method can also be modified to specify num_workers and/or scheduler.
-                        - For example: 'download:2:processed', will use 2 workers and use processed scheduler, while 'download:3' will use 3 workers and default scheduler (threaded), and 'download:processed' will use a single worker and use processed scheduler.
+                        - Downloads the data to the local filesystem to the path specified in environment variable ``DEEPLAKE_DOWNLOAD_PATH``.
+                          This will overwrite ``DEEPLAKE_DOWNLOAD_PATH``.
+                        - Raises an exception if ``DEEPLAKE_DOWNLOAD_PATH`` environment variable is not set or if the dataset does not exist.
+                        - The 'download' access method can be modified to specify num_workers and/or scheduler.
+                          For example: 'download:2:processed' will use 2 workers and use processed scheduler, while 'download:3' will use 3 workers and
+                          default scheduler (threaded), and 'download:processed' will use a single worker and use processed scheduler.
 
                     - 'local'
 
-                        - Used when download was already done in a previous run.
-                        - Doesn't download the data again.
+                        - Downloads dataset if it doesn't already exist, otherwise loads from local storage.
                         - Raises an exception if ``DEEPLAKE_DOWNLOAD_PATH`` environment variable is not set or the dataset is not found in ``DEEPLAKE_DOWNLOAD_PATH``.
+                        - The 'local' access method can be modified to specify num_workers and/or scheduler to be used in case dataset needs to be downloaded.
+                          If dataset needs to be downloaded, 'local:2:processed' will use 2 workers and use processed scheduler, while 'local:3' will use 3 workers
+                          and default scheduler (threaded), and 'local:processed' will use a single worker and use processed scheduler.
 
         Returns:
             Dataset: Dataset loaded using the arguments provided.
@@ -377,6 +388,9 @@ def load(
             InvalidTokenException: If the specified toke is invalid
             TokenPermissionError: When there are permission or other errors related to token
 
+        Warning:
+            Setting ``access_method`` to download will overwrite the local copy of the dataset if it was previously downloaded.
+
         Note:
             Any changes made to the dataset in download / local mode will only be made to the local copy and will not be reflected in the original dataset.
         """
diff --git a/deeplake/util/access_method.py b/deeplake/util/access_method.py
--- a/deeplake/util/access_method.py
+++ b/deeplake/util/access_method.py
@@ -23,9 +23,11 @@ def check_access_method(access_method: str, overwrite: bool):
 
 
 def parse_access_method(access_method: str):
-    num_workers = None
-    scheduler = None
-    if access_method.startswith("download"):
+    num_workers = 0
+    scheduler = "threaded"
+    download = access_method.startswith("download")
+    local = access_method.startswith("local")
+    if download or local:
         split = access_method.split(":")
         if len(split) == 1:
             split.extend(("threaded", "0"))
@@ -38,7 +40,7 @@ def parse_access_method(access_method: str):
                     "Invalid access_method format. Expected format is one of the following: {download, download:scheduler, download:num_workers, download:scheduler:num_workers, download:num_workers:scheduler}"
                 )
 
-        access_method = "download"
+        access_method = "download" if download else "local"
         num_worker_index = 1 if split[1].isnumeric() else 2
         scheduler_index = 3 - num_worker_index
         num_workers = int(split[num_worker_index])
@@ -60,15 +62,14 @@ def get_local_dataset(
     scheduler,
 ):
     local_path = get_local_storage_path(path, os.environ["DEEPLAKE_DOWNLOAD_PATH"])
-    if access_method == "download":
+    download = access_method == "download" or (
+        access_method == "local" and not deeplake.exists(local_path)
+    )
+    if download:
         if not ds_exists:
             raise DatasetHandlerError(
                 f"Dataset {path} does not exist. Cannot use access method 'download'"
             )
-        elif deeplake.exists(local_path):
-            raise DatasetHandlerError(
-                f"A dataset already exists at the download location {local_path}. To reuse the dataset, use access method 'local'. If you want to download the dataset again, delete the dataset at the download location and try again."
-            )
         deeplake.deepcopy(
             path,
             local_path,
@@ -78,10 +79,7 @@ def get_local_dataset(
             scheduler=scheduler,
             progressbar=True,
             verbose=False,
-        )
-    elif not deeplake.exists(local_path):
-        raise DatasetHandlerError(
-            f"A dataset does not exist at the download location {local_path}. Cannot use access method 'local'. Use access method 'download' to first download the dataset and then use access method 'local' in subsequent runs."
+            overwrite=True,
         )
 
     ds = deeplake.load(
@@ -91,7 +89,7 @@ def get_local_dataset(
         memory_cache_size=memory_cache_size,
         local_cache_size=local_cache_size,
     )
-    if access_method == "download":
+    if download:
         ds.storage.next_storage[TIMESTAMP_FILENAME] = time.ctime().encode("utf-8")
     else:
         timestamp = ds.storage.next_storage[TIMESTAMP_FILENAME].decode("utf-8")
diff --git a/deeplake/util/storage.py b/deeplake/util/storage.py
--- a/deeplake/util/storage.py
+++ b/deeplake/util/storage.py
@@ -167,7 +167,7 @@ def get_local_storage_path(path: str, prefix: str):
     local_cache_name = path.replace("://", "_")
     local_cache_name = local_cache_name.replace("/", "_")
     local_cache_name = local_cache_name.replace("\\", "_")
-    return f"{prefix}/{local_cache_name}"
+    return os.path.join(prefix, local_cache_name)
 
 
 def get_pytorch_local_storage(dataset):


Test patch:
diff --git a/deeplake/api/tests/test_access_method.py b/deeplake/api/tests/test_access_method.py
--- a/deeplake/api/tests/test_access_method.py
+++ b/deeplake/api/tests/test_access_method.py
@@ -1,7 +1,16 @@
 import deeplake
 import pytest
 from deeplake.util.exceptions import DatasetHandlerError
-from deeplake.util.access_method import parse_access_method
+from deeplake.util.access_method import parse_access_method, get_local_storage_path
+
+
+def test_local_storage_path():
+    path = "/tmp/deeplake/"
+    dataset_name = "hub://activeloop/mnist-train"
+    assert (
+        get_local_storage_path(dataset_name, path)
+        == "/tmp/deeplake/hub_activeloop_mnist-train"
+    )
 
 
 def test_access_method_parsing():
@@ -10,12 +19,23 @@ def test_access_method_parsing():
     assert parse_access_method("download:5") == ("download", 5, "threaded")
     assert parse_access_method("download:processed:5") == ("download", 5, "processed")
     assert parse_access_method("download:5:processed") == ("download", 5, "processed")
+    assert parse_access_method("local") == ("local", 0, "threaded")
+    assert parse_access_method("local:processed") == ("local", 0, "processed")
+    assert parse_access_method("local:5") == ("local", 5, "threaded")
+    assert parse_access_method("local:processed:5") == ("local", 5, "processed")
+    assert parse_access_method("local:5:processed") == ("local", 5, "processed")
     with pytest.raises(ValueError):
         parse_access_method("download:5:processed:5")
     with pytest.raises(ValueError):
         parse_access_method("download:processed:processed")
     with pytest.raises(ValueError):
         parse_access_method("download:processed:5:processed")
+    with pytest.raises(ValueError):
+        parse_access_method("local:5:processed:5")
+    with pytest.raises(ValueError):
+        parse_access_method("local:processed:processed")
+    with pytest.raises(ValueError):
+        parse_access_method("local:processed:5:processed")
 
 
 def test_access_method(s3_ds_generator):
@@ -25,20 +45,27 @@ def test_access_method(s3_ds_generator):
     with pytest.raises(DatasetHandlerError):
         deeplake.dataset("./some_non_existent_path", access_method="local")
 
-    ds = s3_ds_generator()
-    with ds:
+    with s3_ds_generator() as ds:
         ds.create_tensor("x")
-        for i in range(10):
-            ds.x.append(i)
+        ds.x.extend(list(range(10)))
 
-    ds = s3_ds_generator(access_method="download")
-    with pytest.raises(DatasetHandlerError):
-        # second time download is not allowed
-        s3_ds_generator(access_method="download")
+    ds = s3_ds_generator(access_method="local:2")  # downloads automatically
     assert not ds.path.startswith("s3://")
     for i in range(10):
         assert ds.x[i].numpy() == i
 
+    with s3_ds_generator() as ds:
+        ds.x.extend(list(range(10, 20)))
+
+    ds = s3_ds_generator(access_method="local")  # load downloaded
+    assert not ds.path.startswith("s3://")
+    assert len(ds.x) == 10
+
+    ds = s3_ds_generator(access_method="download")  # download again
+    assert len(ds.x) == 20
+    for i in range(20):
+        assert ds.x[i].numpy() == i
+
     with pytest.raises(ValueError):
         s3_ds_generator(access_method="invalid")
 
@@ -48,9 +75,4 @@ def test_access_method(s3_ds_generator):
     with pytest.raises(ValueError):
         s3_ds_generator(access_method="local", overwrite=True)
 
-    ds = s3_ds_generator(access_method="local")
-    assert not ds.path.startswith("s3://")
-    for i in range(10):
-        assert ds.x[i].numpy() == i
-
     ds.delete()
diff --git a/deeplake/api/tests/test_api.py b/deeplake/api/tests/test_api.py
--- a/deeplake/api/tests/test_api.py
+++ b/deeplake/api/tests/test_api.py
@@ -1766,44 +1766,6 @@ def test_create_branch_when_locked_out(local_ds):
     local_ds.create_tensor("x")
 
 
-def test_access_method(s3_ds_generator):
-    with pytest.raises(DatasetHandlerError):
-        deeplake.dataset("./some_non_existent_path", access_method="download")
-
-    with pytest.raises(DatasetHandlerError):
-        deeplake.dataset("./some_non_existent_path", access_method="local")
-
-    ds = s3_ds_generator()
-    with ds:
-        ds.create_tensor("x")
-        for i in range(10):
-            ds.x.append(i)
-
-    ds = s3_ds_generator(access_method="download")
-    with pytest.raises(DatasetHandlerError):
-        # second time download is not allowed
-        s3_ds_generator(access_method="download")
-    assert not ds.path.startswith("s3://")
-    for i in range(10):
-        assert ds.x[i].numpy() == i
-
-    with pytest.raises(ValueError):
-        s3_ds_generator(access_method="invalid")
-
-    with pytest.raises(ValueError):
-        s3_ds_generator(access_method="download", overwrite=True)
-
-    with pytest.raises(ValueError):
-        s3_ds_generator(access_method="local", overwrite=True)
-
-    ds = s3_ds_generator(access_method="local")
-    assert not ds.path.startswith("s3://")
-    for i in range(10):
-        assert ds.x[i].numpy() == i
-
-    ds.delete()
-
-
 def test_partial_read_then_write(s3_ds_generator):
     ds = s3_ds_generator()
     with ds: