Repository: iterative/dvc

Issue:
fs: don't automatically create buckets
~~We currently do that in azure https://github.com/iterative/dvc/blob/master/dvc/remote/azure.py#L90 .~~

Related https://github.com/iterative/dvc.org/issues/415


Discussion:
Maybe it should be prompted? In case of for example, making mistake in bucket name?
The only place we are creating the bucket (container, TBMP) is the azure, and I think we'd better off if the user creates the buckets explicitly on their end. 
@isidentical Agreed. I think ssh and local are also like that, but `bucket`(aka container_name) is not quite the same as nonexistent dir in local and ssh, so we should be at least consistent within typical cloud group (s3, gs, azure, oss, etc).

Resolving patch:
diff --git a/dvc/fs/azure.py b/dvc/fs/azure.py
--- a/dvc/fs/azure.py
+++ b/dvc/fs/azure.py
@@ -84,14 +84,12 @@ def __init__(self, **config):
 
         url = config.get("url")
         self.path_info = self.PATH_CLS(url)
-        self.bucket = self.path_info.bucket
 
-        if not self.bucket:
+        if not self.path_info.bucket:
             container = self._az_config.get("storage", "container_name", None)
             url = f"azure://{container}"
 
         self.path_info = self.PATH_CLS(url)
-        self.bucket = self.path_info.bucket
 
     def _prepare_credentials(self, **config):
         from azure.identity.aio import DefaultAzureCredential
@@ -167,10 +165,6 @@ def fs(self):
         try:
             with _temp_event_loop():
                 file_system = AzureBlobFileSystem(**self.fs_args)
-                if self.bucket not in [
-                    container.rstrip("/") for container in file_system.ls("/")
-                ]:
-                    file_system.mkdir(self.bucket)
         except (ValueError, AzureError) as e:
             raise AzureAuthError(
                 f"Authentication to Azure Blob Storage via {self.login_method}"
diff --git a/dvc/fs/oss.py b/dvc/fs/oss.py
--- a/dvc/fs/oss.py
+++ b/dvc/fs/oss.py
@@ -45,7 +45,6 @@ def __init__(self, **config):
         url = config.get("url")
         self.path_info = self.PATH_CLS(url) if url else None
 
-        self.bucket = self.path_info.bucket
         self.endpoint = config.get("oss_endpoint") or os.getenv("OSS_ENDPOINT")
 
         self.key_id = (
@@ -68,25 +67,17 @@ def oss_service(self):
         logger.debug(f"key id: {self.key_id}")
         logger.debug(f"key secret: {self.key_secret}")
 
-        auth = oss2.Auth(self.key_id, self.key_secret)
-        bucket = oss2.Bucket(auth, self.endpoint, self.bucket)
-
-        # Ensure bucket exists
-        try:
-            bucket.get_bucket_info()
-        except oss2.exceptions.NoSuchBucket:
-            bucket.create_bucket(
-                oss2.BUCKET_ACL_PUBLIC_READ,
-                oss2.models.BucketCreateConfig(
-                    oss2.BUCKET_STORAGE_CLASS_STANDARD
-                ),
-            )
-        return bucket
+        return oss2.Auth(self.key_id, self.key_secret)
 
-    def _generate_download_url(self, path_info, expires=3600):
-        assert path_info.bucket == self.bucket
+    def _get_bucket(self, bucket):
+        import oss2
 
-        return self.oss_service.sign_url("GET", path_info.path, expires)
+        return oss2.Bucket(self.oss_service, self.endpoint, bucket)
+
+    def _generate_download_url(self, path_info, expires=3600):
+        return self._get_bucket(path_info.bucket).sign_url(
+            "GET", path_info.path, expires
+        )
 
     def exists(self, path_info) -> bool:
         paths = self._list_paths(path_info)
@@ -96,7 +87,7 @@ def _list_paths(self, path_info):
         import oss2
 
         for blob in oss2.ObjectIterator(
-            self.oss_service, prefix=path_info.path
+            self._get_bucket(path_info.bucket), prefix=path_info.path
         ):
             yield blob.key
 
@@ -114,16 +105,17 @@ def remove(self, path_info):
             raise NotImplementedError
 
         logger.debug(f"Removing oss://{path_info}")
-        self.oss_service.delete_object(path_info.path)
+        self._get_bucket(path_info.bucket).delete_object(path_info.path)
 
     def _upload_fobj(self, fobj, to_info):
-        self.oss_service.put_object(to_info.path, fobj)
+        self._get_bucket(to_info.bucket).put_object(to_info.path, fobj)
 
     def _upload(
         self, from_file, to_info, name=None, no_progress_bar=False, **_kwargs
     ):
         with Tqdm(desc=name, disable=no_progress_bar, bytes=True) as pbar:
-            self.oss_service.put_object_from_file(
+            bucket = self._get_bucket(to_info.bucket)
+            bucket.put_object_from_file(
                 to_info.path, from_file, progress_callback=pbar.update_to
             )
 
@@ -133,8 +125,9 @@ def _download(
         with Tqdm(desc=name, disable=no_progress_bar, bytes=True) as pbar:
             import oss2
 
+            bucket = self._get_bucket(from_info.bucket)
             oss2.resumable_download(
-                self.oss_service,
+                bucket,
                 from_info.path,
                 to_file,
                 progress_callback=pbar.update_to,


Test patch:
diff --git a/tests/remotes/azure.py b/tests/remotes/azure.py
--- a/tests/remotes/azure.py
+++ b/tests/remotes/azure.py
@@ -28,19 +28,12 @@ class Azure(Base, CloudURLInfo):
     @cached_property
     def service_client(self):
         # pylint: disable=no-name-in-module
-        from azure.core.exceptions import ResourceNotFoundError
         from azure.storage.blob import BlobServiceClient
 
         service_client = BlobServiceClient.from_connection_string(
             self.CONNECTION_STRING
         )
 
-        container_client = service_client.get_container_client(self.bucket)
-        try:  # verify that container exists
-            container_client.get_container_properties()
-        except ResourceNotFoundError:
-            container_client.create_container()
-
         return service_client
 
     @property
@@ -98,7 +91,16 @@ def _check():
 
 @pytest.fixture
 def azure(azure_server):
+    from azure.core.exceptions import ResourceNotFoundError
+
     url = f"azure://{TEST_AZURE_CONTAINER}/{uuid.uuid4()}"
     ret = Azure(url)
     ret.config = {"url": url, "connection_string": azure_server}
+
+    container = ret.service_client.get_container_client(TEST_AZURE_CONTAINER)
+    try:  # verify that container exists
+        container.get_container_properties()
+    except ResourceNotFoundError:
+        container.create_container()
+
     return ret
diff --git a/tests/remotes/oss.py b/tests/remotes/oss.py
--- a/tests/remotes/oss.py
+++ b/tests/remotes/oss.py
@@ -64,6 +64,8 @@ def _check():
 
 @pytest.fixture
 def oss(oss_server):
+    import oss2
+
     url = OSS.get_url()
     ret = OSS(url)
     ret.config = {
@@ -72,6 +74,19 @@ def oss(oss_server):
         "oss_key_secret": EMULATOR_OSS_ACCESS_KEY_SECRET,
         "oss_endpoint": oss_server,
     }
+
+    auth = oss2.Auth(
+        EMULATOR_OSS_ACCESS_KEY_ID, EMULATOR_OSS_ACCESS_KEY_SECRET
+    )
+    bucket = oss2.Bucket(auth, oss_server, TEST_OSS_REPO_BUCKET)
+    try:
+        bucket.get_bucket_info()
+    except oss2.exceptions.NoSuchBucket:
+        bucket.create_bucket(
+            oss2.BUCKET_ACL_PUBLIC_READ,
+            oss2.models.BucketCreateConfig(oss2.BUCKET_STORAGE_CLASS_STANDARD),
+        )
+
     return ret