Repository: apache/opendal
Pull request #6066: feat(services/b2): Add operation in http context
Author: login: jorgehermo9
Labels: name: releases-note/feat
description: The PR implements a new feature or has a title that begins with "feat"
name: size:L
description: This PR changes 100-499 lines, ignoring generated files.
name: lgtm
description: This PR has been approved by a maintainer
Created: 2025-04-20T12:22:13Z | Merged: 2025-04-27T10:31:15Z

Relates to #5774


Diff:
diff --git a/core/src/services/b2/backend.rs b/core/src/services/b2/backend.rs
index f4b5c0a16ea4..7c6297031d9a 100644
--- a/core/src/services/b2/backend.rs
+++ b/core/src/services/b2/backend.rs
@@ -19,7 +19,6 @@ use std::fmt::Debug;
 use std::fmt::Formatter;
 use std::sync::Arc;
 
-use bytes::Buf;
 use http::Request;
 use http::Response;
 use http::StatusCode;
@@ -30,7 +29,6 @@ use super::core::constants;
 use super::core::parse_file_info;
 use super::core::B2Core;
 use super::core::B2Signer;
-use super::core::ListFileNamesResponse;
 use super::delete::B2Deleter;
 use super::error::parse_error;
 use super::lister::B2Lister;
@@ -289,27 +287,10 @@ impl Access for B2Backend {
         }
 
         let delimiter = if path.ends_with('/') { Some("/") } else { None };
-        let resp = self
-            .core
-            .list_file_names(Some(path), delimiter, None, None)
-            .await?;
-
-        let status = resp.status();
 
-        match status {
-            StatusCode::OK => {
-                let bs = resp.into_body();
-
-                let resp: ListFileNamesResponse =
-                    serde_json::from_reader(bs.reader()).map_err(new_json_deserialize_error)?;
-                if resp.files.is_empty() {
-                    return Err(Error::new(ErrorKind::NotFound, "no such file or directory"));
-                }
-                let meta = parse_file_info(&resp.files[0]);
-                Ok(RpStat::new(meta))
-            }
-            _ => Err(parse_error(resp)),
-        }
+        let file_info = self.core.get_file_info(path, delimiter).await?;
+        let meta = parse_file_info(&file_info);
+        Ok(RpStat::new(meta))
     }
 
     async fn read(&self, path: &str, args: OpRead) -> Result<(RpRead, Self::Reader)> {
@@ -361,28 +342,9 @@ impl Access for B2Backend {
     }
 
     async fn copy(&self, from: &str, to: &str, _args: OpCopy) -> Result<RpCopy> {
-        let resp = self
-            .core
-            .list_file_names(Some(from), None, None, None)
-            .await?;
-
-        let status = resp.status();
-
-        let source_file_id = match status {
-            StatusCode::OK => {
-                let bs = resp.into_body();
+        let file_info = self.core.get_file_info(from, None).await?;
 
-                let resp: ListFileNamesResponse =
-                    serde_json::from_reader(bs.reader()).map_err(new_json_deserialize_error)?;
-                if resp.files.is_empty() {
-                    return Err(Error::new(ErrorKind::NotFound, "no such file or directory"));
-                }
-
-                let file_id = resp.files[0].clone().file_id;
-                Ok(file_id)
-            }
-            _ => Err(parse_error(resp)),
-        }?;
+        let source_file_id = file_info.file_id;
 
         let Some(source_file_id) = source_file_id else {
             return Err(Error::new(ErrorKind::IsADirectory, "is a directory"));
diff --git a/core/src/services/b2/core.rs b/core/src/services/b2/core.rs
index 169f8bfb0c36..8a101522dbaa 100644
--- a/core/src/services/b2/core.rs
+++ b/core/src/services/b2/core.rs
@@ -154,6 +154,8 @@ impl B2Core {
             req = req.header(header::RANGE, range.to_header());
         }
 
+        let req = req.extension(Operation::Read);
+
         let req = req.body(Buffer::new()).map_err(new_request_build_error)?;
 
         self.info.http_client().fetch(req).await
@@ -171,6 +173,8 @@ impl B2Core {
 
         req = req.header(header::AUTHORIZATION, auth_info.authorization_token);
 
+        let req = req.extension(Operation::Write);
+
         // Set body
         let req = req.body(Buffer::new()).map_err(new_request_build_error)?;
 
@@ -264,6 +268,8 @@ impl B2Core {
             req = req.header(header::CONTENT_DISPOSITION, pos)
         }
 
+        let req = req.extension(Operation::Write);
+
         // Set body
         let req = req.body(body).map_err(new_request_build_error)?;
 
@@ -291,6 +297,8 @@ impl B2Core {
             mime.clone_into(&mut start_large_file_request.content_type)
         }
 
+        let req = req.extension(Operation::Write);
+
         let body =
             serde_json::to_vec(&start_large_file_request).map_err(new_json_serialize_error)?;
         let body = bytes::Bytes::from(body);
@@ -314,6 +322,8 @@ impl B2Core {
 
         req = req.header(header::AUTHORIZATION, auth_info.authorization_token);
 
+        let req = req.extension(Operation::Write);
+
         // Set body
         let req = req.body(Buffer::new()).map_err(new_request_build_error)?;
 
@@ -350,6 +360,8 @@ impl B2Core {
 
         req = req.header(X_BZ_CONTENT_SHA1, "do_not_verify");
 
+        let req = req.extension(Operation::Write);
+
         // Set body
         let req = req.body(body).map_err(new_request_build_error)?;
 
@@ -369,6 +381,8 @@ impl B2Core {
 
         req = req.header(header::AUTHORIZATION, auth_info.authorization_token);
 
+        let req = req.extension(Operation::Write);
+
         let body = serde_json::to_vec(&FinishLargeFileRequest {
             file_id: file_id.to_owned(),
             part_sha1_array,
@@ -393,6 +407,8 @@ impl B2Core {
 
         req = req.header(header::AUTHORIZATION, auth_info.authorization_token);
 
+        let req = req.extension(Operation::Write);
+
         let body = serde_json::to_vec(&CancelLargeFileRequest {
             file_id: file_id.to_owned(),
         })
@@ -407,12 +423,45 @@ impl B2Core {
         self.send(req).await
     }
 
+    pub async fn get_file_info(&self, path: &str, delimiter: Option<&str>) -> Result<File> {
+        let resp = self
+            .list_file_names_raw(Some(path), delimiter, None, None, Operation::Stat)
+            .await?;
+
+        let status = resp.status();
+        match status {
+            StatusCode::OK => {
+                let bs = resp.into_body();
+                let mut resp: ListFileNamesResponse =
+                    serde_json::from_reader(bs.reader()).map_err(new_json_deserialize_error)?;
+
+                if resp.files.is_empty() {
+                    return Err(Error::new(ErrorKind::NotFound, "no such file or directory"));
+                }
+                Ok(resp.files.swap_remove(0))
+            }
+            _ => Err(parse_error(resp)),
+        }
+    }
+
     pub async fn list_file_names(
         &self,
         prefix: Option<&str>,
         delimiter: Option<&str>,
         limit: Option<usize>,
         start_after: Option<String>,
+    ) -> Result<Response<Buffer>> {
+        self.list_file_names_raw(prefix, delimiter, limit, start_after, Operation::List)
+            .await
+    }
+
+    async fn list_file_names_raw(
+        &self,
+        prefix: Option<&str>,
+        delimiter: Option<&str>,
+        limit: Option<usize>,
+        start_after: Option<String>,
+        operation: Operation,
     ) -> Result<Response<Buffer>> {
         let auth_info = self.get_auth_info().await?;
 
@@ -444,6 +493,8 @@ impl B2Core {
 
         req = req.header(header::AUTHORIZATION, auth_info.authorization_token);
 
+        req = req.extension(operation);
+
         // Set body
         let req = req.body(Buffer::new()).map_err(new_request_build_error)?;
 
@@ -461,6 +512,8 @@ impl B2Core {
 
         req = req.header(header::AUTHORIZATION, auth_info.authorization_token);
 
+        let req = req.extension(Operation::Copy);
+
         let body = CopyFileRequest {
             source_file_id,
             file_name: to,
@@ -488,6 +541,8 @@ impl B2Core {
 
         req = req.header(header::AUTHORIZATION, auth_info.authorization_token);
 
+        let req = req.extension(Operation::Delete);
+
         let body = HideFileRequest {
             bucket_id: self.bucket_id.clone(),
             file_name: path.to_string(),


Reviews:
user: login: jorgehermo9
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-20T12:22:38Z
user: login: jorgehermo9
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-20T12:23:25Z
user: login: Xuanwo
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-20T14:26:12Z
user: login: jorgehermo9
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-26T16:15:41Z
user: login: Xuanwo
body: Thank you for working on this!
state: APPROVED
author_association: MEMBER
submitted_at: 2025-04-27T10:31:10Z

Review comments:
diff_hunk: @@ -171,6 +173,8 @@ impl B2Core {
 
         req = req.header(header::AUTHORIZATION, auth_info.authorization_token);
 
+        let req = req.extension(Operation::Write);
path: core/src/services/b2/core.rs
user: login: jorgehermo9
body: I'm not sure about this. Getting the upload url is `Operation::Write`, right?
created_at: 2025-04-20T12:22:38Z
author_association: CONTRIBUTOR
diff_hunk: @@ -444,6 +460,8 @@ impl B2Core {
 
         req = req.header(header::AUTHORIZATION, auth_info.authorization_token);
 
+        let req = req.extension(Operation::List);
path: core/src/services/b2/core.rs
user: login: jorgehermo9
body: The stat operation uses this method, so we won't have an `Operation::Stat` for this service... https://github.com/apache/opendal/blob/ed809bec9e4018b18919a6e282a8b79501c3023a/core/src/services/b2/backend.rs#L294
created_at: 2025-04-20T12:23:25Z
author_association: CONTRIBUTOR
diff_hunk: @@ -444,6 +460,8 @@ impl B2Core {
 
         req = req.header(header::AUTHORIZATION, auth_info.authorization_token);
 
+        let req = req.extension(Operation::List);
path: core/src/services/b2/core.rs
user: login: Xuanwo
body: I somehow feel we need to have a seperate `get_file_info` API even it calls `list_file_names` internally.
created_at: 2025-04-20T14:26:12Z
author_association: MEMBER
diff_hunk: @@ -407,12 +423,45 @@ impl B2Core {
         self.send(req).await
     }
 
+    pub async fn get_file_info(&self, path: &str, delimiter: Option<&str>) -> Result<File> {
+        let resp = self
+            .list_file_names_raw(Some(path), delimiter, None, None, Operation::Stat)
+            .await?;
+
+        let status = resp.status();
+        match status {
+            StatusCode::OK => {
+                let bs = resp.into_body();
+                let mut resp: ListFileNamesResponse =
+                    serde_json::from_reader(bs.reader()).map_err(new_json_deserialize_error)?;
path: core/src/services/b2/core.rs
user: login: jorgehermo9
body: While doing this refactor I realized that, In my opinion, we should do response deserialization in all `core.rs` modules. For example, `list_file_names` should return a `ListFileNamesResponse` instead of `Buffer` and then deserialize it in the backend.

I think all methods in core.rs (not only this services, but others) should do something similar to what I'm doing here, since I don't really think it is the backend responsibility.
created_at: 2025-04-26T16:15:41Z
author_association: CONTRIBUTOR

PR comments:
user: login: jorgehermo9
created_at: 2025-04-20T12:35:00Z
body: Should we add `Operation::Info` to those kind of requests? https://github.com/apache/opendal/blob/ed809bec9e4018b18919a6e282a8b79501c3023a/core/src/services/b2/core.rs#L92
author_association: CONTRIBUTOR
user: login: Xuanwo
created_at: 2025-04-20T14:25:42Z
body: > Should we add `Operation::Info` to those kind of requests?

No, let's just ignore those utils API calls.
author_association: MEMBER
user: login: jorgehermo9
created_at: 2025-04-26T16:16:25Z
body: Hi @Xuanwo I refactored what you said in https://github.com/apache/opendal/pull/6066#discussion_r2051736187. What do you think about the changes?
author_association: CONTRIBUTOR
user: login: jorgehermo9
created_at: 2025-04-26T17:00:01Z
body: Do not review this yet, I did a mess with the git history and I mistakenly committed some yandex_disk service changes
author_association: CONTRIBUTOR
user: login: jorgehermo9
created_at: 2025-04-26T20:23:43Z
body: Fixed!
author_association: CONTRIBUTOR