Repository: rust-lang/bors
Pull request #267: Handle Github API timeouts
Author: login: geetanshjuneja
Created: 2025-04-03T09:14:47Z | Merged: 2025-04-10T10:44:24Z

Links to #255 


Diff:
diff --git a/Cargo.lock b/Cargo.lock
index b9f2dcb7..824abd87 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -943,9 +943,9 @@ dependencies = [
 
 [[package]]
 name = "hyper-util"
-version = "0.1.10"
+version = "0.1.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "df2dcfbe0677734ab2f3ffa7fa7bfd4706bfdc1ef393f2ee30184aed67e631b4"
+checksum = "497bbc33a26fdd4af9ed9c70d63f61cf56a938375fbb32df34db9b1cd6d643f2"
 dependencies = [
  "bytes",
  "futures-channel",
@@ -953,6 +953,7 @@ dependencies = [
  "http",
  "http-body",
  "hyper",
+ "libc",
  "pin-project-lite",
  "socket2",
  "tokio",
@@ -1087,9 +1088,9 @@ dependencies = [
 
 [[package]]
 name = "libc"
-version = "0.2.169"
+version = "0.2.171"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a"
+checksum = "c19937216e9d3aa9956d9bb8dfc0b0c8beb6058fc4f7a4dc4d850edf86a237d6"
 
 [[package]]
 name = "libm"
@@ -2043,9 +2044,9 @@ dependencies = [
 
 [[package]]
 name = "socket2"
-version = "0.5.7"
+version = "0.5.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ce305eb0b4296696835b71df73eb912e0f1ffd2556a501fcede6e0c50349191c"
+checksum = "4f5fd57c80058a56cf5c777ab8a126398ece8e442983605d280a44ce79d0edef"
 dependencies = [
  "libc",
  "windows-sys 0.52.0",
diff --git a/Cargo.toml b/Cargo.toml
index 591d7b08..37b81590 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -24,7 +24,7 @@ serde_json = "1"
 toml = "0.8"
 
 # GitHub
-octocrab = "0.44"
+octocrab = { version = "0.44", features = ["timeout"] }
 
 # Async
 futures = "0.3"
diff --git a/src/bors/mod.rs b/src/bors/mod.rs
index a5274197..ccec07dc 100644
--- a/src/bors/mod.rs
+++ b/src/bors/mod.rs
@@ -52,7 +52,7 @@ impl RepositoryState {
     }
 }
 
-#[derive(Clone, Debug, PartialEq, Serialize)]
+#[derive(Clone, Copy, Debug, PartialEq, Serialize)]
 pub enum PullRequestStatus {
     Closed,
     Draft,
diff --git a/src/github/api/client.rs b/src/github/api/client.rs
index 00d31721..cf96b4f2 100644
--- a/src/github/api/client.rs
+++ b/src/github/api/client.rs
@@ -1,6 +1,6 @@
 use anyhow::Context;
+use octocrab::Octocrab;
 use octocrab::models::{App, Repository};
-use octocrab::{Error, Octocrab};
 use tracing::log;
 
 use crate::bors::event::PullRequestComment;
@@ -10,7 +10,7 @@ use crate::database::RunId;
 use crate::github::api::base_github_html_url;
 use crate::github::api::operations::{MergeError, merge_branches, set_branch_to_commit};
 use crate::github::{CommitSha, GithubRepoName, PullRequest, PullRequestNumber};
-use crate::utils::timing::measure_network_request;
+use crate::utils::timing::{measure_network_request, perform_network_request_with_retry};
 
 /// Provides access to a single app installation (repository) using the GitHub API.
 pub struct GithubRepositoryClient {
@@ -55,7 +55,7 @@ impl GithubRepositoryClient {
     /// Loads repository configuration from a file located at `[CONFIG_FILE_PATH]` in the main
     /// branch.
     pub async fn load_config(&self) -> anyhow::Result<RepositoryConfig> {
-        measure_network_request("load_config", || async {
+        perform_network_request_with_retry("load_config", || async {
             let mut response = self
                 .client
                 .repos(&self.repo_name.owner, &self.repo_name.name)
@@ -83,12 +83,12 @@ impl GithubRepositoryClient {
                     Ok(config)
                 })
         })
-        .await
+        .await?
     }
 
     /// Return the current SHA of the given branch.
     pub async fn get_branch_sha(&self, name: &str) -> anyhow::Result<CommitSha> {
-        measure_network_request("get_branch_sha", || async {
+        perform_network_request_with_retry("get_branch_sha", || async {
             // https://docs.github.com/en/rest/branches/branches?apiVersion=2022-11-28#get-a-branch
             let branch: octocrab::models::repos::Branch = self
                 .client
@@ -100,12 +100,12 @@ impl GithubRepositoryClient {
                 .context("Cannot deserialize branch")?;
             Ok(CommitSha(branch.commit.sha))
         })
-        .await
+        .await?
     }
 
     /// Resolve a pull request from this repository by it's number.
     pub async fn get_pull_request(&self, pr: PullRequestNumber) -> anyhow::Result<PullRequest> {
-        measure_network_request("get_pull_request", || async {
+        perform_network_request_with_retry("get_pull_request", || async {
             let pr = self
                 .client
                 .pulls(self.repository().owner(), self.repository().name())
@@ -116,7 +116,7 @@ impl GithubRepositoryClient {
                 })?;
             Ok(pr.into())
         })
-        .await
+        .await?
     }
 
     /// Post a comment to the pull request with the given number.
@@ -126,7 +126,7 @@ impl GithubRepositoryClient {
         pr: PullRequestNumber,
         comment: Comment,
     ) -> anyhow::Result<()> {
-        measure_network_request("post_comment", || async {
+        perform_network_request_with_retry("post_comment", || async {
             self.client
                 .issues(&self.repository().owner, &self.repository().name)
                 .create_comment(pr.0, comment.render())
@@ -134,15 +134,15 @@ impl GithubRepositoryClient {
                 .with_context(|| format!("Cannot post comment to {}", self.format_pr(pr)))?;
             Ok(())
         })
-        .await
+        .await?
     }
 
     /// Set the given branch to a commit with the given `sha`.
     pub async fn set_branch_to_sha(&self, branch: &str, sha: &CommitSha) -> anyhow::Result<()> {
-        measure_network_request("set_branch_to_sha", || async {
+        perform_network_request_with_retry("set_branch_to_sha", || async {
             Ok(set_branch_to_commit(self, branch.to_string(), sha).await?)
         })
-        .await
+        .await?
     }
 
     /// Merge `head` into `base`. Returns the SHA of the merge commit.
@@ -152,10 +152,11 @@ impl GithubRepositoryClient {
         head: &CommitSha,
         commit_message: &str,
     ) -> Result<CommitSha, MergeError> {
-        measure_network_request("merge_branches", || async {
+        perform_network_request_with_retry("merge_branches", || async {
             merge_branches(self, base, head, commit_message).await
         })
         .await
+        .map_err(|_| MergeError::Timeout)?
     }
 
     /// Find all check suites attached to the given commit and branch.
@@ -164,18 +165,18 @@ impl GithubRepositoryClient {
         branch: &str,
         sha: &CommitSha,
     ) -> anyhow::Result<Vec<CheckSuite>> {
-        measure_network_request("get_check_suites_for_commit", || async {
-            #[derive(serde::Deserialize, Debug)]
-            struct CheckSuitePayload {
-                conclusion: Option<String>,
-                head_branch: String,
-            }
+        #[derive(serde::Deserialize, Debug)]
+        struct CheckSuitePayload {
+            conclusion: Option<String>,
+            head_branch: String,
+        }
 
-            #[derive(serde::Deserialize, Debug)]
-            struct CheckSuiteResponse {
-                check_suites: Vec<CheckSuitePayload>,
-            }
+        #[derive(serde::Deserialize, Debug)]
+        struct CheckSuiteResponse {
+            check_suites: Vec<CheckSuitePayload>,
+        }
 
+        perform_network_request_with_retry("get_check_suites_for_commit", || async {
             let response: CheckSuiteResponse = self
                 .client
                 .get(
@@ -218,7 +219,7 @@ impl GithubRepositoryClient {
                 .collect();
             Ok(suites)
         })
-        .await
+        .await?
     }
 
     /// Cancels Github Actions workflows.
@@ -245,7 +246,7 @@ impl GithubRepositoryClient {
 
     /// Add a set of labels to a PR.
     pub async fn add_labels(&self, pr: PullRequestNumber, labels: &[String]) -> anyhow::Result<()> {
-        measure_network_request("add_labels", || async {
+        perform_network_request_with_retry("add_labels", || async {
             let client = self
                 .client
                 .issues(self.repository().owner(), self.repository().name());
@@ -258,7 +259,7 @@ impl GithubRepositoryClient {
 
             Ok(())
         })
-        .await
+        .await?
     }
 
     /// Remove a set of labels from a PR.
@@ -267,7 +268,7 @@ impl GithubRepositoryClient {
         pr: PullRequestNumber,
         labels: &[String],
     ) -> anyhow::Result<()> {
-        measure_network_request("remove_labels", || async {
+        perform_network_request_with_retry("remove_labels", || async {
             let client = self
                 .client
                 .issues(self.repository().owner(), self.repository().name());
@@ -283,7 +284,7 @@ impl GithubRepositoryClient {
                     Err(error) => match error {
                         // This error is returned if we try to remove a label that does not exist on the issue.
                         // This should be a no-op, rather than an error, therefore we swallow this error.
-                        Error::GitHub { source, .. }
+                        octocrab::Error::GitHub { source, .. }
                             if source.message.contains("Label does not exist") =>
                         {
                             log::trace!("Trying to remove label which does not exist on PR {pr}");
@@ -296,7 +297,7 @@ impl GithubRepositoryClient {
                 .context("Cannot remove label(s) from PR")?;
             Ok(())
         })
-        .await
+        .await?
     }
 
     /// Get a workflow url.
diff --git a/src/github/api/mod.rs b/src/github/api/mod.rs
index 8c27f5de..e93475df 100644
--- a/src/github/api/mod.rs
+++ b/src/github/api/mod.rs
@@ -1,5 +1,6 @@
 use std::collections::HashMap;
 use std::sync::Arc;
+use std::time::Duration;
 
 use anyhow::Context;
 use arc_swap::ArcSwap;
@@ -17,6 +18,10 @@ use crate::permissions::TeamApiClient;
 pub mod client;
 pub(crate) mod operations;
 
+pub const DEFAULT_REQUEST_TIMEOUT: Duration = Duration::from_secs(10);
+
+pub const DEFAULT_RETRY_COUNT: u32 = 5;
+
 fn base_github_html_url() -> &'static str {
     "https://github.com"
 }
@@ -32,6 +37,9 @@ pub fn create_github_client(
     Octocrab::builder()
         .base_uri(github_url)?
         .app(app_id, key)
+        .set_read_timeout(Some(DEFAULT_REQUEST_TIMEOUT))
+        .set_write_timeout(Some(DEFAULT_REQUEST_TIMEOUT))
+        .set_connect_timeout(Some(DEFAULT_REQUEST_TIMEOUT))
         .build()
         .context("Could not create octocrab builder")
 }
diff --git a/src/github/api/operations.rs b/src/github/api/operations.rs
index 30547b15..d0326a6f 100644
--- a/src/github/api/operations.rs
+++ b/src/github/api/operations.rs
@@ -17,6 +17,8 @@ pub enum MergeError {
     Unknown { status: StatusCode, text: String },
     #[error("Network error: {0}")]
     NetworkError(#[from] octocrab::Error),
+    #[error("Request timed out")]
+    Timeout,
 }
 
 #[derive(serde::Serialize)]
diff --git a/src/utils/timing.rs b/src/utils/timing.rs
index bc144ae0..44c2a2f7 100644
--- a/src/utils/timing.rs
+++ b/src/utils/timing.rs
@@ -1,6 +1,12 @@
-use std::time::Instant;
+use anyhow::anyhow;
+use std::time::{Duration, Instant};
+use tokio::time;
 use tracing::trace;
 
+use crate::github::api::{DEFAULT_REQUEST_TIMEOUT, DEFAULT_RETRY_COUNT};
+
+const DEFAULT_BACKOFF_TIME: Duration = Duration::from_secs(1);
+
 // Measures the duration of an async operation and logs it using tracing.
 pub async fn measure_operation<T, F, Fut>(operation_name: &str, f: F) -> T
 where
@@ -41,6 +47,63 @@ where
     measure_operation(&format!("network_request:{request_name}"), f).await
 }
 
+// Measures the duration of an async operation and retries it if it fails.
+pub async fn perform_network_request_with_retry<T, F, Fut>(
+    request_name: &str,
+    f: F,
+) -> anyhow::Result<T>
+where
+    F: Fn() -> Fut,
+    Fut: std::future::Future<Output = T>,
+{
+    let operation_name: &str = &format!("network_request:{request_name}");
+    trace!(operation = operation_name, "Starting operation");
+
+    for attempt in 0..=DEFAULT_RETRY_COUNT {
+        let start = Instant::now();
+
+        match time::timeout(DEFAULT_REQUEST_TIMEOUT, f()).await {
+            Ok(result) => {
+                let duration = start.elapsed();
+                trace!(
+                    operation = operation_name,
+                    attempt = attempt,
+                    duration_ms = format!("{:.2}", duration.as_secs_f64() * 1000.0),
+                    "Operation completed successfully"
+                );
+                return Ok(result);
+            }
+            Err(_) => {
+                trace!(
+                    operation = operation_name,
+                    attempt = attempt,
+                    "Operation timed out"
+                );
+                if attempt < DEFAULT_RETRY_COUNT {
+                    trace!(
+                        operation = operation_name,
+                        attempt = attempt + 1,
+                        "Retrying operation..."
+                    );
+                }
+            }
+        }
+
+        time::sleep(DEFAULT_BACKOFF_TIME).await;
+    }
+
+    trace!(
+        operation = operation_name,
+        retries = DEFAULT_RETRY_COUNT,
+        "Operation failed after all retries"
+    );
+    Err(anyhow!(
+        "Operation '{}' timed out after {} retries",
+        operation_name,
+        DEFAULT_RETRY_COUNT
+    ))
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -56,8 +119,8 @@ mod tests {
             42
         }
 
-        let result = measure_operation("test_op", sample_operation).await;
-        assert_eq!(result, 42);
+        let result = perform_network_request_with_retry("test_op", sample_operation).await;
+        assert_eq!(result.unwrap_or_default(), 42);
     }
 
     #[tracing_test::traced_test]
@@ -69,7 +132,9 @@ mod tests {
             Err("test error".to_string())
         }
 
-        let result = measure_operation("error_test", failing_operation).await;
+        let result = perform_network_request_with_retry("error_test", failing_operation)
+            .await
+            .unwrap_or(Ok(0));
         assert!(result.is_err());
         assert_eq!(result.unwrap_err(), "test error");
     }


Reviews:
user: login: Kobzol
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-03T11:37:09Z
user: login: Kobzol
body: The approach looks fine, thanks!
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-07T15:54:12Z
user: login: Kobzol
body: Thanks! Now that you added the timeouts everywhere, it is a bit noisy having the timeout and retry count at all GitHub call-sites. Since they all use the same values, let's just hardcode `DEFAULT_REQUEST_TIMEOUT` and `DEFAULT_RETRY_COUNT` within `perform_network_request`. We can always add a parametrized version in the future if the need arises.
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-09T11:42:17Z
user: login: Kobzol
body: Nice, even the indentation and diff got better. Thanks!
state: APPROVED
author_association: MEMBER
submitted_at: 2025-04-10T07:30:11Z
user: login: Kobzol
body: Thanks! Let's try again.
state: APPROVED
author_association: MEMBER
submitted_at: 2025-04-10T10:39:10Z

Review comments:
diff_hunk: @@ -17,6 +18,9 @@ use crate::permissions::TeamApiClient;
 pub mod client;
 pub(crate) mod operations;
 
+// timeout in seconds
+const TIMEOUT: u64 = 10;
path: src/github/api/mod.rs
user: login: Kobzol
body: ```suggestion
const HTTP_TIMEOUT: Duration = Duration::from_secs(10);
```
And then we don't even need the comment :)
created_at: 2025-04-03T11:32:33Z
author_association: MEMBER
diff_hunk: @@ -12,6 +15,31 @@ use crate::github::api::operations::{MergeError, merge_branches, set_branch_to_c
 use crate::github::{CommitSha, GithubRepoName, PullRequest, PullRequestNumber};
 use crate::utils::timing::measure_network_request;
 
+pub async fn auto_retry_github_request<T, F, Fut>(
+    request_name: &str,
+    f: F,
+) -> Result<T, octocrab::Error>
+where
+    F: Fn() -> Fut + Copy,
+    Fut: std::future::Future<Output = Result<T, octocrab::Error>>,
+{
+    let mut res = measure_network_request(request_name, f).await;
+    if let Err(octocrab::Error::Service { source, .. }) = &res {
path: src/github/api/client.rs
user: login: Kobzol
body: Hmm, I think that this is too ugly in the end. But mainly, I realized that if octocrab changes its internal representation of the errors, then this will silently stop working, and that's not great.

Let's just use the tokio timeout then (but still keep the octocrab timeout setting, that's good practice IMO), at least we will have more control over it.

I would generalize `measure_network_request`, rename it to something like `perform_network_request`, and pass it a timeout and a number of retries that it should perform when a timeout is encountered. By doing the timeouts in that function, we will be able to provide better log messages about what happens, because currently the log would just contain something like "operation started", and the "operation finished" part would be missing.
created_at: 2025-04-03T11:37:07Z
author_association: MEMBER
diff_hunk: @@ -23,6 +25,62 @@ where
     result
 }
 
+// Measures the duration of an async operation and retries it if it fails.
+pub async fn measure_operation_with_retry<T, F, Fut>(
path: src/utils/timing.rs
user: login: Kobzol
body: ```suggestion
pub async fn perform_operation_with_retry<T, F, Fut>(
```
Now the main logic is the retrying, the fact that it is measured is just an implementation detail.
created_at: 2025-04-07T15:45:49Z
author_association: MEMBER
diff_hunk: @@ -33,12 +91,23 @@ where
 }
 
 // Measures the duration of a network request and logs it using tracing.
-pub async fn measure_network_request<T, F, Fut>(request_name: &str, f: F) -> T
+pub async fn perform_network_request<T, F, Fut>(
+    request_name: &str,
+    f: F,
+    timeout: Duration,
+    num_retry: u32,
path: src/utils/timing.rs
user: login: Kobzol
body: ```suggestion
    retry_count: u32,
```
created_at: 2025-04-07T15:48:40Z
author_association: MEMBER
diff_hunk: @@ -17,6 +18,10 @@ use crate::permissions::TeamApiClient;
 pub mod client;
 pub(crate) mod operations;
 
+pub const TIMEOUT: Duration = Duration::from_secs(10);
+
+pub const NUM_RETRY: u32 = 5;
path: src/github/api/mod.rs
user: login: Kobzol
body: ```suggestion
pub const DEFAULT_RETRY_COUNT: u32 = 5;
```
created_at: 2025-04-07T15:49:06Z
author_association: MEMBER
diff_hunk: @@ -17,6 +18,10 @@ use crate::permissions::TeamApiClient;
 pub mod client;
 pub(crate) mod operations;
 
+pub const TIMEOUT: Duration = Duration::from_secs(10);
path: src/github/api/mod.rs
user: login: Kobzol
body: ```suggestion
pub const DEFAULT_REQUEST_TIMEOUT: Duration = Duration::from_secs(10);
```
created_at: 2025-04-07T15:50:43Z
author_association: MEMBER
diff_hunk: @@ -23,6 +25,62 @@ where
     result
 }
 
+// Measures the duration of an async operation and retries it if it fails.
+pub async fn measure_operation_with_retry<T, F, Fut>(
+    operation_name: &str,
+    f: &F,
+    timeout: Duration,
+    num_retries: u32,
+) -> anyhow::Result<T>
+where
+    F: Fn() -> Fut,
+    Fut: std::future::Future<Output = T>,
+{
+    trace!(operation = operation_name, "Starting operation");
+
+    for attempt in 0..=num_retries {
+        let start = Instant::now();
+
+        match time::timeout(timeout, f()).await {
path: src/utils/timing.rs
user: login: Kobzol
body: We might want to have some small back-off here, at least sleep for a second or two (it shouldn't be applied on the first try, so better do it at the end of the loop).
created_at: 2025-04-07T15:54:07Z
author_association: MEMBER
diff_hunk: @@ -164,101 +200,116 @@ impl GithubRepositoryClient {
         branch: &str,
         sha: &CommitSha,
     ) -> anyhow::Result<Vec<CheckSuite>> {
-        measure_network_request("get_check_suites_for_commit", || async {
-            #[derive(serde::Deserialize, Debug)]
-            struct CheckSuitePayload {
-                conclusion: Option<String>,
-                head_branch: String,
-            }
+        perform_network_request(
+            "get_check_suites_for_commit",
+            || async {
+                #[derive(serde::Deserialize, Debug)]
path: src/github/api/client.rs
user: login: Kobzol
body: Let's move these struct definitions outside the closure, to avoid the unnecessary indentation.
created_at: 2025-04-09T11:41:06Z
author_association: MEMBER

PR comments:
user: login: geetanshjuneja
created_at: 2025-04-03T09:16:43Z
body: I've made a change in one place for now to confirm that this is the approach we want to take for handling timeouts.
Also retry logic is still incomplete.

author_association: CONTRIBUTOR
user: login: geetanshjuneja
created_at: 2025-04-04T15:42:27Z
body: @Kobzol I am not sure how can I check rate limit headers with each gh request as octocrab doesn't expose response headers. We can check rate limit through its octocrab [rate limit handler](https://docs.rs/octocrab/latest/octocrab/struct.Octocrab.html#method.ratelimit).
author_association: CONTRIBUTOR
user: login: Kobzol
created_at: 2025-04-04T16:56:33Z
body: Let's deal only with timeouts for now, then. We probably can't do anything about the rate limit anyway.. But hopefully it shouldn't (ever :crossed_fingers:) be triggered, at least not the per hour one (which should be 5k requests, IIRC).
author_association: MEMBER
user: login: geetanshjuneja
created_at: 2025-04-04T18:16:34Z
body: I am done with this, you can review whenever you have a moment.
author_association: CONTRIBUTOR
user: login: Kobzol
created_at: 2025-04-10T07:42:59Z
body: Oh, the test failure showed something interesting. In `GitHubRepositoryClient::cancel_workflows`, we shouldn't retry the whole function, because it actually does several network calls. Let's not retry at all in that function, because it is best-effort anyway.

Seems like retrying is not always the best choice. Please also remove the function to `perform_network_request_with_retry`, or something like that, to make it super obvious that a retry mechanism will be used inside.
author_association: MEMBER
user: login: geetanshjuneja
created_at: 2025-04-10T08:28:39Z
body: > Oh, the test failure showed something interesting. In `GitHubRepositoryClient::cancel_workflows`, we shouldn't retry the whole function, because it actually does several network calls. Let's not retry at all in that function, because it is best-effort anyway.

Since all the workflows are cancelled in parallel, shouldn't we a have retry on each on them?
author_association: CONTRIBUTOR
user: login: Kobzol
created_at: 2025-04-10T08:31:03Z
body: If it was a critical operation, then yes, but cancelling is just a best-effort thing, it doesn't really matter if it fails or not, so we shouldn't block the rest of the bot waiting for it if it takes too long.
author_association: MEMBER