Repository: googleapis/google-cloud-rust
Pull request #1554: impl(auth): introduce TokenCache
Author: login: sai-sunder-s
Created: 2025-03-20T04:14:11Z | Merged: 2025-03-22T14:50:23Z

The token cache has a background task that keeps refreshing in the background before the token expires, irrespective of whether the token is used.

We can optimize in future to refresh only if the token is used in the past hour.


Diff:
diff --git a/src/auth/src/lib.rs b/src/auth/src/lib.rs
index 9c758aecb8..1873c8345f 100644
--- a/src/auth/src/lib.rs
+++ b/src/auth/src/lib.rs
@@ -47,6 +47,9 @@ pub mod credentials;
 /// [Tokens]: https://cloud.google.com/docs/authentication#token
 pub mod token;
 
+/// The token cache
+pub(crate) mod token_cache;
+
 /// A `Result` alias where the `Err` case is
 /// `google_cloud_auth::errors::CredentialError`.
 pub(crate) type Result<T> = std::result::Result<T, crate::errors::CredentialError>;
diff --git a/src/auth/src/token_cache.rs b/src/auth/src/token_cache.rs
new file mode 100644
index 0000000000..a73871ea76
--- /dev/null
+++ b/src/auth/src/token_cache.rs
@@ -0,0 +1,591 @@
+// Copyright 2025 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::Result;
+use crate::token::{Token, TokenProvider};
+use tokio::sync::watch;
+use tokio::time::{Duration, Instant, sleep};
+
+// Different MDS(Metadata Service) backends have different policies to
+// determine when to refresh a token. Most MDS' refresh token 5 mins before
+// expiry, except for Serverless which refresh tokens 4 mins before
+// expiry. So we are using 4 mins as the staleness limit for our refresh logic.
+const NORMAL_REFRESH_SLACK: Duration = Duration::from_secs(240);
+const SHORT_REFRESH_SLACK: Duration = Duration::from_secs(10);
+
+#[derive(Debug, Clone)]
+pub(crate) struct TokenCache {
+    rx_token: watch::Receiver<Option<Result<Token>>>,
+}
+
+// TODO(#1552): Use the token cache in all creds
+#[allow(dead_code)]
+impl TokenCache {
+    pub(crate) fn new<T>(inner: T) -> Self
+    where
+        T: TokenProvider + Send + Sync + 'static,
+    {
+        let (tx_token, rx_token) = watch::channel::<Option<Result<Token>>>(None);
+
+        tokio::spawn(async move {
+            refresh_task(inner, tx_token).await;
+        });
+
+        Self { rx_token }
+    }
+}
+
+#[async_trait::async_trait]
+impl TokenProvider for TokenCache {
+    async fn get_token(&self) -> Result<Token> {
+        let mut rx = self.rx_token.clone();
+        let token_result = rx.borrow_and_update().clone();
+
+        if let Some(token_result) = token_result {
+            match token_result {
+                Ok(token) => match token.expires_at {
+                    None => return Ok(token),
+                    Some(e) => {
+                        if e < Instant::now().into_std() {
+                            // Expired token, wait for refresh
+                            return wait_for_next_token(rx).await;
+                        } else {
+                            // valid token
+                            return Ok(token);
+                        }
+                    }
+                },
+                // An error in the result is still a valid result to propagate to the client library
+                Err(e) => Err(e),
+            }
+        } else {
+            wait_for_next_token(rx).await
+        }
+    }
+}
+
+async fn wait_for_next_token(
+    mut rx_token: watch::Receiver<Option<Result<Token>>>,
+) -> Result<Token> {
+    rx_token.changed().await.unwrap();
+    let token_result = rx_token.borrow().clone();
+
+    token_result.expect("There should always be a token or error in the channel after changed()")
+}
+
+async fn refresh_task<T>(token_provider: T, tx_token: watch::Sender<Option<Result<Token>>>)
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    loop {
+        let token_result = token_provider.get_token().await;
+
+        let _ = tx_token.send(Some(token_result.clone()));
+
+        match token_result {
+            Ok(new_token) => {
+                if let Some(expiry) = new_token.expires_at {
+                    let time_until_expiry =
+                        expiry.checked_duration_since(Instant::now().into_std());
+
+                    match time_until_expiry {
+                        None => {
+                            // We were given a token that is expired, or expires in less than 10 seconds.
+                            // We will immediately restart the loop, and fetch a new token.
+                        }
+                        Some(time_until_expiry) => {
+                            if time_until_expiry > NORMAL_REFRESH_SLACK {
+                                sleep(time_until_expiry - NORMAL_REFRESH_SLACK).await;
+                            } else if time_until_expiry > SHORT_REFRESH_SLACK {
+                                // If expiry is less than 4 mins, try to refresh every 10 seconds
+                                // This is to handle cases where MDS **repeatedly** returns about to expire tokens.
+                                sleep(SHORT_REFRESH_SLACK).await;
+                            }
+                        }
+                    }
+                } else {
+                    // If there is no expiry, the token is valid forever, so no need to refresh
+                    // TODO(#1553): Validate that all auth backends provide expiry and make expiry not optional.
+                    break;
+                }
+            }
+            Err(_) => {
+                // The retry policy has been used already by the inner token provider.
+                // If it ended in an error, just quit the background task.
+                break;
+            }
+        }
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    use crate::errors::CredentialError;
+    use crate::token::test::MockTokenProvider;
+    use std::ops::{Add, Sub};
+    use std::sync::{Arc, Mutex};
+    use tokio::time::{Duration, Instant};
+
+    static TOKEN_VALID_DURATION: Duration = Duration::from_secs(3600);
+
+    #[tokio::test]
+    async fn initial_token_success() {
+        let expected = Token {
+            token: "test-token".to_string(),
+            token_type: "Bearer".to_string(),
+            expires_at: None,
+            metadata: None,
+        };
+        let expected_clone = expected.clone();
+
+        let mut mock = MockTokenProvider::new();
+        mock.expect_get_token()
+            .times(1)
+            .return_once(|| Ok(expected_clone));
+
+        let cache = TokenCache::new(mock);
+        let actual = cache.get_token().await.unwrap();
+        assert_eq!(actual, expected);
+
+        // Verify that we use the cached token instead of making a new request
+        // to the mock token provider.
+        let actual = cache.get_token().await.unwrap();
+        assert_eq!(actual, expected);
+    }
+
+    #[tokio::test]
+    async fn initial_token_failure() {
+        let mut mock = MockTokenProvider::new();
+        mock.expect_get_token()
+            .times(1)
+            .returning(|| Err(CredentialError::non_retryable_from_str("fail")));
+
+        let cache = TokenCache::new(mock);
+        assert!(cache.get_token().await.is_err());
+
+        // Verify that a new request is made to the mock token provider when we
+        // don't have a valid token.
+        assert!(cache.get_token().await.is_err());
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn expired_token_success() {
+        let now = Instant::now();
+
+        let initial = Token {
+            token: "initial-token".to_string(),
+            token_type: "Bearer".to_string(),
+            expires_at: Some((now + TOKEN_VALID_DURATION).into_std()),
+            metadata: None,
+        };
+        let initial_clone = initial.clone();
+
+        let refresh = Token {
+            token: "refreshed-token".to_string(),
+            token_type: "Bearer".to_string(),
+            expires_at: Some((now + 2 * TOKEN_VALID_DURATION).into_std()),
+            metadata: None,
+        };
+        let refresh_clone = refresh.clone();
+
+        let mut mock = MockTokenProvider::new();
+        mock.expect_get_token()
+            .times(1)
+            .return_once(|| Ok(initial_clone));
+
+        mock.expect_get_token()
+            .times(1)
+            .return_once(|| Ok(refresh_clone));
+
+        // fetch an initial token
+        let cache = TokenCache::new(mock);
+        let actual = cache.get_token().await.unwrap();
+        assert_eq!(actual, initial);
+
+        // wait long enough for the token to be expired
+        // get_token should be waiting until the new token is available
+        let sleep = TOKEN_VALID_DURATION.add(Duration::from_secs(100));
+        tokio::time::advance(sleep).await;
+
+        // make sure this is the new token
+        let actual = cache.get_token().await.unwrap();
+        assert_eq!(actual, refresh);
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn expired_token_failure() {
+        let now = Instant::now();
+
+        let initial = Token {
+            token: "initial-token".to_string(),
+            token_type: "Bearer".to_string(),
+            expires_at: Some((now + TOKEN_VALID_DURATION).into_std()),
+            metadata: None,
+        };
+        let initial_clone = initial.clone();
+
+        let mut mock = MockTokenProvider::new();
+        mock.expect_get_token()
+            .times(1)
+            .return_once(|| Ok(initial_clone));
+
+        mock.expect_get_token()
+            .times(1)
+            .return_once(|| Err(CredentialError::non_retryable_from_str("fail")));
+
+        // fetch an initial token
+        let cache = TokenCache::new(mock);
+        let actual = cache.get_token().await.unwrap();
+        assert_eq!(actual, initial);
+
+        // wait long enough for the token to be expired
+        let sleep = TOKEN_VALID_DURATION.add(Duration::from_secs(100));
+        tokio::time::advance(sleep).await;
+
+        // make sure we return the error, not the expired token
+        assert!(cache.get_token().await.is_err());
+    }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 8)]
+    async fn token_cache_multiple_requests_existing_valid_token() {
+        let now = Instant::now();
+
+        let token = Token {
+            token: "initial-token".to_string(),
+            token_type: "Bearer".to_string(),
+            expires_at: Some((now + TOKEN_VALID_DURATION).into_std()),
+            metadata: None,
+        };
+        let token_clone = token.clone();
+
+        let mut mock = MockTokenProvider::new();
+        mock.expect_get_token()
+            .times(1)
+            .return_once(|| Ok(token_clone));
+
+        // fetch an initial token
+        let cache = TokenCache::new(mock);
+        let actual = cache.get_token().await.unwrap();
+        assert_eq!(actual, token);
+
+        // Spawn N tasks, all asking for a token at once.
+        let tasks = (0..1000)
+            .map(|_| {
+                let cache_clone = cache.clone();
+                tokio::spawn(async move { cache_clone.get_token().await })
+            })
+            .collect::<Vec<_>>();
+
+        // Wait for the N token requests to complete, verifying the returned token.
+        for task in tasks {
+            let actual = task.await.unwrap();
+            assert!(actual.is_ok(), "{}", actual.err().unwrap());
+            assert_eq!(actual.unwrap(), token);
+        }
+    }
+
+    #[tokio::test]
+    async fn refresh_task_expired_token_loop() {
+        let now = Instant::now();
+
+        let token1 = Token {
+            token: "token1".to_string(),
+            token_type: "Bearer".to_string(),
+            expires_at: Some(now.into_std()),
+            metadata: None,
+        };
+        let token1_clone = token1.clone();
+
+        let token2 = Token {
+            token: "token2".to_string(),
+            token_type: "Bearer".to_string(),
+            expires_at: Some((now + TOKEN_VALID_DURATION).into_std()),
+            metadata: None,
+        };
+        let token2_clone = token2.clone();
+
+        let mut mock = MockTokenProvider::new();
+        mock.expect_get_token()
+            .times(1)
+            .return_once(|| Ok(token1_clone));
+
+        mock.expect_get_token()
+            .times(1)
+            .return_once(|| Ok(token2_clone));
+
+        let (tx, mut rx) = watch::channel::<Option<Result<Token>>>(None);
+
+        tokio::spawn(async move {
+            refresh_task(mock, tx).await;
+        });
+
+        // Give the refresh task a chance to run
+        sleep(Duration::from_millis(100)).await;
+
+        rx.changed().await.unwrap();
+
+        // Validate that the refresh loop tried getting new token almost immediately
+        assert!(Instant::now() <= now + Duration::from_millis(500));
+
+        let actual = rx.borrow().clone().unwrap().unwrap();
+        assert_eq!(actual, token2.clone());
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn refresh_task_loop() {
+        let now = Instant::now();
+
+        let token1 = Token {
+            token: "token1".to_string(),
+            token_type: "Bearer".to_string(),
+            expires_at: Some((now + TOKEN_VALID_DURATION).into_std()),
+            metadata: None,
+        };
+        let token1_clone = token1.clone();
+
+        let token2 = Token {
+            token: "token2".to_string(),
+            token_type: "Bearer".to_string(),
+            expires_at: Some((now + 2 * TOKEN_VALID_DURATION).into_std()),
+            metadata: None,
+        };
+        let token2_clone = token2.clone();
+
+        let token3 = Token {
+            token: "token3".to_string(),
+            token_type: "Bearer".to_string(),
+            expires_at: Some((now + 3 * TOKEN_VALID_DURATION).into_std()),
+            metadata: None,
+        };
+        let token3_clone = token3.clone();
+
+        let mut mock = MockTokenProvider::new();
+        mock.expect_get_token()
+            .times(1)
+            .return_once(|| Ok(token1_clone));
+
+        mock.expect_get_token()
+            .times(1)
+            .return_once(|| Ok(token2_clone));
+
+        mock.expect_get_token()
+            .times(1)
+            .return_once(|| Ok(token3_clone));
+
+        let (tx, mut rx) = watch::channel::<Option<Result<Token>>>(None);
+
+        // check that channel has None before refresh task starts
+        let actual = rx.borrow().clone();
+        assert!(actual.is_none());
+
+        tokio::spawn(async move {
+            refresh_task(mock, tx).await;
+        });
+
+        rx.changed().await.unwrap();
+        let actual = rx.borrow().clone().unwrap().unwrap();
+        assert_eq!(actual, token1.clone());
+
+        // Validate that it is the same token before it is stale
+        let sleep = Duration::from_secs(120);
+        tokio::time::advance(sleep).await;
+        let actual = rx.borrow().clone().unwrap().unwrap();
+        assert_eq!(actual, token1.clone());
+
+        // time machine takes execution to 3 minutes before expiry
+        tokio::time::advance(TOKEN_VALID_DURATION.sub(Duration::from_secs(300))).await;
+
+        rx.changed().await.unwrap();
+
+        // validate that the token changed less than 4 mins before expiry
+        assert!(Instant::now() < now + TOKEN_VALID_DURATION);
+        let actual = rx.borrow().clone().unwrap().unwrap();
+        assert_eq!(actual, token2);
+
+        // wait long enough for the token to be expired
+        // Adding 500 secs to account for the time manipulation above
+        let sleep = TOKEN_VALID_DURATION.add(Duration::from_secs(500));
+        tokio::time::advance(sleep).await;
+
+        rx.changed().await.unwrap();
+        let actual = rx.borrow().clone().unwrap().unwrap();
+        assert_eq!(actual, token3);
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn refresh_task_loop_less_expiry() {
+        let now = Instant::now();
+
+        let token1 = Token {
+            token: "token1".to_string(),
+            token_type: "Bearer".to_string(),
+            expires_at: Some((now + Duration::from_secs(120)).into_std()),
+            metadata: None,
+        };
+        let token1_clone = token1.clone();
+        let token1_clone2 = token1.clone();
+
+        let token2 = Token {
+            token: "token2".to_string(),
+            token_type: "Bearer".to_string(),
+            expires_at: Some((now + 2 * TOKEN_VALID_DURATION).into_std()),
+            metadata: None,
+        };
+        let token2_clone = token2.clone();
+
+        let mut mock = MockTokenProvider::new();
+        mock.expect_get_token()
+            .times(1)
+            .return_once(|| Ok(token1_clone));
+
+        mock.expect_get_token()
+            .times(1)
+            .return_once(|| Ok(token1_clone2));
+
+        mock.expect_get_token()
+            .times(1)
+            .return_once(|| Ok(token2_clone));
+
+        let (tx, mut rx) = watch::channel::<Option<Result<Token>>>(None);
+
+        // check that channel has None before refresh task starts
+        let actual = rx.borrow().clone();
+        assert!(actual.is_none());
+
+        tokio::spawn(async move {
+            refresh_task(mock, tx).await;
+        });
+
+        rx.changed().await.unwrap();
+        let actual = rx.borrow().clone().unwrap().unwrap();
+        assert_eq!(actual, token1);
+
+        // time machine forwards time by 10 secs
+        tokio::time::advance(Duration::from_secs(10)).await;
+
+        // validate that the same token is obtained and it was
+        // attempted to be refreshed within 10ish seconds
+        assert!(Instant::now() < now + Duration::from_secs(11));
+        rx.changed().await.unwrap();
+        let actual = rx.borrow().clone().unwrap().unwrap();
+        assert_eq!(actual, token1);
+
+        // time machine forwards time by 100 secs
+        tokio::time::advance(Duration::from_secs(100)).await;
+
+        rx.changed().await.unwrap();
+
+        // validate that the token was refreshed within 10ish seconds
+        // before expiry
+        assert!(Instant::now() < now + Duration::from_secs(111));
+        let actual = rx.borrow().clone().unwrap().unwrap();
+        assert_eq!(actual, token2);
+    }
+
+    #[derive(Clone, Debug)]
+    struct FakeTokenProvider {
+        result: Result<Token>,
+        calls: Arc<std::sync::Mutex<i32>>,
+    }
+
+    impl FakeTokenProvider {
+        pub fn new(result: Result<Token>) -> Self {
+            FakeTokenProvider {
+                result,
+                calls: Arc::new(Mutex::new(0)),
+            }
+        }
+
+        pub fn calls(&self) -> i32 {
+            *self.calls.lock().unwrap()
+        }
+    }
+
+    #[async_trait::async_trait]
+    impl TokenProvider for FakeTokenProvider {
+        async fn get_token(&self) -> Result<Token> {
+            // We give enough time for the a thundering herd to pile up waiting for a change notification from watch channel
+            tokio::time::sleep(Duration::from_millis(50)).await;
+
+            // Track how many calls were made to the inner token provider.
+            *self.calls.lock().unwrap() += 1;
+
+            // Return the result.
+            self.result.clone()
+        }
+    }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 8)]
+    async fn no_initial_token_thundering_herd_success() {
+        let token = Token {
+            token: "delayed-token".to_string(),
+            token_type: "Bearer".to_string(),
+            expires_at: Some(std::time::Instant::now()),
+            metadata: None,
+        };
+
+        let tp = FakeTokenProvider::new(Ok(token.clone()));
+
+        let cache = TokenCache::new(tp.clone());
+
+        // Spawn N tasks, all asking for a token at once.
+        let tasks = (0..100)
+            .map(|_| {
+                let cache_clone = cache.clone();
+                tokio::spawn(async move { cache_clone.get_token().await })
+            })
+            .collect::<Vec<_>>();
+
+        // Wait for the N token requests to complete, verifying the returned token.
+        for task in tasks {
+            let actual = task.await.unwrap();
+            assert!(actual.is_ok(), "{}", actual.err().unwrap());
+            assert_eq!(actual.unwrap(), token);
+        }
+
+        let calls = tp.calls();
+        // Only one call to token provider should have been made
+        assert_eq!(calls, 1);
+    }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 8)]
+    async fn no_initial_token_thundering_herd_failure_shares_error() {
+        let err = Err(CredentialError::non_retryable_from_str("epic fail"));
+
+        let tp = FakeTokenProvider::new(err);
+
+        let cache = TokenCache::new(tp.clone());
+
+        // Spawn N tasks, all asking for a token at once.
+        let tasks = (0..100)
+            .map(|_| {
+                let cache_clone = cache.clone();
+                tokio::spawn(async move { cache_clone.get_token().await })
+            })
+            .collect::<Vec<_>>();
+
+        // Wait for the N token requests to complete, verifying the returned error.
+        for task in tasks {
+            let actual = task.await.unwrap();
+            assert!(actual.is_err(), "{:?}", actual.unwrap());
+            let e = format!("{}", actual.err().unwrap());
+            assert!(e.contains("epic fail"), "{e}");
+        }
+
+        let calls = tp.calls();
+        // Only one call to token provider should have been made
+        assert_eq!(calls, 1);
+    }
+}


Reviews:
user: login: dbolduc
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-20T16:34:42Z
user: login: dbolduc
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-20T16:48:37Z
user: login: sai-sunder-s
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-20T16:53:48Z
user: login: dbolduc
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-20T18:25:51Z
user: login: dbolduc
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-20T18:26:59Z
user: login: dbolduc
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-20T18:28:45Z
user: login: sai-sunder-s
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-20T18:29:07Z
user: login: sai-sunder-s
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-20T18:29:57Z
user: login: sai-sunder-s
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-20T18:31:09Z
user: login: dbolduc
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-20T18:31:23Z
user: login: sai-sunder-s
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-20T18:36:11Z
user: login: dbolduc
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-20T18:42:20Z
user: login: dbolduc
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-20T18:50:12Z
user: login: dbolduc
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-21T16:21:50Z
user: login: sai-sunder-s
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-21T16:40:00Z
user: login: sai-sunder-s
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-21T17:11:33Z
user: login: coryan
state: CHANGES_REQUESTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-21T17:45:52Z
user: login: sai-sunder-s
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-21T19:06:12Z
user: login: sai-sunder-s
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-21T19:11:54Z
user: login: sai-sunder-s
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-21T19:44:44Z
user: login: coryan
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-21T19:53:27Z
user: login: coryan
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-03-21T20:00:53Z
user: login: coryan
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-21T20:08:48Z
user: login: sai-sunder-s
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-21T21:15:36Z
user: login: coryan
state: CHANGES_REQUESTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-22T02:01:56Z
user: login: sai-sunder-s
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-22T02:15:54Z
user: login: coryan
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-22T13:25:30Z
user: login: coryan
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-03-22T13:25:40Z

Review comments:
diff_hunk: @@ -0,0 +1,538 @@
+// Copyright 2024 Google LLC
path: src/auth/src/token_cache.rs
user: login: dbolduc
body: nit: 2025
created_at: 2025-03-20T14:56:44Z
author_association: MEMBER
diff_hunk: @@ -0,0 +1,538 @@
+// Copyright 2024 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::token::{Token, TokenProvider};
+use crate::Result;
+use tokio::sync::watch;
+use tokio::time::{sleep, sleep_until, Duration, Instant};
+
+const FIVE_MINUTES: Duration = Duration::from_secs(300);
+const TEN_SECONDS: Duration = Duration::from_secs(10);
+
+#[derive(Debug)]
+pub(crate) struct TokenCache<T>
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    rx_token: watch::Receiver<Option<Result<Token>>>,
+    _phantom: std::marker::PhantomData<T>,
+}
+
+impl<T: TokenProvider> Clone for TokenCache<T> {
+    fn clone(&self) -> TokenCache<T> {
+        TokenCache {
+            rx_token: self.rx_token.clone(),
+            _phantom: std::marker::PhantomData,
+        }
+    }
+}
+
+// TODO 1552: Use the token cache in all creds
path: src/auth/src/token_cache.rs
user: login: dbolduc
body: nit: say `TODO(#1552)`
created_at: 2025-03-20T14:58:42Z
author_association: MEMBER
diff_hunk: @@ -0,0 +1,538 @@
+// Copyright 2024 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::token::{Token, TokenProvider};
+use crate::Result;
+use tokio::sync::watch;
+use tokio::time::{sleep, sleep_until, Duration, Instant};
+
+const FIVE_MINUTES: Duration = Duration::from_secs(300);
+const TEN_SECONDS: Duration = Duration::from_secs(10);
+
+#[derive(Debug)]
+pub(crate) struct TokenCache<T>
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    rx_token: watch::Receiver<Option<Result<Token>>>,
+    _phantom: std::marker::PhantomData<T>,
+}
+
+impl<T: TokenProvider> Clone for TokenCache<T> {
+    fn clone(&self) -> TokenCache<T> {
+        TokenCache {
+            rx_token: self.rx_token.clone(),
+            _phantom: std::marker::PhantomData,
+        }
+    }
+}
+
+// TODO 1552: Use the token cache in all creds
+#[allow(dead_code)]
+impl<T> TokenCache<T>
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    pub(crate) fn new(inner: T) -> Self {
+        let (tx_token, rx_token) = watch::channel::<Option<Result<Token>>>(None);
+
+        tokio::spawn(async move {
+            refresh_task(inner, tx_token).await;
+        });
+
+        Self {
+            rx_token,
+            _phantom: std::marker::PhantomData,
+        }
+    }
+}
+
+#[async_trait::async_trait]
+impl<T> TokenProvider for TokenCache<T>
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    async fn get_token(&self) -> Result<Token> {
+        let mut rx = self.rx_token.clone();
+        let token_result = rx.borrow_and_update().clone();
+
+        if let Some(token_result) = token_result {
+            match token_result {
+                Ok(token) => match token.expires_at {
+                    None => return Ok(token),
+                    Some(e) => {
+                        if e < Instant::now().into_std() {
+                            // Expired token, wait for refresh
+                            return get_token_from_channel(rx).await;
+                        } else {
+                            // valid token
+                            return Ok(token);
+                        }
+                    }
+                },
+                // An error in the result is still a valid result to propagate to the client library
+                Err(e) => Err(e),
+            }
+        } else {
+            get_token_from_channel(rx).await
+        }
+    }
+}
+
+async fn get_token_from_channel(
+    mut rx_token: watch::Receiver<Option<Result<Token>>>,
+) -> Result<Token> {
+    // We have already checked the current token in the channel, wait for new one
+    rx_token.changed().await.unwrap();
+    let token_result = rx_token.borrow().clone();
+
+    token_result.expect("There should always be a token or error in the channel after changed()")
+}
+
+async fn refresh_task<T>(token_provider: T, tx_token: watch::Sender<Option<Result<Token>>>)
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    loop {
+        let token_result = token_provider.get_token().await;
+
+        let _ = tx_token.send(Some(token_result.clone()));
+
+        match token_result {
+            Ok(new_token) => {
+                if let Some(expiry) = new_token.expires_at {
+                    let time_until_expiry = expiry.duration_since(Instant::now().into_std());
+
+                    if time_until_expiry > FIVE_MINUTES {
+                        let time_to_sleep = Instant::now() + time_until_expiry - FIVE_MINUTES;
+                        sleep_until(time_to_sleep).await;
+                    } else if time_until_expiry > TEN_SECONDS {
+                        // If expiry is less than 5 mins, wait till last 10 seconds to refresh
+                        // This is to handle cases where MDS returns about to refresh tokens
+                        let time_to_sleep = Instant::now() + time_until_expiry - TEN_SECONDS;
+                        sleep_until(time_to_sleep).await;
+                    }
+                } else {
+                    // Handle case where expiry is not available
+                    // TODO 1553: Validate that all auth backends provide expiry and make expiry not optional.
+                    sleep(FIVE_MINUTES).await;
path: src/auth/src/token_cache.rs
user: login: dbolduc
body: > make expiry not optional.

We know we have things like API key credentials which can't expire. Their expiries need to be represented with `Option<>`. (Of course, this type of credential would not need the token cache).

Basically, I think we are stuck handling this branch.

I think we should just break out of the loop, killing the refresh task. The token is allegedly valid forever, so we do not need the refresh task.

And I think we leave a comment saying why we do not expect this branch to get hit in practice.
created_at: 2025-03-20T15:16:35Z
author_association: MEMBER
diff_hunk: @@ -0,0 +1,538 @@
+// Copyright 2024 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::token::{Token, TokenProvider};
+use crate::Result;
+use tokio::sync::watch;
+use tokio::time::{sleep, sleep_until, Duration, Instant};
+
+const FIVE_MINUTES: Duration = Duration::from_secs(300);
+const TEN_SECONDS: Duration = Duration::from_secs(10);
+
+#[derive(Debug)]
+pub(crate) struct TokenCache<T>
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    rx_token: watch::Receiver<Option<Result<Token>>>,
+    _phantom: std::marker::PhantomData<T>,
+}
+
+impl<T: TokenProvider> Clone for TokenCache<T> {
+    fn clone(&self) -> TokenCache<T> {
+        TokenCache {
+            rx_token: self.rx_token.clone(),
+            _phantom: std::marker::PhantomData,
+        }
+    }
+}
+
+// TODO 1552: Use the token cache in all creds
+#[allow(dead_code)]
+impl<T> TokenCache<T>
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    pub(crate) fn new(inner: T) -> Self {
+        let (tx_token, rx_token) = watch::channel::<Option<Result<Token>>>(None);
+
+        tokio::spawn(async move {
+            refresh_task(inner, tx_token).await;
+        });
+
+        Self {
+            rx_token,
+            _phantom: std::marker::PhantomData,
+        }
+    }
+}
+
+#[async_trait::async_trait]
+impl<T> TokenProvider for TokenCache<T>
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    async fn get_token(&self) -> Result<Token> {
+        let mut rx = self.rx_token.clone();
+        let token_result = rx.borrow_and_update().clone();
+
+        if let Some(token_result) = token_result {
+            match token_result {
+                Ok(token) => match token.expires_at {
+                    None => return Ok(token),
+                    Some(e) => {
+                        if e < Instant::now().into_std() {
+                            // Expired token, wait for refresh
+                            return get_token_from_channel(rx).await;
+                        } else {
+                            // valid token
+                            return Ok(token);
+                        }
+                    }
+                },
+                // An error in the result is still a valid result to propagate to the client library
+                Err(e) => Err(e),
+            }
+        } else {
+            get_token_from_channel(rx).await
+        }
+    }
+}
+
+async fn get_token_from_channel(
+    mut rx_token: watch::Receiver<Option<Result<Token>>>,
+) -> Result<Token> {
+    // We have already checked the current token in the channel, wait for new one
+    rx_token.changed().await.unwrap();
+    let token_result = rx_token.borrow().clone();
+
+    token_result.expect("There should always be a token or error in the channel after changed()")
+}
+
+async fn refresh_task<T>(token_provider: T, tx_token: watch::Sender<Option<Result<Token>>>)
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    loop {
+        let token_result = token_provider.get_token().await;
path: src/auth/src/token_cache.rs
user: login: dbolduc
body: comment: I assume the retry loop will go here.
created_at: 2025-03-20T15:18:53Z
author_association: MEMBER
diff_hunk: @@ -0,0 +1,538 @@
+// Copyright 2024 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::token::{Token, TokenProvider};
+use crate::Result;
+use tokio::sync::watch;
+use tokio::time::{sleep, sleep_until, Duration, Instant};
+
+const FIVE_MINUTES: Duration = Duration::from_secs(300);
+const TEN_SECONDS: Duration = Duration::from_secs(10);
+
+#[derive(Debug)]
+pub(crate) struct TokenCache<T>
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    rx_token: watch::Receiver<Option<Result<Token>>>,
+    _phantom: std::marker::PhantomData<T>,
+}
+
+impl<T: TokenProvider> Clone for TokenCache<T> {
+    fn clone(&self) -> TokenCache<T> {
+        TokenCache {
+            rx_token: self.rx_token.clone(),
+            _phantom: std::marker::PhantomData,
+        }
+    }
+}
+
+// TODO 1552: Use the token cache in all creds
+#[allow(dead_code)]
+impl<T> TokenCache<T>
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    pub(crate) fn new(inner: T) -> Self {
+        let (tx_token, rx_token) = watch::channel::<Option<Result<Token>>>(None);
+
+        tokio::spawn(async move {
+            refresh_task(inner, tx_token).await;
+        });
+
+        Self {
+            rx_token,
+            _phantom: std::marker::PhantomData,
+        }
+    }
+}
+
+#[async_trait::async_trait]
+impl<T> TokenProvider for TokenCache<T>
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    async fn get_token(&self) -> Result<Token> {
+        let mut rx = self.rx_token.clone();
+        let token_result = rx.borrow_and_update().clone();
+
+        if let Some(token_result) = token_result {
+            match token_result {
+                Ok(token) => match token.expires_at {
+                    None => return Ok(token),
+                    Some(e) => {
+                        if e < Instant::now().into_std() {
+                            // Expired token, wait for refresh
+                            return get_token_from_channel(rx).await;
+                        } else {
+                            // valid token
+                            return Ok(token);
+                        }
+                    }
+                },
+                // An error in the result is still a valid result to propagate to the client library
+                Err(e) => Err(e),
+            }
+        } else {
+            get_token_from_channel(rx).await
+        }
+    }
+}
+
+async fn get_token_from_channel(
+    mut rx_token: watch::Receiver<Option<Result<Token>>>,
+) -> Result<Token> {
+    // We have already checked the current token in the channel, wait for new one
+    rx_token.changed().await.unwrap();
+    let token_result = rx_token.borrow().clone();
+
+    token_result.expect("There should always be a token or error in the channel after changed()")
+}
+
+async fn refresh_task<T>(token_provider: T, tx_token: watch::Sender<Option<Result<Token>>>)
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    loop {
+        let token_result = token_provider.get_token().await;
+
+        let _ = tx_token.send(Some(token_result.clone()));
+
+        match token_result {
+            Ok(new_token) => {
+                if let Some(expiry) = new_token.expires_at {
+                    let time_until_expiry = expiry.duration_since(Instant::now().into_std());
+
+                    if time_until_expiry > FIVE_MINUTES {
+                        let time_to_sleep = Instant::now() + time_until_expiry - FIVE_MINUTES;
+                        sleep_until(time_to_sleep).await;
+                    } else if time_until_expiry > TEN_SECONDS {
+                        // If expiry is less than 5 mins, wait till last 10 seconds to refresh
+                        // This is to handle cases where MDS returns about to refresh tokens
+                        let time_to_sleep = Instant::now() + time_until_expiry - TEN_SECONDS;
+                        sleep_until(time_to_sleep).await;
+                    }
+                } else {
+                    // Handle case where expiry is not available
+                    // TODO 1553: Validate that all auth backends provide expiry and make expiry not optional.
+                    sleep(FIVE_MINUTES).await;
+                }
+            }
+            Err(_) => {
+                // The retry policy has been used already by the inner token provider.
path: src/auth/src/token_cache.rs
user: login: dbolduc
body: Can you clarify where the retry loop will go? I assume it is in this refresh task, not in the inner token provider.

Maybe change the wording to: "The retry loop ended in an error. Just quit the background task."
created_at: 2025-03-20T15:28:33Z
author_association: MEMBER
diff_hunk: @@ -0,0 +1,538 @@
+// Copyright 2024 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::token::{Token, TokenProvider};
+use crate::Result;
+use tokio::sync::watch;
+use tokio::time::{sleep, sleep_until, Duration, Instant};
+
+const FIVE_MINUTES: Duration = Duration::from_secs(300);
+const TEN_SECONDS: Duration = Duration::from_secs(10);
+
+#[derive(Debug)]
+pub(crate) struct TokenCache<T>
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    rx_token: watch::Receiver<Option<Result<Token>>>,
+    _phantom: std::marker::PhantomData<T>,
+}
+
+impl<T: TokenProvider> Clone for TokenCache<T> {
+    fn clone(&self) -> TokenCache<T> {
+        TokenCache {
+            rx_token: self.rx_token.clone(),
+            _phantom: std::marker::PhantomData,
+        }
+    }
+}
+
+// TODO 1552: Use the token cache in all creds
+#[allow(dead_code)]
+impl<T> TokenCache<T>
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    pub(crate) fn new(inner: T) -> Self {
+        let (tx_token, rx_token) = watch::channel::<Option<Result<Token>>>(None);
+
+        tokio::spawn(async move {
+            refresh_task(inner, tx_token).await;
+        });
+
+        Self {
+            rx_token,
+            _phantom: std::marker::PhantomData,
+        }
+    }
+}
+
+#[async_trait::async_trait]
+impl<T> TokenProvider for TokenCache<T>
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    async fn get_token(&self) -> Result<Token> {
+        let mut rx = self.rx_token.clone();
+        let token_result = rx.borrow_and_update().clone();
+
+        if let Some(token_result) = token_result {
+            match token_result {
+                Ok(token) => match token.expires_at {
+                    None => return Ok(token),
+                    Some(e) => {
+                        if e < Instant::now().into_std() {
+                            // Expired token, wait for refresh
+                            return get_token_from_channel(rx).await;
+                        } else {
+                            // valid token
+                            return Ok(token);
+                        }
+                    }
+                },
+                // An error in the result is still a valid result to propagate to the client library
+                Err(e) => Err(e),
+            }
+        } else {
+            get_token_from_channel(rx).await
+        }
+    }
+}
+
+async fn get_token_from_channel(
+    mut rx_token: watch::Receiver<Option<Result<Token>>>,
+) -> Result<Token> {
+    // We have already checked the current token in the channel, wait for new one
+    rx_token.changed().await.unwrap();
+    let token_result = rx_token.borrow().clone();
+
+    token_result.expect("There should always be a token or error in the channel after changed()")
+}
+
+async fn refresh_task<T>(token_provider: T, tx_token: watch::Sender<Option<Result<Token>>>)
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    loop {
+        let token_result = token_provider.get_token().await;
+
+        let _ = tx_token.send(Some(token_result.clone()));
+
+        match token_result {
+            Ok(new_token) => {
+                if let Some(expiry) = new_token.expires_at {
+                    let time_until_expiry = expiry.duration_since(Instant::now().into_std());
+
+                    if time_until_expiry > FIVE_MINUTES {
+                        let time_to_sleep = Instant::now() + time_until_expiry - FIVE_MINUTES;
+                        sleep_until(time_to_sleep).await;
+                    } else if time_until_expiry > TEN_SECONDS {
+                        // If expiry is less than 5 mins, wait till last 10 seconds to refresh
+                        // This is to handle cases where MDS returns about to refresh tokens
+                        let time_to_sleep = Instant::now() + time_until_expiry - TEN_SECONDS;
+                        sleep_until(time_to_sleep).await;
+                    }
+                } else {
+                    // Handle case where expiry is not available
+                    // TODO 1553: Validate that all auth backends provide expiry and make expiry not optional.
+                    sleep(FIVE_MINUTES).await;
+                }
+            }
+            Err(_) => {
+                // The retry policy has been used already by the inner token provider.
+                // If it ended in an error, just quit the background task.
path: src/auth/src/token_cache.rs
user: login: dbolduc
body: I left a comment in the doc also, but it is unfortunate if we kill the background refresh task when the retry loop exits with a retryable error.

Customer applications will see: "So and so failed with a retryable error", but the auth library refuses to retry for them. Applications need to recreate the `Credential` object in order to retry. :shrug: 
created_at: 2025-03-20T15:31:30Z
author_association: MEMBER
diff_hunk: @@ -0,0 +1,538 @@
+// Copyright 2024 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::token::{Token, TokenProvider};
+use crate::Result;
+use tokio::sync::watch;
+use tokio::time::{sleep, sleep_until, Duration, Instant};
+
+const FIVE_MINUTES: Duration = Duration::from_secs(300);
+const TEN_SECONDS: Duration = Duration::from_secs(10);
+
+#[derive(Debug)]
+pub(crate) struct TokenCache<T>
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    rx_token: watch::Receiver<Option<Result<Token>>>,
+    _phantom: std::marker::PhantomData<T>,
+}
+
+impl<T: TokenProvider> Clone for TokenCache<T> {
+    fn clone(&self) -> TokenCache<T> {
+        TokenCache {
+            rx_token: self.rx_token.clone(),
+            _phantom: std::marker::PhantomData,
+        }
+    }
+}
+
+// TODO 1552: Use the token cache in all creds
+#[allow(dead_code)]
+impl<T> TokenCache<T>
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    pub(crate) fn new(inner: T) -> Self {
+        let (tx_token, rx_token) = watch::channel::<Option<Result<Token>>>(None);
+
+        tokio::spawn(async move {
+            refresh_task(inner, tx_token).await;
+        });
+
+        Self {
+            rx_token,
+            _phantom: std::marker::PhantomData,
+        }
+    }
+}
+
+#[async_trait::async_trait]
+impl<T> TokenProvider for TokenCache<T>
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    async fn get_token(&self) -> Result<Token> {
+        let mut rx = self.rx_token.clone();
+        let token_result = rx.borrow_and_update().clone();
+
+        if let Some(token_result) = token_result {
+            match token_result {
+                Ok(token) => match token.expires_at {
+                    None => return Ok(token),
+                    Some(e) => {
+                        if e < Instant::now().into_std() {
+                            // Expired token, wait for refresh
+                            return get_token_from_channel(rx).await;
+                        } else {
+                            // valid token
+                            return Ok(token);
+                        }
+                    }
+                },
+                // An error in the result is still a valid result to propagate to the client library
+                Err(e) => Err(e),
+            }
+        } else {
+            get_token_from_channel(rx).await
+        }
+    }
+}
+
+async fn get_token_from_channel(
+    mut rx_token: watch::Receiver<Option<Result<Token>>>,
+) -> Result<Token> {
+    // We have already checked the current token in the channel, wait for new one
path: src/auth/src/token_cache.rs
user: login: dbolduc
body: If you call this `async fn wait_for_next_token()` the comment will not be necessary.
created_at: 2025-03-20T15:34:09Z
author_association: MEMBER
diff_hunk: @@ -0,0 +1,538 @@
+// Copyright 2024 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::token::{Token, TokenProvider};
+use crate::Result;
+use tokio::sync::watch;
+use tokio::time::{sleep, sleep_until, Duration, Instant};
+
+const FIVE_MINUTES: Duration = Duration::from_secs(300);
+const TEN_SECONDS: Duration = Duration::from_secs(10);
+
+#[derive(Debug)]
+pub(crate) struct TokenCache<T>
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    rx_token: watch::Receiver<Option<Result<Token>>>,
+    _phantom: std::marker::PhantomData<T>,
+}
+
+impl<T: TokenProvider> Clone for TokenCache<T> {
+    fn clone(&self) -> TokenCache<T> {
+        TokenCache {
+            rx_token: self.rx_token.clone(),
+            _phantom: std::marker::PhantomData,
+        }
+    }
+}
+
+// TODO 1552: Use the token cache in all creds
+#[allow(dead_code)]
+impl<T> TokenCache<T>
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    pub(crate) fn new(inner: T) -> Self {
+        let (tx_token, rx_token) = watch::channel::<Option<Result<Token>>>(None);
+
+        tokio::spawn(async move {
+            refresh_task(inner, tx_token).await;
+        });
+
+        Self {
+            rx_token,
+            _phantom: std::marker::PhantomData,
+        }
+    }
+}
+
+#[async_trait::async_trait]
+impl<T> TokenProvider for TokenCache<T>
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    async fn get_token(&self) -> Result<Token> {
+        let mut rx = self.rx_token.clone();
+        let token_result = rx.borrow_and_update().clone();
+
+        if let Some(token_result) = token_result {
+            match token_result {
+                Ok(token) => match token.expires_at {
+                    None => return Ok(token),
+                    Some(e) => {
+                        if e < Instant::now().into_std() {
+                            // Expired token, wait for refresh
+                            return get_token_from_channel(rx).await;
+                        } else {
+                            // valid token
+                            return Ok(token);
+                        }
+                    }
+                },
+                // An error in the result is still a valid result to propagate to the client library
+                Err(e) => Err(e),
+            }
+        } else {
+            get_token_from_channel(rx).await
+        }
+    }
+}
+
+async fn get_token_from_channel(
+    mut rx_token: watch::Receiver<Option<Result<Token>>>,
+) -> Result<Token> {
+    // We have already checked the current token in the channel, wait for new one
+    rx_token.changed().await.unwrap();
path: src/auth/src/token_cache.rs
user: login: dbolduc
body: https://docs.rs/tokio/latest/tokio/sync/watch/struct.Receiver.html#method.changed

```rs
// This method returns an error if and only if the `Sender` is dropped.
```

hmm.. I think we need to handle the error instead of using `unwrap()`, because there is a race.

Consider:

1. we get an expired token on L68

https://github.com/googleapis/google-cloud-rust/blob/3dd8abdebc54ea40d4173eaa451be10607e7c989/src/auth/src/token_cache.rs#L68

2. the background refresh task hits some error, and shuts down, dropping `tx_token`.

https://github.com/googleapis/google-cloud-rust/blob/3dd8abdebc54ea40d4173eaa451be10607e7c989/src/auth/src/token_cache.rs#L135

3. We wait for the next token on L77

https://github.com/googleapis/google-cloud-rust/blob/3dd8abdebc54ea40d4173eaa451be10607e7c989/src/auth/src/token_cache.rs#L77

I think the `unwrap()` will panic in this case.
created_at: 2025-03-20T16:12:01Z
author_association: MEMBER
diff_hunk: @@ -0,0 +1,538 @@
+// Copyright 2024 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::token::{Token, TokenProvider};
+use crate::Result;
+use tokio::sync::watch;
+use tokio::time::{sleep, sleep_until, Duration, Instant};
+
+const FIVE_MINUTES: Duration = Duration::from_secs(300);
+const TEN_SECONDS: Duration = Duration::from_secs(10);
+
+#[derive(Debug)]
+pub(crate) struct TokenCache<T>
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    rx_token: watch::Receiver<Option<Result<Token>>>,
+    _phantom: std::marker::PhantomData<T>,
+}
+
+impl<T: TokenProvider> Clone for TokenCache<T> {
+    fn clone(&self) -> TokenCache<T> {
+        TokenCache {
+            rx_token: self.rx_token.clone(),
+            _phantom: std::marker::PhantomData,
+        }
+    }
+}
+
+// TODO 1552: Use the token cache in all creds
+#[allow(dead_code)]
+impl<T> TokenCache<T>
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    pub(crate) fn new(inner: T) -> Self {
+        let (tx_token, rx_token) = watch::channel::<Option<Result<Token>>>(None);
path: src/auth/src/token_cache.rs
user: login: dbolduc
body: comment: coryan was right about `watch::channel()` being a good construct for this use case.
created_at: 2025-03-20T16:13:15Z
author_association: MEMBER
diff_hunk: @@ -0,0 +1,538 @@
+// Copyright 2024 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::token::{Token, TokenProvider};
+use crate::Result;
+use tokio::sync::watch;
+use tokio::time::{sleep, sleep_until, Duration, Instant};
+
+const FIVE_MINUTES: Duration = Duration::from_secs(300);
+const TEN_SECONDS: Duration = Duration::from_secs(10);
+
+#[derive(Debug)]
+pub(crate) struct TokenCache<T>
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    rx_token: watch::Receiver<Option<Result<Token>>>,
+    _phantom: std::marker::PhantomData<T>,
+}
+
+impl<T: TokenProvider> Clone for TokenCache<T> {
+    fn clone(&self) -> TokenCache<T> {
+        TokenCache {
+            rx_token: self.rx_token.clone(),
+            _phantom: std::marker::PhantomData,
+        }
+    }
+}
+
+// TODO 1552: Use the token cache in all creds
+#[allow(dead_code)]
+impl<T> TokenCache<T>
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    pub(crate) fn new(inner: T) -> Self {
+        let (tx_token, rx_token) = watch::channel::<Option<Result<Token>>>(None);
+
+        tokio::spawn(async move {
+            refresh_task(inner, tx_token).await;
+        });
+
+        Self {
+            rx_token,
+            _phantom: std::marker::PhantomData,
+        }
+    }
+}
+
+#[async_trait::async_trait]
+impl<T> TokenProvider for TokenCache<T>
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    async fn get_token(&self) -> Result<Token> {
+        let mut rx = self.rx_token.clone();
+        let token_result = rx.borrow_and_update().clone();
+
+        if let Some(token_result) = token_result {
+            match token_result {
+                Ok(token) => match token.expires_at {
+                    None => return Ok(token),
+                    Some(e) => {
+                        if e < Instant::now().into_std() {
+                            // Expired token, wait for refresh
+                            return get_token_from_channel(rx).await;
+                        } else {
+                            // valid token
+                            return Ok(token);
+                        }
+                    }
+                },
+                // An error in the result is still a valid result to propagate to the client library
+                Err(e) => Err(e),
+            }
+        } else {
+            get_token_from_channel(rx).await
+        }
+    }
+}
+
+async fn get_token_from_channel(
+    mut rx_token: watch::Receiver<Option<Result<Token>>>,
+) -> Result<Token> {
+    // We have already checked the current token in the channel, wait for new one
+    rx_token.changed().await.unwrap();
+    let token_result = rx_token.borrow().clone();
+
+    token_result.expect("There should always be a token or error in the channel after changed()")
+}
+
+async fn refresh_task<T>(token_provider: T, tx_token: watch::Sender<Option<Result<Token>>>)
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    loop {
+        let token_result = token_provider.get_token().await;
+
+        let _ = tx_token.send(Some(token_result.clone()));
+
+        match token_result {
+            Ok(new_token) => {
+                if let Some(expiry) = new_token.expires_at {
+                    let time_until_expiry = expiry.duration_since(Instant::now().into_std());
+
+                    if time_until_expiry > FIVE_MINUTES {
+                        let time_to_sleep = Instant::now() + time_until_expiry - FIVE_MINUTES;
+                        sleep_until(time_to_sleep).await;
+                    } else if time_until_expiry > TEN_SECONDS {
+                        // If expiry is less than 5 mins, wait till last 10 seconds to refresh
+                        // This is to handle cases where MDS returns about to refresh tokens
+                        let time_to_sleep = Instant::now() + time_until_expiry - TEN_SECONDS;
+                        sleep_until(time_to_sleep).await;
+                    }
+                } else {
+                    // Handle case where expiry is not available
+                    // TODO 1553: Validate that all auth backends provide expiry and make expiry not optional.
+                    sleep(FIVE_MINUTES).await;
+                }
+            }
+            Err(_) => {
+                // The retry policy has been used already by the inner token provider.
+                // If it ended in an error, just quit the background task.
+                break;
+            }
+        }
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    use crate::errors::CredentialError;
+    use crate::token::test::MockTokenProvider;
+    use std::ops::{Add, Sub};
+    use std::sync::{Arc, Mutex};
+    use tokio::time::{Duration, Instant};
+
+    static TOKEN_VALID_DURATION: Duration = Duration::from_secs(3600);
+
+    #[tokio::test]
+    async fn initial_token_success() {
+        let expected = Token {
+            token: "test-token".to_string(),
+            token_type: "Bearer".to_string(),
+            expires_at: None,
+            metadata: None,
+        };
+        let expected_clone = expected.clone();
+
+        let mut mock = MockTokenProvider::new();
+        mock.expect_get_token()
+            .times(1)
+            .return_once(|| Ok(expected_clone));
+
+        let cache = TokenCache::new(mock);
+        let actual = cache.get_token().await.unwrap();
+        assert_eq!(actual, expected);
+
+        // Verify that we use the cached token instead of making a new request
+        // to the mock token provider.
+        let actual = cache.get_token().await.unwrap();
+        assert_eq!(actual, expected);
+    }
+
+    #[tokio::test]
+    async fn initial_token_failure() {
+        let mut mock = MockTokenProvider::new();
+        mock.expect_get_token()
+            .times(1)
+            .returning(|| Err(CredentialError::non_retryable_from_str("fail")));
+
+        let cache = TokenCache::new(mock);
+        assert!(cache.get_token().await.is_err());
+
+        // Verify that a new request is made to the mock token provider when we
+        // don't have a valid token.
+        assert!(cache.get_token().await.is_err());
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn expired_token_success() {
+        let now = Instant::now();
+
+        let initial = Token {
+            token: "initial-token".to_string(),
+            token_type: "Bearer".to_string(),
+            expires_at: Some((now + TOKEN_VALID_DURATION).into_std()),
+            metadata: None,
+        };
+        let initial_clone = initial.clone();
+
+        let refresh = Token {
+            token: "refreshed-token".to_string(),
+            token_type: "Bearer".to_string(),
+            expires_at: Some((now + 2 * TOKEN_VALID_DURATION).into_std()),
+            metadata: None,
+        };
+        let refresh_clone = refresh.clone();
+
+        let mut mock = MockTokenProvider::new();
+        mock.expect_get_token()
+            .times(1)
+            .return_once(|| Ok(initial_clone));
+
+        mock.expect_get_token()
+            .times(1)
+            .return_once(|| Ok(refresh_clone));
+
+        // fetch an initial token
+        let cache = TokenCache::new(mock);
+        let actual = cache.get_token().await.unwrap();
+        assert_eq!(actual, initial);
+
+        // wait long enough for the token to be expired
+        // get_token should be waiting until the new token is available
+        let sleep = TOKEN_VALID_DURATION.add(Duration::from_secs(100));
+        tokio::time::advance(sleep).await;
+
+        // make sure this is the new token
+        let actual = cache.get_token().await.unwrap();
+        assert_eq!(actual, refresh);
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn expired_token_failure() {
+        let now = Instant::now();
+
+        let initial = Token {
+            token: "initial-token".to_string(),
+            token_type: "Bearer".to_string(),
+            expires_at: Some((now + TOKEN_VALID_DURATION).into_std()),
+            metadata: None,
+        };
+        let initial_clone = initial.clone();
+
+        let mut mock = MockTokenProvider::new();
+        mock.expect_get_token()
+            .times(1)
+            .return_once(|| Ok(initial_clone));
+
+        mock.expect_get_token()
+            .times(1)
+            .return_once(|| Err(CredentialError::non_retryable_from_str("fail")));
+
+        // fetch an initial token
+        let cache = TokenCache::new(mock);
+        let actual = cache.get_token().await.unwrap();
+        assert_eq!(actual, initial);
+
+        // wait long enough for the token to be expired
+        let sleep = TOKEN_VALID_DURATION.add(Duration::from_secs(100));
+        tokio::time::advance(sleep).await;
+
+        // make sure we return the error, not the expired token
+        assert!(cache.get_token().await.is_err());
+    }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 8)]
+    async fn token_cache_multiple_requests_existing_valid_token() {
+        let now = Instant::now();
+
+        let token = Token {
+            token: "initial-token".to_string(),
+            token_type: "Bearer".to_string(),
+            expires_at: Some((now + TOKEN_VALID_DURATION).into_std()),
+            metadata: None,
+        };
+        let token_clone = token.clone();
+
+        let mut mock = MockTokenProvider::new();
+        mock.expect_get_token()
+            .times(1)
+            .return_once(|| Ok(token_clone));
+
+        // fetch an initial token
+        let cache = TokenCache::new(mock);
+        let actual = cache.get_token().await.unwrap();
+        assert_eq!(actual, token);
+
+        // Spawn N tasks, all asking for a token at once.
+        let tasks = (0..1000)
+            .map(|_| {
+                let cache_clone = cache.clone();
+                tokio::spawn(async move { cache_clone.get_token().await })
+            })
+            .collect::<Vec<_>>();
+
+        // Wait for the N token requests to complete, verifying the returned token.
+        for task in tasks {
+            let actual = task.await.unwrap();
+            assert!(actual.is_ok(), "{}", actual.err().unwrap());
+            assert_eq!(actual.unwrap(), token);
+        }
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn refresh_task_loop() {
+        let now = Instant::now();
+
+        let token1 = Token {
+            token: "token1".to_string(),
+            token_type: "Bearer".to_string(),
+            expires_at: Some((now + TOKEN_VALID_DURATION).into_std()),
+            metadata: None,
+        };
+        let token1_clone = token1.clone();
+
+        let token2 = Token {
+            token: "token2".to_string(),
+            token_type: "Bearer".to_string(),
+            expires_at: Some((now + 2 * TOKEN_VALID_DURATION).into_std()),
+            metadata: None,
+        };
+        let token2_clone = token2.clone();
+
+        let token3 = Token {
+            token: "token3".to_string(),
+            token_type: "Bearer".to_string(),
+            expires_at: Some((now + 3 * TOKEN_VALID_DURATION).into_std()),
+            metadata: None,
+        };
+        let token3_clone = token3.clone();
+
+        let mut mock = MockTokenProvider::new();
+        mock.expect_get_token()
+            .times(1)
+            .return_once(|| Ok(token1_clone));
+
+        mock.expect_get_token()
+            .times(1)
+            .return_once(|| Ok(token2_clone));
+
+        mock.expect_get_token()
+            .times(1)
+            .return_once(|| Ok(token3_clone));
+
+        let (tx, mut rx) = watch::channel::<Option<Result<Token>>>(None);
+
+        // check that channel has None before refresh task starts
+        let actual = rx.borrow().clone();
+        assert!(actual.is_none());
+
+        tokio::spawn(async move {
+            refresh_task(mock, tx).await;
+        });
+
+        rx.changed().await.unwrap();
+        let actual = rx.borrow().clone().unwrap().unwrap();
+        assert_eq!(actual, token1.clone());
+
+        // Validate that it is the same token before it expires
+        let sleep = Duration::from_secs(120);
+        tokio::time::advance(sleep).await;
+        let actual = rx.borrow().clone().unwrap().unwrap();
+        assert_eq!(actual, token1.clone());
+
+        // time machine takes execution to 3 minutes before expiry
+        tokio::time::advance(TOKEN_VALID_DURATION.sub(Duration::from_secs(300))).await;
+
+        rx.changed().await.unwrap();
+
+        // validate that the token changed less than 5 mins before expiry
+        assert!(Instant::now() < now + TOKEN_VALID_DURATION);
+        let actual = rx.borrow().clone().unwrap().unwrap();
+        assert_eq!(actual, token2);
+
+        // wait long enough for the token to be expired
+        // Adding 500 secs to account for the time manipulation above
+        let sleep = TOKEN_VALID_DURATION.add(Duration::from_secs(500));
+        tokio::time::advance(sleep).await;
+
+        rx.changed().await.unwrap();
+        let actual = rx.borrow().clone().unwrap().unwrap();
+        assert_eq!(actual, token3);
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn refresh_task_loop_less_expiry() {
+        let now = Instant::now();
+
+        let token1 = Token {
+            token: "token1".to_string(),
+            token_type: "Bearer".to_string(),
+            expires_at: Some((now + Duration::from_secs(120)).into_std()),
+            metadata: None,
+        };
+        let token1_clone = token1.clone();
+
+        let token2 = Token {
+            token: "token2".to_string(),
+            token_type: "Bearer".to_string(),
+            expires_at: Some((now + 2 * TOKEN_VALID_DURATION).into_std()),
+            metadata: None,
+        };
+        let token2_clone = token2.clone();
+
+        let mut mock = MockTokenProvider::new();
+        mock.expect_get_token()
+            .times(1)
+            .return_once(|| Ok(token1_clone));
+
+        mock.expect_get_token()
+            .times(1)
+            .return_once(|| Ok(token2_clone));
+
+        let (tx, mut rx) = watch::channel::<Option<Result<Token>>>(None);
+
+        // check that channel has None before refresh task starts
+        let actual = rx.borrow().clone();
+        assert!(actual.is_none());
+
+        tokio::spawn(async move {
+            refresh_task(mock, tx).await;
+        });
+
+        rx.changed().await.unwrap();
+        let actual = rx.borrow().clone().unwrap().unwrap();
+        assert_eq!(actual, token1);
+
+        // time machine forwards time to 10 seconds before expiry
+        tokio::time::advance(Duration::from_secs(110)).await;
+
+        rx.changed().await.unwrap();
+
+        // validate that the token was refreshed within 10ish seconds
+        // before expiry
+        assert!(Instant::now() < now + Duration::from_secs(111));
+        let actual = rx.borrow().clone().unwrap().unwrap();
+        assert_eq!(actual, token2);
+    }
+
+    #[derive(Clone, Debug)]
+    struct FakeTokenProvider {
+        result: Result<Token>,
+        calls: Arc<std::sync::Mutex<i32>>,
+    }
+
+    impl FakeTokenProvider {
+        pub fn new(result: Result<Token>) -> Self {
+            FakeTokenProvider {
+                result,
+                calls: Arc::new(Mutex::new(0)),
+            }
+        }
+
+        pub fn calls(&self) -> i32 {
+            *self.calls.lock().unwrap()
+        }
+    }
+
+    #[async_trait::async_trait]
+    impl TokenProvider for FakeTokenProvider {
+        async fn get_token(&self) -> Result<Token> {
+            // We give enough time for the a thundering herd to pile up waiting for a change notification from watch channel
+            tokio::time::sleep(Duration::from_millis(50)).await;
+
+            // Track how many calls were made to the inner token provider.
+            *self.calls.lock().unwrap() += 1;
+
+            // Return the result.
+            self.result.clone()
+        }
+    }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 8)]
+    async fn no_initial_token_thundering_herd_success() {
+        let token = Token {
+            token: "delayed-token".to_string(),
+            token_type: "Bearer".to_string(),
+            expires_at: Some(std::time::Instant::now()),
+            metadata: None,
+        };
+
+        let tp = FakeTokenProvider::new(Ok(token.clone()));
+
+        let cache = TokenCache::new(tp.clone());
+
+        // Spawn N tasks, all asking for a token at once.
+        let tasks = (0..100)
+            .map(|_| {
+                let cache_clone = cache.clone();
+                tokio::spawn(async move { cache_clone.get_token().await })
+            })
+            .collect::<Vec<_>>();
+
+        // Wait for the N token requests to complete, verifying the returned token.
+        for task in tasks {
+            let actual = task.await.unwrap();
+            assert!(actual.is_ok(), "{}", actual.err().unwrap());
+            assert_eq!(actual.unwrap(), token);
+        }
+
+        let calls = tp.calls();
+        // Only one call to token provider should have been made
+        assert_eq!(calls, 1);
path: src/auth/src/token_cache.rs
user: login: dbolduc
body: This assumes that the N tasks launch within the 50ms allotted. I would not rely on timing like that.

This is what I would do, but maybe you thought the expectation was comically loose? I thought it worked.

https://github.com/dbolduc/google-cloud-rust/blob/607315bca3477f8a478fe27e8ec1d8cc1faee6bd/src/auth/src/token_cache.rs#L465-L472
created_at: 2025-03-20T16:16:57Z
author_association: MEMBER
diff_hunk: @@ -0,0 +1,538 @@
+// Copyright 2024 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::token::{Token, TokenProvider};
+use crate::Result;
+use tokio::sync::watch;
+use tokio::time::{sleep, sleep_until, Duration, Instant};
+
+const FIVE_MINUTES: Duration = Duration::from_secs(300);
+const TEN_SECONDS: Duration = Duration::from_secs(10);
+
+#[derive(Debug)]
+pub(crate) struct TokenCache<T>
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    rx_token: watch::Receiver<Option<Result<Token>>>,
+    _phantom: std::marker::PhantomData<T>,
+}
+
+impl<T: TokenProvider> Clone for TokenCache<T> {
+    fn clone(&self) -> TokenCache<T> {
+        TokenCache {
+            rx_token: self.rx_token.clone(),
+            _phantom: std::marker::PhantomData,
+        }
+    }
+}
+
+// TODO 1552: Use the token cache in all creds
+#[allow(dead_code)]
+impl<T> TokenCache<T>
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    pub(crate) fn new(inner: T) -> Self {
+        let (tx_token, rx_token) = watch::channel::<Option<Result<Token>>>(None);
+
+        tokio::spawn(async move {
+            refresh_task(inner, tx_token).await;
+        });
+
+        Self {
+            rx_token,
+            _phantom: std::marker::PhantomData,
+        }
+    }
+}
+
+#[async_trait::async_trait]
+impl<T> TokenProvider for TokenCache<T>
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    async fn get_token(&self) -> Result<Token> {
+        let mut rx = self.rx_token.clone();
+        let token_result = rx.borrow_and_update().clone();
+
+        if let Some(token_result) = token_result {
+            match token_result {
+                Ok(token) => match token.expires_at {
+                    None => return Ok(token),
+                    Some(e) => {
+                        if e < Instant::now().into_std() {
+                            // Expired token, wait for refresh
+                            return get_token_from_channel(rx).await;
+                        } else {
+                            // valid token
+                            return Ok(token);
+                        }
+                    }
+                },
+                // An error in the result is still a valid result to propagate to the client library
+                Err(e) => Err(e),
+            }
+        } else {
+            get_token_from_channel(rx).await
+        }
+    }
+}
+
+async fn get_token_from_channel(
+    mut rx_token: watch::Receiver<Option<Result<Token>>>,
+) -> Result<Token> {
+    // We have already checked the current token in the channel, wait for new one
+    rx_token.changed().await.unwrap();
+    let token_result = rx_token.borrow().clone();
+
+    token_result.expect("There should always be a token or error in the channel after changed()")
+}
+
+async fn refresh_task<T>(token_provider: T, tx_token: watch::Sender<Option<Result<Token>>>)
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    loop {
+        let token_result = token_provider.get_token().await;
+
+        let _ = tx_token.send(Some(token_result.clone()));
+
+        match token_result {
+            Ok(new_token) => {
+                if let Some(expiry) = new_token.expires_at {
+                    let time_until_expiry = expiry.duration_since(Instant::now().into_std());
+
+                    if time_until_expiry > FIVE_MINUTES {
+                        let time_to_sleep = Instant::now() + time_until_expiry - FIVE_MINUTES;
+                        sleep_until(time_to_sleep).await;
+                    } else if time_until_expiry > TEN_SECONDS {
+                        // If expiry is less than 5 mins, wait till last 10 seconds to refresh
+                        // This is to handle cases where MDS returns about to refresh tokens
+                        let time_to_sleep = Instant::now() + time_until_expiry - TEN_SECONDS;
+                        sleep_until(time_to_sleep).await;
+                    }
path: src/auth/src/token_cache.rs
user: login: dbolduc
body: so if we get a token that expires in < 10 seconds, we immediately restart the loop and ask for another token. Makes sense.

Can you add a comment that makes this obvious?

```
} else {
  // We were given a token that is expired, or expires in less than 10 seconds. We will immediately restart the loop, and fetch a new token.
}
```
created_at: 2025-03-20T16:28:07Z
author_association: MEMBER
diff_hunk: @@ -0,0 +1,538 @@
+// Copyright 2024 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::token::{Token, TokenProvider};
+use crate::Result;
+use tokio::sync::watch;
+use tokio::time::{sleep, sleep_until, Duration, Instant};
+
+const FIVE_MINUTES: Duration = Duration::from_secs(300);
+const TEN_SECONDS: Duration = Duration::from_secs(10);
+
+#[derive(Debug)]
+pub(crate) struct TokenCache<T>
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    rx_token: watch::Receiver<Option<Result<Token>>>,
+    _phantom: std::marker::PhantomData<T>,
+}
+
+impl<T: TokenProvider> Clone for TokenCache<T> {
+    fn clone(&self) -> TokenCache<T> {
+        TokenCache {
+            rx_token: self.rx_token.clone(),
+            _phantom: std::marker::PhantomData,
+        }
+    }
+}
+
+// TODO 1552: Use the token cache in all creds
+#[allow(dead_code)]
+impl<T> TokenCache<T>
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    pub(crate) fn new(inner: T) -> Self {
+        let (tx_token, rx_token) = watch::channel::<Option<Result<Token>>>(None);
+
+        tokio::spawn(async move {
+            refresh_task(inner, tx_token).await;
+        });
+
+        Self {
+            rx_token,
+            _phantom: std::marker::PhantomData,
+        }
+    }
+}
+
+#[async_trait::async_trait]
+impl<T> TokenProvider for TokenCache<T>
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    async fn get_token(&self) -> Result<Token> {
+        let mut rx = self.rx_token.clone();
+        let token_result = rx.borrow_and_update().clone();
+
+        if let Some(token_result) = token_result {
+            match token_result {
+                Ok(token) => match token.expires_at {
+                    None => return Ok(token),
+                    Some(e) => {
+                        if e < Instant::now().into_std() {
+                            // Expired token, wait for refresh
+                            return get_token_from_channel(rx).await;
+                        } else {
+                            // valid token
+                            return Ok(token);
+                        }
+                    }
+                },
+                // An error in the result is still a valid result to propagate to the client library
+                Err(e) => Err(e),
+            }
+        } else {
+            get_token_from_channel(rx).await
+        }
+    }
+}
+
+async fn get_token_from_channel(
+    mut rx_token: watch::Receiver<Option<Result<Token>>>,
+) -> Result<Token> {
+    // We have already checked the current token in the channel, wait for new one
+    rx_token.changed().await.unwrap();
+    let token_result = rx_token.borrow().clone();
+
+    token_result.expect("There should always be a token or error in the channel after changed()")
+}
+
+async fn refresh_task<T>(token_provider: T, tx_token: watch::Sender<Option<Result<Token>>>)
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    loop {
+        let token_result = token_provider.get_token().await;
+
+        let _ = tx_token.send(Some(token_result.clone()));
+
+        match token_result {
+            Ok(new_token) => {
+                if let Some(expiry) = new_token.expires_at {
+                    let time_until_expiry = expiry.duration_since(Instant::now().into_std());
+
+                    if time_until_expiry > FIVE_MINUTES {
+                        let time_to_sleep = Instant::now() + time_until_expiry - FIVE_MINUTES;
+                        sleep_until(time_to_sleep).await;
+                    } else if time_until_expiry > TEN_SECONDS {
+                        // If expiry is less than 5 mins, wait till last 10 seconds to refresh
+                        // This is to handle cases where MDS returns about to refresh tokens
+                        let time_to_sleep = Instant::now() + time_until_expiry - TEN_SECONDS;
+                        sleep_until(time_to_sleep).await;
+                    }
path: src/auth/src/token_cache.rs
user: login: dbolduc
body: Observation: if a backend keeps giving us expired tokens, this background task will chew CPU.

Do we care?
created_at: 2025-03-20T16:28:49Z
author_association: MEMBER
diff_hunk: @@ -0,0 +1,538 @@
+// Copyright 2024 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::token::{Token, TokenProvider};
+use crate::Result;
+use tokio::sync::watch;
+use tokio::time::{sleep, sleep_until, Duration, Instant};
+
+const FIVE_MINUTES: Duration = Duration::from_secs(300);
+const TEN_SECONDS: Duration = Duration::from_secs(10);
+
+#[derive(Debug)]
+pub(crate) struct TokenCache<T>
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    rx_token: watch::Receiver<Option<Result<Token>>>,
+    _phantom: std::marker::PhantomData<T>,
+}
+
+impl<T: TokenProvider> Clone for TokenCache<T> {
+    fn clone(&self) -> TokenCache<T> {
+        TokenCache {
+            rx_token: self.rx_token.clone(),
+            _phantom: std::marker::PhantomData,
+        }
+    }
+}
+
+// TODO 1552: Use the token cache in all creds
+#[allow(dead_code)]
+impl<T> TokenCache<T>
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    pub(crate) fn new(inner: T) -> Self {
+        let (tx_token, rx_token) = watch::channel::<Option<Result<Token>>>(None);
+
+        tokio::spawn(async move {
+            refresh_task(inner, tx_token).await;
+        });
+
+        Self {
+            rx_token,
+            _phantom: std::marker::PhantomData,
+        }
+    }
+}
+
+#[async_trait::async_trait]
+impl<T> TokenProvider for TokenCache<T>
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    async fn get_token(&self) -> Result<Token> {
+        let mut rx = self.rx_token.clone();
+        let token_result = rx.borrow_and_update().clone();
+
+        if let Some(token_result) = token_result {
+            match token_result {
+                Ok(token) => match token.expires_at {
+                    None => return Ok(token),
+                    Some(e) => {
+                        if e < Instant::now().into_std() {
+                            // Expired token, wait for refresh
+                            return get_token_from_channel(rx).await;
+                        } else {
+                            // valid token
+                            return Ok(token);
+                        }
+                    }
+                },
+                // An error in the result is still a valid result to propagate to the client library
+                Err(e) => Err(e),
+            }
+        } else {
+            get_token_from_channel(rx).await
+        }
+    }
+}
+
+async fn get_token_from_channel(
+    mut rx_token: watch::Receiver<Option<Result<Token>>>,
+) -> Result<Token> {
+    // We have already checked the current token in the channel, wait for new one
+    rx_token.changed().await.unwrap();
+    let token_result = rx_token.borrow().clone();
+
+    token_result.expect("There should always be a token or error in the channel after changed()")
+}
+
+async fn refresh_task<T>(token_provider: T, tx_token: watch::Sender<Option<Result<Token>>>)
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    loop {
+        let token_result = token_provider.get_token().await;
+
+        let _ = tx_token.send(Some(token_result.clone()));
path: src/auth/src/token_cache.rs
user: login: dbolduc
body: Don't worry about it in this PR, but a future improvement to think about:

We might send a token to the channel with an expiry earlier than the last token sent to the channel. It would be nice to avoid this.
created_at: 2025-03-20T16:33:38Z
author_association: MEMBER
diff_hunk: @@ -0,0 +1,538 @@
+// Copyright 2024 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::token::{Token, TokenProvider};
+use crate::Result;
+use tokio::sync::watch;
+use tokio::time::{sleep, sleep_until, Duration, Instant};
+
+const FIVE_MINUTES: Duration = Duration::from_secs(300);
+const TEN_SECONDS: Duration = Duration::from_secs(10);
+
+#[derive(Debug)]
+pub(crate) struct TokenCache<T>
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    rx_token: watch::Receiver<Option<Result<Token>>>,
+    _phantom: std::marker::PhantomData<T>,
+}
+
+impl<T: TokenProvider> Clone for TokenCache<T> {
+    fn clone(&self) -> TokenCache<T> {
+        TokenCache {
+            rx_token: self.rx_token.clone(),
+            _phantom: std::marker::PhantomData,
+        }
+    }
+}
+
+// TODO 1552: Use the token cache in all creds
+#[allow(dead_code)]
+impl<T> TokenCache<T>
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    pub(crate) fn new(inner: T) -> Self {
+        let (tx_token, rx_token) = watch::channel::<Option<Result<Token>>>(None);
+
+        tokio::spawn(async move {
+            refresh_task(inner, tx_token).await;
path: src/auth/src/token_cache.rs
user: login: dbolduc
body: Observation: this starts fetching tokens at credential construction time. I think that is probably a good feature. The first token could be available without waiting.
created_at: 2025-03-20T16:48:37Z
author_association: MEMBER
diff_hunk: @@ -0,0 +1,538 @@
+// Copyright 2024 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::token::{Token, TokenProvider};
+use crate::Result;
+use tokio::sync::watch;
+use tokio::time::{sleep, sleep_until, Duration, Instant};
+
+const FIVE_MINUTES: Duration = Duration::from_secs(300);
+const TEN_SECONDS: Duration = Duration::from_secs(10);
+
+#[derive(Debug)]
+pub(crate) struct TokenCache<T>
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    rx_token: watch::Receiver<Option<Result<Token>>>,
+    _phantom: std::marker::PhantomData<T>,
+}
+
+impl<T: TokenProvider> Clone for TokenCache<T> {
+    fn clone(&self) -> TokenCache<T> {
+        TokenCache {
+            rx_token: self.rx_token.clone(),
+            _phantom: std::marker::PhantomData,
+        }
+    }
+}
+
+// TODO 1552: Use the token cache in all creds
+#[allow(dead_code)]
+impl<T> TokenCache<T>
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    pub(crate) fn new(inner: T) -> Self {
+        let (tx_token, rx_token) = watch::channel::<Option<Result<Token>>>(None);
+
+        tokio::spawn(async move {
+            refresh_task(inner, tx_token).await;
path: src/auth/src/token_cache.rs
user: login: sai-sunder-s
body: Yeah I am leaving it this way for now. In future if we get troubles from MDS startup delays, may have to reconsider. Or retry policy could handle the startup delay. Either way future potential problem to solve.
created_at: 2025-03-20T16:53:47Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,538 @@
+// Copyright 2024 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::token::{Token, TokenProvider};
+use crate::Result;
+use tokio::sync::watch;
+use tokio::time::{sleep, sleep_until, Duration, Instant};
+
+const FIVE_MINUTES: Duration = Duration::from_secs(300);
+const TEN_SECONDS: Duration = Duration::from_secs(10);
+
+#[derive(Debug)]
+pub(crate) struct TokenCache<T>
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    rx_token: watch::Receiver<Option<Result<Token>>>,
+    _phantom: std::marker::PhantomData<T>,
+}
+
+impl<T: TokenProvider> Clone for TokenCache<T> {
+    fn clone(&self) -> TokenCache<T> {
+        TokenCache {
+            rx_token: self.rx_token.clone(),
+            _phantom: std::marker::PhantomData,
+        }
+    }
+}
+
+// TODO 1552: Use the token cache in all creds
+#[allow(dead_code)]
+impl<T> TokenCache<T>
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    pub(crate) fn new(inner: T) -> Self {
+        let (tx_token, rx_token) = watch::channel::<Option<Result<Token>>>(None);
+
+        tokio::spawn(async move {
+            refresh_task(inner, tx_token).await;
+        });
+
+        Self {
+            rx_token,
+            _phantom: std::marker::PhantomData,
+        }
+    }
+}
+
+#[async_trait::async_trait]
+impl<T> TokenProvider for TokenCache<T>
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    async fn get_token(&self) -> Result<Token> {
+        let mut rx = self.rx_token.clone();
+        let token_result = rx.borrow_and_update().clone();
+
+        if let Some(token_result) = token_result {
+            match token_result {
+                Ok(token) => match token.expires_at {
+                    None => return Ok(token),
+                    Some(e) => {
+                        if e < Instant::now().into_std() {
+                            // Expired token, wait for refresh
+                            return get_token_from_channel(rx).await;
+                        } else {
+                            // valid token
+                            return Ok(token);
+                        }
+                    }
+                },
+                // An error in the result is still a valid result to propagate to the client library
+                Err(e) => Err(e),
+            }
+        } else {
+            get_token_from_channel(rx).await
+        }
+    }
+}
+
+async fn get_token_from_channel(
+    mut rx_token: watch::Receiver<Option<Result<Token>>>,
+) -> Result<Token> {
+    // We have already checked the current token in the channel, wait for new one
+    rx_token.changed().await.unwrap();
+    let token_result = rx_token.borrow().clone();
+
+    token_result.expect("There should always be a token or error in the channel after changed()")
+}
+
+async fn refresh_task<T>(token_provider: T, tx_token: watch::Sender<Option<Result<Token>>>)
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    loop {
+        let token_result = token_provider.get_token().await;
+
+        let _ = tx_token.send(Some(token_result.clone()));
+
+        match token_result {
+            Ok(new_token) => {
+                if let Some(expiry) = new_token.expires_at {
+                    let time_until_expiry = expiry.duration_since(Instant::now().into_std());
+
+                    if time_until_expiry > FIVE_MINUTES {
+                        let time_to_sleep = Instant::now() + time_until_expiry - FIVE_MINUTES;
+                        sleep_until(time_to_sleep).await;
+                    } else if time_until_expiry > TEN_SECONDS {
+                        // If expiry is less than 5 mins, wait till last 10 seconds to refresh
+                        // This is to handle cases where MDS returns about to refresh tokens
+                        let time_to_sleep = Instant::now() + time_until_expiry - TEN_SECONDS;
+                        sleep_until(time_to_sleep).await;
+                    }
+                } else {
+                    // Handle case where expiry is not available
+                    // TODO 1553: Validate that all auth backends provide expiry and make expiry not optional.
+                    sleep(FIVE_MINUTES).await;
+                }
+            }
+            Err(_) => {
+                // The retry policy has been used already by the inner token provider.
path: src/auth/src/token_cache.rs
user: login: dbolduc
body: We talked offline. The retry loop will be in the inner token provider, so the comment is accurate.

Resolving this...
created_at: 2025-03-20T18:25:51Z
author_association: MEMBER
diff_hunk: @@ -0,0 +1,538 @@
+// Copyright 2024 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::token::{Token, TokenProvider};
+use crate::Result;
+use tokio::sync::watch;
+use tokio::time::{sleep, sleep_until, Duration, Instant};
+
+const FIVE_MINUTES: Duration = Duration::from_secs(300);
+const TEN_SECONDS: Duration = Duration::from_secs(10);
+
+#[derive(Debug)]
+pub(crate) struct TokenCache<T>
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    rx_token: watch::Receiver<Option<Result<Token>>>,
+    _phantom: std::marker::PhantomData<T>,
+}
+
+impl<T: TokenProvider> Clone for TokenCache<T> {
+    fn clone(&self) -> TokenCache<T> {
+        TokenCache {
+            rx_token: self.rx_token.clone(),
+            _phantom: std::marker::PhantomData,
+        }
+    }
+}
+
+// TODO 1552: Use the token cache in all creds
+#[allow(dead_code)]
+impl<T> TokenCache<T>
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    pub(crate) fn new(inner: T) -> Self {
+        let (tx_token, rx_token) = watch::channel::<Option<Result<Token>>>(None);
+
+        tokio::spawn(async move {
+            refresh_task(inner, tx_token).await;
+        });
+
+        Self {
+            rx_token,
+            _phantom: std::marker::PhantomData,
+        }
+    }
+}
+
+#[async_trait::async_trait]
+impl<T> TokenProvider for TokenCache<T>
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    async fn get_token(&self) -> Result<Token> {
+        let mut rx = self.rx_token.clone();
+        let token_result = rx.borrow_and_update().clone();
+
+        if let Some(token_result) = token_result {
+            match token_result {
+                Ok(token) => match token.expires_at {
+                    None => return Ok(token),
+                    Some(e) => {
+                        if e < Instant::now().into_std() {
+                            // Expired token, wait for refresh
+                            return get_token_from_channel(rx).await;
+                        } else {
+                            // valid token
+                            return Ok(token);
+                        }
+                    }
+                },
+                // An error in the result is still a valid result to propagate to the client library
+                Err(e) => Err(e),
+            }
+        } else {
+            get_token_from_channel(rx).await
+        }
+    }
+}
+
+async fn get_token_from_channel(
+    mut rx_token: watch::Receiver<Option<Result<Token>>>,
+) -> Result<Token> {
+    // We have already checked the current token in the channel, wait for new one
+    rx_token.changed().await.unwrap();
+    let token_result = rx_token.borrow().clone();
+
+    token_result.expect("There should always be a token or error in the channel after changed()")
+}
+
+async fn refresh_task<T>(token_provider: T, tx_token: watch::Sender<Option<Result<Token>>>)
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    loop {
+        let token_result = token_provider.get_token().await;
path: src/auth/src/token_cache.rs
user: login: dbolduc
body: The retry loop will be inside the `token_provider.get_token()` call.

Resolving...
created_at: 2025-03-20T18:26:59Z
author_association: MEMBER
diff_hunk: @@ -0,0 +1,538 @@
+// Copyright 2024 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::token::{Token, TokenProvider};
+use crate::Result;
+use tokio::sync::watch;
+use tokio::time::{sleep, sleep_until, Duration, Instant};
+
+const FIVE_MINUTES: Duration = Duration::from_secs(300);
+const TEN_SECONDS: Duration = Duration::from_secs(10);
+
+#[derive(Debug)]
+pub(crate) struct TokenCache<T>
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    rx_token: watch::Receiver<Option<Result<Token>>>,
+    _phantom: std::marker::PhantomData<T>,
+}
+
+impl<T: TokenProvider> Clone for TokenCache<T> {
+    fn clone(&self) -> TokenCache<T> {
+        TokenCache {
+            rx_token: self.rx_token.clone(),
+            _phantom: std::marker::PhantomData,
+        }
+    }
+}
+
+// TODO 1552: Use the token cache in all creds
+#[allow(dead_code)]
+impl<T> TokenCache<T>
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    pub(crate) fn new(inner: T) -> Self {
+        let (tx_token, rx_token) = watch::channel::<Option<Result<Token>>>(None);
+
+        tokio::spawn(async move {
+            refresh_task(inner, tx_token).await;
+        });
+
+        Self {
+            rx_token,
+            _phantom: std::marker::PhantomData,
+        }
+    }
+}
+
+#[async_trait::async_trait]
+impl<T> TokenProvider for TokenCache<T>
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    async fn get_token(&self) -> Result<Token> {
+        let mut rx = self.rx_token.clone();
+        let token_result = rx.borrow_and_update().clone();
+
+        if let Some(token_result) = token_result {
+            match token_result {
+                Ok(token) => match token.expires_at {
+                    None => return Ok(token),
+                    Some(e) => {
+                        if e < Instant::now().into_std() {
+                            // Expired token, wait for refresh
+                            return get_token_from_channel(rx).await;
+                        } else {
+                            // valid token
+                            return Ok(token);
+                        }
+                    }
+                },
+                // An error in the result is still a valid result to propagate to the client library
+                Err(e) => Err(e),
+            }
+        } else {
+            get_token_from_channel(rx).await
+        }
+    }
+}
+
+async fn get_token_from_channel(
+    mut rx_token: watch::Receiver<Option<Result<Token>>>,
+) -> Result<Token> {
+    // We have already checked the current token in the channel, wait for new one
+    rx_token.changed().await.unwrap();
+    let token_result = rx_token.borrow().clone();
+
+    token_result.expect("There should always be a token or error in the channel after changed()")
+}
+
+async fn refresh_task<T>(token_provider: T, tx_token: watch::Sender<Option<Result<Token>>>)
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    loop {
+        let token_result = token_provider.get_token().await;
+
+        let _ = tx_token.send(Some(token_result.clone()));
+
+        match token_result {
+            Ok(new_token) => {
+                if let Some(expiry) = new_token.expires_at {
+                    let time_until_expiry = expiry.duration_since(Instant::now().into_std());
+
+                    if time_until_expiry > FIVE_MINUTES {
+                        let time_to_sleep = Instant::now() + time_until_expiry - FIVE_MINUTES;
+                        sleep_until(time_to_sleep).await;
+                    } else if time_until_expiry > TEN_SECONDS {
+                        // If expiry is less than 5 mins, wait till last 10 seconds to refresh
+                        // This is to handle cases where MDS returns about to refresh tokens
+                        let time_to_sleep = Instant::now() + time_until_expiry - TEN_SECONDS;
+                        sleep_until(time_to_sleep).await;
+                    }
+                } else {
+                    // Handle case where expiry is not available
+                    // TODO 1553: Validate that all auth backends provide expiry and make expiry not optional.
+                    sleep(FIVE_MINUTES).await;
+                }
+            }
+            Err(_) => {
+                // The retry policy has been used already by the inner token provider.
+                // If it ended in an error, just quit the background task.
path: src/auth/src/token_cache.rs
user: login: dbolduc
body: We decided this is fine. We can avoid the case with reasonable defaults.

We will clearly document the expected behavior of credential refreshing so the auth library is predictable.

Resolving...
created_at: 2025-03-20T18:28:45Z
author_association: MEMBER
diff_hunk: @@ -0,0 +1,538 @@
+// Copyright 2024 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::token::{Token, TokenProvider};
+use crate::Result;
+use tokio::sync::watch;
+use tokio::time::{sleep, sleep_until, Duration, Instant};
+
+const FIVE_MINUTES: Duration = Duration::from_secs(300);
+const TEN_SECONDS: Duration = Duration::from_secs(10);
+
+#[derive(Debug)]
+pub(crate) struct TokenCache<T>
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    rx_token: watch::Receiver<Option<Result<Token>>>,
+    _phantom: std::marker::PhantomData<T>,
+}
+
+impl<T: TokenProvider> Clone for TokenCache<T> {
+    fn clone(&self) -> TokenCache<T> {
+        TokenCache {
+            rx_token: self.rx_token.clone(),
+            _phantom: std::marker::PhantomData,
+        }
+    }
+}
+
+// TODO 1552: Use the token cache in all creds
+#[allow(dead_code)]
+impl<T> TokenCache<T>
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    pub(crate) fn new(inner: T) -> Self {
+        let (tx_token, rx_token) = watch::channel::<Option<Result<Token>>>(None);
+
+        tokio::spawn(async move {
+            refresh_task(inner, tx_token).await;
+        });
+
+        Self {
+            rx_token,
+            _phantom: std::marker::PhantomData,
+        }
+    }
+}
+
+#[async_trait::async_trait]
+impl<T> TokenProvider for TokenCache<T>
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    async fn get_token(&self) -> Result<Token> {
+        let mut rx = self.rx_token.clone();
+        let token_result = rx.borrow_and_update().clone();
+
+        if let Some(token_result) = token_result {
+            match token_result {
+                Ok(token) => match token.expires_at {
+                    None => return Ok(token),
+                    Some(e) => {
+                        if e < Instant::now().into_std() {
+                            // Expired token, wait for refresh
+                            return get_token_from_channel(rx).await;
+                        } else {
+                            // valid token
+                            return Ok(token);
+                        }
+                    }
+                },
+                // An error in the result is still a valid result to propagate to the client library
+                Err(e) => Err(e),
+            }
+        } else {
+            get_token_from_channel(rx).await
+        }
+    }
+}
+
+async fn get_token_from_channel(
+    mut rx_token: watch::Receiver<Option<Result<Token>>>,
+) -> Result<Token> {
+    // We have already checked the current token in the channel, wait for new one
+    rx_token.changed().await.unwrap();
+    let token_result = rx_token.borrow().clone();
+
+    token_result.expect("There should always be a token or error in the channel after changed()")
+}
+
+async fn refresh_task<T>(token_provider: T, tx_token: watch::Sender<Option<Result<Token>>>)
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    loop {
+        let token_result = token_provider.get_token().await;
+
+        let _ = tx_token.send(Some(token_result.clone()));
+
+        match token_result {
+            Ok(new_token) => {
+                if let Some(expiry) = new_token.expires_at {
+                    let time_until_expiry = expiry.duration_since(Instant::now().into_std());
+
+                    if time_until_expiry > FIVE_MINUTES {
+                        let time_to_sleep = Instant::now() + time_until_expiry - FIVE_MINUTES;
+                        sleep_until(time_to_sleep).await;
+                    } else if time_until_expiry > TEN_SECONDS {
+                        // If expiry is less than 5 mins, wait till last 10 seconds to refresh
+                        // This is to handle cases where MDS returns about to refresh tokens
+                        let time_to_sleep = Instant::now() + time_until_expiry - TEN_SECONDS;
+                        sleep_until(time_to_sleep).await;
+                    }
+                } else {
+                    // Handle case where expiry is not available
+                    // TODO 1553: Validate that all auth backends provide expiry and make expiry not optional.
+                    sleep(FIVE_MINUTES).await;
path: src/auth/src/token_cache.rs
user: login: sai-sunder-s
body: Good point. I missed that API key also uses same Token struct.
created_at: 2025-03-20T18:29:07Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,538 @@
+// Copyright 2024 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::token::{Token, TokenProvider};
+use crate::Result;
+use tokio::sync::watch;
+use tokio::time::{sleep, sleep_until, Duration, Instant};
+
+const FIVE_MINUTES: Duration = Duration::from_secs(300);
+const TEN_SECONDS: Duration = Duration::from_secs(10);
+
+#[derive(Debug)]
+pub(crate) struct TokenCache<T>
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    rx_token: watch::Receiver<Option<Result<Token>>>,
+    _phantom: std::marker::PhantomData<T>,
+}
+
+impl<T: TokenProvider> Clone for TokenCache<T> {
+    fn clone(&self) -> TokenCache<T> {
+        TokenCache {
+            rx_token: self.rx_token.clone(),
+            _phantom: std::marker::PhantomData,
+        }
+    }
+}
+
+// TODO 1552: Use the token cache in all creds
+#[allow(dead_code)]
+impl<T> TokenCache<T>
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    pub(crate) fn new(inner: T) -> Self {
+        let (tx_token, rx_token) = watch::channel::<Option<Result<Token>>>(None);
+
+        tokio::spawn(async move {
+            refresh_task(inner, tx_token).await;
+        });
+
+        Self {
+            rx_token,
+            _phantom: std::marker::PhantomData,
+        }
+    }
+}
+
+#[async_trait::async_trait]
+impl<T> TokenProvider for TokenCache<T>
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    async fn get_token(&self) -> Result<Token> {
+        let mut rx = self.rx_token.clone();
+        let token_result = rx.borrow_and_update().clone();
+
+        if let Some(token_result) = token_result {
+            match token_result {
+                Ok(token) => match token.expires_at {
+                    None => return Ok(token),
+                    Some(e) => {
+                        if e < Instant::now().into_std() {
+                            // Expired token, wait for refresh
+                            return get_token_from_channel(rx).await;
+                        } else {
+                            // valid token
+                            return Ok(token);
+                        }
+                    }
+                },
+                // An error in the result is still a valid result to propagate to the client library
+                Err(e) => Err(e),
+            }
+        } else {
+            get_token_from_channel(rx).await
+        }
+    }
+}
+
+async fn get_token_from_channel(
+    mut rx_token: watch::Receiver<Option<Result<Token>>>,
+) -> Result<Token> {
+    // We have already checked the current token in the channel, wait for new one
+    rx_token.changed().await.unwrap();
+    let token_result = rx_token.borrow().clone();
+
+    token_result.expect("There should always be a token or error in the channel after changed()")
+}
+
+async fn refresh_task<T>(token_provider: T, tx_token: watch::Sender<Option<Result<Token>>>)
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    loop {
+        let token_result = token_provider.get_token().await;
+
+        let _ = tx_token.send(Some(token_result.clone()));
+
+        match token_result {
+            Ok(new_token) => {
+                if let Some(expiry) = new_token.expires_at {
+                    let time_until_expiry = expiry.duration_since(Instant::now().into_std());
+
+                    if time_until_expiry > FIVE_MINUTES {
+                        let time_to_sleep = Instant::now() + time_until_expiry - FIVE_MINUTES;
+                        sleep_until(time_to_sleep).await;
+                    } else if time_until_expiry > TEN_SECONDS {
+                        // If expiry is less than 5 mins, wait till last 10 seconds to refresh
+                        // This is to handle cases where MDS returns about to refresh tokens
+                        let time_to_sleep = Instant::now() + time_until_expiry - TEN_SECONDS;
+                        sleep_until(time_to_sleep).await;
+                    }
+                } else {
+                    // Handle case where expiry is not available
+                    // TODO 1553: Validate that all auth backends provide expiry and make expiry not optional.
+                    sleep(FIVE_MINUTES).await;
path: src/auth/src/token_cache.rs
user: login: sai-sunder-s
body: Maybe it should not?
created_at: 2025-03-20T18:29:57Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,538 @@
+// Copyright 2024 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::token::{Token, TokenProvider};
+use crate::Result;
+use tokio::sync::watch;
+use tokio::time::{sleep, sleep_until, Duration, Instant};
+
+const FIVE_MINUTES: Duration = Duration::from_secs(300);
+const TEN_SECONDS: Duration = Duration::from_secs(10);
+
+#[derive(Debug)]
+pub(crate) struct TokenCache<T>
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    rx_token: watch::Receiver<Option<Result<Token>>>,
+    _phantom: std::marker::PhantomData<T>,
+}
+
+impl<T: TokenProvider> Clone for TokenCache<T> {
+    fn clone(&self) -> TokenCache<T> {
+        TokenCache {
+            rx_token: self.rx_token.clone(),
+            _phantom: std::marker::PhantomData,
+        }
+    }
+}
+
+// TODO 1552: Use the token cache in all creds
+#[allow(dead_code)]
+impl<T> TokenCache<T>
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    pub(crate) fn new(inner: T) -> Self {
+        let (tx_token, rx_token) = watch::channel::<Option<Result<Token>>>(None);
+
+        tokio::spawn(async move {
+            refresh_task(inner, tx_token).await;
+        });
+
+        Self {
+            rx_token,
+            _phantom: std::marker::PhantomData,
+        }
+    }
+}
+
+#[async_trait::async_trait]
+impl<T> TokenProvider for TokenCache<T>
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    async fn get_token(&self) -> Result<Token> {
+        let mut rx = self.rx_token.clone();
+        let token_result = rx.borrow_and_update().clone();
+
+        if let Some(token_result) = token_result {
+            match token_result {
+                Ok(token) => match token.expires_at {
+                    None => return Ok(token),
+                    Some(e) => {
+                        if e < Instant::now().into_std() {
+                            // Expired token, wait for refresh
+                            return get_token_from_channel(rx).await;
+                        } else {
+                            // valid token
+                            return Ok(token);
+                        }
+                    }
+                },
+                // An error in the result is still a valid result to propagate to the client library
+                Err(e) => Err(e),
+            }
+        } else {
+            get_token_from_channel(rx).await
+        }
+    }
+}
+
+async fn get_token_from_channel(
+    mut rx_token: watch::Receiver<Option<Result<Token>>>,
+) -> Result<Token> {
+    // We have already checked the current token in the channel, wait for new one
+    rx_token.changed().await.unwrap();
+    let token_result = rx_token.borrow().clone();
+
+    token_result.expect("There should always be a token or error in the channel after changed()")
+}
+
+async fn refresh_task<T>(token_provider: T, tx_token: watch::Sender<Option<Result<Token>>>)
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    loop {
+        let token_result = token_provider.get_token().await;
+
+        let _ = tx_token.send(Some(token_result.clone()));
+
+        match token_result {
+            Ok(new_token) => {
+                if let Some(expiry) = new_token.expires_at {
+                    let time_until_expiry = expiry.duration_since(Instant::now().into_std());
+
+                    if time_until_expiry > FIVE_MINUTES {
+                        let time_to_sleep = Instant::now() + time_until_expiry - FIVE_MINUTES;
+                        sleep_until(time_to_sleep).await;
+                    } else if time_until_expiry > TEN_SECONDS {
+                        // If expiry is less than 5 mins, wait till last 10 seconds to refresh
+                        // This is to handle cases where MDS returns about to refresh tokens
+                        let time_to_sleep = Instant::now() + time_until_expiry - TEN_SECONDS;
+                        sleep_until(time_to_sleep).await;
+                    }
+                } else {
+                    // Handle case where expiry is not available
+                    // TODO 1553: Validate that all auth backends provide expiry and make expiry not optional.
+                    sleep(FIVE_MINUTES).await;
+                }
+            }
+            Err(_) => {
+                // The retry policy has been used already by the inner token provider.
+                // If it ended in an error, just quit the background task.
path: src/auth/src/token_cache.rs
user: login: sai-sunder-s
body: Synced offline. We will quit if retries are done.
created_at: 2025-03-20T18:31:09Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,538 @@
+// Copyright 2024 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::token::{Token, TokenProvider};
+use crate::Result;
+use tokio::sync::watch;
+use tokio::time::{sleep, sleep_until, Duration, Instant};
+
+const FIVE_MINUTES: Duration = Duration::from_secs(300);
+const TEN_SECONDS: Duration = Duration::from_secs(10);
+
+#[derive(Debug)]
+pub(crate) struct TokenCache<T>
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    rx_token: watch::Receiver<Option<Result<Token>>>,
+    _phantom: std::marker::PhantomData<T>,
+}
+
+impl<T: TokenProvider> Clone for TokenCache<T> {
+    fn clone(&self) -> TokenCache<T> {
+        TokenCache {
+            rx_token: self.rx_token.clone(),
+            _phantom: std::marker::PhantomData,
+        }
+    }
+}
+
+// TODO 1552: Use the token cache in all creds
+#[allow(dead_code)]
+impl<T> TokenCache<T>
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    pub(crate) fn new(inner: T) -> Self {
+        let (tx_token, rx_token) = watch::channel::<Option<Result<Token>>>(None);
+
+        tokio::spawn(async move {
+            refresh_task(inner, tx_token).await;
+        });
+
+        Self {
+            rx_token,
+            _phantom: std::marker::PhantomData,
+        }
+    }
+}
+
+#[async_trait::async_trait]
+impl<T> TokenProvider for TokenCache<T>
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    async fn get_token(&self) -> Result<Token> {
+        let mut rx = self.rx_token.clone();
+        let token_result = rx.borrow_and_update().clone();
+
+        if let Some(token_result) = token_result {
+            match token_result {
+                Ok(token) => match token.expires_at {
+                    None => return Ok(token),
+                    Some(e) => {
+                        if e < Instant::now().into_std() {
+                            // Expired token, wait for refresh
+                            return get_token_from_channel(rx).await;
+                        } else {
+                            // valid token
+                            return Ok(token);
+                        }
+                    }
+                },
+                // An error in the result is still a valid result to propagate to the client library
+                Err(e) => Err(e),
+            }
+        } else {
+            get_token_from_channel(rx).await
+        }
+    }
+}
+
+async fn get_token_from_channel(
+    mut rx_token: watch::Receiver<Option<Result<Token>>>,
+) -> Result<Token> {
+    // We have already checked the current token in the channel, wait for new one
+    rx_token.changed().await.unwrap();
+    let token_result = rx_token.borrow().clone();
+
+    token_result.expect("There should always be a token or error in the channel after changed()")
+}
+
+async fn refresh_task<T>(token_provider: T, tx_token: watch::Sender<Option<Result<Token>>>)
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    loop {
+        let token_result = token_provider.get_token().await;
+
+        let _ = tx_token.send(Some(token_result.clone()));
+
+        match token_result {
+            Ok(new_token) => {
+                if let Some(expiry) = new_token.expires_at {
+                    let time_until_expiry = expiry.duration_since(Instant::now().into_std());
+
+                    if time_until_expiry > FIVE_MINUTES {
+                        let time_to_sleep = Instant::now() + time_until_expiry - FIVE_MINUTES;
+                        sleep_until(time_to_sleep).await;
+                    } else if time_until_expiry > TEN_SECONDS {
+                        // If expiry is less than 5 mins, wait till last 10 seconds to refresh
+                        // This is to handle cases where MDS returns about to refresh tokens
+                        let time_to_sleep = Instant::now() + time_until_expiry - TEN_SECONDS;
+                        sleep_until(time_to_sleep).await;
+                    }
+                } else {
+                    // Handle case where expiry is not available
+                    // TODO 1553: Validate that all auth backends provide expiry and make expiry not optional.
+                    sleep(FIVE_MINUTES).await;
+                }
+            }
+            Err(_) => {
+                // The retry policy has been used already by the inner token provider.
+                // If it ended in an error, just quit the background task.
+                break;
+            }
+        }
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    use crate::errors::CredentialError;
+    use crate::token::test::MockTokenProvider;
+    use std::ops::{Add, Sub};
+    use std::sync::{Arc, Mutex};
+    use tokio::time::{Duration, Instant};
+
+    static TOKEN_VALID_DURATION: Duration = Duration::from_secs(3600);
+
+    #[tokio::test]
+    async fn initial_token_success() {
+        let expected = Token {
+            token: "test-token".to_string(),
+            token_type: "Bearer".to_string(),
+            expires_at: None,
+            metadata: None,
+        };
+        let expected_clone = expected.clone();
+
+        let mut mock = MockTokenProvider::new();
+        mock.expect_get_token()
+            .times(1)
+            .return_once(|| Ok(expected_clone));
+
+        let cache = TokenCache::new(mock);
+        let actual = cache.get_token().await.unwrap();
+        assert_eq!(actual, expected);
+
+        // Verify that we use the cached token instead of making a new request
+        // to the mock token provider.
+        let actual = cache.get_token().await.unwrap();
+        assert_eq!(actual, expected);
+    }
+
+    #[tokio::test]
+    async fn initial_token_failure() {
+        let mut mock = MockTokenProvider::new();
+        mock.expect_get_token()
+            .times(1)
+            .returning(|| Err(CredentialError::non_retryable_from_str("fail")));
+
+        let cache = TokenCache::new(mock);
+        assert!(cache.get_token().await.is_err());
+
+        // Verify that a new request is made to the mock token provider when we
+        // don't have a valid token.
+        assert!(cache.get_token().await.is_err());
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn expired_token_success() {
+        let now = Instant::now();
+
+        let initial = Token {
+            token: "initial-token".to_string(),
+            token_type: "Bearer".to_string(),
+            expires_at: Some((now + TOKEN_VALID_DURATION).into_std()),
+            metadata: None,
+        };
+        let initial_clone = initial.clone();
+
+        let refresh = Token {
+            token: "refreshed-token".to_string(),
+            token_type: "Bearer".to_string(),
+            expires_at: Some((now + 2 * TOKEN_VALID_DURATION).into_std()),
+            metadata: None,
+        };
+        let refresh_clone = refresh.clone();
+
+        let mut mock = MockTokenProvider::new();
+        mock.expect_get_token()
+            .times(1)
+            .return_once(|| Ok(initial_clone));
+
+        mock.expect_get_token()
+            .times(1)
+            .return_once(|| Ok(refresh_clone));
+
+        // fetch an initial token
+        let cache = TokenCache::new(mock);
+        let actual = cache.get_token().await.unwrap();
+        assert_eq!(actual, initial);
+
+        // wait long enough for the token to be expired
+        // get_token should be waiting until the new token is available
+        let sleep = TOKEN_VALID_DURATION.add(Duration::from_secs(100));
+        tokio::time::advance(sleep).await;
+
+        // make sure this is the new token
+        let actual = cache.get_token().await.unwrap();
+        assert_eq!(actual, refresh);
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn expired_token_failure() {
+        let now = Instant::now();
+
+        let initial = Token {
+            token: "initial-token".to_string(),
+            token_type: "Bearer".to_string(),
+            expires_at: Some((now + TOKEN_VALID_DURATION).into_std()),
+            metadata: None,
+        };
+        let initial_clone = initial.clone();
+
+        let mut mock = MockTokenProvider::new();
+        mock.expect_get_token()
+            .times(1)
+            .return_once(|| Ok(initial_clone));
+
+        mock.expect_get_token()
+            .times(1)
+            .return_once(|| Err(CredentialError::non_retryable_from_str("fail")));
+
+        // fetch an initial token
+        let cache = TokenCache::new(mock);
+        let actual = cache.get_token().await.unwrap();
+        assert_eq!(actual, initial);
+
+        // wait long enough for the token to be expired
+        let sleep = TOKEN_VALID_DURATION.add(Duration::from_secs(100));
+        tokio::time::advance(sleep).await;
+
+        // make sure we return the error, not the expired token
+        assert!(cache.get_token().await.is_err());
+    }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 8)]
+    async fn token_cache_multiple_requests_existing_valid_token() {
+        let now = Instant::now();
+
+        let token = Token {
+            token: "initial-token".to_string(),
+            token_type: "Bearer".to_string(),
+            expires_at: Some((now + TOKEN_VALID_DURATION).into_std()),
+            metadata: None,
+        };
+        let token_clone = token.clone();
+
+        let mut mock = MockTokenProvider::new();
+        mock.expect_get_token()
+            .times(1)
+            .return_once(|| Ok(token_clone));
+
+        // fetch an initial token
+        let cache = TokenCache::new(mock);
+        let actual = cache.get_token().await.unwrap();
+        assert_eq!(actual, token);
+
+        // Spawn N tasks, all asking for a token at once.
+        let tasks = (0..1000)
+            .map(|_| {
+                let cache_clone = cache.clone();
+                tokio::spawn(async move { cache_clone.get_token().await })
+            })
+            .collect::<Vec<_>>();
+
+        // Wait for the N token requests to complete, verifying the returned token.
+        for task in tasks {
+            let actual = task.await.unwrap();
+            assert!(actual.is_ok(), "{}", actual.err().unwrap());
+            assert_eq!(actual.unwrap(), token);
+        }
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn refresh_task_loop() {
+        let now = Instant::now();
+
+        let token1 = Token {
+            token: "token1".to_string(),
+            token_type: "Bearer".to_string(),
+            expires_at: Some((now + TOKEN_VALID_DURATION).into_std()),
+            metadata: None,
+        };
+        let token1_clone = token1.clone();
+
+        let token2 = Token {
+            token: "token2".to_string(),
+            token_type: "Bearer".to_string(),
+            expires_at: Some((now + 2 * TOKEN_VALID_DURATION).into_std()),
+            metadata: None,
+        };
+        let token2_clone = token2.clone();
+
+        let token3 = Token {
+            token: "token3".to_string(),
+            token_type: "Bearer".to_string(),
+            expires_at: Some((now + 3 * TOKEN_VALID_DURATION).into_std()),
+            metadata: None,
+        };
+        let token3_clone = token3.clone();
+
+        let mut mock = MockTokenProvider::new();
+        mock.expect_get_token()
+            .times(1)
+            .return_once(|| Ok(token1_clone));
+
+        mock.expect_get_token()
+            .times(1)
+            .return_once(|| Ok(token2_clone));
+
+        mock.expect_get_token()
+            .times(1)
+            .return_once(|| Ok(token3_clone));
+
+        let (tx, mut rx) = watch::channel::<Option<Result<Token>>>(None);
+
+        // check that channel has None before refresh task starts
+        let actual = rx.borrow().clone();
+        assert!(actual.is_none());
+
+        tokio::spawn(async move {
+            refresh_task(mock, tx).await;
+        });
+
+        rx.changed().await.unwrap();
+        let actual = rx.borrow().clone().unwrap().unwrap();
+        assert_eq!(actual, token1.clone());
+
+        // Validate that it is the same token before it expires
+        let sleep = Duration::from_secs(120);
+        tokio::time::advance(sleep).await;
+        let actual = rx.borrow().clone().unwrap().unwrap();
+        assert_eq!(actual, token1.clone());
+
+        // time machine takes execution to 3 minutes before expiry
+        tokio::time::advance(TOKEN_VALID_DURATION.sub(Duration::from_secs(300))).await;
+
+        rx.changed().await.unwrap();
+
+        // validate that the token changed less than 5 mins before expiry
+        assert!(Instant::now() < now + TOKEN_VALID_DURATION);
+        let actual = rx.borrow().clone().unwrap().unwrap();
+        assert_eq!(actual, token2);
+
+        // wait long enough for the token to be expired
+        // Adding 500 secs to account for the time manipulation above
+        let sleep = TOKEN_VALID_DURATION.add(Duration::from_secs(500));
+        tokio::time::advance(sleep).await;
+
+        rx.changed().await.unwrap();
+        let actual = rx.borrow().clone().unwrap().unwrap();
+        assert_eq!(actual, token3);
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn refresh_task_loop_less_expiry() {
+        let now = Instant::now();
+
+        let token1 = Token {
+            token: "token1".to_string(),
+            token_type: "Bearer".to_string(),
+            expires_at: Some((now + Duration::from_secs(120)).into_std()),
+            metadata: None,
+        };
+        let token1_clone = token1.clone();
+
+        let token2 = Token {
+            token: "token2".to_string(),
+            token_type: "Bearer".to_string(),
+            expires_at: Some((now + 2 * TOKEN_VALID_DURATION).into_std()),
+            metadata: None,
+        };
+        let token2_clone = token2.clone();
+
+        let mut mock = MockTokenProvider::new();
+        mock.expect_get_token()
+            .times(1)
+            .return_once(|| Ok(token1_clone));
+
+        mock.expect_get_token()
+            .times(1)
+            .return_once(|| Ok(token2_clone));
+
+        let (tx, mut rx) = watch::channel::<Option<Result<Token>>>(None);
+
+        // check that channel has None before refresh task starts
+        let actual = rx.borrow().clone();
+        assert!(actual.is_none());
+
+        tokio::spawn(async move {
+            refresh_task(mock, tx).await;
+        });
+
+        rx.changed().await.unwrap();
+        let actual = rx.borrow().clone().unwrap().unwrap();
+        assert_eq!(actual, token1);
+
+        // time machine forwards time to 10 seconds before expiry
+        tokio::time::advance(Duration::from_secs(110)).await;
+
+        rx.changed().await.unwrap();
+
+        // validate that the token was refreshed within 10ish seconds
+        // before expiry
+        assert!(Instant::now() < now + Duration::from_secs(111));
+        let actual = rx.borrow().clone().unwrap().unwrap();
+        assert_eq!(actual, token2);
+    }
+
+    #[derive(Clone, Debug)]
+    struct FakeTokenProvider {
+        result: Result<Token>,
+        calls: Arc<std::sync::Mutex<i32>>,
+    }
+
+    impl FakeTokenProvider {
+        pub fn new(result: Result<Token>) -> Self {
+            FakeTokenProvider {
+                result,
+                calls: Arc::new(Mutex::new(0)),
+            }
+        }
+
+        pub fn calls(&self) -> i32 {
+            *self.calls.lock().unwrap()
+        }
+    }
+
+    #[async_trait::async_trait]
+    impl TokenProvider for FakeTokenProvider {
+        async fn get_token(&self) -> Result<Token> {
+            // We give enough time for the a thundering herd to pile up waiting for a change notification from watch channel
+            tokio::time::sleep(Duration::from_millis(50)).await;
+
+            // Track how many calls were made to the inner token provider.
+            *self.calls.lock().unwrap() += 1;
+
+            // Return the result.
+            self.result.clone()
+        }
+    }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 8)]
+    async fn no_initial_token_thundering_herd_success() {
+        let token = Token {
+            token: "delayed-token".to_string(),
+            token_type: "Bearer".to_string(),
+            expires_at: Some(std::time::Instant::now()),
+            metadata: None,
+        };
+
+        let tp = FakeTokenProvider::new(Ok(token.clone()));
+
+        let cache = TokenCache::new(tp.clone());
+
+        // Spawn N tasks, all asking for a token at once.
+        let tasks = (0..100)
+            .map(|_| {
+                let cache_clone = cache.clone();
+                tokio::spawn(async move { cache_clone.get_token().await })
+            })
+            .collect::<Vec<_>>();
+
+        // Wait for the N token requests to complete, verifying the returned token.
+        for task in tasks {
+            let actual = task.await.unwrap();
+            assert!(actual.is_ok(), "{}", actual.err().unwrap());
+            assert_eq!(actual.unwrap(), token);
+        }
+
+        let calls = tp.calls();
+        // Only one call to token provider should have been made
+        assert_eq!(calls, 1);
path: src/auth/src/token_cache.rs
user: login: dbolduc
body: Nevermind. This implementation caches errors, whereas my implementation would make another request on errors. So I think what you have is better.

Resolving...
created_at: 2025-03-20T18:31:23Z
author_association: MEMBER
diff_hunk: @@ -0,0 +1,538 @@
+// Copyright 2024 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::token::{Token, TokenProvider};
+use crate::Result;
+use tokio::sync::watch;
+use tokio::time::{sleep, sleep_until, Duration, Instant};
+
+const FIVE_MINUTES: Duration = Duration::from_secs(300);
+const TEN_SECONDS: Duration = Duration::from_secs(10);
+
+#[derive(Debug)]
+pub(crate) struct TokenCache<T>
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    rx_token: watch::Receiver<Option<Result<Token>>>,
+    _phantom: std::marker::PhantomData<T>,
+}
+
+impl<T: TokenProvider> Clone for TokenCache<T> {
+    fn clone(&self) -> TokenCache<T> {
+        TokenCache {
+            rx_token: self.rx_token.clone(),
+            _phantom: std::marker::PhantomData,
+        }
+    }
+}
+
+// TODO 1552: Use the token cache in all creds
+#[allow(dead_code)]
+impl<T> TokenCache<T>
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    pub(crate) fn new(inner: T) -> Self {
+        let (tx_token, rx_token) = watch::channel::<Option<Result<Token>>>(None);
+
+        tokio::spawn(async move {
+            refresh_task(inner, tx_token).await;
+        });
+
+        Self {
+            rx_token,
+            _phantom: std::marker::PhantomData,
+        }
+    }
+}
+
+#[async_trait::async_trait]
+impl<T> TokenProvider for TokenCache<T>
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    async fn get_token(&self) -> Result<Token> {
+        let mut rx = self.rx_token.clone();
+        let token_result = rx.borrow_and_update().clone();
+
+        if let Some(token_result) = token_result {
+            match token_result {
+                Ok(token) => match token.expires_at {
+                    None => return Ok(token),
+                    Some(e) => {
+                        if e < Instant::now().into_std() {
+                            // Expired token, wait for refresh
+                            return get_token_from_channel(rx).await;
+                        } else {
+                            // valid token
+                            return Ok(token);
+                        }
+                    }
+                },
+                // An error in the result is still a valid result to propagate to the client library
+                Err(e) => Err(e),
+            }
+        } else {
+            get_token_from_channel(rx).await
+        }
+    }
+}
+
+async fn get_token_from_channel(
+    mut rx_token: watch::Receiver<Option<Result<Token>>>,
+) -> Result<Token> {
+    // We have already checked the current token in the channel, wait for new one
+    rx_token.changed().await.unwrap();
+    let token_result = rx_token.borrow().clone();
+
+    token_result.expect("There should always be a token or error in the channel after changed()")
+}
+
+async fn refresh_task<T>(token_provider: T, tx_token: watch::Sender<Option<Result<Token>>>)
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    loop {
+        let token_result = token_provider.get_token().await;
+
+        let _ = tx_token.send(Some(token_result.clone()));
+
+        match token_result {
+            Ok(new_token) => {
+                if let Some(expiry) = new_token.expires_at {
+                    let time_until_expiry = expiry.duration_since(Instant::now().into_std());
+
+                    if time_until_expiry > FIVE_MINUTES {
+                        let time_to_sleep = Instant::now() + time_until_expiry - FIVE_MINUTES;
+                        sleep_until(time_to_sleep).await;
+                    } else if time_until_expiry > TEN_SECONDS {
+                        // If expiry is less than 5 mins, wait till last 10 seconds to refresh
+                        // This is to handle cases where MDS returns about to refresh tokens
+                        let time_to_sleep = Instant::now() + time_until_expiry - TEN_SECONDS;
+                        sleep_until(time_to_sleep).await;
+                    }
path: src/auth/src/token_cache.rs
user: login: sai-sunder-s
body: Its an extreme edge case to keep receiving expired tokens I believe. And we would not be constantly retrying, we would have backoff in the retry policy, so it should be ok.
created_at: 2025-03-20T18:36:11Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,538 @@
+// Copyright 2024 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::token::{Token, TokenProvider};
+use crate::Result;
+use tokio::sync::watch;
+use tokio::time::{sleep, sleep_until, Duration, Instant};
+
+const FIVE_MINUTES: Duration = Duration::from_secs(300);
+const TEN_SECONDS: Duration = Duration::from_secs(10);
+
+#[derive(Debug)]
+pub(crate) struct TokenCache<T>
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    rx_token: watch::Receiver<Option<Result<Token>>>,
+    _phantom: std::marker::PhantomData<T>,
+}
+
+impl<T: TokenProvider> Clone for TokenCache<T> {
+    fn clone(&self) -> TokenCache<T> {
+        TokenCache {
+            rx_token: self.rx_token.clone(),
+            _phantom: std::marker::PhantomData,
+        }
+    }
+}
+
+// TODO 1552: Use the token cache in all creds
+#[allow(dead_code)]
+impl<T> TokenCache<T>
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    pub(crate) fn new(inner: T) -> Self {
+        let (tx_token, rx_token) = watch::channel::<Option<Result<Token>>>(None);
+
+        tokio::spawn(async move {
+            refresh_task(inner, tx_token).await;
+        });
+
+        Self {
+            rx_token,
+            _phantom: std::marker::PhantomData,
+        }
+    }
+}
+
+#[async_trait::async_trait]
+impl<T> TokenProvider for TokenCache<T>
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    async fn get_token(&self) -> Result<Token> {
+        let mut rx = self.rx_token.clone();
+        let token_result = rx.borrow_and_update().clone();
+
+        if let Some(token_result) = token_result {
+            match token_result {
+                Ok(token) => match token.expires_at {
+                    None => return Ok(token),
+                    Some(e) => {
+                        if e < Instant::now().into_std() {
+                            // Expired token, wait for refresh
+                            return get_token_from_channel(rx).await;
+                        } else {
+                            // valid token
+                            return Ok(token);
+                        }
+                    }
+                },
+                // An error in the result is still a valid result to propagate to the client library
+                Err(e) => Err(e),
+            }
+        } else {
+            get_token_from_channel(rx).await
+        }
+    }
+}
+
+async fn get_token_from_channel(
+    mut rx_token: watch::Receiver<Option<Result<Token>>>,
+) -> Result<Token> {
+    // We have already checked the current token in the channel, wait for new one
+    rx_token.changed().await.unwrap();
+    let token_result = rx_token.borrow().clone();
+
+    token_result.expect("There should always be a token or error in the channel after changed()")
+}
+
+async fn refresh_task<T>(token_provider: T, tx_token: watch::Sender<Option<Result<Token>>>)
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    loop {
+        let token_result = token_provider.get_token().await;
+
+        let _ = tx_token.send(Some(token_result.clone()));
+
+        match token_result {
+            Ok(new_token) => {
+                if let Some(expiry) = new_token.expires_at {
+                    let time_until_expiry = expiry.duration_since(Instant::now().into_std());
+
+                    if time_until_expiry > FIVE_MINUTES {
+                        let time_to_sleep = Instant::now() + time_until_expiry - FIVE_MINUTES;
+                        sleep_until(time_to_sleep).await;
+                    } else if time_until_expiry > TEN_SECONDS {
+                        // If expiry is less than 5 mins, wait till last 10 seconds to refresh
+                        // This is to handle cases where MDS returns about to refresh tokens
+                        let time_to_sleep = Instant::now() + time_until_expiry - TEN_SECONDS;
+                        sleep_until(time_to_sleep).await;
+                    }
path: src/auth/src/token_cache.rs
user: login: dbolduc
body: > we would have backoff in the retry policy

There is no backoff for successful requests.
created_at: 2025-03-20T18:42:20Z
author_association: MEMBER
diff_hunk: @@ -0,0 +1,538 @@
+// Copyright 2024 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::token::{Token, TokenProvider};
+use crate::Result;
+use tokio::sync::watch;
+use tokio::time::{sleep, sleep_until, Duration, Instant};
+
+const FIVE_MINUTES: Duration = Duration::from_secs(300);
+const TEN_SECONDS: Duration = Duration::from_secs(10);
+
+#[derive(Debug)]
+pub(crate) struct TokenCache<T>
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    rx_token: watch::Receiver<Option<Result<Token>>>,
+    _phantom: std::marker::PhantomData<T>,
+}
+
+impl<T: TokenProvider> Clone for TokenCache<T> {
+    fn clone(&self) -> TokenCache<T> {
+        TokenCache {
+            rx_token: self.rx_token.clone(),
+            _phantom: std::marker::PhantomData,
+        }
+    }
+}
+
+// TODO 1552: Use the token cache in all creds
+#[allow(dead_code)]
+impl<T> TokenCache<T>
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    pub(crate) fn new(inner: T) -> Self {
+        let (tx_token, rx_token) = watch::channel::<Option<Result<Token>>>(None);
+
+        tokio::spawn(async move {
+            refresh_task(inner, tx_token).await;
+        });
+
+        Self {
+            rx_token,
+            _phantom: std::marker::PhantomData,
+        }
+    }
+}
+
+#[async_trait::async_trait]
+impl<T> TokenProvider for TokenCache<T>
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    async fn get_token(&self) -> Result<Token> {
+        let mut rx = self.rx_token.clone();
+        let token_result = rx.borrow_and_update().clone();
+
+        if let Some(token_result) = token_result {
+            match token_result {
+                Ok(token) => match token.expires_at {
+                    None => return Ok(token),
+                    Some(e) => {
+                        if e < Instant::now().into_std() {
+                            // Expired token, wait for refresh
+                            return get_token_from_channel(rx).await;
+                        } else {
+                            // valid token
+                            return Ok(token);
+                        }
+                    }
+                },
+                // An error in the result is still a valid result to propagate to the client library
+                Err(e) => Err(e),
+            }
+        } else {
+            get_token_from_channel(rx).await
+        }
+    }
+}
+
+async fn get_token_from_channel(
+    mut rx_token: watch::Receiver<Option<Result<Token>>>,
+) -> Result<Token> {
+    // We have already checked the current token in the channel, wait for new one
+    rx_token.changed().await.unwrap();
+    let token_result = rx_token.borrow().clone();
+
+    token_result.expect("There should always be a token or error in the channel after changed()")
+}
+
+async fn refresh_task<T>(token_provider: T, tx_token: watch::Sender<Option<Result<Token>>>)
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    loop {
+        let token_result = token_provider.get_token().await;
+
+        let _ = tx_token.send(Some(token_result.clone()));
+
+        match token_result {
+            Ok(new_token) => {
+                if let Some(expiry) = new_token.expires_at {
+                    let time_until_expiry = expiry.duration_since(Instant::now().into_std());
+
+                    if time_until_expiry > FIVE_MINUTES {
+                        let time_to_sleep = Instant::now() + time_until_expiry - FIVE_MINUTES;
+                        sleep_until(time_to_sleep).await;
+                    } else if time_until_expiry > TEN_SECONDS {
+                        // If expiry is less than 5 mins, wait till last 10 seconds to refresh
+                        // This is to handle cases where MDS returns about to refresh tokens
+                        let time_to_sleep = Instant::now() + time_until_expiry - TEN_SECONDS;
+                        sleep_until(time_to_sleep).await;
+                    }
+                } else {
+                    // Handle case where expiry is not available
+                    // TODO 1553: Validate that all auth backends provide expiry and make expiry not optional.
+                    sleep(FIVE_MINUTES).await;
path: src/auth/src/token_cache.rs
user: login: dbolduc
body: > Maybe it should not?

That is a good point.

If we end up with something like a `trait HeaderAdder` in gax, then we will just need `create_api_key_credentials()` and `create_access_token_credentials()` to return a type that implements that trait.

The API key creds would not need to know anything about `Token`s.
created_at: 2025-03-20T18:50:12Z
author_association: MEMBER
diff_hunk: @@ -0,0 +1,538 @@
+// Copyright 2024 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::token::{Token, TokenProvider};
+use crate::Result;
+use tokio::sync::watch;
+use tokio::time::{sleep, sleep_until, Duration, Instant};
+
+const FIVE_MINUTES: Duration = Duration::from_secs(300);
+const TEN_SECONDS: Duration = Duration::from_secs(10);
+
+#[derive(Debug)]
+pub(crate) struct TokenCache<T>
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    rx_token: watch::Receiver<Option<Result<Token>>>,
+    _phantom: std::marker::PhantomData<T>,
+}
+
+impl<T: TokenProvider> Clone for TokenCache<T> {
+    fn clone(&self) -> TokenCache<T> {
+        TokenCache {
+            rx_token: self.rx_token.clone(),
+            _phantom: std::marker::PhantomData,
+        }
+    }
+}
+
+// TODO 1552: Use the token cache in all creds
+#[allow(dead_code)]
+impl<T> TokenCache<T>
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    pub(crate) fn new(inner: T) -> Self {
+        let (tx_token, rx_token) = watch::channel::<Option<Result<Token>>>(None);
+
+        tokio::spawn(async move {
+            refresh_task(inner, tx_token).await;
+        });
+
+        Self {
+            rx_token,
+            _phantom: std::marker::PhantomData,
+        }
+    }
+}
+
+#[async_trait::async_trait]
+impl<T> TokenProvider for TokenCache<T>
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    async fn get_token(&self) -> Result<Token> {
+        let mut rx = self.rx_token.clone();
+        let token_result = rx.borrow_and_update().clone();
+
+        if let Some(token_result) = token_result {
+            match token_result {
+                Ok(token) => match token.expires_at {
+                    None => return Ok(token),
+                    Some(e) => {
+                        if e < Instant::now().into_std() {
+                            // Expired token, wait for refresh
+                            return get_token_from_channel(rx).await;
+                        } else {
+                            // valid token
+                            return Ok(token);
+                        }
+                    }
+                },
+                // An error in the result is still a valid result to propagate to the client library
+                Err(e) => Err(e),
+            }
+        } else {
+            get_token_from_channel(rx).await
+        }
+    }
+}
+
+async fn get_token_from_channel(
+    mut rx_token: watch::Receiver<Option<Result<Token>>>,
+) -> Result<Token> {
+    // We have already checked the current token in the channel, wait for new one
+    rx_token.changed().await.unwrap();
path: src/auth/src/token_cache.rs
user: login: dbolduc
body: This convinced me that I was wrong about the `unwrap()` panicking in that case. It will successfully pull the latest `token_result` sent on the channel, even if the `refresh_task` is dead.

```rs
use tokio::sync::watch;
use tokio::time::{sleep, Duration};

#[derive(Clone)]
struct TokenCache {
    rx_token: watch::Receiver<String>,
}

impl TokenCache {
    fn new() -> Self {
        let (tx_token, rx_token) = watch::channel::<String>("expired".into());

        tokio::spawn(async move {
            refresh_task(tx_token).await;
        });

        Self {
            rx_token,
        }
    }

    async fn get_token(&self) -> String {
        let mut rx = self.rx_token.clone();
        let token = rx.borrow_and_update().clone();

        // Sleep for 2 seconds, to make sure this gets scheduled after the refresh task exits
        sleep(Duration::from_secs(2)).await;

        if token == "expired" {
            get_token_from_channel(rx).await
        } else {
            token
        }
    }
}

async fn get_token_from_channel(
    mut rx_token: watch::Receiver<String>,
) -> String {
    rx_token.changed().await.unwrap();
    rx_token.borrow().clone()
}

async fn refresh_task(tx_token: watch::Sender<String>)
{
    // sleep 1 second.
    sleep(Duration::from_secs(1)).await;

    // simulate an error
    let _ = tx_token.send("error".into());

    println!("refresh_task is dropped");
}

#[tokio::main]
async fn main() {
    // create the cache, spawning the refresh task.
    let c1 = TokenCache::new();
    let c2 = c1.clone();
    
    let h1 = tokio::spawn(async move { 
        let token = c1.get_token().await;
        println!("T1 token: {token}");
    });

    let h2 = tokio::spawn(async move {
        // stagger the second thread
        sleep(Duration::from_millis(500)).await;

        let token = c2.get_token().await;
        println!("T2 token: {token}");
    });

    h1.await.unwrap();
    h2.await.unwrap();
}
```

Output:
```
refresh_task is dropped
T1 token: error
T2 token: error
```

---

It looks like the cloned `rx_token`, has its own `changed()` state. I didn't realize that.
created_at: 2025-03-21T16:21:50Z
author_association: MEMBER
diff_hunk: @@ -0,0 +1,538 @@
+// Copyright 2024 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::token::{Token, TokenProvider};
+use crate::Result;
+use tokio::sync::watch;
+use tokio::time::{sleep, sleep_until, Duration, Instant};
+
+const FIVE_MINUTES: Duration = Duration::from_secs(300);
+const TEN_SECONDS: Duration = Duration::from_secs(10);
+
+#[derive(Debug)]
+pub(crate) struct TokenCache<T>
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    rx_token: watch::Receiver<Option<Result<Token>>>,
+    _phantom: std::marker::PhantomData<T>,
+}
+
+impl<T: TokenProvider> Clone for TokenCache<T> {
+    fn clone(&self) -> TokenCache<T> {
+        TokenCache {
+            rx_token: self.rx_token.clone(),
+            _phantom: std::marker::PhantomData,
+        }
+    }
+}
+
+// TODO 1552: Use the token cache in all creds
+#[allow(dead_code)]
+impl<T> TokenCache<T>
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    pub(crate) fn new(inner: T) -> Self {
+        let (tx_token, rx_token) = watch::channel::<Option<Result<Token>>>(None);
+
+        tokio::spawn(async move {
+            refresh_task(inner, tx_token).await;
+        });
+
+        Self {
+            rx_token,
+            _phantom: std::marker::PhantomData,
+        }
+    }
+}
+
+#[async_trait::async_trait]
+impl<T> TokenProvider for TokenCache<T>
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    async fn get_token(&self) -> Result<Token> {
+        let mut rx = self.rx_token.clone();
+        let token_result = rx.borrow_and_update().clone();
+
+        if let Some(token_result) = token_result {
+            match token_result {
+                Ok(token) => match token.expires_at {
+                    None => return Ok(token),
+                    Some(e) => {
+                        if e < Instant::now().into_std() {
+                            // Expired token, wait for refresh
+                            return get_token_from_channel(rx).await;
+                        } else {
+                            // valid token
+                            return Ok(token);
+                        }
+                    }
+                },
+                // An error in the result is still a valid result to propagate to the client library
+                Err(e) => Err(e),
+            }
+        } else {
+            get_token_from_channel(rx).await
+        }
+    }
+}
+
+async fn get_token_from_channel(
+    mut rx_token: watch::Receiver<Option<Result<Token>>>,
+) -> Result<Token> {
+    // We have already checked the current token in the channel, wait for new one
+    rx_token.changed().await.unwrap();
path: src/auth/src/token_cache.rs
user: login: sai-sunder-s
body: Thank you for verifying. Yes every rx clone maintains its own `changed` state, but the `changed()` [documentation](https://github.com/tokio-rs/tokio/blob/0ec4d0db4d24af74cea40cf0ceed6b7a0a5ff183/tokio/src/sync/watch.rs#L712) states there will be error is sender is dropped.

My debugging also yielded similar result as yours. Was going to use thread::park() as a precaution. Lets keep code as it is right now.
created_at: 2025-03-21T16:40:00Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,538 @@
+// Copyright 2024 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::token::{Token, TokenProvider};
+use crate::Result;
+use tokio::sync::watch;
+use tokio::time::{sleep, sleep_until, Duration, Instant};
+
+const FIVE_MINUTES: Duration = Duration::from_secs(300);
+const TEN_SECONDS: Duration = Duration::from_secs(10);
+
+#[derive(Debug)]
+pub(crate) struct TokenCache<T>
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    rx_token: watch::Receiver<Option<Result<Token>>>,
+    _phantom: std::marker::PhantomData<T>,
+}
+
+impl<T: TokenProvider> Clone for TokenCache<T> {
+    fn clone(&self) -> TokenCache<T> {
+        TokenCache {
+            rx_token: self.rx_token.clone(),
+            _phantom: std::marker::PhantomData,
+        }
+    }
+}
+
+// TODO 1552: Use the token cache in all creds
+#[allow(dead_code)]
+impl<T> TokenCache<T>
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    pub(crate) fn new(inner: T) -> Self {
+        let (tx_token, rx_token) = watch::channel::<Option<Result<Token>>>(None);
+
+        tokio::spawn(async move {
+            refresh_task(inner, tx_token).await;
+        });
+
+        Self {
+            rx_token,
+            _phantom: std::marker::PhantomData,
+        }
+    }
+}
+
+#[async_trait::async_trait]
+impl<T> TokenProvider for TokenCache<T>
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    async fn get_token(&self) -> Result<Token> {
+        let mut rx = self.rx_token.clone();
+        let token_result = rx.borrow_and_update().clone();
+
+        if let Some(token_result) = token_result {
+            match token_result {
+                Ok(token) => match token.expires_at {
+                    None => return Ok(token),
+                    Some(e) => {
+                        if e < Instant::now().into_std() {
+                            // Expired token, wait for refresh
+                            return get_token_from_channel(rx).await;
+                        } else {
+                            // valid token
+                            return Ok(token);
+                        }
+                    }
+                },
+                // An error in the result is still a valid result to propagate to the client library
+                Err(e) => Err(e),
+            }
+        } else {
+            get_token_from_channel(rx).await
+        }
+    }
+}
+
+async fn get_token_from_channel(
+    mut rx_token: watch::Receiver<Option<Result<Token>>>,
+) -> Result<Token> {
+    // We have already checked the current token in the channel, wait for new one
+    rx_token.changed().await.unwrap();
+    let token_result = rx_token.borrow().clone();
+
+    token_result.expect("There should always be a token or error in the channel after changed()")
+}
+
+async fn refresh_task<T>(token_provider: T, tx_token: watch::Sender<Option<Result<Token>>>)
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    loop {
+        let token_result = token_provider.get_token().await;
+
+        let _ = tx_token.send(Some(token_result.clone()));
+
+        match token_result {
+            Ok(new_token) => {
+                if let Some(expiry) = new_token.expires_at {
+                    let time_until_expiry = expiry.duration_since(Instant::now().into_std());
+
+                    if time_until_expiry > FIVE_MINUTES {
+                        let time_to_sleep = Instant::now() + time_until_expiry - FIVE_MINUTES;
+                        sleep_until(time_to_sleep).await;
+                    } else if time_until_expiry > TEN_SECONDS {
+                        // If expiry is less than 5 mins, wait till last 10 seconds to refresh
+                        // This is to handle cases where MDS returns about to refresh tokens
+                        let time_to_sleep = Instant::now() + time_until_expiry - TEN_SECONDS;
+                        sleep_until(time_to_sleep).await;
+                    }
+                } else {
+                    // Handle case where expiry is not available
+                    // TODO 1553: Validate that all auth backends provide expiry and make expiry not optional.
+                    sleep(FIVE_MINUTES).await;
path: src/auth/src/token_cache.rs
user: login: sai-sunder-s
body: Yeah we could do something like that.
created_at: 2025-03-21T17:11:33Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,540 @@
+// Copyright 2025 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::token::{Token, TokenProvider};
+use crate::Result;
+use tokio::sync::watch;
+use tokio::time::{sleep_until, Duration, Instant};
+
+const FIVE_MINUTES: Duration = Duration::from_secs(300);
+const TEN_SECONDS: Duration = Duration::from_secs(10);
+
+#[derive(Debug)]
+pub(crate) struct TokenCache<T>
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    rx_token: watch::Receiver<Option<Result<Token>>>,
+    _phantom: std::marker::PhantomData<T>,
+}
+
+impl<T: TokenProvider> Clone for TokenCache<T> {
+    fn clone(&self) -> TokenCache<T> {
+        TokenCache {
+            rx_token: self.rx_token.clone(),
+            _phantom: std::marker::PhantomData,
+        }
+    }
+}
+
+// TODO(#1552): Use the token cache in all creds
+#[allow(dead_code)]
+impl<T> TokenCache<T>
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    pub(crate) fn new(inner: T) -> Self {
+        let (tx_token, rx_token) = watch::channel::<Option<Result<Token>>>(None);
+
+        tokio::spawn(async move {
+            refresh_task(inner, tx_token).await;
+        });
+
+        Self {
+            rx_token,
+            _phantom: std::marker::PhantomData,
+        }
+    }
+}
+
+#[async_trait::async_trait]
+impl<T> TokenProvider for TokenCache<T>
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    async fn get_token(&self) -> Result<Token> {
+        let mut rx = self.rx_token.clone();
+        let token_result = rx.borrow_and_update().clone();
+
+        if let Some(token_result) = token_result {
+            match token_result {
+                Ok(token) => match token.expires_at {
+                    None => return Ok(token),
+                    Some(e) => {
+                        if e < Instant::now().into_std() {
+                            // Expired token, wait for refresh
+                            return wait_for_next_token(rx).await;
+                        } else {
+                            // valid token
+                            return Ok(token);
+                        }
+                    }
+                },
+                // An error in the result is still a valid result to propagate to the client library
+                Err(e) => Err(e),
+            }
+        } else {
+            wait_for_next_token(rx).await
+        }
+    }
+}
+
+async fn wait_for_next_token(
+    mut rx_token: watch::Receiver<Option<Result<Token>>>,
+) -> Result<Token> {
+    rx_token.changed().await.unwrap();
+    let token_result = rx_token.borrow().clone();
+
+    token_result.expect("There should always be a token or error in the channel after changed()")
+}
+
+async fn refresh_task<T>(token_provider: T, tx_token: watch::Sender<Option<Result<Token>>>)
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    loop {
+        let token_result = token_provider.get_token().await;
+
+        let _ = tx_token.send(Some(token_result.clone()));
+
+        match token_result {
+            Ok(new_token) => {
+                if let Some(expiry) = new_token.expires_at {
+                    let time_until_expiry = expiry.duration_since(Instant::now().into_std());
+
+                    if time_until_expiry > FIVE_MINUTES {
+                        let time_to_sleep = Instant::now() + time_until_expiry - FIVE_MINUTES;
+                        sleep_until(time_to_sleep).await;
+                    } else if time_until_expiry > TEN_SECONDS {
+                        // If expiry is less than 5 mins, wait till last 10 seconds to refresh
+                        // This is to handle cases where MDS returns about to refresh tokens
path: src/auth/src/token_cache.rs
user: login: coryan
body: I think:

```suggestion
                        // This is to handle cases where MDS returns about to expire tokens
```
created_at: 2025-03-21T17:19:50Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,540 @@
+// Copyright 2025 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::token::{Token, TokenProvider};
+use crate::Result;
+use tokio::sync::watch;
+use tokio::time::{sleep_until, Duration, Instant};
+
+const FIVE_MINUTES: Duration = Duration::from_secs(300);
+const TEN_SECONDS: Duration = Duration::from_secs(10);
path: src/auth/src/token_cache.rs
user: login: coryan
body: More semantically meaningful names would be useful, such as:

```suggestion
const NORMAL_REFRESH_SLACK: Duration = Duration::from_secs(300);
const SHORT_REFRESH_SLACK: Duration = Duration::from_secs(10);
```

created_at: 2025-03-21T17:20:41Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,540 @@
+// Copyright 2025 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::token::{Token, TokenProvider};
+use crate::Result;
+use tokio::sync::watch;
+use tokio::time::{sleep_until, Duration, Instant};
+
+const FIVE_MINUTES: Duration = Duration::from_secs(300);
+const TEN_SECONDS: Duration = Duration::from_secs(10);
+
+#[derive(Debug)]
+pub(crate) struct TokenCache<T>
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    rx_token: watch::Receiver<Option<Result<Token>>>,
+    _phantom: std::marker::PhantomData<T>,
+}
path: src/auth/src/token_cache.rs
user: login: coryan
body: I think you can simplify this to:

```suggestion
#[derive(Debug)]
pub(crate) struct TokenCache
{
    rx_token: watch::Receiver<Option<Result<Token>>>,
}
```

and then use the generic `T` parameter only in `new()`...
created_at: 2025-03-21T17:22:52Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,540 @@
+// Copyright 2025 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::token::{Token, TokenProvider};
+use crate::Result;
+use tokio::sync::watch;
+use tokio::time::{sleep_until, Duration, Instant};
+
+const FIVE_MINUTES: Duration = Duration::from_secs(300);
+const TEN_SECONDS: Duration = Duration::from_secs(10);
+
+#[derive(Debug)]
+pub(crate) struct TokenCache<T>
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    rx_token: watch::Receiver<Option<Result<Token>>>,
+    _phantom: std::marker::PhantomData<T>,
+}
+
+impl<T: TokenProvider> Clone for TokenCache<T> {
+    fn clone(&self) -> TokenCache<T> {
+        TokenCache {
+            rx_token: self.rx_token.clone(),
+            _phantom: std::marker::PhantomData,
+        }
+    }
+}
+
+// TODO(#1552): Use the token cache in all creds
+#[allow(dead_code)]
+impl<T> TokenCache<T>
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    pub(crate) fn new(inner: T) -> Self {
path: src/auth/src/token_cache.rs
user: login: coryan
body: If you agree with my simplification, this becomes:

```suggestion
impl TokenCache
{
    pub(crate) fn new<T>(inner: T) -> Self
    where
        T: TokenProvider + Send + Sync + 'static,
    {
```
created_at: 2025-03-21T17:23:56Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,538 @@
+// Copyright 2024 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::token::{Token, TokenProvider};
+use crate::Result;
+use tokio::sync::watch;
+use tokio::time::{sleep, sleep_until, Duration, Instant};
+
+const FIVE_MINUTES: Duration = Duration::from_secs(300);
+const TEN_SECONDS: Duration = Duration::from_secs(10);
+
+#[derive(Debug)]
+pub(crate) struct TokenCache<T>
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    rx_token: watch::Receiver<Option<Result<Token>>>,
+    _phantom: std::marker::PhantomData<T>,
+}
+
+impl<T: TokenProvider> Clone for TokenCache<T> {
+    fn clone(&self) -> TokenCache<T> {
+        TokenCache {
+            rx_token: self.rx_token.clone(),
+            _phantom: std::marker::PhantomData,
+        }
+    }
+}
+
+// TODO 1552: Use the token cache in all creds
+#[allow(dead_code)]
+impl<T> TokenCache<T>
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    pub(crate) fn new(inner: T) -> Self {
+        let (tx_token, rx_token) = watch::channel::<Option<Result<Token>>>(None);
path: src/auth/src/token_cache.rs
user: login: coryan
body: It happens from time to time. By accident if nothing else.

created_at: 2025-03-21T17:24:31Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,540 @@
+// Copyright 2025 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::token::{Token, TokenProvider};
+use crate::Result;
+use tokio::sync::watch;
+use tokio::time::{sleep_until, Duration, Instant};
+
+const FIVE_MINUTES: Duration = Duration::from_secs(300);
+const TEN_SECONDS: Duration = Duration::from_secs(10);
+
+#[derive(Debug)]
+pub(crate) struct TokenCache<T>
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    rx_token: watch::Receiver<Option<Result<Token>>>,
+    _phantom: std::marker::PhantomData<T>,
+}
+
+impl<T: TokenProvider> Clone for TokenCache<T> {
+    fn clone(&self) -> TokenCache<T> {
+        TokenCache {
+            rx_token: self.rx_token.clone(),
+            _phantom: std::marker::PhantomData,
+        }
+    }
+}
+
+// TODO(#1552): Use the token cache in all creds
+#[allow(dead_code)]
+impl<T> TokenCache<T>
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    pub(crate) fn new(inner: T) -> Self {
+        let (tx_token, rx_token) = watch::channel::<Option<Result<Token>>>(None);
+
+        tokio::spawn(async move {
+            refresh_task(inner, tx_token).await;
+        });
+
+        Self {
+            rx_token,
+            _phantom: std::marker::PhantomData,
+        }
+    }
+}
+
+#[async_trait::async_trait]
+impl<T> TokenProvider for TokenCache<T>
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    async fn get_token(&self) -> Result<Token> {
+        let mut rx = self.rx_token.clone();
+        let token_result = rx.borrow_and_update().clone();
+
+        if let Some(token_result) = token_result {
+            match token_result {
+                Ok(token) => match token.expires_at {
+                    None => return Ok(token),
+                    Some(e) => {
+                        if e < Instant::now().into_std() {
+                            // Expired token, wait for refresh
+                            return wait_for_next_token(rx).await;
+                        } else {
+                            // valid token
+                            return Ok(token);
+                        }
+                    }
+                },
+                // An error in the result is still a valid result to propagate to the client library
+                Err(e) => Err(e),
+            }
+        } else {
+            wait_for_next_token(rx).await
+        }
+    }
+}
+
+async fn wait_for_next_token(
+    mut rx_token: watch::Receiver<Option<Result<Token>>>,
+) -> Result<Token> {
+    rx_token.changed().await.unwrap();
+    let token_result = rx_token.borrow().clone();
+
+    token_result.expect("There should always be a token or error in the channel after changed()")
+}
+
+async fn refresh_task<T>(token_provider: T, tx_token: watch::Sender<Option<Result<Token>>>)
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    loop {
+        let token_result = token_provider.get_token().await;
+
+        let _ = tx_token.send(Some(token_result.clone()));
+
+        match token_result {
+            Ok(new_token) => {
+                if let Some(expiry) = new_token.expires_at {
+                    let time_until_expiry = expiry.duration_since(Instant::now().into_std());
+
+                    if time_until_expiry > FIVE_MINUTES {
+                        let time_to_sleep = Instant::now() + time_until_expiry - FIVE_MINUTES;
+                        sleep_until(time_to_sleep).await;
+                    } else if time_until_expiry > TEN_SECONDS {
+                        // If expiry is less than 5 mins, wait till last 10 seconds to refresh
+                        // This is to handle cases where MDS returns about to refresh tokens
path: src/auth/src/token_cache.rs
user: login: coryan
body: ```suggestion
                        // This is to handle cases where MDS **repeatedly** returns about to refresh tokens
```


created_at: 2025-03-21T17:38:15Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,540 @@
+// Copyright 2025 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::token::{Token, TokenProvider};
+use crate::Result;
+use tokio::sync::watch;
+use tokio::time::{sleep_until, Duration, Instant};
+
+const FIVE_MINUTES: Duration = Duration::from_secs(300);
+const TEN_SECONDS: Duration = Duration::from_secs(10);
+
+#[derive(Debug)]
+pub(crate) struct TokenCache<T>
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    rx_token: watch::Receiver<Option<Result<Token>>>,
+    _phantom: std::marker::PhantomData<T>,
+}
+
+impl<T: TokenProvider> Clone for TokenCache<T> {
+    fn clone(&self) -> TokenCache<T> {
+        TokenCache {
+            rx_token: self.rx_token.clone(),
+            _phantom: std::marker::PhantomData,
+        }
+    }
+}
+
+// TODO(#1552): Use the token cache in all creds
+#[allow(dead_code)]
+impl<T> TokenCache<T>
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    pub(crate) fn new(inner: T) -> Self {
+        let (tx_token, rx_token) = watch::channel::<Option<Result<Token>>>(None);
+
+        tokio::spawn(async move {
+            refresh_task(inner, tx_token).await;
+        });
+
+        Self {
+            rx_token,
+            _phantom: std::marker::PhantomData,
+        }
+    }
+}
+
+#[async_trait::async_trait]
+impl<T> TokenProvider for TokenCache<T>
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    async fn get_token(&self) -> Result<Token> {
+        let mut rx = self.rx_token.clone();
+        let token_result = rx.borrow_and_update().clone();
+
+        if let Some(token_result) = token_result {
+            match token_result {
+                Ok(token) => match token.expires_at {
+                    None => return Ok(token),
+                    Some(e) => {
+                        if e < Instant::now().into_std() {
+                            // Expired token, wait for refresh
+                            return wait_for_next_token(rx).await;
+                        } else {
+                            // valid token
+                            return Ok(token);
+                        }
+                    }
+                },
+                // An error in the result is still a valid result to propagate to the client library
+                Err(e) => Err(e),
+            }
+        } else {
+            wait_for_next_token(rx).await
+        }
+    }
+}
+
+async fn wait_for_next_token(
+    mut rx_token: watch::Receiver<Option<Result<Token>>>,
+) -> Result<Token> {
+    rx_token.changed().await.unwrap();
+    let token_result = rx_token.borrow().clone();
+
+    token_result.expect("There should always be a token or error in the channel after changed()")
+}
+
+async fn refresh_task<T>(token_provider: T, tx_token: watch::Sender<Option<Result<Token>>>)
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    loop {
+        let token_result = token_provider.get_token().await;
+
+        let _ = tx_token.send(Some(token_result.clone()));
+
+        match token_result {
+            Ok(new_token) => {
+                if let Some(expiry) = new_token.expires_at {
+                    let time_until_expiry = expiry.duration_since(Instant::now().into_std());
+
+                    if time_until_expiry > FIVE_MINUTES {
+                        let time_to_sleep = Instant::now() + time_until_expiry - FIVE_MINUTES;
+                        sleep_until(time_to_sleep).await;
+                    } else if time_until_expiry > TEN_SECONDS {
+                        // If expiry is less than 5 mins, wait till last 10 seconds to refresh
+                        // This is to handle cases where MDS returns about to refresh tokens
+                        let time_to_sleep = Instant::now() + time_until_expiry - TEN_SECONDS;
path: src/auth/src/token_cache.rs
user: login: coryan
body: This is very odd, if we normally think that starting 5 minutes earlier is a good idea, then why wait until there are only 10 seconds left?  maybe we should just sleep for 10 seconds in this case?  Or maybe do some kind of exponentially shorter sleeps (1/2 of the remaining time) if the expiration is within 5 minutes?

created_at: 2025-03-21T17:40:55Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,540 @@
+// Copyright 2025 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::token::{Token, TokenProvider};
+use crate::Result;
+use tokio::sync::watch;
+use tokio::time::{sleep_until, Duration, Instant};
+
+const FIVE_MINUTES: Duration = Duration::from_secs(300);
+const TEN_SECONDS: Duration = Duration::from_secs(10);
+
+#[derive(Debug)]
+pub(crate) struct TokenCache<T>
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    rx_token: watch::Receiver<Option<Result<Token>>>,
+    _phantom: std::marker::PhantomData<T>,
+}
+
+impl<T: TokenProvider> Clone for TokenCache<T> {
+    fn clone(&self) -> TokenCache<T> {
+        TokenCache {
+            rx_token: self.rx_token.clone(),
+            _phantom: std::marker::PhantomData,
+        }
+    }
+}
+
+// TODO(#1552): Use the token cache in all creds
+#[allow(dead_code)]
+impl<T> TokenCache<T>
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    pub(crate) fn new(inner: T) -> Self {
+        let (tx_token, rx_token) = watch::channel::<Option<Result<Token>>>(None);
+
+        tokio::spawn(async move {
+            refresh_task(inner, tx_token).await;
+        });
+
+        Self {
+            rx_token,
+            _phantom: std::marker::PhantomData,
+        }
+    }
+}
+
+#[async_trait::async_trait]
+impl<T> TokenProvider for TokenCache<T>
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    async fn get_token(&self) -> Result<Token> {
+        let mut rx = self.rx_token.clone();
+        let token_result = rx.borrow_and_update().clone();
+
+        if let Some(token_result) = token_result {
+            match token_result {
+                Ok(token) => match token.expires_at {
+                    None => return Ok(token),
+                    Some(e) => {
+                        if e < Instant::now().into_std() {
+                            // Expired token, wait for refresh
+                            return wait_for_next_token(rx).await;
+                        } else {
+                            // valid token
+                            return Ok(token);
+                        }
+                    }
+                },
+                // An error in the result is still a valid result to propagate to the client library
+                Err(e) => Err(e),
+            }
+        } else {
+            wait_for_next_token(rx).await
+        }
+    }
+}
+
+async fn wait_for_next_token(
+    mut rx_token: watch::Receiver<Option<Result<Token>>>,
+) -> Result<Token> {
+    rx_token.changed().await.unwrap();
+    let token_result = rx_token.borrow().clone();
+
+    token_result.expect("There should always be a token or error in the channel after changed()")
+}
+
+async fn refresh_task<T>(token_provider: T, tx_token: watch::Sender<Option<Result<Token>>>)
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    loop {
+        let token_result = token_provider.get_token().await;
+
+        let _ = tx_token.send(Some(token_result.clone()));
+
+        match token_result {
+            Ok(new_token) => {
+                if let Some(expiry) = new_token.expires_at {
+                    let time_until_expiry = expiry.duration_since(Instant::now().into_std());
+
+                    if time_until_expiry > FIVE_MINUTES {
+                        let time_to_sleep = Instant::now() + time_until_expiry - FIVE_MINUTES;
+                        sleep_until(time_to_sleep).await;
+                    } else if time_until_expiry > TEN_SECONDS {
+                        // If expiry is less than 5 mins, wait till last 10 seconds to refresh
+                        // This is to handle cases where MDS returns about to refresh tokens
+                        let time_to_sleep = Instant::now() + time_until_expiry - TEN_SECONDS;
+                        sleep_until(time_to_sleep).await;
+                    }
+                    else{
+                        // We were given a token that is expired, or expires in less than 10 seconds. We will immediately restart the loop, and fetch a new token.
+                    }
+                } else {
+                    // If there is no expiry, the token is valid forever, so no need to refresh
+                    // TODO(#1553): Validate that all auth backends provide expiry and make expiry not optional.
+                    break;
path: src/auth/src/token_cache.rs
user: login: coryan
body: The code coverage says you are missing a test for this case, is that true?
created_at: 2025-03-21T17:44:10Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,540 @@
+// Copyright 2025 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::token::{Token, TokenProvider};
+use crate::Result;
+use tokio::sync::watch;
+use tokio::time::{sleep_until, Duration, Instant};
+
+const FIVE_MINUTES: Duration = Duration::from_secs(300);
+const TEN_SECONDS: Duration = Duration::from_secs(10);
+
+#[derive(Debug)]
+pub(crate) struct TokenCache<T>
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    rx_token: watch::Receiver<Option<Result<Token>>>,
+    _phantom: std::marker::PhantomData<T>,
+}
+
+impl<T: TokenProvider> Clone for TokenCache<T> {
+    fn clone(&self) -> TokenCache<T> {
+        TokenCache {
+            rx_token: self.rx_token.clone(),
+            _phantom: std::marker::PhantomData,
+        }
+    }
+}
+
+// TODO(#1552): Use the token cache in all creds
+#[allow(dead_code)]
+impl<T> TokenCache<T>
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    pub(crate) fn new(inner: T) -> Self {
+        let (tx_token, rx_token) = watch::channel::<Option<Result<Token>>>(None);
+
+        tokio::spawn(async move {
+            refresh_task(inner, tx_token).await;
+        });
+
+        Self {
+            rx_token,
+            _phantom: std::marker::PhantomData,
+        }
+    }
+}
+
+#[async_trait::async_trait]
+impl<T> TokenProvider for TokenCache<T>
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    async fn get_token(&self) -> Result<Token> {
+        let mut rx = self.rx_token.clone();
+        let token_result = rx.borrow_and_update().clone();
+
+        if let Some(token_result) = token_result {
+            match token_result {
+                Ok(token) => match token.expires_at {
+                    None => return Ok(token),
+                    Some(e) => {
+                        if e < Instant::now().into_std() {
+                            // Expired token, wait for refresh
+                            return wait_for_next_token(rx).await;
+                        } else {
+                            // valid token
+                            return Ok(token);
+                        }
+                    }
+                },
+                // An error in the result is still a valid result to propagate to the client library
+                Err(e) => Err(e),
+            }
+        } else {
+            wait_for_next_token(rx).await
+        }
+    }
+}
+
+async fn wait_for_next_token(
+    mut rx_token: watch::Receiver<Option<Result<Token>>>,
+) -> Result<Token> {
+    rx_token.changed().await.unwrap();
+    let token_result = rx_token.borrow().clone();
+
+    token_result.expect("There should always be a token or error in the channel after changed()")
+}
+
+async fn refresh_task<T>(token_provider: T, tx_token: watch::Sender<Option<Result<Token>>>)
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    loop {
+        let token_result = token_provider.get_token().await;
+
+        let _ = tx_token.send(Some(token_result.clone()));
+
+        match token_result {
+            Ok(new_token) => {
+                if let Some(expiry) = new_token.expires_at {
+                    let time_until_expiry = expiry.duration_since(Instant::now().into_std());
+
+                    if time_until_expiry > FIVE_MINUTES {
+                        let time_to_sleep = Instant::now() + time_until_expiry - FIVE_MINUTES;
+                        sleep_until(time_to_sleep).await;
+                    } else if time_until_expiry > TEN_SECONDS {
+                        // If expiry is less than 5 mins, wait till last 10 seconds to refresh
+                        // This is to handle cases where MDS returns about to refresh tokens
+                        let time_to_sleep = Instant::now() + time_until_expiry - TEN_SECONDS;
+                        sleep_until(time_to_sleep).await;
+                    }
+                    else{
+                        // We were given a token that is expired, or expires in less than 10 seconds. We will immediately restart the loop, and fetch a new token.
+                    }
+                } else {
+                    // If there is no expiry, the token is valid forever, so no need to refresh
+                    // TODO(#1553): Validate that all auth backends provide expiry and make expiry not optional.
+                    break;
+                }
+            }
+            Err(_) => {
+                // The retry policy has been used already by the inner token provider.
+                // If it ended in an error, just quit the background task.
+                break;
+            }
+        }
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    use crate::errors::CredentialError;
+    use crate::token::test::MockTokenProvider;
+    use std::ops::{Add, Sub};
+    use std::sync::{Arc, Mutex};
+    use tokio::time::{Duration, Instant};
+
+    static TOKEN_VALID_DURATION: Duration = Duration::from_secs(3600);
+
+    #[tokio::test]
+    async fn initial_token_success() {
+        let expected = Token {
+            token: "test-token".to_string(),
+            token_type: "Bearer".to_string(),
+            expires_at: None,
+            metadata: None,
+        };
+        let expected_clone = expected.clone();
+
+        let mut mock = MockTokenProvider::new();
+        mock.expect_get_token()
+            .times(1)
+            .return_once(|| Ok(expected_clone));
+
+        let cache = TokenCache::new(mock);
+        let actual = cache.get_token().await.unwrap();
+        assert_eq!(actual, expected);
+
+        // Verify that we use the cached token instead of making a new request
+        // to the mock token provider.
+        let actual = cache.get_token().await.unwrap();
+        assert_eq!(actual, expected);
+    }
+
+    #[tokio::test]
+    async fn initial_token_failure() {
+        let mut mock = MockTokenProvider::new();
+        mock.expect_get_token()
+            .times(1)
+            .returning(|| Err(CredentialError::non_retryable_from_str("fail")));
+
+        let cache = TokenCache::new(mock);
+        assert!(cache.get_token().await.is_err());
+
+        // Verify that a new request is made to the mock token provider when we
+        // don't have a valid token.
+        assert!(cache.get_token().await.is_err());
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn expired_token_success() {
+        let now = Instant::now();
+
+        let initial = Token {
+            token: "initial-token".to_string(),
+            token_type: "Bearer".to_string(),
+            expires_at: Some((now + TOKEN_VALID_DURATION).into_std()),
+            metadata: None,
+        };
+        let initial_clone = initial.clone();
+
+        let refresh = Token {
+            token: "refreshed-token".to_string(),
+            token_type: "Bearer".to_string(),
+            expires_at: Some((now + 2 * TOKEN_VALID_DURATION).into_std()),
+            metadata: None,
+        };
+        let refresh_clone = refresh.clone();
+
+        let mut mock = MockTokenProvider::new();
+        mock.expect_get_token()
+            .times(1)
+            .return_once(|| Ok(initial_clone));
+
+        mock.expect_get_token()
+            .times(1)
+            .return_once(|| Ok(refresh_clone));
+
+        // fetch an initial token
+        let cache = TokenCache::new(mock);
+        let actual = cache.get_token().await.unwrap();
+        assert_eq!(actual, initial);
+
+        // wait long enough for the token to be expired
+        // get_token should be waiting until the new token is available
+        let sleep = TOKEN_VALID_DURATION.add(Duration::from_secs(100));
+        tokio::time::advance(sleep).await;
+
+        // make sure this is the new token
+        let actual = cache.get_token().await.unwrap();
+        assert_eq!(actual, refresh);
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn expired_token_failure() {
+        let now = Instant::now();
+
+        let initial = Token {
+            token: "initial-token".to_string(),
+            token_type: "Bearer".to_string(),
+            expires_at: Some((now + TOKEN_VALID_DURATION).into_std()),
+            metadata: None,
+        };
+        let initial_clone = initial.clone();
+
+        let mut mock = MockTokenProvider::new();
+        mock.expect_get_token()
+            .times(1)
+            .return_once(|| Ok(initial_clone));
+
+        mock.expect_get_token()
+            .times(1)
+            .return_once(|| Err(CredentialError::non_retryable_from_str("fail")));
+
+        // fetch an initial token
+        let cache = TokenCache::new(mock);
+        let actual = cache.get_token().await.unwrap();
+        assert_eq!(actual, initial);
+
+        // wait long enough for the token to be expired
+        let sleep = TOKEN_VALID_DURATION.add(Duration::from_secs(100));
+        tokio::time::advance(sleep).await;
+
+        // make sure we return the error, not the expired token
+        assert!(cache.get_token().await.is_err());
+    }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 8)]
+    async fn token_cache_multiple_requests_existing_valid_token() {
+        let now = Instant::now();
+
+        let token = Token {
+            token: "initial-token".to_string(),
+            token_type: "Bearer".to_string(),
+            expires_at: Some((now + TOKEN_VALID_DURATION).into_std()),
+            metadata: None,
+        };
+        let token_clone = token.clone();
+
+        let mut mock = MockTokenProvider::new();
+        mock.expect_get_token()
+            .times(1)
+            .return_once(|| Ok(token_clone));
+
+        // fetch an initial token
+        let cache = TokenCache::new(mock);
+        let actual = cache.get_token().await.unwrap();
+        assert_eq!(actual, token);
+
+        // Spawn N tasks, all asking for a token at once.
+        let tasks = (0..1000)
+            .map(|_| {
+                let cache_clone = cache.clone();
+                tokio::spawn(async move { cache_clone.get_token().await })
+            })
+            .collect::<Vec<_>>();
+
+        // Wait for the N token requests to complete, verifying the returned token.
+        for task in tasks {
+            let actual = task.await.unwrap();
+            assert!(actual.is_ok(), "{}", actual.err().unwrap());
+            assert_eq!(actual.unwrap(), token);
+        }
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn refresh_task_loop() {
+        let now = Instant::now();
+
+        let token1 = Token {
+            token: "token1".to_string(),
+            token_type: "Bearer".to_string(),
+            expires_at: Some((now + TOKEN_VALID_DURATION).into_std()),
+            metadata: None,
+        };
+        let token1_clone = token1.clone();
+
+        let token2 = Token {
+            token: "token2".to_string(),
+            token_type: "Bearer".to_string(),
+            expires_at: Some((now + 2 * TOKEN_VALID_DURATION).into_std()),
+            metadata: None,
+        };
+        let token2_clone = token2.clone();
+
+        let token3 = Token {
+            token: "token3".to_string(),
+            token_type: "Bearer".to_string(),
+            expires_at: Some((now + 3 * TOKEN_VALID_DURATION).into_std()),
+            metadata: None,
+        };
+        let token3_clone = token3.clone();
+
+        let mut mock = MockTokenProvider::new();
+        mock.expect_get_token()
+            .times(1)
+            .return_once(|| Ok(token1_clone));
+
+        mock.expect_get_token()
+            .times(1)
+            .return_once(|| Ok(token2_clone));
+
+        mock.expect_get_token()
+            .times(1)
+            .return_once(|| Ok(token3_clone));
+
+        let (tx, mut rx) = watch::channel::<Option<Result<Token>>>(None);
+
+        // check that channel has None before refresh task starts
+        let actual = rx.borrow().clone();
+        assert!(actual.is_none());
+
+        tokio::spawn(async move {
+            refresh_task(mock, tx).await;
+        });
+
+        rx.changed().await.unwrap();
+        let actual = rx.borrow().clone().unwrap().unwrap();
+        assert_eq!(actual, token1.clone());
+
+        // Validate that it is the same token before it expires
+        let sleep = Duration::from_secs(120);
+        tokio::time::advance(sleep).await;
+        let actual = rx.borrow().clone().unwrap().unwrap();
+        assert_eq!(actual, token1.clone());
+
+        // time machine takes execution to 3 minutes before expiry
+        tokio::time::advance(TOKEN_VALID_DURATION.sub(Duration::from_secs(300))).await;
+
+        rx.changed().await.unwrap();
+
+        // validate that the token changed less than 5 mins before expiry
+        assert!(Instant::now() < now + TOKEN_VALID_DURATION);
+        let actual = rx.borrow().clone().unwrap().unwrap();
+        assert_eq!(actual, token2);
+
+        // wait long enough for the token to be expired
+        // Adding 500 secs to account for the time manipulation above
+        let sleep = TOKEN_VALID_DURATION.add(Duration::from_secs(500));
+        tokio::time::advance(sleep).await;
+
+        rx.changed().await.unwrap();
+        let actual = rx.borrow().clone().unwrap().unwrap();
+        assert_eq!(actual, token3);
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn refresh_task_loop_less_expiry() {
+        let now = Instant::now();
+
+        let token1 = Token {
+            token: "token1".to_string(),
+            token_type: "Bearer".to_string(),
+            expires_at: Some((now + Duration::from_secs(120)).into_std()),
+            metadata: None,
+        };
+        let token1_clone = token1.clone();
+
+        let token2 = Token {
+            token: "token2".to_string(),
+            token_type: "Bearer".to_string(),
+            expires_at: Some((now + 2 * TOKEN_VALID_DURATION).into_std()),
+            metadata: None,
+        };
+        let token2_clone = token2.clone();
+
+        let mut mock = MockTokenProvider::new();
+        mock.expect_get_token()
+            .times(1)
+            .return_once(|| Ok(token1_clone));
+
+        mock.expect_get_token()
+            .times(1)
+            .return_once(|| Ok(token2_clone));
+
+        let (tx, mut rx) = watch::channel::<Option<Result<Token>>>(None);
+
+        // check that channel has None before refresh task starts
+        let actual = rx.borrow().clone();
+        assert!(actual.is_none());
+
+        tokio::spawn(async move {
+            refresh_task(mock, tx).await;
+        });
+
+        rx.changed().await.unwrap();
+        let actual = rx.borrow().clone().unwrap().unwrap();
+        assert_eq!(actual, token1);
+
+        // time machine forwards time to 10 seconds before expiry
+        tokio::time::advance(Duration::from_secs(110)).await;
+
+        rx.changed().await.unwrap();
+
+        // validate that the token was refreshed within 10ish seconds
+        // before expiry
+        assert!(Instant::now() < now + Duration::from_secs(111));
+        let actual = rx.borrow().clone().unwrap().unwrap();
+        assert_eq!(actual, token2);
+    }
+
+    #[derive(Clone, Debug)]
+    struct FakeTokenProvider {
+        result: Result<Token>,
+        calls: Arc<std::sync::Mutex<i32>>,
+    }
+
+    impl FakeTokenProvider {
+        pub fn new(result: Result<Token>) -> Self {
+            FakeTokenProvider {
+                result,
+                calls: Arc::new(Mutex::new(0)),
+            }
+        }
+
+        pub fn calls(&self) -> i32 {
+            *self.calls.lock().unwrap()
+        }
+    }
+
+    #[async_trait::async_trait]
+    impl TokenProvider for FakeTokenProvider {
+        async fn get_token(&self) -> Result<Token> {
+            // We give enough time for the a thundering herd to pile up waiting for a change notification from watch channel
+            tokio::time::sleep(Duration::from_millis(50)).await;
+
+            // Track how many calls were made to the inner token provider.
+            *self.calls.lock().unwrap() += 1;
+
+            // Return the result.
+            self.result.clone()
+        }
+    }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 8)]
+    async fn no_initial_token_thundering_herd_success() {
+        let token = Token {
+            token: "delayed-token".to_string(),
+            token_type: "Bearer".to_string(),
+            expires_at: Some(std::time::Instant::now()),
+            metadata: None,
+        };
+
+        let tp = FakeTokenProvider::new(Ok(token.clone()));
+
+        let cache = TokenCache::new(tp.clone());
+
+        // Spawn N tasks, all asking for a token at once.
+        let tasks = (0..100)
+            .map(|_| {
+                let cache_clone = cache.clone();
+                tokio::spawn(async move { cache_clone.get_token().await })
+            })
+            .collect::<Vec<_>>();
+
+        // Wait for the N token requests to complete, verifying the returned token.
+        for task in tasks {
+            let actual = task.await.unwrap();
+            assert!(actual.is_ok(), "{}", actual.err().unwrap());
+            assert_eq!(actual.unwrap(), token);
+        }
+
+        let calls = tp.calls();
+        // Only one call to token provider should have been made
+        assert_eq!(calls, 1);
+    }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 8)]
+    async fn no_initial_token_thundering_herd_failure_shares_error() {
path: src/auth/src/token_cache.rs
user: login: coryan
body: I am not sure this is a useful test, it seems like you are testing `tokio::sync::watch` more than your own code?

created_at: 2025-03-21T17:45:42Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,540 @@
+// Copyright 2025 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::token::{Token, TokenProvider};
+use crate::Result;
+use tokio::sync::watch;
+use tokio::time::{sleep_until, Duration, Instant};
+
+const FIVE_MINUTES: Duration = Duration::from_secs(300);
+const TEN_SECONDS: Duration = Duration::from_secs(10);
+
+#[derive(Debug)]
+pub(crate) struct TokenCache<T>
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    rx_token: watch::Receiver<Option<Result<Token>>>,
+    _phantom: std::marker::PhantomData<T>,
+}
+
+impl<T: TokenProvider> Clone for TokenCache<T> {
+    fn clone(&self) -> TokenCache<T> {
+        TokenCache {
+            rx_token: self.rx_token.clone(),
+            _phantom: std::marker::PhantomData,
+        }
+    }
+}
+
+// TODO(#1552): Use the token cache in all creds
+#[allow(dead_code)]
+impl<T> TokenCache<T>
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    pub(crate) fn new(inner: T) -> Self {
+        let (tx_token, rx_token) = watch::channel::<Option<Result<Token>>>(None);
+
+        tokio::spawn(async move {
+            refresh_task(inner, tx_token).await;
+        });
+
+        Self {
+            rx_token,
+            _phantom: std::marker::PhantomData,
+        }
+    }
+}
+
+#[async_trait::async_trait]
+impl<T> TokenProvider for TokenCache<T>
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    async fn get_token(&self) -> Result<Token> {
+        let mut rx = self.rx_token.clone();
+        let token_result = rx.borrow_and_update().clone();
+
+        if let Some(token_result) = token_result {
+            match token_result {
+                Ok(token) => match token.expires_at {
+                    None => return Ok(token),
+                    Some(e) => {
+                        if e < Instant::now().into_std() {
+                            // Expired token, wait for refresh
+                            return wait_for_next_token(rx).await;
+                        } else {
+                            // valid token
+                            return Ok(token);
+                        }
+                    }
+                },
+                // An error in the result is still a valid result to propagate to the client library
+                Err(e) => Err(e),
+            }
+        } else {
+            wait_for_next_token(rx).await
+        }
+    }
+}
+
+async fn wait_for_next_token(
+    mut rx_token: watch::Receiver<Option<Result<Token>>>,
+) -> Result<Token> {
+    rx_token.changed().await.unwrap();
+    let token_result = rx_token.borrow().clone();
+
+    token_result.expect("There should always be a token or error in the channel after changed()")
+}
+
+async fn refresh_task<T>(token_provider: T, tx_token: watch::Sender<Option<Result<Token>>>)
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    loop {
+        let token_result = token_provider.get_token().await;
+
+        let _ = tx_token.send(Some(token_result.clone()));
+
+        match token_result {
+            Ok(new_token) => {
+                if let Some(expiry) = new_token.expires_at {
+                    let time_until_expiry = expiry.duration_since(Instant::now().into_std());
+
+                    if time_until_expiry > FIVE_MINUTES {
+                        let time_to_sleep = Instant::now() + time_until_expiry - FIVE_MINUTES;
+                        sleep_until(time_to_sleep).await;
+                    } else if time_until_expiry > TEN_SECONDS {
+                        // If expiry is less than 5 mins, wait till last 10 seconds to refresh
+                        // This is to handle cases where MDS returns about to refresh tokens
+                        let time_to_sleep = Instant::now() + time_until_expiry - TEN_SECONDS;
+                        sleep_until(time_to_sleep).await;
+                    }
+                    else{
+                        // We were given a token that is expired, or expires in less than 10 seconds. We will immediately restart the loop, and fetch a new token.
+                    }
+                } else {
+                    // If there is no expiry, the token is valid forever, so no need to refresh
+                    // TODO(#1553): Validate that all auth backends provide expiry and make expiry not optional.
+                    break;
path: src/auth/src/token_cache.rs
user: login: sai-sunder-s
body: That line is covered by `initial_token_success` case.
created_at: 2025-03-21T19:06:11Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,540 @@
+// Copyright 2025 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::token::{Token, TokenProvider};
+use crate::Result;
+use tokio::sync::watch;
+use tokio::time::{sleep_until, Duration, Instant};
+
+const FIVE_MINUTES: Duration = Duration::from_secs(300);
+const TEN_SECONDS: Duration = Duration::from_secs(10);
+
+#[derive(Debug)]
+pub(crate) struct TokenCache<T>
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    rx_token: watch::Receiver<Option<Result<Token>>>,
+    _phantom: std::marker::PhantomData<T>,
+}
+
+impl<T: TokenProvider> Clone for TokenCache<T> {
+    fn clone(&self) -> TokenCache<T> {
+        TokenCache {
+            rx_token: self.rx_token.clone(),
+            _phantom: std::marker::PhantomData,
+        }
+    }
+}
+
+// TODO(#1552): Use the token cache in all creds
+#[allow(dead_code)]
+impl<T> TokenCache<T>
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    pub(crate) fn new(inner: T) -> Self {
+        let (tx_token, rx_token) = watch::channel::<Option<Result<Token>>>(None);
+
+        tokio::spawn(async move {
+            refresh_task(inner, tx_token).await;
+        });
+
+        Self {
+            rx_token,
+            _phantom: std::marker::PhantomData,
+        }
+    }
+}
+
+#[async_trait::async_trait]
+impl<T> TokenProvider for TokenCache<T>
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    async fn get_token(&self) -> Result<Token> {
+        let mut rx = self.rx_token.clone();
+        let token_result = rx.borrow_and_update().clone();
+
+        if let Some(token_result) = token_result {
+            match token_result {
+                Ok(token) => match token.expires_at {
+                    None => return Ok(token),
+                    Some(e) => {
+                        if e < Instant::now().into_std() {
+                            // Expired token, wait for refresh
+                            return wait_for_next_token(rx).await;
+                        } else {
+                            // valid token
+                            return Ok(token);
+                        }
+                    }
+                },
+                // An error in the result is still a valid result to propagate to the client library
+                Err(e) => Err(e),
+            }
+        } else {
+            wait_for_next_token(rx).await
+        }
+    }
+}
+
+async fn wait_for_next_token(
+    mut rx_token: watch::Receiver<Option<Result<Token>>>,
+) -> Result<Token> {
+    rx_token.changed().await.unwrap();
+    let token_result = rx_token.borrow().clone();
+
+    token_result.expect("There should always be a token or error in the channel after changed()")
+}
+
+async fn refresh_task<T>(token_provider: T, tx_token: watch::Sender<Option<Result<Token>>>)
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    loop {
+        let token_result = token_provider.get_token().await;
+
+        let _ = tx_token.send(Some(token_result.clone()));
+
+        match token_result {
+            Ok(new_token) => {
+                if let Some(expiry) = new_token.expires_at {
+                    let time_until_expiry = expiry.duration_since(Instant::now().into_std());
+
+                    if time_until_expiry > FIVE_MINUTES {
+                        let time_to_sleep = Instant::now() + time_until_expiry - FIVE_MINUTES;
+                        sleep_until(time_to_sleep).await;
+                    } else if time_until_expiry > TEN_SECONDS {
+                        // If expiry is less than 5 mins, wait till last 10 seconds to refresh
+                        // This is to handle cases where MDS returns about to refresh tokens
+                        let time_to_sleep = Instant::now() + time_until_expiry - TEN_SECONDS;
+                        sleep_until(time_to_sleep).await;
+                    }
+                    else{
+                        // We were given a token that is expired, or expires in less than 10 seconds. We will immediately restart the loop, and fetch a new token.
+                    }
+                } else {
+                    // If there is no expiry, the token is valid forever, so no need to refresh
+                    // TODO(#1553): Validate that all auth backends provide expiry and make expiry not optional.
+                    break;
+                }
+            }
+            Err(_) => {
+                // The retry policy has been used already by the inner token provider.
+                // If it ended in an error, just quit the background task.
+                break;
+            }
+        }
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    use crate::errors::CredentialError;
+    use crate::token::test::MockTokenProvider;
+    use std::ops::{Add, Sub};
+    use std::sync::{Arc, Mutex};
+    use tokio::time::{Duration, Instant};
+
+    static TOKEN_VALID_DURATION: Duration = Duration::from_secs(3600);
+
+    #[tokio::test]
+    async fn initial_token_success() {
+        let expected = Token {
+            token: "test-token".to_string(),
+            token_type: "Bearer".to_string(),
+            expires_at: None,
+            metadata: None,
+        };
+        let expected_clone = expected.clone();
+
+        let mut mock = MockTokenProvider::new();
+        mock.expect_get_token()
+            .times(1)
+            .return_once(|| Ok(expected_clone));
+
+        let cache = TokenCache::new(mock);
+        let actual = cache.get_token().await.unwrap();
+        assert_eq!(actual, expected);
+
+        // Verify that we use the cached token instead of making a new request
+        // to the mock token provider.
+        let actual = cache.get_token().await.unwrap();
+        assert_eq!(actual, expected);
+    }
+
+    #[tokio::test]
+    async fn initial_token_failure() {
+        let mut mock = MockTokenProvider::new();
+        mock.expect_get_token()
+            .times(1)
+            .returning(|| Err(CredentialError::non_retryable_from_str("fail")));
+
+        let cache = TokenCache::new(mock);
+        assert!(cache.get_token().await.is_err());
+
+        // Verify that a new request is made to the mock token provider when we
+        // don't have a valid token.
+        assert!(cache.get_token().await.is_err());
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn expired_token_success() {
+        let now = Instant::now();
+
+        let initial = Token {
+            token: "initial-token".to_string(),
+            token_type: "Bearer".to_string(),
+            expires_at: Some((now + TOKEN_VALID_DURATION).into_std()),
+            metadata: None,
+        };
+        let initial_clone = initial.clone();
+
+        let refresh = Token {
+            token: "refreshed-token".to_string(),
+            token_type: "Bearer".to_string(),
+            expires_at: Some((now + 2 * TOKEN_VALID_DURATION).into_std()),
+            metadata: None,
+        };
+        let refresh_clone = refresh.clone();
+
+        let mut mock = MockTokenProvider::new();
+        mock.expect_get_token()
+            .times(1)
+            .return_once(|| Ok(initial_clone));
+
+        mock.expect_get_token()
+            .times(1)
+            .return_once(|| Ok(refresh_clone));
+
+        // fetch an initial token
+        let cache = TokenCache::new(mock);
+        let actual = cache.get_token().await.unwrap();
+        assert_eq!(actual, initial);
+
+        // wait long enough for the token to be expired
+        // get_token should be waiting until the new token is available
+        let sleep = TOKEN_VALID_DURATION.add(Duration::from_secs(100));
+        tokio::time::advance(sleep).await;
+
+        // make sure this is the new token
+        let actual = cache.get_token().await.unwrap();
+        assert_eq!(actual, refresh);
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn expired_token_failure() {
+        let now = Instant::now();
+
+        let initial = Token {
+            token: "initial-token".to_string(),
+            token_type: "Bearer".to_string(),
+            expires_at: Some((now + TOKEN_VALID_DURATION).into_std()),
+            metadata: None,
+        };
+        let initial_clone = initial.clone();
+
+        let mut mock = MockTokenProvider::new();
+        mock.expect_get_token()
+            .times(1)
+            .return_once(|| Ok(initial_clone));
+
+        mock.expect_get_token()
+            .times(1)
+            .return_once(|| Err(CredentialError::non_retryable_from_str("fail")));
+
+        // fetch an initial token
+        let cache = TokenCache::new(mock);
+        let actual = cache.get_token().await.unwrap();
+        assert_eq!(actual, initial);
+
+        // wait long enough for the token to be expired
+        let sleep = TOKEN_VALID_DURATION.add(Duration::from_secs(100));
+        tokio::time::advance(sleep).await;
+
+        // make sure we return the error, not the expired token
+        assert!(cache.get_token().await.is_err());
+    }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 8)]
+    async fn token_cache_multiple_requests_existing_valid_token() {
+        let now = Instant::now();
+
+        let token = Token {
+            token: "initial-token".to_string(),
+            token_type: "Bearer".to_string(),
+            expires_at: Some((now + TOKEN_VALID_DURATION).into_std()),
+            metadata: None,
+        };
+        let token_clone = token.clone();
+
+        let mut mock = MockTokenProvider::new();
+        mock.expect_get_token()
+            .times(1)
+            .return_once(|| Ok(token_clone));
+
+        // fetch an initial token
+        let cache = TokenCache::new(mock);
+        let actual = cache.get_token().await.unwrap();
+        assert_eq!(actual, token);
+
+        // Spawn N tasks, all asking for a token at once.
+        let tasks = (0..1000)
+            .map(|_| {
+                let cache_clone = cache.clone();
+                tokio::spawn(async move { cache_clone.get_token().await })
+            })
+            .collect::<Vec<_>>();
+
+        // Wait for the N token requests to complete, verifying the returned token.
+        for task in tasks {
+            let actual = task.await.unwrap();
+            assert!(actual.is_ok(), "{}", actual.err().unwrap());
+            assert_eq!(actual.unwrap(), token);
+        }
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn refresh_task_loop() {
+        let now = Instant::now();
+
+        let token1 = Token {
+            token: "token1".to_string(),
+            token_type: "Bearer".to_string(),
+            expires_at: Some((now + TOKEN_VALID_DURATION).into_std()),
+            metadata: None,
+        };
+        let token1_clone = token1.clone();
+
+        let token2 = Token {
+            token: "token2".to_string(),
+            token_type: "Bearer".to_string(),
+            expires_at: Some((now + 2 * TOKEN_VALID_DURATION).into_std()),
+            metadata: None,
+        };
+        let token2_clone = token2.clone();
+
+        let token3 = Token {
+            token: "token3".to_string(),
+            token_type: "Bearer".to_string(),
+            expires_at: Some((now + 3 * TOKEN_VALID_DURATION).into_std()),
+            metadata: None,
+        };
+        let token3_clone = token3.clone();
+
+        let mut mock = MockTokenProvider::new();
+        mock.expect_get_token()
+            .times(1)
+            .return_once(|| Ok(token1_clone));
+
+        mock.expect_get_token()
+            .times(1)
+            .return_once(|| Ok(token2_clone));
+
+        mock.expect_get_token()
+            .times(1)
+            .return_once(|| Ok(token3_clone));
+
+        let (tx, mut rx) = watch::channel::<Option<Result<Token>>>(None);
+
+        // check that channel has None before refresh task starts
+        let actual = rx.borrow().clone();
+        assert!(actual.is_none());
+
+        tokio::spawn(async move {
+            refresh_task(mock, tx).await;
+        });
+
+        rx.changed().await.unwrap();
+        let actual = rx.borrow().clone().unwrap().unwrap();
+        assert_eq!(actual, token1.clone());
+
+        // Validate that it is the same token before it expires
+        let sleep = Duration::from_secs(120);
+        tokio::time::advance(sleep).await;
+        let actual = rx.borrow().clone().unwrap().unwrap();
+        assert_eq!(actual, token1.clone());
+
+        // time machine takes execution to 3 minutes before expiry
+        tokio::time::advance(TOKEN_VALID_DURATION.sub(Duration::from_secs(300))).await;
+
+        rx.changed().await.unwrap();
+
+        // validate that the token changed less than 5 mins before expiry
+        assert!(Instant::now() < now + TOKEN_VALID_DURATION);
+        let actual = rx.borrow().clone().unwrap().unwrap();
+        assert_eq!(actual, token2);
+
+        // wait long enough for the token to be expired
+        // Adding 500 secs to account for the time manipulation above
+        let sleep = TOKEN_VALID_DURATION.add(Duration::from_secs(500));
+        tokio::time::advance(sleep).await;
+
+        rx.changed().await.unwrap();
+        let actual = rx.borrow().clone().unwrap().unwrap();
+        assert_eq!(actual, token3);
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn refresh_task_loop_less_expiry() {
+        let now = Instant::now();
+
+        let token1 = Token {
+            token: "token1".to_string(),
+            token_type: "Bearer".to_string(),
+            expires_at: Some((now + Duration::from_secs(120)).into_std()),
+            metadata: None,
+        };
+        let token1_clone = token1.clone();
+
+        let token2 = Token {
+            token: "token2".to_string(),
+            token_type: "Bearer".to_string(),
+            expires_at: Some((now + 2 * TOKEN_VALID_DURATION).into_std()),
+            metadata: None,
+        };
+        let token2_clone = token2.clone();
+
+        let mut mock = MockTokenProvider::new();
+        mock.expect_get_token()
+            .times(1)
+            .return_once(|| Ok(token1_clone));
+
+        mock.expect_get_token()
+            .times(1)
+            .return_once(|| Ok(token2_clone));
+
+        let (tx, mut rx) = watch::channel::<Option<Result<Token>>>(None);
+
+        // check that channel has None before refresh task starts
+        let actual = rx.borrow().clone();
+        assert!(actual.is_none());
+
+        tokio::spawn(async move {
+            refresh_task(mock, tx).await;
+        });
+
+        rx.changed().await.unwrap();
+        let actual = rx.borrow().clone().unwrap().unwrap();
+        assert_eq!(actual, token1);
+
+        // time machine forwards time to 10 seconds before expiry
+        tokio::time::advance(Duration::from_secs(110)).await;
+
+        rx.changed().await.unwrap();
+
+        // validate that the token was refreshed within 10ish seconds
+        // before expiry
+        assert!(Instant::now() < now + Duration::from_secs(111));
+        let actual = rx.borrow().clone().unwrap().unwrap();
+        assert_eq!(actual, token2);
+    }
+
+    #[derive(Clone, Debug)]
+    struct FakeTokenProvider {
+        result: Result<Token>,
+        calls: Arc<std::sync::Mutex<i32>>,
+    }
+
+    impl FakeTokenProvider {
+        pub fn new(result: Result<Token>) -> Self {
+            FakeTokenProvider {
+                result,
+                calls: Arc::new(Mutex::new(0)),
+            }
+        }
+
+        pub fn calls(&self) -> i32 {
+            *self.calls.lock().unwrap()
+        }
+    }
+
+    #[async_trait::async_trait]
+    impl TokenProvider for FakeTokenProvider {
+        async fn get_token(&self) -> Result<Token> {
+            // We give enough time for the a thundering herd to pile up waiting for a change notification from watch channel
+            tokio::time::sleep(Duration::from_millis(50)).await;
+
+            // Track how many calls were made to the inner token provider.
+            *self.calls.lock().unwrap() += 1;
+
+            // Return the result.
+            self.result.clone()
+        }
+    }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 8)]
+    async fn no_initial_token_thundering_herd_success() {
+        let token = Token {
+            token: "delayed-token".to_string(),
+            token_type: "Bearer".to_string(),
+            expires_at: Some(std::time::Instant::now()),
+            metadata: None,
+        };
+
+        let tp = FakeTokenProvider::new(Ok(token.clone()));
+
+        let cache = TokenCache::new(tp.clone());
+
+        // Spawn N tasks, all asking for a token at once.
+        let tasks = (0..100)
+            .map(|_| {
+                let cache_clone = cache.clone();
+                tokio::spawn(async move { cache_clone.get_token().await })
+            })
+            .collect::<Vec<_>>();
+
+        // Wait for the N token requests to complete, verifying the returned token.
+        for task in tasks {
+            let actual = task.await.unwrap();
+            assert!(actual.is_ok(), "{}", actual.err().unwrap());
+            assert_eq!(actual.unwrap(), token);
+        }
+
+        let calls = tp.calls();
+        // Only one call to token provider should have been made
+        assert_eq!(calls, 1);
+    }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 8)]
+    async fn no_initial_token_thundering_herd_failure_shares_error() {
path: src/auth/src/token_cache.rs
user: login: sai-sunder-s
body: It also does test that we do not do any call to the token provider other than through the refresh task and also that multiple threads work fine with our implementation.
created_at: 2025-03-21T19:11:54Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,540 @@
+// Copyright 2025 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::token::{Token, TokenProvider};
+use crate::Result;
+use tokio::sync::watch;
+use tokio::time::{sleep_until, Duration, Instant};
+
+const FIVE_MINUTES: Duration = Duration::from_secs(300);
+const TEN_SECONDS: Duration = Duration::from_secs(10);
+
+#[derive(Debug)]
+pub(crate) struct TokenCache<T>
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    rx_token: watch::Receiver<Option<Result<Token>>>,
+    _phantom: std::marker::PhantomData<T>,
+}
+
+impl<T: TokenProvider> Clone for TokenCache<T> {
+    fn clone(&self) -> TokenCache<T> {
+        TokenCache {
+            rx_token: self.rx_token.clone(),
+            _phantom: std::marker::PhantomData,
+        }
+    }
+}
+
+// TODO(#1552): Use the token cache in all creds
+#[allow(dead_code)]
+impl<T> TokenCache<T>
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    pub(crate) fn new(inner: T) -> Self {
+        let (tx_token, rx_token) = watch::channel::<Option<Result<Token>>>(None);
+
+        tokio::spawn(async move {
+            refresh_task(inner, tx_token).await;
+        });
+
+        Self {
+            rx_token,
+            _phantom: std::marker::PhantomData,
+        }
+    }
+}
+
+#[async_trait::async_trait]
+impl<T> TokenProvider for TokenCache<T>
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    async fn get_token(&self) -> Result<Token> {
+        let mut rx = self.rx_token.clone();
+        let token_result = rx.borrow_and_update().clone();
+
+        if let Some(token_result) = token_result {
+            match token_result {
+                Ok(token) => match token.expires_at {
+                    None => return Ok(token),
+                    Some(e) => {
+                        if e < Instant::now().into_std() {
+                            // Expired token, wait for refresh
+                            return wait_for_next_token(rx).await;
+                        } else {
+                            // valid token
+                            return Ok(token);
+                        }
+                    }
+                },
+                // An error in the result is still a valid result to propagate to the client library
+                Err(e) => Err(e),
+            }
+        } else {
+            wait_for_next_token(rx).await
+        }
+    }
+}
+
+async fn wait_for_next_token(
+    mut rx_token: watch::Receiver<Option<Result<Token>>>,
+) -> Result<Token> {
+    rx_token.changed().await.unwrap();
+    let token_result = rx_token.borrow().clone();
+
+    token_result.expect("There should always be a token or error in the channel after changed()")
+}
+
+async fn refresh_task<T>(token_provider: T, tx_token: watch::Sender<Option<Result<Token>>>)
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    loop {
+        let token_result = token_provider.get_token().await;
+
+        let _ = tx_token.send(Some(token_result.clone()));
+
+        match token_result {
+            Ok(new_token) => {
+                if let Some(expiry) = new_token.expires_at {
+                    let time_until_expiry = expiry.duration_since(Instant::now().into_std());
+
+                    if time_until_expiry > FIVE_MINUTES {
+                        let time_to_sleep = Instant::now() + time_until_expiry - FIVE_MINUTES;
+                        sleep_until(time_to_sleep).await;
+                    } else if time_until_expiry > TEN_SECONDS {
+                        // If expiry is less than 5 mins, wait till last 10 seconds to refresh
+                        // This is to handle cases where MDS returns about to refresh tokens
+                        let time_to_sleep = Instant::now() + time_until_expiry - TEN_SECONDS;
path: src/auth/src/token_cache.rs
user: login: sai-sunder-s
body: updated as per offline discussion.
created_at: 2025-03-21T19:44:44Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,580 @@
+// Copyright 2025 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::Result;
+use crate::token::{Token, TokenProvider};
+use tokio::sync::watch;
+use tokio::time::{Duration, Instant, sleep, sleep_until};
+
+const NORMAL_REFRESH_SLACK: Duration = Duration::from_secs(240);
+const SHORT_REFRESH_SLACK: Duration = Duration::from_secs(10);
+
+#[derive(Debug, Clone)]
+pub(crate) struct TokenCache {
+    rx_token: watch::Receiver<Option<Result<Token>>>,
+}
+
+// TODO(#1552): Use the token cache in all creds
+#[allow(dead_code)]
+impl TokenCache {
+    pub(crate) fn new<T>(inner: T) -> Self
+    where
+        T: TokenProvider + Send + Sync + 'static,
+    {
+        let (tx_token, rx_token) = watch::channel::<Option<Result<Token>>>(None);
+
+        tokio::spawn(async move {
+            refresh_task(inner, tx_token).await;
+        });
+
+        Self { rx_token }
+    }
+}
+
+#[async_trait::async_trait]
+impl TokenProvider for TokenCache {
+    async fn get_token(&self) -> Result<Token> {
+        let mut rx = self.rx_token.clone();
+        let token_result = rx.borrow_and_update().clone();
+
+        if let Some(token_result) = token_result {
+            match token_result {
+                Ok(token) => match token.expires_at {
+                    None => return Ok(token),
+                    Some(e) => {
+                        if e < Instant::now().into_std() {
+                            // Expired token, wait for refresh
+                            return wait_for_next_token(rx).await;
+                        } else {
+                            // valid token
+                            return Ok(token);
+                        }
+                    }
+                },
+                // An error in the result is still a valid result to propagate to the client library
+                Err(e) => Err(e),
+            }
+        } else {
+            wait_for_next_token(rx).await
+        }
+    }
+}
+
+async fn wait_for_next_token(
+    mut rx_token: watch::Receiver<Option<Result<Token>>>,
+) -> Result<Token> {
+    rx_token.changed().await.unwrap();
+    let token_result = rx_token.borrow().clone();
+
+    token_result.expect("There should always be a token or error in the channel after changed()")
+}
+
+async fn refresh_task<T>(token_provider: T, tx_token: watch::Sender<Option<Result<Token>>>)
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    loop {
+        let token_result = token_provider.get_token().await;
+
+        let _ = tx_token.send(Some(token_result.clone()));
+
+        match token_result {
+            Ok(new_token) => {
+                if let Some(expiry) = new_token.expires_at {
+                    let time_until_expiry = expiry.duration_since(Instant::now().into_std());
+
+                    if time_until_expiry > NORMAL_REFRESH_SLACK {
+                        let time_to_sleep =
+                            Instant::now() + time_until_expiry - NORMAL_REFRESH_SLACK;
+                        sleep_until(time_to_sleep).await;
+                    } else if time_until_expiry > SHORT_REFRESH_SLACK {
+                        // If expiry is less than 4 mins, try to refresh every 10 seconds
+                        // This is to handle cases where MDS **repeatedly** returns about to expire tokens
path: src/auth/src/token_cache.rs
user: login: coryan
body: nit: s/tokens/tokens./
created_at: 2025-03-21T19:53:27Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,580 @@
+// Copyright 2025 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::Result;
+use crate::token::{Token, TokenProvider};
+use tokio::sync::watch;
+use tokio::time::{Duration, Instant, sleep, sleep_until};
+
+const NORMAL_REFRESH_SLACK: Duration = Duration::from_secs(240);
path: src/auth/src/token_cache.rs
user: login: coryan
body: Please document why this number is right, a go/ link to an internal document or a b/ link is fine.  We just don't want future developers to wonder about why 4m which is somewhat unusual.

created_at: 2025-03-21T19:55:13Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,540 @@
+// Copyright 2025 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::token::{Token, TokenProvider};
+use crate::Result;
+use tokio::sync::watch;
+use tokio::time::{sleep_until, Duration, Instant};
+
+const FIVE_MINUTES: Duration = Duration::from_secs(300);
+const TEN_SECONDS: Duration = Duration::from_secs(10);
+
+#[derive(Debug)]
+pub(crate) struct TokenCache<T>
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    rx_token: watch::Receiver<Option<Result<Token>>>,
+    _phantom: std::marker::PhantomData<T>,
+}
+
+impl<T: TokenProvider> Clone for TokenCache<T> {
+    fn clone(&self) -> TokenCache<T> {
+        TokenCache {
+            rx_token: self.rx_token.clone(),
+            _phantom: std::marker::PhantomData,
+        }
+    }
+}
+
+// TODO(#1552): Use the token cache in all creds
+#[allow(dead_code)]
+impl<T> TokenCache<T>
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    pub(crate) fn new(inner: T) -> Self {
+        let (tx_token, rx_token) = watch::channel::<Option<Result<Token>>>(None);
+
+        tokio::spawn(async move {
+            refresh_task(inner, tx_token).await;
+        });
+
+        Self {
+            rx_token,
+            _phantom: std::marker::PhantomData,
+        }
+    }
+}
+
+#[async_trait::async_trait]
+impl<T> TokenProvider for TokenCache<T>
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    async fn get_token(&self) -> Result<Token> {
+        let mut rx = self.rx_token.clone();
+        let token_result = rx.borrow_and_update().clone();
+
+        if let Some(token_result) = token_result {
+            match token_result {
+                Ok(token) => match token.expires_at {
+                    None => return Ok(token),
+                    Some(e) => {
+                        if e < Instant::now().into_std() {
+                            // Expired token, wait for refresh
+                            return wait_for_next_token(rx).await;
+                        } else {
+                            // valid token
+                            return Ok(token);
+                        }
+                    }
+                },
+                // An error in the result is still a valid result to propagate to the client library
+                Err(e) => Err(e),
+            }
+        } else {
+            wait_for_next_token(rx).await
+        }
+    }
+}
+
+async fn wait_for_next_token(
+    mut rx_token: watch::Receiver<Option<Result<Token>>>,
+) -> Result<Token> {
+    rx_token.changed().await.unwrap();
+    let token_result = rx_token.borrow().clone();
+
+    token_result.expect("There should always be a token or error in the channel after changed()")
+}
+
+async fn refresh_task<T>(token_provider: T, tx_token: watch::Sender<Option<Result<Token>>>)
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    loop {
+        let token_result = token_provider.get_token().await;
+
+        let _ = tx_token.send(Some(token_result.clone()));
+
+        match token_result {
+            Ok(new_token) => {
+                if let Some(expiry) = new_token.expires_at {
+                    let time_until_expiry = expiry.duration_since(Instant::now().into_std());
+
+                    if time_until_expiry > FIVE_MINUTES {
+                        let time_to_sleep = Instant::now() + time_until_expiry - FIVE_MINUTES;
+                        sleep_until(time_to_sleep).await;
+                    } else if time_until_expiry > TEN_SECONDS {
+                        // If expiry is less than 5 mins, wait till last 10 seconds to refresh
+                        // This is to handle cases where MDS returns about to refresh tokens
+                        let time_to_sleep = Instant::now() + time_until_expiry - TEN_SECONDS;
+                        sleep_until(time_to_sleep).await;
+                    }
+                    else{
+                        // We were given a token that is expired, or expires in less than 10 seconds. We will immediately restart the loop, and fetch a new token.
+                    }
+                } else {
+                    // If there is no expiry, the token is valid forever, so no need to refresh
+                    // TODO(#1553): Validate that all auth backends provide expiry and make expiry not optional.
+                    break;
+                }
+            }
+            Err(_) => {
+                // The retry policy has been used already by the inner token provider.
+                // If it ended in an error, just quit the background task.
+                break;
+            }
+        }
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    use crate::errors::CredentialError;
+    use crate::token::test::MockTokenProvider;
+    use std::ops::{Add, Sub};
+    use std::sync::{Arc, Mutex};
+    use tokio::time::{Duration, Instant};
+
+    static TOKEN_VALID_DURATION: Duration = Duration::from_secs(3600);
+
+    #[tokio::test]
+    async fn initial_token_success() {
+        let expected = Token {
+            token: "test-token".to_string(),
+            token_type: "Bearer".to_string(),
+            expires_at: None,
+            metadata: None,
+        };
+        let expected_clone = expected.clone();
+
+        let mut mock = MockTokenProvider::new();
+        mock.expect_get_token()
+            .times(1)
+            .return_once(|| Ok(expected_clone));
+
+        let cache = TokenCache::new(mock);
+        let actual = cache.get_token().await.unwrap();
+        assert_eq!(actual, expected);
+
+        // Verify that we use the cached token instead of making a new request
+        // to the mock token provider.
+        let actual = cache.get_token().await.unwrap();
+        assert_eq!(actual, expected);
+    }
+
+    #[tokio::test]
+    async fn initial_token_failure() {
+        let mut mock = MockTokenProvider::new();
+        mock.expect_get_token()
+            .times(1)
+            .returning(|| Err(CredentialError::non_retryable_from_str("fail")));
+
+        let cache = TokenCache::new(mock);
+        assert!(cache.get_token().await.is_err());
+
+        // Verify that a new request is made to the mock token provider when we
+        // don't have a valid token.
+        assert!(cache.get_token().await.is_err());
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn expired_token_success() {
+        let now = Instant::now();
+
+        let initial = Token {
+            token: "initial-token".to_string(),
+            token_type: "Bearer".to_string(),
+            expires_at: Some((now + TOKEN_VALID_DURATION).into_std()),
+            metadata: None,
+        };
+        let initial_clone = initial.clone();
+
+        let refresh = Token {
+            token: "refreshed-token".to_string(),
+            token_type: "Bearer".to_string(),
+            expires_at: Some((now + 2 * TOKEN_VALID_DURATION).into_std()),
+            metadata: None,
+        };
+        let refresh_clone = refresh.clone();
+
+        let mut mock = MockTokenProvider::new();
+        mock.expect_get_token()
+            .times(1)
+            .return_once(|| Ok(initial_clone));
+
+        mock.expect_get_token()
+            .times(1)
+            .return_once(|| Ok(refresh_clone));
+
+        // fetch an initial token
+        let cache = TokenCache::new(mock);
+        let actual = cache.get_token().await.unwrap();
+        assert_eq!(actual, initial);
+
+        // wait long enough for the token to be expired
+        // get_token should be waiting until the new token is available
+        let sleep = TOKEN_VALID_DURATION.add(Duration::from_secs(100));
+        tokio::time::advance(sleep).await;
+
+        // make sure this is the new token
+        let actual = cache.get_token().await.unwrap();
+        assert_eq!(actual, refresh);
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn expired_token_failure() {
+        let now = Instant::now();
+
+        let initial = Token {
+            token: "initial-token".to_string(),
+            token_type: "Bearer".to_string(),
+            expires_at: Some((now + TOKEN_VALID_DURATION).into_std()),
+            metadata: None,
+        };
+        let initial_clone = initial.clone();
+
+        let mut mock = MockTokenProvider::new();
+        mock.expect_get_token()
+            .times(1)
+            .return_once(|| Ok(initial_clone));
+
+        mock.expect_get_token()
+            .times(1)
+            .return_once(|| Err(CredentialError::non_retryable_from_str("fail")));
+
+        // fetch an initial token
+        let cache = TokenCache::new(mock);
+        let actual = cache.get_token().await.unwrap();
+        assert_eq!(actual, initial);
+
+        // wait long enough for the token to be expired
+        let sleep = TOKEN_VALID_DURATION.add(Duration::from_secs(100));
+        tokio::time::advance(sleep).await;
+
+        // make sure we return the error, not the expired token
+        assert!(cache.get_token().await.is_err());
+    }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 8)]
+    async fn token_cache_multiple_requests_existing_valid_token() {
+        let now = Instant::now();
+
+        let token = Token {
+            token: "initial-token".to_string(),
+            token_type: "Bearer".to_string(),
+            expires_at: Some((now + TOKEN_VALID_DURATION).into_std()),
+            metadata: None,
+        };
+        let token_clone = token.clone();
+
+        let mut mock = MockTokenProvider::new();
+        mock.expect_get_token()
+            .times(1)
+            .return_once(|| Ok(token_clone));
+
+        // fetch an initial token
+        let cache = TokenCache::new(mock);
+        let actual = cache.get_token().await.unwrap();
+        assert_eq!(actual, token);
+
+        // Spawn N tasks, all asking for a token at once.
+        let tasks = (0..1000)
+            .map(|_| {
+                let cache_clone = cache.clone();
+                tokio::spawn(async move { cache_clone.get_token().await })
+            })
+            .collect::<Vec<_>>();
+
+        // Wait for the N token requests to complete, verifying the returned token.
+        for task in tasks {
+            let actual = task.await.unwrap();
+            assert!(actual.is_ok(), "{}", actual.err().unwrap());
+            assert_eq!(actual.unwrap(), token);
+        }
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn refresh_task_loop() {
+        let now = Instant::now();
+
+        let token1 = Token {
+            token: "token1".to_string(),
+            token_type: "Bearer".to_string(),
+            expires_at: Some((now + TOKEN_VALID_DURATION).into_std()),
+            metadata: None,
+        };
+        let token1_clone = token1.clone();
+
+        let token2 = Token {
+            token: "token2".to_string(),
+            token_type: "Bearer".to_string(),
+            expires_at: Some((now + 2 * TOKEN_VALID_DURATION).into_std()),
+            metadata: None,
+        };
+        let token2_clone = token2.clone();
+
+        let token3 = Token {
+            token: "token3".to_string(),
+            token_type: "Bearer".to_string(),
+            expires_at: Some((now + 3 * TOKEN_VALID_DURATION).into_std()),
+            metadata: None,
+        };
+        let token3_clone = token3.clone();
+
+        let mut mock = MockTokenProvider::new();
+        mock.expect_get_token()
+            .times(1)
+            .return_once(|| Ok(token1_clone));
+
+        mock.expect_get_token()
+            .times(1)
+            .return_once(|| Ok(token2_clone));
+
+        mock.expect_get_token()
+            .times(1)
+            .return_once(|| Ok(token3_clone));
+
+        let (tx, mut rx) = watch::channel::<Option<Result<Token>>>(None);
+
+        // check that channel has None before refresh task starts
+        let actual = rx.borrow().clone();
+        assert!(actual.is_none());
+
+        tokio::spawn(async move {
+            refresh_task(mock, tx).await;
+        });
+
+        rx.changed().await.unwrap();
+        let actual = rx.borrow().clone().unwrap().unwrap();
+        assert_eq!(actual, token1.clone());
+
+        // Validate that it is the same token before it expires
+        let sleep = Duration::from_secs(120);
+        tokio::time::advance(sleep).await;
+        let actual = rx.borrow().clone().unwrap().unwrap();
+        assert_eq!(actual, token1.clone());
+
+        // time machine takes execution to 3 minutes before expiry
+        tokio::time::advance(TOKEN_VALID_DURATION.sub(Duration::from_secs(300))).await;
+
+        rx.changed().await.unwrap();
+
+        // validate that the token changed less than 5 mins before expiry
+        assert!(Instant::now() < now + TOKEN_VALID_DURATION);
+        let actual = rx.borrow().clone().unwrap().unwrap();
+        assert_eq!(actual, token2);
+
+        // wait long enough for the token to be expired
+        // Adding 500 secs to account for the time manipulation above
+        let sleep = TOKEN_VALID_DURATION.add(Duration::from_secs(500));
+        tokio::time::advance(sleep).await;
+
+        rx.changed().await.unwrap();
+        let actual = rx.borrow().clone().unwrap().unwrap();
+        assert_eq!(actual, token3);
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn refresh_task_loop_less_expiry() {
+        let now = Instant::now();
+
+        let token1 = Token {
+            token: "token1".to_string(),
+            token_type: "Bearer".to_string(),
+            expires_at: Some((now + Duration::from_secs(120)).into_std()),
+            metadata: None,
+        };
+        let token1_clone = token1.clone();
+
+        let token2 = Token {
+            token: "token2".to_string(),
+            token_type: "Bearer".to_string(),
+            expires_at: Some((now + 2 * TOKEN_VALID_DURATION).into_std()),
+            metadata: None,
+        };
+        let token2_clone = token2.clone();
+
+        let mut mock = MockTokenProvider::new();
+        mock.expect_get_token()
+            .times(1)
+            .return_once(|| Ok(token1_clone));
+
+        mock.expect_get_token()
+            .times(1)
+            .return_once(|| Ok(token2_clone));
+
+        let (tx, mut rx) = watch::channel::<Option<Result<Token>>>(None);
+
+        // check that channel has None before refresh task starts
+        let actual = rx.borrow().clone();
+        assert!(actual.is_none());
+
+        tokio::spawn(async move {
+            refresh_task(mock, tx).await;
+        });
+
+        rx.changed().await.unwrap();
+        let actual = rx.borrow().clone().unwrap().unwrap();
+        assert_eq!(actual, token1);
+
+        // time machine forwards time to 10 seconds before expiry
+        tokio::time::advance(Duration::from_secs(110)).await;
+
+        rx.changed().await.unwrap();
+
+        // validate that the token was refreshed within 10ish seconds
+        // before expiry
+        assert!(Instant::now() < now + Duration::from_secs(111));
+        let actual = rx.borrow().clone().unwrap().unwrap();
+        assert_eq!(actual, token2);
+    }
+
+    #[derive(Clone, Debug)]
+    struct FakeTokenProvider {
+        result: Result<Token>,
+        calls: Arc<std::sync::Mutex<i32>>,
+    }
+
+    impl FakeTokenProvider {
+        pub fn new(result: Result<Token>) -> Self {
+            FakeTokenProvider {
+                result,
+                calls: Arc::new(Mutex::new(0)),
+            }
+        }
+
+        pub fn calls(&self) -> i32 {
+            *self.calls.lock().unwrap()
+        }
+    }
+
+    #[async_trait::async_trait]
+    impl TokenProvider for FakeTokenProvider {
+        async fn get_token(&self) -> Result<Token> {
+            // We give enough time for the a thundering herd to pile up waiting for a change notification from watch channel
+            tokio::time::sleep(Duration::from_millis(50)).await;
+
+            // Track how many calls were made to the inner token provider.
+            *self.calls.lock().unwrap() += 1;
+
+            // Return the result.
+            self.result.clone()
+        }
+    }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 8)]
+    async fn no_initial_token_thundering_herd_success() {
+        let token = Token {
+            token: "delayed-token".to_string(),
+            token_type: "Bearer".to_string(),
+            expires_at: Some(std::time::Instant::now()),
+            metadata: None,
+        };
+
+        let tp = FakeTokenProvider::new(Ok(token.clone()));
+
+        let cache = TokenCache::new(tp.clone());
+
+        // Spawn N tasks, all asking for a token at once.
+        let tasks = (0..100)
+            .map(|_| {
+                let cache_clone = cache.clone();
+                tokio::spawn(async move { cache_clone.get_token().await })
+            })
+            .collect::<Vec<_>>();
+
+        // Wait for the N token requests to complete, verifying the returned token.
+        for task in tasks {
+            let actual = task.await.unwrap();
+            assert!(actual.is_ok(), "{}", actual.err().unwrap());
+            assert_eq!(actual.unwrap(), token);
+        }
+
+        let calls = tp.calls();
+        // Only one call to token provider should have been made
+        assert_eq!(calls, 1);
+    }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 8)]
+    async fn no_initial_token_thundering_herd_failure_shares_error() {
path: src/auth/src/token_cache.rs
user: login: coryan
body: > and also that multiple threads work fine with our implementation.

Beware: tests can only show that synchronization is incorrect, not that it "works fine".  We still show write them, but be aware of their limitations.

created_at: 2025-03-21T19:57:57Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,580 @@
+// Copyright 2025 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::Result;
+use crate::token::{Token, TokenProvider};
+use tokio::sync::watch;
+use tokio::time::{Duration, Instant, sleep, sleep_until};
+
+const NORMAL_REFRESH_SLACK: Duration = Duration::from_secs(240);
+const SHORT_REFRESH_SLACK: Duration = Duration::from_secs(10);
+
+#[derive(Debug, Clone)]
+pub(crate) struct TokenCache {
+    rx_token: watch::Receiver<Option<Result<Token>>>,
+}
+
+// TODO(#1552): Use the token cache in all creds
+#[allow(dead_code)]
+impl TokenCache {
+    pub(crate) fn new<T>(inner: T) -> Self
+    where
+        T: TokenProvider + Send + Sync + 'static,
+    {
+        let (tx_token, rx_token) = watch::channel::<Option<Result<Token>>>(None);
+
+        tokio::spawn(async move {
+            refresh_task(inner, tx_token).await;
+        });
+
+        Self { rx_token }
+    }
+}
+
+#[async_trait::async_trait]
+impl TokenProvider for TokenCache {
+    async fn get_token(&self) -> Result<Token> {
+        let mut rx = self.rx_token.clone();
+        let token_result = rx.borrow_and_update().clone();
+
+        if let Some(token_result) = token_result {
+            match token_result {
+                Ok(token) => match token.expires_at {
+                    None => return Ok(token),
+                    Some(e) => {
+                        if e < Instant::now().into_std() {
+                            // Expired token, wait for refresh
+                            return wait_for_next_token(rx).await;
+                        } else {
+                            // valid token
+                            return Ok(token);
+                        }
+                    }
+                },
+                // An error in the result is still a valid result to propagate to the client library
+                Err(e) => Err(e),
+            }
+        } else {
+            wait_for_next_token(rx).await
+        }
+    }
+}
+
+async fn wait_for_next_token(
+    mut rx_token: watch::Receiver<Option<Result<Token>>>,
+) -> Result<Token> {
+    rx_token.changed().await.unwrap();
+    let token_result = rx_token.borrow().clone();
+
+    token_result.expect("There should always be a token or error in the channel after changed()")
+}
+
+async fn refresh_task<T>(token_provider: T, tx_token: watch::Sender<Option<Result<Token>>>)
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    loop {
+        let token_result = token_provider.get_token().await;
+
+        let _ = tx_token.send(Some(token_result.clone()));
+
+        match token_result {
+            Ok(new_token) => {
+                if let Some(expiry) = new_token.expires_at {
+                    let time_until_expiry = expiry.duration_since(Instant::now().into_std());
path: src/auth/src/token_cache.rs
user: login: coryan
body: I should have noticed this earlier. Here you  need: https://doc.rust-lang.org/std/time/struct.Instant.html#method.checked_duration_since
created_at: 2025-03-21T20:07:37Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,580 @@
+// Copyright 2025 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::Result;
+use crate::token::{Token, TokenProvider};
+use tokio::sync::watch;
+use tokio::time::{Duration, Instant, sleep, sleep_until};
+
+const NORMAL_REFRESH_SLACK: Duration = Duration::from_secs(240);
+const SHORT_REFRESH_SLACK: Duration = Duration::from_secs(10);
+
+#[derive(Debug, Clone)]
+pub(crate) struct TokenCache {
+    rx_token: watch::Receiver<Option<Result<Token>>>,
+}
+
+// TODO(#1552): Use the token cache in all creds
+#[allow(dead_code)]
+impl TokenCache {
+    pub(crate) fn new<T>(inner: T) -> Self
+    where
+        T: TokenProvider + Send + Sync + 'static,
+    {
+        let (tx_token, rx_token) = watch::channel::<Option<Result<Token>>>(None);
+
+        tokio::spawn(async move {
+            refresh_task(inner, tx_token).await;
+        });
+
+        Self { rx_token }
+    }
+}
+
+#[async_trait::async_trait]
+impl TokenProvider for TokenCache {
+    async fn get_token(&self) -> Result<Token> {
+        let mut rx = self.rx_token.clone();
+        let token_result = rx.borrow_and_update().clone();
+
+        if let Some(token_result) = token_result {
+            match token_result {
+                Ok(token) => match token.expires_at {
+                    None => return Ok(token),
+                    Some(e) => {
+                        if e < Instant::now().into_std() {
+                            // Expired token, wait for refresh
+                            return wait_for_next_token(rx).await;
+                        } else {
+                            // valid token
+                            return Ok(token);
+                        }
+                    }
+                },
+                // An error in the result is still a valid result to propagate to the client library
+                Err(e) => Err(e),
+            }
+        } else {
+            wait_for_next_token(rx).await
+        }
+    }
+}
+
+async fn wait_for_next_token(
+    mut rx_token: watch::Receiver<Option<Result<Token>>>,
+) -> Result<Token> {
+    rx_token.changed().await.unwrap();
+    let token_result = rx_token.borrow().clone();
+
+    token_result.expect("There should always be a token or error in the channel after changed()")
+}
+
+async fn refresh_task<T>(token_provider: T, tx_token: watch::Sender<Option<Result<Token>>>)
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    loop {
+        let token_result = token_provider.get_token().await;
+
+        let _ = tx_token.send(Some(token_result.clone()));
+
+        match token_result {
+            Ok(new_token) => {
+                if let Some(expiry) = new_token.expires_at {
+                    let time_until_expiry = expiry.duration_since(Instant::now().into_std());
+
+                    if time_until_expiry > NORMAL_REFRESH_SLACK {
+                        let time_to_sleep =
+                            Instant::now() + time_until_expiry - NORMAL_REFRESH_SLACK;
+                        sleep_until(time_to_sleep).await;
path: src/auth/src/token_cache.rs
user: login: coryan
body: Simplify to:

```suggestion
                        sleep(time_until_expiry - NORMAL_REFRESH_SLACK).await;
```

created_at: 2025-03-21T20:08:26Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,580 @@
+// Copyright 2025 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::Result;
+use crate::token::{Token, TokenProvider};
+use tokio::sync::watch;
+use tokio::time::{Duration, Instant, sleep, sleep_until};
+
+const NORMAL_REFRESH_SLACK: Duration = Duration::from_secs(240);
path: src/auth/src/token_cache.rs
user: login: sai-sunder-s
body: Added comment
created_at: 2025-03-21T21:15:36Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,594 @@
+// Copyright 2025 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::Result;
+use crate::token::{Token, TokenProvider};
+use tokio::sync::watch;
+use tokio::time::{Duration, Instant, sleep};
+
+// Different MDS(Metadata Service) backends have different policies to
+// determine when to refresh a token. Most MDS' refresh token 5 mins before
+// expiry, except for Serverless which refresh tokens 4 mins before
+// expiry. So we are using 4 mins as the staleness limit for our refresh logic.
+const NORMAL_REFRESH_SLACK: Duration = Duration::from_secs(240);
+const SHORT_REFRESH_SLACK: Duration = Duration::from_secs(10);
+
+#[derive(Debug, Clone)]
+pub(crate) struct TokenCache {
+    rx_token: watch::Receiver<Option<Result<Token>>>,
+}
+
+// TODO(#1552): Use the token cache in all creds
+#[allow(dead_code)]
+impl TokenCache {
+    pub(crate) fn new<T>(inner: T) -> Self
+    where
+        T: TokenProvider + Send + Sync + 'static,
+    {
+        let (tx_token, rx_token) = watch::channel::<Option<Result<Token>>>(None);
+
+        tokio::spawn(async move {
+            refresh_task(inner, tx_token).await;
+        });
+
+        Self { rx_token }
+    }
+}
+
+#[async_trait::async_trait]
+impl TokenProvider for TokenCache {
+    async fn get_token(&self) -> Result<Token> {
+        let mut rx = self.rx_token.clone();
+        let token_result = rx.borrow_and_update().clone();
+
+        if let Some(token_result) = token_result {
+            match token_result {
+                Ok(token) => match token.expires_at {
+                    None => return Ok(token),
+                    Some(e) => {
+                        if e < Instant::now().into_std() {
+                            // Expired token, wait for refresh
+                            return wait_for_next_token(rx).await;
+                        } else {
+                            // valid token
+                            return Ok(token);
+                        }
+                    }
+                },
+                // An error in the result is still a valid result to propagate to the client library
+                Err(e) => Err(e),
+            }
+        } else {
+            wait_for_next_token(rx).await
+        }
+    }
+}
+
+async fn wait_for_next_token(
+    mut rx_token: watch::Receiver<Option<Result<Token>>>,
+) -> Result<Token> {
+    rx_token.changed().await.unwrap();
+    let token_result = rx_token.borrow().clone();
+
+    token_result.expect("There should always be a token or error in the channel after changed()")
+}
+
+async fn refresh_task<T>(token_provider: T, tx_token: watch::Sender<Option<Result<Token>>>)
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    loop {
+        let token_result = token_provider.get_token().await;
+
+        let _ = tx_token.send(Some(token_result.clone()));
+
+        match token_result {
+            Ok(new_token) => {
+                if let Some(expiry) = new_token.expires_at {
+                    let time_until_expiry =
+                        expiry.checked_duration_since(Instant::now().into_std());
+
+                    match time_until_expiry {
+                        None => {
+                            // We were given a token that is expired, or expires in less than 10 seconds.
+                            // We will immediately restart the loop, and fetch a new token.
+                        }
+                        Some(time_until_expiry) => {
+                            if time_until_expiry > NORMAL_REFRESH_SLACK {
+                                sleep(time_until_expiry - NORMAL_REFRESH_SLACK).await;
+                            } else if time_until_expiry > SHORT_REFRESH_SLACK {
+                                // If expiry is less than 4 mins, try to refresh every 10 seconds
+                                // This is to handle cases where MDS **repeatedly** returns about to expire tokens.
+                                sleep(SHORT_REFRESH_SLACK).await;
+                            }
+                        }
+                    }
+                } else {
+                    // If there is no expiry, the token is valid forever, so no need to refresh
+                    // TODO(#1553): Validate that all auth backends provide expiry and make expiry not optional.
+                    break;
+                }
+            }
+            Err(_) => {
+                // The retry policy has been used already by the inner token provider.
+                // If it ended in an error, just quit the background task.
+                break;
+            }
+        }
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    use crate::errors::CredentialError;
+    use crate::token::test::MockTokenProvider;
+    use std::ops::{Add, Sub};
+    use std::sync::{Arc, Mutex};
+    use tokio::time::{Duration, Instant};
+
+    static TOKEN_VALID_DURATION: Duration = Duration::from_secs(3600);
+
+    #[tokio::test]
+    async fn initial_token_success() {
+        let expected = Token {
+            token: "test-token".to_string(),
+            token_type: "Bearer".to_string(),
+            expires_at: None,
+            metadata: None,
+        };
+        let expected_clone = expected.clone();
+
+        let mut mock = MockTokenProvider::new();
+        mock.expect_get_token()
+            .times(1)
+            .return_once(|| Ok(expected_clone));
+
+        let cache = TokenCache::new(mock);
+        let actual = cache.get_token().await.unwrap();
+        assert_eq!(actual, expected);
+
+        // Verify that we use the cached token instead of making a new request
+        // to the mock token provider.
+        let actual = cache.get_token().await.unwrap();
+        assert_eq!(actual, expected);
+    }
+
+    #[tokio::test]
+    async fn initial_token_failure() {
+        let mut mock = MockTokenProvider::new();
+        mock.expect_get_token()
+            .times(1)
+            .returning(|| Err(CredentialError::non_retryable_from_str("fail")));
+
+        let cache = TokenCache::new(mock);
+        assert!(cache.get_token().await.is_err());
+
+        // Verify that a new request is made to the mock token provider when we
+        // don't have a valid token.
+        assert!(cache.get_token().await.is_err());
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn expired_token_success() {
+        let now = Instant::now();
+
+        let initial = Token {
+            token: "initial-token".to_string(),
+            token_type: "Bearer".to_string(),
+            expires_at: Some((now + TOKEN_VALID_DURATION).into_std()),
+            metadata: None,
+        };
+        let initial_clone = initial.clone();
+
+        let refresh = Token {
+            token: "refreshed-token".to_string(),
+            token_type: "Bearer".to_string(),
+            expires_at: Some((now + 2 * TOKEN_VALID_DURATION).into_std()),
+            metadata: None,
+        };
+        let refresh_clone = refresh.clone();
+
+        let mut mock = MockTokenProvider::new();
+        mock.expect_get_token()
+            .times(1)
+            .return_once(|| Ok(initial_clone));
+
+        mock.expect_get_token()
+            .times(1)
+            .return_once(|| Ok(refresh_clone));
+
+        // fetch an initial token
+        let cache = TokenCache::new(mock);
+        let actual = cache.get_token().await.unwrap();
+        assert_eq!(actual, initial);
+
+        // wait long enough for the token to be expired
+        // get_token should be waiting until the new token is available
+        let sleep = TOKEN_VALID_DURATION.add(Duration::from_secs(100));
+        tokio::time::advance(sleep).await;
+
+        // make sure this is the new token
+        let actual = cache.get_token().await.unwrap();
+        assert_eq!(actual, refresh);
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn expired_token_failure() {
+        let now = Instant::now();
+
+        let initial = Token {
+            token: "initial-token".to_string(),
+            token_type: "Bearer".to_string(),
+            expires_at: Some((now + TOKEN_VALID_DURATION).into_std()),
+            metadata: None,
+        };
+        let initial_clone = initial.clone();
+
+        let mut mock = MockTokenProvider::new();
+        mock.expect_get_token()
+            .times(1)
+            .return_once(|| Ok(initial_clone));
+
+        mock.expect_get_token()
+            .times(1)
+            .return_once(|| Err(CredentialError::non_retryable_from_str("fail")));
+
+        // fetch an initial token
+        let cache = TokenCache::new(mock);
+        let actual = cache.get_token().await.unwrap();
+        assert_eq!(actual, initial);
+
+        // wait long enough for the token to be expired
+        let sleep = TOKEN_VALID_DURATION.add(Duration::from_secs(100));
+        tokio::time::advance(sleep).await;
+
+        // make sure we return the error, not the expired token
+        assert!(cache.get_token().await.is_err());
+    }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 8)]
+    async fn token_cache_multiple_requests_existing_valid_token() {
+        let now = Instant::now();
+
+        let token = Token {
+            token: "initial-token".to_string(),
+            token_type: "Bearer".to_string(),
+            expires_at: Some((now + TOKEN_VALID_DURATION).into_std()),
+            metadata: None,
+        };
+        let token_clone = token.clone();
+
+        let mut mock = MockTokenProvider::new();
+        mock.expect_get_token()
+            .times(1)
+            .return_once(|| Ok(token_clone));
+
+        // fetch an initial token
+        let cache = TokenCache::new(mock);
+        let actual = cache.get_token().await.unwrap();
+        assert_eq!(actual, token);
+
+        // Spawn N tasks, all asking for a token at once.
+        let tasks = (0..1000)
+            .map(|_| {
+                let cache_clone = cache.clone();
+                tokio::spawn(async move { cache_clone.get_token().await })
+            })
+            .collect::<Vec<_>>();
+
+        // Wait for the N token requests to complete, verifying the returned token.
+        for task in tasks {
+            let actual = task.await.unwrap();
+            assert!(actual.is_ok(), "{}", actual.err().unwrap());
+            assert_eq!(actual.unwrap(), token);
+        }
+    }
+
+    #[tokio::test]
+    async fn refresh_task_expired_token_loop() {
+        let now = Instant::now();
+
+        let token1 = Token {
+            token: "token1".to_string(),
+            token_type: "Bearer".to_string(),
+            expires_at: Some(now.into_std()),
+            metadata: None,
+        };
+        let token1_clone = token1.clone();
+
+        let token2 = Token {
+            token: "token2".to_string(),
+            token_type: "Bearer".to_string(),
+            expires_at: Some((now + TOKEN_VALID_DURATION).into_std()),
+            metadata: None,
+        };
+        let token2_clone = token2.clone();
+
+        let mut mock = MockTokenProvider::new();
+        mock.expect_get_token()
+            .times(1)
+            .return_once(|| Ok(token1_clone));
+
+        mock.expect_get_token()
+            .times(1)
+            .return_once(|| Ok(token2_clone));
+
+        let (tx, mut rx) = watch::channel::<Option<Result<Token>>>(None);
+
+        // Wait to make sure the first token expires.
+        sleep(Duration::from_millis(10)).await;
+
+        tokio::spawn(async move {
+            refresh_task(mock, tx).await;
+        });
+
+        // Give the refresh task a chance to run
+        sleep(Duration::from_millis(100)).await;
path: src/auth/src/token_cache.rs
user: login: coryan
body: These are race conditions and flakes waiting to happen. Can we write this test using `start_paused = true`?
created_at: 2025-03-22T02:01:40Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,594 @@
+// Copyright 2025 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::Result;
+use crate::token::{Token, TokenProvider};
+use tokio::sync::watch;
+use tokio::time::{Duration, Instant, sleep};
+
+// Different MDS(Metadata Service) backends have different policies to
+// determine when to refresh a token. Most MDS' refresh token 5 mins before
+// expiry, except for Serverless which refresh tokens 4 mins before
+// expiry. So we are using 4 mins as the staleness limit for our refresh logic.
+const NORMAL_REFRESH_SLACK: Duration = Duration::from_secs(240);
+const SHORT_REFRESH_SLACK: Duration = Duration::from_secs(10);
+
+#[derive(Debug, Clone)]
+pub(crate) struct TokenCache {
+    rx_token: watch::Receiver<Option<Result<Token>>>,
+}
+
+// TODO(#1552): Use the token cache in all creds
+#[allow(dead_code)]
+impl TokenCache {
+    pub(crate) fn new<T>(inner: T) -> Self
+    where
+        T: TokenProvider + Send + Sync + 'static,
+    {
+        let (tx_token, rx_token) = watch::channel::<Option<Result<Token>>>(None);
+
+        tokio::spawn(async move {
+            refresh_task(inner, tx_token).await;
+        });
+
+        Self { rx_token }
+    }
+}
+
+#[async_trait::async_trait]
+impl TokenProvider for TokenCache {
+    async fn get_token(&self) -> Result<Token> {
+        let mut rx = self.rx_token.clone();
+        let token_result = rx.borrow_and_update().clone();
+
+        if let Some(token_result) = token_result {
+            match token_result {
+                Ok(token) => match token.expires_at {
+                    None => return Ok(token),
+                    Some(e) => {
+                        if e < Instant::now().into_std() {
+                            // Expired token, wait for refresh
+                            return wait_for_next_token(rx).await;
+                        } else {
+                            // valid token
+                            return Ok(token);
+                        }
+                    }
+                },
+                // An error in the result is still a valid result to propagate to the client library
+                Err(e) => Err(e),
+            }
+        } else {
+            wait_for_next_token(rx).await
+        }
+    }
+}
+
+async fn wait_for_next_token(
+    mut rx_token: watch::Receiver<Option<Result<Token>>>,
+) -> Result<Token> {
+    rx_token.changed().await.unwrap();
+    let token_result = rx_token.borrow().clone();
+
+    token_result.expect("There should always be a token or error in the channel after changed()")
+}
+
+async fn refresh_task<T>(token_provider: T, tx_token: watch::Sender<Option<Result<Token>>>)
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    loop {
+        let token_result = token_provider.get_token().await;
+
+        let _ = tx_token.send(Some(token_result.clone()));
+
+        match token_result {
+            Ok(new_token) => {
+                if let Some(expiry) = new_token.expires_at {
+                    let time_until_expiry =
+                        expiry.checked_duration_since(Instant::now().into_std());
+
+                    match time_until_expiry {
+                        None => {
+                            // We were given a token that is expired, or expires in less than 10 seconds.
+                            // We will immediately restart the loop, and fetch a new token.
+                        }
+                        Some(time_until_expiry) => {
+                            if time_until_expiry > NORMAL_REFRESH_SLACK {
+                                sleep(time_until_expiry - NORMAL_REFRESH_SLACK).await;
+                            } else if time_until_expiry > SHORT_REFRESH_SLACK {
+                                // If expiry is less than 4 mins, try to refresh every 10 seconds
+                                // This is to handle cases where MDS **repeatedly** returns about to expire tokens.
+                                sleep(SHORT_REFRESH_SLACK).await;
+                            }
+                        }
+                    }
+                } else {
+                    // If there is no expiry, the token is valid forever, so no need to refresh
+                    // TODO(#1553): Validate that all auth backends provide expiry and make expiry not optional.
+                    break;
+                }
+            }
+            Err(_) => {
+                // The retry policy has been used already by the inner token provider.
+                // If it ended in an error, just quit the background task.
+                break;
+            }
+        }
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    use crate::errors::CredentialError;
+    use crate::token::test::MockTokenProvider;
+    use std::ops::{Add, Sub};
+    use std::sync::{Arc, Mutex};
+    use tokio::time::{Duration, Instant};
+
+    static TOKEN_VALID_DURATION: Duration = Duration::from_secs(3600);
+
+    #[tokio::test]
+    async fn initial_token_success() {
+        let expected = Token {
+            token: "test-token".to_string(),
+            token_type: "Bearer".to_string(),
+            expires_at: None,
+            metadata: None,
+        };
+        let expected_clone = expected.clone();
+
+        let mut mock = MockTokenProvider::new();
+        mock.expect_get_token()
+            .times(1)
+            .return_once(|| Ok(expected_clone));
+
+        let cache = TokenCache::new(mock);
+        let actual = cache.get_token().await.unwrap();
+        assert_eq!(actual, expected);
+
+        // Verify that we use the cached token instead of making a new request
+        // to the mock token provider.
+        let actual = cache.get_token().await.unwrap();
+        assert_eq!(actual, expected);
+    }
+
+    #[tokio::test]
+    async fn initial_token_failure() {
+        let mut mock = MockTokenProvider::new();
+        mock.expect_get_token()
+            .times(1)
+            .returning(|| Err(CredentialError::non_retryable_from_str("fail")));
+
+        let cache = TokenCache::new(mock);
+        assert!(cache.get_token().await.is_err());
+
+        // Verify that a new request is made to the mock token provider when we
+        // don't have a valid token.
+        assert!(cache.get_token().await.is_err());
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn expired_token_success() {
+        let now = Instant::now();
+
+        let initial = Token {
+            token: "initial-token".to_string(),
+            token_type: "Bearer".to_string(),
+            expires_at: Some((now + TOKEN_VALID_DURATION).into_std()),
+            metadata: None,
+        };
+        let initial_clone = initial.clone();
+
+        let refresh = Token {
+            token: "refreshed-token".to_string(),
+            token_type: "Bearer".to_string(),
+            expires_at: Some((now + 2 * TOKEN_VALID_DURATION).into_std()),
+            metadata: None,
+        };
+        let refresh_clone = refresh.clone();
+
+        let mut mock = MockTokenProvider::new();
+        mock.expect_get_token()
+            .times(1)
+            .return_once(|| Ok(initial_clone));
+
+        mock.expect_get_token()
+            .times(1)
+            .return_once(|| Ok(refresh_clone));
+
+        // fetch an initial token
+        let cache = TokenCache::new(mock);
+        let actual = cache.get_token().await.unwrap();
+        assert_eq!(actual, initial);
+
+        // wait long enough for the token to be expired
+        // get_token should be waiting until the new token is available
+        let sleep = TOKEN_VALID_DURATION.add(Duration::from_secs(100));
+        tokio::time::advance(sleep).await;
+
+        // make sure this is the new token
+        let actual = cache.get_token().await.unwrap();
+        assert_eq!(actual, refresh);
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn expired_token_failure() {
+        let now = Instant::now();
+
+        let initial = Token {
+            token: "initial-token".to_string(),
+            token_type: "Bearer".to_string(),
+            expires_at: Some((now + TOKEN_VALID_DURATION).into_std()),
+            metadata: None,
+        };
+        let initial_clone = initial.clone();
+
+        let mut mock = MockTokenProvider::new();
+        mock.expect_get_token()
+            .times(1)
+            .return_once(|| Ok(initial_clone));
+
+        mock.expect_get_token()
+            .times(1)
+            .return_once(|| Err(CredentialError::non_retryable_from_str("fail")));
+
+        // fetch an initial token
+        let cache = TokenCache::new(mock);
+        let actual = cache.get_token().await.unwrap();
+        assert_eq!(actual, initial);
+
+        // wait long enough for the token to be expired
+        let sleep = TOKEN_VALID_DURATION.add(Duration::from_secs(100));
+        tokio::time::advance(sleep).await;
+
+        // make sure we return the error, not the expired token
+        assert!(cache.get_token().await.is_err());
+    }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 8)]
+    async fn token_cache_multiple_requests_existing_valid_token() {
+        let now = Instant::now();
+
+        let token = Token {
+            token: "initial-token".to_string(),
+            token_type: "Bearer".to_string(),
+            expires_at: Some((now + TOKEN_VALID_DURATION).into_std()),
+            metadata: None,
+        };
+        let token_clone = token.clone();
+
+        let mut mock = MockTokenProvider::new();
+        mock.expect_get_token()
+            .times(1)
+            .return_once(|| Ok(token_clone));
+
+        // fetch an initial token
+        let cache = TokenCache::new(mock);
+        let actual = cache.get_token().await.unwrap();
+        assert_eq!(actual, token);
+
+        // Spawn N tasks, all asking for a token at once.
+        let tasks = (0..1000)
+            .map(|_| {
+                let cache_clone = cache.clone();
+                tokio::spawn(async move { cache_clone.get_token().await })
+            })
+            .collect::<Vec<_>>();
+
+        // Wait for the N token requests to complete, verifying the returned token.
+        for task in tasks {
+            let actual = task.await.unwrap();
+            assert!(actual.is_ok(), "{}", actual.err().unwrap());
+            assert_eq!(actual.unwrap(), token);
+        }
+    }
+
+    #[tokio::test]
+    async fn refresh_task_expired_token_loop() {
+        let now = Instant::now();
+
+        let token1 = Token {
+            token: "token1".to_string(),
+            token_type: "Bearer".to_string(),
+            expires_at: Some(now.into_std()),
+            metadata: None,
+        };
+        let token1_clone = token1.clone();
+
+        let token2 = Token {
+            token: "token2".to_string(),
+            token_type: "Bearer".to_string(),
+            expires_at: Some((now + TOKEN_VALID_DURATION).into_std()),
+            metadata: None,
+        };
+        let token2_clone = token2.clone();
+
+        let mut mock = MockTokenProvider::new();
+        mock.expect_get_token()
+            .times(1)
+            .return_once(|| Ok(token1_clone));
+
+        mock.expect_get_token()
+            .times(1)
+            .return_once(|| Ok(token2_clone));
+
+        let (tx, mut rx) = watch::channel::<Option<Result<Token>>>(None);
+
+        // Wait to make sure the first token expires.
+        sleep(Duration::from_millis(10)).await;
+
+        tokio::spawn(async move {
+            refresh_task(mock, tx).await;
+        });
+
+        // Give the refresh task a chance to run
+        sleep(Duration::from_millis(100)).await;
path: src/auth/src/token_cache.rs
user: login: sai-sunder-s
body: Removed one of the sleeps. Added it while debugging the issue (turned out to be somehow windows cannot give an instant in the past but windows and mac can). 

I do not think start-pasued=true is of use here because we do not want forward time. We want to make sure not much time passes between refreshes i.e. we are running the loop in refresh task without any sleep.
created_at: 2025-03-22T02:15:54Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,594 @@
+// Copyright 2025 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::Result;
+use crate::token::{Token, TokenProvider};
+use tokio::sync::watch;
+use tokio::time::{Duration, Instant, sleep};
+
+// Different MDS(Metadata Service) backends have different policies to
+// determine when to refresh a token. Most MDS' refresh token 5 mins before
+// expiry, except for Serverless which refresh tokens 4 mins before
+// expiry. So we are using 4 mins as the staleness limit for our refresh logic.
+const NORMAL_REFRESH_SLACK: Duration = Duration::from_secs(240);
+const SHORT_REFRESH_SLACK: Duration = Duration::from_secs(10);
+
+#[derive(Debug, Clone)]
+pub(crate) struct TokenCache {
+    rx_token: watch::Receiver<Option<Result<Token>>>,
+}
+
+// TODO(#1552): Use the token cache in all creds
+#[allow(dead_code)]
+impl TokenCache {
+    pub(crate) fn new<T>(inner: T) -> Self
+    where
+        T: TokenProvider + Send + Sync + 'static,
+    {
+        let (tx_token, rx_token) = watch::channel::<Option<Result<Token>>>(None);
+
+        tokio::spawn(async move {
+            refresh_task(inner, tx_token).await;
+        });
+
+        Self { rx_token }
+    }
+}
+
+#[async_trait::async_trait]
+impl TokenProvider for TokenCache {
+    async fn get_token(&self) -> Result<Token> {
+        let mut rx = self.rx_token.clone();
+        let token_result = rx.borrow_and_update().clone();
+
+        if let Some(token_result) = token_result {
+            match token_result {
+                Ok(token) => match token.expires_at {
+                    None => return Ok(token),
+                    Some(e) => {
+                        if e < Instant::now().into_std() {
+                            // Expired token, wait for refresh
+                            return wait_for_next_token(rx).await;
+                        } else {
+                            // valid token
+                            return Ok(token);
+                        }
+                    }
+                },
+                // An error in the result is still a valid result to propagate to the client library
+                Err(e) => Err(e),
+            }
+        } else {
+            wait_for_next_token(rx).await
+        }
+    }
+}
+
+async fn wait_for_next_token(
+    mut rx_token: watch::Receiver<Option<Result<Token>>>,
+) -> Result<Token> {
+    rx_token.changed().await.unwrap();
+    let token_result = rx_token.borrow().clone();
+
+    token_result.expect("There should always be a token or error in the channel after changed()")
+}
+
+async fn refresh_task<T>(token_provider: T, tx_token: watch::Sender<Option<Result<Token>>>)
+where
+    T: TokenProvider + Send + Sync + 'static,
+{
+    loop {
+        let token_result = token_provider.get_token().await;
+
+        let _ = tx_token.send(Some(token_result.clone()));
+
+        match token_result {
+            Ok(new_token) => {
+                if let Some(expiry) = new_token.expires_at {
+                    let time_until_expiry =
+                        expiry.checked_duration_since(Instant::now().into_std());
+
+                    match time_until_expiry {
+                        None => {
+                            // We were given a token that is expired, or expires in less than 10 seconds.
+                            // We will immediately restart the loop, and fetch a new token.
+                        }
+                        Some(time_until_expiry) => {
+                            if time_until_expiry > NORMAL_REFRESH_SLACK {
+                                sleep(time_until_expiry - NORMAL_REFRESH_SLACK).await;
+                            } else if time_until_expiry > SHORT_REFRESH_SLACK {
+                                // If expiry is less than 4 mins, try to refresh every 10 seconds
+                                // This is to handle cases where MDS **repeatedly** returns about to expire tokens.
+                                sleep(SHORT_REFRESH_SLACK).await;
+                            }
+                        }
+                    }
+                } else {
+                    // If there is no expiry, the token is valid forever, so no need to refresh
+                    // TODO(#1553): Validate that all auth backends provide expiry and make expiry not optional.
+                    break;
+                }
+            }
+            Err(_) => {
+                // The retry policy has been used already by the inner token provider.
+                // If it ended in an error, just quit the background task.
+                break;
+            }
+        }
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    use crate::errors::CredentialError;
+    use crate::token::test::MockTokenProvider;
+    use std::ops::{Add, Sub};
+    use std::sync::{Arc, Mutex};
+    use tokio::time::{Duration, Instant};
+
+    static TOKEN_VALID_DURATION: Duration = Duration::from_secs(3600);
+
+    #[tokio::test]
+    async fn initial_token_success() {
+        let expected = Token {
+            token: "test-token".to_string(),
+            token_type: "Bearer".to_string(),
+            expires_at: None,
+            metadata: None,
+        };
+        let expected_clone = expected.clone();
+
+        let mut mock = MockTokenProvider::new();
+        mock.expect_get_token()
+            .times(1)
+            .return_once(|| Ok(expected_clone));
+
+        let cache = TokenCache::new(mock);
+        let actual = cache.get_token().await.unwrap();
+        assert_eq!(actual, expected);
+
+        // Verify that we use the cached token instead of making a new request
+        // to the mock token provider.
+        let actual = cache.get_token().await.unwrap();
+        assert_eq!(actual, expected);
+    }
+
+    #[tokio::test]
+    async fn initial_token_failure() {
+        let mut mock = MockTokenProvider::new();
+        mock.expect_get_token()
+            .times(1)
+            .returning(|| Err(CredentialError::non_retryable_from_str("fail")));
+
+        let cache = TokenCache::new(mock);
+        assert!(cache.get_token().await.is_err());
+
+        // Verify that a new request is made to the mock token provider when we
+        // don't have a valid token.
+        assert!(cache.get_token().await.is_err());
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn expired_token_success() {
+        let now = Instant::now();
+
+        let initial = Token {
+            token: "initial-token".to_string(),
+            token_type: "Bearer".to_string(),
+            expires_at: Some((now + TOKEN_VALID_DURATION).into_std()),
+            metadata: None,
+        };
+        let initial_clone = initial.clone();
+
+        let refresh = Token {
+            token: "refreshed-token".to_string(),
+            token_type: "Bearer".to_string(),
+            expires_at: Some((now + 2 * TOKEN_VALID_DURATION).into_std()),
+            metadata: None,
+        };
+        let refresh_clone = refresh.clone();
+
+        let mut mock = MockTokenProvider::new();
+        mock.expect_get_token()
+            .times(1)
+            .return_once(|| Ok(initial_clone));
+
+        mock.expect_get_token()
+            .times(1)
+            .return_once(|| Ok(refresh_clone));
+
+        // fetch an initial token
+        let cache = TokenCache::new(mock);
+        let actual = cache.get_token().await.unwrap();
+        assert_eq!(actual, initial);
+
+        // wait long enough for the token to be expired
+        // get_token should be waiting until the new token is available
+        let sleep = TOKEN_VALID_DURATION.add(Duration::from_secs(100));
+        tokio::time::advance(sleep).await;
+
+        // make sure this is the new token
+        let actual = cache.get_token().await.unwrap();
+        assert_eq!(actual, refresh);
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn expired_token_failure() {
+        let now = Instant::now();
+
+        let initial = Token {
+            token: "initial-token".to_string(),
+            token_type: "Bearer".to_string(),
+            expires_at: Some((now + TOKEN_VALID_DURATION).into_std()),
+            metadata: None,
+        };
+        let initial_clone = initial.clone();
+
+        let mut mock = MockTokenProvider::new();
+        mock.expect_get_token()
+            .times(1)
+            .return_once(|| Ok(initial_clone));
+
+        mock.expect_get_token()
+            .times(1)
+            .return_once(|| Err(CredentialError::non_retryable_from_str("fail")));
+
+        // fetch an initial token
+        let cache = TokenCache::new(mock);
+        let actual = cache.get_token().await.unwrap();
+        assert_eq!(actual, initial);
+
+        // wait long enough for the token to be expired
+        let sleep = TOKEN_VALID_DURATION.add(Duration::from_secs(100));
+        tokio::time::advance(sleep).await;
+
+        // make sure we return the error, not the expired token
+        assert!(cache.get_token().await.is_err());
+    }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 8)]
+    async fn token_cache_multiple_requests_existing_valid_token() {
+        let now = Instant::now();
+
+        let token = Token {
+            token: "initial-token".to_string(),
+            token_type: "Bearer".to_string(),
+            expires_at: Some((now + TOKEN_VALID_DURATION).into_std()),
+            metadata: None,
+        };
+        let token_clone = token.clone();
+
+        let mut mock = MockTokenProvider::new();
+        mock.expect_get_token()
+            .times(1)
+            .return_once(|| Ok(token_clone));
+
+        // fetch an initial token
+        let cache = TokenCache::new(mock);
+        let actual = cache.get_token().await.unwrap();
+        assert_eq!(actual, token);
+
+        // Spawn N tasks, all asking for a token at once.
+        let tasks = (0..1000)
+            .map(|_| {
+                let cache_clone = cache.clone();
+                tokio::spawn(async move { cache_clone.get_token().await })
+            })
+            .collect::<Vec<_>>();
+
+        // Wait for the N token requests to complete, verifying the returned token.
+        for task in tasks {
+            let actual = task.await.unwrap();
+            assert!(actual.is_ok(), "{}", actual.err().unwrap());
+            assert_eq!(actual.unwrap(), token);
+        }
+    }
+
+    #[tokio::test]
+    async fn refresh_task_expired_token_loop() {
+        let now = Instant::now();
+
+        let token1 = Token {
+            token: "token1".to_string(),
+            token_type: "Bearer".to_string(),
+            expires_at: Some(now.into_std()),
+            metadata: None,
+        };
+        let token1_clone = token1.clone();
+
+        let token2 = Token {
+            token: "token2".to_string(),
+            token_type: "Bearer".to_string(),
+            expires_at: Some((now + TOKEN_VALID_DURATION).into_std()),
+            metadata: None,
+        };
+        let token2_clone = token2.clone();
+
+        let mut mock = MockTokenProvider::new();
+        mock.expect_get_token()
+            .times(1)
+            .return_once(|| Ok(token1_clone));
+
+        mock.expect_get_token()
+            .times(1)
+            .return_once(|| Ok(token2_clone));
+
+        let (tx, mut rx) = watch::channel::<Option<Result<Token>>>(None);
+
+        // Wait to make sure the first token expires.
+        sleep(Duration::from_millis(10)).await;
+
+        tokio::spawn(async move {
+            refresh_task(mock, tx).await;
+        });
+
+        // Give the refresh task a chance to run
+        sleep(Duration::from_millis(100)).await;
path: src/auth/src/token_cache.rs
user: login: coryan
body: Regardless, any test with a time in it is risky.  Maybe you will get lucky, but flaky tests are put out of their misery fast in this project.  I will not hesitate to remove or disable the test.

created_at: 2025-03-22T13:25:30Z
author_association: CONTRIBUTOR

PR comments:
user: login: codecov[bot]
created_at: 2025-03-20T04:23:22Z
body: ## [Codecov](https://app.codecov.io/gh/googleapis/google-cloud-rust/pull/1554?dropdown=coverage&src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=googleapis) Report
Attention: Patch coverage is `90.00000%` with `4 lines` in your changes missing coverage. Please review.
> Project coverage is 95.72%. Comparing base [(`cd3b51f`)](https://app.codecov.io/gh/googleapis/google-cloud-rust/commit/cd3b51fa358a541ebc054ba052f856f68e55e913?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=googleapis) to head [(`12300b3`)](https://app.codecov.io/gh/googleapis/google-cloud-rust/commit/12300b3de5a2b3a99f02c4263aa677d7ada8f282?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=googleapis).
> Report is 6 commits behind head on main.

| [Files with missing lines](https://app.codecov.io/gh/googleapis/google-cloud-rust/pull/1554?dropdown=coverage&src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=googleapis) | Patch % | Lines |
|---|---|---|
| [src/auth/src/token\_cache.rs](https://app.codecov.io/gh/googleapis/google-cloud-rust/pull/1554?src=pr&el=tree&filepath=src%2Fauth%2Fsrc%2Ftoken_cache.rs&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=googleapis#diff-c3JjL2F1dGgvc3JjL3Rva2VuX2NhY2hlLnJz) | 90.00% | [4 Missing :warning: ](https://app.codecov.io/gh/googleapis/google-cloud-rust/pull/1554?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=googleapis) |

<details><summary>Additional details and impacted files</summary>


```diff
@@            Coverage Diff             @@
##             main    #1554      +/-   ##
==========================================
- Coverage   95.78%   95.72%   -0.07%     
==========================================
  Files          42       43       +1     
  Lines        1755     1823      +68     
==========================================
+ Hits         1681     1745      +64     
- Misses         74       78       +4     
```

</details>

[:umbrella: View full report in Codecov by Sentry](https://app.codecov.io/gh/googleapis/google-cloud-rust/pull/1554?dropdown=coverage&src=pr&el=continue&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=googleapis).   
:loudspeaker: Have feedback on the report? [Share it here](https://about.codecov.io/codecov-pr-comment-feedback/?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=googleapis).

<details><summary> :rocket: New features to boost your workflow: </summary>

- :snowflake: [Test Analytics](https://docs.codecov.com/docs/test-analytics): Detect flaky tests, report on failures, and find test suite problems.
</details>
author_association: NONE
user: login: coryan
created_at: 2025-03-21T20:03:45Z
body: I fiddled with the title of this PR, it seemed too repetitive.

author_association: CONTRIBUTOR