SlowGuess/NeurIPS_2023_OCR huggingface dataset:
$$
\begin{array}{l} x _ {t} ^ {s} \Big (G ^ {s} + \gamma \mathbb {E} _ {s ^ {\prime} \sim P ^ {s}} \left[ V _ {t} ^ {s ^ {\prime}} \right] \Big) y ^ {s} - x ^ {s} \Big (G ^ {s} + \gamma \mathbb {E} _ {s ^ {\prime} \sim P ^ {s}} \left[ V _ {t} ^ {s ^ {\prime}} \right] \Big) y _ {t} ^ {s} \\ = \hat {x} _ {\tau} ^ {s} \Big (G ^ {s} + \gamma \mathbb {E} _ {s ^ {\prime} \sim P ^ {s}} \Big [ V _ {t} ^ {s ^ {\prime}} \Big ] \Big) y ^ {s} - x ^ {s} \Big (G ^ {s} + \gamma \mathbb {E} _ {s ^ {\prime} \sim P ^ {s}} \Big [ V _ {t} ^ {s ^ {\prime}} \Big ] \Big) \hat {y} _ {\tau} ^ {s} \\ \leq \hat {x} _ {\tau} ^ {s} \left(G ^ {s} + \gamma \mathbb {E} _ {s ^ {\prime} \sim P ^ {s}} \left[ V _ {t _ {\tau} (s)} ^ {s ^ {\prime}} \right]\right) y ^ {s} - x ^ {s} \left(G ^ {s} + \gamma \mathbb {E} _ {s ^ {\prime} \sim P ^ {s}} \left[ V _ {t _ {\tau} (s)} ^ {s ^ {\prime}} \right]\right) \hat {y} _ {\tau} ^ {s} + 2 \max  _ {s ^ {\prime}} | V _ {t} ^ {s ^ {\prime}} - V _ {t _ {\tau} (s)} ^ {s ^ {\prime}} | \\ \leq \hat {x} _ {\tau \star} ^ {s} \left(G ^ {s} + \gamma \mathbb {E} _ {s ^ {\prime} \sim P ^ {s}} \left[ V _ {t _ {\tau} (s)} ^ {s ^ {\prime}} \right]\right) y ^ {s} - x ^ {s} \left(G ^ {s} + \gamma \mathbb {E} _ {s ^ {\prime} \sim P ^ {s}} \left[ V _ {t _ {\tau} (s)} ^ {s ^ {\prime}} \right]\right) \hat {y} _ {\tau \star} ^ {s} + 2 \max  _ {s ^ {\prime}} | V _ {t} ^ {s ^ {\prime}} - V _ {t _ {\tau} (s)} ^ {s ^ {\prime}} | + \mathcal {O} (\| \hat {z} _ {\tau} ^ {s} - \hat {z} _ {\tau \star} ^ {s} \| _ {1}) \\ \leq 2 \epsilon_ {\tau} \ln (A) + \mathcal {O} \left(\frac {1}{\tau}\right) + 2 \max  _ {s ^ {\prime}} \left| V _ {t} ^ {s ^ {\prime}} - V _ {t _ {\tau} (s)} ^ {s ^ {\prime}} \right| + \mathcal {O} \left(\sqrt {\mathrm {K L} \left(\hat {z} _ {\tau} ^ {s} , \hat {z} _ {\tau \star} ^ {s}\right)}\right) (Lemma9) \\ \leq \mathcal {O} \left(\frac {\ln (A)}{1 - \gamma} \tau^ {- k _ {\epsilon}}\right) + \mathcal {O} \left(\frac {L \ln (S t / \delta) \tau^ {- k _ {\alpha}}}{1 - \gamma}\right) + \mathcal {O} \left(\sqrt {A \ln^ {5} (S A t / \delta) L ^ {2}} \tau^ {- \frac {k _ {\sharp}}{2}}\right). (Lemma 17) \\ \end{array}
$$
Combining the above two inequalities with Lemma 17 and Lemma 20 and the choice of parameters  $k_{\alpha} = \frac{9}{9 + \varepsilon}, k_{\varepsilon} = \frac{1}{9 + \varepsilon}, k_{\beta} = \frac{3}{9 + \varepsilon}$ , and  $k_{\eta} = \frac{5}{9 + \varepsilon}$ , we have  $k_{\sharp} = \min \{k_{\beta} - k_{\epsilon}, k_{\eta} - k_{\beta}, k_{\alpha} - k_{\eta} - 2k_{\epsilon}\} = \frac{2}{9 + \varepsilon}, k_{*} = \min \{k_{\eta}, k_{\beta}, k_{\alpha} - k_{\beta}, k_{\epsilon}\} = \frac{1}{9 + \varepsilon}$ , and
$$
\begin{array}{l} \max _ {s, x, y} \left(V _ {x _ {t}, y} ^ {s} - V _ {x, y _ {t}} ^ {s}\right) \\ \leq \frac {2}{1 - \gamma} \max  _ {s, x, y} \left(x _ {t} ^ {s} \Big (G ^ {s} + \gamma \mathbb {E} _ {s ^ {\prime} \sim P ^ {s}} \Big [ V _ {t} ^ {s ^ {\prime}} \Big ] \Big) y ^ {s} - x ^ {s} \Big (G ^ {s} + \gamma \mathbb {E} _ {s ^ {\prime} \sim P ^ {s}} \Big [ V _ {t} ^ {s ^ {\prime}} \Big ] \Big) y _ {t} ^ {s}\right) + \frac {4 \gamma}{1 - \gamma} \max  _ {s} | V _ {t} ^ {s} - V _ {\star} ^ {s} | \\ \leq \mathcal {O} \left(\frac {\sqrt {A} \ln^ {5 / 2} (S A t / \delta)}{(1 - \gamma) ^ {2}} L \tau^ {- \min  \{k _ {\epsilon}, \frac {k _ {\sharp}}{2}, k _ {\alpha} \}}\right) + \mathcal {O} \left(\frac {A \ln^ {3} (S A t / \delta)}{(1 - \gamma) ^ {3}} \left(\frac {L \ln (S t / \delta)}{1 - \gamma} \ln \frac {t}{1 - \gamma}\right) ^ {\frac {k _ {*}}{1 - k _ {\alpha}}} \left(\frac {L \ln (S t / \delta)}{t}\right) ^ {k _ {*}}\right) \\ \end{array}
$$
$$
= \mathcal {O} \left(\frac {A L ^ {2 + 1 / \varepsilon} \ln^ {4 + 1 / \varepsilon} (S A t / \delta) \ln^ {1 / \varepsilon} (t / (1 - \gamma))}{(1 - \gamma) ^ {3 + 1 / \varepsilon}} \cdot t ^ {- \frac {1}{9 + \varepsilon}}\right). \qquad (\frac {t}{L} \leq \tau \leq t)
$$
E Convergent Analysis of Algorithm 3
E.1 Part I. Basic Iteration Properties
Definition 1. Let  $t_{\tau}(s)$  be the  $\tau$ -th time the players visit state  $s$ . Define  $\hat{x}_{\tau}^{s} = x_{t_{\tau}(s)}^{s}$ ,  $\hat{y}_{\tau}^{s} = y_{t_{\tau}(s)}^{s}$ ,  $\hat{a}_{\tau}^{s} = a_{t_{\tau}(s)}$ ,  $\hat{b}_{\tau}^{s} = b_{t_{\tau}(s)}$ . Furthermore, define
$$
\underline {{f}} _ {\tau} ^ {s} (x, y) \triangleq x ^ {\top} \left(G ^ {s} + \gamma \mathbb {E} _ {s ^ {\prime} \sim P ^ {s}} \left[ \underline {{V}} _ {t _ {\tau} (s)} ^ {s ^ {\prime}} \right]\right) y - \epsilon \phi (x) + \epsilon \phi (y),
$$
$$
\overline {{f}} _ {\tau} ^ {s} (x, y) \triangleq x ^ {\top} \left(G ^ {s} + \gamma \mathbb {E} _ {s ^ {\prime} \sim P ^ {s}} \left[ \overline {{V}} _ {t _ {\tau} (s)} ^ {s ^ {\prime}} \right]\right) y - \epsilon \phi (x) + \epsilon \phi (y).
$$
Lemma 22. For any  $x^s \in \Omega$ ,
$$
\begin{array}{l} \underline {{f}} _ {\tau} ^ {s} (\hat {x} _ {\tau} ^ {s}, \hat {y} _ {\tau} ^ {s}) - \underline {{f}} _ {\tau} ^ {s} (x ^ {s}, \hat {y} _ {\tau} ^ {s}) \\ \leq \frac {(1 - \eta \epsilon) K L (x ^ {s} , \hat {x} _ {\tau} ^ {s}) - K L (x ^ {s} , \hat {x} _ {\tau + 1} ^ {s})}{\eta} + \frac {1 0 \eta A \ln^ {2} (A T)}{(1 - \gamma) ^ {2}} + \frac {2 \eta A}{(1 - \gamma) ^ {2}} \underline {{\lambda}} _ {\tau} ^ {s} + \underline {{\xi}} _ {\tau} ^ {s} + \underline {{\zeta}} _ {\tau} ^ {s} (x ^ {s}). \\ \end{array}
$$
$$
\underline {{\lambda}} _ {\tau} ^ {s} = \frac {1}{A} \sum_ {a} \left(\frac {\mathbf {1} [ \hat {a} _ {\tau} ^ {s} = a ]}{\hat {x} _ {\tau , a} ^ {s} + \beta} - 1\right),
$$
$$
\underline {{\xi}} _ {\tau} ^ {s} = \left(\hat {x} _ {\tau} ^ {s}\right) ^ {\top} \left(\left(G ^ {s} + \gamma \mathbb {E} _ {s ^ {\prime} \sim P s} \left[ \underline {{V}} _ {t} ^ {s ^ {\prime}} \right]\right) \hat {y} _ {\tau} ^ {s} - \frac {\mathbf {1} [ \hat {a} _ {\tau} ^ {s} = a ] (\sigma_ {t} + \gamma \underline {{V}} _ {t} ^ {s _ {t + 1}})}{\hat {x} _ {\tau , a} ^ {s} + \beta}\right),
$$
$$
\underline {{\varsigma}} _ {\tau} ^ {s} (x ^ {s}) = (x ^ {s}) ^ {\top} \left(\frac {\mathbf {1} [ \hat {a} _ {\tau} ^ {s} = a ] (\sigma_ {t} + \gamma \underline {{V}} _ {t} ^ {s _ {t + 1}})}{\hat {x} _ {\tau , a} ^ {s} + \beta} - \left(G ^ {s} + \gamma \mathbb {E} _ {s ^ {\prime} \sim P ^ {s}} [ \underline {{V}} _ {t} ^ {s ^ {\prime}} ]\right) \hat {y} _ {\tau} ^ {s}\right).
$$
Proof. The proof is exactly the same as that of Lemma 16.  $\square$
E.2 Part II. Value Convergence
Lemma 23 (weighted regret bound). There exists a large enough universal constant  $\kappa$  (used in the definition of  $bns_{\tau}$ ) such that with probability  $1 - \mathcal{O}(\delta)$ , for any state  $s$ , visitation count  $\tau$ , and any  $x^{s} \in \Omega$ ,
$$
\sum_ {i = 1} ^ {\tau} \alpha_ {\tau} ^ {i} \left(\underline {{f}} _ {i} ^ {s} \left(\hat {x} _ {i} ^ {s}, \hat {y} _ {i} ^ {s}\right) - \underline {{f}} _ {i} ^ {s} \left(x ^ {s}, \hat {y} _ {i} ^ {s}\right)\right) \leq \frac {1}{2} b n s _ {\tau}.
$$
Proof. Fix state  $s$  and visitation count  $\tau \leq T$ . Applying Lemma 22 and considering the weighted sum of the bounds, we get
$$
\begin{array}{l} \sum_ {i = 1} ^ {\tau} \alpha_ {\tau} ^ {i} \left(\underline {{f}} _ {i} ^ {s} (\hat {x} _ {i} ^ {s}, \hat {y} _ {i} ^ {s}) - \underline {{f}} _ {i} ^ {s} (x ^ {s}, \hat {y} _ {i} ^ {s})\right) \\ \leq \sum_ {i = 1} ^ {\tau} \alpha_ {\tau} ^ {i} \left(\frac {(1 - \eta \epsilon) \mathrm {K L} (x ^ {s} , \hat {x} _ {i} ^ {s}) - \mathrm {K L} (x ^ {s} , \hat {x} _ {i + 1} ^ {s})}{\eta} + \frac {1 0 \eta A \ln^ {2} (A T)}{(1 - \gamma) ^ {2}} + \frac {2 \eta A}{(1 - \gamma) ^ {2}} \underline {{\lambda}} _ {i} ^ {s} + \underline {{\xi}} _ {i} ^ {s} + \underline {{\zeta}} _ {i} ^ {s} (x ^ {s})\right) \\ \leq \underbrace {\frac {\alpha_ {\tau} ^ {1} (1 - \eta \epsilon)}{\eta} \mathrm {K L} (x ^ {s} , \hat {x} _ {1} ^ {s}) + \sum_ {i = 2} ^ {\tau} \left(\frac {\alpha_ {\tau} ^ {i} (1 - \eta \epsilon)}{\eta} \mathrm {K L} (x ^ {s} , \hat {x} _ {i} ^ {s}) - \frac {\alpha_ {\tau} ^ {i - 1}}{\eta} \mathrm {K L} (x ^ {s} , \hat {x} _ {i} ^ {s})\right)} _ {\text {t e r m} _ {0}} \\ \end{array}
$$
$$
+ \underbrace {\frac {1 0 \eta A \ln^ {2} (A T)}{(1 - \gamma) ^ {2}} \sum_ {i = 1} ^ {\tau} \alpha_ {\tau} ^ {i}} _ {\mathbf {t e r m} _ {1}} + \underbrace {\frac {2 \eta A}{(1 - \gamma) ^ {2}} \sum_ {i = 1} ^ {\tau} \alpha_ {\tau} ^ {i} \underline {{\lambda}} _ {i} ^ {s}} _ {\mathbf {t e r m} _ {2}} + \underbrace {\sum_ {i = 1} ^ {\tau} \alpha_ {\tau} ^ {i} \underline {{\xi}} _ {i} ^ {s}} _ {\mathbf {t e r m} _ {3}} + \underbrace {\sum_ {i = 1} ^ {\tau} \alpha_ {\tau} ^ {i} \underline {{\zeta}} _ {i} ^ {s} (x ^ {s})} _ {\mathbf {t e r m} _ {4}}.
$$
Since  $\alpha_{\tau}^{i - 1} = \frac{\alpha_{i - 1}(1 - \alpha_i)}{\alpha_i}\alpha_\tau^i\geq (1 - \alpha_i)\alpha_\tau^i$
$$
\begin{array}{l} \mathbf {t e r m} _ {0} \leq \frac {\alpha_ {\tau} ^ {1}}{\eta} \mathrm {K L} \left(x ^ {s}, \hat {x} _ {1} ^ {s}\right) + \sum_ {i = 2} ^ {\tau} \mathrm {K L} \left(x ^ {s}, \hat {x} _ {i} ^ {s}\right) \left(\frac {\alpha_ {\tau} ^ {i} (1 - \eta \epsilon)}{\eta} - \frac {\alpha_ {\tau} ^ {i} (1 - \alpha_ {i})}{\eta}\right) \\ \leq \ln (A T) \left(\frac {\alpha_ {\tau} ^ {1}}{\eta} + \sum_ {i = 2} ^ {\tau} \frac {\alpha_ {\tau} ^ {i} \alpha_ {i}}{\eta}\right) = \mathcal {O} \left(\frac {\ln (A T) \alpha_ {\tau}}{\eta}\right). \\ \end{array}
$$
We proceed to bound other terms as follows: wht probability at least  $1 - \frac{\delta}{S\tau^2}$
$$
\mathbf {t e r m} _ {1} = \mathcal {O} \left(\frac {A \ln^ {2} (A T) \eta}{(1 - \gamma) ^ {2}}\right), \quad \left(\sum_ {i = 1} ^ {\tau} \alpha_ {\tau} ^ {i} = 1\right)
$$
$$
\begin{array}{l} \mathbf {t e r m} _ {2} = \frac {2 \eta A}{(1 - \gamma) ^ {2}} \sum_ {i = 1} ^ {\tau} \alpha_ {\tau} ^ {i} \left(\frac {1}{A} \sum_ {a} \left(\frac {\mathbf {1} [ \hat {a} _ {i} ^ {s} = a ]}{\hat {x} _ {i , a} ^ {s} + \beta} - 1\right)\right) \\ \leq \frac {2 A}{(1 - \gamma) ^ {2}} \times \mathcal {O} \left(\ln (A S \tau / \delta) \max  _ {i \leq \tau} \frac {\alpha_ {\tau} ^ {i} \eta}{\beta}\right) \tag {byLemma7} \\ = \mathcal {O} \left(\frac {A \ln (A S T / \delta) \alpha_ {\tau}}{(1 - \gamma) ^ {2}} \times \frac {\eta}{\beta}\right), \\ \end{array}
$$
$$
\begin{array}{l} \mathbf {t e r m} _ {3} = \mathcal {O} \left(\frac {A}{1 - \gamma} \sum_ {i = 1} ^ {\tau} \beta \alpha_ {\tau} ^ {i} + \frac {1}{1 - \gamma} \sqrt {\ln (A S \tau / \delta) \sum_ {i = 1} ^ {\tau} \left(\alpha_ {\tau} ^ {i}\right) ^ {2}}\right) \quad (\text {b y}) \\ = \mathcal {O} \left(\frac {A \beta}{1 - \gamma} + \frac {1}{1 - \gamma} \sqrt {\ln (A S \tau / \delta) \sum_ {i = 1} ^ {\tau} \alpha_ {\tau} ^ {i} \alpha_ {i}}\right) \\ = \mathcal {O} \left(\frac {A \ln (A S T / \delta) (\beta + \alpha_ {\tau})}{1 - \gamma}\right), \\ \end{array}
$$
$$
\mathbf {t e r m} _ {4} = \mathcal {O} \left(\frac {\ln (A S \tau / \delta)}{1 - \gamma} \max _ {i \leq \tau} \frac {\alpha_ {\tau} ^ {i}}{\beta}\right) = \mathcal {O} \left(\frac {\ln (A S T / \delta) \alpha_ {\tau}}{(1 - \gamma) \beta}\right).
$$
Combining all terms and applying a union bound over  $s \in S$  and  $\tau$ , we get with probability  $1 - \mathcal{O}(\delta)$  such that for any  $s \in S$ , visitation count  $\tau$ , and  $x^s \in \Omega$ ,
$$
\begin{array}{l} \sum_ {i = 1} ^ {\tau} \alpha_ {\tau} ^ {i} \left(\underline {{f}} _ {i} ^ {s} (\hat {x} _ {i} ^ {s}, \hat {y} _ {i} ^ {s}) - \underline {{f}} _ {i} ^ {s} (x ^ {s}, \hat {y} _ {i} ^ {s})\right) = \mathcal {O} \left(\frac {A \ln^ {2} (A S T / \delta) \left(\eta + \beta + (\eta^ {- 1} + \beta^ {- 1}) \alpha_ {\tau}\right)}{(1 - \gamma) ^ {2}}\right) \\ = \mathcal {O} \left(\frac {A \ln^ {2} (S A T / \delta) (\beta + \alpha_ {\tau} / \eta)}{(1 - \gamma) ^ {2}}\right). \quad \text {(u s i n g} \eta \leq \beta) \\ \end{array}
$$
This implies the conclusion of the lemma.
Lemma 24. For all  $t, s$ ,  $\overline{V}_t^s \geq \underline{V}_t^s$ .
Proof. We prove it by induction on  $t$ . The inequality clearly holds for  $t = 1$  by the initialization. Suppose that the inequality holds for  $1,2,\dots ,t - 1$  and for all  $s$ . Now consider time  $t$  and state  $s$ . Let  $\tau = n_t^s$ , and let  $1\leq t_1 < t_2 < \ldots < t_\tau < t$  be the time indices when the players visit state  $s$ . By the update rule,
$$
\tilde {V} _ {t} ^ {s} - \underline {{V}} _ {t} ^ {s} = \sum_ {i = 1} ^ {\tau} \alpha_ {\tau} ^ {i} \left(\gamma \bar {V} _ {t _ {i}} ^ {s _ {t _ {i}} + 1} - \gamma \underline {{V}} _ {t _ {i}} ^ {s _ {t _ {i}} + 1} + 2 \mathsf {b n s} _ {i}\right) > 0
$$
where the inequality is by the induction hypothesis. Therefore,
$$
\overline {{V}} _ {t} ^ {s} - \underline {{V}} _ {t} ^ {s} = \min  \left\{\tilde {V} _ {t} ^ {s}, H \right\} - \max  \left\{\tilde {V} _ {t} ^ {s}, 0 \right\} > 0.
$$
In the last inequality we also use the fact that  $\tilde{V}_t^s \leq H$  and  $\tilde{V}_t^s \geq 0$ . Note that by the induction hypothesis and the update rule of  $\overline{V}_t^s$  and  $\underline{V}_t^s$ , we have  $0 \leq \underline{V}_i^s < \overline{V}_i^s \leq H$  for all  $s$  and  $1 \leq i \leq t - 1$ . Thus  $\tilde{V}_t^s = \sum_{i=1}^\tau \alpha_\tau^i (\gamma \underline{V}_{t_i}^{s_{t_i + 1}} - \mathsf{bns}_i) \leq H$  and similarly  $\tilde{V}_t^s \geq 0$ .
Lemma 25. Let  $c = (c_{1},\ldots ,c_{T})$  be any non-negative sequence with  $c_{i}\leq c_{\max}\forall i$  and  $\sum_{t = 1}^{T}c_{t} = C$ . Then
$$
\sum_ {t = 1} ^ {T} c _ {t} \left(\overline {{V}} _ {t} ^ {s _ {t + 1}} - \underline {{V}} _ {t} ^ {s _ {t + 1}}\right) \leq \mathcal {O} \left(\frac {C A \ln^ {3} (A S T / \delta) \beta}{(1 - \gamma) ^ {3}} + \frac {c _ {\max} A S \ln^ {4} (A S T / \delta)}{\eta (1 - \gamma) ^ {3}}\right).
$$
Note that  $c_t^\prime$  is another sequence with
$$
c _ {i} ^ {\prime} \leq c _ {\max } ^ {\prime} \leq c _ {\max } \sup  _ {i} \sum_ {\tau = i} ^ {\infty} \alpha_ {\tau} ^ {i} \leq \left(1 + \frac {1}{H}\right) c _ {\max }
$$
$$
\sum_ {t = 1} ^ {T} c _ {t} ^ {\prime} \leq \sum_ {t = 1} ^ {T} c _ {t} = C
$$
since  $\sum_{i=1}^{\tau} \alpha_{\tau}^{i} = 1$  for any  $\tau \geq 1$ . Thus, we can unroll the inequality Eq. (19) for  $H$  times, which gives
$$
\begin{array}{l} Z _ {c} \leq \gamma^ {H} \left(1 + \frac {1}{H}\right) ^ {H} \frac {c _ {\max} T}{1 - \gamma} + H \times \mathcal {O} \left(\frac {C A \ln^ {2} (A S T / \delta) \beta}{(1 - \gamma) ^ {2}} + \frac {c _ {\max} A S \ln^ {3} (A S T / \delta)}{\eta (1 - \gamma) ^ {2}}\right) \\ = \mathcal {O} \left(\frac {C A \ln^ {3} (A S T / \delta) \beta}{(1 - \gamma) ^ {3}} + \frac {c _ {\max} A S \ln^ {4} (A S T / \delta)}{\eta (1 - \gamma) ^ {3}}\right) \\ \end{array}
$$
where in the inequality we use that  $(1 + \frac{1}{H})^H \leq e$  and  $\gamma^H = (1 - (1 - \gamma))^H \leq e^{-(1 - \gamma)H} = \frac{1}{T}$ .
Corollary 2. There exists a universal constant  $C_1 > 0$  such that for any  $\tilde{\epsilon} \geq \frac{C_1 A \ln^3 (AST / \delta) \beta}{(1 - \gamma)^3}$ , with probability at least  $1 - \mathcal{O}(\delta)$ ,
$$
\sum_ {t = 1} ^ {T} \mathbf {1} \left[ x _ {t} ^ {s _ {t} \top} \left(\mathbb {E} _ {s ^ {\prime} \sim P ^ {s _ {t}}} \left[ \overline {{V}} _ {t} ^ {s ^ {\prime}} - \underline {{V}} _ {t} ^ {s ^ {\prime}} \right]\right) y _ {t} ^ {s _ {t}} \geq \tilde {\epsilon} \right] \leq \mathcal {O} \left(\frac {A S \ln^ {4} (A S T / \delta)}{\eta \tilde {\epsilon} (1 - \gamma) ^ {3}}\right).
$$
Proof. We apply Lemma 25 with the following definition of  $c_t$ :
$$
c _ {t} = \mathbf {1} \left[ x _ {t} ^ {s _ {t}} \left(\mathbb {E} _ {s ^ {\prime} \sim P ^ {s _ {t}}} \left[ \overline {{V}} _ {t} ^ {s ^ {\prime}} - \underline {{V}} _ {t} ^ {s ^ {\prime}} \right]\right) y _ {t} ^ {s _ {t}} \geq \tilde {\epsilon} \right],
$$
$$
\sum_ {t = 1} ^ {T} c _ {t} \left(\bar {V} _ {t} ^ {s _ {t + 1}} - \underline {{V}} _ {t} ^ {s _ {t + 1}}\right) \leq C _ {2} \times \left(\frac {C A \ln^ {3} (A S T / \delta) \beta}{(1 - \gamma) ^ {3}} + \frac {A S \ln^ {4} (A S T / \delta)}{\eta (1 - \gamma) ^ {3}}\right) \tag {20}
$$
for some universal constant  $C_2$  and  $C = \sum_{t=1}^{T} c_t$ . By Azuma's inequality, for some universal constant  $C_3 > 0$ , with probability  $1 - \delta$ ,
$$
\begin{array}{l} \sum_ {t = 1} ^ {T} c _ {t} x _ {t} ^ {s _ {t}} \left(\mathbb {E} _ {s ^ {\prime} \sim P ^ {s _ {t}}} \left[ \bar {V} _ {t} ^ {s ^ {\prime}} - \underline {{V}} _ {t} ^ {s ^ {\prime}} \right]\right) y _ {t} ^ {s _ {t}} - \sum_ {t = 1} ^ {T} c _ {t} \left(\bar {V} _ {t} ^ {s _ {t + 1}} - \underline {{V}} _ {t} ^ {s _ {t + 1}}\right) \\ \leq \frac {C _ {3}}{1 - \gamma} \sqrt {\ln (S / \delta) \sum_ {t = 1} ^ {T} c _ {t} ^ {2}} = \frac {C _ {3}}{1 - \gamma} \sqrt {\ln (S / \delta) C} \\ \leq C _ {3} \times \frac {C \beta}{1 - \gamma} + C _ {3} \times \frac {\ln (S / \delta)}{\eta (1 - \gamma)}. \tag {21} \\ \end{array}
$$
Combining Eq. (20) and Eq. (21), we get
$$
\begin{array}{l} \sum_ {t = 1} ^ {T} c _ {t} x _ {t} ^ {s _ {t}} \left(\mathbb {E} _ {s ^ {\prime} \sim P ^ {s _ {t}}} \left[ \overline {{V}} _ {t} ^ {s ^ {\prime}} - \underline {{V}} _ {t} ^ {s ^ {\prime}} \right]\right) y _ {t} ^ {s _ {t}} \\ \leq \left(C _ {2} + C _ {3}\right) \times \left(\frac {C A \ln^ {3} (A S T / \delta) \beta}{(1 - \gamma) ^ {3}} + \frac {A S \ln^ {4} (A S T / \delta)}{\eta (1 - \gamma) ^ {3}}\right). \\ \end{array}
$$
By the definition of  $c_t$ , the left-hand side above is lower bounded by  $\tilde{\epsilon} \sum_{t=1}^{T} c_t = C \tilde{\epsilon}$ . Define  $C_1 = 2(C_2 + C_3)$ . Then by the condition on  $\epsilon'$ , the right-hand side above is above inequality is bounded by
$$
\frac {C \tilde {\epsilon}}{2} + \frac {C _ {1}}{2} \left(\frac {A S \ln^ {4} (A S T / \delta)}{\eta (1 - \gamma) ^ {3}}\right)
$$
by the condition on  $\tilde{\epsilon}$ . Combining the upper bound and the lower bound, we get
$$
C \leq C _ {1} \times \left(\frac {A S \ln^ {4} (A S T / \delta)}{\eta \tilde {\epsilon} (1 - \gamma) ^ {3}}\right).
$$
Lemma 26. With probability at least  $1 - \mathcal{O}(\delta)$ , for any  $t\geq 1$
$$
\underline {{V}} _ {t} ^ {s} \leq V _ {\star} ^ {s} + \mathcal {O} \left(\frac {\epsilon \ln (A T)}{1 - \gamma}\right), \quad \overline {{V}} _ {t} ^ {s} \geq V _ {\star} ^ {s} - \mathcal {O} \left(\frac {\epsilon \ln (A T)}{1 - \gamma}\right).
$$
Proof. Fix a  $t$  and  $s$ , let  $\tau = n_t(s)$ , and let  $t_i$  be the time index in which  $s$  is visited the  $i$ -th time. With probability at least  $1 - \frac{\delta}{ST}$ , we have
$$
\mathcal {V} _ {t} ^ {s} = \sum_ {i = 1} ^ {\tau} \alpha_ {\tau} ^ {i} \left(\sigma_ {t _ {i}} + \gamma \underline {{V}} _ {t _ {i}} ^ {s _ {t _ {i} + 1}} - \mathsf {b n s} _ {i}\right)
$$
$$
\leq \sum_ {i = 1} ^ {\tau} \alpha_ {\tau} ^ {i} \left(\underline {{f}} _ {i} ^ {s} (x _ {t _ {i}} ^ {s}, y _ {t _ {i}} ^ {s}) + \epsilon \phi (x _ {t _ {i}} ^ {s}) - \epsilon \phi (y _ {t _ {i}} ^ {s})\right) + \sum_ {i = 1} ^ {\tau} \alpha_ {\tau} ^ {i} \left(\sigma_ {t _ {i}} + \gamma \underline {{V}} _ {t _ {i}} ^ {s _ {t _ {i}} + 1} - x _ {t _ {i}} ^ {s} ^ {\top} (G ^ {s} + \mathbb {E} _ {s ^ {\prime} \sim P ^ {s t} [ V _ {t _ {i}} ^ {s ^ {\prime}} ]} y _ {t _ {i}} ^ {s}) \right.
$$
$$
- \sum_ {i = 1} ^ {\tau} \alpha_ {\tau} ^ {i} \mathsf {b n s} _ {i}
$$
$$
\leq \sum_ {i = 1} ^ {\tau} \alpha_ {\tau} ^ {i} \left(\underline {{f}} _ {i} ^ {s} \left(x _ {t _ {i}} ^ {s}, y _ {t _ {i}} ^ {s}\right) + \epsilon \phi \left(x _ {t _ {i}} ^ {s}\right) - \epsilon \phi \left(y _ {t _ {i}} ^ {s}\right)\right) + \frac {1}{1 - \gamma} \sqrt {2 \sum_ {i = 1} ^ {\tau} \left(\alpha_ {\tau} ^ {i}\right) ^ {2} \log (S T / \delta)} - \sum_ {i = 1} ^ {\tau} \alpha_ {\tau} ^ {i} \mathsf {b n s} _ {i}
$$
(byHoeffding'sinequality)
$$
\leq \sum_ {i = 1} ^ {\tau} \alpha_ {\tau} ^ {i} \left(\underline {{f}} _ {i} ^ {s} \left(x _ {t _ {i}} ^ {s}, y _ {t _ {i}} ^ {s}\right) + \epsilon \phi \left(x _ {t _ {i}} ^ {s}\right) - \epsilon \phi \left(y _ {t _ {i}} ^ {s}\right)\right) + \frac {1}{1 - \gamma} \sqrt {2 \alpha_ {\tau} \log (S T / \delta)} - \sum_ {i = 1} ^ {\tau} \alpha_ {\tau} ^ {i} \mathsf {b n s} _ {i} (\sum_ {i = 1} ^ {\tau} \alpha_ {\tau} ^ {i} \leq 1)
$$
$$
\leq \sum_ {i = 1} ^ {\tau} \alpha_ {\tau} ^ {i} \left(\underline {{f}} _ {i} ^ {s} \left(x _ {t _ {i}} ^ {s}, y _ {t _ {i}} ^ {s}\right) + \epsilon \phi \left(x _ {t _ {i}} ^ {s}\right) - \epsilon \phi \left(y _ {t _ {i}} ^ {s}\right)\right) - \frac {1}{2} \mathsf {b n s} _ {\tau}
$$
$(\sum_{i=1}^{\tau} \alpha_{\tau}^{i} \geq \frac{1}{2}$  and  $\mathsf{bns}_{\tau}$  is decreasing, and  $\sqrt{\alpha_{\tau}} \leq \frac{\alpha_{\tau}}{\eta} + \eta \leq \frac{\alpha_{\tau}}{\eta} + \beta)$
$$
\leq \min  _ {x \in \Omega} \sum_ {i = 1} ^ {\tau} \alpha_ {\tau} ^ {i} \left(\underline {{f}} _ {i} ^ {s} \left(x ^ {s}, y _ {t _ {i}} ^ {s}\right)\right) + \sum_ {i = 1} ^ {\tau} \alpha_ {\tau} ^ {i} \left(\epsilon \phi \left(x _ {t _ {i}} ^ {s}\right) - \epsilon \phi \left(y _ {t _ {i}} ^ {s}\right)\right) \tag {byLemma23}
$$
$$
\leq \min  _ {x \in \Omega} \sum_ {i = 1} ^ {\tau} \alpha_ {\tau} ^ {i} \left(x ^ {s}\right) ^ {\top} \left(G ^ {s} + \gamma \mathbb {E} _ {s ^ {\prime} \sim P ^ {s}} \left[ \underline {{V}} _ {t _ {i}} ^ {s ^ {\prime}} \right]\right) y _ {t _ {i}} ^ {s} + \mathcal {O} (\epsilon \ln (A T)). (x _ {a} \geq \frac {1}{A T} \text {f o r a n y} x \in \Omega .)
$$
Therefore, using a union bound over  $s$  and  $t$ , we have with probability  $1 - \delta$ , for all  $s$  and  $t$ ,
$$
\underline {{V}} _ {t} ^ {s} = \max  \left\{\underline {{V}} _ {t} ^ {s}, 0 \right\} \leq \min  _ {x} \sum_ {i = 1} ^ {\tau} \alpha_ {\tau} ^ {i} \left(x ^ {s}\right) ^ {\top} \left(G ^ {s} + \gamma \mathbb {E} _ {s ^ {\prime} \sim P ^ {s}} \left[ \underline {{V}} _ {t _ {i}} ^ {s ^ {\prime}} \right]\right) y _ {t _ {i}} ^ {s} + C _ {4} \epsilon \ln (A T) \tag {22}
$$
for some universal constant  $C_4$ . Next, we use induction to show the first inequality. Suppose that
$$
\underline {{V}} _ {t ^ {\prime}} ^ {s} \leq V _ {\star} ^ {s} + \frac {C _ {4} \epsilon \ln (A T)}{1 - \gamma}
$$
for all  $s$  and  $t^{\prime} <   t$  . Then by Eq. (22),
$$
\begin{array}{l} \underline {{V}} _ {t} ^ {s} \leq \min  _ {x} \sum_ {i = 1} ^ {\tau} \alpha_ {\tau} ^ {i} \left(x ^ {s}\right) ^ {\top} \left(G ^ {s} + \gamma \mathbb {E} _ {s ^ {\prime} \sim P ^ {s}} \left[ V _ {\star} ^ {s ^ {\prime}} + \frac {C _ {4} \epsilon \ln (A T)}{1 - \gamma} \right]\right) y _ {t _ {i}} ^ {s} + C _ {4} \epsilon \ln (A T) \\ = \min  _ {x} \sum_ {i = 1} ^ {\tau} \alpha_ {\tau} ^ {i} \left(x ^ {s}\right) ^ {\top} \left(G ^ {s} + \gamma \mathbb {E} _ {s ^ {\prime} \sim P ^ {s}} \left[ V _ {\star} ^ {s ^ {\prime}} \right]\right) y _ {t _ {i}} ^ {s} + \frac {C _ {4} \epsilon \ln (A T)}{1 - \gamma} \\ \leq \min  _ {x} \sum_ {i = 1} ^ {\tau} \max  _ {y} \alpha_ {\tau} ^ {i} \left(x ^ {s}\right) ^ {\top} \left(G ^ {s} + \gamma \mathbb {E} _ {s ^ {\prime} \sim P ^ {s}} \left[ V _ {\star} ^ {s ^ {\prime}} \right]\right) y ^ {s} + \frac {C _ {4} \epsilon \ln (A T)}{1 - \gamma} \\ = \min  _ {x} \max  _ {y} \left(x ^ {s}\right) ^ {\top} \left(G ^ {s} + \gamma \mathbb {E} _ {s ^ {\prime}} \sim P ^ {s} \left[ V _ {\star} ^ {s ^ {\prime}} \right]\right) y ^ {s} + \frac {C _ {4} \epsilon \ln (A T)}{1 - \gamma} \\ = V _ {\star} ^ {s} + \frac {C _ {4} \epsilon \ln (A T)}{1 - \gamma}, \\ \end{array}
$$
which proves the first desired inequality. The other inequality can be proven in the same way.
E.3 Part III. Policy Convergence to the Nash of the Regularized Game
Lemma 27. Let  $0 \leq p \leq 1$  be arbitrarily chosen, and define
$$
\begin{array}{l} f _ {\tau} ^ {s} \left(x ^ {s}, y ^ {s}\right) \triangleq p \underline {{f}} _ {\tau} ^ {s} \left(x ^ {s}, y ^ {s}\right) + (1 - p) \bar {f} _ {\tau} ^ {s} \left(x ^ {s}, y ^ {s}\right) \\ = x ^ {s \top} \left(G ^ {s} + \mathbb {E} _ {s ^ {\prime} \sim P ^ {s}} \left[ p \underline {{V}} _ {t _ {\tau} (s)} ^ {s ^ {\prime}} + (1 - p) \overline {{V}} _ {t _ {\tau} (s)} ^ {s ^ {\prime}} \right]\right) y ^ {s} - \epsilon \phi (x ^ {s}) + \epsilon \phi (y ^ {s}). \\ \end{array}
$$
Furthermore, let  $\hat{z}_{\tau \star}^{s} = (\hat{x}_{\tau \star}^{s}, \hat{y}_{\tau \star}^{s})$  be the equilibrium of  $f_{\tau}^{s}(x,y)$ , and define  $z_{t\star}^{s} = \hat{z}_{\tau \star}^{s}$  where  $\tau = n_t(s)$ . Then with probability at least  $1 - \mathcal{O}(\delta)$ , the following holds for any  $0 < \epsilon' \leq 1$ :