SlowGuess/NeurIPS_2023_OCR huggingface dataset:
$$
\begin{array}{l} \mathbb {E} \left[ \left(\hat {w} _ {\theta_ {t}} ^ {*} - \hat {w} _ {\theta_ {t + 1}} ^ {*}\right) ^ {T} \left(w _ {t} - \hat {w} _ {\theta_ {t}} ^ {*} + \alpha^ {w} \bar {f} \left(w _ {t}, \theta_ {t}\right)\right) | \mathcal {F} _ {t} \right] \\ \leq \left(\sqrt {2} \alpha^ {\theta} L _ {w} \left\| \nabla V _ {\theta_ {t}} ^ {*} \right\| + \sqrt {2} \alpha^ {\theta} L _ {w} \sqrt {c _ {1} \mathcal {N} _ {\theta_ {t}} \left(Q (w _ {t}) - \hat {Q} _ {\theta_ {t}} ^ {*}\right) + c _ {2} \epsilon^ {2}} + \frac {\alpha^ {\theta^ {2}} H _ {w} U _ {g} ^ {2}}{\sqrt {2} (1 - \gamma) ^ {2}}\right) \\ \cdot \sqrt {\left(\frac {1}{\lambda_ {\operatorname* {m i n}}} - 2 \alpha^ {w}\right) \mathcal {N} _ {\theta_ {t}} \left(Q \left(w _ {t}\right) - \hat {Q} _ {\theta_ {t}} ^ {*}\right) + C _ {w , 1}} \\ \leq \frac {1}{\sqrt {2}} \alpha^ {\theta} L _ {w} \left\| \nabla V _ {\theta_ {t}} ^ {*} \right\| ^ {2} + \frac {1}{\sqrt {2}} \alpha^ {\theta} L _ {w} \left(\frac {1}{\lambda_ {\min }} - 2 \alpha^ {w}\right) \mathcal {N} _ {\theta_ {t}} \left(Q (w _ {t}) - \hat {Q} _ {\theta_ {t}} ^ {*}\right) + \frac {1}{\sqrt {2}} \alpha^ {\theta} L _ {w} C _ {w, 1} \\ + \sqrt {2} \alpha^ {\theta} L _ {w} \sqrt {\left[ c _ {1} \mathcal {N} _ {\theta_ {t}} (Q (w _ {t}) - \hat {Q} _ {\theta_ {t}} ^ {*}) + c _ {2} \epsilon^ {2} \right] \cdot \left[ \left(\frac {1}{\lambda_ {\operatorname* {m i n}}} - 2 \alpha^ {w}\right) \mathcal {N} _ {\theta_ {t}} (Q (w _ {t}) - \hat {Q} _ {\theta_ {t}} ^ {*}) + C _ {w , 1} \right]} \\ + \frac {\alpha^ {\theta^ {2}} H _ {w} U _ {g} ^ {2}}{\sqrt {2} (1 - \gamma) ^ {2}} \sqrt {\left(\frac {1}{\lambda_ {\operatorname* {m i n}}} - 2 \alpha^ {w}\right) \mathcal {N} _ {\theta_ {t}} \left(Q \left(w _ {t}\right) - \hat {Q} _ {\theta_ {t}} ^ {*}\right)} + C _ {w, 1} \\ \leq \frac {1}{\sqrt {2}} \alpha^ {\theta} L _ {w} \left\| \nabla V _ {\theta_ {t}} ^ {*} \right\| ^ {2} \\ + \left(\frac {1}{\sqrt {2}} \alpha^ {\theta} L _ {w} \left(\frac {1}{\lambda_ {\min }} - 2 \alpha^ {w}\right) + \sqrt {2} \alpha^ {\theta} L _ {w} \sqrt {c _ {1} \left(\frac {1}{\lambda_ {\min }} - 2 \alpha^ {w}\right)}\right) \cdot \mathcal {N} _ {\theta_ {t}} (Q (w _ {t}) - \hat {Q} _ {\theta_ {t}} ^ {*}) \\ + \left(\sqrt {2} \alpha^ {\theta} L _ {w} \sqrt {c _ {1} C _ {w , 1} + c _ {2} \epsilon^ {2} \left(\frac {1}{\lambda_ {\min}} - 2 \alpha^ {w}\right)} + \frac {\alpha^ {\theta^ {2}} H _ {w} U _ {g} ^ {2}}{\sqrt {2} (1 - \gamma) ^ {2}} \sqrt {\frac {1}{\lambda_ {\min}} - 2 \alpha^ {w}}\right) \sqrt {\mathcal {N} _ {\theta_ {t}} (Q (w _ {t}) - \hat {Q} _ {\theta_ {t}} ^ {*})} \\ + \frac {1}{\sqrt {2}} \alpha^ {\theta} L _ {w} C _ {w, 1} + \sqrt {2} \alpha^ {\theta} L _ {w} \sqrt {c _ {2} C _ {w , 1} \epsilon^ {2}} + \frac {\sqrt {C _ {w , 1}} \alpha^ {\theta^ {2}} H _ {w} U _ {g} ^ {2}}{\sqrt {2} (1 - \gamma) ^ {2}}. \\ \end{array}
$$
After taking expectation on both sides, we know
$$
\begin{array}{l} \mathbb {E} \left[ \left(\hat {w} _ {\theta_ {t}} ^ {*} - \hat {w} _ {\theta_ {t + 1}} ^ {*}\right) ^ {T} \left(w _ {t} - \hat {w} _ {\theta_ {t}} ^ {*} + \alpha^ {w} \bar {f} \left(w _ {t}, \theta_ {t}\right)\right) \right] \\ \leq \frac {1}{\sqrt {2}} \alpha^ {\theta} L _ {w} \mathbb {E} \left[ \left\| \nabla V _ {\theta_ {t}} ^ {*} \right\| ^ {2} \right] \\ + \left(\frac {1}{\sqrt {2}} \alpha^ {\theta} L _ {w} \left(\frac {1}{\lambda_ {\min }} - 2 \alpha^ {w}\right) + \sqrt {2} \alpha^ {\theta} L _ {w} \sqrt {c _ {1} \left(\frac {1}{\lambda_ {\min }} - 2 \alpha^ {w}\right)}\right) \cdot \mathbb {E} \left[ \mathcal {N} _ {\theta_ {t}} \left(Q (w _ {t}) - \hat {Q} _ {\theta_ {t}} ^ {*}\right) \right] \\ + \left(\sqrt {2} \alpha^ {\theta} L _ {w} \sqrt {c _ {1} C _ {w , 1} + c _ {2} \epsilon^ {2} \left(\frac {1}{\lambda_ {\min}} - 2 \alpha^ {w}\right)} + \frac {\alpha^ {\theta^ {2}} H _ {w} U _ {g} ^ {2}}{\sqrt {2} (1 - \gamma) ^ {2}} \sqrt {\frac {1}{\lambda_ {\min}} - 2 \alpha^ {w}}\right) \sqrt {\mathbb {E} \left[ \mathcal {N} _ {\theta_ {t}} (Q (w _ {t}) - \hat {Q} _ {\theta_ {t}} ^ {*}) \right]} \\ + \frac {1}{\sqrt {2}} \alpha^ {\theta} L _ {w} C _ {w, 1} + \sqrt {2} \alpha^ {\theta} L _ {w} \sqrt {c _ {2} C _ {w , 1} \epsilon^ {2}} + \frac {\sqrt {C _ {w , 1}} \alpha^ {\theta^ {2}} H _ {w} U _ {g} ^ {2}}{\sqrt {2} (1 - \gamma) ^ {2}}, \\ \end{array}
$$
which is the bound for  $I_{2,1}$
For  $I_{2,2}$ , by Lemma C.18, we have
This ends the bound for  $I_{2}$ , which, after combining the bound for  $I_{2,1}$  and  $I_{2,2}$ , will be
$$
\begin{array}{l} \mathbb {E} \left[ \left(\hat {w} _ {\theta_ {t}} ^ {*} - \hat {w} _ {\theta_ {t + 1}} ^ {*}\right) ^ {T} \left(w _ {t} - \hat {w} _ {\theta_ {t}} ^ {*} + \alpha^ {w} f \left(O _ {t}, w _ {t}\right)\right) \right] \\ \leq \frac {1}{\sqrt {2}} \alpha^ {\theta} L _ {w} \mathbb {E} \left[ \left\| \nabla V _ {\theta_ {t}} ^ {*} \right\| ^ {2} \right] \\ + \left(\frac {1}{\sqrt {2}} \alpha^ {\theta} L _ {w} \left(\frac {1}{\lambda_ {\min }} - 2 \alpha^ {w}\right) + \sqrt {2} \alpha^ {\theta} L _ {w} \sqrt {c _ {1} \left(\frac {1}{\lambda_ {\min }} - 2 \alpha^ {w}\right)}\right) \cdot \mathbb {E} \left[ \mathcal {N} _ {\theta_ {t}} \left(Q (w _ {t}) - \hat {Q} _ {\theta_ {t}} ^ {*}\right) \right] \\ + \left(\sqrt {2} \alpha^ {\theta} L _ {w} \sqrt {c _ {1} C _ {w , 1} + c _ {2} \epsilon^ {2} \left(\frac {1}{\lambda_ {\min}} - 2 \alpha^ {w}\right)} + \frac {\alpha^ {\theta^ {2}} H _ {w} U _ {g} ^ {2}}{\sqrt {2} (1 - \gamma) ^ {2}} \sqrt {\frac {1}{\lambda_ {\min}} - 2 \alpha^ {w}}\right) \sqrt {\mathbb {E} \left[ \mathcal {N} _ {\theta_ {t}} \left(Q (w _ {t}) - \hat {Q} _ {\theta_ {t}} ^ {*}\right) \right]} \\ + \frac {1}{\sqrt {2}} \alpha^ {\theta} L _ {w} C _ {w, 1} + \sqrt {2} \alpha^ {\theta} L _ {w} \sqrt {c _ {2} C _ {w , 1} \epsilon^ {2}} + \frac {\sqrt {C _ {w , 1}} \alpha^ {\theta^ {2}} H _ {w} U _ {g} ^ {2}}{\sqrt {2} (1 - \gamma) ^ {2}} + 2 \alpha^ {w} U _ {f} \left(\frac {\sqrt {2} \alpha^ {\theta} L _ {w} U _ {g}}{1 - \gamma} + \frac {\alpha^ {\theta^ {2}} H _ {w} U _ {g} ^ {2}}{\sqrt {2} (1 - \gamma) ^ {2}}\right). \\ \end{array}
$$
To analyze  $I_{3}$ , we have
$$
\mathbb {E} \left[ | | \hat {w} _ {\theta_ {t}} ^ {*} - \hat {w} _ {\theta_ {t + 1}} ^ {*} | | ^ {2} | \mathcal {F} _ {t} \right] \leq L _ {w} ^ {2} \mathbb {E} \left[ | | \theta_ {t + 1} - \theta_ {t} | | ^ {2} | \mathcal {F} _ {t} \right].
$$
$$
\theta_ {t + 1} - \theta_ {t} = - \alpha^ {\theta} \nabla V _ {\theta_ {t}} ^ {*} - \alpha^ {\theta} D _ {Q} + \alpha^ {\theta} \bar {g} (w _ {t}, \theta_ {t}) - \frac {\alpha^ {\theta}}{1 - \gamma} g (\hat {O} _ {t}, w _ {t}, \theta_ {t}),
$$
$$
\begin{array}{l} \mathbb {E} \left[ | | \theta_ {t + 1} - \theta_ {t} | | ^ {2} | \mathcal {F} _ {t} \right] \leq | | \alpha^ {\theta} \nabla V _ {\theta_ {t}} ^ {*} + \alpha^ {\theta} D _ {Q} | | ^ {2} + \frac {4}{(1 - \gamma) ^ {2}} \alpha^ {\theta^ {2}} U _ {g} ^ {2} \\ \leq 2 \alpha^ {\theta^ {2}} \left(\left| \left| \nabla V _ {\theta_ {t}} ^ {*} \right| \right| ^ {2} + c _ {1} \cdot \mathcal {N} _ {\theta_ {t}} \left(Q (w _ {t}) - \hat {Q} _ {\theta_ {t}} ^ {*}\right) + c _ {2} \epsilon^ {2}\right) + \frac {4}{(1 - \gamma) ^ {2}} \alpha^ {\theta^ {2}} U _ {g} ^ {2}, \\ \end{array}
$$
where we use the fact that  $\mathbb{E}\left[\bar{g} (w_t,\theta_t) - \frac{1}{1 - \gamma} g(\hat{O}_t,w_t,\theta_t)|\mathcal{F}_t\right] = 0$ . Hence,
$$
\mathbb {E} \left[ | | \hat {w} _ {\theta_ {t}} ^ {*} - \hat {w} _ {\theta_ {t + 1}} ^ {*} | | ^ {2} | \mathcal {F} _ {t} \right] \leq 2 \alpha^ {\theta^ {2}} L _ {w} ^ {2} \left(| | \nabla V _ {\theta_ {t}} ^ {*} | | ^ {2} + c _ {1} \cdot \mathcal {N} _ {\theta_ {t}} (Q (w _ {t}) - \hat {Q} _ {\theta_ {t}} ^ {*}) + c _ {2} \epsilon^ {2}\right) + \frac {4 L _ {w} ^ {2}}{(1 - \gamma) ^ {2}} \alpha^ {\theta^ {2}} U _ {g} ^ {2}.
$$
After taking expectation on both side, we will arrive at the bound for  $I_{3}$ , which is
$$
\mathbb {E} \left[ | | \hat {w} _ {\theta_ {t}} ^ {*} - \hat {w} _ {\theta_ {t + 1}} ^ {*} | | ^ {2} \right] \leq 2 \alpha^ {\theta^ {2}} L _ {w} ^ {2} \left(\mathbb {E} \left[ | | \nabla V _ {\theta_ {t}} ^ {*} | | ^ {2} \right] + c _ {1} \cdot \mathbb {E} \left[ \mathcal {N} _ {\theta_ {t}} (Q (w _ {t}) - \hat {Q} _ {\theta_ {t}} ^ {*}) \right] + c _ {2} \epsilon^ {2}\right) + \frac {4 L _ {w} ^ {2}}{(1 - \gamma) ^ {2}} \alpha^ {\theta^ {2}} U _ {g} ^ {2}.
$$
Now, go back to Eq.(17) and we obtain
$$
\begin{array}{l} \mathbb {E} \left[ | | w _ {t + 1} - \hat {w} _ {\theta_ {t + 1}} ^ {*} | | ^ {2} \right] \\ \leq \mathbb {E} \left[ \left| \left| w _ {t} - \hat {w} _ {\theta_ {t}} ^ {*} \right| \right| ^ {2} \right] + \left(\frac {1}{\sqrt {2}} \alpha^ {\theta} L _ {w} + 2 \alpha^ {\theta^ {2}} L _ {w} ^ {2}\right) \mathbb {E} \left[ \left| \left| \nabla V _ {\theta_ {t}} ^ {*} \right| \right| ^ {2} \right] \\ + \left(\frac {1}{\sqrt {2}} \alpha^ {\theta} L _ {w} \left(\frac {1}{\lambda_ {\operatorname* {m i n}}} - 2 \alpha^ {w}\right) + \sqrt {2} \alpha^ {\theta} L _ {w} \sqrt {c _ {1} \left(\frac {1}{\lambda_ {\operatorname* {m i n}}} - 2 \alpha^ {w}\right)} + 2 \alpha^ {\theta^ {2}} L _ {w} ^ {2} c _ {1} - 2 \alpha^ {w}\right) \cdot \mathbb {E} \left[ \mathcal {N} _ {\theta_ {t}} (Q (w _ {t}) - \hat {Q} _ {\theta_ {t}} ^ {*}) \right] \\ + \left(\sqrt {2} \alpha^ {\theta} L _ {w} \sqrt {c _ {1} C _ {w , 1} + c _ {2} \epsilon^ {2} \left(\frac {1}{\lambda_ {\mathrm {m i n}}} - 2 \alpha^ {w}\right)} + \frac {\alpha^ {\theta^ {2}} H _ {w} U _ {g} ^ {2}}{\sqrt {2} (1 - \gamma) ^ {2}} \sqrt {\frac {1}{\lambda_ {\mathrm {m i n}}} - 2 \alpha^ {w}}\right) \sqrt {\mathbb {E} \left[ \mathcal {N} _ {\theta_ {t}} (Q (w _ {t}) - \hat {Q} _ {\theta_ {t}} ^ {*}) \right]} \\ + C _ {w, 1} + C _ {w, 2} + \frac {1}{\sqrt {2}} \alpha^ {\theta} L _ {w} C _ {w, 1} + \sqrt {2} \alpha^ {\theta} L _ {w} \sqrt {c _ {2} C _ {w , 1} \epsilon^ {2}} + \frac {\sqrt {C _ {w , 1}} \alpha^ {\theta^ {2}} H _ {w} U _ {g} ^ {2}}{\sqrt {2} (1 - \gamma) ^ {2}} \\ + 2 \alpha^ {w} U _ {f} \left(\frac {\sqrt {2} \alpha^ {\theta} L _ {w} U _ {g}}{1 - \gamma} + \frac {\alpha^ {\theta^ {2}} H _ {w} U _ {g} ^ {2}}{\sqrt {2} (1 - \gamma) ^ {2}}\right) + 2 \alpha^ {\theta^ {2}} L _ {w} ^ {2} c _ {2} \epsilon^ {2} + \frac {4 L _ {w} ^ {2}}{(1 - \gamma) ^ {2}} \alpha^ {\theta^ {2}} U _ {g} ^ {2}. \\ \end{array}
$$
Now, we can do a telescoping sum for  $i$  to  $T$ :
$$
\begin{array}{l} \left(2 \alpha^ {w} - \frac {1}{\sqrt {2}} \alpha^ {\theta} L _ {w} \left(\frac {1}{\lambda_ {\mathrm {m i n}}} - 2 \alpha^ {w}\right) - \sqrt {2} \alpha^ {\theta} L _ {w} \sqrt {c _ {1} \left(\frac {1}{\lambda_ {\mathrm {m i n}}} - 2 \alpha^ {w}\right)} - 2 \alpha^ {\theta 2} L _ {w} ^ {2} c _ {1}\right) \cdot \left(\frac {1}{T} \sum_ {t = 1} ^ {T} \mathbb {E} \left[ \mathcal {N} _ {\theta_ {t}} (Q (w _ {t}) - \hat {Q} _ {\theta_ {t}} ^ {*}) \right]\right) \\ \leq \frac {\mathbb {E} \left[ | | w _ {1} - \hat {w} _ {\theta_ {1}} ^ {*} | | ^ {2} \right]}{T} + \left(\frac {1}{\sqrt {2}} \alpha^ {\theta} L _ {w} + 2 \alpha^ {\theta^ {2}} L _ {w} ^ {2}\right) \left(\frac {1}{T} \sum_ {t = 1} ^ {T} \mathbb {E} \left[ \left| | \nabla V _ {\theta_ {t}} ^ {*} \right| ^ {2} \right]\right) \\ + \left(\sqrt {2} \alpha^ {\theta} L _ {w} \sqrt {c _ {1} C _ {w , 1} + c _ {2} \epsilon^ {2} \left(\frac {1}{\lambda_ {\operatorname* {m i n}}} - 2 \alpha^ {w}\right)} + \frac {\alpha^ {\theta^ {2}} H _ {w} U _ {g} ^ {2}}{\sqrt {2} (1 - \gamma) ^ {2}} \sqrt {\frac {1}{\lambda_ {\operatorname* {m i n}}} - 2 \alpha^ {w}}\right) \sqrt {\frac {1}{T} \sum_ {t = 1} ^ {T} \mathbb {E} \left[ \mathcal {N} _ {\theta_ {t}} \left(Q (w _ {t}) - \hat {Q} _ {\theta_ {t}} ^ {*}\right) \right]} \\ + C _ {w, 1} + C _ {w, 2} + \frac {1}{\sqrt {2}} \alpha^ {\theta} L _ {w} C _ {w, 1} + \sqrt {2} \alpha^ {\theta} L _ {w} \sqrt {c _ {2} C _ {w , 1} \epsilon^ {2}} + \frac {\sqrt {C _ {w , 1}} \alpha^ {\theta^ {2}} H _ {w} U _ {g} ^ {2}}{\sqrt {2} (1 - \gamma) ^ {2}} \\ + 2 \alpha^ {w} U _ {f} \left(\frac {\sqrt {2} \alpha^ {\theta} L _ {w} U _ {g}}{1 - \gamma} + \frac {\alpha^ {\theta^ {2}} H _ {w} U _ {g} ^ {2}}{\sqrt {2} (1 - \gamma) ^ {2}}\right) + 2 \alpha^ {\theta^ {2}} L _ {w} ^ {2} c _ {2} \epsilon^ {2} + \frac {4 L _ {w} ^ {2}}{(1 - \gamma) ^ {2}} \alpha^ {\theta^ {2}} U _ {g} ^ {2}, \\ \end{array}
$$
which, if we adopt notations from Eq.(13), can be rewritten as
$$
\begin{array}{l} \left(2 \alpha^ {w} - \frac {1}{\sqrt {2}} \alpha^ {\theta} L _ {w} \left(\frac {1}{\lambda_ {\min }} - 2 \alpha^ {w}\right) - \sqrt {2} \alpha^ {\theta} L _ {w} \sqrt {c _ {1} \left(\frac {1}{\lambda_ {\min }} - 2 \alpha^ {w}\right)} - 2 \alpha^ {\theta 2} L _ {w} ^ {2} c _ {1}\right) \cdot \Delta_ {Q} \\ \leq \frac {\mathbb {E} \left[ | | w _ {1} - \hat {w} _ {\theta_ {1}} ^ {*} | | ^ {2} \right]}{T} + \left(\frac {1}{\sqrt {2}} \alpha^ {\theta} L _ {w} + 2 \alpha^ {\theta^ {2}} L _ {w} ^ {2}\right) \Delta_ {V} \\ + \left(\sqrt {2} \alpha^ {\theta} L _ {w} \sqrt {c _ {1} C _ {w , 1} + c _ {2} \epsilon^ {2} \left(\frac {1}{\lambda_ {\min}} - 2 \alpha^ {w}\right)} + \frac {\alpha^ {\theta^ {2}} H _ {w} U _ {g} ^ {2}}{\sqrt {2} (1 - \gamma) ^ {2}} \sqrt {\frac {1}{\lambda_ {\min}} - 2 \alpha^ {w}}\right) \sqrt {\Delta_ {Q}} \\ + C _ {w, 1} + C _ {w, 2} + \frac {1}{\sqrt {2}} \alpha^ {\theta} L _ {w} C _ {w, 1} + \sqrt {2} \alpha^ {\theta} L _ {w} \sqrt {c _ {2} C _ {w , 1} \epsilon^ {2}} + \frac {\sqrt {C _ {w , 1}} \alpha^ {\theta^ {2}} H _ {w} U _ {g} ^ {2}}{\sqrt {2} (1 - \gamma) ^ {2}} \\ + 2 \alpha^ {w} U _ {f} \left(\frac {\sqrt {2} \alpha^ {\theta} L _ {w} U _ {g}}{1 - \gamma} + \frac {\alpha^ {\theta^ {2}} H _ {w} U _ {g} ^ {2}}{\sqrt {2} (1 - \gamma) ^ {2}}\right) + 2 \alpha^ {\theta^ {2}} L _ {w} ^ {2} c _ {2} \epsilon^ {2} + \frac {4 L _ {w} ^ {2}}{(1 - \gamma) ^ {2}} \alpha^ {\theta^ {2}} U _ {g} ^ {2}. \\ \end{array}
$$
F Critic Update Analysis under the i.i.d. Sampling Case
Lemma F.1. In the i.i.d. sampling case,
$$
\begin{array}{l} \left(2 \alpha^ {w} - \frac {1}{\sqrt {2}} \alpha^ {\theta} L _ {w} \left(\frac {1}{\lambda_ {\mathrm {m i n}}} - 2 \alpha^ {w}\right) - \sqrt {2} \alpha^ {\theta} L _ {w} \sqrt {c _ {1} \left(\frac {1}{\lambda_ {\mathrm {m i n}}} - 2 \alpha^ {w}\right)} - 2 \alpha^ {\theta^ {2}} L _ {w} ^ {2} c _ {1}\right) \cdot \Delta_ {Q} \\ \leq \frac {\mathbb {E} \left[ | | w _ {1} - \hat {w} _ {\theta_ {1}} ^ {*} | | ^ {2} \right]}{T} + \left(\frac {1}{\sqrt {2}} \alpha^ {\theta} L _ {w} + 2 \alpha^ {\theta^ {2}} L _ {w} ^ {2}\right) \Delta_ {V} \\ + \left(\sqrt {2} \alpha^ {\theta} L _ {w} \sqrt {c _ {1} C _ {w , 1} + c _ {2} \epsilon^ {2} \left(\frac {1}{\lambda_ {\min}} - 2 \alpha^ {w}\right)} + \frac {\alpha^ {\theta^ {2}} H _ {w} U _ {g} ^ {2}}{\sqrt {2} (1 - \gamma) ^ {2}} \sqrt {\frac {1}{\lambda_ {\min}} - 2 \alpha^ {w}}\right) \sqrt {\Delta_ {Q}} \\ + C _ {w, 1} + \frac {1}{\sqrt {2}} \alpha^ {\theta} L _ {w} C _ {w, 1} + \sqrt {2} \alpha^ {\theta} L _ {w} \sqrt {c _ {2} C _ {w , 1} \epsilon^ {2}} + \frac {\sqrt {C _ {w , 1}} \alpha^ {\theta^ {2}} H _ {w} U _ {g} ^ {2}}{\sqrt {2} (1 - \gamma) ^ {2}} \\ + 2 \alpha^ {\theta^ {2}} L _ {w} ^ {2} c _ {2} \epsilon^ {2} + \frac {4 L _ {w} ^ {2}}{(1 - \gamma) ^ {2}} \alpha^ {\theta^ {2}} U _ {g} ^ {2}. \\ \end{array}
$$
Proof. The i.i.d. assumption implies that  $f(O_{t},w_{t})$  is replaced by  $\bar{f} (w_{t},\theta_{t})$ . This will bring a change in both the analysis for  $I_{1}$  and  $I_{2}$ .
First, we will figure out how the i.i.d. sampling effect  $I_{1}$ . Now we know that  $I_{1,3}$  in Eq.(18) is 0. That means, for  $I_{1}$  in Eq.(17),
$$
\begin{array}{l} \mathbb {E} \left[ | | w _ {t} - \hat {w} _ {\theta_ {t}} ^ {*} + \alpha^ {w} f (O _ {t}, w _ {t}) | | ^ {2} | \mathcal {F} _ {t} \right] \leq - 2 \alpha^ {w} \mathcal {N} _ {\theta_ {t}} (Q (w _ {t}) - \hat {Q} _ {\theta_ {t}} ^ {*}) + | | w _ {t} - \hat {w} _ {\theta_ {t}} ^ {*} | | ^ {2} \\ + 1 6 \alpha^ {w} (1 + \gamma) L _ {Q} H _ {Q} \sigma_ {w} ^ {3} + 4 \alpha^ {w} (1 + \gamma) L _ {Q} \epsilon \sigma_ {w} + \alpha^ {w ^ {2}} U _ {f} ^ {2} \\ = - 2 \alpha^ {w} \mathcal {N} _ {\theta_ {t}} \left(Q (w _ {t}) - \hat {Q} _ {\theta_ {t}} ^ {*}\right) + | | w _ {t} - \hat {w} _ {\theta_ {t}} ^ {*} | | ^ {2} + C _ {w, 1}, \\ \end{array}
$$
where  $C_{w,1}$  is defined the same as before:
$$
C _ {w, 1} = 1 6 \alpha^ {w} (1 + \gamma) L _ {Q} H _ {Q} \sigma_ {w} ^ {3} + 4 \alpha^ {w} (1 + \gamma) L _ {Q} \epsilon \sigma_ {w} + \alpha^ {w 2} U _ {f} ^ {2}.
$$
Next, after a removal of  $I_{2,2}$  term ( $I_{2,2}$  will just be 0 if we replace  $f$  by  $\bar{f}$ ), we can derive the new bound for  $I_{2}$ , which is
$$
\begin{array}{l} \mathbb {E} \left[ (\hat {w} _ {\theta_ {t}} ^ {*} - \hat {w} _ {\theta_ {t + 1}} ^ {*}) ^ {T} (w _ {t} - \hat {w} _ {\theta_ {t}} ^ {*} + \alpha^ {w} f (O _ {t}, w _ {t})) \right] \\ \leq \frac {1}{\sqrt {2}} \alpha^ {\theta} L _ {w} \mathbb {E} \left[ \left\| \nabla V _ {\theta_ {t}} ^ {*} \right\| ^ {2} \right] \\ + \left(\frac {1}{\sqrt {2}} \alpha^ {\theta} L _ {w} \left(\frac {1}{\lambda_ {\min }} - 2 \alpha^ {w}\right) + \sqrt {2} \alpha^ {\theta} L _ {w} \sqrt {c _ {1} \left(\frac {1}{\lambda_ {\min }} - 2 \alpha^ {w}\right)}\right) \cdot \mathbb {E} \left[ \mathcal {N} _ {\theta_ {t}} \left(Q (w _ {t}) - \hat {Q} _ {\theta_ {t}} ^ {*}\right) \right] \\ + \left(\sqrt {2} \alpha^ {\theta} L _ {w} \sqrt {c _ {1} C _ {w , 1} + c _ {2} \epsilon^ {2} \left(\frac {1}{\lambda_ {\min}} - 2 \alpha^ {w}\right)} + \frac {\alpha^ {\theta^ {2}} H _ {w} U _ {g} ^ {2}}{\sqrt {2} (1 - \gamma) ^ {2}} \sqrt {\frac {1}{\lambda_ {\min}} - 2 \alpha^ {w}}\right) \sqrt {\mathbb {E} \left[ \mathcal {N} _ {\theta_ {t}} (Q (w _ {t}) - \hat {Q} _ {\theta_ {t}} ^ {*}) \right]} \\ + \frac {1}{\sqrt {2}} \alpha^ {\theta} L _ {w} C _ {w, 1} + \sqrt {2} \alpha^ {\theta} L _ {w} \sqrt {c _ {2} C _ {w , 1} \epsilon^ {2}} + \frac {\sqrt {C _ {w , 1}} \alpha^ {\theta^ {2}} H _ {w} U _ {g} ^ {2}}{\sqrt {2} (1 - \gamma) ^ {2}}. \\ \end{array}
$$
Based on the new bounds for  $I_{1}$  and  $I_{2}$ , the critic update now gives the following:
$$
\begin{array}{l} \left(2 \alpha^ {w} - \frac {1}{\sqrt {2}} \alpha^ {\theta} L _ {w} \left(\frac {1}{\lambda_ {\min }} - 2 \alpha^ {w}\right) - \sqrt {2} \alpha^ {\theta} L _ {w} \sqrt {c _ {1} \left(\frac {1}{\lambda_ {\min }} - 2 \alpha^ {w}\right)} - 2 \alpha^ {\theta 2} L _ {w} ^ {2} c _ {1}\right) \cdot \Delta_ {Q} \\ \leq \frac {\mathbb {E} \left[ | | w _ {1} - \hat {w} _ {\theta_ {1}} ^ {*} | | ^ {2} \right]}{T} + \left(\frac {1}{\sqrt {2}} \alpha^ {\theta} L _ {w} + 2 \alpha^ {\theta^ {2}} L _ {w} ^ {2}\right) \Delta_ {V} \\ + \left(\sqrt {2} \alpha^ {\theta} L _ {w} \sqrt {c _ {1} C _ {w , 1} + c _ {2} \epsilon^ {2} \left(\frac {1}{\lambda_ {\min}} - 2 \alpha^ {w}\right)} + \frac {\alpha^ {\theta^ {2}} H _ {w} U _ {g} ^ {2}}{\sqrt {2} (1 - \gamma) ^ {2}} \sqrt {\frac {1}{\lambda_ {\min}} - 2 \alpha^ {w}}\right) \sqrt {\Delta_ {Q}} \\ + C _ {w, 1} + \frac {1}{\sqrt {2}} \alpha^ {\theta} L _ {w} C _ {w, 1} + \sqrt {2} \alpha^ {\theta} L _ {w} \sqrt {c _ {2} C _ {w , 1} \epsilon^ {2}} + \frac {\sqrt {C _ {w , 1}} \alpha^ {\theta^ {2}} H _ {w} U _ {g} ^ {2}}{\sqrt {2} (1 - \gamma) ^ {2}} \\ + 2 \alpha^ {\theta^ {2}} L _ {w} ^ {2} c _ {2} \epsilon^ {2} + \frac {4 L _ {w} ^ {2}}{(1 - \gamma) ^ {2}} \alpha^ {\theta^ {2}} U _ {g} ^ {2}. \\ \end{array}
$$
G Small Gain Theorem and Small Gain Analysis
G.1 Small Gain Theorem
Now we introduce the small gain theorem.
Lemma G.1. Suppose  $x$  and  $y$  satisfy the following two inequalities:
$$
x \leq a _ {1} y + a _ {2},
$$
$$
y \leq b _ {1} x + b _ {2} + b _ {3} \sqrt {y},
$$
where all coefficients are non-negative. Then,  $y$  can be upper bounded by the following inequality:
Proof. Proof of this lemma can be found in Olshevsky & Gharesifard (2022).
G.2 Small Gain Analysis under i.i.d. Sampling
Now recall the result from Actor analysis is
and the one from Critic analysis is
$$
\left(2 \alpha^ {w} - \frac {1}{\sqrt {2}} \alpha^ {\theta} L _ {w} \left(\frac {1}{\lambda_ {\min }} - 2 \alpha^ {w}\right) - \sqrt {2} \alpha^ {\theta} L _ {w} \sqrt {c _ {1} \left(\frac {1}{\lambda_ {\min }} - 2 \alpha^ {w}\right)} - 2 \alpha^ {\theta 2} L _ {w} ^ {2} c _ {1}\right) \cdot \Delta_ {Q}
$$
$$
\begin{array}{l} \leq \frac {\mathbb {E} \left[ | | w _ {1} - \hat {w} _ {\theta_ {1}} ^ {*} | | ^ {2} \right]}{T} + \left(\frac {1}{\sqrt {2}} \alpha^ {\theta} L _ {w} + 2 \alpha^ {\theta^ {2}} L _ {w} ^ {2}\right) \Delta_ {V} \\ + \left(\sqrt {2} \alpha^ {\theta} L _ {w} \sqrt {c _ {1} C _ {w , 1} + c _ {2} \epsilon^ {2} \left(\frac {1}{\lambda_ {\min}} - 2 \alpha^ {w}\right)} + \frac {\alpha^ {\theta^ {2}} H _ {w} U _ {g} ^ {2}}{\sqrt {2} (1 - \gamma) ^ {2}} \sqrt {\frac {1}{\lambda_ {\min}} - 2 \alpha^ {w}}\right) \sqrt {\Delta_ {Q}} \\ + C _ {w, 1} + \frac {1}{\sqrt {2}} \alpha^ {\theta} L _ {w} C _ {w, 1} + \sqrt {2} \alpha^ {\theta} L _ {w} \sqrt {c _ {2} C _ {w , 1} \epsilon^ {2}} + \frac {\sqrt {C _ {w , 1}} \alpha^ {\theta^ {2}} H _ {w} U _ {g} ^ {2}}{\sqrt {2} (1 - \gamma) ^ {2}} \\ + 2 \alpha^ {\theta^ {2}} L _ {w} ^ {2} c _ {2} \epsilon^ {2} + \frac {4 L _ {w} ^ {2}}{(1 - \gamma) ^ {2}} \alpha^ {\theta^ {2}} U _ {g} ^ {2}. \\ \end{array}
$$
What we really care about is the relationship between  $T, \epsilon$  and  $m$ . So from now on, we will use  $O(\cdot)$  and  $\tilde{O}(\cdot)$  ( $\tilde{O}(\cdot)$  hides the potential logarithm factor of  $m$ ) notations and only consider these variables. First, observe the following dependency on  $T, m$  and  $\epsilon$ :
$$
\begin{array}{l} L _ {Q} = O (1), \\ H _ {Q} = \tilde {O} (\frac {1}{\sqrt {m}}), \\ L _ {\delta} = O (1), \\ U _ {\delta} = O (\epsilon) + O (1), \\ L _ {f} = \tilde {O} \left(\frac {1}{\sqrt {m}} (\epsilon + 1)\right) + O (1), \\ U _ {f} = O (\epsilon) + O (1), \\ U _ {g} = O (\epsilon) + O (1), \\ L _ {\bar {f}} = O \left(\left(\log T + 1\right) (\epsilon + 1)\right), \\ L _ {F _ {\theta}} = O \left(\left(\log T + 1\right) (\epsilon + 1)\right), \\ L _ {F _ {w}} = \tilde {O} \left(\left(\frac {1}{\sqrt {m}} + 1\right) (\epsilon + 1)\right). \\ \end{array}
$$
If we choose  $\alpha^w = \alpha^\theta = \frac{1}{\sqrt{T}}$  and given that all other coefficients are independent with  $\epsilon, T$  and  $m$ , we conclude
$$
C _ {w, 1} = \tilde {O} \left(\frac {1}{\sqrt {T}} \cdot \frac {1}{\sqrt {m}}\right) + O \left(\frac {1}{\sqrt {T}} \epsilon\right) + O \left(\frac {1}{\sqrt {T}} \epsilon^ {2}\right) + O \left(\frac {1}{T}\right).
$$
We can set
$$
x = \Delta_ {V},
$$
$$
y = \Delta_ {Q},
$$
$$
a _ {1} = \frac {c _ {1} \left(\alpha_ {\theta} ^ {2} H _ {V} + \frac {\alpha_ {\theta}}{2}\right)}{\frac {\alpha_ {\theta}}{2} - \alpha_ {\theta} ^ {2} H _ {V}} = O (1),
$$
$$
a _ {2} = \frac {\frac {\mathbb {E} \left[ V _ {\theta_ {1}} ^ {*} - V _ {\theta_ {T} + 1} ^ {*} \right]}{T} + c _ {2} \left(\alpha_ {\theta} ^ {2} H _ {V} + \frac {\alpha_ {\theta}}{2}\right) \epsilon^ {2} + \frac {2 H _ {V}}{(1 - \gamma) ^ {2}} \alpha^ {\theta^ {2}} U _ {g} ^ {2}}{\frac {\alpha_ {\theta}}{2} - \alpha_ {\theta} ^ {2} H _ {V}} = O \left(\frac {1}{\sqrt {T}}\right) + O \left(\epsilon^ {2}\right),
$$
$$
b _ {1} = \frac {\frac {1}{\sqrt {2}} \alpha^ {\theta} L _ {w} + 2 \alpha^ {\theta^ {2}} L _ {w} ^ {2}}{2 \alpha^ {w} - \frac {1}{\sqrt {2}} \alpha^ {\theta} L _ {w} \left(\frac {1}{\lambda_ {\mathrm {m i n}}} - 2 \alpha^ {w}\right) - \sqrt {2} \alpha^ {\theta} L _ {w} \sqrt {c _ {1} \left(\frac {1}{\lambda_ {\mathrm {m i n}}} - 2 \alpha^ {w}\right)} - 2 \alpha^ {\theta^ {2}} L _ {w} ^ {2} c _ {1}} = O (1),
$$