SlowGuess/NeurIPS_2023_OCR huggingface dataset:
$$
\begin{array}{l} \geq J (\pi_ {t}) + \langle \hat {g} (\pi_ {t}), \pi - \pi_ {t} \rangle - \sum_ {s} d ^ {\pi_ {t}} (s) \left[ \frac {D _ {\phi^ {*}} (\nabla \phi (\pi_ {t} ^ {s}) - c _ {s} \delta_ {t} ^ {s} , \nabla \phi (\pi_ {t} ^ {s}))}{c _ {s}} + \left(\frac {1}{\eta} + \frac {1}{c _ {s}}\right) D _ {\phi} (\pi^ {s}, \pi_ {t} ^ {s}) \right] \\ J (\pi) \geq J (\pi_ {t}) + \langle \hat {g} (\pi_ {t}), \pi - \pi_ {t} \rangle - \sum_ {s} d ^ {\pi_ {t}} (s) \left(\frac {1}{\eta} + \frac {1}{c _ {s}}\right) D _ {\phi} (\pi^ {s}, \pi_ {t} ^ {s}) - \sum_ {s} \frac {d ^ {\pi_ {t}} (s) D _ {\phi^ {*}} (\nabla \phi (\pi_ {t} ^ {s}) - c _ {s} \delta_ {t} ^ {s} , \nabla \phi (\pi_ {t} ^ {s}))}{c _ {s}} \\ \end{array}
$$
Proposition 4. For the direct representation and negative entropy mirror map,  $c > 0$ ,  $\eta \leq \frac{(1 - \gamma)^3}{2\gamma|A|}$ ,
Proof. For the direct representation,  $\pi_{s,a} = p^{\pi}(a|s)$ . Using the policy gradient theorem,  $[\nabla_{\pi}J(\pi)]_{s,a} = d^{\pi}(s)Q^{\pi}(s,a)$ . We choose  $\hat{g} (\pi)$  such that  $[\hat{g} (\pi)]_{s,a} = d^{\pi}(s)\hat{Q}^{\pi}(s,a)$  as the estimated gradient. Using Vaswani et al. [57, Proposition 2],  $J + \frac{1}{\eta}\Phi$  is convex for  $\eta \leq \frac{(1 - \gamma)^3}{2\gamma|A|}$ . Defining  $\delta_t^s \coloneqq \nabla_{\pi^s}J(\pi_t) - \hat{g}^s (\pi_t) = Q^{\pi_t}(s,\cdot) - \hat{Q}^{\pi_t}(s,\cdot)\in \mathbb{R}^A$ , and using Prop. 12 with  $c_{s} = c$  for all  $s$ ,
$$
J (\pi) \geq J (\pi_ {t}) + \langle \hat {g} (\pi_ {t}), \pi - \pi_ {t} \rangle - \sum_ {s} d ^ {\pi_ {t}} (s) \left(\frac {1}{\eta} + \frac {1}{c}\right) D _ {\phi} (\pi^ {s}, \pi_ {t} ^ {s}) - \sum_ {s} \frac {d ^ {\pi_ {t}} (s) D _ {\phi^ {*}} (\nabla \phi (\pi_ {t} ^ {s}) - c \delta_ {t} ^ {s} , \nabla \phi (\pi_ {t} ^ {s}))}{c}
$$
Since  $\phi (\pi^s) = \phi (p^\pi (\cdot |s)) = \sum_a p^\pi (a|s)\log (p^\pi (a|s))$ , using Lemma 10,  $D_{\phi}(\pi^{s},\pi_{t}^{s}) = \mathrm{KL}(p^{\pi}(\cdot |s)||p^{\pi_{t}}(\cdot |s))$ . Hence,
$$
\begin{array}{l} J (\pi) \geq J (\pi_ {t}) + \sum_ {s} d ^ {\pi_ {t}} (s) \sum_ {a} \hat {Q} ^ {\pi_ {t}} (s, a) [ p ^ {\pi} (a | s) - p ^ {\pi_ {t}} (a | s) ] - \left(\frac {1}{\eta} + \frac {1}{c}\right) \sum_ {s} d ^ {\pi_ {t}} (s) \operatorname {K L} (p ^ {\pi} (\cdot | s) \| p ^ {\pi_ {t}} (\cdot | s)) \\ - \sum_ {s} \frac {d ^ {\pi_ {t}} (s) D _ {\phi^ {*}} (\nabla \phi (\pi_ {t} ^ {s}) - c \delta_ {t} ^ {s} , \nabla \phi (\pi_ {t} ^ {s}))}{c} \\ \end{array}
$$
Using Lemma 7 to simplify the last term,
$$
\begin{array}{l} \sum_ {s} \frac {d ^ {\pi_ {t}} (s) D _ {\phi^ {*}} (\nabla \phi (\pi_ {t} ^ {s}) - c \delta_ {t} ^ {s} , \nabla \phi (\pi_ {t} ^ {s}))}{c} \\ = \frac {1}{c} \left[ \sum_ {s} d ^ {\pi_ {t}} (s) \left[ c \left\langle p ^ {\pi_ {t}} (\cdot | s), \delta_ {t} ^ {s} \right\rangle + \log \left(\sum_ {a} p ^ {\pi_ {t}} (a | s) \exp (- c \delta_ {t} ^ {s} [ a ])\right) \right] \right] \\ = \sum_ {s} d ^ {\pi_ {t}} (s) \left[ \sum_ {a} p ^ {\pi_ {t}} (a | s) \left[ Q ^ {\pi_ {t}} (s, a) - \hat {Q} ^ {\pi_ {t}} (s, a) \right] + \frac {1}{c} \log \left(\sum_ {a} p ^ {\pi_ {t}} (a | s) \exp \left(- c \left[ Q ^ {\pi_ {t}} (s, a) - \hat {Q} ^ {\pi_ {t}} (s, a) \right]\right)\right) \right] \\ \end{array}
$$
$$
\begin{array}{l} J (\pi) \geq J (\pi_ {t}) + \sum_ {s} d ^ {\pi_ {t}} (s) \sum_ {a} \hat {Q} ^ {\pi_ {t}} (s, a) [ p ^ {\pi} (a | s) - p ^ {\pi_ {t}} (a | s) ] - \left(\frac {1}{\eta} + \frac {1}{c}\right) \sum_ {s} d ^ {\pi_ {t}} (s) \operatorname {K L} (p ^ {\pi} (\cdot | s) | | p ^ {\pi_ {t}} (\cdot | s)) \\ \left. \right. - \left[ \sum_ {s} d ^ {\pi_ {t}} (s) \left[ \sum_ {a} p ^ {\pi_ {t}} (a | s) [ Q ^ {\pi_ {t}} (s, a) - \hat {Q} ^ {\pi_ {t}} (s, a) ] + \frac {1}{c} \log \left(\sum_ {a} p ^ {\pi_ {t}} (a | s) \exp \left(- c [ Q ^ {\pi_ {t}} (s, a) - \hat {Q} ^ {\pi_ {t}} (s, a) ]\right)\right)\right]\right] \\ = J(\pi_{t}) - \underbrace{\mathbb{E}_{s\sim d^{\pi_{t}}}\left[\mathbb{E}_{a\sim p^{\pi_{t}}(\cdot|s)}[\hat{Q}^{\pi_{t}}(s,a)]\right]}_{:= -C} + \mathbb{E}_{s\sim d^{\pi_{t}}}\left[\mathbb{E}_{a\sim p^{\pi (\cdot|s)}}\left[\hat{Q}^{\pi_{t}}(s,a) - \left(\frac{1}{\eta} +\frac{1}{c}\right)\log \left(\frac{p^{\pi}(a|s)}{p^{\pi_{t}}(a|s)}\right)\right]\right] \\ \left. \right. - \left[ \sum_ {s} d ^ {\pi_ {t}} (s) \left[ \sum_ {a} p ^ {\pi_ {t}} (a | s) [ Q ^ {\pi_ {t}} (s, a) - \hat {Q} ^ {\pi_ {t}} (s, a) ] + \frac {1}{c} \log \left(\sum_ {a} p ^ {\pi_ {t}} (a | s) \exp \left(- c [ Q ^ {\pi_ {t}} (s, a) - \hat {Q} ^ {\pi_ {t}} (s, a) ]\right)\right)\right]\right] \\ \end{array}
$$
$$
\begin{array}{l} J (\pi) \geq J (\pi_ {t}) + C + \mathbb {E} _ {s \sim d ^ {\pi_ {t}}} \left[ \mathbb {E} _ {a \sim p ^ {\pi_ {t}} (\cdot | s)} \left[ \frac {p ^ {\pi} (a | s)}{p ^ {\pi_ {t}} (a | s)} \left(\hat {Q} ^ {\pi_ {t}} (s, a) - \left(\frac {1}{\eta} + \frac {1}{c}\right) \log \left(\frac {p ^ {\pi} (a | s)}{p ^ {\pi_ {t}} (a | s)}\right)\right) \right] \right] \\ - \mathbb {E} _ {s \sim d ^ {\pi_ {t}}} \left[ \mathbb {E} _ {a \sim p ^ {\pi_ {t}} (\cdot | s)} \left[ Q ^ {\pi_ {t}} (s, a) - \hat {Q} ^ {\pi_ {t}} (s, a) \right] + \frac {1}{c} \log \left(\mathbb {E} _ {a \sim p ^ {\pi_ {t}} (\cdot | s)} \left[ \exp \left(- c \left[ Q ^ {\pi_ {t}} (s, a) - \hat {Q} ^ {\pi_ {t}} (s, a) \right]\right) \right]\right) \right] \\ \end{array}
$$
$$
\begin{array}{l} J (\pi) - J \left(\pi_ {t}\right) \geq \mathbb {E} _ {s \sim d ^ {\pi_ {t}}} \mathbb {E} _ {a \sim p ^ {\pi_ {t}} (· | s)} \left[ \left(\hat {A} ^ {\pi_ {t}} (s, a) + \frac {1}{\eta} + \frac {1}{c}\right) \log \left(\frac {p ^ {\pi} (a | s)}{p ^ {\pi_ {t}} (a | s)}\right) \right] \\ \left. \right. - \frac {1}{c} \mathbb {E} _ {s \sim d ^ {\pi_ {t}}} \mathbb {E} _ {a \sim p ^ {\pi_ {t}} (\cdot | s)} \left[\left(1 - c \left[ A ^ {\pi_ {t}} (s, a) - \hat {A} ^ {\pi_ {t}} (s, a) \right]\right) \log \left(1 - c \left[ A ^ {\pi_ {t}} (s, a) - \hat {A} ^ {\pi_ {t}} (s, a) \right]\right)\right] \\ \end{array}
$$
Proof. For the softmax representation,  $\pi_{s,a} = z(s,a)$  s.t.  $p^{\pi}(a|s) = \frac{\exp(z(s,a))}{\sum_{a'}\exp(z(s,a'))}$ . Using the policy gradient theorem,  $[\nabla_{\pi}J(\pi)]_{s,a} = d^{\pi}(s)p^{\pi}(a|s)A^{\pi}(s,a)$ . We choose  $\hat{g} (\pi)$  such that  $[\hat{g} (\pi)]_{s,a} = d^{\pi}(s)p^{\pi}(a|s)\hat{A}^{\pi}(s,a)$  as the estimated gradient. Using Vaswani et al. [57, Proposition 3],  $J + \frac{1}{\eta}\Phi$  is convex for  $\eta \leq 1 - \gamma$ . Define  $\delta_s\in \mathbb{R}^A$  such that  $\delta_t^s [a]\coloneqq \nabla_{\pi^s}J(\pi_t) - \hat{g}^s (\pi_t) = p^{\pi_t}(a|s)[A^{\pi_t}(s,a) - \hat{A}^{\pi_t}(s,a)]$ . Using Prop. 12 with  $c_{s} = c$  for all  $s$ ,
Since  $\phi (\pi^s) = \phi (z(s,\cdot)) = \log \left(\sum_a\exp (z(s,a))\right)$ , using Lemma 11,  $D_{\phi}(\pi^{s},\pi_{t}^{s}) = \mathrm{KL}(p^{\pi_{t}}(\cdot |s)||p^{\pi}(\cdot |s))$  where  $p^\pi (a|s) = \frac{\exp(z(s,a))}{\sum_{a'}\exp(z(s,a'))}$  and  $p^{\pi_t}(a|s) = \frac{\exp(z_t(s,a))}{\sum_{a'}\exp(z_t(s,a'))}$ . Hence, the above bound can be simplified as,
$$
\begin{array}{l} J (\pi) \geq J (\pi_ {t}) + \sum_ {s} d ^ {\pi_ {t}} (s) \sum_ {a} \hat {A} ^ {\pi_ {t}} (s, a) p ^ {\pi_ {t}} (a | s) [ z (s, a) - z _ {t} (s, a) ] - \left(\frac {1}{\eta} + \frac {1}{c}\right) \sum_ {s} d ^ {\pi_ {t}} (s) \operatorname {K L} (p ^ {\pi_ {t}} (\cdot | s) | | p ^ {\pi} (\cdot | s)) \\ - \sum_ {s} \frac {d ^ {\pi_ {t}} (s) D _ {\phi^ {*}} (\nabla \phi (\pi_ {t} ^ {s}) - c \delta_ {t} ^ {s} , \nabla \phi (\pi_ {t} ^ {s}))}{c} \\ \end{array}
$$
Using Lemma 8 to simplify the last term,
$$
\begin{array}{l} \sum_ {s} \frac {d ^ {\pi_ {t}} (s) D _ {\phi^ {*}} (\nabla \phi (\pi_ {t} ^ {s}) - c \delta_ {t} ^ {s} , \nabla \phi (\pi_ {t} ^ {s}))}{c} \\ = \frac {1}{c} \left[ \sum_ {s} d ^ {\pi_ {t}} (s) \left[ \sum_ {a} \left(p ^ {\pi_ {t}} (a | s) - c \delta_ {t} ^ {s} [ a ]\right) \log \left(\frac {p ^ {\pi_ {t}} (a | s) - c \delta_ {t} ^ {s} [ a ]}{p ^ {\pi_ {t}} (a | s)}\right) \right] \right] \\ = \frac {1}{c} \left[ \sum_ {s} d ^ {\pi_ {t}} (s) \left[ \sum_ {a} \left(p ^ {\pi_ {t}} (a | s) - c \left[ p ^ {\pi_ {t}} (a | s) [ A ^ {\pi_ {t}} (s, a) - \hat {A} ^ {\pi_ {t}} (s, a) ] \right]\right) \log \left(\frac {p ^ {\pi_ {t}} (a | s) - c \left[ p ^ {\pi_ {t}} (a | s) [ A ^ {\pi_ {t}} (s , a) - \hat {A} ^ {\pi_ {t}} (s , a) ] \right]}{p ^ {\pi_ {t}} (a | s)}\right) \right] \right] \\ = \frac {1}{c} \left[ \sum_ {s} d ^ {\pi_ {t}} (s) \left[ \sum_ {a} p ^ {\pi_ {t}} (a | s) \left[ \left(1 - c \left[ A ^ {\pi_ {t}} (s, a) - \hat {A} ^ {\pi_ {t}} (s, a) \right]\right) \log \left(1 - c \left[ A ^ {\pi_ {t}} (s, a) - \hat {A} ^ {\pi_ {t}} (s, a) \right]\right) \right] \right] \right] \\ \end{array}
$$
$$
\begin{array}{l} J (\pi) \geq J (\pi_ {t}) + \sum_ {s} d ^ {\pi_ {t}} (s) \sum_ {a} \hat {A} ^ {\pi_ {t}} (s, a) p ^ {\pi_ {t}} (a | s) [ z (s, a) - z _ {t} (s, a) ] - \left(\frac {1}{\eta} + \frac {1}{c}\right) \sum_ {s} d ^ {\pi_ {t}} (s) \operatorname {K L} (p ^ {\pi_ {t}} (\cdot | s) | | p ^ {\pi} (\cdot | s)) \\ \left. \right. - \frac {1}{c} \left[ \sum_ {s} d ^ {\pi_ {t}} (s) \left[ \sum_ {a} p ^ {\pi_ {t}} (a | s) \left[\left(1 - c \left[ A ^ {\pi_ {t}} (s, a) - \hat {A} ^ {\pi_ {t}} (s, a) \right]\right) \log \left(1 - c \left[ A ^ {\pi_ {t}} (s, a) - \hat {A} ^ {\pi_ {t}} (s, a) \right]\right)\right]\right]\right] \\ = J \left(\pi_ {t}\right) + \sum_ {s} d ^ {\pi_ {t}} (s) \sum_ {a} \left[ p ^ {\pi_ {t}} (a | s) \hat {A} ^ {\pi_ {t}} (s, a) [ z (s, a) - z _ {t} (s, a) ] - \left(\frac {1}{\eta} + \frac {1}{c}\right) p ^ {\pi_ {t}} (a | s) \log \left(\frac {p ^ {\pi_ {t}} (a | s)}{p ^ {\pi} (a | s)}\right) \right] \\ - \frac {1}{c} \mathbb {E} _ {s \sim d ^ {\pi_ {t}}} \mathbb {E} _ {a \sim p ^ {\pi_ {t}} (\cdot | s)} \left[ \left(1 - c \left[ A ^ {\pi_ {t}} (s, a) - \hat {A} ^ {\pi_ {t}} (s, a) \right]\right) \log \left(1 - c \left[ A ^ {\pi_ {t}} (s, a) - \hat {A} ^ {\pi_ {t}} (s, a) \right]\right) \right] \\ \end{array}
$$
Let us focus on simplifying  $\sum_{a}\left[p^{\pi_t}(a|s)\hat{A}^{\pi_t}(s,a)[z(s,a) - z_t(s,a)]\right]$  for a fixed  $s$ . Note that  $\sum_{a}p^{\pi_t}(a|s)\hat{A}^{\pi_t}(s,a) = 0 \Rightarrow \log \left(\sum_{a'}\exp (z(s,a'))\right)$ $\sum_{a}p^{\pi_t}(a|s)\hat{A}^{\pi_t}(s,a) = 0$ .
$$
\begin{array}{l} \sum_ {a} \left[ p ^ {\pi_ {t}} (a | s) \hat {A} ^ {\pi_ {t}} (s, a) z (s, a) \right] = \sum_ {a} \left[ p ^ {\pi_ {t}} (a | s) \hat {A} ^ {\pi_ {t}} (s, a) \left(z (s, a) - \log \left(\sum_ {a ^ {\prime}} \exp (z (s, a ^ {\prime}))\right)\right) \right] \\ = \sum_ {a} \left[ \right. p ^ {\pi_ {t}} (a | s) \hat {A} ^ {\pi_ {t}} (s, a) \left( \right.\log \left(\exp \left(z (s, a)\right) - \log \left(\sum_ {a ^ {\prime}} \exp \left(z (s, a ^ {\prime})\right)\right)\right)\left. \right] \\ = \sum_ {a} \left[ p ^ {\pi_ {t}} (a | s) \hat {A} ^ {\pi_ {t}} (s, a) \log \left(\frac {\exp (z (s , a))}{\sum_ {a ^ {\prime}} \exp (z (s , a ^ {\prime}))}\right) \right] = \mathbb {E} _ {a \sim p ^ {\pi_ {t}} (\cdot | s)} \left[ \hat {A} ^ {\pi_ {t}} (s, a) \log (p ^ {\pi} (a | s)) \right] \\ \end{array}
$$
Similarly, simplifying  $\sum_{a}\left[p^{\pi_t}(a|s)\hat{A}^{\pi_t}(s,a)z_t(s,a)\right]$
$$
\begin{array}{l} \sum_ {a} \left[ p ^ {\pi_ {t}} (a | s) \hat {A} ^ {\pi_ {t}} (s, a) z _ {t} (s, a) \right] = E _ {a \sim p ^ {\pi_ {t}} (\cdot | s)} \left[ \hat {A} ^ {\pi_ {t}} (s, a) \log \left(p ^ {\pi_ {t}} (a | s)\right) \right] \\ \Rightarrow \sum_ {a} \left[ p ^ {\pi_ {t}} (a | s) \hat {A} ^ {\pi_ {t}} (s, a) [ z (s, a) - z _ {t} (s, a) ] \right] = \mathbb {E} _ {a \sim p ^ {\pi_ {t}} (\cdot | s)} \left[ \hat {A} ^ {\pi_ {t}} (s, a) \log \left(\frac {p ^ {\pi} (a | s)}{p ^ {\pi_ {t}} (a | s)}\right) \right] \\ \end{array}
$$
$$
\begin{array}{l} J (\pi) \geq J (\pi_ {t}) + \sum_ {s} d ^ {\pi_ {t}} (s) \mathbb {E} _ {a \sim p ^ {\pi_ {t}} (\cdot | s)} \left[ \hat {A} ^ {\pi_ {t}} (s, a) \log \left(\frac {p ^ {\pi} (a | s)}{p ^ {\pi_ {t}} (a | s)}\right) - \left(\frac {1}{\eta} + \frac {1}{c}\right) \log \left(\frac {p ^ {\pi_ {t}} (a | s)}{p ^ {\pi} (a | s)}\right) \right] \\ - \frac {1}{c} \mathbb {E} _ {s \sim d ^ {\pi_ {t}}} \mathbb {E} _ {a \sim p ^ {\pi_ {t}} (\cdot | s)} \left[ \left(1 - c \left[ A ^ {\pi_ {t}} (s, a) - \hat {A} ^ {\pi_ {t}} (s, a) \right]\right) \log \left(1 - c \left[ A ^ {\pi_ {t}} (s, a) - \hat {A} ^ {\pi_ {t}} (s, a) \right]\right) \right] \\ = J \left(\pi_ {t}\right) + \mathbb {E} _ {s \sim d ^ {\pi_ {t}}} \mathbb {E} _ {a \sim p ^ {\pi_ {t}} (\cdot | s)} \left[ \left(\hat {A} ^ {\pi_ {t}} (s, a) + \frac {1}{\eta} + \frac {1}{c}\right) \log \left(\frac {p ^ {\pi} (a | s)}{p ^ {\pi_ {t}} (a | s)}\right) \right] \\ \left. \right. - \frac {1}{c} \mathbb {E} _ {s \sim d ^ {\pi_ {t}}} \mathbb {E} _ {a \sim p ^ {\pi_ {t}} (\cdot | s)} \left[\left(1 - c \left[ A ^ {\pi_ {t}} (s, a) - \hat {A} ^ {\pi_ {t}} (s, a) \right]\right) \log \left(1 - c \left[ A ^ {\pi_ {t}} (s, a) - \hat {A} ^ {\pi_ {t}} (s, a) \right]\right)\right]. \\ \end{array}
$$
$$
\begin{array}{l} J (\pi) - J (\pi_ {t}) \geq C + \mathbb {E} _ {s \sim d ^ {\pi_ {t}}} \mathbb {E} _ {\varepsilon \sim \chi} \left[ \widehat {\nabla_ {a} Q ^ {\pi_ {t}}} (s, a) \big | _ {a = \pi_ {t} (s, \varepsilon)} \pi (s, \varepsilon) - \frac {1}{2} \left(\frac {1}{\eta} + \frac {1}{c}\right) [ \pi_ {t} (s, \epsilon) - \pi (s, \epsilon) ] ^ {2} \right] \\ - \frac {c}{2} \mathbb {E} _ {s \sim d ^ {\pi_ {t}}} \mathbb {E} _ {\varepsilon \sim \chi} \left[ \right.\left. \nabla_ {a} Q ^ {\pi_ {t}} (s, a) \right| _ {a = \pi_ {t} (s, \varepsilon)} - \widehat {\nabla_ {a} Q ^ {\pi_ {t}}} (s, a) \left. \right| _ {a = \pi_ {t} (s, \varepsilon)} \left. \right] ^ {2} \\ \end{array}
$$
where  $C$  is a constant and  $\widehat{\nabla_a Q^{\pi_t}}(s, a)\big|_{a = \pi_t(s, \varepsilon)}$  is the estimate of the action-value gradients for policy  $\pi$  at state  $s$  and  $a = \pi_t(s, \epsilon)$ .
Proof. For stochastic value gradients with a fixed  $\varepsilon$ ,  $\frac{\partial J(\pi)}{\partial \pi(s, \epsilon)} = d^{\pi}(s) \nabla_{a} Q^{\pi}(s, a)\big|_{a=\pi(s, \epsilon)}$ . We choose  $\hat{g}(\pi)$  such that  $[\hat{g}(\pi)]_{s,a} = d^{\pi}(s) \widehat{\nabla_{a} Q^{\pi}}(s, a)\big|_{a=\pi(s, \epsilon)}$ . Define  $\delta_{t}^{s} \in \mathbb{R}^{A}$  such that  $\delta_{t}^{s}[a] := \left. \nabla_{a} Q^{\pi t}(s, a) \right|_{a=\pi_{t}(s, \epsilon)} - \left. \widehat{\nabla_{a} Q^{\pi t}}(s, a) \right|_{a=\pi_{t}(s, \epsilon)}$ . Using Prop. 12 with  $c_{s} = c$  for all  $s$ ,
$$
\begin{array}{l} J(\pi)\geq J(\pi_{t}) + \mathbb{E}_{\varepsilon \sim \chi}\bigg[\sum_{s}d^{\pi_{t}}(s)\widehat{\nabla_{a}Q^{\pi_{t}}} (s,a)\big|_{a = \pi_{t}(s,\epsilon)}\left[\pi (s,\varepsilon) - \pi_{t}(s,\varepsilon)\right] - \sum_{s}d^{\pi_{t}}(s)\left(\frac{1}{\eta} +\frac{1}{c}\right)D_{\phi}(\pi^{s},\pi_{t}{}^{s}) \\ \left. - \sum_ {s} \frac {d ^ {\pi_ {t}} (s) D _ {\phi^ {*}} \left(\nabla \phi \left(\pi_ {t} ^ {s}\right) - c \delta_ {t} ^ {s} , \nabla \phi \left(\pi_ {t} ^ {s}\right)\right)}{c} \right] \\ \end{array}
$$
For a fixed  $\varepsilon$ , since  $\phi(\pi^s) = \phi(\pi(s, \epsilon)) = \frac{1}{2} [\pi(s, \epsilon)]^2$ ,  $D_{\phi}(\pi^s, \pi_t^s) = \frac{1}{2} [\pi(s, \epsilon) - \pi_t(s, \epsilon)]^2$ . Hence,
$$
\begin{array}{l} J(\pi)\geq J(\pi_{t}) + \mathbb{E}_{\varepsilon \sim \chi}\bigg[\sum_{s}d^{\pi_{t}}(s)\widehat{\nabla_{a}Q^{\pi_{t}}} (s,a)\big|_{a = \pi_{t}(s,\epsilon)}\left[\pi (s,\varepsilon) - \pi_{t}(s,\varepsilon)\right] - \frac{1}{2}\left(\frac{1}{\eta} +\frac{1}{c}\right)\sum_{s}d^{\pi_{t}}(s)\left[\pi (s,\epsilon) - \pi_{t}(s,\epsilon)\right]^{2} \\ \left. - \sum_ {s} \frac {d ^ {\pi_ {t}} (s) D _ {\phi^ {*}} \left(\nabla \phi \left(\pi_ {t} ^ {s}\right) - c \delta_ {t} ^ {s} , \nabla \phi \left(\pi_ {t} ^ {s}\right)\right)}{c} \right] \\ \end{array}
$$
Simplifying the last term, since  $\phi (\pi (s,\epsilon)) = \frac{1}{2} [\pi (s,\epsilon)]^2$
$$
\frac {D _ {\phi^ {*}} \left(\nabla \phi (\pi_ {t} ^ {s}) - c \delta_ {t} ^ {s} , \nabla \phi (\pi_ {t} ^ {s})\right)}{c} = \frac {c}{2} \left[ \delta_ {t} ^ {s} \right] ^ {2} = \frac {c}{2} \left[ \right.\left. \nabla_ {a} Q ^ {\pi_ {t}} (s, a) \right| _ {a = \pi_ {t} (s, \epsilon)} - \widehat {\nabla_ {a} Q ^ {\pi_ {t}}} (s, a) \left. \right| _ {a = \pi_ {t} (s, \epsilon)} \left. \right] ^ {2}
$$
$$
\begin{array}{l} J (\pi) \geq J (\pi_ {t}) + \mathbb {E} _ {\varepsilon \sim \chi} \left[ \sum_ {s} d ^ {\pi_ {t}} (s) \widehat {\nabla_ {a} Q ^ {\pi_ {t}}} (s, a) \big | _ {a = \pi_ {t} (s, \epsilon)} \pi (s, \varepsilon) - \underbrace {\sum_ {s} d ^ {\pi_ {t}} (s) \nabla_ {a} Q ^ {\pi_ {t}} (s , a) \big | _ {a = \pi_ {t} (s , \epsilon)} \pi_ {t} (s , \varepsilon)} _ {:= - C} \right] \\ \left. \right. - \mathbb {E} _ {\varepsilon \sim \chi} \left[ \frac {1}{2} \left(\frac {1}{\eta} + \frac {1}{c}\right) \sum_ {s} d ^ {\pi_ {t}} (s) [ \pi (s, \epsilon) - \pi_ {t} (s, \epsilon) ] ^ {2} - \frac {c}{2} \sum_ {s} d ^ {\pi_ {t}} (s) \left[ \nabla_ {a} Q ^ {\pi_ {t}} (s, a) \big | _ {a = \pi_ {t} (s, \epsilon)} - \widehat {\nabla_ {a} Q ^ {\pi_ {t}}} (s, a) \big | _ {a = \pi_ {t} (s, \epsilon)} \right] ^ {2} \right] \\ \end{array}
$$
$$
\begin{array}{l} J (\pi) \geq J (\pi_ {t}) + C + \mathbb {E} _ {\varepsilon \sim \chi} \left[ \mathbb {E} _ {s \sim d ^ {\pi_ {t}}} \left[ \widehat {\nabla_ {a} Q ^ {\pi_ {t}}} (s, a) \big | _ {a = \pi_ {t} (s, \epsilon)} \pi (s, \varepsilon) - \frac {1}{2} \left(\frac {1}{\eta} + \frac {1}{c}\right) [ \pi (s, \epsilon) - \pi_ {t} (s, \epsilon) ] ^ {2} \right] \right] \\ - \frac {c}{2} \mathbb {E} _ {\varepsilon \sim \chi} \left[ \right. \mathbb {E} _ {s \sim d ^ {\pi_ {t}}} \left[ \right.\left. \nabla_ {a} Q ^ {\pi_ {t}} (s, a) \right| _ {a = \pi_ {t} (s, \epsilon)} - \widehat {\nabla_ {a} Q ^ {\pi_ {t}}} (s, a) \left. \right| _ {a = \pi_ {t} (s, \epsilon)} \left. \right] ^ {2} \left. \right] \\ \end{array}
$$
Proposition 13. For the softmax representation and Euclidean mirror map,  $c > 0$ ,  $\eta \leq \frac{(1 - \gamma)^3}{8}$  then
$$
\begin{array}{l} J (\pi) \geq J (\pi_ {t}) + C + \mathbb {E} _ {s \sim d ^ {\pi_ {t}} (s)} \left[ \mathbb {E} _ {a \sim p ^ {\pi_ {t}} (., | s)} \left[ \hat {A} ^ {\pi_ {t}} (s, a) z (s, a) \right] - \frac {1}{2} \left(\frac {1}{\eta} + \frac {1}{c}\right) | | z (s, \cdot) - z _ {t} (s, \cdot) | | ^ {2} \right] \\ - \frac {c}{2} \mathbb {E} _ {s \sim d ^ {\pi_ {t}}} \mathbb {E} _ {a \sim p ^ {\pi_ {t}} (. | s)} \left[ A ^ {\pi_ {t}} (s, a) - \hat {A} ^ {\pi_ {t}} (s, a) \right] ^ {2} \\ \end{array}
$$
where  $C$  is a constant and  $\hat{A}^{\pi}$  is the estimate of advantage function for policy  $\pi_t$ .
Proof. For the softmax representation,  $\pi_{s,a} = z(s,a)$  s.t.  $p^\pi (a|s) = \frac{\exp(z(s,a))}{\sum_{a'}\exp(z(s,a'))}$ . Using the policy gradient theorem,  $[\nabla_{\pi}J(\pi)]_{s,a} = d^{\pi}(s)p^{\pi}(a|s)A^{\pi}(s,a)$ . We choose  $\hat{g} (\pi)$  such that  $[\hat{g} (\pi)]_{s,a} = d^{\pi}(s)p^{\pi}(a|s)\hat{A}^{\pi}(s,a)$  as the estimated gradient. Define  $\delta_s\in \mathbb{R}^A$  such that  $\delta_t^s [a]\coloneqq \nabla_{\pi^s}J(\pi_t) - \hat{g}^s (\pi_t) = p^{\pi_t}(a|s)[A^{\pi_t}(s,a) - \hat{A}^{\pi_t}(s,a)]$ . Using Mei et al. [37, Lemma 7],  $J + \frac{1}{\eta}\Phi$  is convex for  $\eta \leq 1 - \gamma$ . Using Prop. 12 with  $c_{s} = c$  for all  $s$ ,
$$
J (\pi) \geq J (\pi_ {t}) + \langle \hat {g} (\pi_ {t}), \pi - \pi_ {t} \rangle - \left(\frac {1}{\eta} + \frac {1}{c}\right) \sum_ {s} d ^ {\pi_ {t}} (s) D _ {\phi} (\pi^ {s}, \pi_ {t} ^ {s}) - \sum_ {s} \frac {d ^ {\pi_ {t}} (s) D _ {\phi^ {*}} (\nabla \phi (\pi_ {t} ^ {s}) - c \delta_ {t} ^ {s} , \nabla \phi (\pi_ {t} ^ {s}))}{c}
$$
Since  $\phi (\pi^s) = \phi (z(s,\cdot)) = \frac{1}{2}\sum_a[z_{s,a}]^2$ ,  $D_{\phi}(\pi^{s},\pi_{t}^{s}) = \frac{1}{2}\| z(s,\cdot) - z_{t}(s,\cdot)\|_{2}^{2}$ . Hence,
$$
\begin{array}{l} J (\pi) \geq J (\pi_ {t}) + \sum_ {s} d ^ {\pi_ {t}} (s) \sum_ {a} \hat {A} ^ {\pi_ {t}} (s, a) p ^ {\pi_ {t}} (a | s) [ z (s, a) - z _ {t} (s, a) ] - \frac {1}{2} \left(\frac {1}{\eta} + \frac {1}{c}\right) \sum_ {s} d ^ {\pi_ {t}} (s) \| z (s, \cdot) - z _ {t} (s, \cdot) \| _ {2} ^ {2} \\ - \sum_ {s} \frac {d ^ {\pi_ {t}} (s) D _ {\phi^ {*}} (\nabla \phi (\pi_ {t} ^ {s}) - c \delta_ {t} ^ {s} , \nabla \phi (\pi_ {t} ^ {s}))}{c} \\ \end{array}
$$
Simplifying the last term, since  $\phi (z(\cdot ,a)) = \frac{1}{2} [z(s,a)]^2$
$$
\begin{array}{l} \frac {\sum_ {s} d ^ {\pi_ {t}} (s) D _ {\phi^ {*}} \left(\nabla \phi \left(\pi_ {t} ^ {s}\right) - c \delta_ {t} ^ {s} , \nabla \phi \left(\pi_ {t} ^ {s}\right)\right)}{c} = \frac {c}{2} \sum_ {s} d ^ {\pi_ {t}} (s) \sum_ {a} [ \delta_ {t} ^ {s} (a) ] ^ {2} \\ = \frac {c}{2} \sum_ {s} d ^ {\pi_ {t}} (s) \sum_ {a} p ^ {\pi_ {t}} (a | s) ^ {2} \left[ A ^ {\pi_ {t}} (s, a) - \hat {A} ^ {\pi_ {t}} (s, a) \right] ^ {2} \\ \leq \frac {c}{2} \sum_ {s} d ^ {\pi_ {t}} (s) \sum_ {a} p ^ {\pi_ {t}} (a | s) \left[ A ^ {\pi_ {t}} (s, a) - \hat {A} ^ {\pi_ {t}} (s, a) \right] ^ {2} \quad (\text {S i n c e} p ^ {\pi_ {t}} (a | s) \leq 1) \\ \end{array}
$$
$$
\begin{array}{l} J(\pi)\geq J(\pi_{t}) + \sum_{s}d^{\pi_{t}}(s)\sum_{a}\hat{A}^{\pi_{t}}(s,a)p^{\pi_{t}}(a|s)[z(s,a) - z_{t}(s,a)] - \frac{1}{2}\left(\frac{1}{\eta} +\frac{1}{c}\right)\sum_{s}d^{\pi_{t}}(s)\| z(s,\cdot) - z_{t}(s,\cdot)\|_{2}^{2} \\ - \frac {c}{2} \sum_ {s} d ^ {\pi_ {t}} (s) \sum_ {a} p ^ {\pi_ {t}} (a | s) \left[ A ^ {\pi_ {t}} (s, a) - \hat {A} ^ {\pi_ {t}} (s, a) \right] ^ {2} \\ = J (\pi_ {t}) - \underbrace {\sum_ {s} d ^ {\pi_ {t}} (s) \sum_ {a} \hat {A} ^ {\pi_ {t}} (s , a) p ^ {\pi_ {t}} (a | s) z _ {t} (s , a)} _ {: = - C} \\ + \sum_ {s} d ^ {\pi_ {t}} (s) \left[ \sum_ {a} \hat {A} ^ {\pi_ {t}} (s, a) p ^ {\pi_ {t}} (a | s) z (s, a) - \frac {1}{2} \left(\frac {1}{\eta} + \frac {1}{c}\right) \| z (s, \cdot) - z _ {t} (s, \cdot) \| _ {2} ^ {2} \right] \\ - \frac {c}{2} \sum_ {s} d ^ {\pi_ {t}} (s) \sum_ {a} p ^ {\pi_ {t}} (a | s) \left[ A ^ {\pi_ {t}} (s, a) - \hat {A} ^ {\pi_ {t}} (s, a) \right] ^ {2} \\ = J (\pi_ {t}) + C + \mathbb {E} _ {s \sim d ^ {\pi_ {t}}} \left[ \mathbb {E} _ {a \sim p ^ {\pi_ {t}} (\cdot | s)} \left[ \hat {A} ^ {\pi_ {t}} (s, a) z (s, a) \right] - \frac {1}{2} \left(\frac {1}{\eta} + \frac {1}{c}\right) \| z (s, \cdot) - z _ {t} (s, \cdot) \| _ {2} ^ {2} \right] \\ - \frac {c}{2} \mathbb {E} _ {s \sim d ^ {\pi_ {t}}} \mathbb {E} _ {a \sim p ^ {\pi_ {t}} (\cdot | s)} \left[ A ^ {\pi_ {t}} (s, a) - \hat {A} ^ {\pi_ {t}} (s, a) \right] ^ {2} \\ \end{array}
$$
Proposition 14. For both the direct (with the negative-entropy mirror map) and softmax representations (with the log-sum-exp mirror map), for a fixed state  $s$ , if  $\delta \in \mathbb{R}^A \coloneqq \nabla_{\pi^s} J(\pi_t) - \hat{g}^s(\pi_t)$ , the second-order Taylor expansion of  $f(c) = D_{\phi^*} (\nabla \phi(\pi_t^s) - c\delta, \nabla \phi(\pi_t^s))$  around  $c = 0$  is equal to
$$
f (c) \approx \frac {c ^ {2}}{2} \sum_ {a} p ^ {\pi_ {t}} (a | s) [ A (s, a) - \hat {A} (s, a) ] ^ {2}.
$$
$$
\begin{array}{l} f (c) = D _ {\phi^ {*}} \left(\nabla \phi \left(\pi_ {t} ^ {s}\right) - c \delta , \nabla \phi \left(\pi_ {t} ^ {s}\right)\right) \Longrightarrow f (0) = D _ {\phi} ^ {*} \left(\nabla \phi \left(\pi_ {t} ^ {s}\right), \nabla \phi \left(\pi_ {t} ^ {s}\right)\right) = 0 \\ f (c) = D _ {\phi^ {*}} (\nabla \phi (\pi_ {t} ^ {s}) - c \delta , \nabla \phi (\pi_ {t} ^ {s})) = \phi^ {*} (\nabla \phi (\pi_ {t} ^ {s}) - c \delta) - \phi^ {*} (\nabla \phi (\pi_ {t} ^ {s})) - \langle \nabla \phi^ {*} (\nabla \phi (\pi_ {t} ^ {s})), \nabla \phi (\pi_ {t} ^ {s}) - c \delta - \nabla \phi (\pi_ {t} ^ {s}) \rangle \\ \Rightarrow f ^ {\prime} (c) = \left\langle \nabla \phi^ {*} \left(\nabla \phi \left(\pi_ {t} ^ {s}\right) - c \delta\right), - \delta \right\rangle + \left\langle \pi_ {t} ^ {s}, \delta \right\rangle \Rightarrow f ^ {\prime} (0) = \left\langle \pi_ {t} ^ {s}, - \delta \right\rangle + \left\langle \pi_ {t} ^ {s}, \delta \right\rangle = 0 \\ f ^ {\prime \prime} (c) = \langle \delta , \nabla^ {2} \phi^ {*} (\nabla \phi (\pi_ {t} ^ {s}) - c \delta) \delta \rangle \Longrightarrow f ^ {\prime \prime} (0) = \langle \delta , \nabla^ {2} \phi^ {*} (\nabla \phi (\pi_ {t} ^ {s})) \delta \rangle . \\ \end{array}
$$
By the second-order Taylor series expansion of  $f(c)$  around  $c = 0$ ,