$$
\begin{array}{l} L _ {\tau} \left(\pi_ {\tau} ^ {\star}, \lambda_ {t}\right) - L _ {\tau} \left(\pi_ {\theta_ {t}}, \lambda_ {t}\right) \\ = V _ {r + \lambda_ {t} g} ^ {\pi_ {\tau} ^ {\star}} (\rho) - V _ {r + \lambda_ {t} g} ^ {\pi_ {\theta_ {t}}} (\rho) \\ - \frac {\tau}{1 - \gamma} \sum_ {s, a} d _ {\rho} ^ {\pi_ {\tau} ^ {\star}} (s) \pi_ {\tau} ^ {\star} (a | s) \log \pi_ {\tau} ^ {\star} (a | s) + \frac {\tau}{1 - \gamma} \sum_ {s, a} d _ {\rho} ^ {\pi_ {\theta_ {t}}} (s) \pi_ {\theta_ {t}} (a | s) \log \pi_ {\theta_ {t}} (a | s) \\ = V _ {r + \lambda_ {t} g + \tau \psi_ {t}} ^ {\pi_ {\tau} ^ {\star}} (\rho) - V _ {r + \lambda_ {t} g + \tau \psi_ {t}} ^ {\pi_ {\theta_ {t}}} (\rho) - \tau V _ {\psi_ {t}} ^ {\pi_ {\tau} ^ {\star}} (\rho) + \tau V _ {\psi_ {t}} ^ {\pi_ {\theta_ {t}}} (\rho) \\ - \frac {\tau}{1 - \gamma} \sum_ {s, a} d _ {\rho} ^ {\pi_ {\tau} ^ {\star}} (s) \pi_ {\tau} ^ {\star} (a | s) \log \pi_ {\tau} ^ {\star} (a | s) + \frac {\tau}{1 - \gamma} \sum_ {s, a} d _ {\rho} ^ {\pi_ {\theta_ {t}}} (s) \pi_ {\theta_ {t}} (a | s) \log \pi_ {\theta_ {t}} (a | s) \\ = \frac {1}{1 - \gamma} \sum_ {s, a} d _ {\rho} ^ {\pi_ {\tau} ^ {\star}} (s) \left(\pi_ {\tau} ^ {\star} (a \mid s) - \pi_ {\theta_ {t}} (a \mid s)\right) Q _ {r + \lambda_ {t} g + \tau \psi_ {t}} ^ {\pi_ {\theta_ {t}}} (s, a) \\ + \frac {\tau}{1 - \gamma} \sum_ {s, a} d _ {\rho} ^ {\pi_ {\tau} ^ {\star}} (s) \pi_ {\tau} ^ {\star} (a | s) \log \pi_ {\theta_ {t}} (a | s) - \frac {\tau}{1 - \gamma} \sum_ {s, a} d _ {\rho} ^ {\pi_ {\theta_ {t}}} (s) \pi_ {\theta_ {t}} (a | s) \log \pi_ {\theta_ {t}} (a | s) \\ - \frac {\tau}{1 - \gamma} \sum_ {s, a} d _ {\rho} ^ {\pi_ {\tau} ^ {\star}} (s) \pi_ {\tau} ^ {\star} (a | s) \log \pi_ {\tau} ^ {\star} (a | s) + \frac {\tau}{1 - \gamma} \sum_ {s, a} d _ {\rho} ^ {\pi_ {\theta_ {t}}} (s) \pi_ {\theta_ {t}} (a | s) \log \pi_ {\theta_ {t}} (a | s) \\ = \frac {1}{1 - \gamma} \sum_ {s, a} d _ {\rho} ^ {\pi_ {\tau} ^ {\star}} (s) \left(\pi_ {\tau} ^ {\star} (a \mid s) - \pi_ {\theta_ {t}} (a \mid s)\right) \phi_ {s, a} ^ {\top} w _ {t} - \frac {\tau}{1 - \gamma} \sum_ {s} d _ {\rho} ^ {\pi_ {\tau} ^ {\star}} (s) \mathrm {K L} _ {t} (s) \\ + \frac {1}{1 - \gamma} \sum_ {s, a} d _ {\rho} ^ {\pi_ {\tau} ^ {\star}} (s) \left(\pi_ {\tau} ^ {\star} (a | s) - \pi_ {\theta_ {t}} (a | s)\right) \left(Q _ {r + \lambda_ {t} g + \tau \psi_ {t}} ^ {\pi_ {\theta_ {t}}} (s, a) - \phi_ {s, a} ^ {\top} w _ {t}\right) \\ \leq \sum_ {s} d _ {\rho} ^ {\pi_ {\tau} ^ {\star}} (s) \left(\frac {\mathrm {K L} _ {t} (s) - \mathrm {K L} _ {t + 1} (s)}{\eta (1 - \gamma)}\right) + \eta \left(C _ {W}\right) ^ {2} - \frac {\tau}{1 - \gamma} \sum_ {s} d _ {\rho} ^ {\pi_ {\tau} ^ {\star}} (s) \mathrm {K L} _ {t} (s) \\ + \frac {1}{1 - \gamma} \sum_ {s, a} d _ {\rho} ^ {\pi_ {\tau} ^ {\star}} (s) (\pi_ {\tau} ^ {\star} (a | s) - \pi_ {\theta_ {t}} (a | s)) \left(Q _ {r + \lambda_ {t} g + \tau \psi_ {t}} ^ {\pi_ {\theta_ {t}}} (s, a) - \phi_ {s, a} ^ {\top} w _ {t}\right) \\ = \sum_ {s} d _ {\rho} ^ {\pi_ {\tau} ^ {*}} (s) \left(\frac {(1 - \eta \tau) \mathrm {K L} _ {t} (s) - \mathrm {K L} _ {t + 1} (s)}{\eta (1 - \gamma)}\right) + \eta \left(C _ {W}\right) ^ {2} \\ + \frac {1}{1 - \gamma} \sum_ {s, a} d _ {\rho} ^ {\pi_ {\tau} ^ {\star}} (s) \left(\pi_ {\tau} ^ {\star} (a | s) - \pi_ {\theta_ {t}} (a | s)\right) \left(Q _ {r + \lambda_ {t} g + \tau \psi_ {t}} ^ {\pi_ {\theta_ {t}}} (s, a) - \phi_ {s, a} ^ {\top} w _ {t}\right) \\ = \frac {(1 - \eta \tau) \mathrm {K L} _ {t} (\rho) - \mathrm {K L} _ {t + 1} (\rho)}{\eta (1 - \gamma)} + \eta \left(C _ {W}\right) ^ {2} \\ + \frac {1}{1 - \gamma} \sum_ {s, a} d _ {\rho} ^ {\pi_ {\tau} ^ {\star}} (s) \left(\pi_ {\tau} ^ {\star} (a | s) - \pi_ {\theta_ {t}} (a | s)\right) \left(Q _ {r + \lambda_ {t} g + \tau \psi_ {t}} ^ {\pi_ {\theta_ {t}}} (s, a) - \phi_ {s, a} ^ {\top} w _ {t}\right), \\ \end{array}
$$