$$
\begin{array}{l} \nabla_ {\theta} L (\theta) = \int p _ {g} (s) f ^ {\prime} \left(\frac {p _ {\theta} (s)}{p _ {g} (s)}\right) \left(\frac {\nabla_ {\theta} p _ {\theta} (s)}{p _ {g} (s)}\right) d s \\ = \int \nabla p _ {\theta} (s) f ^ {\prime} \left(\frac {p _ {\theta} (s)}{p _ {g} (s)}\right) d s \\ = \int f ^ {\prime} \left(\frac {p _ {\theta} (s)}{p _ {g} (s)}\right) \left(\frac {1}{Z} \int p (\tau) e ^ {\sum_ {t = 1} ^ {T} \log \pi_ {\theta} (s _ {t})} \eta_ {\tau} (s) \sum_ {t = 1} ^ {T} \nabla_ {\theta} \log \pi_ {\theta} (s _ {t}) d \tau \right. \\ \left. - T p _ {\theta} (s) \mathbb {E} _ {s \sim p _ {\theta} (s)} \left[ \nabla_ {\theta} \log \pi_ {\theta} (s) \right]\right) d s \\ = \frac {1}{Z} \int p (\tau) e ^ {\sum_ {t = 1} ^ {T} \log \pi_ {\theta} (s _ {t})} \sum_ {t = 1} ^ {T} \nabla_ {\theta} \log \pi_ {\theta} (s _ {t}) \int \eta_ {\tau} (s) f ^ {\prime} \left(\frac {p _ {\theta} (s)}{p _ {g} (s)}\right) d s d \tau \\ - \int T p _ {\theta} (s) f ^ {\prime} \left(\frac {p _ {\theta} (s)}{p _ {g} (s)}\right) \mathbb {E} _ {s \sim p _ {\theta} (s)} [ \nabla_ {\theta} \log \pi_ {\theta} (s) ] d s \\ = \frac {1}{Z} \int p (\tau) e ^ {\sum_ {t = 1} ^ {T} \log \pi_ {\theta} (s _ {t})} \sum_ {t = 1} ^ {T} \nabla_ {\theta} \log \pi_ {\theta} (s _ {t}) \sum_ {t = 1} ^ {T} f ^ {\prime} \left(\frac {p _ {\theta} (s _ {t})}{p _ {g} (s _ {t})}\right) d \tau \\ - T \int p _ {\theta} (s) f ^ {\prime} \left(\frac {p _ {\theta} (s)}{p _ {g} (s)}\right) \mathbb {E} _ {s \sim p _ {\theta} (s)} [ \nabla_ {\theta} \log \pi_ {\theta} (s) ] d s \\ = \frac {1}{T} \int p _ {\theta} (\tau) \sum_ {t = 1} ^ {T} \nabla_ {\theta} \log \pi_ {\theta} (s _ {t}) \sum_ {t = 1} ^ {T} f ^ {\prime} \left(\frac {p _ {\theta} (s _ {t})}{p _ {g} (s _ {t})}\right) d \tau \\ - T \mathbb {E} _ {s \sim p _ {\theta} (s)} \left[ f ^ {\prime} \left(\frac {p _ {\theta} (s)}{p _ {g} (s)}\right) \right] \mathbb {E} _ {s \sim p _ {\theta} (s)} \left[ \nabla_ {\theta} \log \pi_ {\theta} (s) \right] \\ = \frac {1}{T} \mathbb {E} _ {\tau \sim p _ {\theta} (\tau)} \left[ \sum_ {t = 1} ^ {T} \nabla_ {\theta} \log \pi_ {\theta} (s _ {t}) \sum_ {t = 1} ^ {T} f ^ {\prime} \left(\frac {p _ {\theta} (s _ {t})}{p _ {g} (s _ {t})}\right) \right] \\ - T \mathbb {E} _ {s \sim p _ {\theta} (s)} \left[ f ^ {\prime} \left(\frac {p _ {\theta} (s)}{p _ {g} (s)}\right) \right] \mathbb {E} _ {s \sim p _ {\theta} (s)} [ \nabla_ {\theta} \log \pi_ {\theta} (s) ] \\ = \frac {1}{T} \left[ \mathbb {E} _ {\tau \sim p _ {\theta} (\tau)} \left[ \sum_ {t = 1} ^ {T} \nabla_ {\theta} \log \pi_ {\theta} (s _ {t}) \sum_ {t = 1} ^ {T} f ^ {\prime} \left(\frac {p _ {\theta} (s _ {t})}{p _ {g} (s _ {t})}\right) \right] \right. \\ \left. - \mathbb {E} _ {s \sim p _ {\theta} (s)} \left[ \sum_ {t = 1} ^ {T} f ^ {\prime} \left(\frac {p _ {\theta} (s _ {t})}{p _ {g} (s _ {t})}\right) \right] \mathbb {E} _ {\tau \sim p _ {\theta} (\tau)} \left[ \sum_ {t = 1} ^ {T} \nabla_ {\theta} \log \pi_ {\theta} (s _ {t}) \right] \right] \\ = \frac {1}{T} \mathbb {E} _ {\tau \sim p _ {\theta} (\tau)} \left[ \sum_ {t = 1} ^ {T} \nabla_ {\theta} \log \pi_ {\theta} (s _ {t}) \sum_ {t = 1} ^ {T} f ^ {\prime} \left(\frac {p _ {\theta} (s _ {t})}{p _ {g} (s _ {t})}\right) \right] \\ \end{array}
$$