$$
\begin{array}{l} \sum_ {k = t W + 1} ^ {(t + 1) W} \mathbb {E} _ {\left(s _ {h}, a _ {h}\right) \sim \left(P ^ {\star , k}, \tilde {\pi} ^ {k}\right)} \left[ \left\| \phi_ {h} ^ {\star , k} \left(s _ {h}, a _ {h}\right) \right\| _ {\left(U _ {h, \phi^ {\star}, k} ^ {k, W}\right) ^ {- 1}} ^ {2} - \left\| \phi_ {h} ^ {\star , k} \left(s _ {h}, a _ {h}\right) \right\| _ {\left(U _ {h, \phi^ {\star}, t W + 1} ^ {k, W}\right) ^ {- 1}} ^ {2} \right] \\ = \sum_ {k = t W + 1} ^ {(t + 1) W} \mathbb {E} _ {\left(s _ {h}, a _ {h}\right) \sim \left(P ^ {\star , k}, \tilde {\pi} ^ {k}\right)} \left[ \phi_ {h} ^ {\star , k} \left(s _ {h}, a _ {h}\right) ^ {\top} \left(\left(U _ {h, \phi^ {\star , k}} ^ {k, W}\right) ^ {- 1} - \left(U _ {h, \phi^ {\star , t W + 1}} ^ {k, W}\right) ^ {- 1}\right) \phi_ {h} ^ {\star , k} \left(s _ {h}, a _ {h}\right) \right] \\ = \sum_ {k = t W + 1} ^ {(t + 1) W} \mathbb {E} _ {\left(s _ {h}, a _ {h}\right) \sim \left(P ^ {\star , k}, \tilde {\pi} ^ {k}\right)} \left[ \phi_ {h} ^ {\star , k} \left(s _ {h}, a _ {h}\right) ^ {\top} \left(U _ {h, \phi^ {\star}, k} ^ {k, W}\right) ^ {- 1} \left(U _ {h, \phi^ {\star}, t W + 1} ^ {k, W} - U _ {h, \phi^ {\star}, k} ^ {k, W}\right) \left(U _ {h, \phi^ {\star}, t W + 1} ^ {k, W}\right) ^ {- 1} \phi_ {h} ^ {\star , k} \left(s _ {h}, a _ {h}\right) \right] \\ \overset {(i)} {\leq} \sum_ {k = t W + 1} ^ {(t + 1) W} \mathbb {E} _ {(s _ {h}, a _ {h}) \sim (P ^ {\star , k}, \bar {\pi} ^ {k})} \left[ \left\| \phi_ {h} ^ {\star , k} (s _ {h}, a _ {h}) \right\| _ {2} \left\| (U _ {h, \phi^ {\star}, k} ^ {k, W}) ^ {- 1} \right\| _ {2} \left\| (U _ {h, \phi^ {\star}, t W + 1} ^ {k, W}) - U _ {h, \phi^ {\star}, k} ^ {k, W}) \right\| _ {2} \left\| (U _ {h, \phi^ {\star}, t W + 1} ^ {k, W}) ^ {- 1} \right\| _ {2} \left\| \phi_ {h} ^ {\star , k} (s _ {h}, a _ {h}) \right\| _ {2} \right] \\ \stackrel {(i i)} {\leq} \frac {1}{\lambda_ {W} ^ {2}} \sum_ {k = t W + 1} ^ {(t + 1) W} \mathbb {E} _ {(s _ {h}, a _ {h}) \sim (P ^ {\star}, k, \tilde {\pi} ^ {k})} \left[ \left\| \sum_ {i = 1 \vee k - W} ^ {k - 1} \mathbb {E} _ {(s _ {h}, a _ {h}) \sim (P ^ {\star}, i, \tilde {\pi} ^ {i})} \left[ \phi_ {h} ^ {\star , t W + 1} (s _ {h}, a _ {h}) \phi_ {h} ^ {\star , t W + 1} (s _ {h}, a _ {h}) ^ {\top} - \phi_ {h} ^ {\star , k} (s _ {h}, a _ {h}) \phi_ {h} ^ {\star , k} (s _ {h}, a _ {h}) ^ {\top} \right] \right\| _ {2} \right] \\ \leq \frac {1}{\lambda_ {W} ^ {2}} \sum_ {k = t W + 1} ^ {(t + 1) W} \sum_ {i = 1 \vee k - W} ^ {k - 1} \mathbb {E} _ {(s _ {h}, a _ {h}) \sim (P ^ {\star , i}, \tilde {\pi} ^ {i})} \left[ \left\| \phi_ {h} ^ {\star , t W + 1} (s _ {h}, a _ {h}) \phi_ {h} ^ {\star , t W + 1} (s _ {h}, a _ {h}) ^ {\top} - \phi_ {h} ^ {\star , k} (s _ {h}, a _ {h}) \phi_ {h} ^ {\star , k} (s _ {h}, a _ {h}) ^ {\top} \right\| _ {2} \right] \\ \leq \frac {1}{\lambda_ {W} ^ {2}} \sum_ {k = t W + 1} ^ {(t + 1) W} \sum_ {i = 1 \vee k - W} ^ {k - 1} \underset {(s _ {h}, a _ {h}) \sim (P ^ {\star , i}, \tilde {\pi} ^ {i})} {\mathbb {E}} \left[ \left\| \phi_ {h} ^ {\star , t W + 1} (s _ {h}, a _ {h}) \phi_ {h} ^ {\star , t W + 1} (s _ {h}, a _ {h}) ^ {\top} - \phi_ {h} ^ {\star , t W + 1} (s _ {h}, a _ {h}) \phi_ {h} ^ {\star , k} (s _ {h}, a _ {h}) ^ {\top} \right\| _ {2} \right. \\ + \left\| \phi_ {h} ^ {\star , t W + 1} (s _ {h}, a _ {h}) \phi_ {h} ^ {\star , k} (s _ {h}, a _ {h}) ^ {\top} - \phi_ {h} ^ {\star , k} (s _ {h}, a _ {h}) \phi_ {h} ^ {\star , k} (s _ {h}, a _ {h}) ^ {\top} \right\| _ {2} \Big ] \\ \end{array}
$$