SlowGuess/NeurIPS_2023_OCR huggingface dataset:
$$
\begin{array}{l} \mathbb {E} _ {\text {s e e d s}} \left[ V _ {t} ^ {(T D)} (s) \left(\sum_ {l = 1} ^ {N} \mathbb {I} _ {S _ {l} = x} \left(R _ {l} + \gamma V _ {t} ^ {(T D)} \left(X _ {l} ^ {\prime}\right) - V _ {t} ^ {(T D)} (x)\right)\right) \right] \\ = \mathbb {E} _ {\text {s e e d s}} \left[ V _ {t} ^ {(T D)} (s) \sum_ {l = 1} ^ {N} \mathbb {E} \left[ \left(\mathbb {I} _ {S _ {l} = x} \left(R _ {l} + \gamma V _ {t} ^ {(T D)} \left(X _ {l} ^ {\prime}\right) - V _ {t} ^ {(T D)} (x)\right)\right) | \text {s e e d} \right] \right], \\ = \mathbb {E} _ {\text {s e e d s}} \left[ V _ {t} ^ {(T D)} (s) N \widehat {d _ {\pi}} (x) \left(\widehat {r _ {\pi}} (x) + \gamma (\widehat {\mathcal {P} _ {\pi}} V _ {t} ^ {(T D)}) (x) - V ^ {(T D)} (x)\right) \right], \\ = N \mathbb {E} _ {s e e d s} \bigl [ \widehat {d _ {\pi}} (x) \widehat {r _ {\pi}} (x) V _ {t} ^ {(T D)} (s) \bigr ] + \gamma N \mathbb {E} _ {s e e d s} \bigl [ \widehat {d _ {\pi}} (x) (\widehat {\mathcal {P} _ {\pi}} V _ {t} ^ {(T D)}) (x) V _ {t} ^ {(T D)} (s) \bigr ] \\ - N \mathbb {E} _ {\text {s e e d s}} \left[ \widehat {d _ {\pi}} (x) V _ {t} ^ {(T D)} (x) V _ {t} ^ {(T D)} (s) \right], \\ = N d _ {\pi} (x) r _ {\pi} (x) m ^ {(T D)} (s) + \gamma N d _ {\pi} (x) \mathbb {E} _ {s e e d s} \left[ \sum_ {x ^ {\prime}} \widehat {\mathcal {P}} _ {\pi} (x ^ {\prime} | x) V _ {t} ^ {(T D)} (x ^ {\prime}) V _ {t} ^ {(T D)} (x) \right] \\ - N d _ {\pi} (x) \Sigma_ {t} ^ {(T D)} (s, x), \\ = N d _ {\pi} (x) \left(r _ {\pi} (x) m _ {t} ^ {(T D)} (s) + \gamma \sum_ {x ^ {\prime}} \mathbb {E} _ {s e e d s} [ \widehat {\mathcal {P}} _ {\pi} (x ^ {\prime} | x) V _ {t} ^ {(T D)} (x ^ {\prime}) V _ {t} ^ {(T D)} (x) ] - \Sigma_ {t} ^ {(T D)} (s, x)\right), \\ = N d _ {\pi} (x) \left(r _ {\pi} (x) m _ {t} ^ {(T D)} (s) + \gamma \sum_ {x ^ {\prime}} \mathcal {P} _ {\pi} (x ^ {\prime} | x) \Sigma_ {t} ^ {(T D)} (x ^ {\prime}, s) - \Sigma_ {t} ^ {(T D)} (s, x)\right), \\ = N d _ {\pi} (x) \left(r _ {\pi} (x) m _ {t} ^ {(T D)} (s) + \gamma \left(\mathcal {P} _ {\pi} \Sigma_ {t} ^ {(T D)}\right) (x, s) - \Sigma_ {t} ^ {(T D)} (s, x)\right). \\ \end{array}
$$
A similar simplification can be done to Eq. (10) which results in:
$$
\boxed {(1 0) = N d _ {\pi} (s) \left(r _ {\pi} (s) m ^ {(T D)} (x) + \gamma (\mathcal {P} _ {\pi} \Sigma_ {t} ^ {(T D)}) (s, x) - \Sigma_ {t} ^ {(T D)} (x, s)\right).}
$$
We will break down Eq. (11) into 9 parts and simplify each of them separately.
$$
\begin{array}{l} (1 1) = \mathbb {E} _ {\text {s e e d s}} \left[ \left(\sum_ {k = 1} ^ {N} \mathbb {I} _ {S _ {k} = s} R _ {k}\right) \left(\sum_ {l = 1} ^ {N} \mathbb {I} _ {S _ {l} = x} R _ {l}\right) \right] (12) \\ + \gamma \mathbb {E} _ {\text {s e e d s}} \left[ \left(\sum_ {k = 1} ^ {N} \mathbb {I} _ {S _ {k} = s} R _ {k}\right) \left(\sum_ {l = 1} ^ {N} \mathbb {I} _ {S _ {l} = x} V _ {t} ^ {(T D)} \left(X _ {l} ^ {\prime}\right)\right) \right] (13) \\ - \mathbb {E} _ {\text {s e e d s}} \left[ \left(\sum_ {k = 1} ^ {N} \mathbb {I} _ {S _ {k} = s} R _ {k}\right) \left(\sum_ {l = 1} ^ {N} \mathbb {I} _ {S _ {l} = x} V _ {t} ^ {(T D)} (x)\right) \right] (14) \\ + \gamma \mathbb {E} _ {\text {s e e d s}} \left[ \left(\sum_ {k = 1} ^ {N} \mathbb {I} _ {S _ {k} = s} V _ {t} ^ {(T D)} \left(S _ {k} ^ {\prime}\right)\right) \left(\sum_ {l = 1} ^ {N} \mathbb {I} _ {S _ {l} = x} R _ {l}\right) \right] (15) \\ + \gamma^ {2} \mathbb {E} _ {\text {s e e d s}} \left[ \left(\sum_ {k = 1} ^ {N} \mathbb {I} _ {S _ {k} = s} V _ {t} ^ {(T D)} \left(S _ {k} ^ {\prime}\right)\right) \left(\sum_ {l = 1} ^ {N} \mathbb {I} _ {S _ {l} = x} V _ {t} ^ {(T D)} \left(X _ {l} ^ {\prime}\right)\right) \right] (16) \\ - \gamma \mathbb {E} _ {\text {s e e d s}} \left[ \left(\sum_ {k = 1} ^ {N} \mathbb {I} _ {S _ {k} = s} V _ {t} ^ {(T D)} \left(S _ {k} ^ {\prime}\right)\right) \left(\sum_ {l = 1} ^ {N} \mathbb {I} _ {S _ {l} = x} V _ {t} ^ {(T D)} (x)\right) \right] (17) \\ - \mathbb {E} _ {\text {s e e d s}} \left[ \left(\sum_ {k = 1} ^ {N} \mathbb {I} _ {S _ {k} = s} V _ {t} ^ {(T D)} (s)\right) \left(\sum_ {l = 1} ^ {N} \mathbb {I} _ {S _ {l} = x} R _ {l}\right) \right] (18) \\ - \gamma \mathbb {E} _ {\text {s e e d s}} \left[ \left(\sum_ {k = 1} ^ {N} \mathbb {I} _ {S _ {k} = s} V _ {t} ^ {(T D)} (s)\right) \left(\sum_ {l = 1} ^ {N} \mathbb {I} _ {S _ {l} = x} V _ {t} ^ {(T D)} \left(X _ {l} ^ {\prime}\right)\right) \right] (19) \\ + \mathbb {E} _ {\text {s e e d s}} \left[ \left(\sum_ {k = 1} ^ {N} \mathbb {I} _ {S _ {k} = s} V _ {t} ^ {(T D)} (s)\right) \left(\sum_ {l = 1} ^ {N} \mathbb {I} _ {S _ {l} = x} V _ {t} ^ {(T D)} (x)\right) \right]. (20) \\ \end{array}
$$
Consider Eq. (12)
$$
\begin{array}{l} \mathbb {E} _ {\text {s e e d s}} \left[ \left(\sum_ {k = 1} ^ {N} \mathbb {I} _ {S _ {k} = s} R _ {k}\right) \left(\sum_ {l = 1} ^ {N} \mathbb {I} _ {S _ {l} = x} R _ {l}\right) \right] = \mathbb {E} _ {\text {s e e d s}} \left[ \mathbb {E} \left[ \left(\sum_ {k = 1} ^ {N} \mathbb {I} _ {S _ {k} = s} R _ {k}\right) \left(\sum_ {l = 1} ^ {N} \mathbb {I} _ {S _ {l} = x} R _ {l}\right) | \text {s e e d} \right] \right], \\ = \mathbb{E}_{\text{\it seeds}}\bigg[\mathbb{E}\bigg[\left(\sum_{k = 1}^{N}\mathbb{I}_{S_{k} = s}R_{k}\right)\left(\sum_{\substack{l = 1,\\ l\neq k}}^{N}\mathbb{I}_{S_{l} = x}R_{l}\right)|\text{\it seed}\bigg]\bigg] + \mathbb{E}_{\text{\it seeds}}\bigg[\mathbb{E}\bigg[\left(\sum_{m = 1}^{N}\mathbb{I}_{S_{m} = s}\mathbb{I}_{S_{m} = x}R_{m}R_{m}\right)|\text{\it seed}\bigg]\bigg], \\ = \mathbb{E}_{\text{\it seeds}}\bigg[\mathbb{E}\bigg[\left(\sum_{k = 1}^{N}\mathbb{I}_{S_{k} = s}R_{k}\right)|\text{\it seed}\bigg]\mathbb{E}\bigg[\left(\sum_{\substack{l = 1,\\ l\neq k}}^{N}\mathbb{I}_{S_{l} = x}R_{l}\right)|\text{\it seed}\bigg]\bigg] + \mathbb{I}_{s = x}\mathbb{E}_{\text{\it seeds}}\bigg[\sum_{m = 1}^{N}\widehat{d_{\pi}} (s)\mathbb{E}[R_{m}^{2}| \text{\it seed},S_{m} = s]\bigg], \\ = \mathbb{E}_{\text{\it seeds}}\bigg[\sum_{k = 1}^{N}\widehat{d_{\pi}} (s)\widehat{r_{\pi}} (s)\sum_{\substack{l = 1,\\ l\neq k}}^{N}\widehat{d_{\pi}} (x)\widehat{r_{\pi}} (x)\bigg] + \mathbb{I}_{s = x}\sum_{m = 1}^{N}\mathbb{E}_{\text{\it seeds}}[\widehat{d_{\pi}} (s)(\widehat{c(s)} +\widehat{r_{\pi}}^{2}(s)], \\ = \sum_{k = 1}^{N}\sum_{\substack{l = 1,\\ l\neq k}}^{N}\mathbb{E}_{\text{seeds}}[\widehat{d_{\pi}} (s)\widehat{r_{\pi}} (s)\widehat{d_{\pi}} (x)\widehat{r_{\pi}} (x)] + \mathbb{I}_{s = x}Nd_{\pi}(s) \left(c(s) + r_{\pi}^{2}(s)\right), \\ = N (N - 1) d _ {\pi} (s) d _ {\pi} (x) r _ {\pi} (s) r _ {\pi} (x) + \mathbb {I} _ {s = x} N d _ {\pi} (s) \left(c (s) + r _ {\pi} ^ {2} (s)\right). \\ \end{array}
$$
We have used the i.i.d assumption to separate the terms. Later, we used the indicator function to indicate the part which is non-zero. Also, we have used the definition of the second moment in the penultimate step.
We do a similar simplification to Eq. (13) now.
$$
\begin{array}{l} \gamma \mathbb {E} _ {s e e d s} \left[ \left(\sum_ {k = 1} ^ {N} \mathbb {I} _ {S _ {k} = s} R _ {k}\right) \left(\sum_ {l = 1} ^ {N} \mathbb {I} _ {S _ {l} = x} V _ {t} ^ {(T D)} (X _ {l} ^ {\prime})\right) \right] \\ = \gamma \mathbb {E} _ {s e e d s} \left[ \sum_ {k = 1} ^ {N} \sum_ {\substack {l = 1, \\ l \neq k}} ^ {N} \mathbb {I} _ {S _ {k} = s} \mathbb {I} _ {S _ {l} = x} R _ {k} V _ {t} ^ {(T D)} (X _ {l} ^ {\prime}) \right] + \gamma \mathbb {E} _ {s e e d s} \left[ \sum_ {m = 1} ^ {N} \mathbb {I} _ {S _ {m} = s} \mathbb {I} _ {S _ {m} = x} R _ {m} V _ {t} ^ {(T D)} (X _ {m} ^ {\prime}) \right], \\ = \gamma \mathbb{E}_{\text{\it{seeds}}}\left[\sum_{k = 1}^{N}\sum_{\substack{l = 1,\\ l\neq k}}^{N}\mathbb{E}[\mathbb{I}_{S_{k} = s}\mathbb{I}_{S_{l} = x}R_{k}V_{t}^{(TD)}(X_{l}^{\prime})| \text{\it{seed}} ]\right] + \gamma \mathbb{I}_{s = x}\sum_{m = 1}^{N}\mathbb{E}_{\text{\it{seeds}}}\left[\mathbb{E}[\mathbb{I}_{S_{m} = s}\mathbb{I}_{S_{m} = x}R_{m}V_{t}^{(TD)}(X_{m}^{\prime})| \text{\it{seed}} ]\right], \\ = \gamma \mathbb{E}_{\text{\it seeds}}\left[\sum_{k = 1}^{N}\sum_{\substack{l = 1,\\ l\neq k}}^{N}\widehat{d_{\pi}} (s)\widehat{d_{\pi}} (x)\widehat{r_{\pi}} (s)(\widehat{\mathcal{P}_{\pi}} V_{t}^{(TD)}) (x)\right] + \gamma \mathbb{I}_{s = x}N\mathbb{E}_{\text{\it seeds}}[\widehat{d_{\pi}} (s)r_{\pi}(s)(\widehat{\mathcal{P}_{\pi}} V_{t}^{(TD)}) (s)], \\ = \gamma N (N - 1) d _ {\pi} (s) d _ {\pi} (x) r _ {\pi} (s) (\mathcal {P} _ {\pi} m _ {t} ^ {(T D)}) (x) + \gamma \mathbb {I} _ {s = x} N d _ {\pi} (s) r _ {\pi} (s) (\mathcal {P} _ {\pi} m _ {t} ^ {(T D)}) (s) \\ \end{array}
$$
We will simplify Eq. (14) now.
$$
\mathbb {E} _ {s e e d s} \left[ \left(\sum_ {k = 1} ^ {N} \mathbb {I} _ {S _ {k} = s} R _ {k}\right) \left(\sum_ {l = 1} ^ {N} \mathbb {I} _ {S _ {l} = x} V _ {t} ^ {(T D)} (x)\right) \right]
$$
$$
\begin{array}{l} = \mathbb{E}_{\text{\it seeds}}\left[\sum_{\substack{k = 1\\ l\neq k}}^{N}\sum_{\substack{l = 1,\\ l\neq k}}^{N}\mathbb{I}_{S_{k} = s}\mathbb{I}_{S_{l} = x}R_{k}V_{t}^{(TD)}(x)\right] + \mathbb{E}_{\text{\it seeds}}\left[\sum_{m = 1}^{N}\mathbb{I}_{S_{m} = s}\mathbb{I}_{S_{m} = x}R_{m}V_{t}^{(TD)}(x)\right] \\ = \mathbb {E} _ {s e e d s} \left[ \sum_ {\substack {k = 1 \\ l \neq k}} ^ {N} \sum_ {\substack {l = 1, \\ l \neq k}} ^ {N} \mathbb {E} [ \mathbb {I} _ {S _ {k} = s} \mathbb {I} _ {S _ {l} = x} R _ {k} V _ {t} ^ {(T D)} (x) | s e e d ] \right] + \mathbb {I} _ {s = x} \sum_ {m = 1} ^ {N} \mathbb {E} _ {s e e d s} [ \mathbb {E} [ \mathbb {I} _ {S _ {m} = s} \mathbb {I} _ {S _ {m} = x} R _ {m} V _ {t} ^ {(T D)} (x) | s e e d ] ], \\ = \mathbb {E} _ {s e e d s} \left[ \sum_ {k = 1} ^ {N} \sum_ {\substack {l = 1, \\ l \neq k}} ^ {N} \widehat {d _ {\pi}} (s) \widehat {d _ {\pi}} (x) r _ {\widehat {\pi}} (s) V _ {t} ^ {(T D)} (x) \right] + \mathbb {I} _ {s = x} N \mathbb {E} _ {s e e d s} [ \widehat {d _ {\pi}} (s) r _ {\pi} (s) V _ {t} ^ {(T D)} (s) ], \\ = N (N - 1) d _ {\pi} (s) d _ {\pi} (x) r _ {\pi} (s) m _ {t} ^ {(T D)} (x) + \mathbb {I} _ {s = x} N d _ {\pi} (s) r _ {\pi} (s) m _ {t} ^ {(T D)} (s) \\ \end{array}
$$
The next expression Eq. (15) is similar to Eq. (13) and therefore, we skip the simplification details.
$$
\begin{array}{l} \gamma \mathbb {E} _ {\text {s e e d s}} \left[ \left(\sum_ {k = 1} ^ {N} \mathbb {I} _ {S _ {k} = s} V _ {t} ^ {(T D)} \left(S _ {k} ^ {\prime}\right)\right) \left(\sum_ {l = 1} ^ {N} \mathbb {I} _ {S _ {l} = x} R _ {l}\right) \right] \\ = \gamma N (N - 1) d _ {\pi} (s) d _ {\pi} (x) r _ {\pi} (x) (\mathcal {P} _ {\pi} m _ {t} ^ {(T D)}) (s) + \gamma \mathbb {I} _ {s = x} N d _ {\pi} (s) r _ {\pi} (s) (\mathcal {P} _ {\pi} m _ {t} ^ {(T D)}) (s) \\ \end{array}
$$
Now, we simplify (16),
$$
\begin{array}{l} \gamma^ {2} \mathbb {E} _ {\text {s e e d s}} \left[ \left(\sum_ {k = 1} ^ {N} \mathbb {I} _ {S _ {k} = s} V _ {t} ^ {(T D)} \left(S _ {k} ^ {\prime}\right)\right) \left(\sum_ {l = 1} ^ {N} \mathbb {I} _ {S _ {l} = x} V _ {t} ^ {(T D)} \left(X _ {l} ^ {\prime}\right)\right) \right] \\ = \gamma^{2}\mathbb{E}_{\text{\it seeds}}\left[\sum_{k = 1}^{N}\sum_{\substack{l = 1,\\ l\neq k}}^{N}\mathbb{I}_{S_{k} = s}\mathbb{I}_{S_{l} = x}V_{t}^{(TD)}(S_{k}^{\prime})V_{t}^{(TD)}(X_{l}^{\prime})\right] + \gamma^{2}\mathbb{E}_{\text{\it seeds}}\left[\sum_{m = 1}^{N}\mathbb{I}_{S_{m} = s}\mathbb{I}_{S_{m} = x}V_{t}^{(TD)}(S_{m}^{\prime})V_{t}^{(TD)}(X_{m}^{\prime})\right] \\ = \gamma^{2}\mathbb{E}_{\text{\it{seeds}}} \left[\sum_{k = 1}^{N}\sum_{\substack{l = 1,\\ l\neq k}}^{N}\mathbb{E}[\mathbb{I}_{S_{k} = s}\mathbb{I}_{S_{l} = x}V_{t}^{(TD)}(S_{k}^{\prime})V_{t}^{(TD)}(X_{l}^{\prime})| \text{\it{seed}} ]\right] \\ + \gamma^ {2} \mathbb {I} _ {s = x} \sum_ {m = 1} ^ {N} \mathbb {E} _ {s e e d s} [ \mathbb {E} [ \mathbb {I} _ {S _ {m} = s} \mathbb {I} _ {S _ {m} = x} V _ {t} ^ {(T D)} (S _ {m} ^ {\prime}) V _ {t} ^ {(T D)} (X _ {m} ^ {\prime}) | s e e d ] ], \\ = \gamma^{2}\mathbb{E}_{\text{\it{seeds}}} \left[\sum_{k = 1}^{N}\sum_{\substack{l = 1,\\ l\neq k}}^{N}\widehat{d_{\pi}} (s)\widehat{d_{\pi}} (x)(\widehat{\mathcal{P}_{\pi}} V_{t}^{(TD)}) (s)(\widehat{\mathcal{P}_{\pi}} V_{t}^{(TD)}) (x)\right] \\ + \gamma^ {2} \mathbb {I} _ {s = x} \sum_ {m = 1} ^ {N} \mathbb {E} _ {s e e d s} [ \widehat {d _ {\pi}} (s) (\widehat {\mathcal {P} _ {\pi}} V _ {t} ^ {(T D)}) (s) (\widehat {\mathcal {P} _ {\pi}} V _ {t} ^ {(T D)}) (x) ], \\ = \gamma^ {2} N (N - 1) d _ {\pi} (s) d _ {\pi} (x) (\mathcal {P} _ {\pi} \Sigma_ {t} ^ {(T D)} \mathcal {P} _ {\pi} ^ {T}) (s, x) + \gamma^ {2} \mathbb {I} _ {s = x} N d _ {\pi} (s) (\mathcal {P} _ {\pi} d i a g (\Sigma_ {t} ^ {(T D)})) (s) \\ \end{array}
$$
We now consider the next piece Eq. (17),
$$
\gamma \mathbb {E} _ {s e e d s} \left[ \left(\sum_ {k = 1} ^ {N} \mathbb {I} _ {S _ {k} = s} V _ {t} ^ {(T D)} (S _ {k} ^ {\prime})\right) \left(\sum_ {l = 1} ^ {N} \mathbb {I} _ {S _ {l} = x} V _ {t} ^ {(T D)} (x)\right) \right]
$$
$$
\begin{array}{l} = \gamma \mathbb{E}_{\text{\it seeds}}\left[\sum_{k = 1}^{N}\sum_{\substack{l = 1,\\ l\neq k}}^{N}\mathbb{I}_{S_{k} = s}\mathbb{I}_{S_{l} = x}V_{t}^{(TD)}(S_{k}^{\prime})V_{t}^{(TD)}(x)\right] + \gamma \mathbb{E}_{\text{\it seeds}}\left[\sum_{m = 1}^{N}\mathbb{I}_{S_{m} = s}\mathbb{I}_{S_{m} = x}V_{t}^{(TD)}(S_{m}^{\prime})V_{t}^{(TD)}(x)\right] \\ = \gamma \mathbb{E}_{\text{\it{seeds}}} \left[\sum_{k = 1}^{N}\sum_{\substack{l = 1,\\ l\neq k}}^{N}\mathbb{E}[\mathbb{I}_{S_{k} = s}\mathbb{I}_{S_{l} = x}V_{t}^{(TD)}(S_{k}^{\prime})V_{t}^{(TD)}(x)|\text{\it{seed}} ]\right] \\ + \gamma \mathbb {I} _ {s = x} \sum_ {m = 1} ^ {N} \mathbb {E} _ {\text {s e e d s}} \left[ \mathbb {E} \left[ \mathbb {I} _ {S _ {m} = s} \mathbb {I} _ {S _ {m} = x} V _ {t} ^ {(T D)} \left(S _ {m} ^ {\prime}\right) V _ {t} ^ {(T D)} (x) | \text {s e e d} \right] \right], \\ = \gamma \mathbb{E}_{\text{\it seeds}}\left[\sum_{k = 1}^{N}\sum_{\substack{l = 1,\\ l\neq k}}^{N}\widehat{d_{\pi}} (s)\widehat{d_{\pi}} (x)(\widehat{\mathcal{P}_{\pi}} V_{t}^{(TD)})(s)V_{t}^{(TD)}(x)\right] + \gamma \mathbb{I}_{s = x}\sum_{m = 1}^{N}\mathbb{E}_{\text{\it seeds}}[\widehat{d_{\pi}} (s) (\widehat{\mathcal{P}_{\pi}} V_{t}^{(TD)})(s)V_{t}^{(TD)}(s)], \\ \end{array}
$$
$$
= \gamma N (N - 1) d _ {\pi} (s) d _ {\pi} (x) (\mathcal {P} _ {\pi} \Sigma_ {t} ^ {(T D)}) (s, x) + \gamma \mathbb {I} _ {s = x} N d _ {\pi} (s) (\mathcal {P} _ {\pi} \Sigma_ {t} ^ {(T D)}) (s, s).
$$
Simplification of Eq. (18) is similar to Eq. (14).
$$
\begin{array}{l} \mathbb {E} _ {\text {s e e d s}} \left[ \left(\sum_ {k = 1} ^ {N} \mathbb {I} _ {S _ {k} = s} V _ {t} ^ {(T D)} (s)\right) \left(\sum_ {l = 1} ^ {N} \mathbb {I} _ {S _ {l} = x} R _ {l}\right) \right] \\ = N (N - 1) d _ {\pi} (s) d _ {\pi} (x) r _ {\pi} (x) m ^ {(T D)} (s) + \mathbb {I} _ {s = x} N d _ {\pi} (s) r _ {\pi} (s) m ^ {(T D)} (s). \\ \end{array}
$$
Also, the simplification of Eq. (19) is similar to that of Eq. (17).
$$
\begin{array}{l} \gamma \mathbb {E} _ {\text {s e e d s}} \left[ \left(\sum_ {k = 1} ^ {N} \mathbb {I} _ {S _ {k} = s} V _ {t} ^ {(T D)} (s)\right) \left(\sum_ {l = 1} ^ {N} \mathbb {I} _ {S _ {l} = x} V _ {t} ^ {(T D)} \left(X _ {l} ^ {\prime}\right)\right) \right] \\ \overline {{= \gamma N (N - 1) d _ {\pi} (s) d _ {\pi} (x) (\mathcal {P} _ {\pi} \Sigma_ {t} ^ {(T D)}) ^ {T} (s , x) + \gamma \mathbb {I} _ {s = x} N d _ {\pi} (s) (\mathcal {P} _ {\pi} \Sigma_ {t} ^ {(T D)}) ^ {T} (s , s)}}. \\ \end{array}
$$
We will simplify the final part i.e., Eq. (20).
$$
\begin{array}{l} \mathbb {E} _ {\text {s e e d s}} \left[ \left(\sum_ {k = 1} ^ {N} \mathbb {I} _ {S _ {k} = s} V _ {t} ^ {(T D)} (s)\right) \left(\sum_ {l = 1} ^ {N} \mathbb {I} _ {S _ {l} = x} V _ {t} ^ {(T D)} (x)\right) \right] \\ = \mathbb{E}_{\text{\it seeds}}\left[\sum_{k = 1}^{N}\sum_{\substack{l = 1,\\ l\neq k}}^{N}\mathbb{I}_{S_{k} = s}\mathbb{I}_{S_{l} = x}V_{t}^{(TD)}(s)V_{t}^{(TD)}(x)\right] + \mathbb{E}_{\text{\it seeds}}\left[\sum_{m = 1}^{N}\mathbb{I}_{S_{m} = s}\mathbb{I}_{S_{m} = x}V_{t}^{(TD)}(s)V_{t}^{(TD)}(x)\right] \\ = \mathbb{E}_{\text{seeds}}\left[\sum_{\substack{k = 1\\ l\neq k}}^{N}\sum_{\substack{l = 1,\\ l\neq k}}^{N}\mathbb{E}[\mathbb{I}_{S_{k} = s}\mathbb{I}_{S_{l} = x}V_{t}^{(TD)}(s)V_{t}^{(TD)}(x)|\text{seed} ]\right] \\ + \gamma \mathbb {I} _ {s = x} \sum_ {m = 1} ^ {N} \mathbb {E} _ {s e e d s} \left[ \mathbb {E} \left[ \mathbb {I} _ {S _ {m} = s} \mathbb {I} _ {S _ {m} = x} V _ {t} ^ {(T D)} (s) V _ {t} ^ {(T D)} (x) | s e e d \right] \right], \\ = \mathbb{E}_{\text{\it seeds}}\left[\sum_{k = 1}^{N}\sum_{\substack{l = 1,\\ l\neq k}}^{N}\widehat{d_{\pi}} (s)\widehat{d_{\pi}} (x)V_{t}^{(TD)}(s)V_{t}^{(TD)}(x)\right] + \gamma \mathbb{I}_{s = x}\sum_{m = 1}^{N}\mathbb{E}_{\text{\it seeds}}[\widehat{d_{\pi}} (s)V_{t}^{(TD)}(s)V_{t}^{(TD)}(s)], \\ \Big | = N (N - 1) d _ {\pi} (s) d _ {\pi} (x) \Sigma_ {t} ^ {(T D)} (s, x) + \gamma \mathbb {I} _ {s = x} N d _ {\pi} (s) \Sigma_ {t} ^ {(T D)} (s, s). \\ \end{array}
$$
We can put all these pieces together to get the desired result.
$$
\boxed {\Sigma_ {t} ^ {(T D)} (s, x) = \Sigma_ {t} ^ {(T D)} (s, x) + \alpha \Omega_ {t} ^ {(T D)} (s, x) + \alpha^ {2} \Psi_ {t} ^ {(T D)} (s, x),}
$$
$$
\begin{array}{l} \Omega_ {t} ^ {(T D)} (s, x) = N d _ {\pi} (x) \left(r _ {\pi} (x) m _ {t} ^ {(T D)} (s) + \gamma (\mathcal {P} _ {\pi} \Sigma_ {t} ^ {(T D)}) (x, s) - \Sigma_ {t} ^ {(T D)} (s, x)\right) \\ + N d _ {\pi} (s) \left(r _ {\pi} (s) m _ {t} ^ {(T D)} (x) + \gamma (\mathcal {P} _ {\pi} \Sigma_ {t} ^ {(T D)}) (s, x) - \Sigma_ {t} ^ {(T D)} (x, s)\right), \\ \end{array}
$$
$$
\begin{array}{l} \Psi_ {t} ^ {(T D)} (s, x) = N (N - 1) d _ {\pi} (s) d _ {\pi} (x) r _ {\pi} (s) r _ {\pi} (x) + \mathbb {I} _ {s = x} N d _ {\pi} (s) \left(c (s) + r _ {\pi} ^ {2} (s)\right) \\ + \gamma N (N - 1) d _ {\pi} (s) d _ {\pi} (x) r _ {\pi} (s) (\mathcal {P} _ {\pi} m _ {t} ^ {(T D)}) (x) + \gamma \mathbb {I} _ {s = x} N d _ {\pi} (s) r _ {\pi} (s) (\mathcal {P} _ {\pi} m _ {t} ^ {(T D)}) (s) \\ - N (N - 1) d _ {\pi} (s) d _ {\pi} (x) r _ {\pi} (s) m _ {t} ^ {(T D)} (x) - \mathbb {I} _ {s = x} N d _ {\pi} (s) r _ {\pi} (s) m _ {t} ^ {(T D)} (s) \\ + \gamma N (N - 1) d _ {\pi} (s) d _ {\pi} (x) r _ {\pi} (x) (\mathcal {P} _ {\pi} m _ {t} ^ {(T D)}) (s) + \gamma \mathbb {I} _ {s = x} N d _ {\pi} (s) r _ {\pi} (s) (\mathcal {P} _ {\pi} m _ {t} ^ {(T D)}) (s) \\ + \gamma^ {2} N (N - 1) d _ {\pi} (s) d _ {\pi} (x) \left(\mathcal {P} _ {\pi} \Sigma_ {t} ^ {(T D)} \mathcal {P} _ {\pi} ^ {T}\right) (s, x) + \gamma^ {2} \mathbb {I} _ {s = x} N d _ {\pi} (s) \left(\mathcal {P} _ {\pi} \operatorname {d i a g} \left(\Sigma_ {t} ^ {(T D)}\right)\right) (s) \\ - \gamma N (N - 1) d _ {\pi} (s) d _ {\pi} (x) (\mathcal {P} _ {\pi} \Sigma_ {t} ^ {(T D)}) (s, x) - \gamma \mathbb {I} _ {s = x} N d _ {\pi} (s) (\mathcal {P} _ {\pi} \Sigma_ {t} ^ {(T D)}) (s, s) \\ - N (N - 1) d _ {\pi} (s) d _ {\pi} (x) r _ {\pi} (x) m _ {t} ^ {(T D)} (s) - \mathbb {I} _ {s = x} N d _ {\pi} (s) r _ {\pi} (s) m _ {t} ^ {(T D)} (s) \\ - \gamma N (N - 1) d _ {\pi} (s) d _ {\pi} (x) \left(\mathcal {P} _ {\pi} \Sigma_ {t} ^ {(T D)}\right) ^ {T} (s, x) - \gamma \mathbb {I} _ {s = x} N d _ {\pi} (s) \left(\mathcal {P} _ {\pi} \Sigma_ {t} ^ {(T D)}\right) ^ {T} (s, s) \\ + N (N - 1) d _ {\pi} (s) d _ {\pi} (x) \Sigma_ {t} ^ {(T D)} (s, x) + \mathbb {I} _ {s = x} N d _ {\pi} (s) \Sigma_ {t} ^ {(T D)} (s, s) \\ \end{array}
$$
We will now prove the final part of the theorem. The mean squared error  $\xi_{t + 1}^{(TD)}$  can be decomposed into bias and covariance terms:
$$
\begin{array}{l} \xi_ {t + 1} ^ {(T D)} = \sum_ {s} (b i a s _ {t + 1} ^ {2} (s) + C o v _ {t + 1} (s, s)), \\ = \sum_ {s} \left(\left(v _ {\pi} (s) - m _ {t + 1} ^ {(T D)} (s)\right) + \left(\Sigma_ {t + 1} ^ {(T D)} (s, s) - m _ {t + 1} ^ {(T D)} (s) m _ {t + 1} ^ {(T D)} (s)\right)\right), \\ = \sum_ {s} (v _ {\pi} ^ {2} (s) - 2 v _ {\pi} (s) m _ {t + 1} ^ {(T D)} (s) + \Sigma_ {t + 1} ^ {(T D)} (s, s)), \\ = \sum_ {s} \Big (v _ {\pi} ^ {2} (s) - 2 v _ {\pi} (s) (m _ {t} ^ {(T D)} (s) + \alpha \Delta_ {t} ^ {(T D)} (s)) + \Sigma_ {t} ^ {(T D)} (s, s) + \alpha \Omega_ {t} ^ {(T D)} (s, s) + \alpha^ {2} \Psi_ {t} ^ {(T D)} (s, s) \Big), \\ = \sum_ {s} \left(v _ {\pi} ^ {2} (s) - 2 v _ {\pi} (s) m _ {t} ^ {(T D)} (s) + \Sigma_ {t} ^ {(T D)} (s, s)\right) \\ + \sum_ {s} \left(- 2 \alpha v _ {\pi} (s) \Delta_ {t} ^ {(T D)} (s)\right) + \alpha \Omega_ {t} ^ {(T D)} (s, s) + \alpha^ {2} \Psi_ {t} ^ {(T D)} (s, s)), \\ \boxed {= \xi_ {t} ^ {(T D)} + \alpha \sum_ {s \in \mathcal {S}} (- 2 v _ {\pi} (s) \Delta_ {t} ^ {(T D)} (s) + \Omega_ {t} ^ {(T D)} (s, s)) + \alpha^ {2} \sum_ {s \in \mathcal {S}} \Psi_ {t} ^ {(T D)} (s, s).} \\ \end{array}
$$
Theorem B.3.4. Let  $\mathbb{E}_{\text{seeds}}$  denote the expectation with respect to seeds. Let
$$
m _ {t} ^ {(P T)} (s) = \mathbb {E} _ {\text {s e e d s}} \left[ V _ {t} ^ {(P T)} (s) \right],
$$
$$
C o v (V _ {t} ^ {(P T)} (s), V _ {t} ^ {(P T)} (x)) = \Sigma_ {t} ^ {(P T)} (s, x) - m _ {t} ^ {(P T)} (s) m _ {t} ^ {(P T)} (x),
$$
where  $\Sigma_t^{(PT)}(s,x) = \mathbb{E}_{\text{seeds}}[V_t^{(PT)}(s)V_t^{(PT)}(s)]$ . Then:
$$
m _ {t + 1} ^ {(P T)} (s) = m _ {t} ^ {(P T)} (s) + \alpha N d _ {\pi} (s) \Delta_ {t} ^ {(P T)} (s),
$$
$$
\Sigma_ {t + 1} ^ {(P T)} (s, x) = \Sigma_ {t} ^ {(P T)} (s, x) + \alpha \Omega_ {t} ^ {(P T)} (s, x) + \alpha^ {2} \Psi_ {t} ^ {(P T)} (s, x),
$$
and, the mean squared error  $\xi_t^{(PT)}$  at timestep  $t$  is
$$
\begin{array}{l} \xi_ {t + 1} ^ {(P T)} = \xi_ {t} ^ {(P T)} + \alpha \sum_ {s \in \mathcal {S}} d _ {\pi} (s) (- 2 v _ {\pi} (s) \Delta_ {t} ^ {(P T)} (s) + \Omega_ {t} ^ {(P T)} (s, s)) \\ + \alpha^ {2} \sum_ {s \in \mathcal {S}} d _ {\pi} (s) \Psi_ {t} ^ {(P T)} (s, s). \\ \end{array}
$$
Proof. The proof is similar to that of TD learning algorithm. We have:
$$
m _ {t + 1} ^ {(P T)} (s) = m _ {t} ^ {(P T)} (s) + \alpha N d _ {\pi} (s) (r _ {\pi} (s) + \gamma (\mathcal {P} _ {\pi} m _ {t} ^ {(P T)}) (s) - m _ {t} ^ {(P T)} (s)),
$$
$$
\Sigma_ {t} ^ {(P T)} (s, x) = \Sigma_ {t} ^ {(P T)} (s, x) + \alpha \Omega_ {t} ^ {(P T)} (s, x) + \alpha^ {2} \Psi_ {t} ^ {(P T)} (s, x),
$$
$$
\begin{array}{l} \Omega_ {t} ^ {(P T)} (s, x) = N d _ {\pi} (x) \left(r _ {\pi} (x) m _ {t} ^ {(P T)} (s) + \gamma \left(\mathcal {P} _ {\pi} \Sigma_ {t} ^ {(P T)}\right) (x, s) - \Sigma_ {t} ^ {(P T)} (s, x)\right) \\ + N d _ {\pi} (s) \left(r _ {\pi} (s) m _ {t} ^ {(P T)} (x) + \gamma (\mathcal {P} _ {\pi} \Sigma_ {t} ^ {(P T)}) (s, x) - \Sigma_ {t} ^ {(P T)} (x, s)\right), \\ \end{array}
$$
$$
\begin{array}{l} \Psi_ {t} ^ {(P T)} (s, x) = N (N - 1) d _ {\pi} (s) d _ {\pi} (x) r _ {\pi} (s) r _ {\pi} (x) + \mathbb {I} _ {s = x} N d _ {\pi} (s) \left(c (s) + r _ {\pi} ^ {2} (s)\right) \\ + \gamma N (N - 1) d _ {\pi} (s) d _ {\pi} (x) r _ {\pi} (s) \left(\mathcal {P} _ {\pi} m _ {t} ^ {(P T)}\right) (x) + \gamma \mathbb {I} _ {s = x} N d _ {\pi} (s) r _ {\pi} (s) \left(\mathcal {P} _ {\pi} m _ {t} ^ {(P T)}\right) (s) \\ - N (N - 1) d _ {\pi} (s) d _ {\pi} (x) r _ {\pi} (s) m _ {t} ^ {(P T)} (x) - \mathbb {I} _ {s = x} N d _ {\pi} (s) r _ {\pi} (s) m _ {t} ^ {(P T)} (s) \\ + \gamma N (N - 1) d _ {\pi} (s) d _ {\pi} (x) r _ {\pi} (x) \left(\mathcal {P} _ {\pi} m _ {t} ^ {(P T)}\right) (s) + \gamma \mathbb {I} _ {s = x} N d _ {\pi} (s) r _ {\pi} (s) \left(\mathcal {P} _ {\pi} m _ {t} ^ {(P T)}\right) (s) \\ + \gamma^ {2} N (N - 1) d _ {\pi} (s) d _ {\pi} (x) \left(\mathcal {P} _ {\pi} \Sigma_ {t} ^ {(P T)} \mathcal {P} _ {\pi} ^ {T}\right) (s, x) + \gamma^ {2} \mathbb {I} _ {s = x} N d _ {\pi} (s) \left(\mathcal {P} _ {\pi} \operatorname {d i a g} \left(\Sigma_ {t} ^ {(P T)}\right)\right) (s) \\ - \gamma N (N - 1) d _ {\pi} (s) d _ {\pi} (x) \left(\mathcal {P} _ {\pi} \Sigma_ {t} ^ {(P T)}\right) (s, x) - \gamma \mathbb {I} _ {s = x} N d _ {\pi} (s) \left(\mathcal {P} _ {\pi} \Sigma_ {t} ^ {(P T)}\right) (s, s) \\ - N (N - 1) d _ {\pi} (s) d _ {\pi} (x) r _ {\pi} (x) m _ {t} ^ {(P T)} (s) - \mathbb {I} _ {s = x} N d _ {\pi} (s) r _ {\pi} (s) m _ {t} ^ {(P T)} (s) \\ - \gamma N (N - 1) d _ {\pi} (s) d _ {\pi} (x) \left(\mathcal {P} _ {\pi} \Sigma_ {t} ^ {(P T)}\right) ^ {T} (s, x) - \gamma \mathbb {I} _ {s = x} N d _ {\pi} (s) \left(\mathcal {P} _ {\pi} \Sigma_ {t} ^ {(P T)}\right) ^ {T} (s, s) \\ + N (N - 1) d _ {\pi} (s) d _ {\pi} (x) \Sigma_ {t} ^ {(P T)} (s, x) + \mathbb {I} _ {s = x} N d _ {\pi} (s) \Sigma_ {t} ^ {(P T)} (s, s) \\ \end{array}
$$
The MSE recursive equation can be obtained similar to that of TD expression. Therefore, we skip the details and give the final expression.