$$
\begin{array}{l} = \left| r (x) - \tilde {r} (\phi (x)) + \gamma \sum_ {x ^ {\prime} \in \mathcal {X}} P \left(x ^ {\prime} \mid x\right) q ^ {\pi_ {\mathrm {e}}} \left(x ^ {\prime}\right) - \gamma \sum_ {\phi \left(x ^ {\prime}\right) \in \tilde {\mathcal {X}}} \tilde {P} \left(\phi \left(x ^ {\prime}\right) \mid \phi (x)\right) \tilde {q} ^ {\pi_ {\mathrm {e}}} \left(\phi \left(x ^ {\prime}\right)\right) \right| \\ \stackrel {(a)} {=} \left| r (x) - \frac {1}{| \phi (x) |} \sum_ {y \in \phi (x)} r (y) + \gamma \sum_ {x ^ {\prime} \in \mathcal {X}} P (x ^ {\prime} | x) q ^ {\pi_ {\mathrm {e}}} (x ^ {\prime}) - \gamma \frac {1}{| \phi (x) |} \sum_ {\phi (x ^ {\prime}) \in \tilde {\mathcal {X}}} \sum_ {y \in \phi (x)} \mathbb {P} (\phi (x ^ {\prime}) | y) \tilde {q} ^ {\pi_ {\mathrm {e}}} (\phi (x ^ {\prime})) \right| \\ \stackrel {(b)} {=} \frac {1}{| \phi (x) |} \left| | \phi (x) | r (x) - \sum_ {y \in \phi (x)} r (y) + \gamma | \phi (x) | \sum_ {x ^ {\prime} \in \mathcal {X}} P \left(x ^ {\prime} | x\right) q ^ {\pi_ {\mathrm {e}}} \left(x ^ {\prime}\right) - \gamma \sum_ {\phi \left(x ^ {\prime}\right) \in \tilde {\mathcal {X}}} \sum_ {y \in \phi (x)} \mathbb {P} \left(\phi \left(x ^ {\prime}\right) | y\right) \tilde {q} ^ {\pi_ {\mathrm {e}}} \left(\phi \left(x ^ {\prime}\right)\right) \right| \\ \stackrel {(c)} {=} \frac {1}{| \phi (x) |} \left| \sum_ {y \in \phi (x)} (r (x) - r (y)) + \sum_ {y \in \phi (x)} \left(\gamma \sum_ {x ^ {\prime} \in \mathcal {X}} P (x ^ {\prime} | x) q ^ {\pi_ {\mathrm {e}}} (x ^ {\prime}) - \gamma \sum_ {\phi (x ^ {\prime}) \in \tilde {\mathcal {X}}} \mathbb {P} (\phi (x ^ {\prime}) | y) \tilde {q} ^ {\pi_ {\mathrm {e}}} (\phi (x ^ {\prime}))\right) \right| \\ \stackrel {(d. 1)} {\leq} \frac {1}{| \phi (x) |} \sum_ {y \in \phi (x)} \left(| r (x) - r (y) | + \gamma \left| \sum_ {x ^ {\prime} \in \mathcal {X}} P (x ^ {\prime} | x) q ^ {\pi_ {\mathrm {e}}} (x ^ {\prime}) - \sum_ {\phi (x ^ {\prime}) \in \tilde {\mathcal {X}}} \mathbb {P} (\phi (x ^ {\prime}) | y) \tilde {q} ^ {\pi_ {\mathrm {e}}} (\phi (x ^ {\prime})) \right|\right) \\ \stackrel {(d. 2)} {=} \frac {1}{| \phi (x) |} \sum_ {y \in \phi (x)} \left(| r (x) - r (y) | + \gamma \left| \sum_ {x ^ {\prime} \in \mathcal {X}} P \left(x ^ {\prime} \mid x\right) q ^ {\pi_ {\mathrm {e}}} \left(x ^ {\prime}\right) - \sum_ {\phi \left(x ^ {\prime}\right) \in \tilde {\mathcal {X}}} \sum_ {z \in \phi \left(x ^ {\prime}\right)} P \left(z \mid y\right) \tilde {q} ^ {\pi_ {\mathrm {e}}} \left(\phi \left(x ^ {\prime}\right)\right) \right|\right) \\ \stackrel {(d. 3)} {=} \frac {1}{| \phi (x) |} \sum_ {y \in \phi (x)} \left(| r (x) - r (y) | + \gamma \left| \sum_ {x ^ {\prime} \in \mathcal {X}} P (x ^ {\prime} | x) q ^ {\pi_ {\mathrm {e}}} (x ^ {\prime}) - \sum_ {x ^ {\prime} \in \mathcal {X}} P (x ^ {\prime} | y) \tilde {q} ^ {\pi_ {\mathrm {e}}} (\phi (x ^ {\prime})) \right|\right) \\ \stackrel {(e)} {\leq} \frac {1}{| \phi (x) |} \sum_ {y \in \phi (x)} \left(| r (x) - r (y) | + \gamma \left| \sum_ {x ^ {\prime} \in \mathcal {X}} \left(P \left(x ^ {\prime} | x\right) q ^ {\pi_ {\mathrm {e}}} \left(x ^ {\prime}\right) - P \left(x ^ {\prime} | y\right) \tilde {q} ^ {\pi_ {\mathrm {e}}} (\phi \left(x ^ {\prime}\right))\right) \right|\right) \\ \stackrel {(f)} {\leq} \frac {1}{| \phi (x) |} \sum_ {y \in \phi (x)} \left(| r (x) - r (y) | + \gamma \left| \sum_ {x ^ {\prime} \in \mathcal {X}} \left(P \left(x ^ {\prime} | x\right) q ^ {\pi_ {\mathrm {e}}} \left(x ^ {\prime}\right) - P \left(x ^ {\prime} | y\right) q ^ {\pi_ {\mathrm {e}}} \left(x ^ {\prime}\right)\right) \right|\right) \\ + \frac {\gamma}{| \phi (x) |} \sum_ {y \in \phi (x)} \left(\left| \sum_ {x ^ {\prime} \in \mathcal {X}} P \left(x ^ {\prime} \mid y\right) \left(q ^ {\pi_ {\mathrm {e}}} \left(x ^ {\prime}\right) - \tilde {q} ^ {\pi_ {\mathrm {e}}} \left(\phi \left(x ^ {\prime}\right)\right)\right) \right|\right) \\ \stackrel {(g)} {\leq} \frac {1}{| \phi (x) |} \sum_ {y \in \phi (x)} \left(| r (x) - r (y) | + \gamma \left| \sum_ {x ^ {\prime} \in \mathcal {X}} \left(P (x ^ {\prime} | x) - P (x ^ {\prime} | y)\right) q ^ {\pi_ {\mathrm {e}}} (x ^ {\prime}) \right| + \gamma \| q - \widetilde {q} \| _ {\infty}\right) \\ \stackrel {(h)} {=} \frac {1}{| \phi (x) |} \sum_ {y \in \phi (x)} \left(| r (x) - r (y) | + \gamma \left| \mathbb {E} _ {x ^ {\prime} \sim P (\cdot | x)} \left[ q ^ {\pi_ {\mathrm {e}}} \left(x ^ {\prime}\right) \right] - \mathbb {E} _ {x ^ {\prime} \sim P (\cdot | y)} \left[ q ^ {\pi_ {\mathrm {e}}} \left(x ^ {\prime}\right) \right] \right| + \gamma \| q - \widetilde {q} \| _ {\infty}\right) \\ \end{array}
$$