SlowGuess/NeurIPS_2023_OCR huggingface dataset:
$$
\| \boldsymbol {x} \| _ {2, i} ^ {2} - \| \boldsymbol {y} \| _ {2} ^ {2} = \frac {1}{p} \sum_ {j = 1} ^ {n} \delta_ {i j} (x _ {j} ^ {2} - y _ {j} ^ {2}) + \sum_ {j = 1} ^ {n} \left(\frac {\delta_ {i j}}{p} - 1\right) y _ {j} ^ {2}.
$$
The first term is bounded as
$$
\left| \frac {1}{p} \sum_ {j = 1} ^ {n} \delta_ {i j} \left(x _ {j} ^ {2} - y _ {j} ^ {2}\right) \right| \leq \| \boldsymbol {x} - \boldsymbol {y} \| _ {\infty} \| \boldsymbol {x} + \boldsymbol {y} \| _ {\infty} \frac {1}{p} \sum_ {j = 1} ^ {n} \delta_ {i j} \lesssim n \| \boldsymbol {x} - \boldsymbol {y} \| _ {\infty} \| \boldsymbol {x} + \boldsymbol {y} \| _ {\infty},
$$
and the second term is bounded as
$$
\left| \sum_ {j = 1} ^ {n} \left(\frac {\delta_ {i j}}{p} - 1\right) y _ {j} ^ {2} \right| \lesssim \sqrt {\frac {\log n}{p}} \| \boldsymbol {y} \| _ {2} \| \boldsymbol {y} \| _ {\infty} + \frac {\log n}{p} \| \boldsymbol {y} \| _ {\infty} ^ {2}
$$
by Lemma G.6.
Lemma G.9. Let  $\mathbf{y}$  be a vector that is independent from the sampling. Then, if  $n^2 p \gtrsim n \log n$ , we have
$$
\left\| \frac {1}{p} \mathcal {P} _ {\Omega} \left(\boldsymbol {x x} ^ {\top}\right) - \boldsymbol {y y} ^ {\top} \right\| \lesssim n \| \boldsymbol {x} - \boldsymbol {y} \| _ {\infty} \left(\| \boldsymbol {x} \| _ {\infty} + \| \boldsymbol {y} \| _ {\infty}\right) + \sqrt {\frac {n \log n}{p}} \| \boldsymbol {y} \| _ {\infty} ^ {2}
$$
Proof. We have the following sequence of inequalities
$$
\begin{array}{l} \left\| \frac {1}{p} \mathcal {P} _ {\Omega} (\boldsymbol {x x} ^ {\top}) - \boldsymbol {y y} ^ {\top} \right\| \leq \left\| \frac {1}{p} \mathcal {P} _ {\Omega} (\boldsymbol {x x} ^ {\top}) - \frac {1}{p} \mathcal {P} _ {\Omega} (\boldsymbol {y y} ^ {\top}) \right\| + \left\| \frac {1}{p} \mathcal {P} _ {\Omega} (\boldsymbol {y y} ^ {\top}) - \boldsymbol {y y} ^ {\top} \right\| \\ \leq \left\| \boldsymbol {x} \boldsymbol {x} ^ {\top} - \boldsymbol {y} \boldsymbol {y} ^ {\top} \right\| _ {\infty} \left\| \frac {1}{p} \mathcal {P} _ {\Omega} (\mathbf {1 1} ^ {\top}) \right\| + \left\| \frac {1}{p} \mathcal {P} _ {\Omega} (\boldsymbol {y} \boldsymbol {y} ^ {\top}) - \boldsymbol {y} \boldsymbol {y} ^ {\top} \right\| \\ \lesssim \| \boldsymbol {x} - \boldsymbol {y} \| _ {\infty} (\| \boldsymbol {x} \| _ {\infty} + \| \boldsymbol {y} \| _ {\infty}) \left\| \frac {1}{p} \mathcal {P} _ {\Omega} \left(\mathbf {1 1} ^ {\top}\right) \right\| + \left\| \frac {1}{p} \mathcal {P} _ {\Omega} \left(\boldsymbol {y y} ^ {\top}\right) - \boldsymbol {y y} ^ {\top} \right\| \\ \lesssim n \| \boldsymbol {x} - \boldsymbol {y} \| _ {\infty} (\| \boldsymbol {x} \| _ {\infty} + \| \boldsymbol {y} \| _ {\infty}) + \sqrt {\frac {n \log n}{p}} \| \boldsymbol {y} \| _ {\infty} ^ {2}, \\ \end{array}
$$
where the second line is derived from a basic inequality  $\| \mathbf{A} \| \leq \| |\mathbf{A}| \|$  that holds for any matrix  $\mathbf{A}$ , and the last line follows by applying Lemma G.3 to  $\mathbf{11}^{\top}$  and  $\mathbf{yy}^{\top}$ .
Lemma G.10. Let  $\mathbf{y}$  be a vector that is independent from the sampling. Then, if  $n^2 p \gtrsim n \log n$ , we have
$$
\begin{array}{l} \left\| \nabla^ {2} g (\boldsymbol {x}) - \left(\| \boldsymbol {y} \| _ {2} ^ {2} \boldsymbol {I} + 2 \boldsymbol {y} \boldsymbol {y} ^ {\top}\right) \right\| \lesssim n \| \boldsymbol {x} - \boldsymbol {y} \| _ {\infty} \left(\| \boldsymbol {x} \| _ {\infty} + \| \boldsymbol {y} \| _ {\infty}\right) \\ + \sqrt {\frac {\log n}{p}} \| \boldsymbol {y} \| _ {2} \| \boldsymbol {y} \| _ {\infty} + \frac {\log n}{p} \| \boldsymbol {y} \| _ {\infty} ^ {2} + \sqrt {\frac {n \log n}{p}} \| \boldsymbol {y} \| _ {\infty} ^ {2} \\ \end{array}
$$
Proof. This follows directly from Lemmas G.8 and G.9.
Let us define an operator  $\mathcal{P}_{\Omega_l}$  such that an entry of  $\mathcal{P}_{\Omega_l}(\boldsymbol{X})$  is equal to that of  $\boldsymbol{X}$  if it is contained both in the  $l$ th row/column and  $\Omega$ , and otherwise 0. We also define an operator  $\mathcal{P}_l$  that makes the entries outside the  $l$ th row/column zero. Then, we have
$$
\frac {1}{p} \mathcal {P} _ {\Omega} (\boldsymbol {X}) - \mathcal {P} _ {\Omega} ^ {(l)} (\boldsymbol {X}) = \frac {1}{p} \mathcal {P} _ {\Omega_ {l}} (\boldsymbol {X}) - \mathcal {P} _ {l} (\boldsymbol {X}).
$$
$$
\frac {1}{p} \mathcal {P} _ {\Omega} (\boldsymbol {E}) - \boldsymbol {E} ^ {(l)} = \frac {1}{p} \mathcal {P} _ {\Omega_ {l}} (E).
$$
The following lemma was also introduced in [14], but we include the proof for completeness.
Lemma G.11. Suppose that a matrix  $M$  and a vector  $\mathbf{v}$  are independent from sampling of the lth row/column. If  $n^2 p \gtrsim n \log n$ , we have
$$
\left\| \left(\frac {1}{p} \mathcal {P} _ {\Omega_ {l}} (\boldsymbol {M}) - \mathcal {P} _ {l} (\boldsymbol {M})\right) \boldsymbol {v} \right\| _ {2} \lesssim \| \boldsymbol {M} \| _ {\infty} \left(\sqrt {\frac {\log n}{p}} \| \boldsymbol {v} \| _ {2} + \frac {\log n}{p} \| \boldsymbol {v} \| _ {\infty} + \sqrt {\frac {n}{p}} \| \boldsymbol {v} \| _ {\infty}\right)
$$
Proof. If we consider the contribution of  $l$ th term and the other terms separately, we have
$$
\begin{array}{l} \left\| \left(\frac {1}{p} \mathcal {P} _ {\Omega_ {l}} (\boldsymbol {M}) - \mathcal {P} _ {l} (\boldsymbol {M})\right) \boldsymbol {v} \right\| _ {2} \leq \left| \sum_ {j = 1} ^ {n} \left(\frac {\delta_ {l j}}{p} - 1\right) M _ {l j} v _ {j} \right| + | v _ {l} | \sqrt {\sum_ {i = 1} ^ {n} \left(\frac {\delta_ {i l}}{p} - 1\right) ^ {2} M _ {i l} ^ {2}} \\ \leq \| M \| _ {\infty} \left(\left| \sum_ {j = 1} ^ {n} \left(\frac {\delta_ {l j}}{p} - 1\right) v _ {j} \right| + \| \boldsymbol {v} \| _ {\infty} \sqrt {\sum_ {i = 1} ^ {n} \left(\frac {\delta_ {i l}}{p} - 1\right) ^ {2}}\right) \\ \end{array}
$$
From Lemma G.6, we have
$$
\left| \sum_ {j = 1} ^ {n} \left(\frac {\delta_ {l j}}{p} - 1\right) v _ {j} \right| \lesssim \sqrt {\frac {\log n}{p}} \| \boldsymbol {v} \| _ {2} + \frac {\log n}{p} \| \boldsymbol {v} \| _ {\infty}
$$
with high probability. Regarding the second term, notice that
$$
\sum_ {i = 1} ^ {n} \left(\frac {\delta_ {i l}}{p} - 1\right) ^ {2} = n + \left(\frac {1}{p} - 2\right) \sum_ {i = 1} ^ {n} \frac {\delta_ {i l}}{p}.
$$
Lemma G.6 implies that  $\sum_{i=1}^{n} \frac{\delta_{il}}{p} \asymp n$  with high probability if  $n^2 p \gtrsim n \log n$ . Hence, we have
$$
\sum_ {i = 1} ^ {n} \left(\frac {\delta_ {i l}}{p} - 1\right) ^ {2} \lesssim \frac {n}{p},
$$
Lemma G.12. Let  $M$  be a matrix and  $\mathbf{v}$ ,  $\mathbf{w}$  be vectors that are independent from sampling of the lth row/column. Then, if  $n^2 p \gtrsim n \log n$ , we have
$$
\begin{array}{l} \left| \boldsymbol {w} ^ {\top} \left(\frac {1}{p} \mathcal {P} _ {\Omega_ {l}} (\boldsymbol {M}) - \mathcal {P} _ {l} (\boldsymbol {M})\right) \boldsymbol {v} \right| \\ \lesssim \| \boldsymbol {M} \| _ {\infty} \left(\sqrt {\frac {\log n}{p}} \left(\| \boldsymbol {v} \| _ {2} \| \boldsymbol {w} \| _ {\infty} + \| \boldsymbol {w} \| _ {2} \| \boldsymbol {v} \| _ {\infty}\right) + \frac {\log n}{p} \| \boldsymbol {v} \| _ {\infty} \| \boldsymbol {w} \| _ {\infty}\right) \\ \end{array}
$$
Proof. We can consider the  $l$ th row and column separately by
$$
\begin{array}{l} \left| \boldsymbol {w} ^ {\top} \left(\frac {1}{p} \mathcal {P} _ {\Omega_ {l}} (\boldsymbol {M}) - \mathcal {P} _ {l} (\boldsymbol {M})\right) \boldsymbol {v} \right| \\ \leq \left| v _ {l} \sum_ {i} \left(\frac {\delta_ {i l}}{p} - 1\right) M _ {i l} w _ {i} \right| + \left| w _ {l} \sum_ {j} \left(\frac {\delta_ {l j}}{p} - 1\right) M _ {l j} v _ {j} \right| + \left| \left(\frac {\delta_ {l l}}{p} - 1\right) M _ {l l} v _ {l} w _ {l} \right| \\ \leq \| M \| _ {\infty} \left(\| \boldsymbol {v} \| _ {\infty} \left| \sum_ {i} \left(\frac {\delta_ {i l}}{p} - 1\right) w _ {i} \right| + \| \boldsymbol {w} \| _ {\infty} \left| \sum_ {j} \left(\frac {\delta_ {l j}}{p} - 1\right) v _ {j} \right| + \frac {1}{p} \| \boldsymbol {v} \| _ {\infty} \| \boldsymbol {w} \| _ {\infty}\right) \\ \end{array}
$$
If we apply Lemma G.6 to the summations, we get the desired result.
Lemma G.13. Let  $\pmb{E}$  be a symmetric matrix whose upper and on diagonal entries are drawn from Gaussian distribution  $\mathcal{N}(0, \sigma^2)$  independently. Let  $\pmb{v}$  be a vector that is independent from sampling of the lth row and column. Then, if  $n^2 p \gtrsim n \log^2 n$ , we have
$$
\left\| \frac {1}{p} \mathcal {P} _ {\Omega_ {l}} (\boldsymbol {E}) \boldsymbol {v} \right\| _ {2} \lesssim \sigma \left(\sqrt {\frac {\log n}{p}} \| \boldsymbol {v} \| _ {2} + \frac {\sqrt {\log^ {3} n}}{p} \| \boldsymbol {v} \| _ {\infty} + \sqrt {\frac {n}{p}} \| \boldsymbol {v} \| _ {\infty}\right)
$$
$$
\left\| \frac {1}{p} \mathcal {P} _ {\Omega_ {l}} (\boldsymbol {E}) \boldsymbol {v} \right\| _ {2} \leq \frac {1}{p} \left| \sum_ {j = 1} ^ {n} \delta_ {l j} E _ {l j} v _ {j} \right| + \frac {1}{p} | v _ {l} | \sqrt {\sum_ {i = 1} ^ {n} \delta_ {i l} E _ {i l} ^ {2}}
$$
For the first term, we will calculate  $V$  and  $L$  of Corollary G.5.  $V$  is calculated as
$$
V = \sum_ {j = 1} ^ {n} \mathbb {E} \big [ (\delta_ {l j} E _ {l j} v _ {j}) ^ {2} \big ] = p \sigma^ {2} \| \boldsymbol {v} \| _ {2} ^ {2}.
$$
To find  $L$ , we first note that  $\|E_{l*}\|_{\infty} \lesssim \sigma \sqrt{\log n}$  with high probability, where  $E_{l*}$  is the  $l$ th row of  $\pmb{E}$ . Thus, for all  $j \in [n]$ , we have
$$
\left| \delta_ {l j} E _ {l j} v _ {j} \right| \lesssim \sigma \sqrt {\log n} \| \boldsymbol {v} \| _ {\infty}.
$$
Corollary G.5 implies that the first term is bounded as
$$
\frac {1}{p} \left| \sum_ {j = 1} ^ {n} \delta_ {l j} E _ {l j} v _ {j} \right| \lesssim \sigma \left(\sqrt {\frac {\log n}{p}} \| \boldsymbol {v} \| _ {2} + \frac {\sqrt {\log^ {3} n}}{p} \| \boldsymbol {v} \| _ {\infty}\right). \tag {G.1}
$$
For the second term, it suffices to bound
$$
\left| \sum_ {i = 1} ^ {n} \delta_ {i l} (E _ {i l} ^ {2} - \sigma^ {2}) \right|.
$$
As before, we obtain  $V$  and  $L$  through
$$
\begin{array}{l} \sum_ {i = 1} ^ {n} \mathbb {E} \left[ \left(\delta_ {i l} \left(E _ {i l} ^ {2} - \sigma^ {2}\right)\right) ^ {2} \right] = p \sum_ {i = 1} ^ {n} \mathbb {E} \left[ E _ {i l} ^ {4} - 2 \sigma^ {2} E _ {i l} ^ {2} + \sigma^ {4} \right] = 2 \sigma^ {4} n p, \\ \left| \delta_ {i l} (E _ {i l} ^ {2} - \sigma^ {2}) \right| \lesssim \sigma^ {2} \log n. \\ \end{array}
$$
Corollary G.5 implies that
$$
\left| \sum_ {i = 1} ^ {n} \delta_ {i l} \left(E _ {i l} ^ {2} - \sigma^ {2}\right) \right| \lesssim \sigma^ {2} \left(\sqrt {n p \log n} + \log^ {2} n\right).
$$
Because  $\sum_{i=1}^{n} \delta_{il} \asymp np$ , we have
$$
\sum_ {i = 1} ^ {n} \delta_ {i l} E _ {i l} ^ {2} \lesssim \sigma^ {2} (n p + \sqrt {n p \log n} + \log^ {2} n) \lesssim \sigma^ {2} n p \tag {G.2}
$$
if  $n^2 p \gtrsim n\log^2 n$ . Combining (G.1) and (G.2), we get the desired bound.
Lemma G.14. Let  $\pmb{E}$  be a symmetric matrix whose upper and on diagonal entries are drawn from Gaussian distribution  $\mathcal{N}(0, \sigma^2)$  independently. Let  $\pmb{v}, \pmb{w}$  be vectors that are independent from sampling of the lth row and column. Then, if  $n^2 p \gtrsim n \log n$ , we have
$$
\frac {1}{p} \big | \pmb {w} ^ {\top} \mathcal {P} _ {\Omega_ {l}} (\pmb {E}) \pmb {v} \big | \lesssim \sigma \left(\sqrt {\frac {\log n}{p}} (\| \pmb {v} \| _ {2} \| \pmb {w} \| _ {\infty} + \| \pmb {w} \| _ {2} \| \pmb {v} \| _ {\infty}) + \frac {\sqrt {\log^ {3} n}}{p} \| \pmb {v} \| _ {\infty} \| \pmb {w} \| _ {\infty}\right).
$$
$$
\begin{array}{l} \frac {1}{p} \left| \boldsymbol {w} ^ {\top} \mathcal {P} _ {\Omega_ {l}} (\boldsymbol {E}) \boldsymbol {v} \right| \leq \frac {1}{p} \left| v _ {l} \sum_ {i} \delta_ {i l} E _ {i l} w _ {i} \right| + \frac {1}{p} \left| w _ {l} \sum_ {j} \delta_ {l j} E _ {l j} v _ {j} \right| + \frac {1}{p} | \delta_ {l l} E _ {l l} v _ {l} w _ {l} | \\ \leq \frac {\| \boldsymbol {v} \| _ {\infty}}{p} \left| \sum_ {i} \delta_ {i l} E _ {i l} w _ {i} \right| + \frac {\| \boldsymbol {w} \| _ {\infty}}{p} \left| \sum_ {j} \delta_ {l j} E _ {l j} v _ {j} \right| + \frac {1}{p} \| \boldsymbol {v} \| _ {\infty} \| \boldsymbol {w} \| _ {\infty} | E _ {l l} |. \\ \end{array}
$$
We bound the two summations similar to (G.1) and for the last term, we note that  $|E_{ll}| \lesssim \sigma \sqrt{\log n}$  with high probability.