SlowGuess/NeurIPS_2023_OCR huggingface dataset:
$$
\delta_ {\mathbf {s}} ^ {\prime} (\kappa) = 2 \mu_ {*} (\mathcal {P}) - 2 \tilde {\mu} _ {\mathbf {s}} \left(\mathcal {C} _ {\mathbf {s}} \left(\sqrt {1 + \kappa} \eta_ {\mathbf {s}} ^ {*}, \mathcal {P}\right)\right) > 0. \tag {18}
$$
From Lemma 12,  $\mathcal{C}_{\mathbf{s}}$  is a continuous correspondence. Since  $D(P_{\mathbf{s}}\parallel P_{\mathbf{s}}^{\prime}) + \epsilon /2\sum_{\mathbf{x}\in \mathcal{I}_{-s}}D(P_{\mathbf{x}}\parallel P_{\mathbf{x}}^{\prime})$  is a continuous function of  $\mathcal{P}^{\prime} = \langle P_{\mathbf{s}}^{\prime}\rangle_{\mathbf{s}\in \mathcal{I}}\in \mathcal{M}_{\mathcal{G}}$ , it follows from Berge's maximum theorem in Lemma 11 that
$$
\max _ {\mathcal {P} ^ {\prime} \in \mathcal {C} _ {\mathbf {s}} ((1 + \kappa) \eta_ {\mathbf {s}}, \bar {\mathcal {P}})} D (\bar {P} _ {\mathbf {s}} \| P _ {\mathbf {s}} ^ {\prime}) + \epsilon / 2 \sum_ {\mathbf {x} \in \mathcal {I} _ {- \mathbf {s}}} D (\bar {P} _ {\mathbf {x}} \| P _ {\mathbf {x}} ^ {\prime})
$$
is a continuous function of  $\bar{\mathcal{P}} = \langle \bar{P}_{\mathbf{s}}\rangle_{\mathbf{s}\in \mathcal{I}}$ . Thus, there exists  $\delta_{\mathbf{s}}^{\prime \prime}(\kappa) > 0$  such that if  $\forall \mathbf{s}\in \mathcal{I}$ :  $\left\| \bar{P}_{\mathbf{s}} - P_{\mathbf{s}}\right\| _1\leq \delta_{\mathbf{s}}^{\prime \prime}(\kappa)$ ,
$$
\begin{array}{l} \max _ {\mathcal {P} ^ {\prime} \in \mathcal {C} _ {\mathbf {s}} ((1 + \kappa) \eta_ {\mathbf {s}} ^ {*}, \bar {\mathcal {P}})} D (P _ {\mathbf {s}} \| P _ {\mathbf {s}} ^ {\prime}) + \epsilon / 2 \sum_ {\mathbf {x} \in \mathcal {I} _ {- \mathbf {s}}} D (P _ {\mathbf {x}} \| P _ {\mathbf {x}} ^ {\prime}) \\ \leq \sqrt {1 + \kappa} \max _ {\mathcal {P} ^ {\prime} \in \mathcal {C} _ {\mathbf {s}} ((1 + \kappa) \eta_ {\mathbf {s}} ^ {*}, \mathcal {P})} D (P _ {\mathbf {s}} \| P _ {\mathbf {s}} ^ {\prime}) + \epsilon / 2 \sum_ {\mathbf {x} \in \mathcal {I} _ {- \mathbf {s}}} D (P _ {\mathbf {x}} \| P _ {\mathbf {x}} ^ {\prime}) \leq \frac {1}{\eta_ {\mathbf {s}} ^ {*} \sqrt {1 + \kappa}}, \\ \end{array}
$$
where the second inequality is due to the definition of  $\mathcal{C}_{\mathbf{s}}((1 + \kappa)\eta_{\mathbf{s}},\mathcal{P})$ . The inequality means for any  $\mathcal{P}' \in \mathcal{C}_{\mathbf{s}}((1 + \kappa)\eta_{\mathbf{s}}^{*},\mathcal{P})$
$$
(\eta_ {\mathbf {s}} ^ {*} \sqrt {1 + \kappa}) D (P _ {\mathbf {s}} \| P _ {\mathbf {s}} ^ {\prime}) + (\eta_ {\mathbf {s}} ^ {*} \sqrt {1 + \kappa}) \epsilon / 2 \sum_ {\mathbf {x} \in \mathcal {I} _ {- \mathbf {s}}} D (P _ {\mathbf {x}} \| P _ {\mathbf {x}} ^ {\prime}) \leq 1,
$$
which means  $\mathcal{C}_{\mathbf{s}}((1 + \kappa)\eta_{\mathbf{s}}^{*},\bar{\mathcal{P}})\subseteq \mathcal{C}_{\mathbf{s}}(\sqrt{1 + \kappa}\eta_{\mathbf{s}}^{*},\mathcal{P})$  . Therefore,
$$
\tilde {\mu} _ {\mathbf {s}} \left(\mathcal {C} _ {\mathbf {s}} \left(\left(1 + \kappa\right) \eta_ {\mathbf {s}} ^ {*}, \bar {\mathcal {P}}\right)\right) \leq \tilde {\mu} _ {\mathbf {s}} \left(\mathcal {C} _ {\mathbf {s}} \left(\sqrt {1 + \kappa} \eta_ {\mathbf {s}} ^ {*}, \mathcal {P}\right)\right). \tag {19}
$$
Let  $\hat{\delta}_{\mathbf{s}}(\kappa) = \min \{\delta_{\mathbf{s}}^{\prime}(\kappa),\delta_{\mathbf{s}}^{\prime \prime}(\kappa)\}$ , and let  $\hat{\delta} (\kappa) = \min_{\mathbf{s}\in \mathcal{I}_{-s_*}}\hat{\delta}_{\mathbf{s}}(\kappa)$ . Then we set  $\delta (\kappa) = \min \{\Delta_{\min},\hat{\delta} (\kappa)\}$ , where  $\Delta_{\min} = \min_{\mathbf{s}\in \mathcal{I}_{-s_*}}\mu_*(\mathcal{P}) - \mu_{\mathbf{s}}(\mathcal{P})$ . Since the reward is bounded by [0, 1], for any  $\mathbf{s}\in \mathcal{I}$ ,  $|\mu_{\mathbf{s}}(\bar{\mathcal{P}}) - \mu_{\mathbf{s}}(\mathcal{P})|\leq \frac{1}{2}\left\| \bar{P}_{\mathbf{s}} - P_{\mathbf{s}}\right\| _1$ . If  $\forall \mathbf{s}\in \mathcal{I}:||\bar{P}_{\mathbf{s}} - P_{\mathbf{s}}||_1\leq \delta (\kappa)$ ,
$$
\mu_ {\mathbf {s}} (\bar {\mathcal {P}}) - \mu_ {\mathbf {s} _ {*}} (\bar {\mathcal {P}}) \leq \mu_ {\mathbf {s}} (\mathcal {P}) - \mu_ {\mathbf {s} _ {*}} (\mathcal {P}) + \delta (\kappa) \leq \mu_ {\mathbf {s}} (\mathcal {P}) - \mu_ {\mathbf {s} _ {*}} (\mathcal {P}) + \Delta_ {\min } \leq 0,
$$
which means intervention  $\mathbf{s}_*$  is also optimal according to  $\bar{\mathcal{P}}$ . Besides, with (18) and (19), we have
$$
\begin{array}{l} \tilde {\mu} _ {\mathbf {s}} (\mathcal {C} _ {\mathbf {s}} ((1 + \kappa) \eta_ {\mathbf {s}} ^ {*}, \bar {\mathcal {P}})) \leq \tilde {\mu} _ {\mathbf {s}} (\mathcal {C} _ {\mathbf {s}} (\sqrt {1 + \kappa} \eta_ {\mathbf {s}} ^ {*}, \mathcal {P})) = \mu_ {*} (\mathcal {P}) - \frac {\delta_ {\mathbf {s}} ^ {\prime} (\kappa)}{2} \\ \leq \mu_ {*} (\mathcal {P}) - \frac {\delta (\kappa)}{2} \leq \mu_ {\mathbf {s} _ {*}} (\mathcal {P}) - \frac {1}{2} \left\| \bar {P} _ {\mathbf {s} _ {*}} + P _ {\mathbf {s} _ {*}} \right\| _ {1} \leq \mu_ {\mathbf {s} _ {*}} (\bar {\mathcal {P}}) = \mu_ {*} (\bar {\mathcal {P}}). \\ \end{array}
$$
Since  $\tilde{\mu}_{\mathbf{s}}(\mathcal{C}_{\mathbf{s}}(\eta_{\mathbf{s}},\bar{\mathcal{P}}))$  is continuous monotonically nonincreasing with  $\eta_{\mathbf{s}}$  and  $\tilde{\mu}_{\mathbf{s}}(\mathcal{C}_{\mathbf{s}}(\zeta_{\mathbf{s}}(\bar{\mathcal{P}}),\bar{\mathcal{P}}))\geq$ $\mu_{*}(\bar{\mathcal{P}})\geq \tilde{\mu}_{\mathbf{s}}(\mathcal{C}_{\mathbf{s}}((1 + \kappa)\eta_{\mathbf{s}}^{*},\bar{\mathcal{P}})),$  we have
$$
\zeta_ {\mathbf {s}} (\bar {\mathcal {P}}) \leq (1 + \kappa) \eta_ {\mathbf {s}} ^ {*} = (1 + \kappa) \zeta_ {\mathbf {s}} (\mathcal {P}),
$$
B.4 Main Proof of Theorem 2
The expected regret is defined as  $R_{T}^{\mathrm{SCM - AAM}} = \sum_{\mathbf{s}\in \mathcal{I}_{-s_{*}}}\mathbb{E}[N_{T}(\mathbf{s})]\Delta_{\mathbf{s}}$ , in which  $N_{T}(\mathbf{s})$  is the total number of times  $\mathbf{s}$  is selected by the SCM-AAM algorithm till  $T$ . So the proof is articulated around bounding  $\mathbb{E}[N_T(\mathbf{s})]$  for each  $\mathbf{s}\in \mathcal{I}_{-s_*}$ .
Proof. At each time  $t$ , consider the event that the true interventional distribution tuple  $\mathcal{P} = \langle P_{\mathbf{s}} \rangle_{\mathbf{s} \in \mathcal{I}}$  is not contained by the confidence set  $\mathcal{C}_t = \left\{\mathcal{P}' \in \mathcal{M}_{\mathcal{G}} \mid \sum_{\mathbf{s} \in \mathcal{I}} N_t(\mathbf{s}) D(\bar{P}_{\mathbf{s},t} \| P_{\mathbf{s}}') \leq (1 + \gamma) \ln t\right\}$ . Due to Corollary 8, there exists  $c_1 > 0$  irrelevant with  $T$
$$
\sum_ {t = 1} ^ {T} \mathbb {P} (\mathcal {P} \notin \mathcal {C} _ {t}) \leq \sum_ {t = 1} ^ {T} \mathbb {P} \left(\sum_ {\mathbf {s} \in \mathcal {I}} N _ {t} (\mathbf {s}) D \left(\bar {P} _ {\mathbf {s}, t} \| P _ {\mathbf {s}}\right) \geq (1 + \gamma) \ln t\right) \leq c _ {1}, \tag {20}
$$
which will only result in finite regret bounded  $c_{1}$  since reward is in [0, 1]. So by only considering time steps with  $\{\mathcal{P} \in \mathcal{C}_t\}$  to be true, we upper bound the number of times a suboptimal arm  $\mathbf{s} \in \mathcal{I}_{-\mathbf{s}_*}$  is selected in the exploration and exploitation phase separately.
Exploitation phase: When  $\{\mathcal{P}\in \mathcal{C}_t\}$  is true, we have
$$
\forall \mathbf {s} \in \mathcal {I}: \tilde {\mu} _ {\mathbf {s}} \left(\mathcal {C} _ {t}\right) = \sum_ {\mathbf {v} \in \Omega (\mathbf {V})} v _ {n} \max  _ {\mathcal {P} ^ {\prime} \in \mathcal {C} _ {t}} P _ {\mathbf {s}} ^ {\prime} (\mathbf {v}) \geq \sum_ {\mathbf {v} \in \Omega (\mathbf {V})} v _ {n} P _ {\mathbf {s}} (\mathbf {v}) = \mu_ {\mathbf {s}}. \tag {21}
$$
According to the design, SCM-AAM enters the exploitation phase at time  $t$  if  $\bar{\mu}_{\mathbf{S}_t,t} > \tilde{\mu}_{\mathbf{s}}(\mathcal{C}_t),\forall \mathbf{s}\in \mathcal{I}\setminus \{\mathbf{S}_t\}$ . It follows from (21), if  $\mathbf{S}_t\neq \mathbf{s}_*$  and  $\mathcal{P}\in \mathcal{C}_t$
$$
\bar {\mu} _ {\mathbf {S} _ {t}, t} > \tilde {\mu} _ {\mathbf {s} _ {*}} (\mathcal {C} _ {t}) \geq \mu_ {*}.
$$
Thus, we can bound the regret from exploitation steps as the following
$$
\sum_ {t = 1} ^ {T} \sum_ {\mathbf {s} \in \mathcal {I} \backslash \left\{\mathbf {s} _ {*} \right\}} \Delta_ {\mathbf {s}} \mathbb {1} \left\{\mathbf {S} _ {t} = \mathbf {s}, \mathcal {P} \in \mathcal {C} _ {t} \right\} \leq \sum_ {t = 1} ^ {T} \sum_ {\mathbf {s} \in \mathcal {I} / \left\{\mathbf {s} _ {*} \right\}} \Delta_ {\mathbf {s}} \mathbb {1} \left\{\mathbf {S} _ {t} = \mathbf {s}, \bar {\mu} _ {\mathbf {s}, t} > \mu_ {*}. \right\} \tag {22}
$$
Let  $\{t_{\mathbf{s},i}\}_{i\geq 1}$  be the sequence of rounds such that  $\mathbf{s}$  is selected by exploitation steps, namely  $\mathbf{S}_{t_{\mathbf{s},i}} = \mathbf{s}$ . Considering the warmup step that selects each intervention once, at each  $t_{\mathbf{s},i}$ , we have  $N_{t_{\mathbf{s},i}}(\mathbf{s})\geq i$ . Since each  $t_{\mathbf{s},i}$  is a stopping time, by applying Lemma 9, we bound the expectation of (22) as
$$
\begin{array}{l} \mathbb {E} \left[ \sum_ {t = 1} ^ {T} \sum_ {\mathbf {s} \in \mathcal {I} _ {- \mathbf {s} _ {*}}} \Delta_ {\mathbf {s}} \mathbb {1} \{\mathbf {S} _ {t} = \mathbf {s}, \mathcal {P} \in \mathcal {C} _ {t} \} \right] \leq \sum_ {\mathbf {s} \in \mathcal {I} _ {- \mathbf {s} _ {*}}} \Delta_ {\mathbf {s}} \sum_ {i \geq 1} P (\bar {\mu} _ {\mathbf {s}, t _ {\mathbf {s}, i}} - \mu_ {\mathbf {s}} > \Delta_ {\mathbf {s}}, t _ {\mathbf {s}, i} \leq T) \\ \leq \sum_ {\mathbf {s} \in \mathcal {I} _ {- \mathbf {s} _ {*}}} \Delta_ {\mathbf {s}} \sum_ {i \geq 1} \exp (- 2 i \Delta_ {\mathbf {s}} ^ {2}) \leq \sum_ {\mathbf {s} \in \mathcal {I} _ {- \mathbf {s} _ {*}}} \frac {1}{2 \Delta_ {\mathbf {s}}} := c _ {2}. \tag {23} \\ \end{array}
$$
Exploration phase with inaccurate  $\bar{\mathcal{P}}_t$ : We define event  $E_{t}$  that  $\bar{P}_t$  is estimated with sufficient accuracy,
$$
E _ {t} = \left\{\forall \mathbf {s} \in \mathcal {I}: \left\| \bar {P} _ {t, \mathbf {s}} - P _ {\mathbf {s}} \right\| _ {1} \leq \delta (\kappa) \right\}, \tag {24}
$$
where  $\delta(\kappa) > 0$  is defined in Lemma 14. Let  $\mathcal{R} = \{t_i\}_{i \geq 1}$  be the sequence of rounds that SCM-AAM enters the exploration phase, and each  $t_i$  is a stopping time. Then we have
$$
\sum_ {t = 1} ^ {T} \mathbb {1} \left\{t \in \mathcal {R}, E _ {t} ^ {\mathbb {C}} \right\} \leq \sum_ {i \geq 1} \mathbb {1} \left\{E _ {t _ {i}} ^ {\mathbb {C}}, t _ {i} \leq T \right\}.
$$
From Lemma 6, at  $t_i$ , we have  $\forall \mathbf{s} \in \mathcal{I}: N_t(\mathbf{s}) \geq \epsilon i / 2$ . With Corollary 10, we apply a union bound
$$
\begin{array}{l} \mathbb {E} \left[ \sum_ {t = 1} ^ {T} \mathbb {1} \{t \in \mathcal {R}, E _ {t} ^ {\mathbb {C}} \} \right] \leq \sum_ {\mathbf {s} \in \mathcal {I}} \sum_ {i \geq 1} P (E _ {t _ {i}} ^ {\mathbb {C}}, t _ {i} \leq T) \leq \sum_ {i \geq 1} P \Big (\left\| \bar {P} _ {t _ {i}, \mathbf {s}} - P _ {\mathbf {s}} \right\| _ {1} > \delta (\kappa), t _ {i} \leq T \Big) \\ \leq \sum_ {\mathbf {s} \in \mathcal {I}} \sum_ {i \geq 1} (2 ^ {| \Omega (\mathbf {V}) |} - 2) \exp \left(\frac {- \epsilon i \delta (\kappa) ^ {2}}{4}\right) = \frac {(2 ^ {| \Omega (\mathbf {V}) | + 2} - 8) | \mathcal {I} |}{\epsilon \delta (\kappa) ^ {2}} = c _ {3}. \tag {25} \\ \end{array}
$$
Greedy approximation exploration with accurate  $\bar{\mathcal{P}}_t$ : For each  $\mathbf{s} \in \mathcal{I}$ , let  $N_t^{\mathrm{e}}(\mathbf{s})$  be the total number of times  $\mathbf{s}$  is selected in exploration phases till  $t$ . Within time steps such that both  $E_t$  defined in (24) and  $\{\mathcal{P} \in \mathcal{C}_t\}$  are true, let the last time  $\mathbf{s} \in \mathcal{I}_{-\mathbf{s}_*}$  being selected by exploration with greedy approximation be
$$
\bar {t} _ {\mathbf {s}} = \max  \left\{t \in \mathcal {R} \mid \overline {{\mathbf {S}}} _ {t} = \mathbf {s}, E _ {t}, \mathcal {P} \in \mathcal {C} _ {t} \right\}
$$
is true. With Lemma 6, we have  $\forall \mathbf{x} \in \mathcal{I}_{-\mathbf{s}}: N_{\tilde{t}_{\mathbf{s}}}(\mathbf{x}) \geq \epsilon N_{\tilde{t}_{\mathbf{s}}}^{\mathrm{e}}(\mathbf{s}) / 2$ , so that
$$
\frac {\sum_ {\mathbf {x} \in \mathcal {I}} N _ {\bar {t} _ {\mathbf {x}}} (\mathbf {x}) D (\bar {P} _ {\mathbf {x} , \bar {t} _ {\mathbf {s}}} \| P _ {\mathbf {x}} ^ {\prime})}{(1 + \gamma) \ln \bar {t} _ {\mathbf {s}}} \geq \frac {\epsilon N _ {\bar {t} _ {\mathbf {s}}} ^ {\mathrm {e}} (\mathbf {s}) \sum_ {\mathbf {x} \in \mathcal {I} - \mathbf {s}} D (\bar {P} _ {\mathbf {x} , \bar {t} _ {\mathbf {s}}} \| P _ {\mathbf {x}} ^ {\prime})}{2 (1 + \gamma) \ln \bar {t} _ {\mathbf {s}}} + \frac {N _ {\bar {t} _ {\mathbf {s}}} ^ {\mathrm {e}} (\mathbf {s}) D (\bar {P} _ {\mathbf {s} , \bar {t} _ {\mathbf {s}}} \| P _ {\mathbf {s}} ^ {\prime})}{(1 + \gamma) \ln \bar {t} _ {\mathbf {s}}}. \tag {26}
$$
Recall the definition of  $\mathcal{C}_t$  and  $\mathcal{C}_{\mathbf{s}}(\eta_{\mathbf{s}},\bar{\mathcal{P}})$
$$
\mathcal {C} _ {t} = \left\{\mathcal {P} ^ {\prime} \in \mathcal {M} _ {\mathcal {G}} \Big | \sum_ {\mathbf {x} \in \mathcal {I}} N _ {t} (\mathbf {x}) D (\bar {P} _ {\mathbf {x}, t} \| P _ {\mathbf {x}} ^ {\prime}) \leq (1 + \gamma) \ln t \right\},
$$
$$
\mathcal {C} _ {\mathbf {s}} (\eta_ {\mathbf {s}}, \bar {\mathcal {P}} _ {t}) = \Big \{\mathcal {P} ^ {\prime} \in \mathcal {M} _ {\mathcal {G}} \Big | \eta_ {\mathbf {s}} D (\bar {P} _ {\mathbf {s}, t} \| P _ {\mathbf {s}} ^ {\prime}) + \epsilon \eta_ {\mathbf {s}} / 2 \sum_ {\mathbf {x} \in \mathcal {I} _ {- \mathbf {s}}} D (\bar {P} _ {\mathbf {x}, t} \| P _ {\mathbf {x}} ^ {\prime}) \leq 1 \Big \}.
$$
With (26), we have  $\mathcal{C}_{\bar{t}_{\mathrm{s}}}\subseteq \mathcal{C}_{\mathrm{s}}\left(\frac{N_{\bar{t}_{\mathrm{s}}}^{\mathrm{c}}(\mathrm{s})}{(1 + \gamma)\ln\bar{t}_{\mathrm{s}}},\bar{\mathcal{P}}_{\bar{t}_{\mathrm{s}}}\right)$  , and as a result,
$$
\tilde {\mu} _ {\mathbf {s}} \left(\mathcal {C} _ {\mathbf {s}} \left(\frac {N _ {\bar {t} _ {\mathbf {s}}} ^ {e} (\mathbf {s})}{(1 + \gamma) \ln \bar {t} _ {\mathbf {s}}}, \bar {\mathcal {P}} _ {t}\right)\right) \geq \tilde {\mu} _ {\mathbf {s}} \left(\mathcal {C} _ {t}\right). \tag {27}
$$
By the algorithm design, we have  $\tilde{\mu}_{\mathbf{s}}(\mathcal{C}_{\vec{t}_{\mathbf{s}}}) \geq \bar{\mu}_{\mathbf{s}_{*}, t}$ , since otherwise SCM-AAM does not enter exploration phase. Therefore, it follows from (27) that
$$
\tilde {\mu} _ {\mathbf {s}} \bigg (\mathcal {C} _ {\mathbf {s}} \Big (\frac {N _ {\bar {t} _ {\mathbf {s}}} ^ {\mathrm {e}} (\mathbf {s})}{(1 + \gamma) \ln \bar {t} _ {\mathbf {s}}}, \bar {\mathcal {P}} _ {\bar {t} _ {\mathbf {s}}} \Big) \bigg) \geq \bar {\mu} _ {\mathbf {s} _ {*}, \bar {t} _ {\mathbf {s}}} = \mu_ {*} (\bar {\mathcal {P}} _ {\bar {t} _ {\mathbf {s}}}),
$$
where the inequality holds since when  $E_{\bar{t}_{\mathbf{s}}}$  happens, Lemma 14 shows  $\mathbf{s}_*$  is also optimal according to  $\bar{\mathcal{P}}_{\bar{t}_{\mathbf{s}}}$ . With Lemma 13,  $\tilde{\mu}_{\mathbf{s}}(\mathcal{C}_{\mathbf{s}}(\eta_{\mathbf{s}},\bar{\mathcal{P}}_{\bar{t}_{\mathbf{s}}}))$  is a continuous nonincreasing function of  $\eta_{\mathbf{s}}$ , then we have
$$
N _ {\bar {t} _ {\mathbf {s}}} ^ {\mathrm {e}} (\mathbf {s}) \leq \zeta_ {\mathbf {s}} (\bar {\mathcal {P}} _ {\bar {t} _ {\mathbf {s}}}) (1 + \gamma) \ln \bar {t} _ {\mathbf {s}} \leq \zeta_ {\mathbf {s}} (\bar {\mathcal {P}} _ {\bar {t} _ {\mathbf {s}}}) (1 + \gamma) \ln T,
$$
where  $\zeta_{\mathbf{s}}(\bar{\mathcal{P}}_{\bar{t}_{\mathbf{s}}})$  defined in (17) is the maximum value of  $\eta_{\mathbf{s}}$  such that  $\tilde{\mu}_{\mathbf{s}}(\mathcal{C}_{\mathbf{s}}(\eta_{\mathbf{s}},\bar{\mathcal{P}}_{\bar{t}_{\mathbf{s}}}))\geq \mu_{*}(\bar{\mathcal{P}}_{\bar{t}_{\mathbf{s}}})$ . Together with Lemma 14, we have
$$
\sum_ {t = 1} ^ {T} \mathbb {1} \left\{\overline {{\mathcal {S}}} _ {t} = \mathbf {s}, E _ {t}, \mathcal {P} \in \mathcal {C} _ {t} \right\} = N _ {\bar {t} _ {\mathrm {s}}} ^ {\mathrm {e}} (\mathbf {s}) \leq \zeta_ {\mathbf {s}} (\mathcal {P}) (1 + \kappa) (1 + \gamma) \ln T. \tag {28}
$$
Forced Exploration: By slightly abusing the notation, at time  $t$ , we denote by  $\{\overline{\mathbf{S}}_t\}$  as the greedy approximation exploration event and by  $\{\underline{\mathbf{S}}_t\}$  as the force exploration event. Let the last time  $\mathbf{s} \in \mathcal{I}$  be selected by forced exploration be  $t_{\mathrm{s}} := \max \left\{t \in \mathcal{R} \mid \underline{\mathbf{S}}_t = \mathbf{s}\right\}$ . When SCM-AAM force exploration at  $t$ , we have  $N_t(\underline{\mathbf{S}}_t) \leq \epsilon N_t^{\mathrm{e}}$ , resulting in
$$
\sum_ {t = 1} ^ {T} \mathbb {1} \left\{\underline {{\mathbf {S}}} _ {t} \right\} \leq \sum_ {\mathbf {s} \in \mathcal {I}} \sum_ {t = 1} ^ {T} \mathbb {1} \left\{\underline {{\mathbf {S}}} _ {t} = \mathbf {s} \right\} \leq \sum_ {\mathbf {s} \in \mathcal {I}} N _ {\underline {{t}} _ {\mathbf {s}}} (\mathbf {s}) \leq \sum_ {\mathbf {s} \in \mathcal {I}} \epsilon N _ {\underline {{t}} _ {\mathbf {s}}} ^ {\mathrm {e}} \leq \epsilon | \mathcal {I} | N _ {T} ^ {\mathrm {e}}. \tag {29}
$$
We want to provide an upper bound  $N_T^{\mathrm{e}}$ , which is
$$
N _ {T} ^ {\mathrm {e}} = \sum_ {t = 1} ^ {T} \mathbb {1} \{t \in \mathcal {R} \} \leq \sum_ {t = 1} ^ {T} \left(\mathbb {1} \{\underline {{\mathbf {S}}} _ {t} \} + \mathbb {1} \{\overline {{\mathbf {S}}} _ {t}, E _ {t}, \mathcal {P} \in \mathcal {C} _ {t} \} + \mathbb {1} \{t \in \mathcal {R}, E _ {t} ^ {\complement} \} + \mathbb {1} \{\mathcal {P} \in \mathcal {C} _ {t} ^ {\complement} \}\right), \tag {30}
$$
Putting together (29) and (30), we get
$$
\sum_ {t = 1} ^ {T} \mathbb {1} \left\{\underline {{\mathbf {S}}} _ {t} \right\} \leq \frac {\epsilon | \mathcal {I} |}{1 - \epsilon | \mathcal {I} |} \sum_ {t = 1} ^ {T} \left(\mathbb {1} \left\{t \in \mathcal {R}, \overline {{\mathbf {S}}} _ {t}, E _ {t}, \mathcal {P} \in \mathcal {C} _ {t} \right\} + \mathbb {1} \left\{t \in \mathcal {R}, E _ {t} ^ {\mathbf {C}} \right\} + \mathbb {1} \left\{\mathcal {P} \in \mathcal {C} _ {t} ^ {\mathbf {C}} \right\}\right).
$$
Also with (28), we have
$$
\begin{array}{l} \sum_ {t = 1} ^ {T} \mathbb {1} \{t \in \mathcal {R}, \overline {{\mathbf {S}}} _ {t}, E _ {t}, \mathcal {P} \in \mathcal {C} _ {t} \} = \sum_ {\mathbf {s} \in \mathcal {I}} \sum_ {t = 1} ^ {T} \mathbb {1} \{t \in \mathcal {R}, \overline {{\mathbf {S}}} _ {t} = \mathbf {s}, E _ {t}, \mathcal {P} \in \mathcal {C} _ {t} \} \\ \leq \sum_ {\mathbf {s} \in \mathcal {I}} \zeta_ {\mathbf {s}} (\bar {\mathcal {P}} _ {\hat {t} _ {\mathbf {s}}}) (1 + \kappa) (1 + \gamma) \ln T. \\ \end{array}
$$
Then, the expected number of forced explorations can be bound as the following,
$$
\mathbb {E} \left[ \sum_ {t = 1} ^ {T} \mathbb {1} \left\{\underline {{\mathbf {S}}} _ {t} \right\} \right] \leq \frac {\epsilon | \mathcal {I} |}{1 - \epsilon | \mathcal {I} |} \left[ c _ {1} + c _ {3} + (1 + \kappa) (1 + \gamma) \sum_ {\mathbf {s} \in \mathcal {I}} \zeta_ {\mathbf {s}} \left(\bar {\mathcal {P}} _ {\bar {t} _ {\mathbf {s}}}\right) \ln T \right], \tag {31}
$$
where  $C_1$  and  $C_3$  are defined in (20) and (25) respectively.
Summary: With reward  $V_{n} \in [0,1]$ , we decomposed the expected regret as
$$
\begin{array}{l} R _ {T} ^ {\mathrm {S C M - A A M}} (\mathcal {P}) \leq \mathbb {E} \left[ \sum_ {t = 1} ^ {T} \sum_ {\mathbf {s} \in \mathcal {I} _ {- \mathbf {s} _ {*}}} \Delta_ {\mathbf {s}} \mathbb {1} \{\mathbf {S} _ {t} = \mathbf {s}, \mathcal {P} \in \mathcal {C} _ {t} \} \right] + \sum_ {\mathbf {s} \in \mathcal {I} _ {- \mathbf {s} _ {*}}} \Delta_ {\mathbf {s}} \sum_ {t \in \mathcal {R}} \mathbb {1} \{\overline {{\mathcal {S}}} _ {t} = \mathbf {s}, E _ {t}, \mathcal {P} \in \mathcal {C} _ {t} \} \\ + \mathbb {E} \bigg [ \sum_ {t = 1} ^ {T} \mathbb {1} \{\underline {{\mathbf {S}}} _ {t} \} \bigg ] + \sum_ {t \in \mathcal {R}} \mathbb {P} (E _ {t} ^ {\mathsf {C}}) + \sum_ {t = 1} ^ {T} \mathbb {P} (\mathcal {P} \not \in \mathcal {C} _ {t}), \\ \end{array}
$$
where the first three terms present regret from exploitation, exploration with greedy approximation, and forced exploration respectively. So the proof can be concluded the proof by substituting (20), (25), (23), (28) and (31) into the above inequality.
C Experiment Details and Additional Experimental Results
In the main paper, we conducted experiments in three causal bandit instances. The causal graph corresponding to each instance is shown in Fig. 2. For  $p \in [0,1]$ , let  $\mathcal{B}(p)$  denote a Bernoulli random variable with probability  $p$  to be 1 and  $1 - p$  to be 0. The detailed structural equations for each causal bandit instance are described below.
Task1:  $U_{V_1} = \mathcal{B}(0.6)$ ,  $U_{V_2} = \mathcal{B}(0.11)$ ,  $U_{V_2V_3} = \mathcal{B}(0.51)$  and  $U_{V_3} = \mathcal{B}(0.15)$ .
$$
f _ {V _ {1}} \left(u _ {V _ {1}}\right) = u _ {V _ {1}}
$$
$$
f _ {V _ {2}} \left(v _ {1}, u _ {V _ {2}}, u _ {V _ {2} V _ {3}}\right) = v _ {1} \oplus u _ {V _ {2}} \oplus u _ {V _ {2} V _ {3}}
$$
$$
f _ {V _ {3}} \left(v _ {2}, u _ {V _ {3}}, u _ {V _ {2} V _ {3}}\right) = v _ {2} \oplus u _ {V _ {3}} \oplus u _ {V _ {2} V _ {3}} \oplus 1
$$
Task2:  $U_{V_1} = \mathcal{B}(0.45)$ ,  $U_{V_2} = \mathcal{B}(0.05)$ ,  $U_{V_5V_2} = \mathcal{B}(0.54)$ ,  $U_{V_3} = \mathcal{B}(0.07)$ ,  $U_{V_3V_4} = \mathcal{B}(0.51)$ ,  $U_{V_4} = \mathcal{B}(0.06)$  and  $U_{V_5} = \mathcal{B}(0.06)$ .
$$
f _ {V _ {2}} \left(u _ {V _ {2}}, u _ {V _ {5} V _ {2}}\right) = u _ {V _ {2}} \oplus u _ {V _ {5} V _ {2}}
$$
$$
f _ {V _ {3}} \left(v _ {1}, u _ {V _ {3}}, u _ {V _ {3} V _ {4}}\right) = v _ {1} \oplus u _ {V _ {3}} \oplus u _ {V _ {3} V _ {4}}
$$
$$
f _ {V _ {4}} \left(v _ {2}, u _ {V _ {4}}, u _ {V _ {3} V _ {4}}\right) = 1 \oplus v _ {2} \oplus u _ {V _ {4}} \oplus u _ {V _ {3} V _ {4}}
$$
$$
f _ {V _ {5}} \left(v _ {3}, v _ {4}, u _ {V _ {5}}, u _ {V _ {5} V _ {2}}\right) = v _ {3} \oplus v _ {4} \oplus u _ {V _ {5}} \oplus u _ {V _ {5} V _ {2}}
$$
Task3:  $U_{V_1} = \mathcal{B}(0.5)$ ,  $U_{V_2} = \mathcal{B}(0.85)$ ,  $U_{V_3} = \mathcal{B}(0.85)$ ,  $U_{V_3V_7} = \mathcal{B}(0.85)$ ,  $U_{V_4} = \mathcal{B}(0.14)$ ,  $U_{V_5} = \mathcal{B}(0.74)$ ,  $U_{V_6} = \mathcal{B}(0.74)$ ,  $U_{V_7} = \mathcal{B}(0.54)$ .
$$
f _ {V _ {2}} \left(u _ {V _ {2}}\right) = u _ {V _ {2}}
$$
$$
f _ {V _ {3}} \left(v _ {1}, u _ {V _ {3}}, u _ {V _ {3} V _ {7}}\right) = \left(v _ {1} \oplus u _ {u _ {V _ {3}}}\right) \vee u _ {u _ {V _ {3} V _ {7}}}
$$
$$
f _ {V _ {4}} \left(v _ {2}, u _ {V _ {4}}\right) = v _ {2} \wedge u _ {V _ {4}}
$$
$$
f _ {V _ {5}} \left(v _ {3}, v _ {4}, u _ {V _ {5}}\right) = \left(v _ {3} \oplus u _ {V _ {5}}\right) \vee v _ {4}
$$
$$
f _ {V _ {6}} \left(v _ {4}, u _ {V _ {6}}\right) = v _ {4} \vee u _ {V _ {6}}
$$
$$
f _ {V _ {7}} \left(v _ {5}, v _ {6}, u _ {V _ {5}}, u _ {V _ {7}}, u _ {V _ {3} V _ {7}}\right) = \left(\left(v _ {5} \oplus v _ {6} \oplus 0 \oplus u _ {V _ {3} V _ {7}}\right) \wedge u _ {V _ {7}}\right) \oplus v _ {6}
$$