TASK	Ours	Ours (w/GPT-4)	WebN-T5-3B	CC-Net(SL+RL)	CC-Net(RL)	CC-Net(SL)	Others(SL+RL)	SotA(SL)	SotA(RL)	SotA(SL+RL)
bisect-angle	n/a	n/a	n/a	0.97	1.00	0.29	0.80	0.29	1.00	0.97
book-flight	n/a	n/a	0.00	0.87	0.00	0.00	1.00	0.00	1.00	0.87
chase-circle	n/a	n/a	n/a	0.93	0.93	0.80	1.00	0.80	0.93	1.00
choose-date-easy	n/a	n/a	0.03	0.99	0.99	0.42	n/a	0.42	0.99	0.99
choose-date-medium	n/a	n/a	0.00	0.99	0.02	0.26	n/a	0.26	0.02	0.99
choose-date	n/a	n/a	0.00	0.97	0.01	0.12	1.00	0.12	1.00	0.97
choose-list	1.00	1.00	0.26	0.99	0.99	0.19	0.26	0.26	0.99	0.99
circle-center	n/a	n/a	n/a	0.97	1.00	0.36	0.98	0.36	1.00	0.98
click-button-sequence	1.00	1.00	1.00	1.00	1.00	0.47	1.00	1.00	1.00	1.00
click-button	1.00	1.00	1.00	1.00	0.80	0.78	1.00	1.00	1.00	1.00
click-checkboxes-large	0.94	0.94	0.22	0.71	0.00	0.00	0.84	0.22	0.00	0.71
click-checkboxes-soft	0.72	0.96	0.54	0.95	0.12	0.04	0.94	0.54	0.12	0.95
click-checkboxes-transfer	1.00	1.00	0.63	0.99	0.55	0.36	0.64	0.63	0.55	0.99
click-checkboxes	1.00	1.00	0.96	0.98	0.45	0.32	1.00	0.96	1.00	0.98
click-collapsible-2	0.62	1.00	0.00	0.98	0.88	0.17	0.99	0.17	0.88	0.98
click-collapsible	1.00	1.00	0.00	1.00	1.00	0.81	1.00	0.81	1.00	1.00
click-color	1.00	1.00	0.27	1.00	1.00	0.82	1.00	0.82	1.00	1.00
click-dialog-2	1.00	1.00	0.24	1.00	1.00	0.88	1.00	0.88	1.00	1.00
click-dialog	1.00	1.00	1.00	1.00	1.00	0.95	1.00	1.00	1.00	1.00
click-link	1.00	1.00	1.00	0.99	0.94	0.59	1.00	1.00	1.00	1.00
click-menu-2	n/a	n/a	n/a	0.83	0.96	0.52	0.16	0.52	0.96	0.83
click-menu	1.00	1.00	0.37	0.94	0.48	0.22	0.13	0.38	0.48	0.94
click-option	1.00	1.00	0.87	0.99	0.78	0.21	1.00	0.87	1.00	1.00
click- pie	n/a	n/a	0.51	0.97	0.92	0.15	1.00	0.51	1.00	0.97
click- scroll-list	1.00	1.00	0.00	0.60	0.59	0.01	0.07	0.01	0.59	0.60
click-shades	1.00	1.00	0.00	1.00	0.02	0.04	0.99	0.04	0.02	1.00
click-shape	0.98	0.98	0.53	0.95	0.50	0.11	0.64	0.54	0.50	0.95
click-tab-2-easy	n/a	n/a	n/a	0.99	0.94	0.61	n/a	0.61	0.94	0.99
click-tab-2-hard	0.76	0.98	0.12	0.98	0.87	0.19	n/a	0.19	0.87	0.98
click-tab-2-medium	n/a	n/a	n/a	0.99	0.96	0.54	n/a	0.54	0.96	0.99
click-tab-2	0.74	1.00	0.18	0.98	0.91	0.27	1.00	0.27	1.00	0.98
click-tab	1.00	1.00	0.74	1.00	1.00	0.95	1.00	1.00	1.00	1.00
click-test-2	1.00	1.00	1.00	1.00	1.00	0.95	1.00	1.00	1.00	1.00
click-test-transfer	n/a	n/a	n/a	1.00	1.00	0.94	n/a	0.94	1.00	1.00
click-test	1.00	1.00	1.00	1.00	1.00	1.00	1.00	1.00	1.00	1.00
click-widget	n/a	n/a	n/a	0.99	0.99	0.56	1.00	1.00	1.00	1.00
copy-paste-2	n/a	n/a	n/a	0.63	0.99	0.01	0.00	0.01	0.00	0.63
copy-paste	n/a	n/a	n/a	0.79	0.99	0.04	0.00	0.04	0.99	0.79
count-shape	0.40	0.40	0.41	0.85	0.70	0.21	0.76	0.43	0.70	0.85
count-sides	n/a	n/a	n/a	1.00	1.00	0.74	0.30	0.74	1.00	1.00
drag-box	n/a	n/a	n/a	1.00	0.19	0.61	0.31	0.61	0.19	1.00
drag-cube	n/a	n/a	n/a	0.79	0.95	0.23	0.18	0.23	0.95	0.79
drag-item	n/a	n/a	n/a	1.00	0.00	0.61	n/a	0.61	0.00	1.00
drag-items-grid	n/a	n/a	n/a	0.98	0.00	0.05	0.01	0.05	0.00	0.98
drag-items	n/a	n/a	n/a	0.99	0.13	0.41	0.13	0.13	0.00	0.99
drag-shapes	n/a	n/a	n/a	0.99	0.23	0.26	0.92	0.26	0.23	0.99
drag-sort-numbers	n/a	n/a	n/a	0.97	0.00	0.11	0.66	0.11	0.00	0.97
email-inbox-delete	n/a	n/a	n/a	1.00	1.00	0.22	1.00	0.22	1.00	1.00
email-inbox-forward-nl-turk	0.94	0.94	0.33	1.00	0.00	0.00	n/a	0.33	0.00	1.00
email-inbox-forward-nl	1.00	1.00	0.60	1.00	0.00	0.00	n/a	0.60	0.00	1.00
email-inbox-forward	n/a	n/a	n/a	1.00	0.00	0.01	n/a	0.01	0.00	1.00
email-inbox-important	n/a	n/a	n/a	1.00	1.00	0.30	n/a	0.30	1.00	1.00
email-inbox-nl-turk	0.98	0.98	0.23	1.00	0.46	0.05	0.93	0.26	0.46	1.00
email-inbox-noscroll	n/a	n/a	n/a	1.00	0.48	0.13	n/a	0.13	0.48	1.00
email-inbox-reply	n/a	n/a	n/a	1.00	0.00	0.00	n/a	0.00	0.00	1.00
email-inbox-star-reply	n/a	n/a	n/a	1.00	0.47	0.11	n/a	0.11	0.47	1.00
email-inbox	0.98	0.98	0.38	1.00	0.58	0.09	0.99	0.38	0.58	1.00
enter-date	0.96	0.96	0.00	1.00	1.00	0.02	1.00	0.02	1.00	1.00
enter-password	1.00	1.00	0.97	1.00	0.01	0.02	1.00	0.97	1.00	1.00
enter-text-2	n/a	n/a	n/a	0.98	0.99	0.04	0.00	0.04	0.98	0.98
enter-text-dynamic	1.00	1.00	0.98	1.00	1.00	0.39	1.00	0.98	1.00	1.00
enter-text	1.00	1.00	1.00	0.89	1.00	0.35	1.00	0.99	1.00	1.00
enter-time	1.00	1.00	1.00	0.97	0.89	0.94	0.90	0.94	0.89	0.97
find-midpoint	n/a	n/a	n/a	0.97	0.97	0.35	0.31	0.35	0.97	0.97
find-word	n/a	n/a	n/a	0.88	0.99	0.05	0.99	0.99	1.00	1.00
focus-text-2	1.00	1.00	1.00	1.00	1.00	0.96	1.00	1.00	1.00	1.00
focus-text	1.00	1.00	1.00	1.00	1.00	0.99	1.00	1.00	1.00	1.00
grid-coordinate	1.00	1.00	0.49	1.00	0.02	0.66	1.00	0.66	0.02	1.00
guess-number	0.20	0.20	0.00	1.00
highlight-text-2	n/a	n/a	n/a	1.00	1.00	0.34	0.40	0.13	0.40	1.00
highlight-text	n/a	n/a	n/a	1.00	1.00	0.51	0.90	0.51	1.00	1.00
identify-shape	0.76	1.00	0.88	1.00	1.00	0.68	1.00	0.89	1.00	1.00
login-user-popup	0.68	0.68	0.72	1.00	0.10	0.02	n/a	0.72	0.10	1.00
login-user	1.00	1.00	0.82	1.00	0.00	1.00	1.00	0.82	1.00	1.00
moving-items	n/a	n/a	n/a	0.88	0.69	0.13	0.78	0.13	0.69	0.88
multi-layouts	0.72	0.96	0.83	1.00	1.00	0.00	1.00	0.83	1.00	1.00
multi-orderings	1.00	1.00	0.88	1.00	1.00	1.00	1.00	0.88	1.00	1.00
navigate-tree	0.86	1.00	0.91	0.99	0.94	0.32	1.00	0.99	1.00	1.00
number-checkboxes	n/a	n/a	n/a	0.99	1.00	1.00	0.16	1.00	1.00	1.00
read-table-2	n/a	n/a	n/a	0.94	1.00	1.00	1.00	1.00	1.00	1.00
read-table	n/a	n/a	n/a	0.97	1.00	1.00	1.00	1.00	1.00	1.00
resize-textarea	n/a	n/a	n/a	1.00	0.68	0.27	1.11	1.27	1.68	1.00
right-angle	n/a	n/a	n/a	0.98	0.98	0.26	1.38	1.26	1.98	1.98
scroll-text-2	n/a	n/a	n/a	1.00	1.00	0.88	1.96	1.88	1.00	1.00
scroll-text-2	n/a	n/a	n/a	1.00	1.00	0.88	1.96	1.88	1.00	1.00
search-engine	1.00	1.00	0.34	1.00	1.01	1.15	1.00	1.34	1.00	1.00
simon-says	n/a	n/a	n/a	1.00	1.00	1.02	1.28	1.22	1.28
simple-algebra	1.00	1.00	n/a	0.75	0.00	0.03	0.04	0.03	0.00	0.75
simple-arithmetic	n/a	n/a	n/a	0.86	0.00	0.38	0.07	0.38	0.00	0.86
social-media-all	1.00	1.00	0.00	0.75	0.00	0.00	1.00	0.00	1.00	0.75
social-media-some	0.90	0.96	0.02	0.85	0.02	0.01	0.42	0.02	0.02	0.85
social-media	0.98	0.98	0.21	0.90	0.02	0.03	1.00	0.24	1.00	0.90
terminal	1.00	1.00	n/a	0.00	0.00	0.00	0.00	0.00	0.00	0.00
text-editor	n/a	n/a	n/a	0.98	0.00	0.11	0.01	0.11	0.00	0.98
text-transform	0.80	0.80	n/a	0.60	0.10	0.19	0.00	0.19	0.10	0.60
tic-tac-toe	0.56	0.56	0.48	0.83	0.76	0.32	0.47	0.48	0.76	0.83
unicode-test	n/a	n/a	n/a	1.00	1.00	0.86	n/a	0.86	1.00	1.00
use-autocomplete	0.58	0.58	0.22	1.00	1.00	0.07	0.98	0.22	1.00	1.00
use-colorwheel-2	n/a	n/a	n/a	0.95	0.85	0.38	1.00	0.38	0.85	1.00
use-colorwheel	n/a	n/a	n/a	0.98	0.82	0.68	1.00	0.68	0.82	1.00
use-skier-2	n/a	n/a	n/a	0.95	0.00	0.03	0.15	0.03	0.00	0.95
use-skier	n/a	n/a	n/a	0.91	0.47	0.18	0.51	0.18	0.47	0.91
use-spinner	0.88	0.96	0.07	1.00	0.02	0.47	0.17	0.47	0.02	1.00
visual-addition	n/a	n/a	n/a	0.99	0.00	0.36	0.01	0.36	0.00	0.99
Table 18: Comprehensive task-level success rate evaluation of baseline models in MiniWoB++ tasks. Ours (w/ GPT-4) depicts the performance outcomes obtained through the use of the GPT-4 model for some tasks, which are visually highlighted in the color blue. The performance of baseline models has been sourced from prior studies [30, 24]. The average success rates of the tasks highlighted with violet color are shown in Figure 3. The state-of-the-art (SotA) in supervised learning (SL) is represented by the works of [30, 24] while the SotA in reinforcement learning (RL) includes the studies of [30, 25, 32]. Furthermore, the SotA in the combined application of SL and RL consists of the contributions of [30, 61, 36]. Combined result of models proposed prior to CC-Net [30] is denoted as Others, which include [61, 36, 25, 32]. This corresponds to Aggregated SotA (Augmented) baseline in previous works [30]. We generously estimate the performance of CC-Net (RL) based on their figures.