Spaces:
Runtime error
Runtime error
Commit
•
675e604
1
Parent(s):
cf5eed6
Fix dataset loading bug
Browse files
app.py
CHANGED
@@ -3,185 +3,211 @@ import matplotlib.pyplot as plt
|
|
3 |
import numpy as np
|
4 |
from functools import partial
|
5 |
|
6 |
-
|
7 |
-
|
8 |
-
|
9 |
-
|
10 |
-
|
11 |
-
|
12 |
-
|
13 |
-
|
14 |
-
|
15 |
-
|
16 |
-
|
17 |
-
|
18 |
-
|
19 |
-
|
20 |
-
|
21 |
-
|
22 |
-
|
23 |
-
|
24 |
-
|
25 |
-
|
26 |
-
|
27 |
-
|
|
|
28 |
|
29 |
dataset_data = {
|
30 |
-
"
|
31 |
-
|
32 |
-
|
33 |
-
|
34 |
-
|
35 |
-
|
36 |
-
|
37 |
-
"
|
38 |
-
|
39 |
-
|
40 |
-
|
41 |
-
|
42 |
-
|
43 |
-
|
44 |
-
"
|
45 |
-
|
46 |
-
|
47 |
-
|
48 |
-
|
49 |
-
|
50 |
-
|
51 |
-
"
|
52 |
-
|
53 |
-
|
54 |
-
|
55 |
-
|
56 |
-
|
57 |
-
|
58 |
-
|
59 |
-
|
60 |
-
|
61 |
-
|
62 |
-
|
63 |
-
|
64 |
-
|
65 |
-
|
66 |
-
|
67 |
-
|
68 |
-
|
69 |
-
|
70 |
-
|
71 |
-
|
72 |
-
|
73 |
-
|
74 |
-
|
75 |
-
|
76 |
-
|
77 |
-
|
78 |
-
|
79 |
-
|
80 |
-
|
81 |
-
|
82 |
-
|
83 |
-
|
84 |
-
|
85 |
-
|
86 |
-
|
87 |
-
|
88 |
-
|
89 |
-
|
90 |
-
|
91 |
-
|
92 |
-
|
93 |
-
|
94 |
-
|
95 |
-
|
96 |
-
|
97 |
-
|
98 |
-
|
99 |
-
|
100 |
-
|
101 |
-
|
102 |
-
|
103 |
-
|
104 |
-
|
105 |
-
|
106 |
-
|
107 |
-
|
108 |
-
|
109 |
-
|
110 |
-
|
111 |
-
|
112 |
-
|
113 |
-
|
114 |
-
|
115 |
-
|
116 |
-
|
117 |
-
|
118 |
-
|
119 |
-
|
120 |
-
|
121 |
-
|
122 |
-
|
123 |
-
|
124 |
-
|
125 |
-
|
126 |
-
|
127 |
-
|
128 |
-
|
129 |
-
|
130 |
-
|
131 |
-
|
132 |
-
|
133 |
-
|
134 |
-
|
135 |
-
|
136 |
-
|
137 |
-
|
138 |
-
|
139 |
-
|
140 |
-
|
141 |
-
|
142 |
-
|
143 |
-
|
144 |
-
|
145 |
-
|
146 |
-
|
147 |
-
|
148 |
-
|
149 |
-
|
150 |
-
|
151 |
-
|
152 |
-
|
153 |
-
|
154 |
-
|
155 |
-
|
156 |
-
|
157 |
-
|
158 |
-
|
159 |
-
|
160 |
-
|
161 |
-
|
162 |
-
|
163 |
-
|
164 |
-
|
165 |
-
|
166 |
-
|
167 |
-
|
168 |
-
|
169 |
-
|
170 |
-
|
171 |
-
|
172 |
-
|
173 |
-
|
174 |
-
|
175 |
-
|
176 |
-
|
177 |
-
|
178 |
-
|
179 |
-
|
180 |
-
|
181 |
-
|
182 |
-
|
183 |
-
|
184 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
185 |
|
186 |
def plt_plot(ratio, dataset, threshold):
|
187 |
x = dataset_data[dataset][ratio]
|
|
|
3 |
import numpy as np
|
4 |
from functools import partial
|
5 |
|
6 |
+
ai4code_ds = load_dataset("CarperAI/pile-v2-small", data_dir="data/AI4Code/data.json")
|
7 |
+
amps_ds = load_dataset("CarperAI/pile-v2-small", data_dir="data/AMPS/data.json")
|
8 |
+
apache_ds = load_dataset("CarperAI/pile-v2-small", data_dir="data/ASFPublicMail/data.json")
|
9 |
+
books3_ds = load_dataset("CarperAI/pile-v2-small", data_dir="data/Books3/data.json")
|
10 |
+
cp_ds = load_dataset("CarperAI/pile-v2-small", data_dir="data/CPDataset/data.json")
|
11 |
+
dmmath_ds = load_dataset("CarperAI/pile-v2-small", data_dir="data/DMMath/data.json")
|
12 |
+
discourse_ds = load_dataset("CarperAI/pile-v2-small", data_dir="data/Discourse/data.json")
|
13 |
+
wiki_ds = load_dataset("CarperAI/pile-v2-small", data_dir="data/Enwiki/data.json")
|
14 |
+
euro_ds = load_dataset("CarperAI/pile-v2-small", data_dir="data/EuroParliamentProceedings/data.json")
|
15 |
+
freelaw_ds = load_dataset("CarperAI/pile-v2-small", data_dir="data/FreeLaw_Options/data.json")
|
16 |
+
ghdiffs_ds = load_dataset("CarperAI/pile-v2-small", data_dir="data/GitHubDiff/data.json")
|
17 |
+
ghissues_ds = load_dataset("CarperAI/pile-v2-small", data_dir="data/GitHubIssues/data.json")
|
18 |
+
gutenberg_ds = load_dataset("CarperAI/pile-v2-small", data_dir="data/Gutenberg/data.json")
|
19 |
+
leet_ds = load_dataset("CarperAI/pile-v2-small", data_dir="data/LeetCode/data.json")
|
20 |
+
pileoflaw_ds = load_dataset("CarperAI/pile-v2-small", data_dir="data/PileOfLaw/data.json")
|
21 |
+
pubmed_ds = load_dataset("CarperAI/pile-v2-small", data_dir="data/PubMed/data.json")
|
22 |
+
s2orc_ds = load_dataset("CarperAI/pile-v2-small", data_dir="data/S2ORC/data.json")
|
23 |
+
se_ds = load_dataset("CarperAI/pile-v2-small", data_dir="data/StackExchange/data.json")
|
24 |
+
usenet_ds = load_dataset("CarperAI/pile-v2-small", data_dir="data/USENET/data.json")
|
25 |
+
uspto_ds = load_dataset("CarperAI/pile-v2-small", data_dir="data/USPTO/data.json")
|
26 |
+
ubuntuirc_ds = load_dataset("CarperAI/pile-v2-small", data_dir="data/UbuntuIRC/data.json")
|
27 |
+
arxiv_ds = load_dataset("CarperAI/pile-v2-small", data_dir="data/arXiv/data.json")
|
28 |
+
|
29 |
|
30 |
dataset_data = {
|
31 |
+
"ai4code" : ai4code_ds["train"],
|
32 |
+
"amps" : amps_ds["train"],
|
33 |
+
"apache" : apache_ds["train"],
|
34 |
+
"books3" : books3_ds["train"],
|
35 |
+
"competitive_programming" : cp_ds["train"],
|
36 |
+
"dmmath" : dmmath_ds["train"],
|
37 |
+
"discourse" : discourse_ds["train"],
|
38 |
+
"enwiki" : wiki_ds["train"],
|
39 |
+
"euro" : euro_ds["train"],
|
40 |
+
"freelaw" : freelaw_ds["train"],
|
41 |
+
"ghdiffs" : ghdiffs_ds["train"],
|
42 |
+
"ghissues" : ghissues_ds["train"],
|
43 |
+
"gutenberg" : gutenberg_ds["train"],
|
44 |
+
"leetcode" : leet_ds["train"],
|
45 |
+
"pileoflaw" : pileoflaw_ds["train"],
|
46 |
+
"pubmed" : pubmed_ds["train"],
|
47 |
+
"s2orc" : s2orc_ds["train"],
|
48 |
+
"se" : se_ds["train"],
|
49 |
+
"usenet" : usenet_ds["train"],
|
50 |
+
"uspto" : uspto_ds["train"],
|
51 |
+
"ubuntuirc" : ubuntuirc_ds["train"],
|
52 |
+
"arxiv" : arxiv_ds["train"]
|
53 |
+
}
|
54 |
+
|
55 |
+
# dataset_data = {
|
56 |
+
# "AI4Code": {
|
57 |
+
# # create fake data for the different ratios
|
58 |
+
# "word_rep_ratios": np.random.randn(1000),
|
59 |
+
# "char_rep_ratios": np.random.randn(1000),
|
60 |
+
# "flagged_word_ratios": np.random.randn(1000),
|
61 |
+
# "num_words": np.random.randint(0, 1000, 1000),
|
62 |
+
# },
|
63 |
+
# "AMPS": {
|
64 |
+
# # create fake data for the different ratios
|
65 |
+
# "word_rep_ratios": np.random.randn(1000),
|
66 |
+
# "char_rep_ratios": np.random.randn(1000),
|
67 |
+
# "flagged_word_ratios": np.random.randn(1000),
|
68 |
+
# "num_words": np.random.randint(0, 1000, 1000),
|
69 |
+
# },
|
70 |
+
# "ASFPublicMail": {
|
71 |
+
# # create fake data for the different ratios
|
72 |
+
# "word_rep_ratios": np.random.randn(1000),
|
73 |
+
# "char_rep_ratios": np.random.randn(1000),
|
74 |
+
# "flagged_word_ratios": np.random.randn(1000),
|
75 |
+
# "num_words": np.random.randint(0, 1000, 1000),
|
76 |
+
# },
|
77 |
+
# "Books3": {
|
78 |
+
# # create fake data for the different ratios
|
79 |
+
# "word_rep_ratios": np.random.randn(1000),
|
80 |
+
# "char_rep_ratios": np.random.randn(1000),
|
81 |
+
# "flagged_word_ratios": np.random.randn(1000),
|
82 |
+
# "num_words": np.random.randint(0, 1000, 1000),
|
83 |
+
# },
|
84 |
+
# "CPDataset": {
|
85 |
+
# # create fake data for the different ratios
|
86 |
+
# "word_rep_ratios": np.random.randn(1000),
|
87 |
+
# "char_rep_ratios": np.random.randn(1000),
|
88 |
+
# "flagged_word_ratios": np.random.randn(1000),
|
89 |
+
# "num_words": np.random.randint(0, 1000, 1000),
|
90 |
+
# },
|
91 |
+
# "DMMath": {
|
92 |
+
# # create fake data for the different ratios
|
93 |
+
# "word_rep_ratios": np.random.randn(1000),
|
94 |
+
# "char_rep_ratios": np.random.randn(1000),
|
95 |
+
# "flagged_word_ratios": np.random.randn(1000),
|
96 |
+
# "num_words": np.random.randint(0, 1000, 1000),
|
97 |
+
# },
|
98 |
+
# "Discourse": {
|
99 |
+
# # create fake data for the different ratios
|
100 |
+
# "word_rep_ratios": np.random.randn(1000),
|
101 |
+
# "char_rep_ratios": np.random.randn(1000),
|
102 |
+
# "flagged_word_ratios": np.random.randn(1000),
|
103 |
+
# "num_words": np.random.randint(0, 1000, 1000),
|
104 |
+
# },
|
105 |
+
# "Enwiki": {
|
106 |
+
# # create fake data for the different ratios
|
107 |
+
# "word_rep_ratios": np.random.randn(1000),
|
108 |
+
# "char_rep_ratios": np.random.randn(1000),
|
109 |
+
# "flagged_word_ratios": np.random.randn(1000),
|
110 |
+
# "num_words": np.random.randint(0, 1000, 1000),
|
111 |
+
# },
|
112 |
+
# "EuroParliamentProceedings": {
|
113 |
+
# # create fake data for the different ratios
|
114 |
+
# "word_rep_ratios": np.random.randn(1000),
|
115 |
+
# "char_rep_ratios": np.random.randn(1000),
|
116 |
+
# "flagged_word_ratios": np.random.randn(1000),
|
117 |
+
# "num_words": np.random.randint(0, 1000, 1000),
|
118 |
+
# },
|
119 |
+
# "FreeLaw_Options": {
|
120 |
+
# # create fake data for the different ratios
|
121 |
+
# "word_rep_ratios": np.random.randn(1000),
|
122 |
+
# "char_rep_ratios": np.random.randn(1000),
|
123 |
+
# "flagged_word_ratios": np.random.randn(1000),
|
124 |
+
# "num_words": np.random.randint(0, 1000, 1000),
|
125 |
+
# },
|
126 |
+
# "GitHubDiff": {
|
127 |
+
# # create fake data for the different ratios
|
128 |
+
# "word_rep_ratios": np.random.randn(1000),
|
129 |
+
# "char_rep_ratios": np.random.randn(1000),
|
130 |
+
# "flagged_word_ratios": np.random.randn(1000),
|
131 |
+
# "num_words": np.random.randint(0, 1000, 1000),
|
132 |
+
# },
|
133 |
+
# "GitHubIssues": {
|
134 |
+
# # create fake data for the different ratios
|
135 |
+
# "word_rep_ratios": np.random.randn(1000),
|
136 |
+
# "char_rep_ratios": np.random.randn(1000),
|
137 |
+
# "flagged_word_ratios": np.random.randn(1000),
|
138 |
+
# "num_words": np.random.randint(0, 1000, 1000),
|
139 |
+
# },
|
140 |
+
# "Gutenberg": {
|
141 |
+
# # create fake data for the different ratios
|
142 |
+
# "word_rep_ratios": np.random.randn(1000),
|
143 |
+
# "char_rep_ratios": np.random.randn(1000),
|
144 |
+
# "flagged_word_ratios": np.random.randn(1000),
|
145 |
+
# "num_words": np.random.randint(0, 1000, 1000),
|
146 |
+
# },
|
147 |
+
# "LeetCode": {
|
148 |
+
# # create fake data for the different ratios
|
149 |
+
# "word_rep_ratios": np.random.randn(1000),
|
150 |
+
# "char_rep_ratios": np.random.randn(1000),
|
151 |
+
# "flagged_word_ratios": np.random.randn(1000),
|
152 |
+
# "num_words": np.random.randint(0, 1000, 1000),
|
153 |
+
# },
|
154 |
+
# "PileOfLaw": {
|
155 |
+
# # create fake data for the different ratios
|
156 |
+
# "word_rep_ratios": np.random.randn(1000),
|
157 |
+
# "char_rep_ratios": np.random.randn(1000),
|
158 |
+
# "flagged_word_ratios": np.random.randn(1000),
|
159 |
+
# "num_words": np.random.randint(0, 1000, 1000),
|
160 |
+
# },
|
161 |
+
# "PubMed": {
|
162 |
+
# # create fake data for the different ratios
|
163 |
+
# "word_rep_ratios": np.random.randn(1000),
|
164 |
+
# "char_rep_ratios": np.random.randn(1000),
|
165 |
+
# "flagged_word_ratios": np.random.randn(1000),
|
166 |
+
# "num_words": np.random.randint(0, 1000, 1000),
|
167 |
+
# },
|
168 |
+
# "S2ORC": {
|
169 |
+
# # create fake data for the different ratios
|
170 |
+
# "word_rep_ratios": np.random.randn(1000),
|
171 |
+
# "char_rep_ratios": np.random.randn(1000),
|
172 |
+
# "flagged_word_ratios": np.random.randn(1000),
|
173 |
+
# "num_words": np.random.randint(0, 1000, 1000),
|
174 |
+
# },
|
175 |
+
# "StackExchange": {
|
176 |
+
# # create fake data for the different ratios
|
177 |
+
# "word_rep_ratios": np.random.randn(1000),
|
178 |
+
# "char_rep_ratios": np.random.randn(1000),
|
179 |
+
# "flagged_word_ratios": np.random.randn(1000),
|
180 |
+
# "num_words": np.random.randint(0, 1000, 1000),
|
181 |
+
# },
|
182 |
+
# "USENET": {
|
183 |
+
# # create fake data for the different ratios
|
184 |
+
# "word_rep_ratios": np.random.randn(1000),
|
185 |
+
# "char_rep_ratios": np.random.randn(1000),
|
186 |
+
# "flagged_word_ratios": np.random.randn(1000),
|
187 |
+
# "num_words": np.random.randint(0, 1000, 1000),
|
188 |
+
# },
|
189 |
+
# "USPTO": {
|
190 |
+
# # create fake data for the different ratios
|
191 |
+
# "word_rep_ratios": np.random.randn(1000),
|
192 |
+
# "char_rep_ratios": np.random.randn(1000),
|
193 |
+
# "flagged_word_ratios": np.random.randn(1000),
|
194 |
+
# "num_words": np.random.randint(0, 1000, 1000),
|
195 |
+
# },
|
196 |
+
# "UbuntuIRC": {
|
197 |
+
# # create fake data for the different ratios
|
198 |
+
# "word_rep_ratios": np.random.randn(1000),
|
199 |
+
# "char_rep_ratios": np.random.randn(1000),
|
200 |
+
# "flagged_word_ratios": np.random.randn(1000),
|
201 |
+
# "num_words": np.random.randint(0, 1000, 1000),
|
202 |
+
# },
|
203 |
+
# "arXiv": {
|
204 |
+
# # create fake data for the different ratios
|
205 |
+
# "word_rep_ratios": np.random.randn(1000),
|
206 |
+
# "char_rep_ratios": np.random.randn(1000),
|
207 |
+
# "flagged_word_ratios": np.random.randn(1000),
|
208 |
+
# "num_words": np.random.randint(0, 1000, 1000),
|
209 |
+
# },
|
210 |
+
# }
|
211 |
|
212 |
def plt_plot(ratio, dataset, threshold):
|
213 |
x = dataset_data[dataset][ratio]
|