nipunsadvilkar commited on
Commit
c3a9018
1 Parent(s): 47ebced

Saving weights and logs of step 500

Browse files
Files changed (3) hide show
  1. run.log +273 -0
  2. run.sh +18 -0
  3. tokens.py +20 -0
run.log ADDED
@@ -0,0 +1,273 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
0
  0%| | 0/5 [00:00<?, ?ba/s]
1
  20%|██ | 1/5 [00:00<00:00, 8.80ba/s]
2
  40%|████ | 2/5 [00:00<00:00, 9.16ba/s]
3
  60%|██████ | 3/5 [00:00<00:00, 9.52ba/s]
4
  80%|████████ | 4/5 [00:00<00:00, 7.13ba/s]
 
5
  0%| | 0/1 [00:00<?, ?ba/s]
 
6
  0%| | 0/5 [00:00<?, ?ba/s]
7
  20%|██ | 1/5 [00:00<00:03, 1.32ba/s]
8
  40%|████ | 2/5 [00:01<00:02, 1.24ba/s]
9
  60%|██████ | 3/5 [00:02<00:01, 1.21ba/s]
10
  80%|████████ | 4/5 [00:03<00:00, 1.13ba/s]
 
11
  0%| | 0/1 [00:00<?, ?ba/s]
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
12
 
 
13
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ [16:04:24] - INFO - __main__ - Training/evaluation parameters TrainingArguments(
2
+ _n_gpu=-1,
3
+ adafactor=False,
4
+ adam_beta1=0.9,
5
+ adam_beta2=0.999,
6
+ adam_epsilon=1e-08,
7
+ dataloader_drop_last=False,
8
+ dataloader_num_workers=0,
9
+ dataloader_pin_memory=True,
10
+ ddp_find_unused_parameters=None,
11
+ debug=[],
12
+ deepspeed=None,
13
+ disable_tqdm=False,
14
+ do_eval=False,
15
+ do_predict=False,
16
+ do_train=False,
17
+ eval_accumulation_steps=None,
18
+ eval_steps=500,
19
+ evaluation_strategy=IntervalStrategy.NO,
20
+ fp16=False,
21
+ fp16_backend=auto,
22
+ fp16_full_eval=False,
23
+ fp16_opt_level=O1,
24
+ gradient_accumulation_steps=1,
25
+ greater_is_better=None,
26
+ group_by_length=False,
27
+ ignore_data_skip=False,
28
+ label_names=None,
29
+ label_smoothing_factor=0.0,
30
+ learning_rate=0.0003,
31
+ length_column_name=length,
32
+ load_best_model_at_end=False,
33
+ local_rank=-1,
34
+ log_level=-1,
35
+ log_level_replica=-1,
36
+ log_on_each_node=True,
37
+ logging_dir=./runs/Jul08_16-04-24_t1v-n-112df4a9-w-0,
38
+ logging_first_step=False,
39
+ logging_steps=500,
40
+ logging_strategy=IntervalStrategy.STEPS,
41
+ lr_scheduler_type=SchedulerType.LINEAR,
42
+ max_grad_norm=1.0,
43
+ max_steps=-1,
44
+ metric_for_best_model=None,
45
+ mp_parameters=,
46
+ no_cuda=False,
47
+ num_train_epochs=8.0,
48
+ output_dir=./,
49
+ overwrite_output_dir=True,
50
+ past_index=-1,
51
+ per_device_eval_batch_size=4,
52
+ per_device_train_batch_size=4,
53
+ prediction_loss_only=False,
54
+ push_to_hub=True,
55
+ push_to_hub_model_id=flax-community/roberta-base-mr,
56
+ push_to_hub_organization=None,
57
+ push_to_hub_token=vdIAyRvCACJNslYtyLHufmNDnUIyknPzUgVDMFiXqJoulvMqjoubonLJzXOJQJczWfRMJumVaMFjGSFVnQAMdswvZkzNIthKrxBeARBXfqnIwjABkKpCbjGEgnkjpjKi,
58
+ remove_unused_columns=True,
59
+ report_to=[],
60
+ resume_from_checkpoint=None,
61
+ run_name=./,
62
+ save_on_each_node=False,
63
+ save_steps=500,
64
+ save_strategy=IntervalStrategy.STEPS,
65
+ save_total_limit=None,
66
+ seed=42,
67
+ sharded_ddp=[],
68
+ skip_memory_metrics=True,
69
+ tpu_metrics_debug=False,
70
+ tpu_num_cores=None,
71
+ use_legacy_prediction_loop=False,
72
+ warmup_ratio=0.0,
73
+ warmup_steps=1000,
74
+ weight_decay=0.0,
75
+ )
76
+ [16:04:24] - DEBUG - urllib3.connectionpool - Starting new HTTPS connection (1): s3.amazonaws.com:443
77
+ [16:04:24] - DEBUG - urllib3.connectionpool - https://s3.amazonaws.com:443 "HEAD /datasets.huggingface.co/datasets/datasets/oscar/oscar.py HTTP/1.1" 404 0
78
+ [16:04:24] - DEBUG - urllib3.connectionpool - Starting new HTTPS connection (1): raw.githubusercontent.com:443
79
+ [16:04:24] - DEBUG - urllib3.connectionpool - https://raw.githubusercontent.com:443 "HEAD /huggingface/datasets/master/datasets/oscar/oscar.py HTTP/1.1" 200 0
80
+ [16:04:24] - DEBUG - urllib3.connectionpool - Starting new HTTPS connection (1): raw.githubusercontent.com:443
81
+ [16:04:24] - DEBUG - urllib3.connectionpool - https://raw.githubusercontent.com:443 "HEAD /huggingface/datasets/master/datasets/oscar/dataset_infos.json HTTP/1.1" 200 0
82
+ [16:04:24] - WARNING - datasets.builder - Reusing dataset oscar (/home/nipunsadvilkar/.cache/huggingface/datasets/oscar/unshuffled_deduplicated_als/1.0.0/84838bd49d2295f62008383b05620571535451d84545037bb94d6f3501651df2)
83
+ [16:04:24] - DEBUG - urllib3.connectionpool - Starting new HTTPS connection (1): s3.amazonaws.com:443
84
+ [16:04:24] - DEBUG - urllib3.connectionpool - https://s3.amazonaws.com:443 "HEAD /datasets.huggingface.co/datasets/datasets/oscar/oscar.py HTTP/1.1" 404 0
85
+ [16:04:24] - DEBUG - urllib3.connectionpool - Starting new HTTPS connection (1): raw.githubusercontent.com:443
86
+ [16:04:24] - DEBUG - urllib3.connectionpool - https://raw.githubusercontent.com:443 "HEAD /huggingface/datasets/master/datasets/oscar/oscar.py HTTP/1.1" 200 0
87
+ [16:04:24] - DEBUG - urllib3.connectionpool - Starting new HTTPS connection (1): raw.githubusercontent.com:443
88
+ [16:04:24] - DEBUG - urllib3.connectionpool - https://raw.githubusercontent.com:443 "HEAD /huggingface/datasets/master/datasets/oscar/dataset_infos.json HTTP/1.1" 200 0
89
+ [16:04:24] - WARNING - datasets.builder - Reusing dataset oscar (/home/nipunsadvilkar/.cache/huggingface/datasets/oscar/unshuffled_deduplicated_als/1.0.0/84838bd49d2295f62008383b05620571535451d84545037bb94d6f3501651df2)
90
+ [16:04:24] - DEBUG - urllib3.connectionpool - Starting new HTTPS connection (1): s3.amazonaws.com:443
91
+ [16:04:25] - DEBUG - urllib3.connectionpool - https://s3.amazonaws.com:443 "HEAD /datasets.huggingface.co/datasets/datasets/oscar/oscar.py HTTP/1.1" 404 0
92
+ [16:04:25] - DEBUG - urllib3.connectionpool - Starting new HTTPS connection (1): raw.githubusercontent.com:443
93
+ [16:04:25] - DEBUG - urllib3.connectionpool - https://raw.githubusercontent.com:443 "HEAD /huggingface/datasets/master/datasets/oscar/oscar.py HTTP/1.1" 200 0
94
+ [16:04:25] - DEBUG - urllib3.connectionpool - Starting new HTTPS connection (1): raw.githubusercontent.com:443
95
+ [16:04:25] - DEBUG - urllib3.connectionpool - https://raw.githubusercontent.com:443 "HEAD /huggingface/datasets/master/datasets/oscar/dataset_infos.json HTTP/1.1" 200 0
96
+ [16:04:25] - WARNING - datasets.builder - Reusing dataset oscar (/home/nipunsadvilkar/.cache/huggingface/datasets/oscar/unshuffled_deduplicated_als/1.0.0/84838bd49d2295f62008383b05620571535451d84545037bb94d6f3501651df2)
97
+
98
  0%| | 0/5 [00:00<?, ?ba/s]
99
  20%|██ | 1/5 [00:00<00:00, 8.80ba/s]
100
  40%|████ | 2/5 [00:00<00:00, 9.16ba/s]
101
  60%|██████ | 3/5 [00:00<00:00, 9.52ba/s]
102
  80%|████████ | 4/5 [00:00<00:00, 7.13ba/s]
103
+
104
  0%| | 0/1 [00:00<?, ?ba/s]
105
+
106
  0%| | 0/5 [00:00<?, ?ba/s]
107
  20%|██ | 1/5 [00:00<00:03, 1.32ba/s]
108
  40%|████ | 2/5 [00:01<00:02, 1.24ba/s]
109
  60%|██████ | 3/5 [00:02<00:01, 1.21ba/s]
110
  80%|████████ | 4/5 [00:03<00:00, 1.13ba/s]
111
+
112
  0%| | 0/1 [00:00<?, ?ba/s]
113
+ [16:04:29] - WARNING - __main__ - Unable to display metrics through TensorBoard because the package is not installed: Please run pip install tensorboard to enable.
114
+ [16:04:29] - INFO - absl - Starting the local TPU driver.
115
+ [16:04:29] - INFO - absl - Unable to initialize backend 'tpu_driver': Not found: Unable to find driver in registry given worker: local://
116
+ [16:04:29] - INFO - absl - Unable to initialize backend 'gpu': Not found: Could not find registered platform with name: "cuda". Available platform names are: Interpreter TPU Host
117
+ /home/nipunsadvilkar/roberta_mr_env/lib/python3.8/site-packages/jax/lib/xla_bridge.py:382: UserWarning: jax.host_count has been renamed to jax.process_count. This alias will eventually be removed; please update your code.
118
+ warnings.warn(
119
+ /home/nipunsadvilkar/roberta_mr_env/lib/python3.8/site-packages/jax/lib/xla_bridge.py:369: UserWarning: jax.host_id has been renamed to jax.process_index. This alias will eventually be removed; please update your code.
120
+ warnings.warn(
121
+
122
+
123
+
124
+
125
+
126
+
127
+
128
+
129
+
130
+
131
+
132
+
133
+
134
+
135
+
136
+
137
+
138
+
139
+
140
+
141
+
142
+
143
+
144
+
145
+
146
+
147
+
148
+
149
+
150
+
151
+
152
+
153
+
154
+
155
+
156
+
157
+
158
+
159
+
160
+
161
+
162
+
163
+
164
+
165
+
166
+
167
+
168
+
169
+
170
+
171
+
172
+
173
+
174
+
175
+
176
+
177
+
178
+
179
+
180
+
181
+
182
+
183
+
184
+
185
+
186
+
187
+
188
+
189
+
190
+
191
+
192
+
193
+
194
+
195
+
196
+
197
+
198
+
199
+
200
+
201
+
202
+
203
+
204
+
205
+
206
+
207
+
208
+
209
+
210
+
211
+
212
+
213
+
214
+
215
+
216
+
217
+
218
+
219
+
220
+
221
+
222
+
223
+
224
+
225
+
226
+
227
+
228
+
229
+
230
+
231
+
232
+
233
+
234
+
235
+
236
+
237
+
238
+
239
+
240
+
241
+
242
+
243
+
244
+
245
+
246
+
247
+
248
+
249
+
250
+
251
+
252
+
253
+
254
 
255
+
256
 
257
+
258
+
259
+
260
+
261
+
262
+
263
+
264
+
265
+ To disable this warning, you can either:
266
+ - Avoid using `tokenizers` before the fork if possible
267
+ - Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)
268
+ huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...
269
+ To disable this warning, you can either:
270
+ - Avoid using `tokenizers` before the fork if possible
271
+ - Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)
272
+
273
+ [16:06:12] - INFO - huggingface_hub.repository - git version 2.25.1
274
+ git-lfs/2.9.2 (GitHub; linux amd64; go 1.13.5)
275
+ huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...
276
+ To disable this warning, you can either:
277
+ - Avoid using `tokenizers` before the fork if possible
278
+ - Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)
279
+ [16:06:12] - DEBUG - huggingface_hub.repository - [Repository] is a valid git repo
280
+ huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...
281
+ To disable this warning, you can either:
282
+ - Avoid using `tokenizers` before the fork if possible
283
+ - Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)
284
+ huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...
285
+ To disable this warning, you can either:
286
+ - Avoid using `tokenizers` before the fork if possible
287
+ - Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)
run.sh ADDED
@@ -0,0 +1,18 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ HUB_TOKEN=`cat $HOME/.huggingface/token`
2
+ python run_mlm_flax.py \
3
+ --output_dir="./" \
4
+ --model_type="roberta" \
5
+ --config_name="./" \
6
+ --tokenizer_name="./" \
7
+ --dataset_name="oscar" \
8
+ --dataset_config_name="unshuffled_deduplicated_als" \
9
+ --max_seq_length="128" \
10
+ --per_device_train_batch_size="4" \
11
+ --per_device_eval_batch_size="4" \
12
+ --learning_rate="3e-4" \
13
+ --warmup_steps="1000" \
14
+ --overwrite_output_dir \
15
+ --num_train_epochs="8" \
16
+ --push_to_hub_model_id="flax-community/roberta-base-mr" \
17
+ --push_to_hub_token="$HUB_TOKEN" \
18
+ --push_to_hub 2>&1 | tee run.log
tokens.py ADDED
@@ -0,0 +1,20 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ from datasets import load_dataset
3
+ from tokenizers import ByteLevelBPETokenizer
4
+ # Load dataset
5
+ dataset = load_dataset("oscar", "unshuffled_deduplicated_als", split="train")
6
+ # Instantiate tokenizer
7
+ tokenizer = ByteLevelBPETokenizer()
8
+ def batch_iterator(batch_size=100_000):
9
+ for i in range(0, len(dataset), batch_size):
10
+ yield dataset["text"][i: i + batch_size]
11
+ # Customized training
12
+ tokenizer.train_from_iterator(batch_iterator(), vocab_size=50265, min_frequency=2, special_tokens=[
13
+ "<s>",
14
+ "<pad>",
15
+ "</s>",
16
+ "<unk>",
17
+ "<mask>",
18
+ ])
19
+ # Save files to disk
20
+ tokenizer.save("./tokenizer.json")