birgermoell
commited on
Commit
•
774265c
1
Parent(s):
84879fe
Saving weights and logs of step 29001
Browse files
combinedataset.py
ADDED
@@ -0,0 +1,5 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
from datasets import interleave_datasets
|
2 |
+
from itertools import islice
|
3 |
+
en_dataset = load_dataset('oscar', "unshuffled_deduplicated_en", split='train', streaming=True)
|
4 |
+
fr_dataset = load_dataset('oscar', "unshuffled_deduplicated_fr", split='train', streaming=True)
|
5 |
+
multilingual_dataset = interleave_datasets([en_dataset, fr_dataset])
|
events.out.tfevents.1625538240.t1v-n-98937c84-w-0.291128.3.v2
CHANGED
@@ -1,3 +1,3 @@
|
|
1 |
version https://git-lfs.github.com/spec/v1
|
2 |
-
oid sha256:
|
3 |
-
size
|
|
|
1 |
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:3bf9bb24193646870ceb7612d94b3be3369614c52bfea2f0f037cfa15e4e0cc1
|
3 |
+
size 4300302
|
flax_model.msgpack
CHANGED
@@ -1,3 +1,3 @@
|
|
1 |
version https://git-lfs.github.com/spec/v1
|
2 |
-
oid sha256:
|
3 |
size 498796983
|
|
|
1 |
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:b45a9cbb800f47fbedaa15be1e2eb324964513b63c52bc7bc52131418631ffae
|
3 |
size 498796983
|