Spaces:
Build error
Build error
PeteBleackley
commited on
Commit
·
7ac34f7
1
Parent(s):
3c28153
Another workaround
Browse files
qarac/corpora/CombinedCorpus.py
CHANGED
|
@@ -60,9 +60,9 @@ class CombinedCorpus(keras.utils.Sequence):
|
|
| 60 |
{},
|
| 61 |
'consistency'),
|
| 62 |
n_samples)
|
| 63 |
-
|
| 64 |
self.pad_token = tokenizer.token_to_id('<pad>')
|
| 65 |
-
|
| 66 |
|
| 67 |
def __len__(self):
|
| 68 |
"""
|
|
@@ -117,7 +117,7 @@ class CombinedCorpus(keras.utils.Sequence):
|
|
| 117 |
Y.update(y)
|
| 118 |
yield (X,Y)
|
| 119 |
|
| 120 |
-
def
|
| 121 |
batch = []
|
| 122 |
n=0
|
| 123 |
for sample in self.samples:
|
|
@@ -129,6 +129,9 @@ class CombinedCorpus(keras.utils.Sequence):
|
|
| 129 |
n=0
|
| 130 |
if n!=0:
|
| 131 |
yield batch
|
|
|
|
|
|
|
|
|
|
| 132 |
|
| 133 |
|
| 134 |
def batch(self,samples):
|
|
|
|
| 60 |
{},
|
| 61 |
'consistency'),
|
| 62 |
n_samples)
|
| 63 |
+
self.batches = None
|
| 64 |
self.pad_token = tokenizer.token_to_id('<pad>')
|
| 65 |
+
self.on_epoch_end()
|
| 66 |
|
| 67 |
def __len__(self):
|
| 68 |
"""
|
|
|
|
| 117 |
Y.update(y)
|
| 118 |
yield (X,Y)
|
| 119 |
|
| 120 |
+
def make_batches(self):
|
| 121 |
batch = []
|
| 122 |
n=0
|
| 123 |
for sample in self.samples:
|
|
|
|
| 129 |
n=0
|
| 130 |
if n!=0:
|
| 131 |
yield batch
|
| 132 |
+
|
| 133 |
+
def on_epoch_end(self):
|
| 134 |
+
self.batches = self.make_batches()
|
| 135 |
|
| 136 |
|
| 137 |
def batch(self,samples):
|