updated minimal line length
Browse files
text_collection/text_collection.py
CHANGED
@@ -110,7 +110,7 @@ class TextCollection(datasets.GeneratorBasedBuilder):
|
|
110 |
with open(path, "r") as f:
|
111 |
print("# Reading", path)
|
112 |
for line in f:
|
113 |
-
if len(line.strip()) >
|
114 |
current_lines.append(line)
|
115 |
elif current_lines:
|
116 |
feature = id_, {"id": id_, "text": "".join(current_lines).rstrip()}
|
|
|
110 |
with open(path, "r") as f:
|
111 |
print("# Reading", path)
|
112 |
for line in f:
|
113 |
+
if len(line.strip()) > -1:
|
114 |
current_lines.append(line)
|
115 |
elif current_lines:
|
116 |
feature = id_, {"id": id_, "text": "".join(current_lines).rstrip()}
|