Spaces:
Runtime error
Runtime error
add * to glob
Browse files
utils.py
CHANGED
@@ -111,7 +111,7 @@ def load_hf_dataset(ds_name: str, ds_config: str = None, ds_split: str = "train"
|
|
111 |
if ds_name == "wikipedia":
|
112 |
pattern = re.compile(r"[^a-zA-Z0-9]")
|
113 |
folder = Path("/data") / pattern.sub("", ds_name+ds_config)
|
114 |
-
files = list(map(str, folder.glob("chunk_")))
|
115 |
|
116 |
return load_dataset("parquet", data_files=files, split="train")
|
117 |
|
|
|
111 |
if ds_name == "wikipedia":
|
112 |
pattern = re.compile(r"[^a-zA-Z0-9]")
|
113 |
folder = Path("/data") / pattern.sub("", ds_name+ds_config)
|
114 |
+
files = list(map(str, folder.glob("chunk_*")))
|
115 |
|
116 |
return load_dataset("parquet", data_files=files, split="train")
|
117 |
|