nbroad HF staff commited on
Commit
2492d76
1 Parent(s): f500dc6

add * to glob

Browse files
Files changed (1) hide show
  1. utils.py +1 -1
utils.py CHANGED
@@ -111,7 +111,7 @@ def load_hf_dataset(ds_name: str, ds_config: str = None, ds_split: str = "train"
111
  if ds_name == "wikipedia":
112
  pattern = re.compile(r"[^a-zA-Z0-9]")
113
  folder = Path("/data") / pattern.sub("", ds_name+ds_config)
114
- files = list(map(str, folder.glob("chunk_")))
115
 
116
  return load_dataset("parquet", data_files=files, split="train")
117
 
 
111
  if ds_name == "wikipedia":
112
  pattern = re.compile(r"[^a-zA-Z0-9]")
113
  folder = Path("/data") / pattern.sub("", ds_name+ds_config)
114
+ files = list(map(str, folder.glob("chunk_*")))
115
 
116
  return load_dataset("parquet", data_files=files, split="train")
117