Spaces:
Sleeping
Sleeping
Commit
•
bfcacbc
1
Parent(s):
2992a41
add options to turn features off
Browse files- dataset_loading.py +1 -1
dataset_loading.py
CHANGED
@@ -26,7 +26,7 @@ def load_local_corpus(corpus_file, columns_to_combine=["title", "text"]):
|
|
26 |
if idx == 0 and "doc_id" in line:
|
27 |
continue
|
28 |
inst = json.loads(line)
|
29 |
-
all_text = " ".join([inst[col] for col in columns_to_combine if col in inst])
|
30 |
if id_key not in inst:
|
31 |
id_key = "doc_id"
|
32 |
did2text[inst[id_key]] = {
|
|
|
26 |
if idx == 0 and "doc_id" in line:
|
27 |
continue
|
28 |
inst = json.loads(line)
|
29 |
+
all_text = " ".join([inst[col] for col in columns_to_combine if col in inst and inst[col] is not None])
|
30 |
if id_key not in inst:
|
31 |
id_key = "doc_id"
|
32 |
did2text[inst[id_key]] = {
|