add options to turn features off

dataset_loading.py

@@ -26,7 +26,7 @@ def load_local_corpus(corpus_file, columns_to_combine=["title", "text"]):
                 if idx == 0 and "doc_id" in line:
                     continue
                 inst = json.loads(line)
-            all_text = " ".join([inst[col] for col in columns_to_combine if col in inst])
+            all_text = " ".join([inst[col] for col in columns_to_combine if col in inst and inst[col] is not None])
             if id_key not in inst:
                 id_key = "doc_id"
             did2text[inst[id_key]] = {