Update app.py

app.py

@@ -14,6 +14,26 @@ __model_output_path = "paavansundar/Medical_QNA_GPT2"
 #prepare data
 def prepareData():
     df=pd.read_csv("MedQuAD.csv")
+    df['Question']=df['Question'].replace(r'^\s*$', np.nan, regex=True)
+    df['Answer']=df['Answer'].replace(r'^\s*$', np.nan, regex=True)
+    df = df.drop_duplicates(subset=['Question', 'Answer'])
+    df=df.dropna()
+    train_ds=df.groupby('Focus').head(100)
+    train_ds=train_ds.groupby('Focus').head(4).reset_index(drop=True)
+    test_ds=train_ds.groupby('Focus').head(1).reset_index(drop=True)
+    train_seq=list()
+    for i in range(len(train_ds)):
+      s='<question>'+train_ds.loc[i,'Question']+'<answer>'+train_ds.loc[i,'Answer']
+      train_seq.append(s)
+    val_seq=list()
+    for i in range(len(test_ds)):
+      s='<question>'+test_ds.loc[i,'Question']+'<answer>'+test_ds.loc[i,'Answer']
+      val_seq.append(s)
+    with open("train.txt", "w") as f:
+        f.writelines(line+'\n' for line in train_seq)
+
+    with open("val.txt", "w") as f:
+        f.writelines(line+'\n' for line in val_seq)
     
 # Create a Data collator object
 data_collator = DataCollatorForLanguageModeling(tokenizer=__tokenizer, mlm=False, return_tensors="pt")