Update app.py

app.py

@@ -7,13 +7,24 @@ from transformers import AutoTokenizer
 import pickle
 import os
 
-github_url = "https://github.com/TheMITTech/shakespeare"
+from glob import glob
 
-with open('shakespeare.pkl', 'wb') as fp:
-    pickle.dump(github_url, fp)
+files = glob("./shakespeare/**/*.html")
 
-with open('shakespeare.pkl', 'rb') as fp:
-    data = pickle.load(fp)
+import shutil
+import os
+
+os.mkdir('./data')
+destination_folder = './data/'
+
+for html_file in files:
+  shutil.move(html_file, destination_folder + html_file.split("/")[-1])
+
+from langchain.document_loaders import BSHTMLLoader, DirectoryLoader
+
+bshtml_dir_loader = DirectoryLoader('./data/', loader_cls=BSHTMLLoader)
+
+data = bshtml_dir_loader.load()
 
 bloomz_tokenizer = AutoTokenizer.from_pretrained('bigscience/bloomz-1b7')