Create train.py

train.py

@@ -0,0 +1,41 @@
+from langchain.document_loaders import  PyPDFLoader
+from langchain_together.embeddings import TogetherEmbeddings
+import faiss
+import os
+import time
+import numpy as np
+import pickle
+
+os.environ["TOGETHER_API_KEY"] = st.secrets["together_api_key"]
+embeddings = TogetherEmbeddings(model="togethercomputer/m2-bert-80M-8k-retrieval")
+
+loader = PyPDFLoader("ship.pdf")
+data = loader.load()
+print (f'You have {len(data)} document(s) in your data')
+print (f'There are {len(data[0].page_content)} characters in your sample document')
+print (f'Here is a sample: {data[0].page_content}')
+
+list_of_texts = []
+list_of_embeddings = []
+for val in data:
+    text_content = val.page_content
+    list_of_texts.append(text_content)
+    embedding_vector = embeddings.embed_query(text_content)
+    list_of_embeddings.append(embedding_vector)
+
+
+embeddings_array = np.array(list_of_embeddings).astype('float32')
+d = len(list_of_embeddings[0])
+index = faiss.IndexFlatL2(d)
+index.add(embeddings_array)
+
+# Save the index
+faiss.write_index(index, "faiss.index")
+# Save the list of texts
+with open("list_of_texts.pkl", 'wb') as f:
+    pickle.dump(list_of_texts, f)
+
+
+
+
+