Spaces:
Sleeping
Sleeping
xavierbarbier
commited on
Update app.py
Browse files
app.py
CHANGED
@@ -30,7 +30,7 @@ print("Start the model init process")
|
|
30 |
model = model = GPT4All(model_name, model_path, allow_download = False, device="cpu")
|
31 |
|
32 |
|
33 |
-
|
34 |
# creating a pdf reader object
|
35 |
reader = PdfReader("./resource/NGAP 01042024.pdf")
|
36 |
text = []
|
@@ -44,7 +44,7 @@ text = ' '.join(text)
|
|
44 |
|
45 |
chunk_size = 2048
|
46 |
chunks = [text[i:i + chunk_size] for i in range(0, len(text), chunk_size)]
|
47 |
-
|
48 |
model_kwargs = {'device': 'cpu'}
|
49 |
encode_kwargs = {'normalize_embeddings': False}
|
50 |
embeddings = HuggingFaceEmbeddings(
|
@@ -56,13 +56,15 @@ embeddings = HuggingFaceEmbeddings(
|
|
56 |
def get_text_embedding(text):
|
57 |
|
58 |
return embeddings.embed_query(text)
|
|
|
|
|
59 |
text_embeddings = np.array([get_text_embedding(chunk) for chunk in chunks])
|
60 |
|
61 |
d = text_embeddings.shape[1]
|
62 |
index = faiss.IndexFlatL2(d)
|
63 |
index.add(text_embeddings)
|
64 |
-
|
65 |
-
|
66 |
|
67 |
print("Finish the model init process")
|
68 |
|
@@ -74,7 +76,7 @@ max_new_tokens = 2048
|
|
74 |
|
75 |
|
76 |
|
77 |
-
|
78 |
|
79 |
|
80 |
|
|
|
30 |
model = model = GPT4All(model_name, model_path, allow_download = False, device="cpu")
|
31 |
|
32 |
|
33 |
+
"""
|
34 |
# creating a pdf reader object
|
35 |
reader = PdfReader("./resource/NGAP 01042024.pdf")
|
36 |
text = []
|
|
|
44 |
|
45 |
chunk_size = 2048
|
46 |
chunks = [text[i:i + chunk_size] for i in range(0, len(text), chunk_size)]
|
47 |
+
"""
|
48 |
model_kwargs = {'device': 'cpu'}
|
49 |
encode_kwargs = {'normalize_embeddings': False}
|
50 |
embeddings = HuggingFaceEmbeddings(
|
|
|
56 |
def get_text_embedding(text):
|
57 |
|
58 |
return embeddings.embed_query(text)
|
59 |
+
|
60 |
+
"""
|
61 |
text_embeddings = np.array([get_text_embedding(chunk) for chunk in chunks])
|
62 |
|
63 |
d = text_embeddings.shape[1]
|
64 |
index = faiss.IndexFlatL2(d)
|
65 |
index.add(text_embeddings)
|
66 |
+
"""
|
67 |
+
index = faiss.read_index("./resourse/embeddings_ngap.faiss")
|
68 |
|
69 |
print("Finish the model init process")
|
70 |
|
|
|
76 |
|
77 |
|
78 |
|
79 |
+
|
80 |
|
81 |
|
82 |
|