xavierbarbier commited on
Commit
3ad6441
·
verified ·
1 Parent(s): 5f20a11

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +7 -5
app.py CHANGED
@@ -30,7 +30,7 @@ print("Start the model init process")
30
  model = model = GPT4All(model_name, model_path, allow_download = False, device="cpu")
31
 
32
 
33
-
34
  # creating a pdf reader object
35
  reader = PdfReader("./resource/NGAP 01042024.pdf")
36
  text = []
@@ -44,7 +44,7 @@ text = ' '.join(text)
44
 
45
  chunk_size = 2048
46
  chunks = [text[i:i + chunk_size] for i in range(0, len(text), chunk_size)]
47
-
48
  model_kwargs = {'device': 'cpu'}
49
  encode_kwargs = {'normalize_embeddings': False}
50
  embeddings = HuggingFaceEmbeddings(
@@ -56,13 +56,15 @@ embeddings = HuggingFaceEmbeddings(
56
  def get_text_embedding(text):
57
 
58
  return embeddings.embed_query(text)
 
 
59
  text_embeddings = np.array([get_text_embedding(chunk) for chunk in chunks])
60
 
61
  d = text_embeddings.shape[1]
62
  index = faiss.IndexFlatL2(d)
63
  index.add(text_embeddings)
64
-
65
-
66
 
67
  print("Finish the model init process")
68
 
@@ -74,7 +76,7 @@ max_new_tokens = 2048
74
 
75
 
76
 
77
- #index = faiss.load_local("resourse/embeddings_ngap.faiss")
78
 
79
 
80
 
 
30
  model = model = GPT4All(model_name, model_path, allow_download = False, device="cpu")
31
 
32
 
33
+ """
34
  # creating a pdf reader object
35
  reader = PdfReader("./resource/NGAP 01042024.pdf")
36
  text = []
 
44
 
45
  chunk_size = 2048
46
  chunks = [text[i:i + chunk_size] for i in range(0, len(text), chunk_size)]
47
+ """
48
  model_kwargs = {'device': 'cpu'}
49
  encode_kwargs = {'normalize_embeddings': False}
50
  embeddings = HuggingFaceEmbeddings(
 
56
  def get_text_embedding(text):
57
 
58
  return embeddings.embed_query(text)
59
+
60
+ """
61
  text_embeddings = np.array([get_text_embedding(chunk) for chunk in chunks])
62
 
63
  d = text_embeddings.shape[1]
64
  index = faiss.IndexFlatL2(d)
65
  index.add(text_embeddings)
66
+ """
67
+ index = faiss.read_index("./resourse/embeddings_ngap.faiss")
68
 
69
  print("Finish the model init process")
70
 
 
76
 
77
 
78
 
79
+
80
 
81
 
82