feat: alterações feitas com peixe em 28/01

_antigos/resumos/serializer.py

@@ -17,7 +17,7 @@ class ResumoPDFSerializer(serializers.Serializer):
     system_prompt = serializers.CharField(required=False)
     user_message = serializers.CharField(required=False, default="")
     model = serializers.CharField(required=False)
-    iterative_refinement = serializers.BooleanField(required=False, default=False)
+    iterative_refinement = serializers.BooleanField(required=False, default=False)  # type: ignore
 
 class ResumoCursorSerializer(serializers.Serializer):
     files = serializers.ListField(child=serializers.FileField(), required=True)

_utils/bubble_integrations/obter_arquivo.py

@@ -1,9 +1,11 @@
 # from setup.easy_imports import PyPDFLoader
 import os
 from langchain_community.document_loaders import PyPDFLoader
-
+import tempfile
 import requests
 
+from _utils.handle_files import return_document_list_with_llama_parser
+
 headers = {"Authorization": f"Bearer {os.environ.get("BUBBLE_TOKEN")}"}
 
 
@@ -14,10 +16,22 @@ headers = {"Authorization": f"Bearer {os.environ.get("BUBBLE_TOKEN")}"}
 #     )
 
 
-def get_pdf_from_bubble(
+async def get_pdf_from_bubble(
     file_url=f"https://vella.app.br/version-test/fileupload/f1735864316650x718601440484441900/Boleto_DIGITICS%20Servic%CC%A7os%20de%20Secretariado%20LTDA_30_12_2024_804841714.pdf",
+    should_use_llama_parse=False,
 ):
-
-    result = PyPDFLoader(file_url, headers=headers)
-
-    return result.load()
+    if should_use_llama_parse:
+        response = requests.get(file_url, headers=headers)
+        response.raise_for_status()
+
+        # Create a NamedTemporaryFile (with a .pdf suffix) to store the file
+        with tempfile.NamedTemporaryFile(delete=False, suffix=".pdf") as tmp_file:
+            # Write the response content to the temporary file
+            tmp_file.write(response.content)
+            return await return_document_list_with_llama_parser(
+                tmp_file.name
+            )  # por enquanto este arquivo não está sendo excluído
+    else:
+        result = PyPDFLoader(file_url, headers=headers)
+
+        return result.load()

_utils/gerar_relatorio_modelo_usuario/EnhancedDocumentSummarizer.py

@@ -1,5 +1,7 @@
 import os
 from typing import List, Dict, Tuple, Optional
+
+from pydantic import SecretStr
 from _utils.vector_stores.Vector_store_class import VectorStore
 from setup.easy_imports import (
     Chroma,
@@ -25,11 +27,11 @@ from asgiref.sync import sync_to_async
 
 
 class EnhancedDocumentSummarizer(DocumentSummarizer):
+    openai_api_key = os.environ.get("OPENAI_API_KEY", "")
+    claude_api_key = os.environ.get("CLAUDE_API_KEY", "")
 
     def __init__(
         self,
-        openai_api_key: str,
-        claude_api_key: str,
         config: RetrievalConfig,
         embedding_model,
         chunk_size,
@@ -45,8 +47,8 @@ class EnhancedDocumentSummarizer(DocumentSummarizer):
         reciprocal_rank_fusion,
     ):
         super().__init__(
-            openai_api_key,
-            os.environ.get("COHERE_API_KEY"),
+            self.openai_api_key,
+            os.environ.get("COHERE_API_KEY", ""),
             embedding_model,
             chunk_size,
             chunk_overlap,
@@ -55,7 +57,7 @@ class EnhancedDocumentSummarizer(DocumentSummarizer):
         )
         self.config = config
         self.contextual_retriever = ContextualRetriever(
-            config, claude_api_key, claude_context_model
+            config, self.claude_api_key, claude_context_model
         )
         self.logger = logging.getLogger(__name__)
         self.prompt_auxiliar = prompt_auxiliar
@@ -208,8 +210,8 @@ class EnhancedDocumentSummarizer(DocumentSummarizer):
 
             llm = ChatOpenAI(
                 temperature=self.gpt_temperature,
-                model_name=self.gpt_model,
-                api_key=self.openai_api_key,
+                model=self.gpt_model,
+                api_key=SecretStr(self.openai_api_key),
             )
 
             prompt_auxiliar = PromptTemplate(

_utils/gerar_relatorio_modelo_usuario/contextual_retriever.py

@@ -165,6 +165,9 @@ class ContextualRetriever:
         ]
         # print("\n\nresult", result)
 
+        if result == "" or result == [""]:
+            print("\n\ncontext", context)
+
         lista_chunks = []
         for index, chunk in enumerate(chunks):
             lista_chunks.append(
@@ -259,6 +262,7 @@ async def get_full_text_and_all_PDFs_chunks(
     listaPDFs: List[str],
     splitterObject: Splitter,
     should_use_llama_parse: bool,
+    isBubble: bool,
 ):
     all_PDFs_chunks = []
 
@@ -266,10 +270,14 @@ async def get_full_text_and_all_PDFs_chunks(
 
     # Load and process document
     for pdf_path in listaPDFs:
-        if should_use_llama_parse:
-            pages = pages + await return_document_list_with_llama_parser(pdf_path)
+        if isBubble:
+            pages = pages + await get_pdf_from_bubble(pdf_path, should_use_llama_parse)
         else:
-            pages = pages + get_pdf_from_bubble(pdf_path)
+            if should_use_llama_parse:
+                pages = pages + await return_document_list_with_llama_parser(pdf_path)
+            else:
+                pages = pages + PyPDFLoader(pdf_path).load()
+
         chunks = splitterObject.load_and_split_document(
             pdf_path, pages, should_use_llama_parse
         )

_utils/gerar_relatorio_modelo_usuario/prompts.py

@@ -180,10 +180,10 @@ Here are the 20 chunks to analyze:
 </user_input>
 Example output structure (do not copy the content, only the format):
 1. <chunk_context>
-[document_id] --- [document_title] --- [brief_context_for_the_chunk]
+[128372552] --- [Certidão de Ocorrência] --- [This chunk concludes the victim's statement, confirming the details of the robbery and his signature on the report. It aligns with the summary's focus on the victim's testimony as a key piece of evidence in the case.]
 </chunk_context>
-2.<chunk_context>
-[document_id] --- [document_title] --- [brief_context_for_the_chunk]
+2. <chunk_context>
+[158665972] --- [Certidão de Antecedentes] --- [This chunk includes the certification by the police clerk, Anatalino Lima Farias, confirming the accuracy of the transcribed report. It supports the summary's reliance on official police records to document the crime and its investigation.]
 </chunk_context>
 [Continue for all 20 chunks]
 Please provide context for all 20 chunks, following this structure. It's OK for this section to be quite long.

_utils/handle_files.py

@@ -2,7 +2,6 @@ import tempfile, os
 from typing import List
 from langchain_core.documents import Document as LangchainDocument
 from llama_index import Document
-from _utils.bubble_integrations.obter_arquivo import get_pdf_from_bubble
 from llama_parse import LlamaParse, ResultType
 
 

_utils/resumo_completo_cursor.py

@@ -38,7 +38,9 @@ os.environ.get("LANGCHAIN_API_KEY")
 os.environ["LANGCHAIN_PROJECT"] = "VELLA"
 
 
-async def get_llm_summary_answer_by_cursor_complete(serializer, listaPDFs=None):
+async def get_llm_summary_answer_by_cursor_complete(
+    serializer, listaPDFs=None, isBubble=False
+):
     """Parâmetro "contexto" só deve ser passado quando quiser utilizar o teste com ragas, e assim, não quiser passar PDFs"""
     # Configuration
     config = RetrievalConfig(
@@ -51,8 +53,6 @@ async def get_llm_summary_answer_by_cursor_complete(serializer, listaPDFs=None):
 
     # Initialize enhanced summarizer
     summarizer = EnhancedDocumentSummarizer(
-        openai_api_key=os.environ.get("OPENAI_API_KEY"),
-        claude_api_key=os.environ.get("CLAUDE_API_KEY"),
         config=config,
         embedding_model=serializer["hf_embedding"],
         chunk_overlap=serializer["chunk_overlap"],
@@ -69,7 +69,7 @@ async def get_llm_summary_answer_by_cursor_complete(serializer, listaPDFs=None):
     )
 
     allPdfsChunks, pages = await get_full_text_and_all_PDFs_chunks(
-        listaPDFs, summarizer.splitter, serializer["should_use_llama_parse"]
+        listaPDFs, summarizer.splitter, serializer["should_use_llama_parse"], isBubble
     )
 
     chunks_passados, is_contextualized_chunk = (

_utils/vector_stores/Vector_store_class.py

@@ -1,3 +1,4 @@
+from pydoc import text
 from typing import List, Dict, Tuple, Optional
 from _utils.models.gerar_relatorio import (
     ContextualizedChunk,
@@ -19,26 +20,38 @@ class VectorStore:
         try:
             # Prepare texts with context
             if is_contextualized_chunk:
-                texts = [f"{chunk.context} {chunk.content}" for chunk in chunks]
+                texts = [
+                    f"Document_id: {chunk.chunk_id}\nDocument_context: {chunk.context}\nDocument_content: {chunk.content}"
+                    for chunk in chunks
+                ]
             else:
                 texts = [f"{chunk.content}" for chunk in chunks]
 
             # Create vector store
             metadatas = []
-            for chunk in chunks:
+            for index, chunk in enumerate(chunks):
                 if is_contextualized_chunk:
-                    context = chunk.context
+                    context = texts[index]
+                    metadatas.append(
+                        {
+                            "chunk_id": chunk.chunk_id,
+                            "page": chunk.page_number,
+                            "start_char": chunk.start_char,
+                            "end_char": chunk.end_char,
+                            "context": context,
+                        }
+                    )
                 else:
-                    context = ""
-                metadatas.append(
-                    {
-                        "chunk_id": chunk.chunk_id,
-                        "page": chunk.page_number,
-                        "start_char": chunk.start_char,
-                        "end_char": chunk.end_char,
-                        "context": context,
-                    }
-                )
+                    context = texts[index]
+                    metadatas.append(
+                        {
+                            "chunk_id": chunk.chunk_id,
+                            "page": chunk.page_number,
+                            "start_char": chunk.start_char,
+                            "end_char": chunk.end_char,
+                            "context": context,
+                        }
+                    )
 
             vector_store = Chroma.from_texts(
                 texts=texts, metadatas=metadatas, embedding=self.embeddings

gerar_documento/serializer.py

@@ -29,16 +29,16 @@ class GerarDocumentoSerializer(ResumoCursorSerializer):
         required=False, default=prompt_gerar_documento
     )
     user_message = serializers.CharField(required=False, default=user_message)
-    num_chunks_retrieval = serializers.IntegerField(default=5)
+    num_chunks_retrieval = serializers.IntegerField(default=20)
     embedding_weight = serializers.FloatField(default=0.5)
     bm25_weight = serializers.FloatField(default=0.5)
     context_window = serializers.IntegerField(default=3)
     chunk_overlap = serializers.IntegerField(default=200)
-    num_k_rerank = serializers.IntegerField(default=5)
+    num_k_rerank = serializers.IntegerField(default=20)
     model_cohere_rerank = serializers.CharField(
         required=False, default="rerank-english-v2.0"
     )
-    more_initial_chunks_for_reranking = serializers.IntegerField(default=20)
+    more_initial_chunks_for_reranking = serializers.IntegerField(default=100)
     claude_context_model = serializers.CharField(
         required=False, default="claude-3-haiku-20240307"
     )
@@ -57,16 +57,16 @@ class GerarDocumentoComPDFProprioSerializer(ResumoCursorSerializer):
         required=False, default=prompt_gerar_documento
     )
     user_message = serializers.CharField(required=False, default=user_message)
-    num_chunks_retrieval = serializers.IntegerField(default=5)
+    num_chunks_retrieval = serializers.IntegerField(default=20)
     embedding_weight = serializers.FloatField(default=0.5)
     bm25_weight = serializers.FloatField(default=0.5)
     context_window = serializers.IntegerField(default=3)
     chunk_overlap = serializers.IntegerField(default=200)
-    num_k_rerank = serializers.IntegerField(default=5)
+    num_k_rerank = serializers.IntegerField(default=20)
     model_cohere_rerank = serializers.CharField(
         required=False, default="rerank-english-v2.0"
     )
-    more_initial_chunks_for_reranking = serializers.IntegerField(default=20)
+    more_initial_chunks_for_reranking = serializers.IntegerField(default=100)
     claude_context_model = serializers.CharField(
         required=False, default="claude-3-haiku-20240307"
     )

gerar_documento/views.py

@@ -43,7 +43,7 @@ class GerarDocumentoView(AsyncAPIView):
             print("\n\nlistaPDFs: ", listaPDFs)
 
             resposta_llm = await get_llm_summary_answer_by_cursor_complete(
-                data, listaPDFs
+                data, listaPDFs, True
             )
             print("\n\nresposta_llm: ", resposta_llm)
 

setup/environment.py

@@ -3,3 +3,4 @@ default_model = "gpt-4o-mini"
 
 api_url = "https://luanpoppe-vella-backend.hf.space"
 # api_url = "http://localhost:8000"
+