Nguyen Ba Hung commited on
Commit ·
ba44ab9
1
Parent(s): 6e77184
change retrieval
Browse files- core/api/server.py +1 -1
- core/api/static/index.html +16 -0
- core/rag/retrieval.py +2 -2
- requirements.txt +8 -3
core/api/server.py
CHANGED
|
@@ -32,7 +32,7 @@ from core.rag.retrieval import Retriever, RetrievalMode, get_retrieval_config
|
|
| 32 |
from core.rag.generator import RAGContextBuilder, SYSTEM_PROMPT
|
| 33 |
|
| 34 |
# Config
|
| 35 |
-
RETRIEVAL_MODE = RetrievalMode.
|
| 36 |
RETRIEVAL_CFG = get_retrieval_config()
|
| 37 |
LLM_MODEL = os.getenv("LLM_MODEL", "qwen/qwen3-32b")
|
| 38 |
LLM_API_BASE = "https://api.groq.com/openai/v1"
|
|
|
|
| 32 |
from core.rag.generator import RAGContextBuilder, SYSTEM_PROMPT
|
| 33 |
|
| 34 |
# Config
|
| 35 |
+
RETRIEVAL_MODE = RetrievalMode.HYBRID
|
| 36 |
RETRIEVAL_CFG = get_retrieval_config()
|
| 37 |
LLM_MODEL = os.getenv("LLM_MODEL", "qwen/qwen3-32b")
|
| 38 |
LLM_API_BASE = "https://api.groq.com/openai/v1"
|
core/api/static/index.html
CHANGED
|
@@ -115,10 +115,26 @@
|
|
| 115 |
|
| 116 |
// 2. Bảo vệ các công thức toán (MathJax) khỏi bị lỗi khi render Markdown
|
| 117 |
let mathBlocks = [];
|
|
|
|
|
|
|
| 118 |
text = text.replace(/\$\$([\s\S]*?)\$\$/g, function(match) {
|
| 119 |
mathBlocks.push(match);
|
| 120 |
return `__MATH_BLOCK_${mathBlocks.length - 1}__`;
|
| 121 |
});
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 122 |
|
| 123 |
// 3. Render Markdown cơ bản
|
| 124 |
let html = text
|
|
|
|
| 115 |
|
| 116 |
// 2. Bảo vệ các công thức toán (MathJax) khỏi bị lỗi khi render Markdown
|
| 117 |
let mathBlocks = [];
|
| 118 |
+
|
| 119 |
+
// Protect block math: $$...$$ and \[...\]
|
| 120 |
text = text.replace(/\$\$([\s\S]*?)\$\$/g, function(match) {
|
| 121 |
mathBlocks.push(match);
|
| 122 |
return `__MATH_BLOCK_${mathBlocks.length - 1}__`;
|
| 123 |
});
|
| 124 |
+
text = text.replace(/\\\[([\s\S]*?)\\\]/g, function(match) {
|
| 125 |
+
mathBlocks.push(match);
|
| 126 |
+
return `__MATH_BLOCK_${mathBlocks.length - 1}__`;
|
| 127 |
+
});
|
| 128 |
+
|
| 129 |
+
// Protect inline math: \(...\) and $...$
|
| 130 |
+
text = text.replace(/\\\(([\s\S]*?)\\\)/g, function(match) {
|
| 131 |
+
mathBlocks.push(match);
|
| 132 |
+
return `__MATH_BLOCK_${mathBlocks.length - 1}__`;
|
| 133 |
+
});
|
| 134 |
+
text = text.replace(/\$([^\$\n]+?)\$/g, function(match) {
|
| 135 |
+
mathBlocks.push(match);
|
| 136 |
+
return `__MATH_BLOCK_${mathBlocks.length - 1}__`;
|
| 137 |
+
});
|
| 138 |
|
| 139 |
// 3. Render Markdown cơ bản
|
| 140 |
let html = text
|
core/rag/retrieval.py
CHANGED
|
@@ -32,8 +32,8 @@ class RetrievalMode(str, Enum):
|
|
| 32 |
class RetrievalConfig:
|
| 33 |
rerank_api_base_url: str = "https://api.siliconflow.com/v1"
|
| 34 |
rerank_model: str = "Qwen/Qwen3-Reranker-8B"
|
| 35 |
-
rerank_top_n: int =
|
| 36 |
-
initial_k: int =
|
| 37 |
top_k: int = 5
|
| 38 |
vector_weight: float = 0.5
|
| 39 |
bm25_weight: float = 0.5
|
|
|
|
| 32 |
class RetrievalConfig:
|
| 33 |
rerank_api_base_url: str = "https://api.siliconflow.com/v1"
|
| 34 |
rerank_model: str = "Qwen/Qwen3-Reranker-8B"
|
| 35 |
+
rerank_top_n: int = 5
|
| 36 |
+
initial_k: int = 50
|
| 37 |
top_k: int = 5
|
| 38 |
vector_weight: float = 0.5
|
| 39 |
bm25_weight: float = 0.5
|
requirements.txt
CHANGED
|
@@ -3,6 +3,8 @@ langchain==1.2.0
|
|
| 3 |
langchain-chroma==1.1.0
|
| 4 |
langchain-openai==1.1.6
|
| 5 |
chromadb==1.3.7
|
|
|
|
|
|
|
| 6 |
|
| 7 |
# LLM & Embeddings
|
| 8 |
groq==0.37.1
|
|
@@ -23,9 +25,12 @@ fastapi==0.115.12
|
|
| 23 |
uvicorn==0.34.2
|
| 24 |
|
| 25 |
# Evaluation
|
| 26 |
-
ragas==0.4.2
|
| 27 |
-
datasets==4.4.1
|
| 28 |
|
| 29 |
# Utils
|
| 30 |
requests==2.32.5
|
| 31 |
-
huggingface-hub==0.36.0
|
|
|
|
|
|
|
|
|
|
|
|
| 3 |
langchain-chroma==1.1.0
|
| 4 |
langchain-openai==1.1.6
|
| 5 |
chromadb==1.3.7
|
| 6 |
+
langchain-classic==1.0.3
|
| 7 |
+
langchain_community==0.4.1
|
| 8 |
|
| 9 |
# LLM & Embeddings
|
| 10 |
groq==0.37.1
|
|
|
|
| 25 |
uvicorn==0.34.2
|
| 26 |
|
| 27 |
# Evaluation
|
| 28 |
+
# ragas==0.4.2
|
| 29 |
+
# datasets==4.4.1
|
| 30 |
|
| 31 |
# Utils
|
| 32 |
requests==2.32.5
|
| 33 |
+
huggingface-hub==0.36.0
|
| 34 |
+
|
| 35 |
+
# Ocr
|
| 36 |
+
docling
|