dev/add-metadata-url (#1)

- update metadata, url and category (f35d932cb20a039ab283571c1ec0c97663948246)

app.py

@@ -5,7 +5,7 @@ from langchain.llms import OpenAI
 from langchain.vectorstores import Qdrant
 from openai.error import InvalidRequestError
 from qdrant_client import QdrantClient
-from config import get_db_config
+from config import DB_CONFIG
 
 
 PERSIST_DIR_NAME = "nvdajp-book"
@@ -13,7 +13,7 @@ PERSIST_DIR_NAME = "nvdajp-book"
 
 def get_retrieval_qa() -> RetrievalQA:
     embeddings = OpenAIEmbeddings()
-    db_url, db_api_key, db_collection_name = get_db_config()
+    db_url, db_api_key, db_collection_name = DB_CONFIG
     client = QdrantClient(url=db_url, api_key=db_api_key)
     db = Qdrant(client=client, collection_name=db_collection_name, embeddings=embeddings)
     retriever = db.as_retriever()
@@ -22,21 +22,17 @@ def get_retrieval_qa() -> RetrievalQA:
     )
 
 
-def _remove_prefix_path(p: str):
-    prefix = "data/rtdocs/nvdajp-book.readthedocs.io/"
-    return p.removeprefix(prefix)
-
-
 def get_related_url(metadata):
-    path = set()
-    url = "https://nvdajp-book.readthedocs.io/"
+    urls = set()
     for m in metadata:
-        p = m['source']
-        pathname = _remove_prefix_path(p)
-        if pathname in path:
+        # p = m['source']
+        url = m["url"]
+        if url in urls:
             continue
-        path.add(pathname)
-        yield f'<p>url: <a href="{url}{pathname}">{pathname}</a></p>'
+        urls.add(url)
+        category = m["category"]
+        # print(m)
+        yield f'<p>URL: <a href="{url}">{url}</a> (category: {category})</p>'
 
 
 def main(query: str):

config.py

@@ -1,8 +1,21 @@
 import os
 
 
+SAAS = True
+
+
 def get_db_config():
     url = os.environ["QDRANT_URL"]
     api_key = os.environ["QDRANT_API_KEY"]
     collection_name = "nvdajp-book"
     return url, api_key, collection_name
+
+
+def get_local_db_congin():
+    url = "localhost"
+    # api_key = os.environ["QDRANT_API_KEY"]
+    collection_name = "nvdajp-book"
+    return url, None, collection_name
+
+
+DB_CONFIG = get_db_config() if SAAS else get_local_db_congin()

store.py

@@ -3,16 +3,29 @@ from langchain.text_splitter import RecursiveCharacterTextSplitter
 from langchain.embeddings import OpenAIEmbeddings
 from langchain.vectorstores import Qdrant
 # from qdrant_client import QdrantClient
-from config import get_db_config
+from config import DB_CONFIG
 
 
 CHUNK_SIZE = 500
 
 
+def _remove_prefix_path(p: str):
+    prefix = "data/rtdocs/nvdajp-book.readthedocs.io/"
+    return p.removeprefix(prefix)
+
+
 def get_documents(path: str):
     loader = ReadTheDocsLoader(path, encoding="utf-8")
     docs = loader.load()
-    return docs
+    base_url = "https://nvdajp-book.readthedocs.io/"
+    add_meta = {"category": "ja-book"}
+    for doc in docs:
+        org_metadata = doc.metadata
+        source = _remove_prefix_path(org_metadata["source"])
+        add_meta = {"category": "ja-book", "source": source, "url": f"{base_url}{source}"}
+        doc.metadata = org_metadata | add_meta
+        yield doc
+    # return docs
 
 
 def get_text_chunk(docs):
@@ -23,7 +36,7 @@ def get_text_chunk(docs):
 
 def store(texts):
     embeddings = OpenAIEmbeddings()
-    db_url, db_api_key, db_collection_name = get_db_config()
+    db_url, db_api_key, db_collection_name = DB_CONFIG
     # client = QdrantClient(url=db_url, api_key=db_api_key, prefer_grpc=True)
     _ = Qdrant.from_documents(
         texts,
@@ -48,6 +61,9 @@ if __name__ == "__main__":
     args = sys.argv
     if len(args) != 2:
         print("No args, you need two args for html_path")
+        docs = get_documents("data/rtdocs/nvdajp-book.readthedocs.io/ja/latest")
+        print(type(docs))
+        breakpoint()
     else:
         path = args[1]
         # dir_name = args[2]