support target_versions

app.py

@@ -1,5 +1,4 @@
 import logging
-import os
 from typing import Optional, Tuple
 
 import gradio as gr
@@ -19,12 +18,6 @@ handler = (
 handler.setLevel(logging.INFO)
 logging.basicConfig(level=logging.INFO)
 
-# Check if an openai key is set as an env. variable
-if os.getenv("OPENAI_API_KEY") is None:
-    print(
-        "Warning: No openai key detected. You can set it with 'export OPENAI_API_KEY=sk-...'."
-    )
-
 # Typehint for chatbot history
 ChatHistory = list[list[Optional[str], Optional[str]]]
 
@@ -114,21 +107,21 @@ with demo:
     examples = gr.Examples(
         examples=[
             "How can I install the library?",
-            "How do I deal with noisy data?",
-            "How do I deal with noisy data in 2 words?",
+            "What dependencies are required?",
         ],
         inputs=question,
     )
 
     gr.Markdown(
-        "This application uses GPT to search the docs for relevant info and answer questions."
+        "This app uses [Buster 🤖](github.com/jerpint/buster) and ChatGPT to search the docs for relevant info and answer questions."
     )
 
     response = gr.State()
 
     # fmt: off
-    submit.click(
-        add_user_question,
+    gr.on(
+        triggers=[submit.click, question.submit],
+        fn=add_user_question,
         inputs=[question],
         outputs=[chatbot]
     ).then(
@@ -141,21 +134,6 @@ with demo:
         outputs=[chatbot]
     )
 
-    question.submit(
-        add_user_question,
-        inputs=[question],
-        outputs=[chatbot],
-    ).then(
-        chat,
-        inputs=[chatbot],
-        outputs=[chatbot, response]
-    ).then(
-        add_sources,
-        inputs=[chatbot, response],
-        outputs=[chatbot]
-    )
-    # fmt: on
-
 
 demo.queue(concurrency_count=16)
 demo.launch(share=False)

cfg.py

@@ -1,5 +1,5 @@
+import os
 import logging
-import sys
 
 from buster.busterbot import Buster, BusterConfig
 from buster.completers import ChatGPTCompleter, DocumentAnswerer
@@ -14,11 +14,17 @@ from rtd_scraper.scrape_rtd import scrape_rtd
 # Set the root logger's level to INFO
 logging.basicConfig(level=logging.INFO)
 
+# Check if an openai key is set as an env. variable
+if os.getenv("OPENAI_API_KEY") is None:
+    print(
+        "Warning: No openai key detected. You can set it with 'export OPENAI_API_KEY=sk-...'."
+    )
 
-homepage_url = "https://buster.readthedocs.io/"
-
+homepage_url = os.getenv("RTD_URL", "https://orion.readthedocs.io/")
+target_version = os.getenv("RTD_VERSION", "en/stable")
 
-scrape_rtd(homepage_url=homepage_url, save_directory="outputs/")
+# scrape and embed content from readthedocs website
+scrape_rtd(homepage_url=homepage_url, save_directory="outputs/", target_version=target_version)
 
 # Disable logging for third-party libraries at DEBUG level
 for name in logging.root.manager.loggerDict:

rtd_scraper/scrape_rtd.py

@@ -5,10 +5,10 @@ from buster.docparser import get_all_documents
 from buster.documents_manager import DeepLakeDocumentsManager
 from buster.parser import SphinxParser
 from scrapy.crawler import CrawlerProcess
-from scrapy.exceptions import CloseSpider
 from scrapy.utils.project import get_project_settings
 
 from rtd_scraper.tutorial.spiders.docs_spider import DocsSpider
+# from tutorial.spiders.docs_spider import DocsSpider
 
 # When using scrapy it seems to set logging for all apps at DEBUG, so simply shut it off here...
 for name in logging.root.manager.loggerDict:
@@ -16,12 +16,9 @@ for name in logging.root.manager.loggerDict:
     logger.setLevel(logging.INFO)
 
 
-def run_spider(homepage_url, save_directory):
-    #  settings_file_path = 'rtd_scraper.tutorial.settings' # The path seen from top-level, ie. from cfg.py
-    #  os.environ.setdefault('SCRAPY_SETTINGS_MODULE', settings_file_path)
-
+def run_spider(homepage_url, save_directory, target_version=None):
     process = CrawlerProcess(settings=get_project_settings())
-    process.crawl(DocsSpider, homepage_url=homepage_url, save_dir=save_directory)
+    process.crawl(DocsSpider, homepage_url=homepage_url, save_dir=save_directory, target_version=target_version)
 
     # Start the crawling process
     process.start()
@@ -30,11 +27,11 @@ def run_spider(homepage_url, save_directory):
     process.stop()
 
 
-def scrape_rtd(homepage_url, save_directory):
+def scrape_rtd(homepage_url, save_directory, target_version=None):
     # Crawl the website using scrapy
-    run_spider(homepage_url, save_directory=save_directory)
+    run_spider(homepage_url, save_directory=save_directory, target_version=target_version)
 
-    # Convert the .html pages into chunks using Buster's SphinxParser
+    # # Convert the .html pages into chunks using Buster's SphinxParser
     root_dir = os.path.join(save_directory, homepage_url.split("https://")[1])
 
     # root_dir is the folder containing the scraped content e.g. crawled_outputs/buster.readthedocs.io/
@@ -49,23 +46,23 @@ def scrape_rtd(homepage_url, save_directory):
     # Add the source column
     df["source"] = "readthedocs"
 
-    #  #  Initialize the DeepLake vector store
-    #  dm = DeepLakeDocumentsManager(
-    #      vector_store_path=os.path.join(save_directory, "deeplake_store"),
-    #      overwrite=True,
-    #      required_columns=["url", "content", "source", "title"],
-    #  )
-    #
-    #  # Add all embeddings to the vector store
-    #  dm.batch_add(
-    #      df=df,
-    #      batch_size=3000,
-    #      min_time_interval=60,
-    #      num_workers=32,
-    #  )
-    #
+    #  Initialize the DeepLake vector store
+    dm = DeepLakeDocumentsManager(
+        vector_store_path=os.path.join(save_directory, "deeplake_store"),
+        overwrite=True,
+        required_columns=["url", "content", "source", "title"],
+    )
+
+    # Add all embeddings to the vector store
+    dm.batch_add(
+        df=df,
+        batch_size=3000,
+        min_time_interval=60,
+        num_workers=32,
+    )
+
 
 
 if __name__ == "__main__":
-    homepage_url = "https://buster.readthedocs.io/"
-    scrape_rtd(homepage_url=homepage_url, save_directory="outputs/")
+    homepage_url = "https://orion.readthedocs.io/"
+    scrape_rtd(homepage_url=homepage_url, target_version="v0.2.7", save_directory="outputs/")

rtd_scraper/tutorial/spiders/docs_spider.py

@@ -6,20 +6,39 @@ import scrapy
 
 logging.basicConfig(format="%(levelname)s: %(message)s", level=logging.ERROR)
 
+from urllib.parse import urlparse
+
+def extract_domain(url):
+    """
+    Extract the domain (including subdomains) from a given URL.
+
+    Args:
+    - url (str): The URL from which the domain needs to be extracted.
+
+    Returns:
+    - str: The domain (with subdomains) extracted from the URL.
+           For example, 'www.example.com' for the URL 'https://www.example.com/path/to/something'.
+
+    """
+    parsed_uri = urlparse(url)
+    # The netloc attribute will contain the domain name
+    domain = parsed_uri.netloc
+    return domain
+
 
 class DocsSpider(scrapy.Spider):
     name = "docs"
 
-    def __init__(self, homepage_url: str, save_dir="crawled_pages", *args, **kwargs):
+    def __init__(self, homepage_url: str, save_dir="crawled_pages", target_version=None, *args, **kwargs):
         super(DocsSpider, self).__init__(*args, **kwargs)
 
         if not homepage_url.startswith("https://"):
             homepage_url = "https://" + homepage_url
 
-        project: str = homepage_url.split(".")[0].split("https://")[1]
-        self.allowed_domains = [f"{project}.readthedocs.io"]
+        self.allowed_domains = [extract_domain(homepage_url)]
         self.start_urls = [homepage_url]
         self.base_dir = Path(save_dir)
+        self.target_version = target_version
 
     def parse(self, response):
         parsed_uri = urlparse(response.url)
@@ -39,6 +58,13 @@ class DocsSpider(scrapy.Spider):
         with open(filepath, "wb") as f:
             f.write(response.body)
 
-        # Follow links to other documentation pages
+        # Follow links to other documentation pages only if they contain the target version in the full URL
         for href in response.css("a::attr(href)").getall():
-            yield response.follow(href, self.parse)
+            if self.target_version:
+                # A version was specified, check to see if it's the correct version from url
+                full_url = response.urljoin(href)  # Expand href to a full URL
+                if self.target_version in full_url:
+                    yield response.follow(href, self.parse)
+            else:
+                # no version specified, follow all links
+                yield response.follow(href, self.parse)