update

ingest.py

@@ -6,6 +6,8 @@ from langchain.text_splitter import RecursiveCharacterTextSplitter, CharacterTex
 from langchain.vectorstores.faiss import FAISS
 import itertools
 import os
+import fsspec
+from pathlib import Path
 
 def get_text(content):
     relevant_part = content.find("div", {"class": "markdown"})
@@ -16,62 +18,65 @@ def get_text(content):
 
 def ingest_docs(urls=[]):
     """Get documents from web pages."""
+    folders=[]
     documents = []
-    text_splitter = RecursiveCharacterTextSplitter(
-        chunk_size=1000,
-        chunk_overlap=200,
-    )
     for url in urls:
         try:
-            url = url[0]
-            if len([i for i in map(''.join, itertools.product(*zip('sitemap'.upper(), 'sitemap'.lower()))) if i in url]) > 0:
-                loader = SitemapLoader(
-                    web_path=url, parsing_function=get_text
-                )
-            elif len([i for i in map(''.join, itertools.product(*zip('readthedocs'.upper(), 'readthedocs'.lower()))) if i in url]) > 0:
-                loader = ReadTheDocsLoader(
-                        path=url
-                    )
-            elif "local:" in url:
-                local_repo_path_1 = url.split('local:')[1]
-                loaders = []
-                known_exts = [".py", ".md"]
-                paths_by_ext = {}
-                docs_by_ext = {}
-                for ext in known_exts + ["other"]:
-                    docs_by_ext[ext] = []
-                    paths_by_ext[ext] = []
-                for root, dirs, files in os.walk(local_repo_path_1):
-                    for file in files:
-                        file_path = os.path.join(root, file)
-                        rel_file_path = os.path.relpath(file_path, local_repo_path_1)
-                        for ext in paths_by_ext.keys():
-                            if '.' not in [i[0] for i in rel_file_path.split('/')]:
-                                if rel_file_path.endswith(ext):
-                                    paths_by_ext[ext].append(rel_file_path)
-                                else:
-                                    paths_by_ext["other"].append(rel_file_path)
 
-                # for each extension, load the files and split them
-                for ext in paths_by_ext.keys():
-                    for i in range(len(paths_by_ext[ext])):
-                        try:
-                            docs_by_ext[ext] += TextLoader(os.path.join(local_repo_path_1, paths_by_ext[ext][i])).load()
-                        except Exception as e:
-                            print(e)
-                            continue
-                py_splitter = PythonCodeTextSplitter(chunk_size=1000, chunk_overlap=0)
-                text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=0)
-                md_splitter = MarkdownTextSplitter(chunk_size=1000, chunk_overlap=0)
-                for ext in docs_by_ext.keys():
-                    if ext == ".py":
-                        documents += py_splitter.split_documents(docs_by_ext[ext])
-                    elif ext == ".md":
-                        documents += md_splitter.split_documents(docs_by_ext[ext])
-                    else:
-                        documents += text_splitter.split_documents(docs_by_ext[ext])
+            if "local:" in url:
+                folders.append(url.split('local:')[1])
             else:
-                raise ValueError("No loader found for this url")
+                url = url[0]
+                if url[0] == '/':
+                    url = url[1:]
+                if url[-1] != '/':
+                    url += '/'
+                org = url.split('/')[0]
+                repo = url.split('/')[1]
+                # join all strings after 2nd slash
+                folder = '/'.join(url.split('/')[2:])
+                if folder[-1] != '/':
+                    folder += '/'
+                fs = fsspec.filesystem("github", org=org, repo=repo)
+                # recursive copy
+                destination = url
+                destination.mkdir(exist_ok=True, parents=True)
+                fs.get(fs.ls(folder), destination.as_posix(), recursive=True)
+                folders.append(destination)
+        except Exception as e:
+            print(e)
+    for folder in folders:
+        try:
+            py_splitter = PythonCodeTextSplitter(chunk_size=1000, chunk_overlap=0)
+            text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=0)
+            md_splitter = MarkdownTextSplitter(chunk_size=1000, chunk_overlap=0)
+            local_repo_path_1 = folder
+            known_exts = [".py", ".md", ".rst"]
+            paths_by_ext = {}
+            docs_by_ext = {}
+            for ext in known_exts + ["other"]:
+                docs_by_ext[ext] = []
+                paths_by_ext[ext] = []
+            for root, dirs, files in os.walk(local_repo_path_1):
+                for file in files:
+                    file_path = os.path.join(root, file)
+                    rel_file_path = os.path.relpath(file_path, local_repo_path_1)
+                    for ext in paths_by_ext.keys():
+                        if '.' not in [i[0] for i in rel_file_path.split('/')]:
+                            if rel_file_path.endswith(ext):
+                                paths_by_ext[ext].append(rel_file_path)
+                                docs_by_ext[ext].append(TextLoader(os.path.join(local_repo_path_1, rel_file_path)).load())
+                            else:
+                                paths_by_ext["other"].append(rel_file_path)
+                                docs_by_ext["other"].append(TextLoader(os.path.join(local_repo_path_1, rel_file_path)).load())
+
+            for ext in docs_by_ext.keys():
+                if ext == ".py":
+                    documents += py_splitter.split_documents(docs_by_ext[ext])
+                elif ext == ".md" or ext == ".rst":
+                    documents += md_splitter.split_documents(docs_by_ext[ext])
+                else:
+                    documents += text_splitter.split_documents(docs_by_ext[ext])
         except Exception as e:
             print(e)
             continue