LangchainBot-space-creator

Running

App Files Files Community

LangchainBot-space-creator / app.py

ysharma HF staff

update duplicate instructions

3dc9117 over 1 year ago

raw history blame

No virus

8.28 kB

	from langchain.llms import OpenAI
	from langchain.chains.qa_with_sources import load_qa_with_sources_chain
	from langchain.docstore.document import Document
	import requests
	import pathlib
	import subprocess
	import tempfile
	import os
	import gradio as gr
	import pickle
	from huggingface_hub import HfApi, upload_folder
	from huggingface_hub import whoami, list_models

	# using a vector space for our search
	from langchain.embeddings.openai import OpenAIEmbeddings
	from langchain.vectorstores.faiss import FAISS
	from langchain.text_splitter import CharacterTextSplitter


	#Code for extracting the markdown fies from a Repo
	#To get markdowns from github for any/your repo
	def get_github_docs(repo_link):
	repo_owner, repo_name = repo_link.split('/')[-2], repo_link.split('/')[-1]

	with tempfile.TemporaryDirectory() as d:
	subprocess.check_call(
	f"git clone https://github.com/{repo_owner}/{repo_name}.git .",
	cwd=d,
	shell=True,
	)
	git_sha = (
	subprocess.check_output("git rev-parse HEAD", shell=True, cwd=d)
	.decode("utf-8")
	.strip()
	)
	repo_path = pathlib.Path(d)
	markdown_files = list(repo_path.rglob("*.md")) + list(
	repo_path.rglob("*.mdx")
	)
	for markdown_file in markdown_files:
	try:
	with open(markdown_file, "r") as f:
	relative_path = markdown_file.relative_to(repo_path)
	github_url = f"https://github.com/{repo_owner}/{repo_name}/blob/{git_sha}/{relative_path}"
	yield Document(page_content=f.read(), metadata={"source": github_url})
	except FileNotFoundError:
	print(f"Could not open file: {markdown_file}")

	#Code for creating a new space for the user
	def create_space(repo_link, hf_token):
	print("*********INSIDE CREATE SPACE*************")
	repo_name = repo_link.split('/')[-1]
	api = HfApi(token=hf_token)
	repo_url = api.create_repo(
	repo_id=f'LangChain_{repo_name}Bot', #example - ysharma/LangChain_GradioBot
	exist_ok = True,
	repo_type="space",
	space_sdk="gradio",
	private=False)

	#Code for creating the search index
	#Saving search index to disk
	def create_search_index(repo_link, openai_api_key):
	print("*********INSIDE CREATE SEARCH INDEX*************")
	#openai = OpenAI(temperature=0, openai_api_key=openai_api_key )
	sources = get_github_docs(repo_link) #"gradio-app", "gradio"
	source_chunks = []
	splitter = CharacterTextSplitter(separator=" ", chunk_size=1024, chunk_overlap=0)
	for source in sources:
	for chunk in splitter.split_text(source.page_content):
	source_chunks.append(Document(page_content=chunk, metadata=source.metadata))

	search_index = FAISS.from_documents(source_chunks, OpenAIEmbeddings(openai_api_key=openai_api_key))

	#saving FAISS search index to disk
	with open("search_index.pickle", "wb") as f:
	pickle.dump(search_index, f)
	return "search_index.pickle"

	def upload_files_to_space(repo_link, hf_token):
	print("*********INSIDE UPLOAD FILES TO SPACE*************")
	repo_name = repo_link.split('/')[-1]
	api = HfApi(token=hf_token)
	user_name = whoami(token=hf_token)['name']

	#Replacing the repo namein app.py
	with open("template/app_og.py", "r") as f:
	app = f.read()
	app = app.replace("$RepoName", repo_name)
	#app = app.replace("$space_id", whoami(token=token)["name"] + "/" + model_id.split("/")[-1])

	#Saving the new app.py file to disk
	with open("template/app.py", "w") as f:
	f.write(app)

	#Uploading the new app.py to the new space
	api.upload_file(
	path_or_fileobj = "template/app.py",
	path_in_repo = "app.py",
	repo_id = f'{user_name}/LangChain_{repo_name}Bot', #model_id,
	token = hf_token,
	repo_type="space",)
	#Uploading the new search_index file to the new space
	api.upload_file(
	path_or_fileobj = "search_index.pickle",
	path_in_repo = "search_index.pickle",
	repo_id = f'{user_name}/LangChain_{repo_name}Bot', #model_id,
	token = hf_token,
	repo_type="space",)
	#Upload requirements.txt to the space
	api.upload_file(
	path_or_fileobj="template/requirements.txt",
	path_in_repo="requirements.txt",
	repo_id=f'{user_name}/LangChain_{repo_name}Bot', #model_id,
	token=hf_token,
	repo_type="space",)
	#Deleting the files - search_index and app.py file
	os.remove("template/app.py")
	os.remove("search_index.pickle")

	repo_url = f"https://huggingface.co/spaces/{user_name}/LangChain_{repo_name}Bot"
	space_name = f"{user_name}/LangChain_{repo_name}Bot"
	#return f"Successfully created the Chatbot at: <a href="+ repo_url + " target='_blank'>" + space_name + "</a>"
	return "<p style='color: orange; text-align: center; font-size: 24px; background-color: lightgray;'>Successfully created the Chatbot at: <a href="+ repo_url + " target='_blank'>" + space_name + "</a></p>"
	#<p style="color: orange; text-align: center; font-size: 24px; background-color: lightgray; display: inline-block;">This is some text</p>


	def driver(repo_link, hf_token):
	#create search index openai_api_key=openai_api_key
	#search_index_pickle = create_search_index(repo_link, openai_api_key)
	#create a new space
	print("*********INSIDE DRIVER*************")
	create_space(repo_link, hf_token)
	#upload files to the new space
	html_tag = upload_files_to_space(repo_link, hf_token)
	print(f"html tag is : {html_tag}")
	return html_tag

	def set_state():
	return gr.update(visible=True), gr.update(visible=True)

	#Gradio code for Repo as input and search index as output file
	with gr.Blocks() as demo:
	gr.HTML("""<div style="text-align: center; max-width: 700px; margin: 0 auto;">
	<div
	style="
	display: inline-flex;
	align-items: center;
	gap: 0.8rem;
	font-size: 1.75rem;
	"
	>
	<h1 style="font-weight: 900; margin-bottom: 7px; margin-top: 5px;">
	QandA Chatbot Creator for Github Repos - Automation done using LangChain, Gradio, and Spaces
	</h1>
	</div>
	<p style="margin-bottom: 10px; font-size: 94%">
	Hi, I'm an <b>automated</b> Q & A Chatbot creator. Just copy and paste your <b>Github Repo link</b>, type in your OpenAI API key, and press the button!
	The app will generate the FAISS embedding vector space and will also make it available for you to download. Next, enter your Huggingface Token and press the final button to create your own QandA Chatbot under your profile on HF.
	Press the link displayed for your new space and enjoy your own state-of-the-art QandA chatbot!
	<br>Built using <a href="https://langchain.readthedocs.io/en/latest/" target="_blank">LangChain</a> and <a href="https://github.com/gradio-app/gradio" target="_blank">Gradio</a>
	<a href="https://huggingface.co/spaces/ysharma/LangchainBot-space-creator?duplicate=true"><img src="https://bit.ly/3gLdBN6" alt="Duplicate Space"></a>
	</p>
	</div>""")
	with gr.Row() :
	with gr.Column():
	repo_link = gr.Textbox(label="Enter Github repo name")
	openai_api_key = gr.Textbox(type='password', label="Enter your OpenAI API key here")
	btn_faiss = gr.Button("Create Search index")
	search_index_file = gr.File(label= 'Search index vector')
	with gr.Row():
	hf_token_in = gr.Textbox(type='password', label="Enter hf-token name", visible=False)
	btn_create_space = gr.Button("Create Your Chatbot", visible=False)
	html_out = gr.HTML()

	btn_faiss.click(create_search_index, [repo_link, openai_api_key],search_index_file )
	btn_faiss.click(fn=set_state, inputs=[] , outputs=[hf_token_in, btn_create_space])
	btn_create_space.click(driver, [repo_link, hf_token_in], html_out)

	demo.queue()
	demo.launch(debug=True)