Application file, KKMS repo and utils files

app.py

@@ -0,0 +1,226 @@
+import gradio as gr
+import openai
+import os
+import datetime
+import kkms_kssw as kkms_kssw
+import utils.constants as constants_utils
+
+
+# Create and launch Gradio Web UI
+class DomState:
+	def __init__(self):
+		self.relevant_paragraphs = ''
+		self.answer = ''
+		self.summary = ''
+		self.mandi_price = ''
+		self.mandi_from_date = (datetime.datetime.now() - datetime.timedelta(days=2)).strftime('%Y-%m-%d')
+		self.mandi_to_date = datetime.datetime.now().strftime('%Y-%m-%d')
+		self.weather_info = ''
+		self.weather_forecast = ''
+		self.weather_forecast_summary = ''
+		self.indic_lang_answer = ''
+
+
+		# Initialize index (vector store)
+		self.kkms_kssw_obj = kkms_kssw.KKMS_KSSW()
+		self.kkms_kssw_obj.initialize_index(constants_utils.INDEX_FILENAME, index_type='GPTSimpleVectorIndex')
+		
+	
+	def click_handler_for_get_relevant_paragraphs(self,
+									 question,
+									 mode='default',
+									 response_mode='default',
+									 similarity_top_k=2,
+		):
+		self.relevant_paragraphs = self.kkms_kssw_obj.query(question,
+							mode=mode,
+							response_mode=response_mode,
+							similarity_top_k=similarity_top_k,
+							# required_keywords=required_keywords_list,
+							# exclude_keywords=exclude_keywords_list,
+		)
+		return self.relevant_paragraphs
+	
+	
+	def click_handler_for_summary(self, answer):
+		self.sumamry = self.kkms_kssw_obj.langchain_utils_obj.get_textual_summary(answer)
+		return self.sumamry
+	
+	
+	def click_handler_for_get_answer(self,
+									 relevant_paragraphs,
+									 question
+		):
+		self.answer = self.kkms_kssw_obj.langchain_utils_obj.get_answer_from_para(relevant_paragraphs, question)
+		return self.answer
+	
+	
+	def click_handler_for_mandi_price(self, 
+									  state_name,
+									  apmc_name,
+									  commodity_name,
+									  from_date,
+									  to_date
+									 ):
+		if state_name and apmc_name and commodity_name and from_date and to_date:
+			self.mandi_price = self.kkms_kssw_obj.mandi_utils_obj.get_mandi_price(state_name, apmc_name, commodity_name, from_date, to_date)
+		return self.mandi_price
+	
+	
+	def click_handler_for_get_weather(self, city):
+		time, info, temperature = self.kkms_kssw_obj.weather_utils_obj.get_weather(city)
+		self.weather_info = f'Weather in {city.capitalize()} on {time} is {temperature} with {info}.'
+		return self.weather_info
+	
+	
+	def click_handler_for_get_weather_forecast(self, state, district):
+		self.weather_forecast = self.kkms_kssw_obj.weather_utils_obj.get_weather_forecast(state, district)
+		return self.weather_forecast
+	
+	
+	def click_handler_for_weather_forecast_summary(self, weather_forecast):
+		self.weather_forecast_summary = self.kkms_kssw_obj.langchain_utils_obj.get_weather_forecast_summary(weather_forecast)
+		return self.weather_forecast_summary
+	
+	
+	def click_handler_for_get_indic_answer(self, eng_ans, language='Hindi'):
+		self.indic_lang_answer = self.kkms_kssw_obj.translator_utils_obj.get_indic_google_translate(eng_ans, language)
+		return self.indic_lang_answer
+	
+	
+	def select_widget(self, choice):
+		if choice == "General":
+			return [
+				gr.update(visible=True),
+				gr.update(visible=False),
+				gr.update(visible=False)
+			]
+		
+		elif choice == "Mandi Price":
+			return [
+				gr.update(visible=False),
+				gr.update(visible=True),
+				gr.update(visible=False)
+			]
+		
+		elif choice == "Weather":
+			return [
+				gr.update(visible=False),
+				gr.update(visible=False),
+				gr.update(visible=True)
+			]
+		
+		else:
+			return gr.update(visible=False)
+
+
+with gr.Blocks(title='KKMS-KSSW Demo') as demo:
+	dom = DomState()
+	
+	radio = gr.Radio(
+		["General", "Mandi Price", "Weather"], label="Query related to"
+	)
+	
+	########################### Widget for Govt. Policy #################################################
+	with gr.Row(visible=True) as rowGeneral:
+		with gr.Column(scale=1, min_width=600):
+			with gr.Tab(label='Relevant paragraphs'):
+				question = gr.Textbox(label="Enter your question", placeholder='Type the question here')
+				# Get the Relevant paragraphs for the question asked
+				relevant_paragraphs = gr.Textbox(label="Relevant paragraphs are:", value=dom.relevant_paragraphs, interactive=False)
+				b_relevant_paragraphs = gr.Button("Get Relevant paragraphs").style(size='sm')
+				b_relevant_paragraphs.click(fn=dom.click_handler_for_get_relevant_paragraphs, inputs=question, outputs=[relevant_paragraphs])
+
+		with gr.Column(scale=1, min_width=600):
+			with gr.Tab(label='Extractive Summary'):
+				# Get the extractive text summary from the retrieved Relevant paragraphs
+				summary = gr.Textbox(label="Extractive Summary is:", value=dom.summary, interactive=False)
+				b_summary = gr.Button("Extract Summary").style(size='sm')
+				b_summary.click(fn=dom.click_handler_for_summary, inputs=relevant_paragraphs, outputs=[summary])
+		
+		# Get the exact answer for the question asked from the retrieved Relevant paragraphs
+		with gr.Row():
+			with gr.Column(scale=1, min_width=600):
+				with gr.Tab(label='Answer'):
+					answer = gr.Textbox(label="Answer is:", value=dom.answer, interactive=False)
+					b_answer = gr.Button("Get Answer").style(size='sm')
+					b_answer.click(fn=dom.click_handler_for_get_answer, inputs=[relevant_paragraphs, question], outputs=[answer])
+		
+		# Covert the answer to Indian language
+		with gr.Row():
+			with gr.Column(scale=1, min_width=600):
+				with gr.Tab(label='Answer in selected language'):
+					# Select the language
+					language = gr.Dropdown(
+						['English', 'Hindi', 'Gujarati', 'Marathi', 'Kannada', 'Bengali', 'Panjabi', 'Telugu', 'Tamil', 'Malayalam'],
+						label="Select language")
+					indic_lang_answer = gr.Textbox(label="Answer in the selected language is:", value=dom.indic_lang_answer, interactive=False)
+					b_indic_lang_answer = gr.Button("Get answer in selected language").style(size='sm')
+					b_indic_lang_answer.click(fn=dom.click_handler_for_get_indic_answer, inputs=[answer, language], outputs=[indic_lang_answer])
+	
+	
+	########################## Widget for Mandi Price ###################################################
+	with gr.Row(visible=False) as rowMandiPrice:
+		with gr.Column(scale=1, min_width=600):
+			# Select State
+			state_name = gr.Dropdown(['ANDAMAN AND NICOBAR ISLANDS', 'ANDHRA PRADESH', 'ASSAM', 'BIHAR', 'CHANDIGARH', 'CHHATTISGARH', 'GOA', 'GUJARAT', 'HARYANA', 'HIMACHAL PRADESH', 'JAMMU AND KASHMIR', 'JHARKHAND', 'KARNATAKA', 'KERALA', 'MADHYA PRADESH', 'MAHARASHTRA', 'NAGALAND', 'ODISHA', 'PUDUCHERRY', 'PUNJAB', 'RAJASTHAN', 'TAMIL NADU', 'TELANGANA', 'TRIPURA', 'UTTAR PRADESH', 'UTTARAKHAND', 'WEST BENGAL'], label="Select state")
+			
+			# APMC name
+			apmc_name = gr.Textbox(label="Enter APMC name", placeholder='Type the APMC name here')
+			
+			# APMC name
+			commodity_name = gr.Textbox(label="Enter Commodity name", placeholder='Type the Commodity name here')
+					  
+			# From/To date in yyyy-mm-dd format
+			from_date = gr.Textbox(label="From date?", value=dom.mandi_from_date, placeholder='Please enter the From date here in yyyy-mm-dd format')
+			to_date = gr.Textbox(label="To date?", value=dom.mandi_to_date, placeholder='Please enter the To date here in yyyy-mm-dd format')
+		
+		with gr.Column(scale=1, min_width=600):
+			mandi_price = gr.Textbox(label=f"Mandi Price is:", value=dom.mandi_price, interactive=False)
+			b_summary = gr.Button("Get Mandi Price").style(size='sm')
+			b_summary.click(fn=dom.click_handler_for_mandi_price, inputs=[state_name, apmc_name, commodity_name, from_date, to_date], outputs=[mandi_price])
+	
+	
+	########################## Widget for Weather Info ###################################################
+	with gr.Row(visible=False) as rowWeather:
+		with gr.Column(scale=1, min_width=600):
+			with gr.Tab(label='Weather Info'):
+				city = gr.Textbox(label="Enter city name", placeholder='Type the city name here')
+				weather = gr.Textbox(label=f"Current weather is:", value=dom.weather_info, interactive=False)
+				b_weather = gr.Button("Get weather info").style(size='sm')
+				b_weather.click(fn=dom.click_handler_for_get_weather, inputs=city, outputs=[weather])
+		
+		########### Weather Forecast ###########
+		with gr.Column(scale=1, min_width=600):
+			with gr.Tab(label='Weather Forecast for next 5 days'):
+				# Select the State
+				state = gr.Dropdown(
+					['Andaman-Nicobar', 'Andhra-Pradesh', 'Arunachal-Pradesh', 'Assam', 'Bihar', 'Chandigarh', 'Chhattisgarh', 'Dadra-and-Nagar-Haveli', 'Daman-and-Diu', 'Delhi', 'Goa', 'Gujarat', 'Haryana', 'Himachal-Pradesh', 'Jammu-Kashmir', 'Jharkhand', 'Karnataka', 'Kerala', 'Lakshadweep', 'Madhya-Pradesh', 'Maharashtra', 'Manipur', 'Meghalaya', 'Mizoram', 'Nagaland', 'Odisha', 'Pondicherry', 'Punjab', 'Rajasthan', 'Sikkim', 'Tamilnadu', 'Telangana', 'Tripura', 'Uttar-Pradesh', 'Uttarakhand', 'West-Bengal'],
+					label="Select state"
+				)
+
+				# # Select district
+				# district = gr.Dropdown(
+				#     weather_utils.STATES.get(state, {}),
+				#     label="Select district"
+				# )
+
+				district = gr.Textbox(label="Enter district name", placeholder='Type the district name here')
+				district_weather = gr.Textbox(label=f"Weather forecast is:", value=dom.weather_forecast, interactive=False)
+				bd_weather = gr.Button("Get weather forecast").style(size='sm')
+				bd_weather.click(fn=dom.click_handler_for_get_weather_forecast, inputs=[state, district], outputs=[district_weather])
+				
+		
+		with gr.Column(scale=1, min_width=600):
+			with gr.Tab(label='Weather Forecast Summary'):
+				# Get the summary of the weather forecast
+				weather_forecast_summary = gr.Textbox(label="Weather Forecast Summary is:", value=dom.weather_forecast_summary, interactive=False)
+				b_weather_forecast_summary = gr.Button("Get Weather Forecast Summary").style(size='sm')
+				b_weather_forecast_summary.click(fn=dom.click_handler_for_weather_forecast_summary, inputs=district_weather, outputs=[weather_forecast_summary])
+
+	
+	radio.change(fn=dom.select_widget, inputs=radio, outputs=[rowGeneral, rowMandiPrice, rowWeather])
+
+
+demo.launch(share=True)
+run()

kkms_kssw.py

@@ -0,0 +1,125 @@
+import os
+
+import utils.constants as constants_utils
+import utils.data_loader as data_loader_utils
+import utils.langchain_utils as langchain_utils
+import utils.weather as weather_utils
+import utils.mandi_price as mandi_utils
+import utils.translator as translator_utils
+
+from llama_index import GPTSimpleVectorIndex, SimpleDirectoryReader, GPTListIndex
+from langchain.indexes import VectorstoreIndexCreator
+from langchain.embeddings.openai import OpenAIEmbeddings
+from langchain.vectorstores import FAISS
+
+import warnings
+warnings.filterwarnings('ignore')
+
+
+
+class KKMS_KSSW:
+	def __init__(self):
+		self.index = None
+		self.documents = []
+		self.response = None
+
+		# Instantiate langchain_utils class object
+		self.langchain_utils_obj = langchain_utils.LANGCHAIN_UTILS()
+		# Instantiate Mandi Price utils class object
+		self.mandi_utils_obj = mandi_utils.MANDI_PRICE()
+		# Instantiate Weather class object
+		self.weather_utils_obj = weather_utils.WEATHER()
+		# Instantiate translator_utils class object
+		self.translator_utils_obj = translator_utils.TRANSLATOR()
+
+
+	# Initialize index (vector store)
+	def initialize_index(self, save_index_to_disk=True, index_type='GPTSimpleVectorIndex'):
+		# Delete the old file
+		if os.path.exists(constants_utils.INDEX_FILENAME):
+			# os.system(f'rm -rf {constants_utils.INDEX_FILENAME}')
+			print(f'{constants_utils.INDEX_FILENAME} deleted.')
+		
+		if os.path.exists(constants_utils.INDEX_FILENAME):
+			# Load the index from the saved index.json file
+			print(f'Loading pre-generated index from: {constants_utils.INDEX_FILENAME}')
+			self.index = self.langchain_utils_obj.load_index(index_type='GPTSimpleVectorIndex', filepath=constants_utils.INDEX_FILENAME)
+		else:
+			# Load data from Docs
+			if os.path.exists(constants_utils.DATA_PATH):
+				doc_documents = SimpleDirectoryReader(constants_utils.DATA_PATH).load_data()
+				
+				# Load data from PDFs only
+				# pdf_documents = data_loader_utils.load_document(doc_type='pdf', doc_filepath=doc_filepath)
+			
+			# Load data from URLs & append it to the documents that we read from PDFs
+			# url_documents = data_loader_utils.load_document(doc_type='url', urls=urls)
+			
+			# Merge documents of different data sources
+			self.documents = doc_documents[:]
+			# self.documents.extend(url_documents)
+			
+			# Build the Vector store for docs
+			if index_type == 'GPTSimpleVectorIndex':
+				self.index = GPTSimpleVectorIndex.from_documents(self.documents)
+			elif index_type == 'FAISS':
+				self.index = FAISS.from_documents(
+					self.documents,
+					OpenAIEmbeddings(openai_api_key=os.getenv('OPENAI_API_KEY'))
+				)
+
+			
+			def merge_documents_from_different_sources(doc_documents, url_documents):
+				# Build the Vector store for docs
+				doc_index = GPTSimpleVectorIndex.from_documents(doc_documents)
+				# Build the Vector store for URLs
+				url_index = GPTSimpleVectorIndex.from_documents(url_documents)
+			
+				# Set summary of each index
+				doc_index.set_text("index_from_docs")
+				url_index.set_text("index_from_urls")
+
+				# Merge index of different data sources
+				self.index = GPTListIndex([doc_index])
+				self.index.insert(url_index)   # can also be passed directly as GPTListIndex([doc_index, url_index])
+				
+				return self.index
+			
+			
+			if save_index_to_disk:
+				# Save index to a index.json file
+				print(f'Saving newly generated index: {constants_utils.INDEX_FILENAME}')
+				
+				if index_type == 'GPTSimpleVectorIndex':
+					self.index.save_to_disk(constants_utils.INDEX_FILENAME)
+				elif index_type == 'FAISS':
+					self.index.save_local(constants_utils.INDEX_FILENAME)
+
+
+
+	# Define query on index to retrieve the most relevant top K documents from the vector store
+	def query(self,
+		question,
+		mode='default',
+		response_mode="default",
+		similarity_top_k=1,
+		required_keywords=[],
+		exclude_keywords=[],
+		verbose=False
+	):
+		'''
+			Args:
+				mode: can be any of [default, embedding]
+				response_mode: can be any of [default, compact, tree_summarize]
+		'''
+		
+		# Querying the index
+		self.response = self.index.query(question,
+							   mode=mode,
+							   response_mode=response_mode,
+							   similarity_top_k=similarity_top_k,
+							   required_keywords=required_keywords,
+							   exclude_keywords=exclude_keywords,
+							   verbose=verbose)
+		
+		return self.response

utils/constants.py

@@ -0,0 +1,42 @@
+DATA_PATH = './data/crops/'
+OUTPUT_PATH = './output'
+INDEX_FILENAME = f'{OUTPUT_PATH}/index.json'
+
+URLS = [
+	'https://dmi.gov.in/Documents/GrantCAGrapes.pdf',
+	'https://dmi.gov.in/Documents/organicfaq.pdf',
+	'https://dmi.gov.in/Documents/CAGMOrganic-III.pdf',
+	'https://dmi.gov.in/GradesStandard.aspx',
+	'https://www.india.gov.in/topics/agriculture',
+	'https://www.india.gov.in/farmers-portal',
+	
+	# Pest Management related
+	'https://niphm.gov.in/IPMPackages/Maize.pdf',
+	
+	# Mandi Price related
+	'https://agmarknet.gov.in/',
+	'https://enam.gov.in/web/dashboard/trade-data',
+	
+	# General information related: Information of interests are present on the 2nd level url
+	'https://agricoop.nic.in/#gsc.tab=0',
+	'https://www.manage.gov.in/nf/nf.asp',
+	
+	# Weather forecast related
+	'https://nwp.imd.gov.in/blf/blf_temp/',   # need to select state -> district (on the new page) -> displays detailed table -> can get info at the block level as well from the same page on selection
+	'https://nwp.imd.gov.in/blf/blf_temp/dis.php?value=12gujarat',   # to get weather forecast for the given state
+	'https://nwp.imd.gov.in/blf/blf_temp/block.php?dis=12BHAVNAGAR',   # to get the weather forecast for the given district
+]
+
+
+# Supported Indian laguages for translating the English text to Indian language
+INDIC_LANGUAGE = {
+	'Hindi': 'hi',
+	'Gujarati': 'gu',
+	'Kannada': 'kn',
+	'Marathi': 'mr',
+	'Panjabi': 'pa',
+	'Bengali': "bn",
+	'Telugu': 'te',
+	'Tamil': 'ta',
+	'Malayalam': 'ml',
+}

utils/data_loader.py

@@ -0,0 +1,104 @@
+import os
+import pandas as pd
+from pathlib import Path
+from llama_index import GPTSimpleVectorIndex, download_loader
+from langchain.agents import initialize_agent, Tool
+from langchain.llms import OpenAI
+from langchain.chains.conversation.memory import ConversationBufferMemory
+
+
+
+class DATA_LOADER:
+	def __init__(self):
+		print()
+
+
+	def clean_df(self, df, dropna=True, fillna=False):
+		if fillna:
+			df.fillna('', inplace=True)
+		if dropna:
+			df.dropna(inplace=True)
+			# df = df[~df.isna()]
+		df = df.drop_duplicates().reset_index(drop=True)
+		return df
+
+
+	def load_external_links_used_by_FTAs(self,
+		sheet_filepath='./data/urls_used_by_ftas/external_links_used_by_FTAs.xlsx'
+	):
+		xls = pd.ExcelFile(sheet_filepath)
+		df = pd.DataFrame(columns=['S.No.', 'Link used for', 'Link type', 'Link'])
+		for sheet_name in xls.sheet_names:
+			sheet = pd.read_excel(xls, sheet_name)
+			if sheet.shape[0] > 0:
+				df = pd.concat([df, sheet])
+			else:
+				print(f'{sheet_name} has no content.')
+
+		df = df[['Link used for', 'Link type', 'Link']]
+		# Clean df
+		df = clean_df(df)
+		print(f'Total links available across all cities: {df.shape[0]}')
+		return df
+
+
+	def load_document(self,
+		doc_type='pdf',
+		doc_filepath='',
+		urls=[]
+	):
+		documents = []
+		
+		if doc_type == 'pdf':
+			PDFReader = download_loader("PDFReader")
+			loader = PDFReader()
+			if os.path.exists(doc_filepath):
+				documents = loader.load_data(file=Path(doc_filepath))
+		
+		elif doc_type == 'url':
+			BeautifulSoupWebReader = download_loader("BeautifulSoupWebReader")
+			loader = BeautifulSoupWebReader()
+			if len(urls) > 0:
+				# Load data from URLs
+				documents = loader.load_data(urls=urls)
+		
+		elif doc_type == 'url-kb':
+			KnowledgeBaseWebReader = download_loader("KnowledgeBaseWebReader")
+			loader = KnowledgeBaseWebReader()
+			for url in urls:
+				doc = loader.load_data(
+					root_url=url, 
+					link_selectors=['.article-list a', '.article-list a'],
+					article_path='/articles',
+					body_selector='.article-body',
+					title_selector='.article-title',
+					subtitle_selector='.article-subtitle',
+				)
+				documents.extend(doc)
+
+		elif doc_type == 'url-chatgpt':
+			BeautifulSoupWebReader = download_loader("BeautifulSoupWebReader")
+			loader = BeautifulSoupWebReader()
+			if len(urls) > 0:
+				# Load data from URLs
+				documents = loader.load_data(urls=urls)
+				# Build the Vector database
+				index = GPTSimpleVectorIndex(documents)
+				tools = [
+					Tool(
+						name="Website Index",
+						func=lambda q: index.query(q),
+						description=f"Useful when you want answer questions about the text retrieved from websites.",
+					),
+				]
+				
+				# Call ChatGPT API
+				llm = OpenAI(temperature=0)    # Keep temperature=0 to search from the given urls only
+				memory = ConversationBufferMemory(memory_key="chat_history")
+				agent_chain = initialize_agent(
+					tools, llm, agent="zero-shot-react-description", memory=memory
+				)
+
+				output = agent_chain.run(input="What language is on this website?")
+		
+		return documents

utils/langchain_utils.py

@@ -0,0 +1,169 @@
+from langchain.llms import OpenAI
+from langchain.text_splitter import CharacterTextSplitter
+from langchain.chains.summarize import load_summarize_chain
+from langchain.docstore.document import Document
+from langchain.embeddings.openai import OpenAIEmbeddings
+from langchain.vectorstores import Chroma
+from langchain.chains.question_answering import load_qa_chain
+from langchain.chains.qa_with_sources import load_qa_with_sources_chain
+from langchain.prompts import PromptTemplate
+from llama_index import GPTSimpleVectorIndex
+from langchain.vectorstores import FAISS
+
+import pickle
+import os
+os.environ['OPENAI_API_KEY'] = os.getenv('OPENAI_API_KEY')
+os.environ['HUGGINGFACEHUB_API_TOKEN'] = os.getenv('HUGGINGFACEHUB_API_TOKEN')
+
+
+
+class LANGCHAIN_UTILS:
+	def __init__(self):
+		print()
+
+
+	def generate_prompt_template(self, prompt_type='general'):
+		prompt_template = ''
+		
+		if prompt_type == 'general':
+			prompt_template = """Write a concise summary of the following:
+
+			{text}
+
+			CONCISE SUMMARY IN ENGLISH:"""
+		
+		elif prompt_type == 'weather':
+			prompt_template = """
+				What would be the weather based on the below data:
+				{text}
+			"""
+			
+		return prompt_template
+
+
+
+	def get_textual_summary(self,
+		text,
+		chain_type="stuff",
+		custom_prompt=True,
+		prompt_type='general'
+	):
+		texts = [text]
+		docs = [Document(page_content=t) for t in texts[:3]]
+		
+		llm = OpenAI(temperature=0)
+		if custom_prompt:
+			prompt_template = self.generate_prompt_template(prompt_type)
+			PROMPT = PromptTemplate(template=prompt_template, input_variables=["text"])
+			chain = load_summarize_chain(llm, chain_type=chain_type, prompt=PROMPT)
+		else:
+			chain = load_summarize_chain(llm, chain_type=chain_type)
+		
+		text_summary = chain.run(docs)
+		return text_summary
+
+
+	def get_weather_forecast_summary(self,
+		text,
+		chain_type="stuff"
+	):
+		text = f"""
+			What would be the weather based on the below data:
+			{text}
+			
+			Give simple response without technical numbers which can be explained to human.
+		"""
+		texts = [text]
+		docs = [Document(page_content=t) for t in texts[:3]]
+		
+		llm = OpenAI(temperature=0)
+		chain = load_summarize_chain(llm, chain_type=chain_type)
+		text_summary = chain.run(docs)
+		
+		return text_summary
+
+
+	def get_answer_from_para(self,
+		para,
+		question,
+		chain_type="stuff",
+		custom_prompt=True
+	):
+		# Prepare data (Split paragraph into chunks of small documents)
+		text_splitter = CharacterTextSplitter(chunk_size=1000, chunk_overlap=0)
+		texts = text_splitter.split_text(para)
+
+		# Find similar docs that are relevant to the question
+		embeddings = OpenAIEmbeddings()
+		docsearch = Chroma.from_texts(
+			texts, embeddings,
+			metadatas=[{"source": str(i)} for i in range(len(texts))]
+		)
+
+		# Search for the similar docs
+		docs = docsearch.similarity_search(question, k=1)
+		
+		llm = OpenAI(temperature=0)
+		# Create a Chain for question answering
+		if custom_prompt:
+			prompt_template = """Use the following pieces of context to answer the question at the end. If you don't know the answer, just say that you don't know, don't try to make up an answer.
+
+			{context}
+
+			Question: {question}
+			Answer in English:"""
+
+			PROMPT = PromptTemplate(
+				template=prompt_template, input_variables=["context", "question"]
+			)
+			chain = load_qa_chain(llm, chain_type=chain_type, prompt=PROMPT)
+		else:
+			# chain = load_qa_with_sources_chain(llm, chain_type=chain_type)
+			chain = load_qa_chain(llm, chain_type=chain_type)
+			# chain.run(input_documents=docs, question=question)
+		
+		out_dict = chain({"input_documents": docs, "question": question}, return_only_outputs=True)
+		return out_dict['output_text']
+
+
+	def store_index(self,
+		index,
+		index_type='GPTSimpleVectorIndex',
+		filepath='./output/index.json'
+	):
+		if index_type == 'GPTSimpleVectorIndex':
+			index.save_to_disk(filepath)
+		
+		elif index_type == 'pickle':
+			with open(filepath, "wb") as f:
+				pickle.dump(index, f)
+		
+		elif index_type == 'FAISS':
+			index.save_local(filepath)
+
+
+	def load_index(self,
+		index_type='GPTSimpleVectorIndex',
+		filepath='./output/index.json'
+	):
+		if index_type == 'GPTSimpleVectorIndex':
+			index = GPTSimpleVectorIndex.load_from_disk(filepath)
+		
+		elif index_type == 'pickle':
+			with open(filepath, "rb") as f:
+				index = pickle.load(f)
+		
+		elif index_type == 'FAISS':
+			index = FAISS.load_local(filepath, OpenAIEmbeddings())   # can we use open-source embeddings?
+		
+		return index
+
+
+	def convert_text_to_documents(self, text_list=[]):
+		"""
+			Converts the list of text data to Documents format that can be feed to GPT API to build the Vector store
+		"""
+		
+		from llama_index import Document
+		documents = [Document(t) for t in text_list]
+		return documents

utils/mandi_price.py

@@ -0,0 +1,33 @@
+import requests
+
+
+class MANDI_PRICE:
+	def __init__(self):
+		self.base_url = "https://enam.gov.in/web/Ajax_ctrl/trade_data_list"
+		# "https://enam.gov.in/web/dashboard/trade-data",
+		# "https://enam.gov.in/web/dashboard/trade_data_list",
+
+
+	def get_mandi_price(self,
+		state_name,
+		apmc_name,
+		commodity_name,
+		from_date,
+		to_date
+	):
+		# Prepare the payload for POST request
+		payload = f"language=en&stateName={state_name}&apmcName={apmc_name}&commodityName={commodity_name}&fromDate={from_date}&toDate={to_date}"
+
+		headers = {
+			"Content-type": "application/x-www-form-urlencoded; charset=UTF-8",
+			"Referer": "https://enam.gov.in/web/dashboard/trade-data",
+			"Accept": "application/json, text/javascript, */*; q=0.01",
+		}
+
+		response = requests.post(
+			self.base_url,
+			json=payload,
+			headers=headers,
+		)
+
+		return response.json()

utils/translator.py

@@ -0,0 +1,61 @@
+import utils.constants as constants_utils
+import requests
+from transformers import AutoModelForSeq2SeqLM, AutoTokenizer
+from mosestokenizer import *
+from indicnlp.tokenize import sentence_tokenize
+from googletrans import Translator, constants
+
+
+class TRANSLATOR:
+	def __init__(self):
+		print()
+
+
+	def split_sentences(self, paragraph, language):
+		if language == "en":
+			with MosesSentenceSplitter(language) as splitter:
+				return splitter([paragraph])
+		elif language in constants_utils.INDIC_LANGUAGE:
+			return sentence_tokenize.sentence_split(paragraph, lang=language)
+
+
+	def get_in_hindi(self, payload):
+		tokenizer = AutoTokenizer.from_pretrained("facebook/nllb-200-distilled-600M")
+		model = AutoModelForSeq2SeqLM.from_pretrained("facebook/nllb-200-distilled-600M")
+		article = self.split_sentences(payload['inputs'], 'en')
+		# inputs = tokenizer(payload['input'], return_tensors="pt")
+		out_text = ""
+		for a in article:
+			inputs = tokenizer(a, return_tensors="pt")
+			translated_tokens = model.generate(**inputs, forced_bos_token_id=tokenizer.lang_code_to_id["hin_Deva"], max_length=100)
+			translated_sent = tokenizer.batch_decode(translated_tokens, skip_special_tokens=True)[0]
+			out_text = out_text.join(translated_sent)
+		return out_text
+
+
+	def get_in_indic(self, text, language='Hindi'):
+		tokenizer = AutoTokenizer.from_pretrained("facebook/nllb-200-distilled-600M")
+		model = AutoModelForSeq2SeqLM.from_pretrained("facebook/nllb-200-distilled-600M")
+		inputs = tokenizer(text, return_tensors="pt")
+		
+		code = "eng_Latn"
+		if language == 'Hindi':
+			code= "hin_Deva"
+		elif language == 'Marathi':
+			code = "mar_Deva"
+
+		translated_tokens = model.generate(
+			**inputs,
+			forced_bos_token_id=tokenizer.lang_code_to_id[code],
+			max_length=1000
+		)
+		
+		out_text = tokenizer.batch_decode(translated_tokens, skip_special_tokens=True)[0]
+		return out_text
+
+
+	def get_indic_google_translate(self, text, language='Hindi'):
+		# Init the Google API translator
+		translator = Translator()
+		translations = translator.translate(text, dest=constants_utils.INDIC_LANGUAGE.get(language, 'en'))
+		return str(translations.text)

utils/weather.py

@@ -0,0 +1,202 @@
+import requests
+from bs4 import BeautifulSoup as bs
+
+
+STATE_CODES = {
+	'Andaman-Nicobar': '01',
+	'Andhra-Pradesh': '02',
+	'Arunachal-Pradesh': '03',
+	'Assam': '04',
+	'Bihar': '05',
+	'Chandigarh': '06',
+	'Chhattisgarh': '07',
+	'Dadra-and-Nagar-Haveli': '08',
+	'Daman-and-Diu': '09',
+	'Delhi': '10',
+	'Goa': '11',
+	'Gujarat': '12',
+	'Haryana': '13',
+	# 14
+	'Himachal-Pradesh': '15',
+	'Jammu-Kashmir': '16',
+	'Jharkhand': '17',
+	'Karnataka': '18',
+	'Kerala': '19',
+	'Lakshadweep': '20',
+	'Madhya-Pradesh': '21',
+	'Maharashtra': '22',
+	'Manipur': '23',
+	'Meghalaya': '24',
+	'Mizoram': '25',
+	'Nagaland': '26',
+	'Odisha': '27',
+	'Pondicherry': '28',
+	'Punjab': '29',
+	'Rajasthan': '30',
+	'Sikkim': '31',
+	'Tamilnadu': '32',
+	'Telangana': '33',
+	'Tripura': '34',
+	'Uttar-Pradesh': '35',
+	'Uttarakhand': '36',
+	'West-Bengal': '37',
+}
+
+# List of states that are given as the input selection to https://nwp.imd.gov.in/blf/blf_temp/ to get the weather forecast
+STATES = {
+	'Andaman-Nicobar': {},
+	
+	'Andhra-Pradesh': {},
+	
+	'Arunachal-Pradesh': {},
+	
+	'Assam': {},
+	
+	'Bihar': {},
+	
+	'Chandigarh': {},
+	
+	'Chhattisgarh': {},
+	
+	'Dadra-and-Nagar-Haveli': {},
+	
+	'Daman-and-Diu': {},
+	
+	'Delhi': {
+		'CENTRAL-DELHI': ['CENTRAL-DELHI'],
+		'EAST-DELHI': ['EAST-DELHI'],
+		'NEW-DELHI': ['NEW-DELHI'],
+		'NORTH-DELHI': ['NORTH-DELHI'],
+		'NORTH-EAST-DELHI': ['NORTH-EAST-DELHI'],
+		'NORTH-WEST-DELHI': ['NORTH-WEST-DELHI'],
+		'SHAHDARA': ['SHAHDARA'],
+		'SOUTH-DELHI': ['SOUTH-DELHI'],
+		'SOUTH-EAST-DELHI': ['SOUTH-EAST-DELHI'],
+		'SOUTH-WEST-DELHI': ['SOUTH-WEST-DELHI'],
+		'WEST-DELHI': ['WEST-DELHI'],
+	},
+	
+	'Goa': {},
+	
+	'Gujarat': {
+		'AHMADABAD': ['AHMEDABAD-CITY', 'BAVLA', 'DASKROI', 'DETROJ-RAMPURA', 'DHANDHUKA', 'DHOLERA', 'DHOLKA', 'MANDAL', 'SANAND', 'VIRAMGAM'],
+		'AMRELI': ['AMRELI', 'BABRA', 'BAGASARA', 'DHARI', 'JAFRABAD', 'KHAMBHA', 'KUNKAVAV-VADIA', 'LATHI', 'LILIA', 'RAJULA', 'SAVERKUNDLA'],
+		'ANAND': [],
+		'ARVALLI': [],
+		'BANASKANTHA': [],
+		'BHARUCH': [],
+		'BHAVNAGAR': [],
+		'BOTAD': [],
+		'CHHOTAUDEPUR': [],
+		'DANG': [],
+		'DEVBHUMI-DWARKA': [],
+		'DOHAD': [],
+		'GANDHINAGAR': [],
+		'GIR-SOMNATH': [],
+		'JAMNAGAR': [],
+		'JUNAGADH': [],
+		'KACHCHH': [],
+		'KHEDA': [],
+		'MAHESANA': [],
+		'MAHISAGAR': [],
+		'MORBI': [],
+		'NARMADA': [],
+		'NAVSARI': [],
+		'PANCH-MAHALS': [],
+		'PATAN': [],
+		'PORBANDAR': [],
+		'RAJKOT': [],
+		'SABAR-KANTHA': [],
+		'SURAT': ['BARDOLI', 'CHORASI', 'KAMREJ', 'MAHUVA', 'MANDVI', 'MANGROL', 'OLPAD', 'PALSANA', 'SURAT-CITY', 'UMARPADA'],
+		'SURENDRANAGAR': [],
+		'TAPI': [],
+		'VADODARA': [],
+		'VALSAD': [],
+	},
+	
+	'Haryana': {},
+	
+	'Himachal-Pradesh': {},
+	
+	'Jammu-Kashmir': {},
+	
+	'Jharkhand': {},
+	
+	'Karnataka': {},
+	
+	'Kerala': {},
+	
+	'Lakshadweep': {},
+	
+	'Madhya-Pradesh': {},
+	
+	'Maharashtra': {},
+	
+	'Manipur': {},
+	
+	'Meghalaya': {},
+	
+	'Mizoram': {},
+	
+	'Nagaland': {},
+	
+	'Odisha': {},
+	
+	'Pondicherry': {},
+	
+	'Punjab': {},
+	
+	'Rajasthan': {},
+	
+	'Sikkim': {},
+	
+	'Tamilnadu': {},
+	
+	'Telangana': {},
+	
+	'Tripura': {},
+	
+	'Uttar-Pradesh': {},
+	
+	'Uttarakhand': {},
+	
+	'West-Bengal': {},
+}
+
+
+
+class WEATHER:
+	def __init__(self):
+		self.base_url = 'https://nwp.imd.gov.in/blf/blf_temp'
+		
+
+	# Weather forecast from Govt. website
+	def get_weather_forecast(self, state, district, is_block_level=False):
+		self.district_url = f"{self.base_url}/block.php?dis={STATE_CODES.get(state, '') + district}"
+		self.block_url = f'{self.base_url}/table2.php'
+
+		response = requests.get(self.district_url if not is_block_level else self.block_url)
+		soup = bs(response.text, 'html.parser')
+		scripts = soup.findAll('font')[0]
+		return scripts.text
+
+
+	# Weather using Google weather API
+	def get_weather(self, city):
+		city = city + " weather"
+		city = city.replace(" ", "+")
+
+		headers = {
+			'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3'
+		}
+		response = requests.get(
+			f'https://www.google.com/search?q={city}&oq={city}&aqs=chrome.0.35i39l2j0l4j46j69i60.6128j1j7&sourceid=chrome&ie=UTF-8', headers=headers)
+
+		soup = bs(response.text, 'html.parser')
+		location = soup.select('#wob_loc')[0].getText().strip()
+		time = soup.select('#wob_dts')[0].getText().strip()
+		info = soup.select('#wob_dc')[0].getText().strip()
+		temperature = soup.select('#wob_tm')[0].getText().strip()
+		temperature = temperature + "°C"
+
+		return time, info, temperature