Spaces:
Sleeping
Sleeping
File size: 2,645 Bytes
b30ed6a |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 |
from bs4 import BeautifulSoup
import requests
from requests.auth import HTTPBasicAuth
import run_localGPT
def start_training():
training_status = ingest.main()
return training_status
def replace_substring_and_following(input_string, substring):
index = input_string.find(substring)
if index != -1:
return input_string[:index]
else:
return input_string
def ask_question(strQuestion):
answer = run_localGPT.main(device_type='cpu', strQuery=strQuestion)
answer_cleaned = replace_substring_and_following(answer, "Unhelpful Answer")
return answer_cleaned
def transcript(page_id):
url = f"https://srikanthnm.atlassian.net/wiki/rest/api/content/{page_id}?expand=body.storage" # Replace with the actual URL you want to access
username = "srikanth.nm@gmail.com"
password = "ATATT3xFfGF09rugcjiT06v8xMayt5ggayMNiwz4b6w07PWQxPvpi4fMDzwwHxKt-v8dGx49uiulIMKHUUYroeS8cXvMKYfi7sQnFsYNfGslPVqSq1BQrzPhTio-xmYOHcit5ijzU9cSGGa7eLXUMxQTsSQjLhtZ-EQPI8h6aki690_-evLFZmU=3910FFD4"
response = requests.get(url, auth=HTTPBasicAuth(username, password))
# Check if the request was successful (status code 200)
if response.status_code == 200:
# Process the response data (if applicable)
data = response.json()
else:
data = f"Error: Unable to access the URL. Status code: {response.status_code}"
soup = BeautifulSoup(data['body']['storage']['value'],"html.parser")
page_content = soup.get_text()
page_content_cleaned = page_content.replace('\xa0',' ')
page_content_cleaned
with open('SOURCE_DOCUMENTS/confluence.txt', 'w') as outfile:
outfile.write(page_content_cleaned[:1998])
return page_content_cleaned[:1998]
def summarize():
from langchain import PromptTemplate, LLMChain
from langchain.text_splitter import CharacterTextSplitter
from langchain.chains.mapreduce import MapReduceChain
from langchain.prompts import PromptTemplate
model_id = "TheBloke/Llama-2-7B-Chat-GGML"
model_basename = "llama-2-7b-chat.ggmlv3.q4_0.bin"
llm = run_localGPT.load_model(device_type='cpu', model_id=model_id, model_basename=model_basename)
text_splitter = CharacterTextSplitter()
with open("SOURCE_DOCUMENTS/confluence.txt") as f:
file_content = f.read()
texts = text_splitter.split_text(file_content)
from langchain.docstore.document import Document
docs = [Document(page_content=t) for t in texts]
from langchain.chains.summarize import load_summarize_chain
chain = load_summarize_chain(llm, chain_type="map_reduce")
summary = chain.run(docs)
return summary |