File size: 7,713 Bytes
acc4ffe |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 |
### Set up environment variables:
#
# export IMUN_URL="https://cognitivewudev.azure-api.net/computervision/imageanalysis:analyze"
# export IMUN_SUBSCRIPTION_KEY=a*
# export IMUN_SUBSCRIPTION_KEY2=a*
# export IMUN_OCR_SUBSCRIPTION_KEY=a*
# export IMUN_PARAMS="api-version=2023-02-01-preview&model-version=latest&features=denseCaptions,Tags"
# export OPENAI_API_KEY=sk-*
#
import os
import requests
from langchain import ConversationChain, LLMChain
from langchain.agents import load_tools, initialize_agent, Tool
from langchain.tools.bing_search.tool import BingSearchRun, BingSearchAPIWrapper
from langchain.chains.conversation.memory import ConversationBufferMemory
from langchain.chains import PALChain
from langchain.llms import OpenAI, AzureOpenAI
from langchain.utilities import ImunAPIWrapper, ImunMultiAPIWrapper
MAX_TOKENS = 512
# llm = OpenAI(temperature=0, max_tokens=MAX_TOKENS)
# llm = AzureOpenAI(deployment_name="text-chat-davinci-002", model_name="text-chat-davinci-002", temperature=1, top_p=0.9, max_tokens=MAX_TOKENS)
# llm = AzureOpenAI(deployment_name="text-chat-davinci-002", model_name="text-003", temperature=0, max_tokens=MAX_TOKENS)
llm = AzureOpenAI(deployment_name="text-chat-davinci-002", model_name="text-chat-davinci-002", temperature=0, max_tokens=MAX_TOKENS)
# llm = AzureOpenAI(deployment_name="gpt-35-turbo-version-0301", model_name="gpt-35-turbo (version 0301)", temperature=0, max_tokens=MAX_TOKENS)
# tool_names = ['imun']
# tools = load_tools(tool_names, llm=llm)
memory = ConversationBufferMemory(memory_key="chat_history")
imun_dense = ImunAPIWrapper(
imun_url="https://ehazarwestus.cognitiveservices.azure.com/computervision/imageanalysis:analyze",
params="api-version=2023-02-01-preview&model-version=latest&features=denseCaptions",
imun_subscription_key=os.environ["IMUN_SUBSCRIPTION_KEY2"])
imun = ImunAPIWrapper()
imun = ImunMultiAPIWrapper(imuns=[imun, imun_dense])
imun_celeb = ImunAPIWrapper(
imun_url="https://cvfiahmed.cognitiveservices.azure.com/vision/v3.2/models/celebrities/analyze",
params="")
imun_read = ImunAPIWrapper(
imun_url="https://vigehazar.cognitiveservices.azure.com/formrecognizer/documentModels/prebuilt-read:analyze",
params="api-version=2022-08-31",
imun_subscription_key=os.environ["IMUN_OCR_SUBSCRIPTION_KEY"])
imun_receipt = ImunAPIWrapper(
imun_url="https://vigehazar.cognitiveservices.azure.com/formrecognizer/documentModels/prebuilt-receipt:analyze",
params="api-version=2022-08-31",
imun_subscription_key=os.environ["IMUN_OCR_SUBSCRIPTION_KEY"])
imun_businesscard = ImunAPIWrapper(
imun_url="https://vigehazar.cognitiveservices.azure.com/formrecognizer/documentModels/prebuilt-businessCard:analyze",
params="api-version=2022-08-31",
imun_subscription_key=os.environ["IMUN_OCR_SUBSCRIPTION_KEY"])
imun_layout = ImunAPIWrapper(
imun_url="https://vigehazar.cognitiveservices.azure.com/formrecognizer/documentModels/prebuilt-layout:analyze",
params="api-version=2022-08-31",
imun_subscription_key=os.environ["IMUN_OCR_SUBSCRIPTION_KEY"])
bing = BingSearchAPIWrapper(k=2)
def edit_photo(query: str) -> str:
endpoint = "http://10.123.124.92:7863/"
query = query.strip()
url_idx = query.rfind(" ")
img_url = query[url_idx + 1:].strip()
if img_url.endswith((".", "?")):
img_url = img_url[:-1]
if not img_url.startswith(("http://", "https://")):
return "Invalid image URL"
img_url = img_url.replace("0.0.0.0", "10.123.124.92")
instruction = query[:url_idx]
# This should be some internal IP to wherever the server runs
job = {"image_path": img_url, "instruction": instruction}
response = requests.post(endpoint, json=job)
if response.status_code != 200:
return "Could not finish the task try again later!"
return "Here is the edited image " + endpoint + response.json()["edited_image"]
# these tools should not step on each other's toes
tools = [
Tool(
name="PAL-MATH",
func=PALChain.from_math_prompt(llm).run,
description=(
"A wrapper around calculator. "
"A language model that is really good at solving complex word math problems."
"Input should be a fully worded hard word math problem."
)
),
Tool(
name = "Image Understanding",
func=imun.run,
description=(
"A wrapper around Image Understanding. "
"Useful for when you need to understand what is inside an image (objects, texts, people)."
"Input should be an image url, or path to an image file (e.g. .jpg, .png)."
)
),
Tool(
name = "OCR Understanding",
func=imun_read.run,
description=(
"A wrapper around OCR Understanding (Optical Character Recognition). "
"Useful after Image Understanding tool has found text or handwriting is present in the image tags."
"This tool can find the actual text, written name, or product name in the image."
"Input should be an image url, or path to an image file (e.g. .jpg, .png)."
)
),
Tool(
name = "Receipt Understanding",
func=imun_receipt.run,
description=(
"A wrapper receipt understanding. "
"Useful after Image Understanding tool has recognized a receipt in the image tags."
"This tool can find the actual receipt text, prices and detailed items."
"Input should be an image url, or path to an image file (e.g. .jpg, .png)."
)
),
Tool(
name = "Business Card Understanding",
func=imun_businesscard.run,
description=(
"A wrapper around business card understanding. "
"Useful after Image Understanding tool has recognized businesscard in the image tags."
"This tool can find the actual business card text, name, address, email, website on the card."
"Input should be an image url, or path to an image file (e.g. .jpg, .png)."
)
),
Tool(
name = "Layout Understanding",
func=imun_layout.run,
description=(
"A wrapper around layout and table understanding. "
"Useful after Image Understanding tool has recognized businesscard in the image tags."
"This tool can find the actual business card text, name, address, email, website on the card."
"Input should be an image url, or path to an image file (e.g. .jpg, .png)."
)
),
Tool(
name = "Celebrity Understanding",
func=imun_celeb.run,
description=(
"A wrapper around celebrity understanding. "
"Useful after Image Understanding tool has recognized people in the image tags that could be celebrities."
"This tool can find the name of celebrities in the image."
"Input should be an image url, or path to an image file (e.g. .jpg, .png)."
)
),
BingSearchRun(api_wrapper=bing),
Tool(
name = "Photo Editing",
func=edit_photo,
description=(
"A wrapper around photo editing. "
"Useful to edit an image with a given instruction."
"Input should be an image url, or path to an image file (e.g. .jpg, .png)."
)
),
]
# chain = initialize_agent(tools, llm, agent="conversational-react-description", verbose=True, memory=memory)
chain = initialize_agent(tools, llm, agent="conversational-assistant", verbose=True, memory=memory, return_intermediate_steps=True, max_iterations=4)
# output = chain.conversation("what is the meaning of life")
output = chain.conversation("translate these words to English: Montrealde, La poubelle, l'éboueur et, le citoyen:, C'est un, ménage à trois.")
print (output)
|