File size: 2,673 Bytes
7abe6e1 6b17270 7abe6e1 85a4de7 408fcde 332de37 6b17270 50a12bf 6b17270 b34c465 50a12bf 6b17270 7abe6e1 555ef07 7abe6e1 2c1aa3b 899d9e8 2c1aa3b f9bc4fc ef6a2a0 e207b76 ef6a2a0 709bce3 ef6a2a0 9e8e54f af7cd6b 7abe6e1 9468ae4 72e40fb dfa34b3 7abe6e1 f9bc4fc 555ef07 04e5ddd 9468ae4 dfa34b3 7abe6e1 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 |
import gradio as gr
import boto3, json, os, wandb
from dotenv import load_dotenv, find_dotenv
_ = load_dotenv(find_dotenv())
aws_access_key_id = os.environ["AWS_ACCESS_KEY_ID"]
aws_secret_access_key = os.environ["AWS_SECRET_ACCESS_KEY"]
wandb_api_key = os.environ["WANDB_API_KEY"]
config = {
"max_tokens_to_sample": 300,
"model": "anthropic.claude-v2",
"temperature": 0,
"top_k": 250,
"top_p": 0.999,
}
bedrock_runtime = boto3.client(
aws_access_key_id = aws_access_key_id,
aws_secret_access_key = aws_secret_access_key,
service_name = "bedrock-runtime",
region_name = "us-west-2"
)
def wandb_log(prompt, completion):
wandb.login(key = wandb_api_key)
wandb.init(project = "bedrock-llm", config = config)
wandb.log({"prompt": str(prompt), "completion": str(completion)})
wandb.finish()
def invoke(prompt):
if (prompt == ""):
raise gr.Error("Prompt is required.")
completion = ""
try:
body = json.dumps({"prompt": "\n\nHuman: " + prompt + "\n\nAssistant: ",
"max_tokens_to_sample": config["max_tokens_to_sample"],
"temperature": config["temperature"],
"top_k": config["top_k"],
"top_p": config["top_p"],
"stop_sequences": ["\n\nHuman: "]
})
modelId = config["model"]
accept = "application/json"
contentType = "application/json"
response = bedrock_runtime.invoke_model(body = body, modelId = modelId, accept = accept, contentType = contentType)
response_body = json.loads(response.get("body").read())
completion = response_body["completion"]
except Exception as e:
completion = e
raise gr.Error(e)
finally:
wandb_log(prompt, completion)
return completion
#return "🛑 Execution is commented out. To view the source code see https://huggingface.co/spaces/bstraehle/amazon-bedrock-llm/tree/main."
description = """<a href='https://www.gradio.app/'>Gradio</a> UI using <a href='https://aws.amazon.com/bedrock/'>Amazon Bedrock</a> API
with <a href='https://www.anthropic.com/'>Anthropic</a> Claude 2 foundation model.
RAG evaluation via <a href='https://wandb.ai/bstraehle'>Weights & Biases</a>."""
gr.close_all()
demo = gr.Interface(fn=invoke,
inputs = [gr.Textbox(label = "Prompt", lines = 1)],
outputs = [gr.Textbox(label = "Completion", lines = 1)],
title = "Generative AI - LLM",
description = description)
demo.launch() |