\u001b[0m:\u001b[94m1\u001b[0m \u001b[31m│\u001b[0m\n",
"\u001b[31m╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\u001b[0m\n",
"\u001b[1;91mAttributeError: \u001b[0m\u001b[32m'str'\u001b[0m object has no attribute \u001b[32m'generation_config'\u001b[0m\n"
],
"text/html": [
"╭─────────────────────────────── Traceback (most recent call last) ────────────────────────────────╮\n",
"│ in <cell line: 1>:1 │\n",
"╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\n",
"AttributeError: 'str' object has no attribute 'generation_config'\n",
"\n"
]
},
"metadata": {}
}
]
},
{
"cell_type": "code",
"source": [
"pipeline = transformers.pipeline(\n",
" \"text-generation\",\n",
" model=model,\n",
" tokenizer=tokenizer,\n",
")"
],
"metadata": {
"id": "WwAGjdxPYwul"
},
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"source": [
"prompt = f\"\"\"\n",
": How can I create an account?\n",
":\n",
"\"\"\".strip()\n",
"\n",
"result = pipeline(\n",
" prompt,\n",
" generation_config=generation_config,\n",
")"
],
"metadata": {
"id": "jAUZTOtacqPs"
},
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"source": [
"result"
],
"metadata": {
"id": "o8GCr1qtkHvT"
},
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"source": [
"print(result[0][\"generated_text\"])"
],
"metadata": {
"id": "6nESn6AalyWc"
},
"execution_count": null,
"outputs": []
},
{
"cell_type": "markdown",
"source": [
"## Handler"
],
"metadata": {
"id": "NqAKsbbLxoCN"
}
},
{
"cell_type": "code",
"source": [
"from typing import Any, Dict, List\n",
"\n",
"import torch\n",
"import transformers\n",
"from transformers import AutoModelForCausalLM, AutoTokenizer\n",
"\n",
"dtype = torch.bfloat16 if torch.cuda.get_device_capability()[0] == 8 else torch.float16\n",
"\n",
"\n",
"class EndpointHandler:\n",
" def __init__(self, path=\"\"):\n",
" tokenizer = AutoTokenizer.from_pretrained(path, trust_remote_code=True)\n",
" model = AutoModelForCausalLM.from_pretrained(\n",
" path,\n",
" return_dict=True,\n",
" device_map=\"auto\",\n",
" load_in_8bit=True,\n",
" torch_dtype=dtype,\n",
" trust_remote_code=True,\n",
" )\n",
"\n",
" generation_config = model.generation_config\n",
" generation_config.max_new_tokens = 60\n",
" generation_config.temperature = 0\n",
" generation_config.num_return_sequences = 1\n",
" generation_config.pad_token_id = tokenizer.eos_token_id\n",
" generation_config.eos_token_id = tokenizer.eos_token_id\n",
" self.generation_config = generation_config\n",
"\n",
" self.pipeline = transformers.pipeline(\n",
" \"text-generation\", model=model, tokenizer=tokenizer\n",
" )\n",
"\n",
" def __call__(self, data: Dict[str, Any]) -> Dict[str, Any]:\n",
" prompt = data.pop(\"inputs\", data)\n",
" result = self.pipeline(prompt, generation_config=self.generation_config)\n",
" return result"
],
"metadata": {
"id": "lHyAbDi2p3nR"
},
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"source": [
"MODEL_ID = \"curiousily/falcon-7b-qlora-chat-support-bot-faq-merged\"\n",
"my_handler = EndpointHandler(path=MODEL_ID)"
],
"metadata": {
"id": "X3Xm2cxkupbL"
},
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"source": [
"prompt = f\"\"\"\n",
": How can I create an account?\n",
":\n",
"\"\"\".strip()\n",
"\n",
"payload = {\"inputs\": prompt}"
],
"metadata": {
"id": "RXuHuN9ButKw"
},
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"source": [
"prediction = my_handler(payload)"
],
"metadata": {
"id": "ZKvctyk2u7DV"
},
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"source": [
"prediction"
],
"metadata": {
"id": "RF-ApPuEvH8X"
},
"execution_count": null,
"outputs": []
},
{
"cell_type": "markdown",
"source": [
"## Use the API"
],
"metadata": {
"id": "CJc_5W7XeHoy"
}
},
{
"cell_type": "code",
"source": [
"import requests"
],
"metadata": {
"id": "C-pZ8PWgeGwu"
},
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"source": [
"API_URL = \"https://o7oz3w0gfm6mo8sj.us-east-1.aws.endpoints.huggingface.cloud\"\n",
"API_TOKEN = \"YOUR_HUGGINGFACE_TOKEN\"\n",
"headers = {\"Authorization\": f\"Bearer {API_TOKEN}\"}\n",
"\n",
"prompt = f\"\"\"\n",
": How can I create an account?\n",
":\n",
"\"\"\".strip()\n",
"\n",
"payload = {\"inputs\": prompt}"
],
"metadata": {
"id": "jQXxn0qze6yR"
},
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"source": [
"resp = requests.post(API_URL, json=payload, headers=headers)\n",
"payload = resp.json()"
],
"metadata": {
"id": "7IOjZ9pKfCGH"
},
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"source": [
"payload"
],
"metadata": {
"id": "qNX0S8aJhPTE"
},
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"source": [
"print(payload[0][\"generated_text\"])"
],
"metadata": {
"id": "5lFMHXLBhGSQ"
},
"execution_count": null,
"outputs": []
}
]
}