{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip install sagemaker --upgrade"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "!tar -cf model.tar.gz --use-compress-program=pigz *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sagemaker.config INFO - Not applying SDK defaults from location: /Library/Application Support/sagemaker/config.yaml\n",
      "sagemaker.config INFO - Not applying SDK defaults from location: /Users/tom/Library/Application Support/sagemaker/config.yaml\n",
      "sagemaker.config INFO - Not applying SDK defaults from location: /Library/Application Support/sagemaker/config.yaml\n",
      "sagemaker.config INFO - Not applying SDK defaults from location: /Users/tom/Library/Application Support/sagemaker/config.yaml\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Couldn't call 'get_role' to get Role ARN from role name arn:aws:iam::297308036828:root to get Role path.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sagemaker.config INFO - Not applying SDK defaults from location: /Library/Application Support/sagemaker/config.yaml\n",
      "sagemaker.config INFO - Not applying SDK defaults from location: /Users/tom/Library/Application Support/sagemaker/config.yaml\n",
      "sagemaker role arn: arn:aws:iam::297308036828:role/service-role/AmazonSageMaker-ExecutionRole-20231008T201275\n",
      "sagemaker bucket: sagemaker-us-west-2-297308036828\n",
      "sagemaker session region: us-west-2\n"
     ]
    }
   ],
   "source": [
    "import sagemaker\n",
    "import boto3\n",
    "sess = sagemaker.Session()\n",
    "# sagemaker session bucket -> used for uploading data, models and logs\n",
    "# sagemaker will automatically create this bucket if it not exists\n",
    "sagemaker_session_bucket=None\n",
    "if sagemaker_session_bucket is None and sess is not None:\n",
    "    # set to default bucket if a bucket name is not given\n",
    "    sagemaker_session_bucket = sess.default_bucket()\n",
    "\n",
    "try:\n",
    "    role = sagemaker.get_execution_role()\n",
    "except ValueError:\n",
    "    iam = boto3.client('iam')\n",
    "    role = iam.get_role(RoleName='AmazonSageMaker-ExecutionRole-20231008T201275')['Role']['Arn']\n",
    "\n",
    "sess = sagemaker.Session(default_bucket=sagemaker_session_bucket)\n",
    "\n",
    "print(f\"sagemaker role arn: {role}\")\n",
    "print(f\"sagemaker bucket: {sess.default_bucket()}\")\n",
    "print(f\"sagemaker session region: {sess.boto_region_name}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sagemaker.config INFO - Not applying SDK defaults from location: /Library/Application Support/sagemaker/config.yaml\n",
      "sagemaker.config INFO - Not applying SDK defaults from location: /Users/tom/Library/Application Support/sagemaker/config.yaml\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "model uploaded to: s3://sagemaker-us-west-2-297308036828/llava-v1.5-7b/model.tar.gz\n"
     ]
    }
   ],
   "source": [
    "from sagemaker.s3 import S3Uploader\n",
    "\n",
    "# upload model.tar.gz to s3\n",
    "s3_model_uri = S3Uploader.upload(local_path=\"./model.tar.gz\", desired_s3_uri=f\"s3://{sess.default_bucket()}/llava-v1.5-7b\")\n",
    "\n",
    "print(f\"model uploaded to: {s3_model_uri}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# s3_model_uri = \"s3://sagemaker-us-west-2-297308036828/llava-v1.5-7b/model.tar.gz\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sagemaker.config INFO - Not applying SDK defaults from location: /Library/Application Support/sagemaker/config.yaml\n",
      "sagemaker.config INFO - Not applying SDK defaults from location: /Users/tom/Library/Application Support/sagemaker/config.yaml\n",
      "sagemaker.config INFO - Not applying SDK defaults from location: /Library/Application Support/sagemaker/config.yaml\n",
      "sagemaker.config INFO - Not applying SDK defaults from location: /Users/tom/Library/Application Support/sagemaker/config.yaml\n",
      "------------------!"
     ]
    }
   ],
   "source": [
    "\n",
    "from sagemaker.huggingface.model import HuggingFaceModel\n",
    "\n",
    "# create Hugging Face Model Class\n",
    "huggingface_model = HuggingFaceModel(\n",
    "   model_data=s3_model_uri,      # path to your model and script\n",
    "   role=role,                    # iam role with permissions to create an Endpoint\n",
    "   transformers_version=\"4.28.1\",  # transformers version used\n",
    "   pytorch_version=\"2.0.0\",       # pytorch version used\n",
    "   py_version='py310',            # python version used\n",
    "   model_server_workers=1\n",
    ")\n",
    "\n",
    "# deploy the endpoint endpoint\n",
    "predictor = huggingface_model.deploy(\n",
    "    initial_instance_count=1,\n",
    "    instance_type=\"ml.g5.xlarge\",\n",
    "    # container_startup_health_check_timeout=600, # increase timeout for large models\n",
    "    # model_data_download_timeout=600, # increase timeout for large models\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(optional)\n",
      "\n",
      "The image is a black and white photograph of a man standing in front of a building. The man is wearing a suit and tie, and he appears to be looking off into the distance. The building in the background is large and imposing, with many windows and a prominent clock tower. The overall atmosphere of the image is one of elegance and sophistication.\n"
     ]
    }
   ],
   "source": [
    "data = {\n",
    "    \"image\" : 'https://raw.githubusercontent.com/haotian-liu/LLaVA/main/images/llava_logo.png', \n",
    "    \"question\" : \"Describe the image and color details.\"\n",
    "}\n",
    "\n",
    "# max_new_tokens = data.pop(\"max_new_tokens\", 1024)\n",
    "# temperature = data.pop(\"temperature\", 0.2)\n",
    "# stop_str = data.pop(\"stop_str\", \"###\")\n",
    "\n",
    "# request\n",
    "output = predictor.predict(data)\n",
    "print(output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The image features a unique and eye-catching toy, which is a red and orange plastic horse with a pair of glasses on its face. The horse has a fire effect, giving it a fiery appearance. The glasses on the horse's face add a whimsical touch to the toy. The overall color scheme of the toy is predominantly red and orange, with the fire effect further enhancing the vibrant colors.\n"
     ]
    }
   ],
   "source": [
    "from llava.conversation import conv_templates, SeparatorStyle\n",
    "from llava.constants import (\n",
    "IMAGE_TOKEN_INDEX,\n",
    "DEFAULT_IMAGE_TOKEN,\n",
    "DEFAULT_IM_START_TOKEN,\n",
    "DEFAULT_IM_END_TOKEN,\n",
    ")\n",
    "\n",
    "raw_prompt = \"Describe the image and color details.\"\n",
    "image_path = \"https://raw.githubusercontent.com/haotian-liu/LLaVA/main/images/llava_logo.png\"\n",
    "\n",
    "conv_mode = \"llava_v1\"\n",
    "conv = conv_templates[conv_mode].copy()\n",
    "roles = conv.roles\n",
    "inp = f\"{roles[0]}: {raw_prompt}\"\n",
    "inp = (\n",
    "    DEFAULT_IM_START_TOKEN + DEFAULT_IMAGE_TOKEN + DEFAULT_IM_END_TOKEN + \"\\n\" + inp\n",
    ")\n",
    "conv.append_message(conv.roles[0], inp)\n",
    "conv.append_message(conv.roles[1], None)\n",
    "prompt = conv.get_prompt()\n",
    "\n",
    "stop_str = conv.sep if conv.sep_style != SeparatorStyle.TWO else conv.sep2\n",
    "\n",
    "\n",
    "data = {\"image\" : image_path, \"question\" : prompt, \"stop_str\" : stop_str}\n",
    "output = predictor.predict(data)\n",
    "print(output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "predictor.delete_endpoint()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sagemaker.huggingface.model import HuggingFacePredictor\n",
    "\n",
    "# initial the endpoint predictor\n",
    "predictor = HuggingFacePredictor(\n",
    "    endpoint_name=\"\",\n",
    "    sagemaker_session=sess\n",
    ")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "llava",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}