{"cells":[{"cell_type":"code","execution_count":3,"metadata":{"executionInfo":{"elapsed":173,"status":"ok","timestamp":1702156660830,"user":{"displayName":"Sebastián M.","userId":"05974347998580401662"},"user_tz":360},"id":"pEb6CJl3ZFKP"},"outputs":[],"source":["#! pip install tiktoken"]},{"cell_type":"code","execution_count":4,"metadata":{"executionInfo":{"elapsed":2,"status":"ok","timestamp":1702156660831,"user":{"displayName":"Sebastián M.","userId":"05974347998580401662"},"user_tz":360},"id":"zj6dSJC3Y01I"},"outputs":[],"source":["import json\n","import tiktoken # for token counting\n","import numpy as np\n","from collections import defaultdict"]},{"cell_type":"markdown","metadata":{"id":"SmN-A6KJZQPo"},"source":["<br>\n","<br>\n","\n","## Data Loading"]},{"cell_type":"code","execution_count":6,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":6,"status":"ok","timestamp":1702156661009,"user":{"displayName":"Sebastián M.","userId":"05974347998580401662"},"user_tz":360},"id":"-geBXCMTZObN","outputId":"924e87bd-4c01-4eff-ac93-10f07e27fe41"},"outputs":[{"name":"stdout","output_type":"stream","text":["\n","Num examples: 670\n","First example:\n","{'role': 'system', 'content': 'CARSE es un chatbot que imita el estilo en que Sebas conversa. Fue creado con amor solo para Carmen. Su objetivo es ofrecer una experiencia de chat divertida y familiar que exprese la personalidad, el humor y el amor de Sebas hacia Carmen.'}\n","{'role': 'user', 'content': 'Tengo mucho sueño aún'}\n","{'role': 'assistant', 'content': 'Ay amorcito, a ver si te echas a dormir en el coche un rato'}\n"]}],"source":["data_path = \"Training_data/Training_Prompts.jsonl\"\n","\n","# Load the dataset\n","with open(data_path, 'r', encoding='utf-8') as f:\n","    dataset = [json.loads(line) for line in f]\n","\n","# Initial dataset stats\n","print(\"\\nNum examples:\", len(dataset))\n","print(\"First example:\")\n","for message in dataset[0][\"messages\"]:\n","    print(message)"]},{"cell_type":"markdown","metadata":{"id":"UfxL9TQMaCCG"},"source":["<br>\n","<br>\n","\n","## Format Validation"]},{"cell_type":"code","execution_count":7,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":4,"status":"ok","timestamp":1702156661009,"user":{"displayName":"Sebastián M.","userId":"05974347998580401662"},"user_tz":360},"id":"bOXkqnLgZakM","outputId":"8c716779-0f83-4fbf-c302-56d834735327"},"outputs":[{"name":"stdout","output_type":"stream","text":["\n","No errors found\n"]}],"source":["# Format error checks\n","format_errors = defaultdict(int)\n","\n","for ex in dataset:\n","    if not isinstance(ex, dict):\n","        format_errors[\"data_type\"] += 1\n","        continue\n","\n","    messages = ex.get(\"messages\", None)\n","    if not messages:\n","        format_errors[\"missing_messages_list\"] += 1\n","        continue\n","\n","    for message in messages:\n","        if \"role\" not in message or \"content\" not in message:\n","            format_errors[\"message_missing_key\"] += 1\n","\n","        if any(k not in (\"role\", \"content\", \"name\", \"function_call\") for k in message):\n","            format_errors[\"message_unrecognized_key\"] += 1\n","\n","        if message.get(\"role\", None) not in (\"system\", \"user\", \"assistant\", \"function\"):\n","            format_errors[\"unrecognized_role\"] += 1\n","\n","        content = message.get(\"content\", None)\n","        function_call = message.get(\"function_call\", None)\n","\n","        if (not content and not function_call) or not isinstance(content, str):\n","            format_errors[\"missing_content\"] += 1\n","\n","    if not any(message.get(\"role\", None) == \"assistant\" for message in messages):\n","        format_errors[\"example_missing_assistant_message\"] += 1\n","\n","if format_errors:\n","    print(\"\\nFound errors:\")\n","    for k, v in format_errors.items():\n","        print(f\"{k}: {v}\")\n","else:\n","    print(\"\\nNo errors found\")"]},{"cell_type":"markdown","metadata":{"id":"UXrIdBKtaHkx"},"source":["<br>\n","<br>\n","\n","## Token Counting Utilities"]},{"cell_type":"code","execution_count":8,"metadata":{"executionInfo":{"elapsed":3,"status":"ok","timestamp":1702156661009,"user":{"displayName":"Sebastián M.","userId":"05974347998580401662"},"user_tz":360},"id":"A6ev3s_TaGeJ"},"outputs":[],"source":["encoding = tiktoken.get_encoding(\"cl100k_base\")\n","\n","# not exact!\n","# simplified from https://github.com/openai/openai-cookbook/blob/main/examples/How_to_count_tokens_with_tiktoken.ipynb\n","def num_tokens_from_messages(messages, tokens_per_message=3, tokens_per_name=1):\n","    num_tokens = 0\n","    for message in messages:\n","        num_tokens += tokens_per_message\n","        for key, value in message.items():\n","            num_tokens += len(encoding.encode(value))\n","            if key == \"name\":\n","                num_tokens += tokens_per_name\n","    num_tokens += 3\n","    return num_tokens\n","\n","def num_assistant_tokens_from_messages(messages):\n","    num_tokens = 0\n","    for message in messages:\n","        if message[\"role\"] == \"assistant\":\n","            num_tokens += len(encoding.encode(message[\"content\"]))\n","    return num_tokens\n","\n","def print_distribution(values, name):\n","    print(f\"\\n#### Distribution of {name}:\")\n","    print(f\"min / max: {min(values)}, {max(values)}\")\n","    print(f\"mean / median: {np.mean(values)}, {np.median(values)}\")\n","    print(f\"p5 / p95: {np.quantile(values, 0.1)}, {np.quantile(values, 0.9)}\")"]},{"cell_type":"markdown","metadata":{"id":"2duSc8L7aYki"},"source":["<br>\n","<br>\n","\n","## Data Warnings and Token Counts"]},{"cell_type":"code","execution_count":9,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":3,"status":"ok","timestamp":1702156661009,"user":{"displayName":"Sebastián M.","userId":"05974347998580401662"},"user_tz":360},"id":"CfDEmpgiaLuS","outputId":"f1e8ed95-f77b-4340-80ce-b08ee6025a2e"},"outputs":[{"name":"stdout","output_type":"stream","text":["\n","Num examples missing system message: 0\n","Num examples missing user message: 0\n","\n","#### Distribution of num_messages_per_example:\n","min / max: 3, 3\n","mean / median: 3.0, 3.0\n","p5 / p95: 3.0, 3.0\n","\n","#### Distribution of num_total_tokens_per_example:\n","min / max: 77, 263\n","mean / median: 102.54776119402985, 97.0\n","p5 / p95: 84.0, 120.10000000000002\n","\n","#### Distribution of num_assistant_tokens_per_example:\n","min / max: 2, 184\n","mean / median: 19.9955223880597, 15.0\n","p5 / p95: 6.0, 34.0\n","\n","0 examples may be over the 4096 token limit, they will be truncated during fine-tuning\n"]}],"source":["# Warnings and tokens counts\n","n_missing_system = 0\n","n_missing_user = 0\n","n_messages = []\n","convo_lens = []\n","assistant_message_lens = []\n","\n","for ex in dataset:\n","    messages = ex[\"messages\"]\n","    if not any(message[\"role\"] == \"system\" for message in messages):\n","        n_missing_system += 1\n","    if not any(message[\"role\"] == \"user\" for message in messages):\n","        n_missing_user += 1\n","    n_messages.append(len(messages))\n","    convo_lens.append(num_tokens_from_messages(messages))\n","    assistant_message_lens.append(num_assistant_tokens_from_messages(messages))\n","\n","print(\"\\nNum examples missing system message:\", n_missing_system)\n","print(\"Num examples missing user message:\", n_missing_user)\n","print_distribution(n_messages, \"num_messages_per_example\")\n","print_distribution(convo_lens, \"num_total_tokens_per_example\")\n","print_distribution(assistant_message_lens, \"num_assistant_tokens_per_example\")\n","n_too_long = sum(l > 4096 for l in convo_lens)\n","print(f\"\\n{n_too_long} examples may be over the 4096 token limit, they will be truncated during fine-tuning\")"]},{"cell_type":"markdown","metadata":{"id":"x8GusuUOapUB"},"source":["<br>\n","<br>\n","\n","## Cost Estimation"]},{"cell_type":"code","execution_count":10,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":194,"status":"ok","timestamp":1702156661200,"user":{"displayName":"Sebastián M.","userId":"05974347998580401662"},"user_tz":360},"id":"EnAWqf3SadOj","outputId":"3d0527e9-a3cf-4b6e-9ce4-89a99611d219"},"outputs":[{"name":"stdout","output_type":"stream","text":["\n","Dataset has ~68707 tokens that will be charged for during training\n","By default, you'll train for 3 epochs on this dataset\n","By default, you'll be charged for ~206121 tokens\n"]}],"source":["# Pricing and default n_epochs estimate\n","MAX_TOKENS_PER_EXAMPLE = 4096\n","\n","TARGET_EPOCHS = 3\n","MIN_TARGET_EXAMPLES = 100\n","MAX_TARGET_EXAMPLES = 25000\n","MIN_DEFAULT_EPOCHS = 1\n","MAX_DEFAULT_EPOCHS = 25\n","\n","n_epochs = TARGET_EPOCHS\n","n_train_examples = len(dataset)\n","if n_train_examples * TARGET_EPOCHS < MIN_TARGET_EXAMPLES:\n","    n_epochs = min(MAX_DEFAULT_EPOCHS, MIN_TARGET_EXAMPLES // n_train_examples)\n","elif n_train_examples * TARGET_EPOCHS > MAX_TARGET_EXAMPLES:\n","    n_epochs = max(MIN_DEFAULT_EPOCHS, MAX_TARGET_EXAMPLES // n_train_examples)\n","\n","n_billing_tokens_in_dataset = sum(min(MAX_TOKENS_PER_EXAMPLE, length) for length in convo_lens)\n","print(f\"\\nDataset has ~{n_billing_tokens_in_dataset} tokens that will be charged for during training\")\n","print(f\"By default, you'll train for {n_epochs} epochs on this dataset\")\n","print(f\"By default, you'll be charged for ~{n_epochs * n_billing_tokens_in_dataset} tokens\")"]},{"cell_type":"code","execution_count":11,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":2,"status":"ok","timestamp":1702156777183,"user":{"displayName":"Sebastián M.","userId":"05974347998580401662"},"user_tz":360},"id":"-a820sVicdat","outputId":"c96c5167-c91e-4e1c-e8a4-0bd16a6aeb70"},"outputs":[{"data":{"text/plain":["1.648968"]},"execution_count":11,"metadata":{},"output_type":"execute_result"}],"source":["# Suponiendo que cargan $0.008/1k tokens\n","value = 0.008\n","\n","final_money = (n_epochs * n_billing_tokens_in_dataset)*value/1000\n","final_money"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"0anBNWtKczWR"},"outputs":[],"source":[]}],"metadata":{"colab":{"authorship_tag":"ABX9TyMzN+lsyb4smMC9hf/gYgCs","provenance":[]},"kernelspec":{"display_name":"Python 3","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.9.7"}},"nbformat":4,"nbformat_minor":0}