{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "## All package installation and libraries imports\n", "### Packages installation" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true, "id": "rd5vZMt_2wrC" }, "outputs": [], "source": [ "#run this cell \n", "!pip install accelerate\n", "!pip install bitsandbytes\n", "!pip install optimum\n", "!pip install auto-gptq\n", "!pip install gradio\n", "\n", "#text-to-speech and speech to text\n", "!pip install TTS\n", "!pip install 'transformers == 4.36'\n", "!pip install numpy\n", "!pip install openai-whisper #Whisper models\n", "\n", "!pip install geopy\n", "\n", "!pip uninstall transformer-engine -y\n", "\n", "\n", "!pip install langchain\n", "!pip install text_generation" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### libraries import" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true, "id": "oOnNfKjX4IAV" }, "outputs": [], "source": [ "#gradio interface\n", "import gradio as gr\n", "\n", "from transformers import AutoModelForCausalLM,AutoTokenizer\n", "import torch\n", "\n", "#STT (speech to text)\n", "from transformers import WhisperProcessor, WhisperForConditionalGeneration\n", "import librosa\n", "\n", "#TTS (text to speech)\n", "import torch\n", "from TTS.api import TTS\n", "from IPython.display import Audio\n", "\n", "#json request for APIs\n", "import requests\n", "import json\n", "\n", "#regular expressions\n", "import re\n", "\n", "#langchain and function calling\n", "from typing import List, Literal, Union\n", "import requests\n", "from functools import partial\n", "from geopy.geocoders import Nominatim\n", "import math\n", "\n", "\n", "#langchain, not used anymore since I had to find another way fast to stop using the endpoint, but could be interesting to reuse \n", "from langchain.tools.base import StructuredTool\n", "from langchain.agents import (\n", " Tool,\n", " AgentExecutor,\n", " LLMSingleActionAgent,\n", " AgentOutputParser,\n", ")\n", "from langchain.schema import AgentAction, AgentFinish, OutputParserException\n", "from langchain.prompts import StringPromptTemplate\n", "from langchain.llms import HuggingFaceTextGenInference\n", "from langchain.chains import LLMChain\n", "\n", "\n", "\n", "from datetime import datetime, timedelta, timezone\n", "from transformers import pipeline\n", "import inspect" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Models loads" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true, "id": "JNALTDb0LT90" }, "outputs": [], "source": [ "# load model and processor for speech-to-text\n", "processor = WhisperProcessor.from_pretrained(\"openai/whisper-small\")\n", "modelw = WhisperForConditionalGeneration.from_pretrained(\"openai/whisper-small\")\n", "modelw.config.forced_decoder_ids = None\n", "\n", "#load model for text to speech\n", "device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n", "tts = TTS(\"tts_models/multilingual/multi-dataset/xtts_v1.1\").to(device)\n", "\n", "#load model language recognition\n", "model_ckpt = \"papluca/xlm-roberta-base-language-detection\"\n", "pipe_language = pipeline(\"text-classification\", model=model_ckpt)\n", "\n", "#load model llama2\n", "mn = 'stabilityai/StableBeluga-7B' #mn = \"TheBloke/Llama-2-7b-Chat-GPTQ\" --> other possibility \n", "model = AutoModelForCausalLM.from_pretrained(mn, device_map=0, load_in_4bit=True) #torch_dtype=torch.float16\n", "tokr = AutoTokenizer.from_pretrained(mn, load_in_4bit=True) #tokenizer\n", "\n", "#NexusRaven for function calling\n", "model_id = \"Nexusflow/NexusRaven-13B\"\n", "tokenizer = AutoTokenizer.from_pretrained(model_id)\n", "modelNexus = AutoModelForCausalLM.from_pretrained(model_id, device_map=0, load_in_4bit=True)\n", "pipe = pipeline(\"text-generation\", model=modelNexus, tokenizer = tokenizer)\n", "\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Function calling with NexusRaven " ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "#FUNCTION CALLING \n", "\n", "#API keys\n", "TOMTOM_KEY= \"your_key\" \n", "WHEATHER_API_KEY = \"your_key\" \n", "\n", "##########################################################\n", "# Step 1: Define the functions you want to articulate. ###\n", "##########################################################\n", "\n", "########################################################################################\n", "# Functions called in the articulated functions (not directly called by the model): ###\n", "########################################################################################\n", "\n", "geolocator = Nominatim(user_agent=\"MyApp\")\n", "\n", "def find_precise_place(lat, lon):\n", " location = geolocator.reverse(str(lat) +\", \" + str(lon))\n", " return location.raw.get('display_name', {})\n", "\n", "def find_coordinates(address):\n", " coord = geolocator.geocode(address)\n", " lat = coord.latitude\n", " lon = coord.longitude\n", " return(lat,lon)\n", "\n", "\n", "def check_city_coordinates(lat = \"\", lon = \"\", city = \"\", **kwargs):\n", " \"\"\"\n", " :param lat: latitude\n", " :param lon: longitude\n", " :param city: name of the city\n", "\n", " Checks if the coordinates correspond to the city, if not update the coordinate to correspond to the city\n", " \"\"\"\n", " if lat != \"0\" and lon != \"0\":\n", " reverse = partial(geolocator.reverse, language=\"en\")\n", " location = reverse(f\"{lat}, {lon}\")\n", " address = location.raw.get('address', {})\n", " city = address.get('city') or address.get('town') or address.get('village') or address.get('county')\n", " else : \n", " reverse = partial(geolocator.reverse, language=\"en\")\n", " location = reverse(f\"{lat}, {lon}\")\n", " address = location.raw.get('address', {})\n", " city_name = address.get('city') or address.get('town') or address.get('village') or address.get('county')\n", " if city_name is None :\n", " city_name = 'not_found'\n", " print(city_name)\n", " if city_name.lower() != city.lower():\n", " coord = geolocator.geocode(city )\n", " if coord is None:\n", " coord = geolocator.geocode(city)\n", " lat = coord.latitude\n", " lon = coord.longitude\n", " return lat, lon, city\n", "\n", "# Select coordinates at equal distance, including the last one\n", "def select_equally_spaced_coordinates(coords, number_of_points=10):\n", " n = len(coords)\n", " selected_coords = []\n", " interval = max((n - 1) / (number_of_points - 1), 1)\n", " for i in range(number_of_points):\n", " # Calculate the index, ensuring it doesn't exceed the bounds of the list\n", " index = int(round(i * interval))\n", " if index < n:\n", " selected_coords.append(coords[index])\n", " return selected_coords\n", "\n", "###################################################\n", "# Functions we want to articulate (APIs calls): ###\n", "###################################################\n", "\n", "def search_along_route(latitude_depart, longitude_depart, city_destination, type_of_poi):\n", " \"\"\"\n", " Return some of the closest points of interest along the route from the depart point, specified by its coordinates and a city destination.\n", " :param latitude_depart (string): Required. Latitude of depart location\n", " :param longitude_depart (string): Required. Longitude of depart location\n", " :param city_destination (string): Required. City destination\n", " :param type_of_poi (string): Required. type of point of interest depending on what the user wants to do.\n", " \"\"\"\n", " \n", " lat_dest, lon_dest = find_coordinates(city_destination)\n", " print(lat_dest)\n", " \n", " r = requests.get('https://api.tomtom.com/routing/1/calculateRoute/{0},{1}:{2},{3}/json?key={4}'.format(\n", " latitude_depart,\n", " longitude_depart,\n", " lat_dest,\n", " lon_dest,\n", " TOMTOM_KEY\n", " ))\n", " \n", " coord_route = select_equally_spaced_coordinates(r.json()['routes'][0]['legs'][0]['points'])\n", "\n", " # The API endpoint for searching along a route\n", " url = f'https://api.tomtom.com/search/2/searchAlongRoute/{type_of_poi}.json?key={TOMTOM_KEY}&maxDetourTime=700&limit=20&sortBy=detourTime'\n", "\n", " # The data payload\n", " payload = {\n", " \"route\": {\n", " \"points\": [\n", " {\"lat\": float(latitude_depart), \"lon\": float(longitude_depart)},\n", " {\"lat\": float(coord_route[1]['latitude']), \"lon\": float(coord_route[1]['longitude'])},\n", " {\"lat\": float(coord_route[2]['latitude']), \"lon\": float(coord_route[2]['longitude'])},\n", " {\"lat\": float(coord_route[3]['latitude']), \"lon\": float(coord_route[3]['longitude'])},\n", " {\"lat\": float(coord_route[4]['latitude']), \"lon\": float(coord_route[4]['longitude'])},\n", " {\"lat\": float(coord_route[5]['latitude']), \"lon\": float(coord_route[5]['longitude'])},\n", " {\"lat\": float(coord_route[6]['latitude']), \"lon\": float(coord_route[6]['longitude'])},\n", " {\"lat\": float(coord_route[7]['latitude']), \"lon\": float(coord_route[7]['longitude'])},\n", " {\"lat\": float(coord_route[8]['latitude']), \"lon\": float(coord_route[8]['longitude'])},\n", " {\"lat\": float(lat_dest), \"lon\": float(lon_dest)},\n", " ]\n", " }\n", " }\n", "\n", " # Make the POST request\n", " response = requests.post(url, json=payload)\n", "\n", " # Check if the request was successful\n", " if response.status_code == 200:\n", " # Parse the JSON response\n", " data = response.json()\n", " print(json.dumps(data, indent=4))\n", " else:\n", " print('Failed to retrieve data:', response.status_code)\n", " answer = \"\"\n", " for result in data['results']:\n", " name = result['poi']['name']\n", " address = result['address']['freeformAddress']\n", " detour_time = result['detourTime']\n", " answer = answer + f\" \\nAlong the route to {city_destination}, there is the {name} at {address} that would represent a detour of {int(detour_time/60)} minutes.\"\n", " \n", " return answer\n", "\n", "\n", "def find_points_of_interest(lat=\"0\", lon=\"0\", city=\"\", type_of_poi=\"restaurant\", **kwargs):\n", " \"\"\"\n", " Return some of the closest points of interest for a specific location and type of point of interest. The more parameters there are, the more precise.\n", " :param lat (string): latitude\n", " :param lon (string): longitude\n", " :param city (string): Required. city\n", " :param type_of_poi (string): Required. type of point of interest depending on what the user wants to do.\n", " \"\"\"\n", " lat, lon, city = check_city_coordinates(lat,lon,city)\n", "\n", " r = requests.get(f'https://api.tomtom.com/search/2/search/{type_of_poi}'\n", " '.json?key={0}&lat={1}&lon={2}&radius=10000&idxSet=POI&limit=100'.format(\n", " TOMTOM_KEY,\n", " lat,\n", " lon\n", " ))\n", "\n", " # Parse JSON from the response\n", " data = r.json()\n", " #print(data)\n", " # Extract results\n", " results = data['results']\n", "\n", " # Sort the results based on distance\n", " sorted_results = sorted(results, key=lambda x: x['dist'])\n", " #print(sorted_results)\n", "\n", " # Format and limit to top 5 results\n", " formatted_results = [\n", " f\"The {type_of_poi} {result['poi']['name']} is {int(result['dist'])} meters away\"\n", " for result in sorted_results[:5]\n", " ]\n", "\n", "\n", " return \". \".join(formatted_results)\n", "\n", "def find_route(lat_depart=\"0\", lon_depart=\"0\", city_depart=\"\", address_destination=\"\", depart_time =\"\", **kwargs):\n", " \"\"\"\n", " Return the distance and the estimated time to go to a specific destination from the current place, at a specified depart time.\n", " :param lat_depart (string): latitude of depart\n", " :param lon_depart (string): longitude of depart\n", " :param city_depart (string): Required. city of depart\n", " :param address_destination (string): Required. The destination\n", " :param depart_time (string): departure hour, in the format '08:00:20'.\n", " \"\"\"\n", " print(address_destination)\n", " date = \"2025-03-29T\"\n", " departure_time = '2024-02-01T' + depart_time\n", " lat, lon, city = check_city_coordinates(lat_depart,lon_depart,city_depart)\n", " lat_dest, lon_dest = find_coordinates(address_destination)\n", " #print(lat_dest, lon_dest)\n", " \n", " #print(departure_time)\n", "\n", " r = requests.get('https://api.tomtom.com/routing/1/calculateRoute/{0},{1}:{2},{3}/json?key={4}&departAt={5}'.format(\n", " lat_depart,\n", " lon_depart,\n", " lat_dest,\n", " lon_dest,\n", " TOMTOM_KEY,\n", " departure_time\n", " ))\n", "\n", " # Parse JSON from the response\n", " data = r.json()\n", " #print(data)\n", " \n", " #print(data)\n", " \n", " result = data['routes'][0]['summary']\n", "\n", " # Calculate distance in kilometers (1 meter = 0.001 kilometers)\n", " distance_km = result['lengthInMeters'] * 0.001\n", "\n", " # Calculate travel time in minutes (1 second = 1/60 minutes)\n", " time_minutes = result['travelTimeInSeconds'] / 60\n", " if time_minutes < 60:\n", " time_display = f\"{time_minutes:.0f} minutes\"\n", " else:\n", " hours = int(time_minutes / 60)\n", " minutes = int(time_minutes % 60)\n", " time_display = f\"{hours} hours\" + (f\" and {minutes} minutes\" if minutes > 0 else \"\")\n", " \n", " # Extract arrival time from the JSON structure\n", " arrival_time_str = result['arrivalTime']\n", "\n", " # Convert string to datetime object\n", " arrival_time = datetime.fromisoformat(arrival_time_str)\n", "\n", " # Extract and display the arrival hour in HH:MM format\n", " arrival_hour_display = arrival_time.strftime(\"%H:%M\")\n", "\n", "\n", " # return the distance and time\n", " return(f\"The route to go to {address_destination} is {distance_km:.2f} km and {time_display}. Leaving now, the arrival time is estimated at {arrival_hour_display} \" )\n", "\n", " \n", " # Sort the results based on distance\n", " #sorted_results = sorted(results, key=lambda x: x['dist'])\n", "\n", " #return \". \".join(formatted_results)\n", "\n", "#current weather API\n", "def get_weather(city_name:str= \"\", **kwargs):\n", " \"\"\"\n", " Returns the CURRENT weather in a specified city.\n", " Args:\n", " city_name (string) : Required. The name of the city.\n", " \"\"\"\n", " # The endpoint URL provided by WeatherAPI\n", " url = f\"http://api.weatherapi.com/v1/current.json?key={WEATHER_API_KEY}&q={city_name}&aqi=no\"\n", "\n", " # Make the API request\n", " response = requests.get(url)\n", "\n", " if response.status_code == 200:\n", " # Parse the JSON response\n", " weather_data = response.json()\n", "\n", " # Extracting the necessary pieces of data\n", " location = weather_data['location']['name']\n", " region = weather_data['location']['region']\n", " country = weather_data['location']['country']\n", " time = weather_data['location']['localtime']\n", " temperature_c = weather_data['current']['temp_c']\n", " condition_text = weather_data['current']['condition']['text']\n", " wind_mph = weather_data['current']['wind_mph']\n", " humidity = weather_data['current']['humidity']\n", " feelslike_c = weather_data['current']['feelslike_c']\n", "\n", " # Formulate the sentences\n", " weather_sentences = (\n", " f\"The current weather in {location}, {region}, {country} is {condition_text} \"\n", " f\"with a temperature of {temperature_c}°C that feels like {feelslike_c}°C. \"\n", " f\"Humidity is at {humidity}%. \"\n", " f\"Wind speed is {wind_mph} mph.\"\n", " )\n", " return weather_sentences\n", " else:\n", " # Handle errors\n", " return f\"Failed to get weather data: {response.status_code}, {response.text}\"\n", " \n", "#forecast API\n", "def get_forecast(city_name:str= \"\", when = 0, **kwargs):\n", " \"\"\"\n", " Returns the weather forecast in a specified number of days for a specified city .\n", " Args:\n", " city_name (string) : Required. The name of the city.\n", " when (int) : Required. in number of days (until the day for which we want to know the forecast) (example: tomorrow is 1, in two days is 2, etc.)\n", " \"\"\"\n", " #print(when)\n", " when +=1\n", " # The endpoint URL provided by WeatherAPI\n", " url = f\"http://api.weatherapi.com/v1/forecast.json?key={WEATHER_API_KEY}&q={city_name}&days={str(when)}&aqi=no\"\n", "\n", "\n", " # Make the API request\n", " response = requests.get(url)\n", "\n", " if response.status_code == 200:\n", " # Parse the JSON response\n", " data = response.json()\n", " \n", " # Initialize an empty string to hold our result\n", " forecast_sentences = \"\"\n", "\n", " # Extract city information\n", " location = data.get('location', {})\n", " city_name = location.get('name', 'the specified location')\n", " \n", " #print(data)\n", " \n", "\n", " # Extract the forecast days\n", " forecast_days = data.get('forecast', {}).get('forecastday', [])[when-1:]\n", " #number = 0\n", " \n", " #print (forecast_days)\n", "\n", " for day in forecast_days:\n", " date = day.get('date', 'a specific day')\n", " conditions = day.get('day', {}).get('condition', {}).get('text', 'weather conditions')\n", " max_temp_c = day.get('day', {}).get('maxtemp_c', 'N/A')\n", " min_temp_c = day.get('day', {}).get('mintemp_c', 'N/A')\n", " chance_of_rain = day.get('day', {}).get('daily_chance_of_rain', 'N/A')\n", " \n", " if when == 1:\n", " number_str = 'today'\n", " elif when == 2:\n", " number_str = 'tomorrow'\n", " else:\n", " number_str = f'in {when-1} days'\n", "\n", " # Generate a sentence for the day's forecast\n", " forecast_sentence = f\"On {date} ({number_str}) in {city_name}, the weather will be {conditions} with a high of {max_temp_c}°C and a low of {min_temp_c}°C. There's a {chance_of_rain}% chance of rain. \"\n", " \n", " #number = number + 1\n", " # Add the sentence to the result\n", " forecast_sentences += forecast_sentence\n", " return forecast_sentences\n", " else:\n", " # Handle errors\n", " print( f\"Failed to get weather data: {response.status_code}, {response.text}\")\n", " return f'error {response.status_code}'\n", "\n", "\n", "#############################################################\n", "# Step 2: Let's define some utils for building the prompt ###\n", "#############################################################\n", "\n", "\n", "def format_functions_for_prompt(*functions):\n", " formatted_functions = []\n", " for func in functions:\n", " source_code = inspect.getsource(func)\n", " docstring = inspect.getdoc(func)\n", " formatted_functions.append(\n", " f\"OPTION:\\n{source_code}\\n\\n{docstring}\\n\"\n", " )\n", " return \"\\n\".join(formatted_functions)\n", "\n", "\n", "##############################\n", "# Step 3: Construct Prompt ###\n", "##############################\n", "\n", "\n", "def construct_prompt(user_query: str, context):\n", " formatted_prompt = format_functions_for_prompt(get_weather, find_points_of_interest, find_route, get_forecast, search_along_route)\n", " formatted_prompt += f'\\n\\nContext : {context}'\n", " formatted_prompt += f\"\\n\\nUser Query: Question: {user_query}\\n\"\n", "\n", " prompt = (\n", " \":\\n\"\n", " + formatted_prompt\n", " + \"Please pick a function from the above options that best answers the user query and fill in the appropriate arguments.\"\n", " )\n", " return prompt\n", "\n", "#######################################\n", "# Step 4: Execute the function call ###\n", "#######################################\n", "\n", "\n", "def execute_function_call(model_output):\n", " # Ignore everything after \"Reflection\" since that is not essential.\n", " function_call = (\n", " model_output[0][\"generated_text\"]\n", " .strip()\n", " .split(\"\\n\")[1]\n", " .replace(\"Initial Answer:\", \"\")\n", " .strip()\n", " )\n", "\n", " try:\n", " return eval(function_call)\n", " except Exception as e:\n", " return str(e)\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true }, "outputs": [], "source": [ "# might be deleted\n", "# Compute a Simple equation\n", "print(\"before everything: \")\n", "!nvidia-smi\n", "prompt = construct_prompt(\"What restaurants are there on the road from Luxembourg Gare, which coordinates are lat 49.5999681, lon 6.1342493, to Thionville?\", \"\")\n", "print(\"after creating prompt: \")\n", "!nvidia-smi\n", "model_output = pipe(\n", " prompt, do_sample=False, max_new_tokens=300, return_full_text=False\n", " )\n", "print(model_output[0][\"generated_text\"])\n", "\n", "print(\"creating the pipe of model output: \")\n", "!nvidia-smi\n", "result = execute_function_call(model_output)\n", "print(\"execute function call: \")\n", "!nvidia-smi\n", "del model_output\n", "import gc # garbage collect library\n", "gc.collect()\n", "torch.cuda.empty_cache() \n", "\n", "#print(\"Model Output:\", model_output)\n", "print(\"Execution Result:\", result)\n", "\n", "\n", "#execute_function_call(pipe(construct_prompt(\"Is it raining in Belval, ?\"), do_sample=False, max_new_tokens=300, return_full_text=False))" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## functions to process the anwser and the question" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "#generation of text with Stable beluga \n", "def gen(p, maxlen=15, sample=True):\n", " toks = tokr(p, return_tensors=\"pt\")\n", " res = model.generate(**toks.to(\"cuda\"), max_new_tokens=maxlen, do_sample=sample).to('cpu')\n", " return tokr.batch_decode(res)\n", "\n", "#to have a prompt corresponding to the specific format required by the fine-tuned model Stable Beluga\n", "def mk_prompt(user, syst=\"### System:\\nYou are a useful AI assistant in a car, that follows instructions extremely well. Help as much as you can. Answer questions concisely and do not mention what you base your reply on.\\n\\n\"): return f\"{syst}### User: {user}\\n\\n### Assistant:\\n\"" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "yAJI0WyOLE8G" }, "outputs": [], "source": [ "def car_answer_only(complete_answer, general_context):\n", " \"\"\"returns only the AI assistant answer, without all context, to reply to the user\"\"\"\n", " pattern = r\"Assistant:\\\\n(.*)(|[.!?](\\s|$))\" #pattern = r\"Assistant:\\\\n(.*?)\"\n", "\n", " match = re.search(pattern, complete_answer, re.DOTALL)\n", "\n", " if match:\n", " # Extracting the text\n", " model_answer = match.group(1)\n", " #print(complete_answer)\n", " else:\n", " #print(complete_answer)\n", " model_answer = \"There has been an error with the generated response.\" \n", "\n", " general_context += model_answer\n", " return (model_answer, general_context)\n", "#print(model_answer)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "ViCEgogaENNV" }, "outputs": [], "source": [ "def FnAnswer(general_context, ques, place, time, delete_history, state):\n", " \"\"\"function to manage the two different llms (function calling and basic answer) and call them one after the other\"\"\"\n", " # Initialize state if it is None\n", " if delete_history == \"Yes\":\n", " state = None\n", " if state is None:\n", " conv_context = []\n", " conv_context.append(general_context)\n", " state = {}\n", " state['context'] = conv_context\n", " state['number'] = 0\n", " state['last_question'] = \"\"\n", " \n", " if type(ques) != str: \n", " ques = ques[0]\n", " \n", " place = definePlace(place) #which on the predefined places it is\n", " \n", " formatted_context = '\\n'.join(state['context'])\n", " \n", " #updated at every question\n", " general_context = f\"\"\"\n", " Recent conversation history: '{formatted_context}' (If empty, this indicates the beginning of the conversation).\n", "\n", " Previous question from the user: '{state['last_question']}' (This may or may not be related to the current question).\n", "\n", " User information: The user is inside a car in {place[0]}, with latitude {place[1]} and longitude {place[2]}. The user is mobile and can drive to different destinations. It is currently {time}\n", "\n", " \"\"\"\n", " #first llm call (function calling model, NexusRaven)\n", " model_output= pipe(construct_prompt(ques, general_context), do_sample=False, max_new_tokens=300, return_full_text=False)\n", " call = execute_function_call(model_output) #call variable is formatted to as a call to a specific function with the required parameters\n", " print(call)\n", " #this is what will erase the model_output from the GPU memory to free up space\n", " del model_output\n", " import gc # garbage collect library\n", " gc.collect()\n", " torch.cuda.empty_cache() \n", " \n", " #updated at every question\n", " general_context += f'This information might be of help, use if it seems relevant, and ignore if not relevant to reply to the user: \"{call}\". '\n", " \n", " #question formatted for the StableBeluga llm (second llm), using the output of the first llm as context in general_context\n", " question=f\"\"\"Reply to the user and answer any question with the help of the provided context.\n", "\n", " ## Context\n", "\n", " {general_context} .\n", "\n", " ## Question\n", "\n", " {ques}\"\"\"\n", "\n", " complete_answer = str(gen(mk_prompt(question), 100)) #answer generation with StableBeluga (2nd llm)\n", "\n", " model_answer, general_context= car_answer_only(complete_answer, general_context) #to retrieve only the car answer \n", " \n", " language = pipe_language(model_answer, top_k=1, truncation=True)[0]['label'] #detect the language of the answer, to modify the text-to-speech consequently\n", " \n", " state['last_question'] = ques #add the current question as 'last question' for the next question's context\n", " \n", " state['number']= state['number'] + 1 #adds 1 to the number of interactions with the car\n", "\n", " state['context'].append(str(state['number']) + '. User question: '+ ques + ', Model answer: ' + model_answer) #modifies the context\n", " \n", " #print(\"contexte : \" + '\\n'.join(state['context']))\n", " \n", " if len(state['context'])>5: #6 questions maximum in the context to avoid having too many information\n", " state['context'] = state['context'][1:]\n", "\n", " return model_answer, state['context'], state, language" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "9WQlYePVLrTN" }, "outputs": [], "source": [ "def transcript(general_context, link_to_audio, voice, place, time, delete_history, state):\n", " \"\"\"this function manages speech-to-text to input Fnanswer function and text-to-speech with the Fnanswer output\"\"\"\n", " # load audio from a specific path\n", " audio_path = link_to_audio\n", " audio_array, sampling_rate = librosa.load(link_to_audio, sr=16000) # \"sr=16000\" ensures that the sampling rate is as required\n", "\n", "\n", " # process the audio array\n", " input_features = processor(audio_array, sampling_rate, return_tensors=\"pt\").input_features\n", "\n", "\n", " predicted_ids = modelw.generate(input_features)\n", "\n", " transcription = processor.batch_decode(predicted_ids, skip_special_tokens=True)\n", "\n", " quest_processing = FnAnswer(general_context, transcription, place, time, delete_history, state)\n", " \n", " state=quest_processing[2]\n", " \n", " print(\"langue \" + quest_processing[3])\n", "\n", " tts.tts_to_file(text= str(quest_processing[0]),\n", " file_path=\"output.wav\",\n", " speaker_wav=f'Audio_Files/{voice}.wav',\n", " language=quest_processing[3],\n", " emotion = \"angry\")\n", "\n", " audio_path = \"output.wav\"\n", " return audio_path, state['context'], state" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "def definePlace(place):\n", " if(place == 'Luxembourg Gare, Luxembourg'):\n", " return('Luxembourg Gare', '49.5999681', '6.1342493' )\n", " elif (place =='Kirchberg Campus, Kirchberg'):\n", " return('Kirchberg Campus, Luxembourg', '49.62571206478235', '6.160082636815114')\n", " elif (place =='Belval Campus, Belval'):\n", " return('Belval-Université, Esch-sur-Alzette', '49.499531', '5.9462903')\n", " elif (place =='Eiffel Tower, Paris'):\n", " return('Eiffel Tower, Paris', '48.8582599', '2.2945006')\n", " elif (place=='Thionville, France'):\n", " return('Thionville, France', '49.357927', '6.167587')" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Interfaces (text and audio)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true }, "outputs": [], "source": [ "#INTERFACE WITH ONLY TEXT\n", "\n", "# Generate options for hours (00-23) \n", "hour_options = [f\"{i:02d}:00:00\" for i in range(24)]\n", "\n", "model_answer= ''\n", "general_context= ''\n", "# Define the initial state with some initial context.\n", "print(general_context)\n", "initial_state = {'context': general_context}\n", "initial_context= initial_state['context']\n", "# Create the Gradio interface.\n", "iface = gr.Interface(\n", " fn=FnAnswer,\n", " inputs=[\n", " gr.Textbox(value=initial_context, visible=False),\n", " gr.Textbox(lines=2, placeholder=\"Type your message here...\"),\n", " gr.Radio(choices=['Luxembourg Gare, Luxembourg', 'Kirchberg Campus, Kirchberg', 'Belval Campus, Belval', 'Eiffel Tower, Paris', 'Thionville, France'], label='Choose a location for your car', value= 'Kirchberg Campus, Kirchberg', show_label=True),\n", " gr.Dropdown(choices=hour_options, label=\"What time is it?\", value = \"08:00:00\"),\n", " gr.Radio([\"Yes\", \"No\"], label=\"Delete the conversation history?\", value = 'No'),\n", " gr.State() # This will keep track of the context state across interactions.\n", " ],\n", " outputs=[\n", " gr.Textbox(),\n", " gr.Textbox(visible=False),\n", " gr.State()\n", " ]\n", ")\n", "gr.close_all()\n", "# Launch the interface.\n", "iface.launch(debug=True, share=True, server_name=\"0.0.0.0\", server_port=7860)\n", "#contextual=gr.Textbox(value=general_context, visible=False)\n", "#demo = gr.Interface(fn=FnAnswer, inputs=[contextual,\"text\"], outputs=[\"text\", contextual])\n", "\n", "#demo.launch()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true, "id": "mZTt3y3_KOOF" }, "outputs": [], "source": [ "#INTERFACE WITH AUDIO TO AUDIO\n", "\n", "#to be able to use the microphone on chrome, you will have to go to chrome://flags/#unsafely-treat-insecure-origin-as-secure and enter http://10.186.115.21:7860/ \n", "#in \"Insecure origins treated as secure\", enable it and relaunch chrome\n", "\n", "#example question: \n", "# what's the weather like outside?\n", "# What's the closest restaurant from here?\n", "\n", "\n", "\n", "# Generate options for hours (00-23) \n", "hour_options = [f\"{i:02d}:00:00\" for i in range(24)]\n", "\n", "model_answer= ''\n", "general_context= ''\n", "# Define the initial state with some initial context.\n", "print(general_context)\n", "initial_state = {'context': general_context}\n", "initial_context= initial_state['context']\n", "# Create the Gradio interface.\n", "iface = gr.Interface(\n", " fn=transcript,\n", " inputs=[\n", " gr.Textbox(value=initial_context, visible=False),\n", " gr.Audio( type='filepath', label = 'input audio'),\n", " gr.Radio(choices=['Donald Trump', 'Eddie Murphy'], label='Choose a voice', value= 'Donald Trump', show_label=True), # Radio button for voice selection\n", " gr.Radio(choices=['Luxembourg Gare, Luxembourg', 'Kirchberg Campus, Kirchberg', 'Belval Campus, Belval', 'Eiffel Tower, Paris', 'Thionville, France'], label='Choose a location for your car', value= 'Kirchberg Campus, Kirchberg', show_label=True),\n", " gr.Dropdown(choices=hour_options, label=\"What time is it?\", value = \"08:00:00\"),\n", " gr.Radio([\"Yes\", \"No\"], label=\"Delete the conversation history?\", value = 'No'),\n", " gr.State() # This will keep track of the context state across interactions.\n", " ],\n", " outputs=[\n", " gr.Audio(label = 'output audio'),\n", " gr.Textbox(visible=False),\n", " gr.State()\n", " ]\n", ")\n", "#close all interfaces open to make the port available\n", "gr.close_all()\n", "# Launch the interface.\n", "iface.launch(debug=True, share=True, server_name=\"0.0.0.0\", server_port=7860, ssl_verify=False)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Other possible APIs to use" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "\n", "def search_nearby(lat, lon, city, key):\n", " \"\"\"\n", " :param lat: latitude\n", " :param lon: longitude\n", " :param key: api key\n", " :param type: type of poi\n", " :return: [5] results ['poi']['name']/['freeformAddress'] || ['position']['lat']/['lon']\n", " \"\"\"\n", " results = []\n", "\n", " r = requests.get('https://api.tomtom.com/search/2/nearbySearch/.json?key={0}&lat={1}&lon={2}&radius=10000&limit=50'.format(\n", " key,\n", " lat,\n", " lon\n", " ))\n", "\n", " for result in r.json()['results']:\n", " results.append(f\"The {' '.join(result['poi']['categories'])} {result['poi']['name']} is {int(result['dist'])} meters far from {city}\")\n", " if len(results) == 7:\n", " break\n", "\n", " return \". \".join(results)\n", "\n", "\n", "print(search_nearby('49.625892805337514', '6.160417066963513', 'your location', TOMTOM_KEY))" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "accelerator": "GPU", "colab": { "gpuType": "T4", "provenance": [] }, "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.12" } }, "nbformat": 4, "nbformat_minor": 1 }