|
from modules.lightning import sdxl_lightning |
|
from modules.audio_ldm import audio_generation |
|
from modules.graph_plotter import plot_equation |
|
from modules.search_web import search |
|
from modules.vqa import moondream |
|
from modules.extra_functions import calculator, execute_code, none |
|
from llm_modules.llm import llm_usage |
|
from rw_modules.xtts_voice import xtts_model |
|
|
|
xtts = xtts_model() |
|
llm_model = llm_usage() |
|
sdxl_model = sdxl_lightning() |
|
moondream_model = moondream() |
|
search_engine = search() |
|
audio_gen = audio_generation() |
|
|
|
function_dict = { |
|
"robot": none, |
|
"image_gen": sdxl_model.infer, |
|
"code": execute_code, |
|
"music_gen": audio_gen.infer, |
|
"search": search_engine.search_text, |
|
"search_img": search_engine.search_image, |
|
"save": none, |
|
"plot_graph": plot_equation, |
|
"math": calculator, |
|
"change_volume": none, |
|
"current_news": search_engine.search_news, |
|
"weather": search_engine.search_text, |
|
"none": none, |
|
"listen": none, |
|
"screen": none, |
|
"VisualQ": moondream_model.infer |
|
} |
|
|
|
class FunctionExecutor: |
|
def __init__(self): |
|
pass |
|
|
|
def split_variables(self, h, prefixes_to_remove): |
|
for prefix in prefixes_to_remove: |
|
h = h.replace(prefix, '') |
|
if "'" in h: |
|
variables = [item.strip() for item in h.split("',")] |
|
elif '"' in h: |
|
variables = [item.strip() for item in h.split('",')] |
|
else: |
|
variables = [item.strip() for item in h.split(',')] |
|
for vars in variables: |
|
result = vars[1:-1] if vars.startswith('"') and vars.endswith('"') else vars |
|
result = vars[1:-1] if vars.startswith("'") and vars.endswith("'") else vars |
|
full_results.append(result) |
|
return full_results |
|
|
|
def execute(self, function, args): |
|
try: |
|
return function(*args) |
|
except (ImportError, AttributeError): |
|
print("Error executing the function.") |
|
|
|
def get_function(self, action, param): |
|
chosen_function = function_dict.get(action) |
|
if chosen_function: |
|
function_args = ["num_images=", "description=", "prompt=", "code=", "length=", "query=", "input=", "graph=", "equation=", "volume=", "place=", "percentage="] |
|
params_list = self.split_variables(str(param), function_args) |
|
result = self.execute(chosen_function, params_list) |
|
return result |
|
else: |
|
print("Function not found.") |
|
|
|
function_executor = FunctionExecutor() |
|
|
|
prompt = "Hello, how are you today? Please generate a nice image of a man." |
|
personality = "a caring, nice assistant named hermes" |
|
name = "Hermes" |
|
previous_prompts = [] |
|
previous_answers = [] |
|
final_answer = False |
|
|
|
def llm_speech(prompt, functions, function_executor, personality, name, previous_prompts, previous_answers): |
|
speech_output = "" |
|
for out in llm_model.process_llm(prompt, functions, function_executor=function_executor, personality=personality, name=name, previous_prompts=previous_prompts, previous_answers=previous_answers): |
|
token = out['token'] |
|
if token: |
|
|
|
yield {'token': token, 'speech': None} |
|
if "answer:" in full_output: |
|
speech_output += token |
|
symbols = ['!', '.', '?'] |
|
if any(speech_output.endswith(symbol) for symbol in symbols): |
|
for speech in xtts.infer(speech_output): |
|
yield {'speech': speech, 'token': None} |
|
speech_output = "" |
|
|
|
for output in llm_speech(prompt, functions, function_executor=function_executor, personality=personality, name=name, previous_prompts=previous_prompts, previous_answers=previous_answers) |
|
if output['token']: |
|
print(token, end="") |
|
else: |
|
print("only in ohio") |
|
|