import json import os from together import Together # def rerank_best_answer(json_files, config_file='config.json', model="meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo"): # # Load API key from configuration file # together_ai_key = os.getenv("TOGETHER_AI") # if not together_ai_key: # raise ValueError("TOGETHER_AI environment variable not found. Please set it before running the script.") # # Initialize Together client # client = Together(api_key=together_ai_key) # # Combine all JSON files into a single structure # combined_prompts = {} # for json_file in json_files: # with open(json_file, 'r') as file: # data = json.load(file) # # Format the input for the prompt # for item in data: # query_id = item['query_id'] # if query_id not in combined_prompts: # combined_prompts[query_id] = { # "question": item['input'], # "answers": {} # } # combined_prompts[query_id]["answers"][json_file] = item['response'] # responses = [] # for query_id, prompt in combined_prompts.items(): # # Generate the prompt text # prompt_text = f"""Input JSON: # {json.dumps(prompt, indent=4)} # For the above question, identify which model gave the best response based on accuracy. Ensure the chosen response is an answer and not a follow-up question. Provide the output in the format: # {{ # "best_model": "", # "best_answer": "" # }} # Just output this JSON and nothing else. # """ # # Generate response from Together API # response = client.chat.completions.create( # model=model, # messages=[{"role": "user", "content": prompt_text}], # ) # response_content = response.choices[0].message.content # # print(response_content) # prompt_text_extract_bestModel = f"""Content: # {response_content} # Whats the best_model from above? # """ # prompt_text_extract_bestAnswer = f"""Content: # {response_content} # Whats the best_answer from above? # """ # print(prompt_text_extract_bestModel) # print(prompt_text_extract_bestAnswer) # response_bestModel = client.chat.completions.create( # model=model, # messages=[{"role": "user", "content": prompt_text_extract_bestModel}], # ) # response_bestAnswer = client.chat.completions.create( # model=model, # messages=[{"role": "user", "content": prompt_text_extract_bestAnswer}], # ) # # print({"query_id": query_id, "question": prompt["question"], "Ranker_Output": response.choices[0].message.content}) # responses.append({"query_id": query_id, "question": prompt["question"], "best_model": response_bestModel.choices[0].message.content, "best_answer": response_bestAnswer.choices[0].message.content}) # print(response_bestModel.choices[0].message.content) # return responses def rankerAgent(prompt, model="meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo"): # Load API key from configuration file together_ai_key = os.getenv("TOGETHER_AI") if not together_ai_key: raise ValueError("TOGETHER_AI environment variable not found. Please set it before running the script.") # Initialize Together client client = Together(api_key=together_ai_key) prompt_text = f"""Input JSON: {json.dumps(prompt, indent=4)} For the above question, identify which model gave the best response based on accuracy. Ensure the chosen response is an answer and not a follow-up question. The best_answer should be from the best_model only, as given in the above content. Provide the output in the format: {{ "best_model": "", "best_answer": "" }} Just output this JSON and nothing else. """ # Generate response from Together API response = client.chat.completions.create( model=model, messages=[{"role": "user", "content": prompt_text}], ) response_content = response.choices[0].message.content # print(response_content) prompt_text_extract_bestModel = f"""Content: {response_content} Whats the best_model from above? """ prompt_text_extract_bestAnswer = f"""Content: {response_content} Whats the best_answer from above? """ response_bestModel = client.chat.completions.create( model=model, messages=[{"role": "user", "content": prompt_text_extract_bestModel}], ) response_bestAnswer = client.chat.completions.create( model=model, messages=[{"role": "user", "content": prompt_text_extract_bestAnswer}], ) return response_bestModel.choices[0].message.content, response_bestAnswer.choices[0].message.content