File size: 1,808 Bytes
01377f7 9a9a2f5 01377f7 9a9a2f5 01377f7 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 |
import re
import json
with open("./datasets/glaive/glaive-function-calling-v2.json", "r") as f:
data = json.load(f)
def process_sample(sample):
system_text = sample.get("system", "")
chat_text = sample.get("chat", "")
# Extract tools
tools_match = re.findall(
r'"name": "(.*?)",.*?"description": "(.*?)"', system_text, re.S
)
if not tools_match:
return None # Skip samples without tools
tools = [{"name": name, "description": desc} for name, desc in tools_match]
# Extract function calls and corresponding instructions
function_calls = re.findall(r'<functioncall> {"name": "(.*?)"', chat_text)
if not function_calls:
return None # Skip samples without function calls
# Extract ASSISTANT responses and filter for valid instructions
assistant_responses = re.findall(
r"ASSISTANT: (.*?)<\|endoftext\|>", chat_text, re.S
)
instructions = []
for call, response in zip(function_calls, assistant_responses):
# Find the user prompt just before this ASSISTANT response
user_prompt_match = re.search(
rf'USER: (.*?)\n.*?<functioncall> {{.*?"{call}".*?}}', chat_text, re.S
)
if user_prompt_match:
instructions.append(
{
"instruction": user_prompt_match.group(1).strip(),
"tools": tools,
"used_tools": [call], # Single tool for this instruction
}
)
return instructions
from tqdm import tqdm
results = []
for sample in tqdm(data, desc="Processing GLAIVE samples"):
processed = process_sample(sample)
if processed:
results.extend(processed)
with open("./datasets/glaive/output.json", "w") as f:
json.dump(results, f, indent=2)
|