File size: 1,980 Bytes
f9396fc |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 |
import json
import pandas as pd
# Read dataset.txt
input_file = "dataset.txt"
output_file = "dataset.json"
# Initialize a list to store dataset entries
data = []
# Read the text file line by line
with open(input_file, 'r', encoding='utf-8') as f:
for line in f:
try:
# Parse each line as a JSON object
entry = json.loads(line.strip())
# Ensure the entry has 'input' and 'response' keys
if "input" in entry and "response" in entry:
data.append({
"input": entry["input"],
"response": entry["response"]
})
else:
print(f"Skipping invalid entry: {line.strip()}")
except json.JSONDecodeError:
print(f"Error parsing line: {line.strip()}")
# Save to JSON file
with open(output_file, 'w', encoding='utf-8') as f:
json.dump(data, f, indent=4)
print(f"Converted {len(data)} entries to {output_file}")
import json
# Read dataset.txt
input_file = "dataset.txt"
output_file = "dataset.json"
# Read the entire file content
try:
with open(input_file, 'r', encoding='utf-8') as f:
content = f.read()
# Parse the entire content as a JSON array
data = json.loads(content)
except json.JSONDecodeError as e:
print(f"Error parsing dataset.txt: {e}")
print("Please check the JSON format in dataset.txt")
exit(1)
# Validate and filter entries
valid_data = []
for entry in data:
if isinstance(entry, dict) and "input" in entry and "response" in entry:
valid_data.append({
"input": entry["input"],
"response": entry["response"]
})
else:
print(f"Skipping invalid entry: {entry}")
# Save to JSON file
with open(output_file, 'w', encoding='utf-8') as f:
json.dump(valid_data, f, indent=4)
print(f"Converted {len(valid_data)} entries to {output_file}") |