import json
from collections import defaultdict
 
def load_dataset(filepath):
    """Load the dataset JSON file."""
    with open(filepath, "r") as file:
        return json.load(file)
 
def normalize_output(output):
    """Normalize the Output dictionary by converting lists and dicts inside it to hashable types."""
    normalized_output = {}
    for key, value in output.items():
        if isinstance(value, list):  # Convert list to tuple
            normalized_output[key] = tuple(normalize_list(value))  
        elif isinstance(value, dict):  # Convert dict to tuple of key-value pairs
            normalized_output[key] = tuple(normalize_dict(value))
        else:
            normalized_output[key] = value  # Keep non-dict/non-list values as is
    return normalized_output
 
def normalize_list(value):
    """Recursively convert lists and their inner elements to hashable structures."""
    return [tuple(normalize_list(item)) if isinstance(item, list) else
            tuple(normalize_dict(item)) if isinstance(item, dict) else
            item for item in value]
 
def normalize_dict(value):
    """Recursively convert a dictionary to a hashable tuple of key-value pairs."""
    return ((k, tuple(normalize_list(v)) if isinstance(v, list) else 
                 tuple(normalize_dict(v)) if isinstance(v, dict) else v)
            for k, v in value.items())
 
def check_dataset_health(dataset):
    """Check for incomplete data, count entries by sector, and flag structural issues."""
    incomplete_entries = []       # Track incomplete entries
    sector_counts = defaultdict(int)  # Count entries by sector
 
    for i, entry in enumerate(dataset):
        # Increment count for the sector
        sector = entry.get("Output", {}).get("SectorClassification", "Unknown")
        sector_counts[sector] += 1
 
        # Check for missing Input or Output
        if not entry.get("Input") or not entry.get("Output"):
            incomplete_entries.append((i, "Missing Input or Output entirely"))
            continue
 
        # Check if Input tables have at least 4 columns
        if not all(isinstance(table, list) and len(table) >= 4 for table in entry["Input"]):
            incomplete_entries.append((i, "Some Input tables are incomplete or less than 4 columns"))
 
        # Check Output fields are missing
        output = entry["Output"]
        required_fields = ["SectorClassification", "Overview", "ImportantColumns", "SuggestedPlots", "PlotExplanation", "Insights"]
        missing_fields = [field for field in required_fields if field not in output]
        if missing_fields:
            incomplete_entries.append((i, f"Missing Output fields: {missing_fields}"))
 
    return sector_counts, incomplete_entries
 
def remove_duplicates(dataset):
    """Remove duplicate entries from the dataset."""
    seen_entries = set()      # To track unique entries
    cleaned_dataset = []      # To store unique entries
 
    for entry in dataset:
        # Normalize Input and Output sections for comparison
        input_entry = tuple([tuple(table) for table in entry["Input"]])  # Normalize Input
        output_entry = tuple(normalize_output(entry["Output"]).items())  # Normalize Output
        
        # Combine Input and Output for full entry uniqueness
        full_entry = (input_entry, output_entry)
        
        if full_entry not in seen_entries:  # If entry is not a duplicate
            seen_entries.add(full_entry)   # Mark entry as seen
            cleaned_dataset.append(entry)  # Add unique entry to the cleaned dataset
 
    return cleaned_dataset
 
def print_sector_counts(sector_counts, stage):
    """Print sector counts in a readable format."""
    print(f"\n{stage} Sector Counts:")
    for sector, count in sector_counts.items():
        print(f"    {sector}: {count}")
 
def save_to_json(dataset, filepath):
    """Save the cleaned dataset back to the original file."""
    with open(filepath, "w") as file:
        json.dump(dataset, file, indent=4)
    print(f"Cleaned dataset saved back to {filepath}")
 
def main(filepath):
    # Load the original dataset
    dataset = load_dataset(filepath)
    print(f"Initial dataset contains {len(dataset)} entries.")
 
    # Check dataset health
    sector_counts, incomplete_entries = check_dataset_health(dataset)
    print_sector_counts(sector_counts, "Initial")  # Print initial sector counts
    if incomplete_entries:
        print(f"\nIncomplete Entries Found: {len(incomplete_entries)}")
        for entry in incomplete_entries:
            print(f" - Entry {entry[0]}: {entry[1]}")
 
    # Remove duplicates
    cleaned_dataset = remove_duplicates(dataset)
    print(f"\nCleaned dataset contains {len(cleaned_dataset)} unique entries (duplicates removed).")
 
    # Recalculate sector counts for cleaned dataset
    cleaned_sector_counts, _ = check_dataset_health(cleaned_dataset)
    print_sector_counts(cleaned_sector_counts, "Cleaned")  # Print cleaned sector counts
 
    # Save cleaned dataset back to the original file
    save_to_json(cleaned_dataset, filepath)
 
    print("\nAnalysis Complete. Dataset cleaned and saved back to the original file.")
 
if __name__ == "__main__":
    # Provide the path to your dataset JSON file
    filepath = "/Users/magi/Documents/projects/AI-CRM/LLM-dataset.json"  # Input dataset path
    main(filepath)


this code you gave me, gave this error:

{
	"name": "AttributeError",
	"message": "'list' object has no attribute 'get'",
	"stack": "---------------------------------------------------------------------------
AttributeError                            Traceback (most recent call last)
Cell In[67], line 121
    118 if __name__ == \"__main__\":
    119     # Provide the path to your dataset JSON file
    120     filepath = \"/Users/magi/Documents/projects/AI-CRM/LLM-dataset.json\"  # Input dataset path
--> 121     main(filepath)

Cell In[67], line 98, in main(filepath)
     95 print(f\"Initial dataset contains {len(dataset)} entries.\")
     97 # Check dataset health
---> 98 sector_counts, incomplete_entries = check_dataset_health(dataset)
     99 print_sector_counts(sector_counts, \"Initial\")  # Print initial sector counts
    100 if incomplete_entries:

Cell In[67], line 40, in check_dataset_health(dataset)
     36 sector_counts = defaultdict(int)  # Count entries by sector
     38 for i, entry in enumerate(dataset):
     39     # Increment count for the sector
---> 40     sector = entry.get(\"Output\", {}).get(\"SectorClassification\", \"Unknown\")
     41     sector_counts[sector] += 1
     43     # Check for missing Input or Output

AttributeError: 'list' object has no attribute 'get'"
}