I am trying to make this function work but my tests are failing:

def filter_categories(categories: Dict[str, List[str]]) -> Dict[str, List[str]]:
    """
    Filter the provided categories to only include words or phrases that are in the predefined categories.
    Handles case-insensitive matching, removes unknown categories, and properly handles whitespace and special characters.
    Removes leading and trailing whitespace and punctuation.

    Args:
        categories: A dictionary of categories with lists of words or phrases.

    Returns:
        A dictionary of categories with filtered entries for recognized categories.
    """
    filtered_categories = {}
    for category, entries in categories.items():
        if category.lower() in {cat.lower() for cat in PREDEFINED_CATEGORIES}:
            filtered_entries = []
            predefined_category = next(cat for cat in PREDEFINED_CATEGORIES if cat.lower() == category.lower())

            # Handle both list and string inputs
            if isinstance(entries, str):
                entries = [entry.strip() for entry in entries.split(',')]

            for entry in entries:
                # Remove trailing whitespace and punctuation, but keep leading whitespace
                cleaned_entry = entry.strip().rstrip('.,!?')
                # Find a case-insensitive match in the predefined categories
                match = next((pred for pred in PREDEFINED_CATEGORIES[predefined_category]
                              if pred.lower() == cleaned_entry.lower()), None)
                if match:
                    filtered_entries.append(match)
                else:
                    # Try matching without special characters
                    cleaned_entry_no_special = re.sub(r'[^\w\s]', '', cleaned_entry.lower())
                    match = next((pred for pred in PREDEFINED_CATEGORIES[predefined_category]
                                  if re.sub(r'[^\w\s]', '', pred.lower()) == cleaned_entry_no_special), None)
                    if match:
                        filtered_entries.append(match)
                    else:
                        logger.debug(f"Removed entry '{entry}' from category '{category}' as it doesn't match predefined categories.")

            if filtered_entries:
                filtered_categories[predefined_category] = filtered_entries
    return filtered_categories

tests/lib/categories_test.py::test_filter_categories_exact_match PASSED  [  8%]
tests/lib/categories_test.py::test_filter_categories_case_insensitive PASSED [ 16%]
tests/lib/categories_test.py::test_filter_categories_remove_invalid PASSED [ 25%]
tests/lib/categories_test.py::test_filter_categories_unknown_category PASSED [ 33%]
tests/lib/categories_test.py::test_filter_categories_empty_input PASSED  [ 41%]
tests/lib/categories_test.py::test_filter_categories_empty_lists FAILED  [ 50%]
tests/lib/categories_test.py::test_filter_categories_mixed_valid_and_invalid PASSED [ 58%]
tests/lib/categories_test.py::test_filter_categories_all_invalid FAILED  [ 66%]
tests/lib/categories_test.py::test_filter_categories_mixed_known_and_unknown PASSED [ 75%]
tests/lib/categories_test.py::test_filter_categories_whitespace_and_punctuation PASSED [ 83%]
tests/lib/categories_test.py::test_filter_categories_all_predefined PASSED [ 91%]
tests/lib/categories_test.py::test_filter_categories_comma_separated_string FAILED [100%]

=================================== FAILURES ===================================
______________________ test_filter_categories_empty_lists ______________________

    def test_filter_categories_empty_lists():
        input_categories = {
            "core_technologies": [],
            "threat_vectors": [],
            "impacted_systems": [],
            "cybersecurity_domains": []
        }
        expected_output = input_categories.copy()
    
        result = filter_categories(input_categories)
>       assert result == expected_output
E       AssertionError: assert {} == {'core_technologies': [], 'threat_vectors': [], 'impacted_systems': [], 'cybersecurity_domains': []}
E         
E         Right contains 4 more items:
E         {'core_technologies': [],
E          'cybersecurity_domains': [],
E          'impacted_systems': [],
E          'threat_vectors': []}
E         
E         Full diff:
E         + {}
E         - {
E         -     'core_technologies': [],
E         -     'cybersecurity_domains': [],
E         -     'impacted_systems': [],
E         -     'threat_vectors': [],
E         - }

tests/lib/categories_test.py:79: AssertionError
______________________ test_filter_categories_all_invalid ______________________

    def test_filter_categories_all_invalid():
        input_categories = {
            "core_technologies": ["InvalidTech1", "InvalidTech2"],
            "threat_vectors": ["InvalidThreat1", "InvalidThreat2"]
        }
        expected_output = {
            "core_technologies": [],
            "threat_vectors": []
        }
        result = filter_categories(input_categories)
>       assert result == expected_output
E       AssertionError: assert {} == {'core_technologies': [], 'threat_vectors': []}
E         
E         Right contains 2 more items:
E         {'core_technologies': [], 'threat_vectors': []}
E         
E         Full diff:
E         + {}
E         - {
E         -     'core_technologies': [],
E         -     'threat_vectors': [],
E         - }

tests/lib/categories_test.py:103: AssertionError
----------------------------- Captured stderr call -----------------------------
2024-10-06 13:52:16.003 | DEBUG    | lib.categories:filter_categories:162 - Removed entry 'InvalidTech1' from category 'core_technologies' as it doesn't match predefined categories.
2024-10-06 13:52:16.003 | DEBUG    | lib.categories:filter_categories:162 - Removed entry 'InvalidTech2' from category 'core_technologies' as it doesn't match predefined categories.
2024-10-06 13:52:16.003 | DEBUG    | lib.categories:filter_categories:162 - Removed entry 'InvalidThreat1' from category 'threat_vectors' as it doesn't match predefined categories.
2024-10-06 13:52:16.003 | DEBUG    | lib.categories:filter_categories:162 - Removed entry 'InvalidThreat2' from category 'threat_vectors' as it doesn't match predefined categories.
________________ test_filter_categories_comma_separated_string _________________

    def test_filter_categories_comma_separated_string():
        input_categories = {
            "core_technologies": "Python, Java, InvalidTech, C++",
            "threat_vectors": "Malware, InvalidThreat, Phishing, DDoS",
            "impacted_systems": "Windows, InvalidSystem, Linux, Android",
            "cybersecurity_domains": "Network Security, InvalidDomain, Cloud Security, IoT Security"
        }
        expected_output = {
            "core_technologies": ["Python", "Java", "C++"],
            "threat_vectors": ["Malware", "Phishing", "DDoS"],
            "impacted_systems": ["Windows", "Linux", "Android"],
            "cybersecurity_domains": ["Network Security", "Cloud Security", "IoT Security"]
        }
    
        result = filter_categories(input_categories)
        assert result == expected_output
    
        input_categories = {
            "core_technologies": "Rust",
            "threat_vectors": "Ransomware, Data Breach, Unauthorized Access, Social Engineering",
            "impacted_systems": "Windows, Linux, Medical Devices",
            "cybersecurity_domains": "Incident Response, Data Protection, Threat Intelligence, Malware Analysis"
        }
    
        expected_output = {
            "core_technologies": ["Rust"],
            "threat_vectors": ["Ransomware", "Data Breach", "Unauthorized Access", "Social Engineering"],
            "impacted_systems": ["Windows", "Linux", "Medical Devices"],
            "cybersecurity_domains": ["Incident Response", "Data Protection", "Threat Intelligence", "Malware Analysis"]
        }
    
        result = filter_categories(input_categories)
>       assert result == expected_output
E       AssertionError: assert {'core_technologies': ['Rust'], 'threat_vectors': ['Ransomware', 'Social Engineering'], 'impacted_systems': ['Windows', 'Linux', 'Medical Devices'], 'cybersecurity_domains': ['Incident Response', 'Data Protection', 'Threat Intelligence', 'Malware Analysis']} == {'core_technologies': ['Rust'], 'threat_vectors': ['Ransomware', 'Data Breach', 'Unauthorized Access', 'Social Engineering'], 'impacted_systems': ['Windows', 'Linux', 'Medical Devices'], 'cybersecurity_domains': ['Incident Response', 'Data Protection', 'Threat Intelligence', 'Malware Analysis']}
E         
E         Common items:
E         {'core_technologies': ['Rust'],
E          'cybersecurity_domains': ['Incident Response',
E                                    'Data Protection',
E                                    'Threat Intelligence',
E                                    'Malware Analysis'],
E          'impacted_systems': ['Windows', 'Linux', 'Medical Devices']}
E         Differing items:
E         {'threat_vectors': ['Ransomware', 'Social Engineering']} != {'threat_vectors': ['Ransomware', 'Data Breach', 'Unauthorized Access', 'Social Engineering']}
E         
E         Full diff:
E           {
E               'core_technologies': [
E                   'Rust',
E               ],
E               'cybersecurity_domains': [
E                   'Incident Response',
E                   'Data Protection',
E                   'Threat Intelligence',
E                   'Malware Analysis',
E               ],
E               'impacted_systems': [
E                   'Windows',
E                   'Linux',
E                   'Medical Devices',
E               ],
E               'threat_vectors': [
E                   'Ransomware',
E         -         'Data Breach',
E         -         'Unauthorized Access',
E                   'Social Engineering',
E               ],
E           }

tests/lib/categories_test.py:175: AssertionError
----------------------------- Captured stderr call -----------------------------
2024-10-06 13:52:16.010 | DEBUG    | lib.categories:filter_categories:162 - Removed entry 'InvalidTech' from category 'core_technologies' as it doesn't match predefined categories.
2024-10-06 13:52:16.010 | DEBUG    | lib.categories:filter_categories:162 - Removed entry 'InvalidThreat' from category 'threat_vectors' as it doesn't match predefined categories.
2024-10-06 13:52:16.010 | DEBUG    | lib.categories:filter_categories:162 - Removed entry 'InvalidSystem' from category 'impacted_systems' as it doesn't match predefined categories.
2024-10-06 13:52:16.010 | DEBUG    | lib.categories:filter_categories:162 - Removed entry 'InvalidDomain' from category 'cybersecurity_domains' as it doesn't match predefined categories.
2024-10-06 13:52:16.010 | DEBUG    | lib.categories:filter_categories:162 - Removed entry 'Data Breach' from category 'threat_vectors' as it doesn't match predefined categories.
2024-10-06 13:52:16.010 | DEBUG    | lib.categories:filter_categories:162 - Removed entry 'Unauthorized Access' from category 'threat_vectors' as it doesn't match predefined categories.
=========================== short test summary info ============================
FAILED tests/lib/categories_test.py::test_filter_categories_empty_lists - AssertionError: assert {} == {'core_technologies': [], 'threat_vectors': [], 'impacted_systems': [], 'cybersecurity_domains': []}
  
  Right contains 4 more items:
  {'core_technologies': [],
   'cybersecurity_domains': [],
   'impacted_systems': [],
   'threat_vectors': []}
  
  Full diff:
  + {}
  - {
  -     'core_technologies': [],
  -     'cybersecurity_domains': [],
  -     'impacted_systems': [],
  -     'threat_vectors': [],
  - }
FAILED tests/lib/categories_test.py::test_filter_categories_all_invalid - AssertionError: assert {} == {'core_technologies': [], 'threat_vectors': []}
  
  Right contains 2 more items:
  {'core_technologies': [], 'threat_vectors': []}
  
  Full diff:
  + {}
  - {
  -     'core_technologies': [],
  -     'threat_vectors': [],
  - }
FAILED tests/lib/categories_test.py::test_filter_categories_comma_separated_string - AssertionError: assert {'core_technologies': ['Rust'], 'threat_vectors': ['Ransomware', 'Social Engineering'], 'impacted_systems': ['Windows', 'Linux', 'Medical Devices'], 'cybersecurity_domains': ['Incident Response', 'Data Protection', 'Threat Intelligence', 'Malware Analysis']} == {'core_technologies': ['Rust'], 'threat_vectors': ['Ransomware', 'Data Breach', 'Unauthorized Access', 'Social Engineering'], 'impacted_systems': ['Windows', 'Linux', 'Medical Devices'], 'cybersecurity_domains': ['Incident Response', 'Data Protection', 'Threat Intelligence', 'Malware Analysis']}
  
  Common items:
  {'core_technologies': ['Rust'],
   'cybersecurity_domains': ['Incident Response',
                             'Data Protection',
                             'Threat Intelligence',
                             'Malware Analysis'],
   'impacted_systems': ['Windows', 'Linux', 'Medical Devices']}
  Differing items:
  {'threat_vectors': ['Ransomware', 'Social Engineering']} != {'threat_vectors': ['Ransomware', 'Data Breach', 'Unauthorized Access', 'Social Engineering']}
  
  Full diff:
    {
        'core_technologies': [
            'Rust',
        ],
        'cybersecurity_domains': [
            'Incident Response',
            'Data Protection',
            'Threat Intelligence',
            'Malware Analysis',
        ],
        'impacted_systems': [
            'Windows',
            'Linux',
            'Medical Devices',
        ],
        'threat_vectors': [
            'Ransomware',
  -         'Data Breach',
  -         'Unauthorized Access',
            'Social Engineering',
        ],
    }
========================= 3 failed, 9 passed in 0.06s ==========================

from lib.categories import PREDEFINED_CATEGORIES, filter_categories


def test_filter_categories_exact_match():
    input_categories = {
        "core_technologies": ["Python", "Java", "C++"],
        "threat_vectors": ["Malware", "Phishing", "DDoS"],
        "impacted_systems": ["Windows", "Linux", "Android"],
        "cybersecurity_domains": ["Network Security", "Cloud Security", "IoT Security"]
    }
    expected_output = input_categories.copy()

    result = filter_categories(input_categories)
    assert result == expected_output

def test_filter_categories_case_insensitive():
    input_categories = {
        "core_technologies": ["python", "JAVA", "C++"],
        "threat_vectors": ["malware", "PHISHING", "DdoS"],
        "impacted_systems": ["windows", "LINUX", "AnDrOiD"],
        "cybersecurity_domains": ["network security", "CLOUD SECURITY", "IoT Security"]
    }
    expected_output = {
        "core_technologies": ["Python", "Java", "C++"],
        "threat_vectors": ["Malware", "Phishing", "DDoS"],
        "impacted_systems": ["Windows", "Linux", "Android"],
        "cybersecurity_domains": ["Network Security", "Cloud Security", "IoT Security"]
    }

    result = filter_categories(input_categories)
    assert result == expected_output

def test_filter_categories_remove_invalid():
    input_categories = {
        "core_technologies": ["Python", "InvalidTech", "Java"],
        "threat_vectors": ["Malware", "InvalidThreat", "Phishing"],
        "impacted_systems": ["Windows", "InvalidSystem", "Linux"],
        "cybersecurity_domains": ["Network Security", "InvalidDomain", "Cloud Security"]
    }
    expected_output = {
        "core_technologies": ["Python", "Java"],
        "threat_vectors": ["Malware", "Phishing"],
        "impacted_systems": ["Windows", "Linux"],
        "cybersecurity_domains": ["Network Security", "Cloud Security"]
    }

    result = filter_categories(input_categories)
    assert result == expected_output

def test_filter_categories_unknown_category():
    input_categories = {
        "core_technologies": ["Python", "Java"],
        "unknown_category": ["Item1", "Item2"]
    }
    expected_output = {
        "core_technologies": ["Python", "Java"],
    }

    result = filter_categories(input_categories)
    assert result == expected_output

def test_filter_categories_empty_input():
    input_categories = {}
    expected_output = {}

    result = filter_categories(input_categories)
    assert result == expected_output

def test_filter_categories_empty_lists():
    input_categories = {
        "core_technologies": [],
        "threat_vectors": [],
        "impacted_systems": [],
        "cybersecurity_domains": []
    }
    expected_output = input_categories.copy()

    result = filter_categories(input_categories)
    assert result == expected_output

def test_filter_categories_mixed_valid_and_invalid():
    input_categories = {
        "core_technologies": ["Python", "InvalidTech", "Java", "AnotherInvalid", "C++", "Git"],
        "threat_vectors": ["Malware", "InvalidThreat", "Phishing", "AnotherInvalid", "DDoS"]
    }
    expected_output = {
        "core_technologies": ["Python", "Java", "C++", "Git"],
        "threat_vectors": ["Malware", "Phishing", "DDoS"]
    }
    result = filter_categories(input_categories)
    assert result == expected_output

def test_filter_categories_all_invalid():
    input_categories = {
        "core_technologies": ["InvalidTech1", "InvalidTech2"],
        "threat_vectors": ["InvalidThreat1", "InvalidThreat2"]
    }
    expected_output = {
        "core_technologies": [],
        "threat_vectors": []
    }
    result = filter_categories(input_categories)
    assert result == expected_output

def test_filter_categories_mixed_known_and_unknown():
    input_categories = {
        "core_technologies": ["Python", "Java"],
        "unknown_category1": ["Item1", "Item2"],
        "threat_vectors": ["Malware", "Phishing"],
        "unknown_category2": ["Item3", "Item4"]
    }
    expected_output = {
        "core_technologies": ["Python", "Java"],
        "threat_vectors": ["Malware", "Phishing"],
    }
    result = filter_categories(input_categories)
    assert result == expected_output

def test_filter_categories_whitespace_and_punctuation():
    input_categories = {
        "core_technologies": [" Python ", "Java!", "C++,"],
        "threat_vectors": ["Malware.", " Phishing ", "DDoS!"]
    }
    expected_output = {
        "core_technologies": ["Python", "Java", "C++"],
        "threat_vectors": ["Malware", "Phishing", "DDoS"]
    }
    result = filter_categories(input_categories)
    assert result == expected_output

def test_filter_categories_all_predefined():
    input_categories = {
        "core_technologies": PREDEFINED_CATEGORIES["core_technologies"],
        "threat_vectors": PREDEFINED_CATEGORIES["threat_vectors"],
        "impacted_systems": PREDEFINED_CATEGORIES["impacted_systems"],
        "cybersecurity_domains": PREDEFINED_CATEGORIES["cybersecurity_domains"]
    }
    expected_output = input_categories.copy()

    result = filter_categories(input_categories)
    assert result == expected_output, f"Differences: {set(sum(expected_output.values(), [])) - set(sum(result.values(), []))}"

def test_filter_categories_comma_separated_string():
    input_categories = {
        "core_technologies": "Python, Java, InvalidTech, C++",
        "threat_vectors": "Malware, InvalidThreat, Phishing, DDoS",
        "impacted_systems": "Windows, InvalidSystem, Linux, Android",
        "cybersecurity_domains": "Network Security, InvalidDomain, Cloud Security, IoT Security"
    }
    expected_output = {
        "core_technologies": ["Python", "Java", "C++"],
        "threat_vectors": ["Malware", "Phishing", "DDoS"],
        "impacted_systems": ["Windows", "Linux", "Android"],
        "cybersecurity_domains": ["Network Security", "Cloud Security", "IoT Security"]
    }

    result = filter_categories(input_categories)
    assert result == expected_output

    input_categories = {
        "core_technologies": "Rust",
        "threat_vectors": "Ransomware, Data Breach, Unauthorized Access, Social Engineering",
        "impacted_systems": "Windows, Linux, Medical Devices",
        "cybersecurity_domains": "Incident Response, Data Protection, Threat Intelligence, Malware Analysis"
    }

    expected_output = {
        "core_technologies": ["Rust"],
        "threat_vectors": ["Ransomware", "Data Breach", "Unauthorized Access", "Social Engineering"],
        "impacted_systems": ["Windows", "Linux", "Medical Devices"],
        "cybersecurity_domains": ["Incident Response", "Data Protection", "Threat Intelligence", "Malware Analysis"]
    }

    result = filter_categories(input_categories)
    assert result == expected_output