GET THROUGH THE CODE CAREFULLY AND UNDERTAND IT, THEN FOLLOW THE INSTRUCTION OF REDACTING BELOW

import os
import shutil
import pandas as pd

# Paths to your folders and Excel file
extract_folder = r'C:\Users\larar\OneDrive\Documentos\Escritorio\Histopathological_Diagnosis\EXTRACT_annotated'
usable_folder = r'C:\Users\larar\OneDrive\Documentos\Escritorio\Histopathological_Diagnosis\USABLE_annotated'
excel_path = 'C:/Users/larar/OneDrive/Documentos/Escritorio/Histopathological_Diagnosis-3/Excel Files/HP_WSI-CoordAnnotatedPatches.xlsx'

# Load the Excel file into a DataFrame
excel_df = pd.read_excel(excel_path)
excel_df['Pat_ID'] = excel_df['Pat_ID'].astype(str)
excel_df['Window_ID'] = excel_df['Window_ID'].astype(str)

# Ensure the 'Pat_ID', 'Window_ID', and 'Presence' columns are present
if not all(col in excel_df.columns for col in ['Pat_ID', 'Window_ID', 'Presence']):
    raise ValueError("The Excel file must contain 'Pat_ID', 'Window_ID', and 'Presence' columns.")

# Create the USABLE folder if it doesn't already exist
os.makedirs(usable_folder, exist_ok=True)

# Prepare a list to hold records for the TRAIN_DATA.csv file
train_data_records = []

counter = 0
# Iterate through each file in the EXTRACT folder
for file_name in os.listdir(extract_folder):
    # Skip if not an image file (e.g., in this case we can assume JPEG or PNG files)
    if not (file_name.endswith('.jpg') or file_name.endswith('.png')):
        continue
    
    base_name = os.path.splitext(file_name)[0]
    
    # Split filename by underscore to get Pat_ID and Window_ID
    parts = base_name.split('_')

    # Extract Pat_ID (before the first underscore) and Window_ID (between the first and second underscores)
    pat_id = parts[0]
    window_id_str = parts[1]
    # Remove leading zeros from Window_ID by converting it to an integer and back to string
    window_id = str(int(window_id_str))  # This removes all leading zeros
    
    # Filter the Excel DataFrame for matching Pat_ID and Window_ID
    match = excel_df[(excel_df['Pat_ID'] == pat_id) & (excel_df['Window_ID'] == window_id)]
    
    # If a match is found, and it's in the same row, proceed
    if not match.empty:
        if len(parts) == 2:
            presence_value = match['Presence'].iloc[0]
            if presence_value != 0:
                # Copy the file to the USABLE folder
                src_path = os.path.join(extract_folder, file_name)
                dst_path = os.path.join(usable_folder, file_name)
                shutil.copy(src_path, dst_path)
                
                # Add the matched data to the train_data_records list
                train_data_records.append({
                    'Pat_ID': pat_id,
                    'Window_ID': window_id,
                    'Presence': presence_value
                })
                counter += 1
        else:
            src_path = os.path.join(extract_folder, file_name)
            dst_path = os.path.join(usable_folder, file_name)
            shutil.copy(src_path, dst_path)
            
            # Add the matched data to the train_data_records list
            train_data_records.append({
                'Pat_ID': pat_id,
                'Window_ID': window_id,
                'Presence': -1
            })
            counter += 1

    else:
        # Copy the file to the USABLE folder
        src_path = os.path.join(extract_folder, file_name)
        dst_path = os.path.join(usable_folder, file_name)
        shutil.copy(src_path, dst_path)
        
        # Add the matched data to the train_data_records list
        train_data_records.append({
            'Pat_ID': pat_id,
            'Window_ID': window_id,
            'Presence': -1
        })
        counter += 1


# Save the train_data_records to a CSV file named TRAIN_DATA_annotated.csv
train_data_df = pd.DataFrame(train_data_records)
train_data_df.to_csv(os.path.join(r"C:\Users\larar\OneDrive\Documentos\Escritorio\Histopathological_Diagnosis", 'TRAIN_DATA_annotated.csv'), index=False)

# Print completion message and total count
print("Filtered images have been copied to the USABLE folder.")
print(f"Total images copied to the USABLE folder: {counter}") 
print("TRAIN_DATA_annotated.csv has been saved with matching data records.")

import os
import shutil
import pandas as pd

# Path to the CSV file
csv_path = r"C:\Users\larar\OneDrive\Documentos\Escritorio\PROJECT_VISION2\PatientDiagnosis_cropped.csv"
extract_folder = r"C:\Users\larar\OneDrive\Documentos\Escritorio\EXTRACT_cropped"
usable_folder = r'C:\Users\larar\OneDrive\Documentos\Escritorio\USABLE_cropped'

# Load the CSV file into a DataFrame
csv_df = pd.read_csv(csv_path)

# Ensure the 'CODI' and 'DENSITAT' columns are present
if not all(col in csv_df.columns for col in ['CODI', 'DENSITAT']):
    raise ValueError("The CSV file must contain 'CODI' and 'DENSITAT' columns.")

# Define a mapping for 'DENSITAT' values
density_mapping = {
    'NEGATIVA': 0,
    'BAIXA': 1,
    'ALTA': 1
}

# Apply the mapping to the 'DENSITAT' column
csv_df['DENSITAT'] = csv_df['DENSITAT'].map(density_mapping)

# Prepare the output file path for the new CSV before filtering
train_output_path = r"C:\Users\larar\OneDrive\Documentos\Escritorio\Histopathological_Diagnosis-5\DATA_TRAIN_cropped.csv"

# Save the updated DataFrame to a new CSV file (before filtering)
csv_df[['CODI', 'DENSITAT']].to_csv(train_output_path, index=False)

print(f"DATA_TRAIN_cropped.csv has been saved with the new DENSITAT mapping.")

# Calculate the number of patients removed
total_patients_before_filtering = len(csv_df)
print(total_patients_before_filtering)
filtered_df = csv_df[csv_df['DENSITAT'] == 0]
total_patients_after_filtering = len(filtered_df)

# Calculate how many patients were removed (erased)
patients_erased = total_patients_before_filtering - total_patients_after_filtering
print(f"Number of patients erased: {patients_erased}")


# Create a list of valid CODI values (those with DENSITAT == 0)
valid_codes = filtered_df['CODI'].astype(str).tolist()

# List all files in the extract folder
image_files = os.listdir(extract_folder)

# Check if the usable folder exists, if not, create it
if not os.path.exists(usable_folder):
    os.makedirs(usable_folder)
    print(f"Created the directory: {usable_folder}")

# Iterate over the image files
for image in image_files:
    # Extract the part of the image name before the first underscore
    image_code = image.split('_')[0]

    # Check if the image code is in the list of valid CODI values
    if image_code in valid_codes:
        # Construct the source and destination file paths
        source_path = os.path.join(extract_folder, image)
        destination_path = os.path.join(usable_folder, image)

        # Copy the image to the usable folder
        shutil.copy(source_path, destination_path)
        print(f"Copied: {image} to usable folder")

import zipfile
import os
import shutil

# Paths
crossvalidation_zip_path = r"C:\Users\larar\OneDrive\Documentos\Escritorio\CHALLENGE#2_VISION\CrossValidation.zip"
holdout_zip_path = r"C:\Users\larar\OneDrive\Documentos\Escritorio\CHALLENGE#2_VISION\HoldOut.zip"
extraction_dir_annotated = r"C:\Users\larar\OneDrive\Documentos\Escritorio\Histopathological_Diagnosis-5\EXTRACT_annotated"
extraction_dir_cropped = r"C:\Users\larar\OneDrive\Documentos\Escritorio\EXTRACT_cropped"
extract_dir_holdout = r"C:\Users\larar\OneDrive\Documentos\Escritorio\EXTRACT_holdout"

# User input
level_choice = "patient level"  # or "patch level"

# Extract CrossValidation.zip and locate Annotated.zip or Cropped.zip within it
def extract_crossvalidation_zip(cross_zip_path, extract_to_annotated, extract_to_cropped):
    with zipfile.ZipFile(cross_zip_path, 'r') as cross_zip:
        cross_zip.extractall(extract_to_annotated if level_choice == "patch level" else extract_to_cropped)
        
        # Paths to nested zips
        annotated_zip_path = os.path.join(extract_to_annotated, 'CrossValidation', 'Annotated.zip')
        cropped_zip_path = os.path.join(extract_to_cropped, 'CrossValidation', 'Cropped.zip')
        
        # Extract based on user choice
        if level_choice == "patch level" and os.path.exists(annotated_zip_path):
            print(f"Extracting {annotated_zip_path}...")
            extract_nested_zip(annotated_zip_path, extract_to_annotated)
        elif level_choice == "patient level" and os.path.exists(cropped_zip_path):
            print(f"Extracting {cropped_zip_path}...")
            extract_nested_zip(cropped_zip_path, extract_to_cropped)
            rename_and_move_patient_images(extract_to_cropped)
        else:
            print("Selected zip file not found inside CrossValidation.zip")

# Extract HoldOut.zip and locate HoldOut.zip inside it
def extract_holdout_zip(holdout_zip_path, extract_to_holdout):
    with zipfile.ZipFile(holdout_zip_path, 'r') as holdout_zip:
        holdout_zip.extractall(extract_to_holdout)
        
        # Path to nested HoldOut.zip inside the extracted HoldOut folder
        holdout_inner_zip_path = os.path.join(extract_to_holdout, 'HoldOut', 'HoldOut.zip')
        
        if os.path.exists(holdout_inner_zip_path):
            print(f"Extracting {holdout_inner_zip_path}...")
            extract_nested_zip(holdout_inner_zip_path, extract_to_holdout)
            rename_and_move_patient_images(extract_to_holdout)
        else:
            print("HoldOut.zip not found inside HoldOut.zip")

# Extract nested zip files
def extract_nested_zip(zip_path, extract_to):
    with zipfile.ZipFile(zip_path, 'r') as outer_zip:
        for item in outer_zip.namelist():
            if item.endswith('.zip'):
                with outer_zip.open(item) as nested_zip_file:
                    nested_zip_path = os.path.join(extract_to, item)
                    with open(nested_zip_path, 'wb') as temp_zip:
                        temp_zip.write(nested_zip_file.read())
                    with zipfile.ZipFile(nested_zip_path, 'r') as inner_zip:
                        inner_zip.extractall(extract_to)
                    os.remove(nested_zip_path)
            else:
                outer_zip.extract(item, extract_to)

# Rename images inside each patient folder and move them to EXTRACT_cropped or EXTRACT_holdout
def rename_and_move_patient_images(base_dir):
    for root, dirs, _ in os.walk(base_dir):
        for patient_folder in dirs:
            patient_folder_path = os.path.join(root, patient_folder)

            # Extract patient name up until the first underscore (_)
            patient_name = patient_folder.split('_')[0]

            # Initialize a list to store the renamed images
            renamed_images = []

            for i, image_file in enumerate(os.listdir(patient_folder_path)):
                image_path = os.path.join(patient_folder_path, image_file)
                if os.path.isfile(image_path):
                    # Construct the new name based on patient name and sequence number
                    new_name = os.path.join(base_dir, f"{patient_name}_{i+1}.png")
                    
                    # Rename and move the image to the respective extraction directory
                    shutil.move(image_path, new_name)
                    renamed_images.append(new_name)
                    print(f"Moved and renamed {image_path} to {new_name}")
            
            # Clean up: delete all ZIP files (but keep renamed images)
            for item in os.listdir(patient_folder_path):
                item_path = os.path.join(patient_folder_path, item)
                if item.endswith('.zip') and os.path.isfile(item_path):
                    os.remove(item_path)
                    print(f"Deleted {item_path}")

# Run extraction for CrossValidation.zip
extract_crossvalidation_zip(crossvalidation_zip_path, extraction_dir_annotated, extraction_dir_cropped)

# Run extraction for HoldOut.zip
extract_holdout_zip(holdout_zip_path, extract_dir_holdout)

import os
import shutil
import random
from collections import defaultdict

# Define directories
data_dir = r"C:\Users\larar\OneDrive\Documentos\Escritorio\EXTRACT_cropped"  # Replace with the path to your data_dir
output_dir = 'USABLE_cropped'  # Path to store the cropped images

# Create the output directory if it doesn't exist
os.makedirs(output_dir, exist_ok=True)

# Dictionary to store images by CODI tag
codi_dict = defaultdict(list)

# Traverse the data directory to categorize images by their CODI tag
for filename in os.listdir(data_dir):
    if filename.endswith(('.jpg', '.png', '.jpeg')):  # Add more extensions if needed
        # Extract the CODI tag (part before the first underscore)
        codi_tag = filename.split('_')[0]
        # Store the file path under the corresponding CODI tag
        codi_dict[codi_tag].append(filename)

# For each CODI tag, select the first 3 images and copy them
for codi_tag, filenames in codi_dict.items():
    # Ensure there are at least 3 images to select from
    if len(filenames) >= 3:
        # Shuffle the list of filenames for random selection
        random.shuffle(filenames)
        
        # Select the first 3 random images
        selected_files = filenames[:10]

        # Copy the selected images to the output directory
        for selected_file in selected_files:
            # Define the source file path
            src_file = os.path.join(data_dir, selected_file)
            # Define the destination file path
            dst_file = os.path.join(output_dir, selected_file)
            
            # Copy the file to the new directory
            shutil.copy(src_file, dst_file)

print(f"3 random images per CODI tag have been copied to {output_dir}")

import pandas as pd

# Load both CSV files
annotations_file = pd.read_csv(r"TRAIN_DATA_annotated.csv")
annotations_file_cropped = pd.read_csv(r"TRAIN_DATA_cropped.csv")

# Step 1: Identify patients based on DENSITY in the cropped file
# Filter patients with DENSITY = 1 and DENSITY = 0
positive_patients = annotations_file_cropped[annotations_file_cropped['DENSITAT'] == 1]['CODI'].unique()
negative_patients = annotations_file_cropped[annotations_file_cropped['DENSITAT'] == 0]['CODI'].unique()

# Step 2: Remove rows without a valid patch-level Presence value in the annotations file
# Filter patches to include only those with non-null Presence values
annotations_file = annotations_file.dropna(subset=['Presence'])

# Step 3: Filter patches from annotations_file for each patient group
# Select patches from positive patients and negative patients
positive_patches = annotations_file[annotations_file['Pat_ID'].isin(positive_patients)]
negative_patches = annotations_file[annotations_file['Pat_ID'].isin(negative_patients)]

# Step 4: Save the filtered patches to separate CSV files
positive_patches.to_csv("positive_patches.csv", index=False)
negative_patches.to_csv("negative_patches.csv", index=False)

print("CSV files created:")
print(" - positive_patches.csv: Patches from patients with DENSITY = 1 and valid Presence values")
print(" - negative_patches.csv: Patches from patients with DENSITY = 0 and valid Presence values")



# Track and save mean accuracy
model_accuracies = {}

model_decision = 0 #int(input("Select the method you want to proceed ( 0 = classifier and 1 = autoencoder): "))
if model_decision == 0:

    """
    #patient_data = pd.read_csv(r"C:/Users/larar/OneDrive/Documentos/Escritorio/Histopathological_Diagnosis-5/TRAIN_DATA_cropped.csv")
    #patient_data = patient_data.rename(columns={"CODI": "Pat_ID"})
    # Step 1: Load the Positive and Negative Patient Data
    #positive_patches = pd.read_csv("positive_patches.csv")
    #negative_patches = pd.read_csv("negative_patches.csv")
    # Concatenate to form a single DataFrame containing all patch-level data
    #all_annotations = pd.concat([positive_patches, negative_patches], ignore_index=True)
    # Set up binary Presence values for classification
    #all_annotations['Presence'] = all_annotations['Presence'].map({-1: 0, 1: 1})
    # Merge all_annotations with patient_data, adding `DENSITY` as a new column `patient_Diagnosis`
    #all_annotations = all_annotations.merge(patient_data[['Pat_ID', 'DENSITAT']], on="Pat_ID", how="left")
    # Rename `DENSITY` to `patient_Diagnosis`
    #all_annotations = all_annotations.rename(columns={"DENSITAT": "patient_Diagnosis"})
    # Save to a CSV file to inspect the merged data
    #all_annotations.to_csv("all_annotations_with_patient_diagnosis.csv", index=False)
    #print("Merged data with patient_Diagnosis column saved to 'all_annotations_with_patient_diagnosis.csv'.")
    
    """

    #annotated_csv = pd.read_csv(r"C:\Users\larar\OneDrive\Documentos\Escritorio\Histopathological_Diagnosis-5\all_annotations_with_patient_diagnosis.csv")
    annotated_csv = pd.read_csv(r"/export/fhome/vlia04/MyVirtualEnv/Histopathological_Diagnosis/all_annotations_with_patient_diagnosis.csv")

    # Load all images from `USABLE_annotated` using all_annotations
    data_dir = r"/export/fhome/vlia04/MyVirtualEnv/Histopathological_Diagnosis/USABLE_annotated"
    img_list = LoadAnnotated(annotated_csv, data_dir)  # Assuming this function loads all images for the patches specified
    transform = transforms.Compose([transforms.Resize((224, 224)), transforms.ToTensor()])
    dataset = StandardImageDataset(annotated_csv, img_list, transform=transform)

    print(f"Total number of configurations: {len(all_configurations)}")
    print("CREATE TRAIN AND TEST SUBSET WITH KFOLD")

    # Set up folder to save models
    #save_folder = "saved_models"
    save_folder = r"/export/fhome/vlia04/MyVirtualEnv/Histopathological_Diagnosis/best_saved_models"
    os.makedirs(save_folder, exist_ok=True)

    # Set up folder to save validations
    save_folder1 = r"/export/fhome/vlia04/MyVirtualEnv/Histopathological_Diagnosis/validation_and_training_data"
    os.makedirs(save_folder1, exist_ok=True)

    # Group by `Pat_ID` and use the first `Presence` value for each patient as the stratification label
    patient_labels = annotated_csv.groupby('Pat_ID')['patient_Diagnosis'].first()

    # Set up Stratified K-Fold at patient level
    k_folds = 3
    strat_kfold = StratifiedKFold(n_splits=k_folds, shuffle=False)
    fold_indices = []
    fold_accuracies = {}
    for fold, (train_patient_idx, val_patient_idx) in enumerate(strat_kfold.split(patient_labels.index, patient_labels)):
        print(f"Starting fold {fold + 1}/{k_folds}")
        
        # Select train and validation patient IDs
        train_patient_ids = patient_labels.index[train_patient_idx]
        val_patient_ids = patient_labels.index[val_patient_idx]
        
        # Filter patches based on these patient IDs
        train_df = annotated_csv[annotated_csv['Pat_ID'].isin(train_patient_ids)]
        val_df = annotated_csv[annotated_csv['Pat_ID'].isin(val_patient_ids)]
        
        # Collect indices for train and validation patches
        train_idx = train_df.index.tolist()
        val_idx = val_df.index.tolist()

        # For the training set, you already have the necessary information
        train_labels = train_df['patient_Diagnosis'].values.tolist()  # Patient-level labels for the training set
        train_subset_info = {
            'train_patient_ids': train_patient_ids,
            'train_labels': train_labels,  # Save the patient-level labels
            'train_indices': train_idx
        }
        torch.save(train_subset_info, os.path.join(save_folder1, f"train_subset_info_fold{fold+1}.pth"))

        # For the validation set, add the patient-level labels along with indices
        val_labels = val_df['patient_Diagnosis'].values.tolist()  # Patient-level labels for the validation set
        val_subset_info = {
            'val_patient_ids': val_patient_ids,
            'val_labels': val_labels,  # Save the patient-level labels
            'val_indices': val_idx
        }
        torch.save(val_subset_info, os.path.join(save_folder1, f"val_subset_info_fold{fold+1}.pth"))

        # Create subsets for training and validation
        train_subset = Subset(dataset, train_idx)
        val_subset = Subset(dataset, val_idx)

        # Data loaders
        train_loader = DataLoader(train_subset, batch_size=32, shuffle=False)
        val_loader = DataLoader(val_subset, batch_size=32, shuffle=False)

        fold_accuracies[fold + 1] = []

        # Loop through each model configuration
        
        for config_idx, config in enumerate(all_configurations, start=1):
            print(f"Training config {config_idx}/{len(all_configurations)} in fold {fold + 1}")
            print(f"Configuration details: {config}")
            
            filename = (f"{config['model_name']}_{config['num_layers']}layers_"
                        f"{'_'.join(map(str, config['units_per_layer']))}_dropout{config['dropout']}_fold{fold + 1}.pth")
            
            save_path = os.path.join(save_folder, filename)

            if os.path.exists(save_path):
                print(f"Model {filename} exists. jump...")
                continue

            # Initialize the custom model
            custom_model = CustomModel(
                model_name=config["model_name"],
                embedding_dim=None,
                fc_layers=config["units_per_layer"],
                activations=["relu"] * config["num_layers"],
                batch_norms=[None] * config["num_layers"],
                dropout=config["dropout"],
                num_classes=2
            )
            
            # Set device
            device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
            if torch.cuda.is_available():
                print("CUDA está disponible. Usando GPU:", torch.cuda.get_device_name(0))
            else:
                print("CUDA no está disponible. Usando CPU.")
            
            custom_model.to(device)
            
            # Define loss and optimizer
            pos_weight, neg_weight = weights(annotated_csv)  # Assumes this function calculates class weights
            weight = torch.tensor([pos_weight, neg_weight], device=device)
            criterion = nn.CrossEntropyLoss(weight=weight)
            optimizer = torch.optim.Adam(custom_model.parameters(), lr= 1e-6) 
            
            # Train the model
            epoch_losses, epoch_accuracies, epoch_recall = train_model(custom_model, train_loader, criterion, optimizer, device, epochs=30)
            
            model_filename = (f"{config['model_name']}_{config['num_layers']}layers_"
                        f"{'_'.join(map(str, config['units_per_layer']))}_dropout{config['dropout']}_fold{fold + 1}")
            plot_loss_curve(epoch_losses, epoch_accuracies, epoch_recall, model_filename)

            # Calculate and save mean accuracy for this fold
            mean_fold_accuracy = np.mean(epoch_accuracies)
            fold_accuracies[fold + 1].append(mean_fold_accuracy)
            # Save the model after training
            filename = (f"{config['model_name']}_{config['num_layers']}layers_"
                        f"{'_'.join(map(str, config['units_per_layer']))}_dropout{config['dropout']}_fold{fold + 1}.pth")
            save_path = os.path.join(save_folder, filename)
            # Move model to CPU before saving
            custom_model.to('cpu')
            torch.save(custom_model.state_dict(), save_path)
            print(f"Saved model: {filename}")
            # Frre cache from GPU
            torch.cuda.empty_cache()
            # Free memory from the trash collector
            gc.collect()

import os
import torch
import torch.nn as nn
from torchvision import models
import numpy as np
from collections import OrderedDict
from itertools import product

# Set a custom cache directory for PyTorch models
#torch.hub.set_dir("C:/Users/larar/torch_cache")
torch.hub.set_dir(r"C:\Users\migue\torch_cache")

# Step 1: Generic function to load a model and return its feature size
def get_model_and_features(model_name, pretrained=True):
    if model_name == "resnet50":
        model = models.resnet50(weights="IMAGENET1K_V1" if pretrained else None)
        num_features = model.fc.in_features
        model.fc = nn.Identity()  # Remove final classifier layer
    elif model_name == "densenet201":
        model = models.densenet201(weights="IMAGENET1K_V1" if pretrained else None)
        num_features = model.classifier.in_features
        model.classifier = nn.Identity()  # Remove final classifier layer
    else:
        raise ValueError(f"Model '{model_name}' is not supported. Choose 'resnet50' or 'densenet201'.")
    return model, num_features

# Step 2: Generic function to save model parameters
def save_model_parameters(model_name, folder, filename=None):
    # Define the filename based on the model name if not provided
    if filename is None:
        filename = f"{model_name}_params.npz"
    # Ensure the folder exists
    os.makedirs(folder, exist_ok=True)
    file_path = os.path.join(folder, filename)
    
    # Load the specified model
    model, _ = get_model_and_features(model_name)
    
    # Convert each parameter to numpy and save to .npz file
    params = {k: v.cpu().numpy() for k, v in model.state_dict().items()}
    np.savez(file_path, **params)
    
    print(f"Model parameters saved to {file_path}")

# Step 3: Generic function to load parameters into a model
def load_model_parameters(model_name, filename):
    model, num_features = get_model_and_features(model_name, pretrained=False)  # Initialize without pretrained weights
    params = np.load(filename)  # Load parameters from .npz file
    state_dict = model.state_dict()
    
    # Update model's state_dict with loaded parameters
    for k in state_dict.keys():
        if k in params:
            state_dict[k] = torch.tensor(params[k])  # Convert numpy array back to tensor
    model.load_state_dict(state_dict, strict=False)  # Load state dict with missing keys allowed
    
    return model, num_features

# Custom model class that supports both ResNet and DenseNet
class CustomModel(nn.Module):
    def __init__(self, model_name, embedding_dim, fc_layers, activations, batch_norms, dropout=0.25, pretrained_params=None, num_classes=2):
        super(CustomModel, self).__init__()
        
        # Load the specified model (ResNet or DenseNet) with optional pretrained parameters
        if pretrained_params:
            self.model, self.model_channels = load_model_parameters(model_name, pretrained_params)
        else:
            self.model, self.model_channels = get_model_and_features(model_name, pretrained=True)

        # Embedding layer setup
        if embedding_dim is not None:
            self.embedding = nn.Linear(self.model_channels, embedding_dim)  # If embedding is specified, reduce dimensionality
            self.in_features = embedding_dim
        else:
            self.embedding = None
            self.in_features = self.model_channels

        # Define fully connected layers
        self.fc_block = self.build_fc_block(self.in_features, fc_layers, activations, batch_norms, dropout)
        
        # Final classifier layer (output logits for classification)
        self.classifier = nn.Linear(fc_layers[-1], num_classes)  # Assuming the last layer in fc_layers is the output size

    def build_fc_block(self, in_features, fc_layers, activations, batch_norms, dropout):
        layers = OrderedDict()  # Sequential layer builder
        
        # Add layers to Sequential
        input_dim = in_features
        for i, output_dim in enumerate(fc_layers):
            # Fully connected layer
            layers[f'fc{i+1}'] = nn.Linear(input_dim, output_dim)
            
            # Activation function
            if activations[i] == 'relu':
                layers[f'relu{i+1}'] = nn.ReLU(inplace=True)
            elif activations[i] == 'tanh':
                layers[f'tanh{i+1}'] = nn.Tanh()
            elif activations[i] == 'relu6':
                layers[f'relu6{i+1}'] = nn.ReLU6(inplace=True)
            
            # Batch normalization if specified
            if batch_norms[i] == 'batch':
                layers[f'batchnorm{i+1}'] = nn.BatchNorm1d(output_dim)
            elif batch_norms[i] is None:
                pass  # No batch normalization

            # Dropout layer
            layers[f'dropout{i+1}'] = nn.Dropout(dropout)
            
            # Update input_dim for next layer
            input_dim = output_dim

        return nn.Sequential(layers)
    
    def forward(self, x):
        # Forward through model backbone (without classifier)
        x = self.model(x)
        x = torch.flatten(x, 1)  # Flatten the output
        
        # Forward through embedding layer if defined
        if self.embedding is not None:
            x = self.embedding(x)
        
        # Forward through FC block
        x = self.fc_block(x)
        
        # Forward through final classifier to get logits
        x = self.classifier(x)
        
        return x

I WANT YOU TO EXPLAIN THE FOLLOWING THINGS:
- WHICH DATA DO WE HAVE. WHICH CSVS OR WHATEVER DO WE USE. FOR INSTANCE, THE USABLE (THAT IS WITH THE EXTRACTION OF THE ZIP ETC, EXPLAIN HOW DO WE OBTAIN THEM. EXPLAIN WHAT EACH FILE OF DATA WE USE IS FOR AND WHAT DOES IT OBTAIN.
- EXPLAIN THAT WE HAVE EXPLORED THE DATA.
- DO A SECTION FOR EXPLAINING THAT WE CAN CHOOSE 0 FOR CLASSIFIER AND 1 FOR DETECTION. AND DEVELOP THE CLASSIFIER PART.
- DO A DIFFERENT SECTION TO EXPLAIN THE K-FOLD.

DO IT IN THIS ORDER, BE FORMAL AND DON'T USE MUCH BULLET LISTS, ONLY IF NECESSARY.