import numpy as np
import pandas as pd
import os
import pickle
from tqdm import tqdm
import logging
import time

# Set up logging
logging.basicConfig(level=logging.INFO, format='%(asctime)s %(levelname)s:%(message)s')

# Paths to save the results
os.makedirs(output_dir, exist_ok=True)

# Function to perform mean pooling
def mean_pooling(embeddings_list):
    return np.mean(embeddings_list, axis=0)

# Function to perform max pooling
def max_pooling(embeddings_list):
    return np.max(embeddings_list, axis=0)

# Aggregation function with progress tracking
def aggregate_embeddings(embeddings_dict, pooling_method):
    embeddings_list = []
    note_id_list = []

    for note_id, embeddings in tqdm(embeddings_dict.items(), desc=f'Aggregating with {pooling_method.__name__}'):
        pooled_embedding = pooling_method(embeddings)
        embeddings_list.append(pooled_embedding)
        note_id_list.append(note_id)

    embeddings_df = pd.DataFrame(embeddings_list, index=note_id_list)
    return embeddings_df

# Load datasets with progress tracking and logging
logging.info("Loading datasets...")

# Timing for loading data
start_time = time.time()

with open(events_w_value_features_embeddings_embed_chunked_final, 'rb') as f:
    events_df = pickle.load(f)
with open(oc_fp, 'rb') as f:
    oc = pickle.load(f)

# Log loading stats
end_time = time.time()
logging.info(f"Datasets loaded in {end_time - start_time:.2f} seconds.")
logging.info(f"Events DataFrame: {events_df.shape[0]} rows and {events_df.shape[1]} columns")
logging.info(f"OC DataFrame: {oc.shape[0]} rows and {oc.shape[1]} columns")

# Step 1: Extract unique subjects and shuffle for train-valid-test split
logging.info("Splitting data into train/valid/test sets...")
start_time = time.time()

all_sup_subjects = oc['SUBJECT_ID'].unique()
np.random.seed(0)
np.random.shuffle(all_sup_subjects)

# Split dataset into train (64%), valid (16%), test (20%)
S = len(all_sup_subjects)
bp1, bp2 = int(0.64 * S), int(0.8 * S)

train_sub = all_sup_subjects[:bp1]
valid_sub = all_sup_subjects[bp1:bp2]
test_sub = all_sup_subjects[bp2:]

# Assign ts_ids to the respective sets based on SUBJECT_ID
train_ids = np.array(oc.loc[oc['SUBJECT_ID'].isin(train_sub), 'ts_id'])
valid_ids = np.array(oc.loc[oc['SUBJECT_ID'].isin(valid_sub), 'ts_id'])
test_ids = np.array(oc.loc[oc['SUBJECT_ID'].isin(test_sub), 'ts_id'])

end_time = time.time()
logging.info(f"Data splitting completed in {end_time - start_time:.2f} seconds.")
logging.info(f"Train set: {len(train_ids)} ts_ids")
logging.info(f"Validation set: {len(valid_ids)} ts_ids")
logging.info(f"Test set: {len(test_ids)} ts_ids")

# Step 2: Filter 'events' based on the train, valid, and test ids
logging.info("Filtering events data by ts_id for each split...")
start_time = time.time()

events_train = events_df[events_df['ts_id'].isin(train_ids)]
events_valid = events_df[events_df['ts_id'].isin(valid_ids)]
events_test = events_df[events_df['ts_id'].isin(test_ids)]

end_time = time.time()
logging.info(f"Filtering completed in {end_time - start_time:.2f} seconds.")
logging.info(f"Train Events: {events_train.shape[0]} rows")
logging.info(f"Validation Events: {events_valid.shape[0]} rows")
logging.info(f"Test Events: {events_test.shape[0]} rows")

# Placeholder: Load your pre-existing embeddings
logging.info("Loading embeddings...")
start_time = time.time()


with open(embeddings_with_value, 'rb') as f:
    embeddings_dict_value = pickle.load(f)
with open(embeddings_with_feature, 'rb') as f:
    embeddings_dict_feature = pickle.load(f)

end_time = time.time()
logging.info(f"Embeddings loaded in {end_time - start_time:.2f} seconds.")
logging.info(f"Value embeddings: {len(embeddings_dict_value)} items")
logging.info(f"Feature embeddings: {len(embeddings_dict_feature)} items")

# Step 3: Aggregate and save embeddings for each split
def process_and_save_embeddings(events, embeddings_dict_value, embeddings_dict_feature, split_name):
    logging.info(f"Processing and saving embeddings for {split_name}...")

    # Start time for processing
    start_time = time.time()

    # Extract ts_ids for the current split
    split_ts_ids = events['ts_id'].unique()
    
    logging.info(f"Number of unique ts_ids in {split_name} split: {len(split_ts_ids)}")
    
    # Filter embeddings_dict by split ts_ids
    split_value_embeddings = {k: v for k, v in embeddings_dict_value.items() if k in split_ts_ids}
    split_feature_embeddings = {k: v for k, v in embeddings_dict_feature.items() if k in split_ts_ids}

    # Debugging: Log the size of filtered embeddings
    logging.info(f"{split_name} Value Embeddings: {len(split_value_embeddings)}")
    logging.info(f"{split_name} Feature Embeddings: {len(split_feature_embeddings)}")

    if not split_value_embeddings or not split_feature_embeddings:
        logging.error(f"Error: No embeddings found for {split_name}.")
        return

    # Perform mean pooling
    logging.info(f"Aggregating {split_name} embeddings using mean pooling...")
    embeddings_df_value_mean = aggregate_embeddings(split_value_embeddings, mean_pooling)
    embeddings_df_feature_mean = aggregate_embeddings(split_feature_embeddings, mean_pooling)

    # Save mean-pooled embeddings
    value_mean_path = os.path.join(output_dir, f'{split_name}_embeddings_value_mean.pkl')
    feature_mean_path = os.path.join(output_dir, f'{split_name}_embeddings_feature_mean.pkl')

    embeddings_df_value_mean.to_pickle(value_mean_path)
    embeddings_df_feature_mean.to_pickle(feature_mean_path)
    logging.info(f"Mean-pooled embeddings for {split_name} saved to {value_mean_path} and {feature_mean_path}")

    # Perform max pooling
    logging.info(f"Aggregating {split_name} embeddings using max pooling...")
    embeddings_df_value_max = aggregate_embeddings(split_value_embeddings, max_pooling)
    embeddings_df_feature_max = aggregate_embeddings(split_feature_embeddings, max_pooling)

    # Save max-pooled embeddings
    value_max_path = os.path.join(output_dir, f'{split_name}_embeddings_value_max.pkl')
    feature_max_path = os.path.join(output_dir, f'{split_name}_embeddings_feature_max.pkl')

    embeddings_df_value_max.to_pickle(value_max_path)
    embeddings_df_feature_max.to_pickle(feature_max_path)
    logging.info(f"Max-pooled embeddings for {split_name} saved to {value_max_path} and {feature_max_path}")

    end_time = time.time()
    logging.info(f"Processing for {split_name} completed in {end_time - start_time:.2f} seconds.")

# Process embeddings for train, valid, and test splits
process_and_save_embeddings(events_train, embeddings_dict_value, embeddings_dict_feature, 'train')
process_and_save_embeddings(events_valid, embeddings_dict_value, embeddings_dict_feature, 'valid')
process_and_save_embeddings(events_test, embeddings_dict_value, embeddings_dict_feature, 'test')

logging.info("All processing completed.")


Is the above code valid? does it align with the following code snippet?


# Get train-valid-test split for sup task.
all_sup_subjects = icu.SUBJECT_ID.unique()
np.random.seed(0)
np.random.shuffle(all_sup_subjects)
S = len(all_sup_subjects)
bp1, bp2 = int(0.64*S), int(0.8*S)
train_sub = all_sup_subjects[:bp1]
valid_sub = all_sup_subjects[bp1:bp2]
test_sub = all_sup_subjects[bp2:]
icu.rename(columns={'ICUSTAY_ID':'ts_id'}, inplace=True)
train_ids = np.array(icu.loc[icu.SUBJECT_ID.isin(train_sub)].ts_id)
valid_ids = np.array(icu.loc[icu.SUBJECT_ID.isin(valid_sub)].ts_id)
test_ids = np.array(icu.loc[icu.SUBJECT_ID.isin(test_sub)].ts_id)

# Filter columns.
events = events[['ts_id', 'minute', 'variable', 'value', 'TABLE']]

# Aggregate data.
events['value'] = events['value'].astype(float)
events.loc[events['TABLE'].isna(), 'TABLE'] = 'N/A'
events = events.groupby(['ts_id', 'minute', 'variable']).agg(
                {'value':'mean', 'TABLE':'unique'}).reset_index()
def f(x):
    if len(x)==0:
        return x[0]
    else:
        return ','.join(x)
events['TABLE'] = events['TABLE'].apply(f)

# Save data.
os.makedirs('../data/processed', exist_ok=True)
pickle.dump([events, oc, train_ids, valid_ids, test_ids], 
            open('../data/processed/mimic_iii.pkl','wb'))