error on the combined model code # Import necessary libraries
import os
import pandas as pd
import numpy as np
import tensorflow as tf
import tensorflow_hub as hub
import librosa
import librosa.display
from sklearn.preprocessing import LabelEncoder, OneHotEncoder
from sklearn.model_selection import train_test_split
from sklearn.utils import class_weight
from keras.models import Model
from keras.layers import Input, Dense, Dropout, BatchNormalization, Conv1D, MaxPooling1D, Flatten, concatenate
from keras.callbacks import EarlyStopping
import matplotlib.pyplot as plt
import seaborn as sns

from google.colab import drive
drive.mount('/content/drive')


# Function to preprocess text data
def preprocess_text(file_path):
    data = pd.read_csv(file_path, sep=';')
    happy_emotions = ['joy', 'love', 'surprise']
    data['label'] = data['emotion'].apply(lambda x: 1 if x in happy_emotions else 0)
    return data

# Load and preprocess text datasets
dataset_dir = '/content/'  # Adjust if necessary
train_text_path = os.path.join(dataset_dir, 'train.txt')
val_text_path = os.path.join(dataset_dir, 'val.txt')

train_text_data = preprocess_text(train_text_path)
val_text_data = preprocess_text(val_text_path)


# Prepare audio file paths and labels
audio_dataset_dir = '/content/drive/MyDrive/emotion_dataset/Emotions/'  # Adjust path
audio_paths = []
audio_labels = []

for dirname, _, filenames in os.walk(audio_dataset_dir):
    for filename in filenames:
        if filename.endswith('.wav'):
            file_path = os.path.join(dirname, filename)
            audio_paths.append(file_path)
            label = os.path.basename(dirname).lower()
            audio_labels.append(label)

# Create a DataFrame for audio data
audio_df = pd.DataFrame({'speech': audio_paths, 'emotion': audio_labels})

# Map emotions to binary labels consistent with text data
happy_emotions = ['joy', 'love', 'surprise']
audio_df['label'] = audio_df['emotion'].apply(lambda x: 1 if x in happy_emotions else 0)

# For demonstration, we'll create synthetic IDs to merge datasets
train_text_data['id'] = range(len(train_text_data))
audio_df['id'] = range(len(audio_df))

# Merge text and audio data on 'id' (ensure alignment in your actual data)
multimodal_data = pd.merge(train_text_data, audio_df, on='id', suffixes=('_text', '_audio'))

# Drop unnecessary columns
multimodal_data = multimodal_data[['text', 'speech', 'label_text']]
multimodal_data.rename(columns={'label_text': 'label'}, inplace=True)



def extract_mfcc(filename, n_mfcc=40):
    try:
        y, sr = librosa.load(filename, duration=3, offset=0.5)
        mfcc = librosa.feature.mfcc(y=y, sr=sr, n_mfcc=n_mfcc)
        mfcc_scaled = np.mean(mfcc.T, axis=0)
        return mfcc_scaled
    except Exception as e:
        print(f"Error processing {filename}: {e}")
        return None

# Apply MFCC extraction
multimodal_data['mfcc'] = multimodal_data['speech'].apply(lambda x: extract_mfcc(x))

# Drop rows where MFCC extraction failed
multimodal_data = multimodal_data.dropna(subset=['mfcc'])


# Ensure MFCC extraction is completed first
audio_df['mfcc'] = audio_df['speech'].apply(lambda x: extract_mfcc(x))

# Drop any rows where MFCC may have failed
audio_df.dropna(subset=['mfcc'], inplace=True)

# Ensure IDs or unique keys are consistent if used for merging
train_text_data['id'] = train_text_data.index
audio_df['id'] = audio_df.index

# Merge datasets on 'id' (ensuring proper alignment)
multimodal_data = pd.merge(train_text_data, audio_df, on='id', suffixes=('_text', '_audio'))

# Check the resulting DataFrame columns
print("Columns in multimodal_data after merge:", multimodal_data.columns)

# Select only relevant columns
if 'mfcc' in multimodal_data.columns:
    multimodal_data = multimodal_data[['text', 'mfcc', 'label_text']]

# Rename 'label_text' to 'label'
multimodal_data.rename(columns={'label_text': 'label'}, inplace=True)

print(multimodal_data.head())


# Include 'mfcc' in the merged DataFrame
multimodal_data = pd.merge(train_text_data, audio_df, on='id', suffixes=('_text', '_audio'))

# Select relevant columns, including 'mfcc'
multimodal_data = multimodal_data[['text', 'mfcc', 'label_text']]
multimodal_data.rename(columns={'label_text': 'label'}, inplace=True)

# Drop rows where MFCC extraction failed
multimodal_data = multimodal_data.dropna(subset=['mfcc'])

# Ensure 'mfcc' feature vectors are consistent in length
mfcc_lengths = multimodal_data['mfcc'].apply(lambda x: len(x))
if len(mfcc_lengths.unique()) > 1:
    print("Inconsistent MFCC feature lengths detected.")
    # Handle inconsistencies here (e.g., padding/truncating)

# Convert 'mfcc' column to a NumPy array
audio_features = np.array(multimodal_data['mfcc'].tolist())
print('audio_features.shape:', audio_features.shape)

# Reshape 'audio_features' if necessary
if len(audio_features.shape) == 2:
    audio_features = audio_features.reshape(audio_features.shape[0], audio_features.shape[1], 1)
else:
    print("Unexpected shape for 'audio_features':", audio_features.shape)
    # Handle the error or adjust the reshaping accordingly

# Proceed with labels
labels = multimodal_data['label'].values

# Split data into training and validation sets
X_text_train, X_text_val, X_audio_train, X_audio_val, y_train, y_val = train_test_split(
    multimodal_data['text'], audio_features, labels, test_size=0.2, random_state=42, stratify=labels) ---------------------------------------------------------------------------
KeyError                                  Traceback (most recent call last)
<ipython-input-23-2b90e0397987> in <cell line: 5>()
      3 
      4 # Select relevant columns, including 'mfcc'
----> 5 multimodal_data = multimodal_data[['text', 'mfcc', 'label_text']]
      6 multimodal_data.rename(columns={'label_text': 'label'}, inplace=True)
      7 

2 frames
/usr/local/lib/python3.10/dist-packages/pandas/core/indexes/base.py in _raise_if_missing(self, key, indexer, axis_name)
   6250 
   6251             not_found = list(ensure_index(key)[missing_mask.nonzero()[0]].unique())
-> 6252             raise KeyError(f"{not_found} not in index")
   6253 
   6254     @overload

KeyError: "['mfcc'] not in index"