my loss goes to be negative, check and let me know about my mistakes, the loss is cosine sim. import pandas as pd
import numpy as np
import tensorflow as tf
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score, roc_auc_score
from sklearn.metrics import mean_absolute_error, explained_variance_score
import matplotlib.pyplot as plt
from sklearn.cross_decomposition import CCA

# Load data


d1 = pd.read_csv("/zi/home/sajad.rezaei/clip/clip/02_Full_Pipeline/data/psytools/IMAGEN-IMGN_ADRS_CHILD_FU2-IMAGEN_DIGEST.csv")
d2 = pd.read_csv("/zi/home/sajad.rezaei/clip/clip/02_Full_Pipeline/data/psytools/IMAGEN-IMGN_ANXDX_CHILD_FU2-IMAGEN_DIGEST.csv")
d3 = pd.read_csv("/zi/home/sajad.rezaei/clip/clip/02_Full_Pipeline/data/psytools/IMAGEN-IMGN_AUDIT_CHILD_FU2-IMAGEN_DIGEST.csv")
d4 = pd.read_csv("/zi/home/sajad.rezaei/clip/clip/02_Full_Pipeline/data/psytools/IMAGEN-IMGN_BIS_CHILD_FU2-IMAGEN_DIGEST.csv")
d5 = pd.read_csv("//zi/home/sajad.rezaei/clip/clip/02_Full_Pipeline/data/psytools/IMAGEN-IMGN_CAPE_CHILD_FU2-IMAGEN_DIGEST.csv")
d6 = pd.read_csv("/zi/home/sajad.rezaei/clip/clip/02_Full_Pipeline/data/psytools/IMAGEN-IMGN_CSI_CHILD_FU2-IMAGEN_DIGEST.csv")
d7 = pd.read_csv("/zi/home/sajad.rezaei/clip/clip/02_Full_Pipeline/data/psytools/IMAGEN-IMGN_CTQ_CHILD_FU2-IMAGEN_DIGEST.csv")
d8 = pd.read_csv("/zi/home/sajad.rezaei/clip/clip/02_Full_Pipeline/data/psytools/IMAGEN-IMGN_EDEQ_CHILD_FU2-IMAGEN_DIGEST.csv")
d9= pd.read_csv("/zi/home/sajad.rezaei/clip/clip/02_Full_Pipeline/data/psytools/IMAGEN-IMGN_JVQ_CHILD_FU2-IMAGEN_DIGEST.csv")
d10= pd.read_csv("/zi/home/sajad.rezaei/clip/clip/02_Full_Pipeline/data/psytools/IMAGEN-IMGN_TFEQ_CHILD_FU2-IMAGEN_DIGEST.csv")
data_df= pd.read_csv('/zi/home/sajad.rezaei/clip/clip/01_scripts/00_Text_processing/00_data/01_IMAGEN/00_data/IMAGEN_FU2.csv')


# Preprocess data

d7["User code"] = d7["User code"].str.replace("-I", "-C")
df_list = [d1, d2, d3, d4, d5, d6, d7, d8, d9, d10]


# In[13]:


patterns = [
    "adrs", "ANXDX", "audit", "BIS", "CAPE42", "item",
    "CTQ", "EDEQ", "SCID", "IRI", "JVQ", "PAAQ",
    "RRS", "tci", "TFEQ", "User code"
]



def process_dataframe(df, patterns, drop_zero_rows=False):
    df = df[df.columns[df.columns.str.contains('|'.join(patterns))]]
    df.loc[:,"User code"]= df.loc[:,"User code"].str.replace("-C", "")
    df.loc[:,"User code"]= df.loc[:,"User code"].str.lstrip("0")
    
    # Convert all elements to numeric, forcing non-numeric to NaN
    df_numeric = df.apply(pd.to_numeric, errors='coerce')

    # Replace NaN values with 0
    df_numeric.fillna(0, inplace=True)
    
    # Replace empty strings with 0
    df_numeric.replace("", 0, inplace=True)
    
    # remove the row with all zeros besiders the first column
    if drop_zero_rows:
        df_numeric = df_numeric.loc[(df_numeric.iloc[:, 1:] != 0).any(axis=1)]
    return df_numeric



df_list_processed = [process_dataframe(df, patterns, drop_zero_rows=True) for df in df_list]
# drop rows with NaN
df_list_processed = [df.dropna() for df in df_list_processed]

# merge all df on User code
#merge all df on User code
merged_df = df_list_processed[0]
for i, df in enumerate(df_list_processed, start=2):
    merged_df = pd.merge(merged_df, df, on='User code', suffixes=('', f'_df{i}'))


# drop all row of data_df besides the first and the last
data_df = data_df.drop(data_df.columns[1:-1], axis=1)
data_df.rename(columns={'Unnamed: 0': 'User code'}, inplace=True)


# merge the merged_df with data_df
merged_df = pd.merge(data_df, merged_df, on='User code')

# Pre-process dataframe for TensorFlow
df = merged_df.drop(columns=['User code', "FilePath"])
non_binary_columns = [col for col in df.columns if len(df[col].unique()) > 2]
df = pd.get_dummies(df, columns=non_binary_columns, drop_first=True)
df = df.astype(float)
# Split data into train and validation sets
X_train, X_val = train_test_split(df.values, test_size=0.20, random_state=42)

# Define the Autoencoder model
# class Autoencoder(tf.keras.Model):
#     def __init__(self, input_size):
#         super().__init__()
#         self.encoder = tf.keras.Sequential([
#             tf.keras.layers.Dense(128, activation="relu", input_shape=(input_size,)),
#             tf.keras.layers.BatchNormalization(),
#             tf.keras.layers.Dense(64, activation="relu"),
#             tf.keras.layers.BatchNormalization(),
#             tf.keras.layers.Dense(32, activation="relu"),
#             tf.keras.layers.Dense(16, activation="relu")
#         ])
#         self.decoder = tf.keras.Sequential([
#             tf.keras.layers.Dense(32, activation="relu"),
#             tf.keras.layers.BatchNormalization(),
#             tf.keras.layers.Dense(64, activation="relu"),
#             tf.keras.layers.BatchNormalization(),
#             tf.keras.layers.Dense(128, activation="relu"),
#             tf.keras.layers.Dense(input_size, activation="sigmoid")
#         ])

#     def call(self, x):
#         encoded = self.encoder(x)
#         decoded = self.decoder(encoded)
        # return decoded

# Define the CNN Autoencoder model

# Define the CNN Autoencoder model
class Autoencoder(tf.keras.Model):
    def __init__(self, input_size):
        super(Autoencoder, self).__init__()
        self.encoder = tf.keras.Sequential([
            tf.keras.layers.Input(shape=(input_size, 1 )),
            tf.keras.layers.Conv1D(128, kernel_size=3, activation="relu", padding="same"),
            tf.keras.layers.BatchNormalization(),
            tf.keras.layers.MaxPooling1D(pool_size=3, padding="same"),
            tf.keras.layers.Conv1D(64, kernel_size=3, activation="relu", padding="same"),
            tf.keras.layers.BatchNormalization(),
            tf.keras.layers.MaxPooling1D(pool_size=3),
        ])
        self.decoder = tf.keras.Sequential([
            tf.keras.layers.UpSampling1D(size=3),
            tf.keras.layers.Conv1DTranspose(64, kernel_size=3,  activation="relu", padding="same"),
            tf.keras.layers.BatchNormalization(),
            tf.keras.layers.UpSampling1D(size=3),
            tf.keras.layers.Conv1DTranspose(128, kernel_size=3,  activation="relu"),
            tf.keras.layers.BatchNormalization(),
            tf.keras.layers.ZeroPadding1D(padding=2),
            # tf.keras.layers.Dense(input_size, activation="sigmoid")

            tf.keras.layers.Conv1D(1, kernel_size=3, activation="sigmoid")
        ])

    def call(self, x):
        x = tf.expand_dims(x, axis=-1)  # Add channel dimension
        encoded = self.encoder(x)
        decoded = self.decoder(encoded)
        decoded = tf.squeeze(decoded, axis=-1)  # Remove channel dimension
        return decoded
cos = tf.keras.losses.CosineSimilarity(
    axis=-1, reduction="none", name="cosine_similarity"
)
# Compile and train the model
# cos = tf.keras.losses.MeanSquaredLogarithmicError(
#     reduction="sum_over_batch_size", name="mean_squared_logarithmic_error"
# )
input_size = df.shape[1]
autoencoder = Autoencoder(input_size)
autoencoder.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=0.01), loss="LogCosh")

early_stopping = tf.keras.callbacks.EarlyStopping(monitor="val_loss", patience=10, restore_best_weights=True)

history = autoencoder.fit(X_train, X_train,
                          epochs=100,
                          batch_size=512,
                          validation_data=(X_val, X_val),
                          callbacks=[early_stopping])

# Evaluate the model
train_predictions = autoencoder.predict(X_train)
val_predictions = autoencoder.predict(X_val)

train_mse = np.mean(np.square(train_predictions - X_train))
val_mse = np.mean(np.square(val_predictions - X_val))
train_mae = mean_absolute_error(X_train, train_predictions)
val_mae = mean_absolute_error(X_val, val_predictions)

train_explained_var = explained_variance_score(X_train, train_predictions)
val_explained_var = explained_variance_score(X_val, val_predictions)

# print(f"Training MSE: {train_mse:.4f}, Validation MSE: {val_mse:.4f}")

print(f"Training MSE: {train_mse:.4f}, Validation MSE: {val_mse:.4f}")
print(f"Training MAE: {train_mae:.4f}, Validation MAE: {val_mae:.4f}")
print(f"Training Explained Variance: {train_explained_var:.4f}, Validation Explained Variance: {val_explained_var:.4f}")
# Compute Canonical Correlations
# cca = CCA(n_components=min(X_train.shape[1], train_predictions.shape[1]))
num_components = min(X_train.shape[0], train_predictions.shape[0])
cca = CCA(n_components=num_components)
cca.fit(X_train, train_predictions)
X_train_c, train_predictions_c = cca.transform(X_train, train_predictions)

# Compute the correlation for each pair of canonical variables
cca_correlations = np.corrcoef(X_train_c.T, train_predictions_c.T)[:X_train_c.shape[1], X_train_c.shape[1]:]

print("Canonical Correlations:")
for i, corr in enumerate(cca_correlations.diagonal()):
    print(f"Canonical correlation {i+1}: {corr:.4f}")


# Plot loss curves for training and validation
plt.plot(history.history["loss"], label="Training Loss")
plt.plot(history.history["val_loss"], label="Validation Loss")
plt.legend()
plt.xlabel("Epochs")
plt.ylabel("Loss")
plt.title("Training and Validation Loss")
plt.show()

# Calculate accuracy and AUC if needed
# Note: Since this is an autoencoder, calculating binary accuracy or AUC might not be meaningful in a reconstruction context