import pandas as pd
import numpy as np
import tensorflow as tf
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score, roc_auc_score
from sklearn.metrics import mean_absolute_error, explained_variance_score
import matplotlib.pyplot as plt
from sklearn.cross_decomposition import CCA

# Load data


d1 = pd.read_csv(“/zi/home/sajad.rezaei/clip/clip/02_Full_Pipeline/data/psytools/IMAGEN-IMGN_ADRS_CHILD_FU2-IMAGEN_DIGEST.csv”)
d2 = pd.read_csv(“/zi/home/sajad.rezaei/clip/clip/02_Full_Pipeline/data/psytools/IMAGEN-IMGN_ANXDX_CHILD_FU2-IMAGEN_DIGEST.csv”)
d3 = pd.read_csv(“/zi/home/sajad.rezaei/clip/clip/02_Full_Pipeline/data/psytools/IMAGEN-IMGN_AUDIT_CHILD_FU2-IMAGEN_DIGEST.csv”)
d4 = pd.read_csv(“/zi/home/sajad.rezaei/clip/clip/02_Full_Pipeline/data/psytools/IMAGEN-IMGN_BIS_CHILD_FU2-IMAGEN_DIGEST.csv”)
d5 = pd.read_csv(“//zi/home/sajad.rezaei/clip/clip/02_Full_Pipeline/data/psytools/IMAGEN-IMGN_CAPE_CHILD_FU2-IMAGEN_DIGEST.csv”)
d6 = pd.read_csv(“/zi/home/sajad.rezaei/clip/clip/02_Full_Pipeline/data/psytools/IMAGEN-IMGN_CSI_CHILD_FU2-IMAGEN_DIGEST.csv”)
d7 = pd.read_csv(“/zi/home/sajad.rezaei/clip/clip/02_Full_Pipeline/data/psytools/IMAGEN-IMGN_CTQ_CHILD_FU2-IMAGEN_DIGEST.csv”)
d8 = pd.read_csv(“/zi/home/sajad.rezaei/clip/clip/02_Full_Pipeline/data/psytools/IMAGEN-IMGN_EDEQ_CHILD_FU2-IMAGEN_DIGEST.csv”)
d9= pd.read_csv(“/zi/home/sajad.rezaei/clip/clip/02_Full_Pipeline/data/psytools/IMAGEN-IMGN_JVQ_CHILD_FU2-IMAGEN_DIGEST.csv”)
d10= pd.read_csv(“/zi/home/sajad.rezaei/clip/clip/02_Full_Pipeline/data/psytools/IMAGEN-IMGN_TFEQ_CHILD_FU2-IMAGEN_DIGEST.csv”)
data_df= pd.read_csv(‘/zi/home/sajad.rezaei/clip/clip/01_scripts/00_Text_processing/00_data/01_IMAGEN/00_data/IMAGEN_FU2.csv’)


# Preprocess data

d7[“User code”] = d7[“User code”].str.replace(“-I”, “-C”)
df_list = [d1, d2, d3, d4, d5, d6, d7, d8, d9, d10]


# In[13]:


patterns = [
“adrs”, “ANXDX”, “audit”, “BIS”, “CAPE42”, “item”,
“CTQ”, “EDEQ”, “SCID”, “IRI”, “JVQ”, “PAAQ”,
“RRS”, “tci”, “TFEQ”, “User code”
]



def process_dataframe(df, patterns, drop_zero_rows=False):
df = df[df.columns[df.columns.str.contains(‘|’.join(patterns))]]
df.loc[:,“User code”]= df.loc[:,“User code”].str.replace(“-C”, “”)
df.loc[:,“User code”]= df.loc[:,“User code”].str.lstrip(“0”)

# Convert all elements to numeric, forcing non-numeric to NaN
df_numeric = df.apply(pd.to_numeric, errors=‘coerce’)

# Replace NaN values with 0
df_numeric.fillna(0, inplace=True)

# Replace empty strings with 0
df_numeric.replace(“”, 0, inplace=True)

# remove the row with all zeros besiders the first column
if drop_zero_rows:
df_numeric = df_numeric.loc[(df_numeric.iloc[:, 1:] != 0).any(axis=1)]
return df_numeric



df_list_processed = [process_dataframe(df, patterns, drop_zero_rows=True) for df in df_list]
# drop rows with NaN
df_list_processed = [df.dropna() for df in df_list_processed]

# merge all df on User code
#merge all df on User code
merged_df = df_list_processed[0]
for i, df in enumerate(df_list_processed, start=2):
merged_df = pd.merge(merged_df, df, on=‘User code’, suffixes=(‘’, f’_df{i}'))


# drop all row of data_df besides the first and the last
data_df = data_df.drop(data_df.columns[1:-1], axis=1)
data_df.rename(columns={‘Unnamed: 0’: ‘User code’}, inplace=True)


# merge the merged_df with data_df
merged_df = pd.merge(data_df, merged_df, on=‘User code’)

# Pre-process dataframe for TensorFlow
df = merged_df.drop(columns=[‘User code’, “FilePath”])
non_binary_columns = [col for col in df.columns if len(df[col].unique()) > 2]
df = pd.get_dummies(df, columns=non_binary_columns, drop_first=True)
df = df.astype(float)
# Split data into train and validation sets
X_train, X_val = train_test_split(df.values, test_size=0.20, random_state=42)

# Define the Autoencoder model
# class Autoencoder(tf.keras.Model):
# def init(self, input_size):
# super().init()
# self.encoder = tf.keras.Sequential([
# tf.keras.layers.Dense(128, activation=“relu”, input_shape=(input_size,)),
# tf.keras.layers.BatchNormalization(),
# tf.keras.layers.Dense(64, activation=“relu”),
# tf.keras.layers.BatchNormalization(),
# tf.keras.layers.Dense(32, activation=“relu”),
# tf.keras.layers.Dense(16, activation=“relu”)
# ])
# self.decoder = tf.keras.Sequential([
# tf.keras.layers.Dense(32, activation=“relu”),
# tf.keras.layers.BatchNormalization(),
# tf.keras.layers.Dense(64, activation=“relu”),
# tf.keras.layers.BatchNormalization(),
# tf.keras.layers.Dense(128, activation=“relu”),
# tf.keras.layers.Dense(input_size, activation=“sigmoid”)
# ])

# def call(self, x):
# encoded = self.encoder(x)
# decoded = self.decoder(encoded)
# return decoded

# Define the CNN Autoencoder model
class Autoencoder(tf.keras.Model):
def init(self, input_size):
super().init()
self.encoder = tf.keras.Sequential([
# self.encoder = tf.keras.Sequential([
tf.keras.layers.Input(shape=(input_size, 1)),
tf.keras.layers.Conv1D(128, kernel_size=3, activation=“gelu”,padding=“same”),
tf.keras.layers.BatchNormalization(),
tf.keras.layers.MaxPooling1D(pool_size=2,padding=“same”),
tf.keras.layers.Conv1D(64, kernel_size=3, activation=“gelu”,padding=“causal”),
tf.keras.layers.BatchNormalization(),
tf.keras.layers.MaxPooling1D(pool_size=2,padding=“same”),
])
self.decoder = tf.keras.Sequential([
tf.keras.layers.UpSampling1D(size=2),
tf.keras.layers.Conv1D(64, kernel_size=3, activation=“gelu” ,padding=“same”),
tf.keras.layers.BatchNormalization(),
tf.keras.layers.UpSampling1D(size=2),
tf.keras.layers.Conv1DTranspose(128, kernel_size=3, activation=“gelu”,padding=“same”),
tf.keras.layers.BatchNormalization(),
tf.keras.layers.Conv1D(1, kernel_size=3, activation=“sigmoid” ,padding=“same”)
])

def call(self, x):
x = tf.expand_dims(x, axis=-1) # Add channel dimension
encoded = self.encoder(x)
decoded = self.decoder(encoded)
decoded = tf.squeeze(decoded, axis=-1) # Remove channel dimension
return decoded

# Compile and train the model
input_size = df.shape[1]
autoencoder = Autoencoder(input_size)
autoencoder.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=0.02), loss=“mse”)

early_stopping = tf.keras.callbacks.EarlyStopping(monitor=“val_loss”, patience=10, restore_best_weights=True)

history = autoencoder.fit(X_train, X_train,
epochs=100,
batch_size=512,
validation_data=(X_val, X_val),
callbacks=[early_stopping])

# Evaluate the model
train_predictions = autoencoder.predict(X_train)
val_predictions = autoencoder.predict(X_val)

train_mse = np.mean(np.square(train_predictions - X_train))
val_mse = np.mean(np.square(val_predictions - X_val))
train_mae = mean_absolute_error(X_train, train_predictions)
val_mae = mean_absolute_error(X_val, val_predictions)

train_explained_var = explained_variance_score(X_train, train_predictions)
val_explained_var = explained_variance_score(X_val, val_predictions)

# print(f"Training MSE: {train_mse:.4f}, Validation MSE: {val_mse:.4f}“)

print(f"Training MSE: {train_mse:.4f}, Validation MSE: {val_mse:.4f}”)
print(f"Training MAE: {train_mae:.4f}, Validation MAE: {val_mae:.4f}“)
print(f"Training Explained Variance: {train_explained_var:.4f}, Validation Explained Variance: {val_explained_var:.4f}”)
# Compute Canonical Correlations
# cca = CCA(n_components=min(X_train.shape[1], train_predictions.shape[1]))
num_components = min(X_train.shape[0], train_predictions.shape[0])
cca = CCA(n_components=num_components)
cca.fit(X_train, train_predictions)
X_train_c, train_predictions_c = cca.transform(X_train, train_predictions)

# Compute the correlation for each pair of canonical variables
cca_correlations = np.corrcoef(X_train_c.T, train_predictions_c.T)[:X_train_c.shape[1], X_train_c.shape[1]:]

print(“Canonical Correlations:”)
for i, corr in enumerate(cca_correlations.diagonal()):
print(f"Canonical correlation {i+1}: {corr:.4f}")


# Plot loss curves for training and validation
plt.plot(history.history[“loss”], label=“Training Loss”)
plt.plot(history.history[“val_loss”], label=“Validation Loss”)
plt.legend()
plt.xlabel(“Epochs”)
plt.ylabel(“Loss”)
plt.title(“Training and Validation Loss”)
plt.show()

Epoch 1/100
Traceback (most recent call last):
File “/zi/home/sajad.rezaei/clip/clip/02_Full_Pipeline/src/auto.py”, line 155, in <module>
history = autoencoder.fit(X_train, X_train,
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File “/zi/home/sajad.rezaei/.conda/envs/CLIP/lib/python3.12/site-packages/keras/src/utils/traceback_utils.py”, line 122, in error_handler
raise e.with_traceback(filtered_tb) from None
File “/zi/home/sajad.rezaei/.conda/envs/CLIP/lib/python3.12/site-packages/keras/src/losses/losses.py”, line 1286, in mean_squared_error
return ops.mean(ops.square(y_true - y_pred), axis=-1)
~^~
ValueError: Dimensions must be equal, but are 1291 and 1292 for ‘{{node compile_loss/mse/sub}} = Sub[T=DT_FLOAT](data_1, autoencoder_1/Squeeze)’ with input shapes: [?,1291], [?,1292].