My goal is to create in Python a time series regression between a set of vessel motion timeseries (heave, heave velocity, surge, surge velocity, sway, sway velocity) and cable tension (output). The data structure is the following, each variable is contained in a .txt file where each row represents a timestep and each column refers to a different simulation (in other words, I have data for all variables timeseries which correspond to the rows and columns of the txt files. Note that columns are delimited by commas. My initial Python script is as follows, although its giving me problems. Please correct or change as deemed necessary: import pandas as pd
import numpy as np
import pickle
from sklearn.preprocessing import MinMaxScaler
from sklearn.model_selection import train_test_split
from tensorflow.keras.models import Sequential, load_model
from tensorflow.keras.layers import LSTM, Dense, Dropout, Masking
from tensorflow.keras.callbacks import EarlyStopping, Callback
from sklearn.metrics import mean_squared_error, mean_absolute_error, r2_score
import matplotlib.pyplot as plt
from tensorflow.keras.preprocessing.sequence import pad_sequences
import time

Set simulation data loading option (only "option1" is available now)
simulation_data = "option1"

Initialize lists to hold features and targets
X_all = []
y_all = []

if simulation_data == "option1":

Load data from individual text files
file_paths = {
'heave': 'heave_data.txt',
'heaveVelocity': 'heaveVelocity_data.txt',
'surge': 'surge_data.txt',
'surgeVelocity': 'surgeVelocity_data.txt',
'sway': 'sway_data.txt',
'swayVelocity': 'swayVelocity_data.txt',
'cable_tension': 'Top_tension_max_data.txt'
}

Load each variable's data from the respective text files data_dict = {} for variable, file_path in file_paths.items(): data_dict[variable] = pd.read_csv(file_path, header=None) # Each column represents a separate simulation num_simulations = data_dict['heave'].shape[1] for i in range(num_simulations): # Extract a single simulation (i-th column from each variable) features = np.column_stack([ data_dict['heave'].iloc[:, i], data_dict['heaveVelocity'].iloc[:, i], data_dict['surge'].iloc[:, i], data_dict['surgeVelocity'].iloc[:, i], data_dict['sway'].iloc[:, i], data_dict['swayVelocity'].iloc[:, i], ]) target = data_dict['cable_tension'].iloc[:, i] # Normalize the features and target using MinMaxScaler scaler_features = MinMaxScaler() scaler_target = MinMaxScaler() # Fit and transform the features and target features_scaled = scaler_features.fit_transform(features) target_scaled = scaler_target.fit_transform(target.values.reshape(-1, 1)) X_all.append(features_scaled) y_all.append(target_scaled)
Zero-padding and Masking
max_length = max([x.shape[0] for x in X_all])

Pad the sequences to ensure uniform length
X_all_padded = pad_sequences(X_all, maxlen=max_length, padding='post', dtype='float32')
y_all_padded = pad_sequences(y_all, maxlen=max_length, padding='post', dtype='float32')

Convert to numpy arrays and reshape for LSTM input
X_all_padded = np.array(X_all_padded)

Reshape y_all_padded to have 3D shape for LSTM (samples, time steps, features)
y_all_padded = np.array(y_all_padded).reshape((len(y_all), max_length, 1))

Split into training and test sets
X_train, X_test, y_train, y_test = train_test_split(X_all_padded, y_all_padded, test_size=0.2, random_state=42)

Custom callback to track epoch time
class TimeHistory(Callback):
def on_train_begin(self, logs=None):
self.times = []

def on_epoch_begin(self, epoch, logs=None): self.epoch_time_start = time.time() def on_epoch_end(self, epoch, logs=None): epoch_time = time.time() - self.epoch_time_start self.times.append(epoch_time) print(f"Epoch {epoch + 1} time: {epoch_time:.2f} seconds")
Initialize the custom time-tracking callback
time_callback = TimeHistory()

Build the LSTM model with Masking
model = Sequential()
model.add(Masking(mask_value=0.0, input_shape=(max_length, X_train.shape[2])))

Adding LSTM layers with Dropout
model.add(LSTM(units=50, return_sequences=True))
model.add(Dropout(0.2))
model.add(LSTM(units=50, return_sequences=False))
model.add(Dropout(0.2))

Dense layer to produce the final output (cable tension)
model.add(Dense(units=1))

Compiling the model with mean_squared_error loss for regression
model.compile(optimizer='adam', loss='mean_squared_error')

Start the timer before model training
start_time = time.time()

EarlyStopping to prevent overfitting
early_stopping = EarlyStopping(monitor='val_loss', patience=10, restore_best_weights=True)

Train the model with the time-tracking callback
history = model.fit(X_train, y_train, epochs=100, batch_size=1, validation_split=0.2,
callbacks=[early_stopping, time_callback], shuffle=False)

End the timer after training completes
end_time = time.time()
total_time = end_time - start_time
print(f"Total Training Time: {total_time:.2f} seconds")

Save the model and scalers
model.save('cable_tension_lstm_model.h5')
print("Model saved to disk as 'cable_tension_lstm_model.h5'")

Save the scalers
with open('scaler_features.pkl', 'wb') as f:
pickle.dump(scaler_features, f)
with open('scaler_target.pkl', 'wb') as f:
pickle.dump(scaler_target, f)
print("Scalers saved to disk as 'scaler_features.pkl' and 'scaler_target.pkl'")

Load the model
loaded_model = load_model('cable_tension_lstm_model.h5')

Load the scalers
with open('scaler_features.pkl', 'rb') as f:
loaded_scaler_features = pickle.load(f)
with open('scaler_target.pkl', 'rb') as f:
loaded_scaler_target = pickle.load(f)

Predicting on the test set using the loaded model and scalers
y_pred_scaled = loaded_model.predict(X_test)

Inverse transform the predictions and actual values to get back to original scale
y_pred = loaded_scaler_target.inverse_transform(y_pred_scaled)
y_test_original = loaded_scaler_target.inverse_transform(y_test.reshape(-1, 1))

Remove zero-padded entries from y_test_original
y_test_original = y_test_original[y_test_original != 0]

Match the length of y_pred with y_test_original
y_pred = y_pred[:len(y_test_original)]

Calculate performance metrics
mse = mean_squared_error(y_test_original, y_pred)
rmse = np.sqrt(mse)
mae = mean_absolute_error(y_test_original, y_pred)
mape = np.mean(np.abs((y_test_original - y_pred) / y_test_original)) * 100
r2 = r2_score(y_test_original, y_pred)

print(f"Mean Squared Error: {mse}")
print(f"Root Mean Squared Error: {rmse}")
print(f"Mean Absolute Error: {mae}")
print(f"Mean Absolute Percentage Error: {mape}")
print(f"R2 Score: {r2}")

Time Series Plot for Visual Comparison
plt.figure(figsize=(10, 6))
plt.plot(y_test_original, label='Actual Cable Tension')
plt.plot(y_pred, label='Predicted Cable Tension')
plt.title('Actual vs Predicted Cable Tension')
plt.xlabel('Samples')
plt.ylabel('Cable Tension (kN)')
plt.legend()