Video Classification networker hanging on loss.backwards()

maupind: <p>Hey all,</p>
<p>Currently using a neural network to perform a binary classification of video data. However, the model seems to hang on loss.backwards(). I expect this could take awhile as I am running on the CPU, but I let it run for 2-3 hours and it was not able to complete one pass.</p>
<p>Below is my model structure</p>
<blockquote>
<p>class VestibularNetwork(nn.Module):<br>
def <strong>init</strong>(self, input_shape: int, hidden_units: int, output_shape: int, num_classes: int, batch_size: int = 1):<br>
super().<strong>init</strong>()<br>
self.video_block = nn.Sequential(<br>
nn.Conv2d(in_channels=input_shape,<br>
out_channels=hidden_units,<br>
kernel_size=3, # how big is the square that’s going over the image?<br>
stride=1, # default<br>
padding=1),# options = “valid” (no padding) or “same” (output has same shape as input) or int for specific number<br>
nn.ReLU(),<br>
nn.Conv2d(in_channels=hidden_units,<br>
out_channels=hidden_units,<br>
kernel_size=3,<br>
stride=1,<br>
padding=1),<br>
nn.ReLU(),<br>
nn.MaxPool2d(kernel_size=2,<br>
stride=2) # default stride value is same as kernel_size<br>
)<br>
<span class="hashtag-raw">#self</span>.label_block = nn.SequenThe batch_size parameter is set to 1 by default, but it’s also used as an argument in the method signature. This could lead to confusion about which value is actually being used for the batch size during the forward pass.tial(<br>
#   nn.Linear(in_features=1, out_features=1),<br>
#   nn.ReLU()<br>
#)</p>
</blockquote>
<pre><code>    self.common_block = nn.Sequential(
        nn.Conv2d(in_channels=hidden_units, out_channels=hidden_units, kernel_size=3, padding=1),
        nn.ReLU(),
        nn.Conv2d(hidden_units, hidden_units, 3, padding=1),
        nn.ReLU(),
        nn.MaxPool2d(2)
    )

    # Define LSTM layer
    self.lstm = nn.LSTM(input_size=hidden_units, hidden_size=hidden_units, batch_first=True)


    self.classifier = nn.Linear(in_features=hidden_units, out_features=1)

def forward(self, video_frames, batch_size):
    # Process video frames
    video_frames = video_frames.float()
    (batch_size, sequence_length, channels, height, width) = video_frames.size()
    video_frames_reshaped = video_frames.view(batch_size*sequence_length, channels, height, width)
    #video_frames = video_frames.permute(2, 0, 1).unsqueeze(0)
    #video_frames = video_frames.repeat(batch_size, 1, 1, 1)
    print("Video size:", video_frames_reshaped.shape)
    x_list = []
    for i in range(batch_size):
        # Process each batch separately
        x_batch = self.video_block(video_frames_reshaped[i * sequence_length:(i + 1) * sequence_length])
        x_batch = self.common_block(x_batch)
        x_batch = torch.mean(x_batch, dim=[2, 3])  # Pooling or flattening operation, adjust as needed
        x_list.append(x_batch)

        # Concatenate the outputs of all batches
    x = torch.cat(x_list, dim=0)
    print(f"after cat: {x.shape}")

    # Reshape for LSTM input
    x = x.view(batch_size, sequence_length, -1)

    # Apply LSTM
    lstm_out, _ = self.lstm(x)

    # Aggregate predictions using the final state of the LSTM
    lstm_out = lstm_out[:, -1, :]

    # Apply classifier
    x = self.classifier(lstm_out)
    print(f"classifier: {x.shape}")

    return x
</code></pre>
<p>Any tips or insight would be great as I am not sure how long it should take or if there is some issue in my underlying code causing inefficiencies.</p>
<p>Below is some additional code for my dataloader, train loop, and Bayesian optimization.</p>
<p><em>Dataloader code</em></p>
<blockquote>
<p>class VideoDataset(Dataset):<br>
def <strong>init</strong>(self, video_dir, video_transform = None, clip_length = 10):<br>
super(VideoDataset).<strong>init</strong>()<br>
self.video_dir = video_dir<br>
self.video_files = [file for file in os.listdir(video_dir) if file.endswith(‘.mp4’)]<br>
self.labels = [self.extract_label(file) for file in self.video_files]<br>
self.outcomes = [label[2] for label in self.labels]<br>
self.label_features = [self.extract_label(file)[:2] for file in self.video_files]  # Ignore outcome label<br>
self.video_transform = video_transform<br>
self.clip_length = clip_length</p>
<pre><code>def __len__(self):
    return len(self.video_files)

def __getitem__(self, index):
    video_file= self.video_files[index]
    label = self.labels[index]
    # Ensuring outcome is a 1D tensor
    outcome = torch.tensor(label[2])
    # Create a new tuple for label without modifying the original label
    label_without_outcome = (torch.tensor(label[0]), torch.tensor(label[1]))

    # Update the variable name to reflect the change
    label = label_without_outcome

    video_path = os.path.join(self.video_dir, video_file)

    # Open video file
    cap = cv2.VideoCapture(video_path)

    # Read frames dynamically
    frames = []
    frame_count = 0
    while True:
        ret, frame = cap.read()
        if not ret:
            break
        frames.append(frame)
        frame_count += 1

        # Check if enough frames are read
        if frame_count &gt;= self.clip_length:
            break

    # Release video capture object
    cap.release()

    # Apply transformations if specified
    if self.video_transform:
        frames = [self.video_transform(frame) for frame in frames]

    # Convert frames to tensor
    frames = torch.stack(frames)

    return frames, outcome

def extract_label(self, filename):
    # Example: Extract labels using regular expression
    match = re.search(r'clip\d+_(\w+)_(\w+)_(\w+)', filename)
    if match:
        eye_side = 1 if match.group(1) == 'L' else 0
        test_pos = 1 if match.group(2) == 'LHPD' else 0
        outcome = 0 if match.group(3) == 'N' else 1
        return (eye_side, test_pos, outcome)
    else:
        return (-1, -1, -1)  # Default labels if not found



def adjust_clip_length(self, frames):
    # Adjust clip length by selecting a subset of frames
    if len(frames) &gt; self.clip_length:
        start_idx = (len(frames) - self.clip_length) // 2
        frames = frames[start_idx:start_idx + self.clip_length]
    return frames
</code></pre>
<p>def create_dataloaders(<br>
data_dir: str,<br>
transform: transforms.Compose,<br>
test_size: int,<br>
batch_size: int,<br>
num_workers: int=NUM_WORKERS,<br>
random_seed: int = 33<br>
):</p>
</blockquote>
<p>Here is the train loop</p>
<blockquote>
<h1>Utilise device agnostic code</h1>
<p>device = torch.device(‘cuda’ if torch.cuda.is_available() else ‘cpu’)</p>
<p>botorch_model = BotorchModel(acquisition_function_type=‘qExpectedImprovement’)</p>
<p>init_notebook_plotting()</p>
<h1>Set up BotorchModel for Bayesian optimization</h1>
<p>botorch_model = BotorchModel(acquisition_function_type=‘qExpectedImprovement’)</p>
<p>ax_client = AxClient()<br>
ax_client.create_experiment(<br>
name=“test_visualizations”,<br>
parameters = [<br>
{<br>
“name”: “batch_size”,<br>
“type”: “range”,<br>
“bounds”: [16, 128]<br>
},<br>
{<br>
“name”: “max_epochs”,<br>
“type”: “range”,<br>
“bounds”: [10, 100]<br>
},<br>
{<br>
“name”: “learning_rate”,<br>
“type”: “range”,<br>
“bounds”: [1e-4, 1e-1],<br>
“log_scale”: True<br>
},<br>
{<br>
“name”: “weight_decay”,<br>
“type”: “range”,<br>
“bounds”: [1e-6, 1e-2],<br>
“log_scale”: True<br>
},<br>
],<br>
objective_name= “auc_score_fold”,<br>
minimize= False,<br>
)</p>
<p>for i in range(5):<br>
parameters, trial_index = ax_client.get_next_trial()<br>
ax_client.complete_trial(trial_index=trial_index, raw_data=evaluate(parameters=parameters,<br>
model=vest_model,<br>
dataloader=train_dataloader,<br>
loss_fn=torch.nn.BCEWithLogitsLoss(),<br>
optimizer=optim.Adam(vest_model.parameters()),<br>
device=device))<br>
cv = cross_validate(model=ax_client.generation_strategy.model, folds=-1)<br>
cv.evaluate(parameters=parameters)</p>
<p>model = ax_client.generation_strategy.model<br>
render(interact_contour(model=model, metric_name=“auc_score_fold”))</p>
<p>render(ax_client.get_optimization_trace())</p>
</blockquote>
<p>And lastly, the bayesian optimisation</p>
<blockquote>
<h1>Define the evaluation function for Bayesian optimization</h1>
<p>def evaluate(parameters: str,<br>
model: torch.nn.Module,<br>
dataloader: torch.utils.data.DataLoader,<br>
loss_fn: torch.nn.Module,<br>
optimizer: torch.optim.Optimizer,<br>
device: torch.device) → Tuple[float, float]:<br>
auc_values = <span class="chcklst-box fa fa-square-o fa-fw"></span><br>
loss_values = <span class="chcklst-box fa fa-square-o fa-fw"></span><br>
X_train_list = <span class="chcklst-box fa fa-square-o fa-fw"></span><br>
y_train_list = <span class="chcklst-box fa fa-square-o fa-fw"></span></p>
<pre><code>max_norm = 1.0

model = model.to(device)
model.train()

# Your existing code for setting up the optimizer

for batch_idx, (video_frames_batch, outcomes) in enumerate(dataloader):
    X_train_tensor = torch.tensor(video_frames_batch, dtype=torch.float).to(device)
    y_train_tensor = outcomes.float().unsqueeze(1).to(device)
    if torch.any(torch.isnan(X_train_tensor)) | (torch.any(torch.isnan(y_train_tensor))):
        print("nan values found")

    print(f"show the train tensor {X_train_tensor}")

    # Perform forward pass
    outputs = model(X_train_tensor, batch_size = 1)
    print(f"show the outputs {outputs}")
    #print(f"show the train tensor {X_train_tensor}")
    #print(f"show the true outcomes {y_train_tensor}")
    # Compute loss
    print("Output shape:", outputs.shape)
    #print("Target shape:", y_train_tensor.shape)
    loss = loss_fn(outputs, y_train_tensor)

    print("calculate loss")
    # Perform backward pass and optimization
    optimizer.zero_grad()
    print("zero grad")
    print(f"show the loss {loss}")
    print(f"clip grads")
    torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm)
    print(f"start backwards")
    loss.backward()
    print("backward loss")
    optimizer.step()
    print("optimiser")
    loss_values.append(loss.item())
    # Compute ROC AUC score
    auc_value = roc_auc_score(y_train_tensor.cpu().numpy(), outputs.detach().cpu().numpy())
    auc_values.append(auc_value)
    print("auc complete")


# Compute average loss
avg_loss = sum(loss_values) / len(loss_values)

# Compute average ROC AUC score
avg_auc = sum(auc_values) / len(auc_values)

return avg_loss, avg_auc
</code></pre>
</blockquote>
<p>Thank you and please let me know if there’s anything else needed.</p>

maupind: <p>A little update, for some reason it is now running through multiple passes, but still hangs after a few passes at the same point, loss.backwards()</p>