I will give you a code , you have to change one structure ....
in the code there was written vgg16 , you have use lstm insteed of vgg16 and all others things will be same
here is the code - 
" 
here is the code modify it - 
"#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Mon Mar 16 15:16:08 2020

@author: kalpita
"""

#matplotlib inline
import torch
import numpy as np
import torch.nn as nn
import torch.nn.functional as F
import matplotlib.pyplot as plt
from torchvision import transforms, datasets
import torchvision.datasets as datasets
import torchvision.models as models

import os

My_dir = os.listdir('/media/nibaran/New Volume/Jaya/deep_learning_LineData')
print(My_dir)
print(len(My_dir))
num_epochs = len(My_dir)
for epoch in range(num_epochs):
  print(My_dir[epoch])
  Ptrain = os.path.join('/media/nibaran/New Volume/Jaya/deep_learning_LineData' + '/' + My_dir[epoch] + '/' + 'train'+ '/')
  print(Ptrain)

  PATH = os.path.join('/media/nibaran/New Volume/Jaya/line_result/Train_WTTest' + My_dir[epoch] +'.pth' )

  Ptest = os.path.join('/media/nibaran/New Volume/Jaya/deep_learning_LineData' + '/' + My_dir[epoch] + '/' + 'test' + '/')
  print(Ptest)

  PATH1 = os.path.join('/media/nibaran/New Volume/Jaya/line_result/ModelTest' + My_dir[epoch] +'.pth' )
# Load data
  apply_transform = transforms.Compose([transforms.RandomResizedCrop(224),transforms.RandomHorizontalFlip(), transforms.ToTensor()])
  BatchSize = 1 # change according to system specs

#trainset = datasets.MNIST(root='./MNIST', train=True, download=True, transform=apply_transform)
#train_path='/media/nibaran/New Volume/Jaya/nEW_rUN/bangla_data_deep/86/train/' 
  trainset=datasets.ImageFolder(Ptrain, transform=apply_transform)
  trainLoader = torch.utils.data.DataLoader(trainset, batch_size=BatchSize,
                                          shuffle=True, num_workers=1) # Creating dataloader

# Validation set with random rotations in the range [-90,90]
#testset = datasets.MNIST(root='./MNIST', train=False, download=True, transform=apply_transform)
#test_path='/media/nibaran/New Volume/Jaya/nEW_rUN/bangla_data_deep/86/test/'
  testset=datasets.ImageFolder(Ptest, transform=apply_transform)
  testLoader = torch.utils.data.DataLoader(testset, batch_size=BatchSize,
                                         shuffle=False, num_workers=1) # Creating dataloader

# Size of train and test datasets
  print('No. of samples in train set: '+str(len(trainLoader.dataset)))
  print('No. of samples in test set: '+str(len(testLoader.dataset)))
  Notrain = len(trainLoader.dataset)
  Notest = len(testLoader.dataset)
#Define model architecture

  use_gpu = torch.cuda.is_available()
  net = models.vgg16(pretrained=True)
  net.classifier._modules['6'] = nn.Linear(4096, 2)
  print(net)
  if use_gpu:
     print('GPU is avaialble!')
     net = net.cuda()
    
# train model
  criterion = nn.CrossEntropyLoss() 
  learning_rate = 0.0001
  num_epochs = 100

  train_loss = []
  train_acc = []
  for epoch in range(num_epochs):
    
     running_loss = 0.0 
     running_corr = 0
        
     for i,data in enumerate(trainLoader):
         inputs,labels = data
         if use_gpu:
             inputs, labels = inputs.cuda(),labels.cuda() 
        # Initializing model gradients to zero
         net.zero_grad() 
        # Data feed-forward through the network
         outputs = net(inputs)
        # Predicted class is the one with maximum probability
         preds = torch.argmax(outputs,dim=1)
        # Finding the loss
         loss = criterion(outputs, labels)
        # Accumulating the loss for each batch
         running_loss += loss 
        # Accumulate number of correct predictions
         running_corr += torch.sum(preds==labels)    
        
     totalLoss = running_loss/(i+1)
    # Calculating gradients
     totalLoss.backward()
    # Updating the model parameters
     for f in net.parameters():
         f.data.sub_(f.grad.data * learning_rate)
        
     epoch_loss = running_loss.item()/(i+1)   #Total loss for one epoch
     epoch_acc = running_corr.item()/Notrain
    
    
         
     train_loss.append(epoch_loss) #Saving the loss over epochs for plotting the graph
     train_acc.append(epoch_acc) #Saving the accuracy over epochs for plotting the graph
       
        
     print('Epoch {:.0f}/{:.0f} : Training loss: {:.4f} | Training Accuracy: {:.4f}'.format(epoch+1,num_epochs,epoch_loss,epoch_acc*100))
  fig = plt.figure(figsize=[15,5]) 
  plt.subplot(121)
  plt.plot(range(num_epochs),train_loss,'r-',label='Loss') 
  plt.legend(loc='upper right')
  plt.xlabel('Epochs')
  plt.ylabel('Training')
  plt.subplot(122)
  plt.plot(range(num_epochs),train_acc,'g-',label='Accuracy') 
  plt.legend(loc='upper right')
  plt.xlabel('Epochs')
  plt.ylabel('Training')

#Evaluation of trained model
  correct_pred = 0
  for data in testLoader:
     inputs,labels = data
     if use_gpu:
         inputs, labels = inputs.cuda(),labels.cuda()
    # Feedforward train data batch through model
     output = net(inputs) 
    # Predicted class is the one with maximum probability
     preds = torch.argmax(output,dim=1)
     correct_pred += torch.sum(preds==labels)

  test_accuracy = correct_pred.item()/Notest
  print('Testing accuracy = ',test_accuracy*100)

# Save Model
#PATH = os.path.join('/media/nibaran/New Volume/Jaya/nEW_rUN/Results/Resut_test' + )
#PATH = '/media/nibaran/New Volume/Jaya/nEW_rUN/Results/bangla_data_deepR/Train_WTTest_86w.pt'
#PATH1 = '/media/nibaran/New Volume/Jaya/nEW_rUN/Results/bangla_data_deepR/ModelTest_86w.pt'
  torch.save(net.state_dict(), PATH)
  torch.save(net, PATH1)
"
"