Cifar10 resnet how to increase accuracy?

isaac_Lin: <p>import torch</p>
<p>from PIL import Image</p>
<p>from torch import nn, save, load</p>
<p>from torch.utils.data import DataLoader</p>
<p>import torchvision</p>
<p>from torchvision import datasets</p>
<p>from torchvision.transforms import ToTensor</p>
<p>import torchvision.transforms as transforms</p>
<p>from torchvision import models</p>
<p>if torch.cuda.is_available():<br>
print(“using CUDA”)<br>
device = ‘cuda’<br>
dtype = torch.float32<br>
elif torch.backends.mps.is_available():<br>
print(“using MPS”)<br>
device = ‘mps’<br>
dtype = torch.float32<br>
else:<br>
print(“using CPU”)<br>
device = ‘cpu’<br>
dtype = torch.float32</p>
<p>transform = transforms.Compose([<br>
transforms.Resize(32),<br>
transforms.ToTensor(),<br>
transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])</p>
<p>])</p>
<p>trainset = torchvision.datasets.CIFAR10(root=‘./data’,train=True, download=True, transform = transform)<br>
trainloader = torch.utils.data.DataLoader(trainset, batch_size=32, shuffle=True, num_workers=2)</p>
<p>testset = torchvision.datasets.CIFAR10(root=‘./data’,train=False, download=True, transform = transform)<br>
testloader = torch.utils.data.DataLoader(testset, batch_size=32, shuffle=False, num_workers=2)</p>
<p>net = models.resnet50(pretrained=True)<br>
num_ftrs = net.fc.in_features<br>
net.fc = nn.Sequential(<br>
nn.Linear(num_ftrs, 500),<br>
nn.ReLU(),<br>
nn.Linear(500, 300),<br>
nn.Dropout(0.6),<br>
nn.Linear(300,100),<br>
nn.ReLU(),<br>
nn.Dropout(0.5),<br>
nn.Linear(100,50),<br>
nn.ReLU(),  # Corrected here<br>
nn.Dropout(0.35),<br>
nn.Linear(50,25),<br>
nn.ReLU(),  # Corrected here<br>
nn.Dropout(0.3),<br>
nn.Linear(25,10),<br>
nn.ReLU()  # Corrected here<br>
)</p>
<p>print(“Model Info:”)</p>
<p>print(“ResNet50,Pretrained,weight adj. LR=0.01,Mom=0.3,WD=0.0001”)</p>
<p>print(“Schedule step=1,gamma=0.7, 20 epoches”)</p>
<h1><a name="move-the-model-to-the-gpu-1" class="anchor" href="#move-the-model-to-the-gpu-1"></a>Move the model to the GPU</h1>
<p>resnet50v2 = net.to(device, dtype=torch.float32)</p>
<p>optimizer = torch.optim.SGD(resnet50v2.parameters(), lr=0.01,momentum=0.3,weight_decay=0.0001)</p>
<p>criterion = nn.CrossEntropyLoss()</p>
<p>scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=5, gamma=0.5)</p>
<p>train_losses = <span class="chcklst-box fa fa-square-o fa-fw"></span></p>
<p>test_losses = <span class="chcklst-box fa fa-square-o fa-fw"></span></p>
<p>accuracies = <span class="chcklst-box fa fa-square-o fa-fw"></span></p>
<p>train_acc = <span class="chcklst-box fa fa-square-o fa-fw"></span></p>
<p>import matplotlib.pyplot as plt<br>
from tqdm import tqdm</p>
<p>for epoch in range(32):  # loop over the dataset multiple times<br>
running_loss = 0.0<br>
correctTrain = 0<br>
totalTrain = 0<br>
pbar = tqdm(enumerate(trainloader, 0), total=len(trainloader), desc=“Epoch {}”.format(epoch+1))<br>
for i, data in pbar:<br>
# get the inputs; data is a list of [inputs, labels]<br>
inputs, labels = data[0].to(device,dtype=torch.float32), data[1].to(device)</p>
<pre><code>    # zero the parameter gradients
    optimizer.zero_grad()

    # forward + backward + optimize
    outputs = resnet50v2(inputs)
    loss = criterion(outputs, labels)
    loss.backward()
    optimizer.step()

    # print statistics
    running_loss += loss.item()
    _, predicted_train = torch.max(outputs.data, 1)
    totalTrain += labels.size(0)
    correctTrain += (predicted_train == labels).sum().item()
    pbar.set_postfix({'loss': running_loss/(i+1)})
train_accuracy = 100 * correctTrain / totalTrain
train_acc.append(train_accuracy)
print(f'Epoch {epoch + 1} loss: {running_loss / len(trainloader):.3f}')

# Start of testing phase
resnet50v2.eval()  # Set the model to evaluation mode
test_loss = 0.0
correct = 0
total = 0
with torch.no_grad():
    for data in testloader:
        images, labels = data[0].to(device,dtype=torch.float32), data[1].to(device)
        outputs = resnet50v2(images)
        loss = criterion(outputs, labels)
        test_loss += loss.item()
        _, predicted = torch.max(outputs.data, 1)
        total += labels.size(0)
        correct += (predicted == labels).sum().item()

print(f'Epoch {epoch + 1} Test loss: {test_loss / len(testloader):.3f}, Accuracy: {100 * correct / total:.2f}%')
#print the learning rate
print(f'Epoch {epoch + 1} Learning rate: {optimizer.param_groups[0]["lr"]}')
train_losses.append(running_loss / len(trainloader))
test_losses.append(test_loss / len(testloader))
accuracies.append(100 * correct / total)

resnet50v2.train() # Set the model back to training model
scheduler.step()
</code></pre>
<p>print(‘Finished Training’)</p>
<p>plt.figure(figsize=(10, 5))<br>
plt.plot(train_losses, label=‘Training Loss’)<br>
plt.plot(test_losses, label=‘Test Loss’)<br>
plt.xlabel(‘Epochs’)<br>
plt.ylabel(‘Loss’)<br>
plt.legend()<br>
plt.show()</p>
<p>plt.figure(figsize=(10, 5))<br>
plt.plot(accuracies, label=‘Accuracy’)<br>
plt.plot(train_acc, label=‘Training Accuracy’)<br>
plt.xlabel(‘Epochs’)<br>
plt.ylabel(‘Accuracy (%)’)<br>
plt.legend()<br>
plt.show()</p>
<p>print(“Model Info:”)<br>
print(“ResNet50,Pretrained,weight adj. LR=0.01,Mom=0.3,WD=0.0001”)<br>
print(“Schedule step=1,gamma=0.7, 20 epoches”)</p>

MyMonkeyFriend: <p>This is a the Resnet Model you can try to fine tune and edit with <img src="https://discuss.pytorch.org/images/emoji/apple/smiley.png?v=12" title=":smiley:" class="emoji" alt=":smiley:" loading="lazy" width="20" height="20"></p>
<pre><code class="lang-auto">import torch
import torch.nn as nn

class BasicBlock(nn.Module):
    expansion = 1

    def __init__(self, in_channels, out_channels, stride=1, downsample=None):
        super(BasicBlock, self).__init__()
        self.conv1 = nn.Conv2d(in_channels, out_channels, kernel_size=3, stride=stride, padding=1, bias=False)
        self.bn1 = nn.BatchNorm2d(out_channels)

        self.relu = nn.ReLU(inplace=True)

        self.conv2 = nn.Conv2d(out_channels, out_channels, kernel_size=3, stride=1, padding=1, bias=False)
        self.bn2 = nn.BatchNorm2d(out_channels)

        self.downsample = downsample
        self.stride = stride

    def forward(self, x):
        identity = x

        out = self.conv1(x)
        out = self.bn1(out)
        out = self.relu(out)

        out = self.conv2(out)
        out = self.bn2(out)
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            
        if self.downsample is not None:
            identity = self.downsample(x)

        out += identity
        out = self.relu(out)

        return out
    

class Bottleneck(nn.Module):
    expansion = 4

    def __init__(self, in_channels, out_channels, stride=1, downsample=None):
        super(Bottleneck, self).__init__()
        self.conv1 = nn.Conv2d(in_channels, out_channels, kernel_size=1, bias=False)
        self.bn1 = nn.BatchNorm2d(out_channels)

        self.conv2 = nn.Conv2d(out_channels, out_channels, kernel_size=3, stride=stride, padding=1, bias=False)
        self.bn2 = nn.BatchNorm2d(out_channels)

        self.conv3 = nn.Conv2d(out_channels, out_channels * self.expansion, kernel_size=1, bias=False)
        self.bn3 = nn.BatchNorm2d(out_channels * self.expansion)

        self.relu = nn.ReLU(inplace=True)
        self.downsample = downsample
        self.stride = stride

    def forward(self, x):
        identity = x

        out = self.conv1(x)
        out = self.bn1(out)
        out = self.relu(out)

        out = self.conv2(out)
        out = self.bn2(out)
        out = self.relu(out)

        out = self.conv3(out)
        out = self.bn3(out)

        if self.downsample is not None:
            identity = self.downsample(x)

        out += identity
        out = self.relu(out)

        return out



class ResNet(nn.Module):
    def __init__(self, block, layers, num_classes=1000):
        super(ResNet, self).__init__()
        self.in_channels = 64
        self.conv1 = nn.Conv2d(3, self.in_channels, kernel_size=7, stride=2, padding=3, bias=False)
        self.bn1 = nn.BatchNorm2d(self.in_channels)
        self.relu = nn.ReLU(inplace=True)
        self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)


        self.layer1 = self._make_layer(block, 64, layers[0])
        self.layer2 = self._make_layer(block, 128, layers[1], stride=2)
        self.layer3 = self._make_layer(block, 256, layers[2], stride=2)
        self.layer4 = self._make_layer(block, 512, layers[3], stride=2)
        
        self.avgpool = nn.AdaptiveAvgPool2d((1, 1))
        self.fc = nn.Linear(512 * block.expansion, num_classes)

    def _make_layer(self, block, out_channels, blocks, stride=1):
        downsample = None

        if stride != 1 or self.in_channels != out_channels * block.expansion:
            downsample = nn.Sequential(
                nn.Conv2d(self.in_channels, out_channels * block.expansion, kernel_size=1, stride=stride, bias=False),
                nn.BatchNorm2d(out_channels * block.expansion),
            )

        layers = []
        layers.append(block(self.in_channels, out_channels, stride, downsample))
        self.in_channels = out_channels * block.expansion
        for _ in range(1, blocks):
            layers.append(block(self.in_channels, out_channels))

        return nn.Sequential(*layers)

    def forward(self, x):
        x = self.conv1(x)
        x = self.bn1(x)
        x = self.relu(x)
        
        x = self.maxpool(x)

        x = self.layer1(x)
        x = self.layer2(x)
        x = self.layer3(x)
        x = self.layer4(x)

        x = self.avgpool(x)
        x = torch.flatten(x, 1)
        x = self.fc(x)

        return x
</code></pre>

milktea_islife: <p>Hi isaac, below is the model and code that I used I achieved 86% ACC, hope this helps!</p>
<pre><code class="lang-auto">import torch
import torchvision
import torchvision.transforms as transforms
from tqdm import tqdm
from torch import nn
import matplotlib.pyplot as plt

#include to ensure fallback to other accerleration methods
if torch.cuda.is_available():
    print("using CUDA")
    device = 'cuda'
    dtype = torch.float32
elif torch.backends.mps.is_available():
    print("using MPS")
    device = 'mps'
    dtype = torch.float32
else:
    print("using CPU")
    device = 'cpu'
    dtype = torch.float32
</code></pre>
<pre><code class="lang-auto">transformRes = transforms.Compose([
    transforms.ToTensor(),
    transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
])

trainsetRes = torchvision.datasets.CIFAR10(root='./data', train=True, download=True, transform=transformRes)
trainloaderRes16 = torch.utils.data.DataLoader(trainsetRes, batch_size=16, shuffle=True, num_workers=10)

testsetRes = torchvision.datasets.CIFAR10(root='./data', train=False, download=True, transform=transformRes)
testloaderRes16 = torch.utils.data.DataLoader(testsetRes, batch_size=16, shuffle=False, num_workers=10)
</code></pre>
<pre><code class="lang-auto">import torchvision.models as models

# Load the pretrained model from pytorch
resnet50v2 = models.resnet50(pretrained=True)

# Freeze the parameters of the model
for param in resnet50v2.parameters():
    param.requires_grad = True


# Change the final layer to match the number of classes in the CIFAR-10 dataset
num_ftrs = resnet50v2.fc.in_features
resnet50v2.fc = nn.Sequential(
    nn.Linear(num_ftrs, 500),
    nn.ReLU(),
    nn.Linear(500, 200),
    nn.Dropout(0.5),
    nn.Linear(200,40),
    nn.ReLU(),
    nn.Dropout(0.3),
    nn.Linear(40,10),
    nn.ReLU()

)
print("Model Info:")
print("ResNet50,Pretrained,weight adj. LR=0.01,Mom=0.3,WD=0.0001")
print("Schedule step=1,gamma=0.7, 20 epoches")

# Move the model to the GPU
resnet50v2 = resnet50v2.to(device, dtype=torch.float32)


optimizer = torch.optim.SGD(resnet50v2.parameters(), lr=0.01,momentum=0.3,weight_decay=0.0001)
criterion = nn.CrossEntropyLoss()
scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=5, gamma=0.5)
train_losses = []
test_losses = []
accuracies = []
train_acc = []

for epoch in range(20):  # loop over the dataset multiple times
    running_loss = 0.0
    correctTrain = 0
    totalTrain = 0
    pbar = tqdm(enumerate(trainloaderRes16, 0), total=len(trainloaderRes16), desc="Epoch {}".format(epoch+1))
    for i, data in pbar:
        # get the inputs; data is a list of [inputs, labels]
        inputs, labels = data[0].to(device,dtype=torch.float32), data[1].to(device)

        # zero the parameter gradients
        optimizer.zero_grad()

        # forward + backward + optimize
        outputs = resnet50v2(inputs)
        loss = criterion(outputs, labels)
        loss.backward()
        optimizer.step()

        # print statistics
        running_loss += loss.item()
        _, predicted_train = torch.max(outputs.data, 1)
        totalTrain += labels.size(0)
        correctTrain += (predicted_train == labels).sum().item()
        pbar.set_postfix({'loss': running_loss/(i+1)})
    train_accuracy = 100 * correctTrain / totalTrain
    train_acc.append(train_accuracy)
    print(f'Epoch {epoch + 1} loss: {running_loss / len(trainloaderRes16):.3f}')

    # Start of testing phase
    resnet50v2.eval()  # Set the model to evaluation mode
    test_loss = 0.0
    correct = 0
    total = 0
    with torch.no_grad():
        for data in testloaderRes16:
            images, labels = data[0].to(device,dtype=torch.float32), data[1].to(device)
            outputs = resnet50v2(images)
            loss = criterion(outputs, labels)
            test_loss += loss.item()
            _, predicted = torch.max(outputs.data, 1)
            total += labels.size(0)
            correct += (predicted == labels).sum().item()

    print(f'Epoch {epoch + 1} Test loss: {test_loss / len(testloaderRes16):.3f}, Accuracy: {100 * correct / total:.2f}%')
    #print the learning rate
    print(f'Epoch {epoch + 1} Learning rate: {optimizer.param_groups[0]["lr"]}')
    train_losses.append(running_loss / len(trainloaderRes16))
    test_losses.append(test_loss / len(testloaderRes16))
    accuracies.append(100 * correct / total)

    resnet50v2.train() # Set the model back to training model
    scheduler.step()



print('Finished Training')

plt.figure(figsize=(10, 5))
plt.plot(train_losses, label='Training Loss')
plt.plot(test_losses, label='Test Loss')
plt.xlabel('Epochs')
plt.ylabel('Loss')
plt.legend()
plt.show()

plt.figure(figsize=(10, 5))
plt.plot(accuracies, label='Accuracy')
plt.plot(train_acc, label='Training Accuracy')
plt.xlabel('Epochs')
plt.ylabel('Accuracy (%)')
plt.legend()
plt.show()

</code></pre>

isaac_Lin: <p>Hi , thanks for the help! How about this?<br>
import pandas as pd<br>
import numpy as np<br>
import torch<br>
from torch.nn.utils.rnn import pad_sequence<br>
from torchtext.vocab import build_vocab_from_iterator<br>
data = pd.read_csv(‘emotion_sentimen_dataset.csv’)</p>
<p>import torch<br>
if torch.backends.mps.is_available():<br>
mps_device = torch.device(“mps”)<br>
x = torch.ones(1, device=mps_device)<br>
print (x)<br>
else:<br>
print (“MPS device not found.”)</p>
<p>data = data.drop(‘Unnamed: 0’, axis=1)</p>
<p>data.head()</p>
<h2><a name="get-unique-values-of-emotion-1" class="anchor" href="#get-unique-values-of-emotion-1"></a>get unique values of emotion</h2>
<p>data[‘Emotion’].unique()</p>
<p>sentences = data[‘text’].tolist()</p>
<p>emotion_mapping = {emotion: i for i, emotion in enumerate(data[‘Emotion’].unique())}<br>
data[‘Emotions’] = data[‘Emotion’].map(emotion_mapping)<br>
labels = data[‘Emotions’].tolist()</p>
<h1><a name="tokenization-and-vocabulary-building-2" class="anchor" href="#tokenization-and-vocabulary-building-2"></a>Tokenization and vocabulary building</h1>
<p>tokenize = lambda x: x.split()<br>
def yield_tokens(data):<br>
for text in data:<br>
yield tokenize(text)</p>
<p>vocab = build_vocab_from_iterator(yield_tokens(sentences), specials=[“”, “”])<br>
vocab.set_default_index(vocab[“”])</p>
<h1><a name="tokenizing-and-creating-padded-sequences-3" class="anchor" href="#tokenizing-and-creating-padded-sequences-3"></a>Tokenizing and creating padded sequences</h1>
<p>text_pipeline = lambda x: [vocab[token] for token in tokenize(x)]<br>
vectorized_seqs = [torch.tensor(text_pipeline(sentence)) for sentence in sentences]<br>
padded_sequences = pad_sequence(vectorized_seqs, batch_first=True, padding_value=vocab[‘’])<br>
labels_tensor = torch.tensor(labels, dtype=torch.long)</p>
<p>from sklearn.model_selection import train_test_split</p>
<h1><a name="split-data-into-training-and-validation-sets-4" class="anchor" href="#split-data-into-training-and-validation-sets-4"></a>Split data into training and validation sets</h1>
<p>import torch.nn as nn<br>
import torch.nn.functional as F</p>
<p>class TextCNN(nn.Module):<br>
def <strong>init</strong>(self, vocab_size, embed_dim, num_filters, filter_sizes, num_classes):<br>
super(TextCNN, self).<strong>init</strong>()<br>
self.embedding = nn.Embedding(vocab_size, embed_dim)<br>
self.convs = nn.ModuleList(<br>
[nn.Conv1d(in_channels=embed_dim, out_channels=num_filters, kernel_size=fs) for fs in filter_sizes]<br>
)<br>
self.dropout = nn.Dropout(0.5)<br>
self.pool = nn.MaxPool1d(kernel_size=2)  # Max pooling layer<br>
self.fc = nn.Linear(num_filters * len(filter_sizes) // 2, num_classes)  # Adjusted for max pooling</p>
<pre><code>def forward(self, x):
    x = self.embedding(x)
    x = x.permute(0, 2, 1)
    x = [F.relu(conv(x)).max(dim=2)[0] for conv in self.convs]
    x = torch.cat(x, 1)
    x = self.pool(x)  # Apply max pooling
    x = self.dropout(x)
    x = self.fc(x)
    return F.log_softmax(x, dim=1)  # Using log_softmax for numerical stability
</code></pre>
<h1><a name="hyperparameters-5" class="anchor" href="#hyperparameters-5"></a>Hyperparameters</h1>
<p>vocab_size = len(vocab)<br>
embed_dim = 16<br>
num_filters = 100  # Changed to 100 as per your text<br>
filter_sizes = [3, 4, 5]  # Changed to [3, 4, 5] as per your text<br>
num_classes = 13</p>
<p>model = TextCNN(vocab_size, embed_dim, num_filters, filter_sizes, num_classes).to(mps_device)<br>
print(model)</p>
<h1><a name="define-loss-function-and-optimizer-6" class="anchor" href="#define-loss-function-and-optimizer-6"></a>Define loss function and optimizer</h1>
<p>criterion = nn.NLLLoss()<br>
optimizer = torch.optim.Adam(model.parameters(), lr=0.001)</p>
<p>from torch.utils.data import DataLoader, TensorDataset<br>
from tqdm import tqdm</p>
<p>train_sequences, val_sequences, train_labels, val_labels = train_test_split(padded_sequences, labels_tensor, test_size=0.2, random_state=42)<br>
batch_size = 32</p>
<h1><a name="create-data-loaders-7" class="anchor" href="#create-data-loaders-7"></a>Create data loaders</h1>
<h1><a name="create-data-loaders-8" class="anchor" href="#create-data-loaders-8"></a>Create data loaders</h1>
<p>train_data = TensorDataset(train_sequences.to(mps_device), train_labels.to(mps_device))<br>
train_loader = DataLoader(train_data, batch_size=batch_size, shuffle=True)</p>
<p>val_data = TensorDataset(val_sequences.to(mps_device), val_labels.to(mps_device))<br>
val_loader = DataLoader(val_data, batch_size=batch_size, shuffle=False)</p>
<h1><a name="training-loop-9" class="anchor" href="#training-loop-9"></a>Training loop</h1>
<p>for epoch in range(2):<br>
# Training phase<br>
model.train()<br>
train_loss = 0.0<br>
train_correct = 0<br>
for sequences, labels in train_loader:<br>
optimizer.zero_grad()<br>
outputs = model(sequences).squeeze()<br>
loss = criterion(outputs, labels)<br>
loss.backward()<br>
optimizer.step()<br>
_, predicted = torch.max(outputs, 1)<br>
train_correct += (predicted == labels).sum().item()<br>
train_loss += loss.item()<br>
train_accuracy = train_correct / len(train_loader.dataset)</p>
<h1><a name="validation-phase-10" class="anchor" href="#validation-phase-10"></a>Validation phase</h1>
<p>val_loss = 0.0<br>
val_correct = 0<br>
with torch.no_grad():<br>
for sequences, labels in val_loader:<br>
val_outputs = model(sequences).squeeze()<br>
loss = criterion(val_outputs, labels)<br>
_, val_predicted = torch.max(val_outputs, 1)<br>
val_correct += (val_predicted == labels).sum().item()<br>
val_loss += loss.item()<br>
val_accuracy = val_correct / len(val_loader.dataset)</p>
<p>print(f’Epoch {epoch+1}, Loss: {train_loss / len(train_loader)}, Accuracy: {train_accuracy}, Val Loss: {val_loss / len(val_loader)}, Val Accuracy: {val_accuracy}')</p>
<h1><a name="prediction-with-class-output-11" class="anchor" href="#prediction-with-class-output-11"></a>Prediction with class output</h1>
<p>test_sentences = [“Why did you have to do this you incompetent little shit!”, “I love you so much that i want to love you even more”, “Dont make me worried so much please”, “What a relief”, “Lets do something fun and exciting!”]<br>
test_seqs = [torch.tensor(text_pipeline(sentence)) for sentence in test_sentences]<br>
test_padded = pad_sequence(test_seqs, batch_first=True, padding_value=vocab[‘’]).to(mps_device)</p>
<p>with torch.no_grad():<br>
predictions = model(test_padded)<br>
class_predictions = torch.argmax(predictions, dim=1)</p>
<pre><code>print(class_predictions)  # Outputs emotions directly
</code></pre>
<p>{‘hate’: 0, ‘neutral’: 1, ‘anger’: 2, ‘love’: 3, ‘worry’: 4, ‘relief’: 5, ‘happiness’: 6, ‘fun’: 7, ‘empty’: 8, ‘enthusiasm’: 9, ‘sadness’: 10, ‘surprise’: 11, ‘boredom’: 12}</p>