PyTorch備忘録1

# Pytorch学習(1)
以下のyoutubeチャンネルで学習した内容をノートにまとめたものです。
https://www.youtube.com/playlist?list=PLqnslRFeH2UrcDBWF5mfPGpqQDSta6VK4

## * PyTorchのモデル学習pipeline
1. モデルのデザイン (input, output, size, forward pass)
2. 損失関数と最適化の構築
3. 学習ループ
      1. forward pass : compute prediction
      2. backward pass : gradients
      3. update weight

## * PyTorchコード例1
~~~
# 線形回帰モデル例
import torch
import torch.nn as nn

X = torch.tensor([[1], [2], [3], [4]], dtype=torch.float)
Y = torch.tensor([2, 4, 6, 8], dtype=torch.float)

x_test = torch.tensor([5], dtype=torch.float)
n_samples, n_features = X.shape

input_size = n_features
output_size = n_features

# モデル定義
model = nn.Linear(input_size, output_size)

# Training
learning_rate = 0.01
n_iters = 100

print('before pred: ' ,model(x_test).item())
# -> before pred:  1.0004366636276245

# 損失関数定義
loss = nn.MSELoss()
# 最適化関数定義
optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate)

# 訓練ループ
for epoch in range(n_iters):
  y_pred = model(X)
  l = loss(Y, y_pred)
  l.backward()

  # update weights
  optimizer.step()

  # zero gradients
  optimizer.zero_grad()

  # モデル評価
  y_pred = model(X)
  print(y_pred)
  ~~~

## PyTorchコード例2：ロジスティック回帰
### 0. 学習データの用意と正規化
~~~
import torch
import torch.nn as nn
import numpy as np
from sklearn import datasets
from sklearn. preprocessing import StandardScaler
from sklearn.model_selection import train_test_split

# 0) prepare data
bc = datasets.load_breast_cancer()
X, y = bc.data, bc.target
n_samples, n_features = X.shape

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)

# Scaling
sc = StandardScaler()
X_train = sc.fit_transform(X_train)
X_test = sc.transform(X_test)

X_train = torch.from_numpy(X_train.astype(np.float32))
X_test = torch.from_numpy(X_test.astype(np.float32))
y_train = torch.from_numpy(y_train.astype(np.float32))
y_test = torch.from_numpy(y_test.astype(np.float32))

y_train = y_train.view(y_train.shape[0], 1)
y_test = y_test.view(y_test.shape[0], 1)
~~~
* 正解データの次元確認(2次元に変換必要)
~~~
y_train.view(y_train.shape[0], 1)[:10]
'''
tensor([[1.],
        [0.],
        [1.],
        [1.],
        [0.],
        [1.],
        [1.],
        [1.],
        [1.],
        [1.]])
'''
~~~

### 1. モデル構築
~~~
# Linear model f = wx + b , sigmoid at the end
class Model(nn.Module):
    def __init__(self, n_input_features):
        super(Model, self).__init__()
        self.linear = nn.Linear(n_input_features, 1)

    def forward(self, x):
        y_pred = torch.sigmoid(self.linear(x))
        return y_pred

model = Model(n_features)
~~~

### 2. 損失関数と最適化の設定
~~~
num_epochs = 100
learning_rate = 0.01
criterion = nn.BCELoss()
optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate)
~~~

### 3. 学習ループ
~~~
for epoch in range(num_epochs):
    # Forward pass and loss
    y_pred = model(X_train)
    # loss = criterion(y_pred, y_train)
    loss = criterion(y_pred.squeeze(), y_train)

    # Backward pass and update
    loss.backward()
    optimizer.step()

    # zero grad before new step
    optimizer.zero_grad()

    if (epoch+1) % 10 == 0:
        print(f'epoch: {epoch+1}, loss = {loss.item():.4f}')
~~~
* 学習実行結果
~~~
epoch: 10, loss = 0.4717
epoch: 20, loss = 0.4052
epoch: 30, loss = 0.3598
epoch: 40, loss = 0.3268
epoch: 50, loss = 0.3016
epoch: 60, loss = 0.2818
epoch: 70, loss = 0.2657
epoch: 80, loss = 0.2524
epoch: 90, loss = 0.2411
epoch: 100, loss = 0.2315
~~~

### 4. 精度確認
~~~
with torch.no_grad():
    y_predicted = model(X_test)
    y_predicted_cls = y_predicted.round()
    acc = y_predicted_cls.eq(y_test).sum() / float(y_test.shape[0])
    print(f'accuracy: {acc.item():.4f}')

'''
精度
accuracy: 59.9474
'''
~~~

## DatasetとDataloader
* レコード１つ１つで勾配を求めるととても時間がかかるので、データをいくつかに分けて（バッチ）、バッチごとに（バッチサイズ）まとめて勾配を求めることで時間短縮を図るための方法

## 用語
* epoch : 1 forward and backward pass of ALL training samples
* bach_size : number of training samples in one forward & backward pass
* number of iterations : number of passes, each pass using [batch_size] number of samples
  ex) 100 samples, batch_size=20 -> 100/20=5 iterations for 1epoch

## DatasetとDataloaderを使ったコード例
~~~
import torch
import torchvision
from torch.utils.data import Dataset, DataLoader
import numpy as np
import math

class WineDataset(Dataset):

    def __init__(self):
        # Initialize data, download, etc.
        # read with numpy or pandas
        xy = np.loadtxt('./data/wine/wine.csv', delimiter=',', dtype=np.float32, skiprows=1)
        self.n_samples = xy.shape[0]

        # here the first column is the class label, the rest are the features
        self.x_data = torch.from_numpy(xy[:, 1:]) # size [n_samples, n_features]
        self.y_data = torch.from_numpy(xy[:, [0]]) # size [n_samples, 1]

    # support indexing such that dataset[i] can be used to get i-th sample
    def __getitem__(self, index):
        return self.x_data[index], self.y_data[index]

    # we can call len(dataset) to return the size
    def __len__(self):
        return self.n_samples

# create dataset
dataset = WineDataset()


# get first sample and unpack
first_data = dataset[0]
features, labels = first_data
print(features, labels)


# Load whole dataset with DataLoader
# shuffle: shuffle data, good for training
# num_workers: faster loading with multiple subprocesses
# !!! IF YOU GET AN ERROR DURING LOADING, SET num_workers TO 0 !!!
train_loader = DataLoader(dataset=dataset,
                          batch_size=4,
                          shuffle=True,
                          num_workers=2)


# convert to an iterator and look at one random sample
dataiter = iter(train_loader)
data = next(dataiter)
features, labels = data
print(features, labels)


# Dummy Training loop
num_epochs = 2
total_samples = len(dataset)
n_iterations = math.ceil(total_samples/4)
print(total_samples, n_iterations)
for epoch in range(num_epochs):
    for i, (inputs, labels) in enumerate(train_loader):

        # here: 178 samples, batch_size = 4, n_iters=178/4=44.5 -> 45 iterations
        # Run your training process
        if (i+1) % 5 == 0:
            print(f'Epoch: {epoch+1}/{num_epochs}, Step {i+1}/{n_iterations}| Inputs {inputs.shape} | Labels {labels.shape}')

~~~

## Softmax と Cross Entropy
* Softmax：0 or 1 で出力
~~~
S(y_i) = e^y_i / Σ(e^y_i) # 0~1で出力
~~~
* Cross Entropy：マルチラベルでの出力値、各ラベルの発生確率を出力
~~~
D(^y , Y = -1/NΣ(Y_i * log(^y_i))) # マルチ分類問題での出力値
#  Y=[1, 0, 0], ^y=[0.7, 0.2, 0.1] -> D(^y, Y)= 0.35
#  Y=[1, 0, 0], ^y=[0.1, 0.3, 0.6] -> D(^y, Y)= 2.30 -> 数字が小さいほどYと^yの差が小さい
~~~

* pytorchのCrossEntropyLoss()クラス利用時の注意点
    * 内部でsoftmaxが適用されるので、事前にsoftmaxを行う必要はない
    * Yのラベルはone-hot変換不要。実際のラベルを入れるだけ

### マルチ分類のコード例
~~~
# Multiclass problem

class NuralNet2(nn.Module):
  def __init__(self, input_size, hidden_size, num_classes):
    super(NuralNet2, self).__init__()
    self.linear1 = nn.Linear(input_size, hidden_size)
    self.relu = nn.ReLU()
    self.linear2 == nn.Linear(hidden_size, num_classes) # num_classes:出力の数。３クラスなら３つ 例:[2.0(クラス1の値), 0.5(クラス2の値), 5.2(クラス3の値)]

  def forward(self, x):
    out = self.linear1(x)
    out = self.relu(out)
    out = self.linear2(out)
    # no softmax at the end
    return out

model = NuralNet2(input_size = 28 * 28, hidden_size=5, num_classes=3)

criterion = nn.CrossEntropyLoss() # apply Softmax （softmax関数により、各クラスの出力値を0~1の確率値に変換）
~~~

### 2値分類のコード例
~~~
# Binary classification

class NuralNet1(nn.Module):
  def __init__(self, input_size, hidden_size):
    super(NeuralNet1, self).__init__()
    self.linear1 = nn.Linear(input_size, hidden_size)
    self.relu = nn.RELU()
    self.linear2 = nn.Linear(hidden_size, 1) # この1は出力値が1個のみで確率値になる

  def forwad(self, x):
    out = self.linear1(x)
    out = self.relu(out)
    out = self.linear2(out)
    # sigmoid at the end
    y_pred = torch.sigmoid(out)
    return y_pred

model = NeuralNet1(input_size=28*28, hidden_size=5)
criterion = nn.BCELoss() # バイナリークロスエントロピーロス
~~~

## 非線形変換用関数（値を0~1範囲の値に変換する関数）の種類
1. Step function
2. Sigmoid
3. TanH
4. ReLU
5. Leaky ReLU
6. Softmax

## 転移学習（Transfer Learning）
* 例えば、犬猫分類用のモデルの最後の層だけ変更して、虫鳥分類用モデルに使うといったモデル
* これにより、モデル作りの時間短縮が図れる
* 転移学習コード例
    * https://github.com/patrickloeber/pytorchTutorial/blob/master/15_transfer_learning.py