ディープラーニング入門からトランスフォーマー応用






第1部：ディープラーニング入門（基礎概念とニューラルネット）

単元	内容
1.1 ニューラルネットワークとは	人工ニューロン、活性化関数（ReLU, Sigmoid）
1.2 順伝播と逆伝播	重みの更新、損失関数（クロスエントロピー）
1.3 学習アルゴリズム	勾配降下法（SGD, Adam）
1.4 過学習と汎化	ドロップアウト・正則化の導入
1.5 Pythonで実装	NumPyで簡単な2層NNを構築


```python
#  修正済み・合体済み：NumPyによる2層ニューラルネットワーク（分類・ReLU/Dropout/L2正則化）

import numpy as np
import matplotlib.pyplot as plt
from sklearn.datasets import make_classification
from sklearn.preprocessing import OneHotEncoder
from sklearn.model_selection import train_test_split

# 活性化関数 / Activation functions
def relu(x):
    return np.maximum(0, x)

def relu_deriv(x):
    return (x > 0).astype(float)

def sigmoid(x):
    return 1 / (1 + np.exp(-x))

def sigmoid_deriv(x):
    s = sigmoid(x)
    return s * (1 - s)

# ソフトマックス / Softmax
def softmax(x):
    e_x = np.exp(x - np.max(x, axis=1, keepdims=True))
    return e_x / np.sum(e_x, axis=1, keepdims=True)

# クロスエントロピー損失 / Cross-entropy loss
def cross_entropy(y_pred, y_true):
    m = y_true.shape[0]
    return -np.sum(y_true * np.log(y_pred + 1e-9)) / m

# Dropout適用 / Apply dropout
def apply_dropout(a1, dropout_rate):
    mask = (np.random.rand(*a1.shape) > dropout_rate).astype(float)
    return a1 * mask, mask

# 重み初期化 / Initialize weights
def initialize(input_dim, hidden_dim, output_dim):
    np.random.seed(0)
    W1 = np.random.randn(input_dim, hidden_dim) * 0.01
    b1 = np.zeros((1, hidden_dim))
    W2 = np.random.randn(hidden_dim, output_dim) * 0.01
    b2 = np.zeros((1, output_dim))
    return W1, b1, W2, b2

# 順伝播 / Forward propagation
def forward(x, W1, b1, W2, b2, activation='relu', dropout_rate=0.0, train=True):
    z1 = x @ W1 + b1
    a1 = relu(z1) if activation == 'relu' else sigmoid(z1)
    if train and dropout_rate > 0:
        a1, dropout_mask = apply_dropout(a1, dropout_rate)
    else:
        dropout_mask = np.ones_like(a1)
    z2 = a1 @ W2 + b2
    y_pred = softmax(z2)
    return z1, a1, dropout_mask, z2, y_pred

# 逆伝播 / Backpropagation
def backward(x, y_true, z1, a1, dropout_mask, y_pred, W1, b1, W2, b2, lr, activation='relu', lambda_l2=0.0):
    m = x.shape[0]
    dz2 = (y_pred - y_true) / m
    dW2 = a1.T @ dz2 + lambda_l2 * W2
    db2 = np.sum(dz2, axis=0, keepdims=True)

    da1 = dz2 @ W2.T
    dz1 = da1 * relu_deriv(z1) if activation == 'relu' else da1 * sigmoid_deriv(z1)
    dz1 *= dropout_mask
    dW1 = x.T @ dz1 + lambda_l2 * W1
    db1 = np.sum(dz1, axis=0, keepdims=True)

    W1 -= lr * dW1
    b1 -= lr * db1
    W2 -= lr * dW2
    b2 -= lr * db2
    return W1, b1, W2, b2

# 学習関数 / Training function
def train_model(X, y, hidden_dim=10, epochs=500, lr=0.1, activation='relu', dropout_rate=0.2, lambda_l2=0.01):
    input_dim, output_dim = X.shape[1], y.shape[1]
    W1, b1, W2, b2 = initialize(input_dim, hidden_dim, output_dim)
    loss_history = []

    for epoch in range(epochs):
        z1, a1, dropout_mask, z2, y_pred = forward(X, W1, b1, W2, b2, activation, dropout_rate)
        loss = cross_entropy(y_pred, y)
        loss_history.append(loss)
        W1, b1, W2, b2 = backward(X, y, z1, a1, dropout_mask, y_pred, W1, b1, W2, b2, lr, activation, lambda_l2)
        if epoch % 100 == 0:
            print(f"Epoch {epoch}, Loss: {loss:.4f}")
    return W1, b1, W2, b2, loss_history

# データ生成と前処理 / Generate and preprocess data
X, y_labels = make_classification(n_samples=300, n_features=2, n_classes=3, n_informative=2,
                                  n_redundant=0, n_clusters_per_class=1, random_state=42)

try:
    encoder = OneHotEncoder(sparse_output=False)
except TypeError:
    encoder = OneHotEncoder(sparse=False)

y = encoder.fit_transform(y_labels.reshape(-1, 1))

# モデル訓練 / Train the model
W1, b1, W2, b2, loss_history = train_model(X, y, hidden_dim=10, epochs=500, lr=0.1,
                                           activation='relu', dropout_rate=0.2, lambda_l2=0.01)

# 損失可視化 / Plot training loss
plt.plot(loss_history)
plt.xlabel("Epoch")
plt.ylabel("Cross-Entropy Loss")
plt.title("Training Loss Curve")
plt.grid(True)
plt.tight_layout()
plt.show()


```




⸻

第2部：MNIST手書き数字認識タスク（CNNと分類）

単元	内容
2.1 MNISTとは	28×28ピクセルのグレースケール画像（数字0〜9）
2.2 畳み込みニューラルネット（CNN）	Convolution → Pooling → Flatten → Dense
2.3 モデル構成	Conv2D, MaxPool2D, Dense, Softmax
2.4 損失関数と評価	CrossEntropyLoss・Accuracyでモデル評価
2.5 実装演習（PyTorch/TensorFlow）	CNNによる分類モデルを実装・訓練


```python
# 第2部：PyTorchによるMNIST分類CNNモデル実装（2.1〜2.5合体）

import torch
import torch.nn as nn
import torch.optim as optim
import torchvision
import torchvision.transforms as transforms
import matplotlib.pyplot as plt

# 2.1 データ準備：MNISTロード（28x28ピクセルのグレースケール画像、0〜9の数字）
transform = transforms.Compose([transforms.ToTensor()])
train_set = torchvision.datasets.MNIST(root='./data', train=True, download=True, transform=transform)
test_set = torchvision.datasets.MNIST(root='./data', train=False, download=True, transform=transform)
train_loader = torch.utils.data.DataLoader(train_set, batch_size=64, shuffle=True)
test_loader = torch.utils.data.DataLoader(test_set, batch_size=64, shuffle=False)

# 2.2〜2.3 CNNモデル構成：Conv2D → ReLU → MaxPool → Flatten → Dense → Softmax
class CNNModel(nn.Module):
    def __init__(self):
        super(CNNModel, self).__init__()
        self.conv1 = nn.Conv2d(1, 16, kernel_size=3, padding=1)
        self.pool = nn.MaxPool2d(2, 2)
        self.conv2 = nn.Conv2d(16, 32, kernel_size=3, padding=1)
        self.fc1 = nn.Linear(32 * 7 * 7, 128)
        self.fc2 = nn.Linear(128, 10)
        self.relu = nn.ReLU()
        
    def forward(self, x):
        x = self.pool(self.relu(self.conv1(x)))  # 28x28 → 14x14
        x = self.pool(self.relu(self.conv2(x)))  # 14x14 → 7x7
        x = x.view(-1, 32 * 7 * 7)  # Flatten
        x = self.relu(self.fc1(x))
        x = self.fc2(x)  # SoftmaxはCrossEntropyLossに含まれる
        return x

# モデル初期化
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
model = CNNModel().to(device)

# 2.4 損失関数と最適化手法
criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(model.parameters(), lr=0.001)

# 2.5 訓練ループ
def train_model(model, train_loader, criterion, optimizer, epochs=5):
    loss_history = []
    for epoch in range(epochs):
        model.train()
        running_loss = 0.0
        for images, labels in train_loader:
            images, labels = images.to(device), labels.to(device)
            optimizer.zero_grad()
            outputs = model(images)
            loss = criterion(outputs, labels)
            loss.backward()
            optimizer.step()
            running_loss += loss.item()
        avg_loss = running_loss / len(train_loader)
        loss_history.append(avg_loss)
        print(f"Epoch {epoch+1}, Loss: {avg_loss:.4f}")
    return loss_history

loss_history = train_model(model, train_loader, criterion, optimizer)

# 評価（Accuracy）
def evaluate_model(model, test_loader):
    model.eval()
    correct, total = 0, 0
    with torch.no_grad():
        for images, labels in test_loader:
            images, labels = images.to(device), labels.to(device)
            outputs = model(images)
            _, predicted = torch.max(outputs, 1)
            total += labels.size(0)
            correct += (predicted == labels).sum().item()
    accuracy = correct / total
    print(f"Test Accuracy: {accuracy:.4f}")
    return accuracy

accuracy = evaluate_model(model, test_loader)

# 学習曲線描画
plt.plot(loss_history)
plt.xlabel("Epoch")
plt.ylabel("Training Loss")
plt.title("CNN Training Loss on MNIST")
plt.grid(True)
plt.tight_layout()
plt.show()


```
⸻

第3部：トランスフォーマーモデル基礎（視覚的応用を含む）

単元	内容
3.1 トランスフォーマーの誕生と意義	RNNとの違い、並列処理、BERT・GPTの基盤
3.2 自己注意機構（Self-Attention）	Query, Key, Value、スケーリング付き内積
3.3 マルチヘッドアテンション	複数の視点で意味をとらえる仕組み
3.4 ポジショナルエンコーディング	順序情報の導入（正弦・余弦関数）
3.5 Vision Transformer（ViT）	MNIST画像をパッチに分割してTransformerへ入力
3.6 簡易Transformer実装	PyTorchで小型ViTをMNISTに適用して分類タスク実行



```python
# 第3部：簡易Vision Transformer（ViT）によるMNIST分類（3.1〜3.6）

import torch
import torch.nn as nn
import torch.optim as optim
import torchvision
import torchvision.transforms as transforms
from torch.utils.data import DataLoader
import matplotlib.pyplot as plt

# デバイス設定 / Device
device = torch.device("cpu")

# 3.1 MNISTデータセットのロード（28x28グレースケール画像）
transform = transforms.Compose([
    transforms.ToTensor()
])
train_set = torchvision.datasets.MNIST(root='./data', train=True, download=True, transform=transform)
test_set = torchvision.datasets.MNIST(root='./data', train=False, download=True, transform=transform)
train_loader = DataLoader(train_set, batch_size=64, shuffle=True)
test_loader = DataLoader(test_set, batch_size=64, shuffle=False)

# 3.5 Vision Transformer用：画像をパッチに分割し線形変換
class PatchEmbedding(nn.Module):
    def __init__(self, img_size=28, patch_size=7, in_channels=1, embed_dim=64):
        super().__init__()
        self.n_patches = (img_size // patch_size) ** 2
        self.proj = nn.Conv2d(in_channels, embed_dim, kernel_size=patch_size, stride=patch_size)

    def forward(self, x):
        x = self.proj(x)  # [B, embed_dim, H', W']
        x = x.flatten(2)  # [B, embed_dim, N]
        x = x.transpose(1, 2)  # [B, N, embed_dim]
        return x

# 3.2〜3.4 Transformer Encoder（Self-Attention, Positional Encoding含む）
class SimpleTransformer(nn.Module):
    def __init__(self, embed_dim=64, num_heads=4, ff_dim=128, dropout=0.1):
        super().__init__()
        self.attn = nn.MultiheadAttention(embed_dim, num_heads, dropout=dropout, batch_first=True)
        self.norm1 = nn.LayerNorm(embed_dim)
        self.ff = nn.Sequential(
            nn.Linear(embed_dim, ff_dim),
            nn.ReLU(),
            nn.Linear(ff_dim, embed_dim)
        )
        self.norm2 = nn.LayerNorm(embed_dim)

    def forward(self, x):
        attn_output, _ = self.attn(x, x, x)
        x = self.norm1(x + attn_output)
        x = self.norm2(x + self.ff(x))
        return x

class PositionalEncoding(nn.Module):
    def __init__(self, dim, max_len=100):
        super().__init__()
        pe = torch.zeros(max_len, dim)
        pos = torch.arange(0, max_len).unsqueeze(1)
        div_term = torch.exp(torch.arange(0, dim, 2) * -(torch.log(torch.tensor(10000.0)) / dim))
        pe[:, 0::2] = torch.sin(pos * div_term)
        pe[:, 1::2] = torch.cos(pos * div_term)
        self.pe = pe.unsqueeze(0)

    def forward(self, x):
        return x + self.pe[:, :x.size(1), :].to(x.device)

# 3.6 ViTモデル本体（パッチ分割 → 埋め込み → Transformer → 分類）
class ViT(nn.Module):
    def __init__(self, img_size=28, patch_size=7, in_channels=1, num_classes=10, embed_dim=64, num_heads=4):
        super().__init__()
        self.patch_embed = PatchEmbedding(img_size, patch_size, in_channels, embed_dim)
        self.pos_embed = PositionalEncoding(embed_dim)
        self.transformer = SimpleTransformer(embed_dim, num_heads)
        self.classifier = nn.Linear(embed_dim, num_classes)

    def forward(self, x):
        x = self.patch_embed(x)           # [B, N, D]
        x = self.pos_embed(x)             # [B, N, D]
        x = self.transformer(x)           # [B, N, D]
        x = x.mean(dim=1)                 # Global average pooling
        return self.classifier(x)         # [B, num_classes]

# モデル初期化
model = ViT().to(device)
criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(model.parameters(), lr=0.001)

# 学習ループ
def train_vit(model, loader, criterion, optimizer, epochs=5):
    model.train()
    loss_list = []
    for epoch in range(epochs):
        total_loss = 0
        for images, labels in loader:
            images, labels = images.to(device), labels.to(device)
            optimizer.zero_grad()
            outputs = model(images)
            loss = criterion(outputs, labels)
            loss.backward()
            optimizer.step()
            total_loss += loss.item()
        avg_loss = total_loss / len(loader)
        loss_list.append(avg_loss)
        print(f"Epoch {epoch+1}, Loss: {avg_loss:.4f}")
    return loss_list

losses = train_vit(model, train_loader, criterion, optimizer)

# 評価（分類精度）
def evaluate(model, loader):
    model.eval()
    correct = total = 0
    with torch.no_grad():
        for images, labels in loader:
            images, labels = images.to(device), labels.to(device)
            outputs = model(images)
            preds = torch.argmax(outputs, dim=1)
            correct += (preds == labels).sum().item()
            total += labels.size(0)
    acc = correct / total
    print(f"Test Accuracy: {acc:.4f}")
    return acc

acc = evaluate(model, test_loader)

# 学習曲線プロット
plt.plot(losses)
plt.xlabel("Epoch")
plt.ylabel("Loss")
plt.title("Vision Transformer Training Loss")
plt.grid(True)
plt.tight_layout()
plt.show()


```
```Python
import numpy as np
import matplotlib.pyplot as plt

# -------------------- パラメータ設定 --------------------
d_model = 64           # エンコーディングの次元数
max_len = 100          # 最大系列長
position = np.arange(max_len)[:, np.newaxis]         # (max_len, 1)
div_term = np.exp(np.arange(0, d_model, 2) * (-np.log(10000.0) / d_model))  # (d_model/2,)

# -------------------- Positional Encoding --------------------
pe = np.zeros((max_len, d_model))
pe[:, 0::2] = np.sin(position * div_term)
pe[:, 1::2] = np.cos(position * div_term)

# -------------------- プロット --------------------
plt.figure(figsize=(12, 6))
for i in range(8):  # 上位8次元だけ描画
    plt.plot(pe[:, i], label=f"dim {i}")
plt.title("Positional Encoding (Sinusoidal)")
plt.xlabel("Position")
plt.ylabel("Encoding Value")
plt.legend()
plt.grid(True)
plt.tight_layout()
plt.show()

```


```Python
import numpy as np
import matplotlib.pyplot as plt

# -------------------- ヘルパー関数：スケールドドット積アテンション --------------------
def scaled_dot_product_attention(Q, K, V):
    d_k = Q.shape[-1]
    scores = np.dot(Q, K.T) / np.sqrt(d_k)
    weights = softmax(scores)
    return np.dot(weights, V), weights

def softmax(x):
    e_x = np.exp(x - np.max(x, axis=-1, keepdims=True))
    return e_x / np.sum(e_x, axis=-1, keepdims=True)

# -------------------- パラメータ設定 --------------------
np.random.seed(42)
seq_len = 5      # 入力系列長
d_model = 8      # 入力ベクトルの次元
num_heads = 2    # ヘッド数（マルチヘッド）

# -------------------- 入力ベクトル生成 --------------------
X = np.random.rand(seq_len, d_model)

# -------------------- ヘッドごとの線形変換行列 --------------------
def init_projection(heads, d_model, d_head):
    return [np.random.rand(d_model, d_head) for _ in range(heads)]

d_head = d_model // num_heads
W_Q = init_projection(num_heads, d_model, d_head)
W_K = init_projection(num_heads, d_model, d_head)
W_V = init_projection(num_heads, d_model, d_head)

# -------------------- 各ヘッドでアテンション計算 --------------------
head_outputs = []
attention_weights = []

for i in range(num_heads):
    Q = X @ W_Q[i]
    K = X @ W_K[i]
    V = X @ W_V[i]
    head_output, attn = scaled_dot_product_attention(Q, K, V)
    head_outputs.append(head_output)
    attention_weights.append(attn)

# -------------------- 出力の統合（Concat） --------------------
multi_head_output = np.concatenate(head_outputs, axis=-1)

# -------------------- アテンション重み可視化 --------------------
fig, axes = plt.subplots(1, num_heads, figsize=(12, 4))
for i in range(num_heads):
    im = axes[i].imshow(attention_weights[i], cmap='viridis')
    axes[i].set_title(f"Head {i+1} Attention")
    axes[i].set_xlabel("Key Positions")
    axes[i].set_ylabel("Query Positions")
    fig.colorbar(im, ax=axes[i])
plt.tight_layout()
plt.show()

```



```Python
# Program Name: self_attention_demo.py
# Creation Date: 20250606
# Overview: Minimal implementation of Scaled Dot-Product Self-Attention
# Usage: Run in Python environment with numpy and matplotlib installed

# -------------------- Install Required Libraries --------------------
!pip install numpy matplotlib --quiet

# -------------------- Import Libraries --------------------
import numpy as np
import matplotlib.pyplot as plt

# -------------------- Parameters --------------------
np.random.seed(42)
seq_len = 4      # トークン数（例：文の長さ） / Number of tokens
d_k = 4          # クエリ・キーの次元 / Dimension of Q, K
d_v = 4          # バリューの次元 / Dimension of V

# -------------------- Input: Randomly Initialize Q, K, V --------------------
Q = np.random.rand(seq_len, d_k)  # Query
K = np.random.rand(seq_len, d_k)  # Key
V = np.random.rand(seq_len, d_v)  # Value

# -------------------- Step 1: Compute Scaled Dot-Product --------------------
scores = np.dot(Q, K.T) / np.sqrt(d_k)  # QK^T / sqrt(d_k)

# -------------------- Step 2: Softmax over scores --------------------
def softmax(x):
    e_x = np.exp(x - np.max(x, axis=-1, keepdims=True))
    return e_x / e_x.sum(axis=-1, keepdims=True)

attention_weights = softmax(scores)

# -------------------- Step 3: Apply Attention Weights to V --------------------
output = np.dot(attention_weights, V)

# -------------------- Step 4: Visualize Attention Weights --------------------
fig, ax = plt.subplots()
im = ax.imshow(attention_weights, cmap='Blues')

ax.set_xticks(np.arange(seq_len))
ax.set_yticks(np.arange(seq_len))
ax.set_xticklabels([f"K{i}" for i in range(seq_len)])
ax.set_yticklabels([f"Q{i}" for i in range(seq_len)])
plt.title("Self-Attention Weights (Scaled Dot Product)")
plt.colorbar(im)
plt.show()

```