TransformerのためのPython数学

https://qiita.com/birdwatcher/items/b3e4428f63f708db37b7

1. **Transformer** - 注意機構に基づく深層学習モデルの代表構造
2. **Self-Attention** - 入力系列全体から特徴を抽出する注意機構
3. **Multi-Head Attention** - 複数の注意機構を並列処理して多様な特徴を学習
4. **Query (Q)** - 検索ベクトル
5. **Key (K)** - 検索対象ベクトル
6. **Value (V)** - 取得する情報ベクトル
7. **Scaled Dot-Product Attention** - スケーリング付き内積注意計算
8. **Softmax** - 重みを確率的に正規化する関数
9. **Positional Encoding** - 位置情報を付与するためのベクトル変換
10. **Residual Connection** - 残差接続、勾配消失の防止
11. **Layer Normalization** - 層ごとの正規化処理
12. **Feed Forward Network (FFN)** - 全結合層による非線形変換
13. **Dropout** - 過学習防止のためのランダム無効化
14. **Encoder** - 入力系列を処理して特徴ベクトルに変換するモジュール
15. **Decoder** - 出力系列を生成するモジュール
16. **Masked Self-Attention** - 未来情報を遮断する注意機構（言語モデル用）
17. **Cross-Attention** - 異なる系列間の注意機構
18. **Attention Map** - 注意の重み分布
19. **Tokenization** - テキストをトークン（単語・サブワード）に分割
20. **Embedding** - トークンをベクトル化

```Python
# -*- coding: utf-8 -*-
# Transformer基本用語解説付きPython実装サンプル
# Comments are written in Japanese and English for educational purposes

import torch
import torch.nn.functional as F
import numpy as np

# ============================================
# 1. Transformer 構造の中核 / Core of Transformer Architecture
# ============================================

# Scaled Dot-Product Attention
def scaled_dot_product_attention(Q, K, V):
    """
    スケールド・ドットプロダクト・アテンションの実装
    Implementation of Scaled Dot-Product Attention
    """
    # QとKの内積を計算し、次元数の平方根でスケーリング / Scale dot product by sqrt of dimension
    d_k = Q.size(-1)
    scores = torch.matmul(Q, K.transpose(-2, -1)) / np.sqrt(d_k)
    print("Attention Scores (Before Softmax):\n", scores)

    # ソフトマックス正規化 / Softmax normalization
    weights = F.softmax(scores, dim=-1)
    print("Attention Weights (After Softmax):\n", weights)

    # 重み付き合計 / Weighted sum with V
    output = torch.matmul(weights, V)
    return output, weights

# ============================================
# 2. Positional Encoding / 位置情報付与
# ============================================
def positional_encoding(seq_len, d_model):
    """
    Positional Encoding to inject sequence order information
    シーケンスの位置情報をベクトルに追加するエンコーディング
    """
    pos = torch.arange(seq_len).unsqueeze(1)
    i = torch.arange(0, d_model, 2)
    angle_rates = 1 / torch.pow(10000, (i.float() / d_model))
    angle_rads = pos * angle_rates

    # 偶数次元にsin、奇数次元にcosを適用 / Apply sin to even and cos to odd dimensions
    pos_encoding = torch.zeros(seq_len, d_model)
    pos_encoding[:, 0::2] = torch.sin(angle_rads)
    pos_encoding[:, 1::2] = torch.cos(angle_rads)

    print("Positional Encoding:\n", pos_encoding)
    return pos_encoding

# ============================================
# 3. デモ実行 / Demonstration
# ============================================

# シーケンス長と次元設定 / Define sequence length and model dimension
seq_len, d_model = 4, 8

# サンプルQ, K, Vベクトル / Sample Q, K, V vectors
Q = torch.rand(1, seq_len, d_model)
K = torch.rand(1, seq_len, d_model)
V = torch.rand(1, seq_len, d_model)

# 位置情報を加算 / Add positional encoding
pos_enc = positional_encoding(seq_len, d_model)
Q += pos_enc.unsqueeze(0)
K += pos_enc.unsqueeze(0)
V += pos_enc.unsqueeze(0)

# アテンション計算 / Compute attention
attn_output, attn_weights = scaled_dot_product_attention(Q, K, V)

# 出力表示 / Display output and weights
print("Attention Output:\n", attn_output)
print("Attention Weights:\n", attn_weights)

```


```Python
# -*- coding: utf-8 -*-
# Q, K, Vの意味と役割をPython実演しながら解説
import torch
import torch.nn.functional as F

# Q, K, V それぞれの役割説明用関数
def explain_qkv(Q, K, V):
    print("=== Query (Q): 検索したい内容を表すベクトル ===")
    print(Q)
    print("\n=== Key (K): 検索対象（インデックス・ラベル）を表すベクトル ===")
    print(K)
    print("\n=== Value (V): 取得したい実データや意味情報を表すベクトル ===")
    print(V)
    print("\n=== Q と K の類似度を計算し、重み付けして V から情報を取り出します ===\n")

    # 類似度計算 + スケーリング
    d_k = Q.size(-1)
    scores = torch.matmul(Q, K.transpose(-2, -1)) / d_k**0.5
    print("スコア行列 (Q x K^T / sqrt(d_k)):\n", scores)

    # ソフトマックスで重み付け
    weights = F.softmax(scores, dim=-1)
    print("\nソフトマックス重み:\n", weights)

    # 重み付き合計でVから情報抽出
    result = torch.matmul(weights, V)
    print("\n重み付き合計結果 (Attention Output):\n", result)

# デモデータ定義（シーケンス長2、特徴次元4）
Q = torch.tensor([[[1.0, 0.0, 1.0, 0.0],
                   [0.0, 1.0, 0.0, 1.0]]])  # 検索クエリ / Search Intent

K = torch.tensor([[[1.0, 0.0, 1.0, 0.0],
                   [0.0, 1.0, 0.0, 1.0]]])  # 検索対象 / Search Index

V = torch.tensor([[[10.0, 20.0, 30.0, 40.0],
                   [50.0, 60.0, 70.0, 80.0]]])  # 取得する情報 / Information to Return

# Q, K, Vの説明と実演
explain_qkv(Q, K, V)

```



```Python
# Transformer構成要素 11～20 実装付き Python 解説サンプル

import torch
import torch.nn as nn
import torch.nn.functional as F

# 11. Layer Normalization / 層ごとの正規化処理
layer_norm = nn.LayerNorm(4)
x = torch.randn(2, 4)
x_norm = layer_norm(x)
print("Layer Normalization Output:\n", x_norm)

# 12. Feed Forward Network (FFN) / 全結合層による非線形変換
class FeedForward(nn.Module):
    def __init__(self, d_model, d_ff):
        super().__init__()
        self.ffn = nn.Sequential(
            nn.Linear(d_model, d_ff),
            nn.ReLU(),
            nn.Linear(d_ff, d_model)
        )

    def forward(self, x):
        return self.ffn(x)

ffn = FeedForward(4, 16)
x_ffn = ffn(x_norm)
print("Feed Forward Network Output:\n", x_ffn)

# 13. Dropout / ランダム無効化
dropout_layer = nn.Dropout(p=0.5)
x_dropout = dropout_layer(x_ffn)
print("After Dropout:\n", x_dropout)

# 14. Encoder Module
class Encoder(nn.Module):
    def __init__(self, d_model, d_ff):
        super().__init__()
        self.ffn = FeedForward(d_model, d_ff)
        self.norm = nn.LayerNorm(d_model)

    def forward(self, x):
        x = self.norm(x + self.ffn(x))
        return x

encoder = Encoder(4, 16)
x_enc = encoder(x)
print("Encoder Output:\n", x_enc)

# 15. Decoder Module with Cross-Attention placeholder
class Decoder(nn.Module):
    def __init__(self, d_model, d_ff):
        super().__init__()
        self.ffn = FeedForward(d_model, d_ff)
        self.norm = nn.LayerNorm(d_model)

    def forward(self, x, encoder_output):
        # 通常はCross-Attentionがここに入る / Cross-Attention would be here
        x = self.norm(x + self.ffn(x))
        return x

decoder = Decoder(4, 16)
x_dec = decoder(x, x_enc)
print("Decoder Output:\n", x_dec)

# 16. Masked Self-Attention Example (causal mask)
def causal_mask(seq_len):
    return torch.triu(torch.ones(seq_len, seq_len), diagonal=1).bool()

mask = causal_mask(4)
print("Causal Mask:\n", mask)

# 17. Cross-Attention Placeholder (identity function for demonstration)
def cross_attention(Q, K, V):
    return Q  # 本来は Q, K, V からAttentionを計算

# 18. Attention Map Example
attention_map = torch.rand(4, 4)  # ダミーAttentionマップ / Dummy attention map
print("Attention Map:\n", attention_map)

# 19. Tokenization Example (Simple whitespace split)
sentence = "hello world transformer"
tokens = sentence.split()
print("Tokens:\n", tokens)

# 20. Embedding Example
vocab_size = 10
embedding_dim = 4
embedding = nn.Embedding(vocab_size, embedding_dim)
token_indices = torch.tensor([1, 2, 3])
embedded_tokens = embedding(token_indices)
print("Embedded Tokens:\n", embedded_tokens)

```



```Python
import torch
import torch.nn as nn
import torch.nn.functional as F
import numpy as np

# Scaled Dot-Product Attention
def scaled_dot_product_attention(Q, K, V):
    d_k = Q.size(-1)
    scores = torch.matmul(Q, K.transpose(-2, -1)) / np.sqrt(d_k)
    weights = F.softmax(scores, dim=-1)
    return torch.matmul(weights, V), weights

# Multi-Head Attention
class MultiHeadAttention(nn.Module):
    def __init__(self, d_model, num_heads):
        super().__init__()
        assert d_model % num_heads == 0
        self.num_heads = num_heads
        self.depth = d_model // num_heads
        self.Wq = nn.Linear(d_model, d_model)
        self.Wk = nn.Linear(d_model, d_model)
        self.Wv = nn.Linear(d_model, d_model)
        self.out = nn.Linear(d_model, d_model)

    def split_heads(self, x):
        batch_size, seq_len, d_model = x.size()
        x = x.view(batch_size, seq_len, self.num_heads, self.depth)
        return x.transpose(1, 2)

    def forward(self, x):
        Q = self.split_heads(self.Wq(x))
        K = self.split_heads(self.Wk(x))
        V = self.split_heads(self.Wv(x))
        attn_output, weights = scaled_dot_product_attention(Q, K, V)
        concat = attn_output.transpose(1, 2).contiguous().view(x.size(0), -1, self.num_heads * self.depth)
        return self.out(concat), weights

# Transformer Block
class TransformerBlock(nn.Module):
    def __init__(self, d_model, num_heads, d_ff):
        super().__init__()
        self.mha = MultiHeadAttention(d_model, num_heads)
        self.norm1 = nn.LayerNorm(d_model)
        self.ffn = nn.Sequential(
            nn.Linear(d_model, d_ff),
            nn.ReLU(),
            nn.Linear(d_ff, d_model)
        )
        self.norm2 = nn.LayerNorm(d_model)

    def forward(self, x):
        attn_output, weights = self.mha(x)
        x = self.norm1(x + attn_output)
        ffn_output = self.ffn(x)
        x = self.norm2(x + ffn_output)
        return x, weights

# 実行例
batch_size, seq_len, d_model, num_heads, d_ff = 1, 4, 8, 2, 16
x = torch.randn(batch_size, seq_len, d_model)

transformer_block = TransformerBlock(d_model, num_heads, d_ff)
output, attn_weights = transformer_block(x)

print("Transformer Output:\n", output)
print("Attention Weights:\n", attn_weights)

```