Delete model.py
Browse files
model.py
DELETED
@@ -1,168 +0,0 @@
|
|
1 |
-
from transformers import PreTrainedModel
|
2 |
-
import torch
|
3 |
-
import torch.nn as nn
|
4 |
-
from torch.nn import functional as F
|
5 |
-
from config import BharatAIConfig
|
6 |
-
batch_size = 4
|
7 |
-
block_size = 128
|
8 |
-
max_iters = 10
|
9 |
-
learning_rate = 3e-4
|
10 |
-
eval_iters = 100
|
11 |
-
device = 'cuda' if torch.cuda.is_available() else 'cpu'
|
12 |
-
print(device)
|
13 |
-
# n_embd = 384
|
14 |
-
# n_head = 4
|
15 |
-
# n_layer = 4
|
16 |
-
dropout = 0.2
|
17 |
-
# vocab_size=1000
|
18 |
-
# model architecture
|
19 |
-
class Head(nn.Module):
|
20 |
-
""" one head of self-attention """
|
21 |
-
|
22 |
-
def __init__(self, n_embd, head_size):
|
23 |
-
super().__init__()
|
24 |
-
self.key = nn.Linear(n_embd, head_size, bias=False)
|
25 |
-
self.query = nn.Linear(n_embd, head_size, bias=False)
|
26 |
-
self.value = nn.Linear(n_embd, head_size, bias=False)
|
27 |
-
self.register_buffer('tril', torch.tril(torch.ones(block_size, block_size)))
|
28 |
-
|
29 |
-
self.dropout = nn.Dropout(dropout)
|
30 |
-
|
31 |
-
def forward(self, x):
|
32 |
-
# input of size (batch, time-step, channels)
|
33 |
-
# output of size (batch, time-step, head size)
|
34 |
-
B,T,C = x.shape
|
35 |
-
k = self.key(x) # (B,T,hs)
|
36 |
-
q = self.query(x) # (B,T,hs)
|
37 |
-
# compute attention scores ("affinities")
|
38 |
-
wei = q @ k.transpose(-2,-1) * k.shape[-1]**-0.5 # (B, T, hs) @ (B, hs, T) -> (B, T, T)
|
39 |
-
wei = wei.masked_fill(self.tril[:T, :T] == 0, float('-inf')) # (B, T, T)
|
40 |
-
wei = F.softmax(wei, dim=-1) # (B, T, T)
|
41 |
-
wei = self.dropout(wei)
|
42 |
-
# perform the weighted aggregation of the values
|
43 |
-
v = self.value(x) # (B,T,hs)
|
44 |
-
out = wei @ v # (B, T, T) @ (B, T, hs) -> (B, T, hs)
|
45 |
-
return out
|
46 |
-
|
47 |
-
|
48 |
-
class MultiHeadAttention(nn.Module):
|
49 |
-
""" multiple heads of self-attention in parallel """
|
50 |
-
|
51 |
-
def __init__(self, num_heads, head_size, n_embd):
|
52 |
-
super().__init__()
|
53 |
-
self.heads = nn.ModuleList([Head(n_embd,head_size) for _ in range(num_heads)])
|
54 |
-
self.proj = nn.Linear(head_size * num_heads, n_embd)
|
55 |
-
self.dropout = nn.Dropout(dropout)
|
56 |
-
|
57 |
-
def forward(self, x):
|
58 |
-
out = torch.cat([h(x) for h in self.heads], dim=-1) # (B, T, F) -> (B, T, [h1, h1, h1, h1, h2, h2, h2, h2, h3, h3, h3, h3])
|
59 |
-
out = self.dropout(self.proj(out))
|
60 |
-
return out
|
61 |
-
|
62 |
-
class FeedFoward(nn.Module):
|
63 |
-
""" a simple linear layer followed by a non-linearity """
|
64 |
-
|
65 |
-
def __init__(self, n_embd):
|
66 |
-
super().__init__()
|
67 |
-
self.net = nn.Sequential(
|
68 |
-
nn.Linear(n_embd, 4 * n_embd),
|
69 |
-
nn.ReLU(),
|
70 |
-
nn.Linear(4 * n_embd, n_embd),
|
71 |
-
nn.Dropout(dropout),
|
72 |
-
)
|
73 |
-
|
74 |
-
def forward(self, x):
|
75 |
-
return self.net(x)
|
76 |
-
|
77 |
-
class Block(nn.Module):
|
78 |
-
""" Transformer block: communication followed by computation """
|
79 |
-
|
80 |
-
def __init__(self, n_embd, n_head):
|
81 |
-
# n_embd: embedding dimension, n_head: the number of heads we'd like
|
82 |
-
super().__init__()
|
83 |
-
head_size = n_embd // n_head
|
84 |
-
self.sa = MultiHeadAttention(n_head, head_size,n_embd)
|
85 |
-
self.ffwd = FeedFoward(n_embd)
|
86 |
-
self.ln1 = nn.LayerNorm(n_embd)
|
87 |
-
self.ln2 = nn.LayerNorm(n_embd)
|
88 |
-
|
89 |
-
|
90 |
-
def forward(self, x):
|
91 |
-
y = self.sa(x)
|
92 |
-
x = self.ln1(x + y)
|
93 |
-
y = self.ffwd(x)
|
94 |
-
x = self.ln2(x + y)
|
95 |
-
return x
|
96 |
-
|
97 |
-
class BharatAI(PreTrainedModel):
|
98 |
-
config_class = BharatAIConfig
|
99 |
-
def __init__(self, config):
|
100 |
-
super().__init__(config)
|
101 |
-
self.token_embedding_table = nn.Embedding(config.vocab_size, config.n_embd)
|
102 |
-
self.position_embedding_table = nn.Embedding(block_size, config.n_embd)
|
103 |
-
self.blocks = nn.Sequential(*[Block(config.n_embd, n_head=config.n_head) for _ in range(config.n_layer)])
|
104 |
-
self.ln_f = nn.LayerNorm(config.n_embd) # final layer norm
|
105 |
-
self.lm_head = nn.Linear(config.n_embd, config.vocab_size)
|
106 |
-
|
107 |
-
self.apply(self._init_weights)
|
108 |
-
|
109 |
-
|
110 |
-
|
111 |
-
def _init_weights(self, module):
|
112 |
-
if isinstance(module, nn.Linear):
|
113 |
-
torch.nn.init.normal_(module.weight, mean=0.0, std=0.02)
|
114 |
-
if module.bias is not None:
|
115 |
-
torch.nn.init.zeros_(module.bias)
|
116 |
-
elif isinstance(module, nn.Embedding):
|
117 |
-
torch.nn.init.normal_(module.weight, mean=0.0, std=0.02)
|
118 |
-
|
119 |
-
def forward(self, index, targets=None):
|
120 |
-
B, T = index.shape
|
121 |
-
|
122 |
-
|
123 |
-
# idx and targets are both (B,T) tensor of integers
|
124 |
-
tok_emb = self.token_embedding_table(index) # (B,T,C)
|
125 |
-
pos_emb = self.position_embedding_table(torch.arange(T, device=device)) # (T,C)
|
126 |
-
x = tok_emb + pos_emb # (B,T,C)
|
127 |
-
x = self.blocks(x) # (B,T,C)
|
128 |
-
x = self.ln_f(x) # (B,T,C)
|
129 |
-
logits = self.lm_head(x) # (B,T,vocab_size)
|
130 |
-
|
131 |
-
if targets is None:
|
132 |
-
loss = None
|
133 |
-
else:
|
134 |
-
B, T, C = logits.shape
|
135 |
-
logits = logits.view(B*T, C)
|
136 |
-
targets = targets.view(B*T)
|
137 |
-
loss = F.cross_entropy(logits, targets)
|
138 |
-
|
139 |
-
return logits, loss
|
140 |
-
|
141 |
-
def generate(self, index, max_new_tokens):
|
142 |
-
# index is (B, T) array of indices in the current context
|
143 |
-
for _ in range(max_new_tokens):
|
144 |
-
# crop idx to the last block_size tokens
|
145 |
-
index_cond = index[:, -block_size:]
|
146 |
-
# get the predictions
|
147 |
-
logits, loss = self.forward(index_cond)
|
148 |
-
# focus only on the last time step
|
149 |
-
logits = logits[:, -1, :] # becomes (B, C)
|
150 |
-
# apply softmax to get probabilities
|
151 |
-
probs = F.softmax(logits, dim=-1) # (B, C)
|
152 |
-
# sample from the distribution
|
153 |
-
index_next = torch.multinomial(probs, num_samples=1) # (B, 1)
|
154 |
-
# append sampled index to the running sequence
|
155 |
-
index = torch.cat((index, index_next), dim=1) # (B, T+1)
|
156 |
-
return index
|
157 |
-
# config = BharatAIConfig(
|
158 |
-
# vocab_size=1000,
|
159 |
-
# n_embd=384,
|
160 |
-
# n_head=4,
|
161 |
-
# n_layer=4,
|
162 |
-
# dropout=0.2,
|
163 |
-
# # any other parameters you want to set
|
164 |
-
# )
|
165 |
-
# model = CustomAI(config)
|
166 |
-
# # model.load_state_dict(torch.load('/content/BharatAI.pth'))
|
167 |
-
|
168 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|