Upload EpsteinGPT V1

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+EpsteinGPT.ptl filter=lfs diff=lfs merge=lfs -text

EpsteinGPT.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2760dea55d021be0ec5243890e9911eaa2bb88c7ea0a376a9eac023ad4f29aa3
+size 336554701

EpsteinGPT.ptl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c668c8c6153d5ee229f9531304692b229b491ff6e2a13d8a565cb2e029995d18
+size 113733862

README.md

@@ -1,3 +1,57 @@
----
-license: mit
----
+# EpsteinGPT - Minimal GPT Model
+
+This repository contains a Minimal GPT (MVT) model trained on the Epstein email threads dataset.
+
+## Model Details
+
+This is a custom-built Causal Transformer model (`MinimalGPT`) inspired by nanoGPT/minGPT architectures. It was trained from scratch using a custom Byte-Pair Encoding (BPE) tokenizer.
+
+### Configuration (`config.json`)
+```json
+{
+  "vocab_size": 5000,
+  "block_size": 256,
+  "n_layer": 8,
+  "n_head": 8,
+  "n_embd": 512,
+  "batch_size": 16,
+  "dropout": 0.1,
+  "bias": false
+}
+```
+
+## Files Included
+
+*   `epsteingpt_tokenizer.json`: The custom BPE tokenizer used for encoding and decoding text.
+*   `EpsteinGPT.pt`: The PyTorch checkpoint containing the trained model weights.
+*   `EpsteinGPT.ptl`: The TorchScript Lite version of the trained model, optimized for deployment.
+*   `model.py`: Defines the `MVTConfig` class and the `MinimalGPT` model architecture.
+*   `config.json`: Model configuration in JSON format.
+*   `README.md`: This file.
+
+## How to Use
+
+To use this model, you would typically:
+
+1.  Load the tokenizer:
+    ```python
+    from tokenizers import Tokenizer
+    tokenizer = Tokenizer.from_file("epsteingpt_tokenizer.json")
+    ```
+2.  Load the model architecture and configuration (from `model.py` and `config.json`).
+3.  Load the trained weights from `EpsteinGPT.pt` into the model.
+4.  Use the model for text generation or other tasks.
+
+For generation, you can refer to the `generate.py` script used during development.
+
+## Training
+
+The model was trained on a dataset of Epstein email threads. The training process involved:
+
+1.  **Tokenizer Training:** A BPE tokenizer was trained on the raw text data.
+2.  **Data Preparation:** The text data was tokenized and converted into a numerical format.
+3.  **Model Training:** The `MinimalGPT` model was trained using a custom training loop.
+
+## Further Information
+
+For more details on the model architecture and training process, refer to the `model.py` and `train.py` scripts.

config.json

@@ -0,0 +1,10 @@
+{
+  "vocab_size": 5000,
+  "block_size": 256,
+  "n_layer": 8,
+  "n_head": 8,
+  "n_embd": 512,
+  "batch_size": 16,
+  "dropout": 0.1,
+  "bias": false
+}

model.py

@@ -0,0 +1,123 @@
+import math
+import torch
+import torch.nn as nn
+from torch.nn import functional as F
+
+# Configuration Dataclass (equivalent to GPTConfig in nanoGPT)
+class MVTConfig:
+    vocab_size = 5000  # V: Set by custom tokenizer
+    block_size = 256   # T_ctx: Context length
+    n_layer = 8        # N_layer: Number of decoder blocks
+    n_head = 8         # N_head: Number of attention heads
+    n_embd = 512       # D_embd: Embedding dimension
+    batch_size = 16    # B: Batch size
+    dropout = 0.1
+    bias = False       # Optional bias for linear layers
+
+# Initializing device setup
+device = 'cuda' if torch.cuda.is_available() else 'cpu'
+
+# --- 1. Causal Self-Attention Mechanism ---
+class CausalSelfAttention(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        assert config.n_embd % config.n_head == 0
+        self.c_attn = nn.Linear(config.n_embd, 3 * config.n_embd, bias=config.bias)
+        self.c_proj = nn.Linear(config.n_embd, config.n_embd, bias=config.bias)
+        self.attn_dropout = nn.Dropout(config.dropout)
+        self.resid_dropout = nn.Dropout(config.dropout)
+        self.n_head = config.n_head
+        self.n_embd = config.n_embd
+        self.dropout = config.dropout
+        self.block_size = config.block_size
+        self.register_buffer("mask", torch.tril(torch.ones(config.block_size, config.block_size))
+                             .view(1, 1, config.block_size, config.block_size))
+        nn.init.normal_(self.c_proj.weight, mean=0.0, std=0.02 / math.sqrt(2 * config.n_layer))
+
+    def forward(self, x):
+        B, T, C = x.size()
+        q, k, v = self.c_attn(x).split(self.n_embd, dim=2)
+        k = k.view(B, T, self.n_head, C // self.n_head).transpose(1, 2)
+        q = q.view(B, T, self.n_head, C // self.n_head).transpose(1, 2)
+        v = v.view(B, T, self.n_head, C // self.n_head).transpose(1, 2)
+        att = (q @ k.transpose(-2, -1)) * (1.0 / math.sqrt(k.size(-1)))
+        att = att.masked_fill(self.mask[:, :, :T, :T] == 0, float('-inf'))
+        att = F.softmax(att, dim=-1)
+        att = self.attn_dropout(att)
+        y = att @ v
+        y = y.transpose(1, 2).contiguous().view(B, T, C)
+        y = self.resid_dropout(self.c_proj(y))
+        return y
+
+# --- 2. Feed-Forward Network (MLP) ---
+class MLP(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        self.c_fc = nn.Linear(config.n_embd, 4 * config.n_embd, bias=config.bias)
+        self.gelu = nn.GELU()
+        self.c_proj = nn.Linear(4 * config.n_embd, config.n_embd, bias=config.bias)
+        self.dropout = nn.Dropout(config.dropout)
+        nn.init.normal_(self.c_proj.weight, mean=0.0, std=0.02 / math.sqrt(2 * config.n_layer))
+
+    def forward(self, x):
+        x = self.c_fc(x)
+        x = self.gelu(x)
+        x = self.c_proj(x)
+        x = self.dropout(x)
+        return x
+
+# --- 3. Transformer Block ---
+class Block(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        self.ln_1 = nn.LayerNorm(config.n_embd, bias=config.bias)
+        self.attn = CausalSelfAttention(config)
+        self.ln_2 = nn.LayerNorm(config.n_embd, bias=config.bias)
+        self.mlp = MLP(config)
+
+    def forward(self, x):
+        x = x + self.attn(self.ln_1(x))
+        x = x + self.mlp(self.ln_2(x))
+        return x
+
+# --- 4. The MinimalGPT Model ---
+class MinimalGPT(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        # Store config parameters as instance attributes for TorchScript compatibility
+        self.vocab_size = config.vocab_size
+        self.block_size = config.block_size
+        self.n_layer = config.n_layer
+        self.n_head = config.n_head
+        self.n_embd = config.n_embd
+        self.dropout = config.dropout
+        self.bias = config.bias
+
+        self.transformer = nn.ModuleDict(dict(
+            wte=nn.Embedding(self.vocab_size, self.n_embd),
+            wpe=nn.Embedding(self.block_size, self.n_embd),
+            drop=nn.Dropout(self.dropout),
+            h=nn.ModuleList([Block(config) for _ in range(self.n_layer)]),
+            ln_f=nn.LayerNorm(self.n_embd, bias=self.bias),
+        ))
+        self.lm_head = nn.Linear(self.n_embd, self.vocab_size, bias=False)
+        self.transformer.wte.weight = self.lm_head.weight
+        print(f"Minimal GPT Model initialized: {sum(p.numel() for p in self.parameters())/1e6:.2f}M parameters")
+
+    def forward(self, idx, targets=None):
+        B, T = idx.size()
+        assert T <= self.block_size, f"Input sequence length {T} exceeds block size {self.block_size}"
+        pos = torch.arange(0, T, dtype=torch.long, device=idx.device)
+        tok_emb = self.transformer.wte(idx)
+        pos_emb = self.transformer.wpe(pos)
+        x = self.transformer.drop(tok_emb + pos_emb)
+        for block in self.transformer.h:
+            x = block(x)
+        x = self.transformer.ln_f(x)
+        logits = self.lm_head(x)
+        loss = None
+        if targets is not None:
+            loss = F.cross_entropy(logits.view(-1, logits.size(-1)), targets.view(-1), ignore_index=-1)
+        else: # Return a dummy loss tensor if targets is None for TorchScript compatibility
+            loss = torch.tensor(0.0, device=idx.device)
+        return logits, loss