Upload 17 files

checkpoints/best_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1bf35017912c87c7681e3c232ffe5a2481c97ec4e166ef55e5a4f7f9e780c5a5
+size 13068032

checkpoints/checkpoint_step_100.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a384cf7782c53e58fef5a5d5ba3ffa0c1724fa45e707fe94a9cb413620a99e68
+size 13068032

checkpoints/checkpoint_step_25.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ff3858e7c4ea74c99ee40ac94096a284f443274bcd5bb3c3c650fa4083a1f723
+size 13068032

checkpoints/checkpoint_step_50.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b98d0bded63833efd902dcc514e0472352ae972912c4e763b17ced507c9b405f
+size 13068032

checkpoints/final_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:459b6ab5c6fe9c32084c28426145ed187e8b9c50e40fdecdedbdb2b170525672
+size 13068016

config.json

@@ -0,0 +1,19 @@
+{
+  "architectures": [
+    "VicAIModel"
+  ],
+  "vocab_size": 2000,
+  "dim": 128,
+  "n_layers": 4,
+  "n_heads": 4,
+  "n_kv_heads": 4,
+  "hidden_dim": 256,
+  "max_seq_len": 512,
+  "tie_weights": false,
+  "model_type": "vicai",
+  "tokenizer_class": "ByteLevelBPETokenizer",
+  "pad_token_id": 1,
+  "eos_token_id": 0,
+  "unk_token_id": 2,
+  "bos_token_id": 3
+}

dataset.py

@@ -0,0 +1,347 @@
+"""
+VicAI Dataset
+Dataset handling for training on Wikipedia and other text sources.
+"""
+
+import os
+import random
+import re
+from typing import Dict, Iterator, List, Optional
+
+import requests
+import torch
+from torch.utils.data import Dataset, IterableDataset
+
+
+class WikipediaDataset(IterableDataset):
+    """Stream Wikipedia articles for training."""
+    
+    def __init__(
+        self,
+        tokenizer,
+        max_length: int = 2048,
+        languages: List[str] = ['en'],
+        min_article_length: int = 100,
+    ):
+        self.tokenizer = tokenizer
+        self.max_length = max_length
+        self.languages = languages
+        self.min_article_length = min_article_length
+        self.base_url = "https://en.wikipedia.org/w/api.php"
+    
+    def _fetch_random_article(self) -> Optional[str]:
+        """Fetch a random Wikipedia article."""
+        try:
+            params = {
+                'action': 'query',
+                'format': 'json',
+                'generator': 'random',
+                'grnnamespace': 0,
+                'grnlimit': 1,
+                'prop': 'extracts',
+                'explaintext': True,
+                'exsentences': 50,
+            }
+            response = requests.get(self.base_url, params=params, timeout=10)
+            data = response.json()
+            
+            pages = data['query']['pages']
+            for page_id, page_data in pages.items():
+                text = page_data.get('extract', '')
+                if len(text) > self.min_article_length:
+                    return text
+            return None
+        except Exception as e:
+            print(f"Error fetching article: {e}")
+            return None
+    
+    def _fetch_article_by_title(self, title: str) -> Optional[str]:
+        """Fetch a specific Wikipedia article by title."""
+        try:
+            params = {
+                'action': 'query',
+                'format': 'json',
+                'titles': title,
+                'prop': 'extracts',
+                'explaintext': True,
+                'exlimit': 1,
+            }
+            response = requests.get(self.base_url, params=params, timeout=10)
+            data = response.json()
+            
+            pages = data['query']['pages']
+            for page_id, page_data in pages.items():
+                if page_id != '-1':
+                    return page_data.get('extract', '')
+            return None
+        except Exception as e:
+            print(f"Error fetching article: {e}")
+            return None
+    
+    def _clean_text(self, text: str) -> str:
+        """Clean Wikipedia text."""
+        # Remove citation markers
+        text = re.sub(r'\[\d+\]', '', text)
+        # Remove multiple spaces
+        text = re.sub(r'\s+', ' ', text)
+        # Remove special characters but keep basic punctuation
+        text = re.sub(r'[^\w\s.,!?;:\'\"()-]', ' ', text)
+        return text.strip()
+    
+    def _tokenize_text(self, text: str) -> List[int]:
+        """Tokenize text and create chunks."""
+        cleaned = self._clean_text(text)
+        tokens = self.tokenizer.encode(cleaned, add_special_tokens=True)
+        return tokens
+    
+    def __iter__(self):
+        """Iterate over Wikipedia articles."""
+        while True:
+            text = self._fetch_random_article()
+            if text:
+                tokens = self._tokenize_text(text)
+                
+                # Create chunks of max_length
+                for i in range(0, len(tokens), self.max_length):
+                    chunk = tokens[i:i + self.max_length]
+                    if len(chunk) > 10:  # Minimum chunk size
+                        # Pad if necessary
+                        if len(chunk) < self.max_length:
+                            chunk.extend([self.tokenizer.pad_token_id] * (self.max_length - len(chunk)))
+                        
+                        input_ids = torch.tensor(chunk[:-1])
+                        labels = torch.tensor(chunk[1:])
+                        
+                        yield {
+                            'input_ids': input_ids,
+                            'labels': labels,
+                            'attention_mask': (input_ids != self.tokenizer.pad_token_id).long(),
+                        }
+
+
+class TextFileDataset(Dataset):
+    """Dataset from local text files."""
+    
+    def __init__(
+        self,
+        file_path: str,
+        tokenizer,
+        max_length: int = 2048,
+        stride: int = 512,
+    ):
+        self.tokenizer = tokenizer
+        self.max_length = max_length
+        self.stride = stride
+        
+        print(f"Loading dataset from {file_path}...")
+        with open(file_path, 'r', encoding='utf-8') as f:
+            text = f.read()
+        
+        # Tokenize full text
+        self.tokens = tokenizer.encode(text, add_special_tokens=False)
+        
+        # Create chunks
+        self.chunks = []
+        for i in range(0, len(self.tokens) - max_length, stride):
+            chunk = self.tokens[i:i + max_length + 1]
+            if len(chunk) == max_length + 1:
+                self.chunks.append(chunk)
+        
+        print(f"Created {len(self.chunks)} chunks from {len(self.tokens)} tokens")
+    
+    def __len__(self):
+        return len(self.chunks)
+    
+    def __getitem__(self, idx):
+        chunk = self.chunks[idx]
+        input_ids = torch.tensor(chunk[:-1])
+        labels = torch.tensor(chunk[1:])
+        
+        return {
+            'input_ids': input_ids,
+            'labels': labels,
+            'attention_mask': torch.ones_like(input_ids),
+        }
+
+
+class MixedDataset(IterableDataset):
+    """Mix multiple data sources."""
+    
+    def __init__(
+        self,
+        datasets: List[IterableDataset],
+        weights: Optional[List[float]] = None,
+    ):
+        self.datasets = datasets
+        self.weights = weights or [1.0] * len(datasets)
+        assert len(self.datasets) == len(self.weights)
+        
+        # Normalize weights
+        total = sum(self.weights)
+        self.weights = [w / total for w in self.weights]
+    
+    def __iter__(self):
+        """Sample from datasets according to weights."""
+        iterators = [iter(ds) for ds in self.datasets]
+        
+        while True:
+            # Choose dataset based on weights
+            dataset_idx = random.choices(range(len(self.datasets)), weights=self.weights)[0]
+            
+            try:
+                yield next(iterators[dataset_idx])
+            except StopIteration:
+                # Restart iterator if exhausted
+                iterators[dataset_idx] = iter(self.datasets[dataset_idx])
+                yield next(iterators[dataset_idx])
+
+
+class PretokenizedDataset(Dataset):
+    """Dataset from pre-tokenized binary files."""
+    
+    def __init__(self, data_dir: str, max_length: int = 2048):
+        self.data_dir = data_dir
+        self.max_length = max_length
+        
+        # Load all .pt files
+        self.files = []
+        for fname in os.listdir(data_dir):
+            if fname.endswith('.pt'):
+                self.files.append(os.path.join(data_dir, fname))
+        
+        self.files.sort()
+        print(f"Found {len(self.files)} pre-tokenized files")
+        
+        # Load metadata
+        self.lengths = []
+        for f in self.files:
+            data = torch.load(f, map_location='cpu')
+            self.lengths.append(len(data) // max_length)
+        
+        self.total_length = sum(self.lengths)
+    
+    def __len__(self):
+        return self.total_length
+    
+    def __getitem__(self, idx):
+        # Find which file contains this index
+        file_idx = 0
+        remaining = idx
+        for i, length in enumerate(self.lengths):
+            if remaining < length:
+                file_idx = i
+                break
+            remaining -= length
+        
+        # Load data
+        data = torch.load(self.files[file_idx], map_location='cpu')
+        start = remaining * self.max_length
+        chunk = data[start:start + self.max_length + 1]
+        
+        input_ids = chunk[:-1]
+        labels = chunk[1:]
+        
+        return {
+            'input_ids': input_ids,
+            'labels': labels,
+            'attention_mask': torch.ones_like(input_ids),
+        }
+
+
+def download_wikipedia_dump(output_dir: str, language: str = 'en'):
+    """Download Wikipedia dump for offline processing."""
+    os.makedirs(output_dir, exist_ok=True)
+    
+    # Wikipedia dump URLs
+    base_url = f"https://dumps.wikimedia.org/{language}wiki/latest/"
+    files = [
+        f"{language}wiki-latest-pages-articles-multistream.xml.bz2",
+    ]
+    
+    for filename in files:
+        url = base_url + filename
+        output_path = os.path.join(output_dir, filename)
+        
+        if os.path.exists(output_path):
+            print(f"{filename} already exists")
+            continue
+        
+        print(f"Downloading {filename}...")
+        try:
+            response = requests.get(url, stream=True)
+            response.raise_for_status()
+            
+            with open(output_path, 'wb') as f:
+                for chunk in response.iter_content(chunk_size=8192):
+                    f.write(chunk)
+            
+            print(f"Downloaded {filename}")
+        except Exception as e:
+            print(f"Error downloading {filename}: {e}")
+
+
+def create_sample_corpus(output_file: str = "sample_corpus.txt", num_articles: int = 1000):
+    """Create a sample corpus by fetching Wikipedia articles."""
+    print(f"Creating sample corpus with {num_articles} articles...")
+    
+    dataset = WikipediaDataset(
+        tokenizer=None,  # We'll use raw text
+        max_length=100000,  # Large to get full articles
+    )
+    
+    articles = []
+    for i, item in enumerate(dataset):
+        if i >= num_articles:
+            break
+        
+        # Get raw text from the article fetch
+        text = dataset._fetch_random_article()
+        if text:
+            articles.append(text)
+        
+        if (i + 1) % 100 == 0:
+            print(f"  Fetched {i + 1}/{num_articles} articles")
+    
+    # Write to file
+    with open(output_file, 'w', encoding='utf-8') as f:
+        for article in articles:
+            f.write(article + '\n\n<|endoftext|>\n\n')
+    
+    print(f"Sample corpus saved to {output_file}")
+    return output_file
+
+
+def prepare_openwebtext_data(output_dir: str, max_files: int = 100):
+    """
+    Download and prepare OpenWebText corpus.
+    Note: This is a placeholder - actual OpenWebText requires specific download.
+    """
+    os.makedirs(output_dir, exist_ok=True)
+    print(f"OpenWebText data preparation placeholder")
+    print(f"Please download OpenWebText from https://github.com/jcpeterson/openwebtext")
+    print(f"and place files in {output_dir}")
+
+
+if __name__ == "__main__":
+    # Test dataset
+    from tokenizer import BPETokenizer
+    
+    # Create sample tokenizer
+    sample_texts = [
+        "This is a sample text for testing.",
+        "Wikipedia contains many interesting articles.",
+        "Machine learning models need lots of data.",
+    ]
+    tokenizer = BPETokenizer(vocab_size=1000)
+    tokenizer.train(sample_texts)
+    
+    # Test Wikipedia dataset
+    print("\nTesting Wikipedia dataset...")
+    wiki_dataset = WikipediaDataset(tokenizer, max_length=128)
+    
+    for i, batch in enumerate(wiki_dataset):
+        if i >= 3:
+            break
+        print(f"\nBatch {i + 1}:")
+        print(f"  Input shape: {batch['input_ids'].shape}")
+        print(f"  Labels shape: {batch['labels'].shape}")

generate.py

@@ -0,0 +1,330 @@
+"""
+VicAI Text Generation
+Interactive text generation and sampling utilities.
+"""
+
+import argparse
+import sys
+
+import torch
+
+from model import VicAIModel, VicAIConfig, create_vicai_5b
+from tokenizer import ByteLevelBPETokenizer, BPETokenizer
+from utils import get_logger
+
+
+def generate_interactive(
+    model,
+    tokenizer,
+    device,
+    max_new_tokens: int = 256,
+    temperature: float = 0.8,
+    top_k: int = 50,
+    top_p: float = 0.9,
+    repetition_penalty: float = 1.1,
+):
+    """Interactive text generation loop."""
+    print("\n" + "=" * 60)
+    print("VicAI Interactive Generation")
+    print("=" * 60)
+    print("Commands:")
+    print("  /quit    - Exit the program")
+    print("  /config  - Show current generation settings")
+    print("  /temp X  - Set temperature (0.1 - 2.0)")
+    print("  /topk X  - Set top-k (1 - 100)")
+    print("  /topp X  - Set top-p (0.0 - 1.0)")
+    print("  /reppen X - Set repetition penalty (1.0 - 2.0)")
+    print("  /maxlen X - Set max new tokens")
+    print("=" * 60 + "\n")
+    
+    # Current settings
+    settings = {
+        'temperature': temperature,
+        'top_k': top_k,
+        'top_p': top_p,
+        'repetition_penalty': repetition_penalty,
+        'max_new_tokens': max_new_tokens,
+    }
+    
+    while True:
+        try:
+            # Get prompt
+            prompt = input("\nPrompt: ").strip()
+            
+            # Handle commands
+            if prompt == '/quit':
+                print("Goodbye!")
+                break
+            
+            if prompt == '/config':
+                print("\nCurrent settings:")
+                for key, value in settings.items():
+                    print(f"  {key}: {value}")
+                continue
+            
+            if prompt.startswith('/temp '):
+                try:
+                    settings['temperature'] = float(prompt.split()[1])
+                    print(f"Temperature set to {settings['temperature']}")
+                except (ValueError, IndexError):
+                    print("Invalid temperature value")
+                continue
+            
+            if prompt.startswith('/topk '):
+                try:
+                    settings['top_k'] = int(prompt.split()[1])
+                    print(f"Top-k set to {settings['top_k']}")
+                except (ValueError, IndexError):
+                    print("Invalid top-k value")
+                continue
+            
+            if prompt.startswith('/topp '):
+                try:
+                    settings['top_p'] = float(prompt.split()[1])
+                    print(f"Top-p set to {settings['top_p']}")
+                except (ValueError, IndexError):
+                    print("Invalid top-p value")
+                continue
+            
+            if prompt.startswith('/reppen '):
+                try:
+                    settings['repetition_penalty'] = float(prompt.split()[1])
+                    print(f"Repetition penalty set to {settings['repetition_penalty']}")
+                except (ValueError, IndexError):
+                    print("Invalid repetition penalty value")
+                continue
+            
+            if prompt.startswith('/maxlen '):
+                try:
+                    settings['max_new_tokens'] = int(prompt.split()[1])
+                    print(f"Max new tokens set to {settings['max_new_tokens']}")
+                except (ValueError, IndexError):
+                    print("Invalid max new tokens value")
+                continue
+            
+            if not prompt:
+                continue
+            
+            # Encode prompt
+            input_ids = torch.tensor([tokenizer.encode(prompt)], device=device)
+            
+            # Generate
+            print("\nGenerating...")
+            with torch.no_grad():
+                output_ids = model.generate(
+                    input_ids,
+                    max_new_tokens=settings['max_new_tokens'],
+                    temperature=settings['temperature'],
+                    top_k=settings['top_k'],
+                    top_p=settings['top_p'],
+                    repetition_penalty=settings['repetition_penalty'],
+                    eos_token_id=tokenizer.eos_token_id,
+                )
+            
+            # Decode and print
+            generated_text = tokenizer.decode(output_ids[0].tolist())
+            # Remove the original prompt from output
+            prompt_text = tokenizer.decode(input_ids[0].tolist())
+            if generated_text.startswith(prompt_text):
+                generated_text = generated_text[len(prompt_text):].strip()
+            
+            print("\n" + "-" * 60)
+            print("Generated:")
+            print("-" * 60)
+            print(generated_text)
+            print("-" * 60)
+            
+            # Print token info
+            num_tokens = output_ids.shape[1] - input_ids.shape[1]
+            print(f"\nTokens generated: {num_tokens}")
+            
+        except KeyboardInterrupt:
+            print("\n\nInterrupted by user. Type /quit to exit.")
+        except Exception as e:
+            print(f"\nError: {e}")
+
+
+def generate_batch(
+    model,
+    tokenizer,
+    prompts: list,
+    device,
+    max_new_tokens: int = 256,
+    temperature: float = 0.8,
+    top_k: int = 50,
+    top_p: float = 0.9,
+):
+    """Generate completions for multiple prompts."""
+    results = []
+    
+    for prompt in prompts:
+        input_ids = torch.tensor([tokenizer.encode(prompt)], device=device)
+        
+        with torch.no_grad():
+            output_ids = model.generate(
+                input_ids,
+                max_new_tokens=max_new_tokens,
+                temperature=temperature,
+                top_k=top_k,
+                top_p=top_p,
+                eos_token_id=tokenizer.eos_token_id,
+            )
+        
+        generated_text = tokenizer.decode(output_ids[0].tolist())
+        prompt_text = tokenizer.decode(input_ids[0].tolist())
+        
+        if generated_text.startswith(prompt_text):
+            generated_text = generated_text[len(prompt_text):].strip()
+        
+        results.append({
+            'prompt': prompt,
+            'completion': generated_text,
+        })
+    
+    return results
+
+
+def benchmark_generation(
+    model,
+    tokenizer,
+    device,
+    num_runs: int = 10,
+    max_new_tokens: int = 128,
+    prompt: str = "The future of artificial intelligence is",
+):
+    """Benchmark generation speed."""
+    import time
+    
+    print(f"\nBenchmarking generation ({num_runs} runs)...")
+    
+    input_ids = torch.tensor([tokenizer.encode(prompt)], device=device)
+    
+    # Warmup
+    with torch.no_grad():
+        _ = model.generate(input_ids, max_new_tokens=10)
+    
+    torch.cuda.synchronize()
+    
+    # Benchmark
+    times = []
+    tokens_generated = []
+    
+    for i in range(num_runs):
+        start = time.time()
+        
+        with torch.no_grad():
+            output = model.generate(
+                input_ids,
+                max_new_tokens=max_new_tokens,
+                temperature=1.0,
+            )
+        
+        torch.cuda.synchronize()
+        elapsed = time.time() - start
+        
+        num_tokens = output.shape[1] - input_ids.shape[1]
+        times.append(elapsed)
+        tokens_generated.append(num_tokens)
+        
+        print(f"  Run {i+1}: {num_tokens} tokens in {elapsed:.2f}s ({num_tokens/elapsed:.1f} tok/s)")
+    
+    avg_time = sum(times) / len(times)
+    avg_tokens = sum(tokens_generated) / len(tokens_generated)
+    avg_speed = avg_tokens / avg_time
+    
+    print(f"\nAverage: {avg_tokens:.1f} tokens in {avg_time:.2f}s ({avg_speed:.1f} tok/s)")
+
+
+def main():
+    parser = argparse.ArgumentParser(description='Generate text with VicAI')
+    
+    parser.add_argument('--checkpoint', type=str, required=True, help='Path to model checkpoint')
+    parser.add_argument('--tokenizer', type=str, default='tokenizer.pkl', help='Path to tokenizer')
+    parser.add_argument('--prompt', type=str, default=None, help='Single prompt to generate from')
+    parser.add_argument('--interactive', action='store_true', help='Interactive mode')
+    parser.add_argument('--max-new-tokens', type=int, default=256, help='Maximum tokens to generate')
+    parser.add_argument('--temperature', type=float, default=0.8, help='Sampling temperature')
+    parser.add_argument('--top-k', type=int, default=50, help='Top-k sampling')
+    parser.add_argument('--top-p', type=float, default=0.9, help='Top-p (nucleus) sampling')
+    parser.add_argument('--repetition-penalty', type=float, default=1.1, help='Repetition penalty')
+    parser.add_argument('--benchmark', action='store_true', help='Run generation benchmark')
+    parser.add_argument('--device', type=str, default='cuda', help='Device to use')
+    
+    args = parser.parse_args()
+    
+    # Setup device
+    device = torch.device(args.device if torch.cuda.is_available() else 'cpu')
+    print(f"Using device: {device}")
+    
+    # Load tokenizer
+    print(f"Loading tokenizer from {args.tokenizer}...")
+    # Use ByteLevelBPETokenizer by default (our trained tokenizer)
+    tokenizer = ByteLevelBPETokenizer()
+    tokenizer.load(args.tokenizer)
+    print(f"Tokenizer loaded: {len(tokenizer)} tokens")
+    
+    # Load model
+    print(f"Loading model from {args.checkpoint}...")
+    checkpoint = torch.load(args.checkpoint, map_location=device)
+    
+    # Create model (assuming 5B config)
+    model = create_vicai_5b(vocab_size=len(tokenizer))
+    
+    # Load weights
+    state_dict = checkpoint.get('model', checkpoint)
+    model.load_state_dict(state_dict)
+    model = model.to(device)
+    model.eval()
+    
+    print(f"Model loaded: ~{model.get_num_params() / 1e9:.2f}B parameters")
+    
+    # Run benchmark if requested
+    if args.benchmark:
+        benchmark_generation(model, tokenizer, device)
+        return
+    
+    # Interactive mode
+    if args.interactive or args.prompt is None:
+        generate_interactive(
+            model,
+            tokenizer,
+            device,
+            max_new_tokens=args.max_new_tokens,
+            temperature=args.temperature,
+            top_k=args.top_k,
+            top_p=args.top_p,
+            repetition_penalty=args.repetition_penalty,
+        )
+    else:
+        # Single prompt generation
+        print(f"\nPrompt: {args.prompt}")
+        print("-" * 60)
+        
+        input_ids = torch.tensor([tokenizer.encode(args.prompt)], device=device)
+        
+        with torch.no_grad():
+            output_ids = model.generate(
+                input_ids,
+                max_new_tokens=args.max_new_tokens,
+                temperature=args.temperature,
+                top_k=args.top_k,
+                top_p=args.top_p,
+                repetition_penalty=args.repetition_penalty,
+                eos_token_id=tokenizer.eos_token_id,
+            )
+        
+        generated_text = tokenizer.decode(output_ids[0].tolist())
+        prompt_text = tokenizer.decode(input_ids[0].tolist())
+        
+        if generated_text.startswith(prompt_text):
+            generated_text = generated_text[len(prompt_text):].strip()
+        
+        print(generated_text)
+        print("-" * 60)
+        
+        num_tokens = output_ids.shape[1] - input_ids.shape[1]
+        print(f"\nGenerated {num_tokens} tokens")
+
+
+if __name__ == '__main__':
+    main()

model.py

@@ -0,0 +1,403 @@
+"""
+VicAI Model Architecture
+A 5B parameter decoder-only transformer language model.
+"""
+
+import math
+from typing import Optional, Tuple
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+
+class RMSNorm(nn.Module):
+    """Root Mean Square Layer Normalization."""
+    
+    def __init__(self, dim: int, eps: float = 1e-6):
+        super().__init__()
+        self.eps = eps
+        self.weight = nn.Parameter(torch.ones(dim))
+    
+    def forward(self, x):
+        return x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + self.eps) * self.weight
+
+
+class RotaryPositionalEmbedding(nn.Module):
+    """Rotary Position Embedding (RoPE)."""
+    
+    def __init__(self, dim: int, max_seq_len: int = 8192, base: float = 10000.0):
+        super().__init__()
+        self.dim = dim
+        self.max_seq_len = max_seq_len
+        self.base = base
+        
+        inv_freq = 1.0 / (self.base ** (torch.arange(0, dim, 2).float() / dim))
+        self.register_buffer("inv_freq", inv_freq)
+        
+        t = torch.arange(max_seq_len)
+        freqs = torch.einsum("i,j->ij", t, inv_freq)
+        emb = torch.cat((freqs, freqs), dim=-1)
+        self.register_buffer("cos_cached", emb.cos()[None, None, :, :])
+        self.register_buffer("sin_cached", emb.sin()[None, None, :, :])
+    
+    def rotate_half(self, x):
+        x1, x2 = x.chunk(2, dim=-1)
+        return torch.cat((-x2, x1), dim=-1)
+    
+    def apply_rotary_pos_emb(self, q, k, cos, sin):
+        q_embed = (q * cos) + (self.rotate_half(q) * sin)
+        k_embed = (k * cos) + (self.rotate_half(k) * sin)
+        return q_embed, k_embed
+    
+    def forward(self, q, k, seq_len: int):
+        cos = self.cos_cached[:, :, :seq_len, :]
+        sin = self.sin_cached[:, :, :seq_len, :]
+        return self.apply_rotary_pos_emb(q, k, cos, sin)
+
+
+class GroupedQueryAttention(nn.Module):
+    """Grouped Query Attention (GQA) for efficient inference."""
+    
+    def __init__(
+        self,
+        dim: int,
+        n_heads: int,
+        n_kv_heads: int,
+        dropout: float = 0.0,
+    ):
+        super().__init__()
+        self.dim = dim
+        self.n_heads = n_heads
+        self.n_kv_heads = n_kv_heads
+        self.head_dim = dim // n_heads
+        self.n_rep = n_heads // n_kv_heads
+        
+        self.wq = nn.Linear(dim, n_heads * self.head_dim, bias=False)
+        self.wk = nn.Linear(dim, n_kv_heads * self.head_dim, bias=False)
+        self.wv = nn.Linear(dim, n_kv_heads * self.head_dim, bias=False)
+        self.wo = nn.Linear(n_heads * self.head_dim, dim, bias=False)
+        
+        self.attn_dropout = nn.Dropout(dropout)
+        self.resid_dropout = nn.Dropout(dropout)
+        
+        self.rope = RotaryPositionalEmbedding(self.head_dim)
+        
+    def forward(
+        self,
+        x: torch.Tensor,
+        mask: Optional[torch.Tensor] = None,
+        past_key_value: Optional[Tuple[torch.Tensor, torch.Tensor]] = None,
+    ):
+        bsz, seq_len, _ = x.shape
+        
+        q = self.wq(x).view(bsz, seq_len, self.n_heads, self.head_dim).transpose(1, 2)
+        k = self.wk(x).view(bsz, seq_len, self.n_kv_heads, self.head_dim).transpose(1, 2)
+        v = self.wv(x).view(bsz, seq_len, self.n_kv_heads, self.head_dim).transpose(1, 2)
+        
+        q, k = self.rope(q, k, seq_len)
+        
+        if past_key_value is not None:
+            past_k, past_v = past_key_value
+            k = torch.cat([past_k, k], dim=2)
+            v = torch.cat([past_v, v], dim=2)
+        
+        past_key_value = (k, v)
+        
+        # Repeat k/v for grouped query attention
+        k = k.repeat_interleave(self.n_rep, dim=1)
+        v = v.repeat_interleave(self.n_rep, dim=1)
+        
+        scores = torch.matmul(q, k.transpose(-2, -1)) / math.sqrt(self.head_dim)
+        
+        if mask is not None:
+            scores = scores + mask
+        
+        attn = F.softmax(scores, dim=-1)
+        attn = self.attn_dropout(attn)
+        
+        out = torch.matmul(attn, v)
+        out = out.transpose(1, 2).contiguous().view(bsz, seq_len, self.dim)
+        out = self.wo(out)
+        out = self.resid_dropout(out)
+        
+        return out, past_key_value
+
+
+class FeedForward(nn.Module):
+    """SwiGLU Feed-Forward Network."""
+    
+    def __init__(self, dim: int, hidden_dim: int, dropout: float = 0.0):
+        super().__init__()
+        self.w1 = nn.Linear(dim, hidden_dim, bias=False)
+        self.w2 = nn.Linear(hidden_dim, dim, bias=False)
+        self.w3 = nn.Linear(dim, hidden_dim, bias=False)
+        self.dropout = nn.Dropout(dropout)
+    
+    def forward(self, x):
+        return self.w2(F.silu(self.w1(x)) * self.w3(x))
+
+
+class TransformerBlock(nn.Module):
+    """Single transformer block with pre-normalization."""
+    
+    def __init__(
+        self,
+        dim: int,
+        n_heads: int,
+        n_kv_heads: int,
+        hidden_dim: int,
+        dropout: float = 0.0,
+    ):
+        super().__init__()
+        self.attention_norm = RMSNorm(dim)
+        self.attention = GroupedQueryAttention(dim, n_heads, n_kv_heads, dropout)
+        self.ffn_norm = RMSNorm(dim)
+        self.feed_forward = FeedForward(dim, hidden_dim, dropout)
+    
+    def forward(
+        self,
+        x: torch.Tensor,
+        mask: Optional[torch.Tensor] = None,
+        past_key_value: Optional[Tuple[torch.Tensor, torch.Tensor]] = None,
+    ):
+        # Attention with residual
+        attn_out, past_key_value = self.attention(
+            self.attention_norm(x), mask, past_key_value
+        )
+        x = x + attn_out
+        
+        # FFN with residual
+        x = x + self.feed_forward(self.ffn_norm(x))
+        
+        return x, past_key_value
+
+
+class VicAIConfig:
+    """Configuration for VicAI model."""
+    
+    def __init__(
+        self,
+        vocab_size: int = 32000,
+        dim: int = 4096,
+        n_layers: int = 32,
+        n_heads: int = 32,
+        n_kv_heads: int = 8,
+        hidden_dim: int = 14336,
+        max_seq_len: int = 8192,
+        dropout: float = 0.0,
+        tie_weights: bool = False,
+    ):
+        self.vocab_size = vocab_size
+        self.dim = dim
+        self.n_layers = n_layers
+        self.n_heads = n_heads
+        self.n_kv_heads = n_kv_heads
+        self.hidden_dim = hidden_dim
+        self.max_seq_len = max_seq_len
+        self.dropout = dropout
+        self.tie_weights = tie_weights
+    
+    @property
+    def num_parameters(self):
+        """Calculate approximate parameter count."""
+        # Embedding
+        params = self.vocab_size * self.dim
+        # Attention per layer
+        attn_params = 4 * self.dim * self.dim  # q, k, v, o projections
+        # FFN per layer
+        ffn_params = 3 * self.dim * self.hidden_dim  # w1, w2, w3
+        # Layers
+        params += self.n_layers * (attn_params + ffn_params)
+        # Output
+        params += self.vocab_size * self.dim
+        return params
+
+
+class VicAIModel(nn.Module):
+    """
+    VicAI: A 5B parameter decoder-only transformer language model.
+    
+    Architecture details:
+    - 32 layers
+    - 4096 model dimension
+    - 32 attention heads (8 key-value heads for GQA)
+    - SwiGLU FFN with 14336 hidden dimension
+    - RoPE positional embeddings
+    - RMSNorm pre-normalization
+    - ~5.1B total parameters
+    """
+    
+    def __init__(self, config: VicAIConfig):
+        super().__init__()
+        self.config = config
+        
+        self.token_embedding = nn.Embedding(config.vocab_size, config.dim)
+        self.dropout = nn.Dropout(config.dropout)
+        
+        self.layers = nn.ModuleList([
+            TransformerBlock(
+                config.dim,
+                config.n_heads,
+                config.n_kv_heads,
+                config.hidden_dim,
+                config.dropout,
+            )
+            for _ in range(config.n_layers)
+        ])
+        
+        self.norm = RMSNorm(config.dim)
+        self.lm_head = nn.Linear(config.dim, config.vocab_size, bias=False)
+        
+        if config.tie_weights:
+            self.lm_head.weight = self.token_embedding.weight
+        
+        self.apply(self._init_weights)
+        
+        # Print model info
+        total_params = self.get_num_params()
+        print(f"VicAI Model initialized with {total_params / 1e9:.2f}B parameters")
+    
+    def _init_weights(self, module):
+        if isinstance(module, nn.Linear):
+            torch.nn.init.normal_(module.weight, mean=0.0, std=0.02)
+            if module.bias is not None:
+                torch.nn.init.zeros_(module.bias)
+        elif isinstance(module, nn.Embedding):
+            torch.nn.init.normal_(module.weight, mean=0.0, std=0.02)
+    
+    def get_num_params(self, non_embedding=True):
+        n_params = sum(p.numel() for p in self.parameters())
+        if non_embedding:
+            n_params -= self.token_embedding.weight.numel()
+        return n_params
+    
+    def forward(
+        self,
+        input_ids: torch.Tensor,
+        targets: Optional[torch.Tensor] = None,
+        past_key_values: Optional[list] = None,
+    ):
+        bsz, seq_len = input_ids.shape
+        
+        # Create causal mask
+        mask = torch.triu(
+            torch.ones(seq_len, seq_len, device=input_ids.device),
+            diagonal=1
+        ).bool()
+        mask = mask.unsqueeze(0).unsqueeze(0)
+        mask = mask.to(input_ids.device)
+        mask = torch.where(mask, float('-inf'), 0.0)
+        
+        x = self.token_embedding(input_ids)
+        x = self.dropout(x)
+        
+        new_key_values = []
+        for i, layer in enumerate(self.layers):
+            past_kv = past_key_values[i] if past_key_values is not None else None
+            x, kv = layer(x, mask, past_kv)
+            new_key_values.append(kv)
+        
+        x = self.norm(x)
+        logits = self.lm_head(x)
+        
+        loss = None
+        if targets is not None:
+            loss = F.cross_entropy(
+                logits.view(-1, logits.size(-1)),
+                targets.view(-1),
+                ignore_index=-100
+            )
+        
+        return {
+            'logits': logits,
+            'loss': loss,
+            'past_key_values': new_key_values,
+        }
+    
+    @torch.no_grad()
+    def generate(
+        self,
+        input_ids: torch.Tensor,
+        max_new_tokens: int = 100,
+        temperature: float = 1.0,
+        top_k: int = 50,
+        top_p: float = 0.9,
+        repetition_penalty: float = 1.0,
+        eos_token_id: Optional[int] = None,
+    ):
+        """Generate text autoregressively."""
+        self.eval()
+        
+        batch_size = input_ids.shape[0]
+        device = input_ids.device
+        past_key_values = None
+        
+        for _ in range(max_new_tokens):
+            outputs = self(input_ids, past_key_values=past_key_values)
+            logits = outputs['logits']
+            past_key_values = outputs['past_key_values']
+            
+            # Get logits for last token
+            logits = logits[:, -1, :] / temperature
+            
+            # Apply repetition penalty
+            if repetition_penalty != 1.0:
+                for i in range(batch_size):
+                    for token_id in set(input_ids[i].tolist()):
+                        logits[i, token_id] /= repetition_penalty
+            
+            # Top-k filtering
+            if top_k > 0:
+                indices_to_remove = logits < torch.topk(logits, top_k)[0][..., -1, None]
+                logits[indices_to_remove] = float('-inf')
+            
+            # Top-p (nucleus) filtering
+            if top_p < 1.0:
+                sorted_logits, sorted_indices = torch.sort(logits, descending=True)
+                cumulative_probs = torch.cumsum(F.softmax(sorted_logits, dim=-1), dim=-1)
+                sorted_indices_to_remove = cumulative_probs > top_p
+                sorted_indices_to_remove[..., 1:] = sorted_indices_to_remove[..., :-1].clone()
+                sorted_indices_to_remove[..., 0] = 0
+                indices_to_remove = sorted_indices_to_remove.scatter(
+                    1, sorted_indices, sorted_indices_to_remove
+                )
+                logits[indices_to_remove] = float('-inf')
+            
+            probs = F.softmax(logits, dim=-1)
+            next_token = torch.multinomial(probs, num_samples=1)
+            
+            input_ids = torch.cat([input_ids, next_token], dim=1)
+            
+            # Early stopping if EOS token generated
+            if eos_token_id is not None and (next_token == eos_token_id).all():
+                break
+        
+        return input_ids
+
+
+def create_vicai_5b(vocab_size: int = 32000) -> VicAIModel:
+    """Create a 5B parameter VicAI model."""
+    config = VicAIConfig(
+        vocab_size=vocab_size,
+        dim=4096,
+        n_layers=32,
+        n_heads=32,
+        n_kv_heads=8,
+        hidden_dim=14336,
+        max_seq_len=8192,
+        dropout=0.0,
+    )
+    return VicAIModel(config)
+
+
+if __name__ == "__main__":
+    # Test model creation
+    model = create_vicai_5b()
+    print(f"Total parameters: {model.get_num_params() / 1e9:.2f}B")
+    
+    # Test forward pass
+    x = torch.randint(0, 32000, (2, 128))
+    outputs = model(x)
+    print(f"Output shape: {outputs['logits'].shape}")
+    print(f"Loss: {outputs['loss']}")

readme.md

@@ -0,0 +1,303 @@
+# VicAI
+
+A 5B parameter decoder-only transformer language model built from scratch in PyTorch.
+
+## Overview
+
+VicAI is a state-of-the-art language model featuring:
+- **5.1B parameters** with 32 transformer layers
+- **Grouped Query Attention (GQA)** for efficient inference
+- **Rotary Position Embeddings (RoPE)** for better long-context modeling
+- **SwiGLU activation** in feed-forward layers
+- **RMSNorm** pre-normalization
+- **Byte-level BPE tokenization** (32K vocabulary)
+
+## Architecture
+
+| Component | Specification |
+|-----------|---------------|
+| Parameters | ~5.1B |
+| Layers | 32 |
+| Hidden Dim | 4096 |
+| FFN Dim | 14336 |
+| Attention Heads | 32 |
+| KV Heads | 8 (GQA) |
+| Context Length | 8192 |
+| Vocabulary | 32,000 |
+
+## File Structure
+
+```
+vicai/
+├── model.py           # Model architecture and VicAI 5B config
+├── tokenizer.py       # BPE tokenizer implementation
+├── dataset.py         # Data loading (Wikipedia + custom sources)
+├── train.py           # Distributed training script
+├── utils.py           # Training utilities and helpers
+├── generate.py        # Text generation and inference
+├── requirements.txt   # Dependencies
+└── README.md          # This file
+```
+
+## Installation
+
+```bash
+# Clone the repository
+git clone https://github.com/yourusername/vicai.git
+cd vicai
+
+# Create virtual environment
+python -m venv venv
+source venv/bin/activate  # On Windows: venv\Scripts\activate
+
+# Install dependencies
+pip install -r requirements.txt
+```
+
+## Quick Start
+
+### 1. Prepare Training Data
+
+Option A: Create sample corpus from Wikipedia
+```bash
+python -c "from dataset import create_sample_corpus; create_sample_corpus('data/train.txt', num_articles=10000)"
+```
+
+Option B: Use your own text files
+```bash
+# Place your text files in data/ directory
+# Format: plain text with <|endoftext|> markers between documents
+```
+
+### 2. Train Tokenizer
+
+```python
+from tokenizer import ByteLevelBPETokenizer
+from dataset import create_sample_corpus
+
+# Create corpus
+corpus = create_sample_corpus('data/train.txt', num_articles=1000)
+
+# Read texts
+with open(corpus, 'r') as f:
+    texts = f.read().split('<|endoftext|>')
+
+# Train tokenizer
+tokenizer = ByteLevelBPETokenizer(vocab_size=32000)
+tokenizer.train([t for t in texts if t.strip()])
+tokenizer.save('tokenizer.pkl')
+```
+
+### 3. Train Model
+
+Single GPU:
+```bash
+python train.py \
+    --train-data data/train.txt \
+    --val-data data/val.txt \
+    --tokenizer tokenizer.pkl \
+    --batch-size 4 \
+    --max-steps 100000 \
+    --output-dir checkpoints
+```
+
+Multi-GPU (DDP):
+```bash
+torchrun --nproc_per_node=4 train.py \
+    --train-data data/train.txt \
+    --val-data data/val.txt \
+    --batch-size 1 \
+    --max-steps 100000 \
+    --output-dir checkpoints
+```
+
+Multi-GPU (FSDP):
+```bash
+torchrun --nproc_per_node=8 train.py \
+    --use-fsdp \
+    --train-data data/train.txt \
+    --batch-size 1 \
+    --output-dir checkpoints
+```
+
+### 4. Generate Text
+
+Interactive mode:
+```bash
+python generate.py \
+    --checkpoint checkpoints/best_model.pt \
+    --tokenizer tokenizer.pkl \
+    --interactive
+```
+
+Single prompt:
+```bash
+python generate.py \
+    --checkpoint checkpoints/best_model.pt \
+    --tokenizer tokenizer.pkl \
+    --prompt "The future of AI is" \
+    --max-new-tokens 256
+```
+
+## Training Configuration
+
+### Default Hyperparameters
+
+| Parameter | Value |
+|-----------|-------|
+| Learning Rate | 3e-4 |
+| Min LR | 3e-5 |
+| Warmup Steps | 2,000 |
+| Weight Decay | 0.1 |
+| Batch Size | 4 (per device) |
+| Max Steps | 100,000 |
+| Beta1 | 0.9 |
+| Beta2 | 0.95 |
+
+### Training Tips
+
+- **Memory constrained?** Reduce batch size or use gradient accumulation
+- **Longer context?** Increase `--max-seq-len` (up to 8192)
+- **Faster training?** Enable `--compile` for torch.compile optimization
+- **Better quality?** Train longer or use larger dataset
+
+## Generation Parameters
+
+| Parameter | Default | Description |
+|-----------|---------|-------------|
+| temperature | 0.8 | Lower = more focused, higher = more random |
+| top_k | 50 | Consider only top-k tokens |
+| top_p | 0.9 | Nucleus sampling threshold |
+| repetition_penalty | 1.1 | Penalize repeated tokens |
+| max_new_tokens | 256 | Maximum tokens to generate |
+
+## Data Sources
+
+The model can be trained on:
+
+1. **Wikipedia** (streaming via API)
+2. **OpenWebText** (Common Crawl filtered)
+3. **Custom text files** (your own data)
+4. **Mixed datasets** (combine multiple sources)
+
+## Hardware Requirements
+
+### Training
+
+| GPUs | VRAM per GPU | Config |
+|------|--------------|--------|
+| 1x A100 (80GB) | 80GB | batch_size=4, compile=True |
+| 4x A100 (40GB) | 40GB | batch_size=1, DDP |
+| 8x A100 (40GB) | 40GB | batch_size=1, FSDP |
+| 1x RTX 4090 | 24GB | batch_size=1, smaller model |
+
+### Inference
+
+- Minimum: 1x GPU with 16GB VRAM (with quantization)
+- Recommended: 1x GPU with 24GB+ VRAM
+
+## Model Architecture Details
+
+### Grouped Query Attention
+
+Uses 8 key-value heads instead of 32, reducing memory bandwidth during inference while maintaining quality.
+
+### Rotary Position Embeddings
+
+Rotary embeddings are applied to queries and keys, providing better relative position encoding than absolute embeddings.
+
+### SwiGLU Feed-Forward
+
+```python
+FFN(x) = (silu(W1 @ x) * (W3 @ x)) @ W2
+```
+
+This has been shown to improve training stability and performance.
+
+## Example Usage
+
+```python
+from model import create_vicai_5b
+from tokenizer import ByteLevelBPETokenizer
+import torch
+
+# Load tokenizer
+tokenizer = ByteLevelBPETokenizer()
+tokenizer.load('tokenizer.pkl')
+
+# Create model
+model = create_vicai_5b(vocab_size=len(tokenizer))
+
+# Load checkpoint
+checkpoint = torch.load('checkpoints/best_model.pt')
+model.load_state_dict(checkpoint['model'])
+model = model.cuda()
+
+# Generate
+text = "Artificial intelligence will"
+input_ids = torch.tensor([tokenizer.encode(text)]).cuda()
+
+with torch.no_grad():
+    output = model.generate(
+        input_ids,
+        max_new_tokens=100,
+        temperature=0.8,
+        top_k=50,
+        top_p=0.9,
+    )
+
+generated = tokenizer.decode(output[0].tolist())
+print(generated)
+```
+
+## Citation
+
+If you use VicAI in your research, please cite:
+
+```bibtex
+@software{vicai2024,
+  title = {VicAI: A 5B Parameter Language Model from Scratch},
+  author = {Your Name},
+  year = {2024},
+  url = {https://github.com/yourusername/vicai}
+}
+```
+
+## License
+
+This project is licensed under the MIT License.
+
+## Acknowledgments
+
+- Transformer architecture based on "Attention Is All You Need"
+- RoPE embeddings from RoFormer
+- GQA from the Llama 2 paper
+- SwiGLU from the PaLM paper
+
+## Contributing
+
+Contributions are welcome! Please feel free to submit a Pull Request.
+
+## Troubleshooting
+
+### CUDA Out of Memory
+- Reduce batch size
+- Enable gradient checkpointing
+- Use FSDP for multi-GPU training
+- Reduce sequence length
+
+### Slow Training
+- Enable `--compile` flag
+- Use mixed precision (AMP)
+- Ensure data is on fast storage (SSD)
+- Use DataLoader `num_workers > 0`
+
+### Poor Generation Quality
+- Train longer
+- Use larger, higher quality dataset
+- Adjust sampling parameters (temperature, top_p)
+- Check tokenizer was trained on similar data
+
+## Contact
+
+For questions or issues, please open a GitHub issue or contact the maintainers.

requirements.txt

@@ -0,0 +1,22 @@
+torch>=2.0.0
+torchvision>=0.15.0
+torchaudio>=2.0.0
+numpy>=1.24.0
+tqdm>=4.65.0
+requests>=2.28.0
+transformers>=4.30.0
+datasets>=2.12.0
+accelerate>=0.20.0
+sentencepiece>=0.1.99
+protobuf>=3.20.0
+wandb>=0.15.0
+tensorboard>=2.13.0
+matplotlib>=3.7.0
+scipy>=1.10.0
+scikit-learn>=1.2.0
+pandas>=2.0.0
+pyyaml>=6.0
+regex>=2023.0.0
+filelock>=3.12.0
+packaging>=23.0
+psutil>=5.9.0

tokenizer.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:862358e10e9e9f7c70f593dd3e8d2aa9da1ceca56947cff0545204d943c27baf
+size 71877

tokenizer.py

@@ -0,0 +1,396 @@
+"""
+VicAI Tokenizer
+Byte-Pair Encoding (BPE) tokenizer implementation.
+"""
+
+import json
+import pickle
+import re
+from collections import defaultdict
+from typing import Dict, List, Optional, Union
+
+
+class BPETokenizer:
+    """Byte-Pair Encoding Tokenizer."""
+    
+    def __init__(self, vocab_size: int = 32000):
+        self.vocab_size = vocab_size
+        self.vocab = {}
+        self.merges = []
+        self.special_tokens = {
+            '<pad>': 0,
+            '<unk>': 1,
+            '<s>': 2,
+            '</s>': 3,
+            '<mask>': 4,
+        }
+        self.pad_token_id = 0
+        self.unk_token_id = 1
+        self.bos_token_id = 2
+        self.eos_token_id = 3
+        self.mask_token_id = 4
+        
+    def _get_stats(self, vocab):
+        """Get counts of all symbol pairs."""
+        pairs = defaultdict(int)
+        for word, freq in vocab.items():
+            symbols = word.split()
+            for i in range(len(symbols) - 1):
+                pairs[(symbols[i], symbols[i + 1])] += freq
+        return pairs
+    
+    def _merge_vocab(self, pair, vocab):
+        """Merge all occurrences of pair in vocab."""
+        bigram = re.escape(' '.join(pair))
+        pattern = re.compile(r'(?<!\S)' + bigram + r'(?!\S)')
+        new_vocab = {}
+        for word in vocab:
+            new_word = pattern.sub(''.join(pair), word)
+            new_vocab[new_word] = vocab[word]
+        return new_vocab
+    
+    def _pre_tokenize(self, text: str) -> List[str]:
+        """Pre-tokenize text into words."""
+        # Simple whitespace and punctuation tokenization
+        pattern = r"'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"
+        return re.findall(pattern, text)
+    
+    def train(self, texts: List[str]):
+        """Train BPE on a list of texts."""
+        print(f"Training BPE tokenizer with vocab_size={self.vocab_size}")
+        
+        # Initialize vocabulary with special tokens
+        self.vocab = {token: i for token, i in self.special_tokens.items()}
+        
+        # Build word frequency dictionary
+        vocab = defaultdict(int)
+        for text in texts:
+            words = self._pre_tokenize(text.lower())
+            for word in words:
+                # End word with </w>
+                word = ' '.join(list(word)) + ' </w>'
+                vocab[tuple(word.split())] += 1
+        
+        # Convert to string format
+        vocab = {' '.join(k): v for k, v in vocab.items()}
+        
+        # Add individual characters to vocab
+        for word in vocab:
+            for char in word.split():
+                if char not in self.vocab:
+                    self.vocab[char] = len(self.vocab)
+        
+        # BPE training
+        num_merges = self.vocab_size - len(self.vocab)
+        for i in range(num_merges):
+            pairs = self._get_stats(vocab)
+            if not pairs:
+                break
+            
+            best = max(pairs, key=pairs.get)
+            vocab = self._merge_vocab(best, vocab)
+            self.merges.append(best)
+            
+            # Add merged token to vocab
+            merged_token = ''.join(best)
+            if merged_token not in self.vocab:
+                self.vocab[merged_token] = len(self.vocab)
+            
+            if (i + 1) % 1000 == 0:
+                print(f"  Completed {i + 1}/{num_merges} merges")
+        
+        print(f"Final vocabulary size: {len(self.vocab)}")
+    
+    def encode(self, text: str, add_special_tokens: bool = True) -> List[int]:
+        """Encode text to token IDs."""
+        words = self._pre_tokenize(text)
+        token_ids = []
+        
+        if add_special_tokens:
+            token_ids.append(self.bos_token_id)
+        
+        for word in words:
+            word = word.lower()
+            word_tokens = ' '.join(list(word)) + ' </w>'
+            
+            # Apply BPE merges
+            for merge in self.merges:
+                bigram = re.escape(' '.join(merge))
+                pattern = re.compile(r'(?<!\S)' + bigram + r'(?!\S)')
+                word_tokens = pattern.sub(''.join(merge), word_tokens)
+            
+            # Convert to IDs
+            for token in word_tokens.split():
+                token_ids.append(self.vocab.get(token, self.unk_token_id))
+        
+        if add_special_tokens:
+            token_ids.append(self.eos_token_id)
+        
+        return token_ids
+    
+    def decode(self, token_ids: List[int], skip_special_tokens: bool = True) -> str:
+        """Decode token IDs to text."""
+        # Build reverse vocab
+        reverse_vocab = {v: k for k, v in self.vocab.items()}
+        
+        tokens = []
+        for token_id in token_ids:
+            if token_id in self.special_tokens.values() and skip_special_tokens:
+                continue
+            tokens.append(reverse_vocab.get(token_id, '<unk>'))
+        
+        text = ''.join(tokens)
+        text = text.replace('</w>', ' ')
+        return text.strip()
+    
+    def save(self, path: str):
+        """Save tokenizer to file."""
+        data = {
+            'vocab': self.vocab,
+            'merges': self.merges,
+            'special_tokens': self.special_tokens,
+            'vocab_size': self.vocab_size,
+        }
+        with open(path, 'wb') as f:
+            pickle.dump(data, f)
+        print(f"Tokenizer saved to {path}")
+    
+    def load(self, path: str):
+        """Load tokenizer from file."""
+        with open(path, 'rb') as f:
+            data = pickle.load(f)
+        self.vocab = data['vocab']
+        self.merges = data['merges']
+        self.special_tokens = data['special_tokens']
+        self.vocab_size = data['vocab_size']
+        
+        self.pad_token_id = self.special_tokens['<pad>']
+        self.unk_token_id = self.special_tokens['<unk>']
+        self.bos_token_id = self.special_tokens['<s>']
+        self.eos_token_id = self.special_tokens['</s>']
+        self.mask_token_id = self.special_tokens['<mask>']
+        print(f"Tokenizer loaded from {path}")
+    
+    def batch_encode(
+        self,
+        texts: List[str],
+        max_length: int = 512,
+        padding: bool = True,
+        truncation: bool = True,
+    ) -> Dict[str, List]:
+        """Batch encode texts."""
+        encoded = [self.encode(text) for text in texts]
+        
+        if truncation:
+            encoded = [seq[:max_length] for seq in encoded]
+        
+        if padding:
+            max_len = min(max(len(seq) for seq in encoded), max_length)
+            attention_mask = []
+            for seq in encoded:
+                mask = [1] * len(seq) + [0] * (max_len - len(seq))
+                seq.extend([self.pad_token_id] * (max_len - len(seq)))
+                attention_mask.append(mask[:max_len])
+        else:
+            attention_mask = [[1] * len(seq) for seq in encoded]
+        
+        return {
+            'input_ids': encoded,
+            'attention_mask': attention_mask,
+        }
+    
+    def __len__(self):
+        return len(self.vocab)
+
+
+class ByteLevelBPETokenizer:
+    """Byte-level BPE tokenizer (similar to GPT-2/3)."""
+    
+    def __init__(self, vocab_size: int = 32000):
+        self.vocab_size = vocab_size
+        self.vocab = {}
+        self.merges = []
+        self.byte_encoder = {i: chr(i + 128) for i in range(256)}  # Shift to printable range
+        self.byte_decoder = {chr(i + 128): i for i in range(256)}
+        
+        self.special_tokens = {
+            '<|endoftext|>': 0,
+            '<|pad|>': 1,
+        }
+        self.eos_token_id = 0
+        self.pad_token_id = 1
+    
+    def _bytes_to_unicode(self, text: str) -> str:
+        """Convert string to byte-level representation."""
+        return ''.join(self.byte_encoder[b] for b in text.encode('utf-8'))
+    
+    def _unicode_to_bytes(self, text: str) -> str:
+        """Convert byte-level representation back to string."""
+        return bytes(self.byte_decoder[c] for c in text).decode('utf-8', errors='replace')
+    
+    def train(self, texts: List[str]):
+        """Train byte-level BPE."""
+        print(f"Training byte-level BPE tokenizer with vocab_size={self.vocab_size}")
+        
+        # Initialize vocab with special tokens and all bytes
+        self.vocab = {token: i for token, i in self.special_tokens.items()}
+        for i in range(256):
+            byte_char = self.byte_encoder[i]
+            if byte_char not in self.vocab:
+                self.vocab[byte_char] = len(self.vocab)
+        
+        # Build corpus as byte sequences
+        corpus = []
+        for text in texts:
+            byte_text = self._bytes_to_unicode(text)
+            corpus.extend(list(byte_text))
+        
+        # Get initial word frequencies
+        vocab = defaultdict(int)
+        for text in texts:
+            byte_text = self._bytes_to_unicode(text)
+            # Add end token
+            byte_text += '<|endoftext|>'
+            vocab[tuple(byte_text)] += 1
+        
+        # BPE training
+        num_merges = self.vocab_size - len(self.vocab)
+        
+        for i in range(num_merges):
+            pairs = self._get_stats(vocab)
+            if not pairs:
+                break
+            
+            best = max(pairs, key=pairs.get)
+            vocab = self._merge_vocab(best, vocab)
+            self.merges.append(best)
+            
+            merged = ''.join(best)
+            if merged not in self.vocab:
+                self.vocab[merged] = len(self.vocab)
+            
+            if (i + 1) % 1000 == 0:
+                print(f"  Completed {i + 1}/{num_merges} merges")
+        
+        print(f"Final vocabulary size: {len(self.vocab)}")
+    
+    def _get_stats(self, vocab):
+        pairs = defaultdict(int)
+        for word, freq in vocab.items():
+            symbols = list(word)
+            for i in range(len(symbols) - 1):
+                pairs[(symbols[i], symbols[i + 1])] += freq
+        return pairs
+    
+    def _merge_vocab(self, pair, vocab):
+        new_vocab = {}
+        bigram = pair[0] + pair[1]
+        for word in vocab:
+            new_word = []
+            i = 0
+            while i < len(word):
+                if i < len(word) - 1 and word[i] == pair[0] and word[i + 1] == pair[1]:
+                    new_word.append(bigram)
+                    i += 2
+                else:
+                    new_word.append(word[i])
+                    i += 1
+            new_vocab[tuple(new_word)] = vocab[word]
+        return new_vocab
+    
+    def encode(self, text: str, add_special_tokens: bool = True) -> List[int]:
+        """Encode text to token IDs."""
+        byte_text = self._bytes_to_unicode(text)
+        if add_special_tokens:
+            byte_text += '<|endoftext|>'
+        
+        # Apply merges
+        word = list(byte_text)
+        for merge in self.merges:
+            new_word = []
+            i = 0
+            while i < len(word):
+                if i < len(word) - 1 and word[i] == merge[0] and word[i + 1] == merge[1]:
+                    new_word.append(merge[0] + merge[1])
+                    i += 2
+                else:
+                    new_word.append(word[i])
+                    i += 1
+            word = new_word
+        
+        # Convert to IDs
+        return [self.vocab.get(token, self.special_tokens['<|pad|>']) for token in word]
+    
+    def decode(self, token_ids: List[int]) -> str:
+        """Decode token IDs to text."""
+        reverse_vocab = {v: k for k, v in self.vocab.items()}
+        text = ''.join(reverse_vocab.get(id, '') for id in token_ids)
+        text = text.replace('<|endoftext|>', '')
+        return self._unicode_to_bytes(text)
+    
+    def save(self, path: str):
+        """Save tokenizer to file."""
+        data = {
+            'vocab': self.vocab,
+            'merges': self.merges,
+            'special_tokens': self.special_tokens,
+            'vocab_size': self.vocab_size,
+            'byte_encoder': self.byte_encoder,
+            'byte_decoder': self.byte_decoder,
+        }
+        with open(path, 'wb') as f:
+            pickle.dump(data, f)
+        print(f"Tokenizer saved to {path}")
+    
+    def load(self, path: str):
+        """Load tokenizer from file."""
+        with open(path, 'rb') as f:
+            data = pickle.load(f)
+        self.vocab = data['vocab']
+        self.merges = data['merges']
+        self.special_tokens = data['special_tokens']
+        self.vocab_size = data['vocab_size']
+        self.byte_encoder = data.get('byte_encoder', self.byte_encoder)
+        self.byte_decoder = data.get('byte_decoder', self.byte_decoder)
+        
+        # Ensure all special tokens exist
+        if '<|endoftext|>' not in self.special_tokens:
+            self.special_tokens['<|endoftext|>'] = 0
+        if '<|pad|>' not in self.special_tokens:
+            self.special_tokens['<|pad|>'] = 1
+        
+        self.eos_token_id = self.special_tokens.get('<|endoftext|>', 0)
+        self.pad_token_id = self.special_tokens.get('<|pad|>', 1)
+        print(f"Tokenizer loaded from {path}")
+    
+    def __len__(self):
+        return len(self.vocab)
+
+
+def create_and_train_tokenizer(texts: List[str], vocab_size: int = 32000, output_path: str = "tokenizer.pkl"):
+    """Create and train a tokenizer on the given texts."""
+    tokenizer = ByteLevelBPETokenizer(vocab_size=vocab_size)
+    tokenizer.train(texts)
+    tokenizer.save(output_path)
+    return tokenizer
+
+
+if __name__ == "__main__":
+    # Test tokenizer
+    sample_texts = [
+        "Hello, world! This is a test.",
+        "The quick brown fox jumps over the lazy dog.",
+        "Machine learning is fascinating.",
+        "Artificial intelligence will change the world.",
+    ]
+    
+    tokenizer = BPETokenizer(vocab_size=1000)
+    tokenizer.train(sample_texts)
+    
+    test_text = "Hello world!"
+    encoded = tokenizer.encode(test_text)
+    decoded = tokenizer.decode(encoded)
+    
+    print(f"\nOriginal: {test_text}")
+    print(f"Encoded: {encoded}")
+    print(f"Decoded: {decoded}")

train.py

@@ -0,0 +1,402 @@
+"""
+VicAI Training Script
+Distributed training with FSDP/DDP support.
+"""
+
+import argparse
+import os
+import time
+from contextlib import nullcontext
+from pathlib import Path
+
+import torch
+import torch.distributed as dist
+from torch.distributed.fsdp import FullyShardedDataParallel as FSDP
+from torch.distributed.fsdp.wrap import transformer_auto_wrap_policy
+from torch.nn.parallel import DistributedDataParallel as DDP
+from torch.utils.data import DataLoader
+from torch.utils.data.distributed import DistributedSampler
+
+from model import VicAIModel, VicAIConfig, create_vicai_5b
+from tokenizer import ByteLevelBPETokenizer, BPETokenizer
+from dataset import (
+    WikipediaDataset,
+    TextFileDataset,
+    MixedDataset,
+    create_sample_corpus,
+)
+from utils import (
+    get_logger,
+    load_checkpoint,
+    save_checkpoint,
+    get_lr_scheduler,
+    estimate_loss,
+    configure_optimizers,
+)
+
+
+def setup_distributed():
+    """Initialize distributed training."""
+    if 'RANK' in os.environ and 'WORLD_SIZE' in os.environ:
+        rank = int(os.environ['RANK'])
+        world_size = int(os.environ['WORLD_SIZE'])
+        local_rank = int(os.environ.get('LOCAL_RANK', 0))
+    else:
+        rank = 0
+        world_size = 1
+        local_rank = 0
+    
+    if world_size > 1:
+        dist.init_process_group(backend='nccl', rank=rank, world_size=world_size)
+        torch.cuda.set_device(local_rank)
+    
+    return rank, world_size, local_rank
+
+
+def cleanup_distributed():
+    """Cleanup distributed training."""
+    if dist.is_initialized():
+        dist.destroy_process_group()
+
+
+def get_data_loader(dataset, batch_size, world_size, rank, shuffle=True):
+    """Create distributed data loader."""
+    if world_size > 1:
+        sampler = DistributedSampler(
+            dataset,
+            num_replicas=world_size,
+            rank=rank,
+            shuffle=shuffle,
+        )
+    else:
+        sampler = None
+    
+    loader = DataLoader(
+        dataset,
+        batch_size=batch_size,
+        sampler=sampler,
+        num_workers=4,
+        pin_memory=True,
+        drop_last=True,
+    )
+    
+    return loader, sampler
+
+
+def train_step(model, batch, optimizer, scaler, device, use_amp):
+    """Single training step."""
+    model.train()
+    
+    input_ids = batch['input_ids'].to(device)
+    labels = batch['labels'].to(device)
+    
+    optimizer.zero_grad()
+    
+    with torch.cuda.amp.autocast(enabled=use_amp):
+        outputs = model(input_ids, targets=labels)
+        loss = outputs['loss']
+    
+    if use_amp:
+        scaler.scale(loss).backward()
+        scaler.step(optimizer)
+        scaler.update()
+    else:
+        loss.backward()
+        optimizer.step()
+    
+    return loss.item()
+
+
+def train(
+    model,
+    train_loader,
+    val_loader,
+    optimizer,
+    lr_scheduler,
+    scaler,
+    device,
+    args,
+    logger,
+):
+    """Main training loop."""
+    best_val_loss = float('inf')
+    step = 0
+    
+    model.train()
+    train_iterator = iter(train_loader)
+    
+    for epoch in range(args.max_epochs):
+        if hasattr(train_loader.sampler, 'set_epoch'):
+            train_loader.sampler.set_epoch(epoch)
+        
+        epoch_start_time = time.time()
+        
+        while step < args.max_steps:
+            try:
+                batch = next(train_iterator)
+            except StopIteration:
+                train_iterator = iter(train_loader)
+                batch = next(train_iterator)
+            
+            # Training step
+            loss = train_step(model, batch, optimizer, scaler, device, args.use_amp)
+            lr_scheduler.step()
+            
+            step += 1
+            
+            # Logging
+            if step % args.log_interval == 0 and args.rank == 0:
+                lr = optimizer.param_groups[0]['lr']
+                logger.info(
+                    f"Step {step}/{args.max_steps} | "
+                    f"Loss: {loss:.4f} | LR: {lr:.2e}"
+                )
+            
+            # Evaluation
+            if step % args.eval_interval == 0:
+                val_loss = evaluate(model, val_loader, device, args.use_amp)
+                
+                if args.rank == 0:
+                    logger.info(f"Validation loss: {val_loss:.4f}")
+                    
+                    # Save best model
+                    if val_loss < best_val_loss:
+                        best_val_loss = val_loss
+                        save_checkpoint(
+                            model,
+                            optimizer,
+                            scaler,
+                            step,
+                            val_loss,
+                            args.output_dir / 'best_model.pt',
+                        )
+                        logger.info(f"Saved best model with loss {val_loss:.4f}")
+                
+                model.train()
+            
+            # Regular checkpointing
+            if step % args.save_interval == 0 and args.rank == 0:
+                save_checkpoint(
+                    model,
+                    optimizer,
+                    scaler,
+                    step,
+                    loss,
+                    args.output_dir / f'checkpoint_step_{step}.pt',
+                )
+                logger.info(f"Saved checkpoint at step {step}")
+            
+            if step >= args.max_steps:
+                break
+        
+        epoch_time = time.time() - epoch_start_time
+        if args.rank == 0:
+            logger.info(f"Epoch {epoch + 1} completed in {epoch_time:.2f}s")
+    
+    return step
+
+
+def evaluate(model, data_loader, device, use_amp):
+    """Evaluate model on validation set."""
+    model.eval()
+    total_loss = 0
+    num_batches = 0
+    
+    with torch.no_grad():
+        for batch in data_loader:
+            input_ids = batch['input_ids'].to(device)
+            labels = batch['labels'].to(device)
+            
+            with torch.cuda.amp.autocast(enabled=use_amp):
+                outputs = model(input_ids, targets=labels)
+                loss = outputs['loss']
+            
+            total_loss += loss.item()
+            num_batches += 1
+            
+            if num_batches >= 100:  # Limit eval batches
+                break
+    
+    # Average across all processes
+    avg_loss = total_loss / num_batches
+    
+    if dist.is_initialized():
+        loss_tensor = torch.tensor([avg_loss], device=device)
+        dist.all_reduce(loss_tensor, op=dist.ReduceOp.AVG)
+        avg_loss = loss_tensor.item()
+    
+    return avg_loss
+
+
+def main():
+    parser = argparse.ArgumentParser(description='Train VicAI')
+    
+    # Model args
+    parser.add_argument('--vocab-size', type=int, default=32000)
+    parser.add_argument('--dim', type=int, default=4096)
+    parser.add_argument('--n-layers', type=int, default=32)
+    parser.add_argument('--n-heads', type=int, default=32)
+    parser.add_argument('--n-kv-heads', type=int, default=8)
+    parser.add_argument('--hidden-dim', type=int, default=14336)
+    
+    # Training args
+    parser.add_argument('--batch-size', type=int, default=4)
+    parser.add_argument('--max-seq-len', type=int, default=2048)
+    parser.add_argument('--max-steps', type=int, default=100000)
+    parser.add_argument('--max-epochs', type=int, default=10)
+    parser.add_argument('--learning-rate', type=float, default=3e-4)
+    parser.add_argument('--min-lr', type=float, default=3e-5)
+    parser.add_argument('--warmup-steps', type=int, default=2000)
+    parser.add_argument('--weight-decay', type=float, default=0.1)
+    parser.add_argument('--grad-clip', type=float, default=1.0)
+    parser.add_argument('--beta1', type=float, default=0.9)
+    parser.add_argument('--beta2', type=float, default=0.95)
+    
+    # Data args
+    parser.add_argument('--train-data', type=str, default='data/train.txt')
+    parser.add_argument('--val-data', type=str, default='data/val.txt')
+    parser.add_argument('--tokenizer-path', type=str, default='tokenizer.pkl')
+    
+    # System args
+    parser.add_argument('--output-dir', type=str, default='checkpoints')
+    parser.add_argument('--resume', type=str, default=None)
+    parser.add_argument('--eval-interval', type=int, default=1000)
+    parser.add_argument('--save-interval', type=int, default=5000)
+    parser.add_argument('--log-interval', type=int, default=100)
+    parser.add_argument('--use-amp', action='store_true', default=True)
+    parser.add_argument('--use-fsdp', action='store_true', default=False)
+    parser.add_argument('--compile', action='store_true', default=False)
+    
+    args = parser.parse_args()
+    
+    # Setup
+    args.rank, args.world_size, args.local_rank = setup_distributed()
+    args.is_distributed = args.world_size > 1
+    
+    # Create output directory
+    args.output_dir = Path(args.output_dir)
+    if args.rank == 0:
+        args.output_dir.mkdir(parents=True, exist_ok=True)
+    
+    # Logger
+    logger = get_logger('vicai_train', args.output_dir / 'train.log' if args.rank == 0 else None)
+    
+    if args.rank == 0:
+        logger.info(f"Starting VicAI training with {args.world_size} GPUs")
+        logger.info(f"Arguments: {args}")
+    
+    # Device
+    device = torch.device(f'cuda:{args.local_rank}' if torch.cuda.is_available() else 'cpu')
+    
+    # Load tokenizer
+    if os.path.exists(args.tokenizer_path):
+        logger.info(f"Loading tokenizer from {args.tokenizer_path}")
+        tokenizer = ByteLevelBPETokenizer()
+        tokenizer.load(args.tokenizer_path)
+    else:
+        logger.warning(f"Tokenizer not found at {args.tokenizer_path}, creating default")
+        tokenizer = ByteLevelBPETokenizer(vocab_size=args.vocab_size)
+        # Train on sample data
+        if args.rank == 0:
+            sample_file = create_sample_corpus(num_articles=100)
+            with open(sample_file, 'r') as f:
+                texts = f.read().split('<|endoftext|>')
+            tokenizer.train([t for t in texts if t.strip()])
+            tokenizer.save(args.tokenizer_path)
+        
+        if args.is_distributed:
+            dist.barrier()
+        
+        if args.rank != 0:
+            tokenizer.load(args.tokenizer_path)
+    
+    # Create model
+    logger.info("Creating model...")
+    config = VicAIConfig(
+        vocab_size=len(tokenizer),
+        dim=args.dim,
+        n_layers=args.n_layers,
+        n_heads=args.n_heads,
+        n_kv_heads=args.n_kv_heads,
+        hidden_dim=args.hidden_dim,
+        max_seq_len=args.max_seq_len,
+        dropout=0.0,
+    )
+    
+    if args.rank == 0:
+        logger.info(f"Model config: {config.__dict__}")
+        logger.info(f"Model parameters: ~{config.num_parameters / 1e9:.2f}B")
+    
+    model = VicAIModel(config)
+    
+    if args.use_fsdp and args.is_distributed:
+        model = FSDP(model, device_id=device)
+    elif args.is_distributed:
+        model = DDP(model, device_ids=[args.local_rank])
+    else:
+        model = model.to(device)
+    
+    if args.compile and hasattr(torch, 'compile'):
+        logger.info("Compiling model...")
+        model = torch.compile(model)
+    
+    # Create datasets
+    logger.info("Creating datasets...")
+    
+    if os.path.exists(args.train_data):
+        train_dataset = TextFileDataset(args.train_data, tokenizer, args.max_seq_len)
+        val_dataset = TextFileDataset(args.val_data, tokenizer, args.max_seq_len) if os.path.exists(args.val_data) else train_dataset
+    else:
+        logger.warning("Training data not found, using Wikipedia streaming dataset")
+        train_dataset = WikipediaDataset(tokenizer, max_length=args.max_seq_len)
+        val_dataset = WikipediaDataset(tokenizer, max_length=args.max_seq_len)
+    
+    train_loader, train_sampler = get_data_loader(train_dataset, args.batch_size, args.world_size, args.rank)
+    val_loader, _ = get_data_loader(val_dataset, args.batch_size, args.world_size, args.rank, shuffle=False)
+    
+    # Optimizer
+    optimizer = configure_optimizers(model, args)
+    
+    # Learning rate scheduler
+    lr_scheduler = get_lr_scheduler(optimizer, args)
+    
+    # Gradient scaler for AMP
+    scaler = torch.cuda.amp.GradScaler(enabled=args.use_amp)
+    
+    # Resume from checkpoint
+    start_step = 0
+    if args.resume:
+        logger.info(f"Resuming from {args.resume}")
+        start_step = load_checkpoint(model, optimizer, scaler, args.resume, device)
+    
+    # Training
+    logger.info("Starting training...")
+    final_step = train(
+        model,
+        train_loader,
+        val_loader,
+        optimizer,
+        lr_scheduler,
+        scaler,
+        device,
+        args,
+        logger,
+    )
+    
+    # Save final model
+    if args.rank == 0:
+        save_checkpoint(
+            model,
+            optimizer,
+            scaler,
+            final_step,
+            0.0,
+            args.output_dir / 'final_model.pt',
+        )
+        logger.info("Training completed!")
+    
+    cleanup_distributed()
+
+
+if __name__ == '__main__':
+    main()

utils.py

@@ -0,0 +1,359 @@
+"""
+VicAI Utilities
+Helper functions for training and evaluation.
+"""
+
+import json
+import logging
+import math
+import os
+import sys
+from pathlib import Path
+from typing import Dict, Optional
+
+import torch
+import torch.distributed as dist
+from torch.optim import AdamW
+
+
+def get_logger(name: str, log_file: Optional[Path] = None) -> logging.Logger:
+    """Create a logger with file and console handlers."""
+    logger = logging.getLogger(name)
+    logger.setLevel(logging.INFO)
+    
+    # Clear existing handlers
+    logger.handlers = []
+    
+    # Formatter
+    formatter = logging.Formatter(
+        '%(asctime)s - %(name)s - %(levelname)s - %(message)s',
+        datefmt='%Y-%m-%d %H:%M:%S'
+    )
+    
+    # Console handler
+    console_handler = logging.StreamHandler(sys.stdout)
+    console_handler.setLevel(logging.INFO)
+    console_handler.setFormatter(formatter)
+    logger.addHandler(console_handler)
+    
+    # File handler
+    if log_file:
+        log_file.parent.mkdir(parents=True, exist_ok=True)
+        file_handler = logging.FileHandler(log_file)
+        file_handler.setLevel(logging.INFO)
+        file_handler.setFormatter(formatter)
+        logger.addHandler(file_handler)
+    
+    return logger
+
+
+def save_checkpoint(
+    model,
+    optimizer,
+    scaler,
+    step: int,
+    loss: float,
+    path: Path,
+):
+    """Save model checkpoint."""
+    path.parent.mkdir(parents=True, exist_ok=True)
+    
+    # Unwrap model if using DDP/FSDP
+    state_dict = model.state_dict()
+    if hasattr(model, 'module'):
+        state_dict = model.module.state_dict()
+    
+    checkpoint = {
+        'model': state_dict,
+        'optimizer': optimizer.state_dict(),
+        'scaler': scaler.state_dict() if scaler else None,
+        'step': step,
+        'loss': loss,
+    }
+    
+    torch.save(checkpoint, path)
+
+
+def load_checkpoint(
+    model,
+    optimizer,
+    scaler,
+    path: str,
+    device,
+):
+    """Load model checkpoint."""
+    checkpoint = torch.load(path, map_location=device)
+    
+    # Handle both wrapped and unwrapped models
+    state_dict = checkpoint['model']
+    if hasattr(model, 'module'):
+        model.module.load_state_dict(state_dict)
+    else:
+        model.load_state_dict(state_dict)
+    
+    optimizer.load_state_dict(checkpoint['optimizer'])
+    
+    if scaler and checkpoint.get('scaler'):
+        scaler.load_state_dict(checkpoint['scaler'])
+    
+    return checkpoint.get('step', 0)
+
+
+def get_lr_scheduler(optimizer, args):
+    """Create learning rate scheduler with warmup and cosine decay."""
+    
+    def lr_lambda(current_step):
+        if current_step < args.warmup_steps:
+            # Linear warmup
+            return current_step / args.warmup_steps
+        else:
+            # Cosine decay
+            progress = (current_step - args.warmup_steps) / (args.max_steps - args.warmup_steps)
+            progress = min(1.0, progress)
+            cosine_decay = 0.5 * (1 + math.cos(math.pi * progress))
+            return args.min_lr / args.learning_rate + (1 - args.min_lr / args.learning_rate) * cosine_decay
+    
+    from torch.optim.lr_scheduler import LambdaLR
+    return LambdaLR(optimizer, lr_lambda)
+
+
+def configure_optimizers(model, args):
+    """Configure optimizer with weight decay."""
+    # Separate parameters that should and shouldn't have weight decay
+    decay_params = []
+    no_decay_params = []
+    
+    for name, param in model.named_parameters():
+        if not param.requires_grad:
+            continue
+        
+        # Don't apply weight decay to bias and normalization parameters
+        if 'bias' in name or 'norm' in name or 'embedding' in name:
+            no_decay_params.append(param)
+        else:
+            decay_params.append(param)
+    
+    param_groups = [
+        {'params': decay_params, 'weight_decay': args.weight_decay},
+        {'params': no_decay_params, 'weight_decay': 0.0},
+    ]
+    
+    optimizer = AdamW(
+        param_groups,
+        lr=args.learning_rate,
+        betas=(args.beta1, args.beta2),
+        eps=1e-8,
+    )
+    
+    return optimizer
+
+
+def estimate_loss(model, data_loader, device, num_batches=10):
+    """Estimate loss on a data loader."""
+    model.eval()
+    total_loss = 0
+    
+    with torch.no_grad():
+        for i, batch in enumerate(data_loader):
+            if i >= num_batches:
+                break
+            
+            input_ids = batch['input_ids'].to(device)
+            labels = batch['labels'].to(device)
+            
+            outputs = model(input_ids, targets=labels)
+            total_loss += outputs['loss'].item()
+    
+    model.train()
+    return total_loss / num_batches
+
+
+def get_grad_norm(model):
+    """Calculate gradient norm."""
+    total_norm = 0.0
+    for p in model.parameters():
+        if p.grad is not None:
+            total_norm += p.grad.data.norm(2).item() ** 2
+    return total_norm ** 0.5
+
+
+def clip_gradients(model, max_norm):
+    """Clip gradients by norm."""
+    torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm)
+
+
+class AverageMeter:
+    """Track running average of metrics."""
+    
+    def __init__(self):
+        self.reset()
+    
+    def reset(self):
+        self.val = 0
+        self.avg = 0
+        self.sum = 0
+        self.count = 0
+    
+    def update(self, val, n=1):
+        self.val = val
+        self.sum += val * n
+        self.count += n
+        self.avg = self.sum / self.count
+
+
+class EarlyStopping:
+    """Early stopping to prevent overfitting."""
+    
+    def __init__(self, patience=5, min_delta=0.0):
+        self.patience = patience
+        self.min_delta = min_delta
+        self.counter = 0
+        self.best_loss = None
+        self.early_stop = False
+    
+    def __call__(self, val_loss):
+        if self.best_loss is None:
+            self.best_loss = val_loss
+        elif val_loss > self.best_loss - self.min_delta:
+            self.counter += 1
+            if self.counter >= self.patience:
+                self.early_stop = True
+        else:
+            self.best_loss = val_loss
+            self.counter = 0
+        
+        return self.early_stop
+
+
+def count_parameters(model):
+    """Count trainable parameters."""
+    return sum(p.numel() for p in model.parameters() if p.requires_grad)
+
+
+def format_num_parameters(num_params):
+    """Format parameter count for display."""
+    if num_params >= 1e9:
+        return f"{num_params / 1e9:.2f}B"
+    elif num_params >= 1e6:
+        return f"{num_params / 1e6:.2f}M"
+    elif num_params >= 1e3:
+        return f"{num_params / 1e3:.2f}K"
+    else:
+        return str(num_params)
+
+
+def get_device_info():
+    """Get information about available GPUs."""
+    if not torch.cuda.is_available():
+        return "No CUDA available"
+    
+    info = []
+    for i in range(torch.cuda.device_count()):
+        props = torch.cuda.get_device_properties(i)
+        info.append(
+            f"GPU {i}: {props.name} ({props.total_memory / 1e9:.1f} GB)"
+        )
+    
+    return "\n".join(info)
+
+
+def print_model_summary(model):
+    """Print a summary of the model architecture."""
+    print("\n" + "=" * 60)
+    print("MODEL SUMMARY")
+    print("=" * 60)
+    
+    total_params = 0
+    trainable_params = 0
+    
+    print(f"\n{'Layer':<40} {'Parameters':>15} {'Trainable':>10}")
+    print("-" * 70)
+    
+    for name, param in model.named_parameters():
+        num_params = param.numel()
+        total_params += num_params
+        if param.requires_grad:
+            trainable_params += num_params
+            trainable = "Yes"
+        else:
+            trainable = "No"
+        
+        print(f"{name:<40} {num_params:>15,} {trainable:>10}")
+    
+    print("-" * 70)
+    print(f"{'Total':<40} {total_params:>15,}")
+    print(f"{'Trainable':<40} {trainable_params:>15,}")
+    print(f"{'Non-trainable':<40} {total_params - trainable_params:>15,}")
+    print("=" * 60 + "\n")
+
+
+def save_training_config(args, output_path: Path):
+    """Save training configuration to JSON."""
+    config = vars(args)
+    with open(output_path, 'w') as f:
+        json.dump(config, f, indent=2)
+
+
+def load_training_config(config_path: Path):
+    """Load training configuration from JSON."""
+    with open(config_path, 'r') as f:
+        return json.load(f)
+
+
+def all_reduce_dict(data: Dict, device):
+    """All reduce dictionary values across processes."""
+    if not dist.is_initialized():
+        return data
+    
+    reduced_data = {}
+    for key, value in data.items():
+        if isinstance(value, (int, float)):
+            tensor = torch.tensor([value], device=device)
+            dist.all_reduce(tensor, op=dist.ReduceOp.AVG)
+            reduced_data[key] = tensor.item()
+        else:
+            reduced_data[key] = value
+    
+    return reduced_data
+
+
+def set_seed(seed: int):
+    """Set random seed for reproducibility."""
+    import random
+    import numpy as np
+    
+    random.seed(seed)
+    np.random.seed(seed)
+    torch.manual_seed(seed)
+    torch.cuda.manual_seed_all(seed)
+    
+    # For deterministic operations (may be slower)
+    torch.backends.cudnn.deterministic = True
+    torch.backends.cudnn.benchmark = False
+
+
+def get_memory_usage():
+    """Get current memory usage."""
+    if torch.cuda.is_available():
+        allocated = torch.cuda.memory_allocated() / 1e9
+        reserved = torch.cuda.memory_reserved() / 1e9
+        max_allocated = torch.cuda.max_memory_allocated() / 1e9
+        return {
+            'allocated_gb': allocated,
+            'reserved_gb': reserved,
+            'max_allocated_gb': max_allocated,
+        }
+    return {'allocated_gb': 0, 'reserved_gb': 0, 'max_allocated_gb': 0}
+
+
+if __name__ == "__main__":
+    # Test utilities
+    logger = get_logger("test")
+    logger.info("Testing logger")
+    
+    print(get_device_info())
+    
+    meter = AverageMeter()
+    for i in range(10):
+        meter.update(i)
+    print(f"Average: {meter.avg}")

vocab.json

@@ -0,0 +1,2002 @@
+{
+  "<|endoftext|>": 0,
+  "<|pad|>": 1,
+  "€": 2,
+  "": 3,
+  "‚": 4,
+  "ƒ": 5,
+  "„": 6,
+  "…": 7,
+  "†": 8,
+  "‡": 9,
+  "ˆ": 10,
+  "‰": 11,
+  "Š": 12,
+  "‹": 13,
+  "Œ": 14,
+  "": 15,
+  "Ž": 16,
+  "": 17,
+  "": 18,
+  "‘": 19,
+  "’": 20,
+  "“": 21,
+  "”": 22,
+  "•": 23,
+  "–": 24,
+  "—": 25,
+  "˜": 26,
+  "™": 27,
+  "š": 28,
+  "›": 29,
+  "œ": 30,
+  "": 31,
+  "ž": 32,
+  "Ÿ": 33,
+  " ": 34,
+  "¡": 35,
+  "¢": 36,
+  "£": 37,
+  "¤": 38,
+  "¥": 39,
+  "¦": 40,
+  "§": 41,
+  "¨": 42,
+  "©": 43,
+  "ª": 44,
+  "«": 45,
+  "¬": 46,
+  "­": 47,
+  "®": 48,
+  "¯": 49,
+  "°": 50,
+  "±": 51,
+  "²": 52,
+  "³": 53,
+  "´": 54,
+  "µ": 55,
+  "¶": 56,
+  "·": 57,
+  "¸": 58,
+  "¹": 59,
+  "º": 60,
+  "»": 61,
+  "¼": 62,
+  "½": 63,
+  "¾": 64,
+  "¿": 65,
+  "À": 66,
+  "Á": 67,
+  "Â": 68,
+  "Ã": 69,
+  "Ä": 70,
+  "Å": 71,
+  "Æ": 72,
+  "Ç": 73,
+  "È": 74,
+  "É": 75,
+  "Ê": 76,
+  "Ë": 77,
+  "Ì": 78,
+  "Í": 79,
+  "Î": 80,
+  "Ï": 81,
+  "Ð": 82,
+  "Ñ": 83,
+  "Ò": 84,
+  "Ó": 85,
+  "Ô": 86,
+  "Õ": 87,
+  "Ö": 88,
+  "×": 89,
+  "Ø": 90,
+  "Ù": 91,
+  "Ú": 92,
+  "Û": 93,
+  "Ü": 94,
+  "Ý": 95,
+  "Þ": 96,
+  "ß": 97,
+  "à": 98,
+  "á": 99,
+  "â": 100,
+  "ã": 101,
+  "ä": 102,
+  "å": 103,
+  "æ": 104,
+  "ç": 105,
+  "è": 106,
+  "é": 107,
+  "ê": 108,
+  "ë": 109,
+  "ì": 110,
+  "í": 111,
+  "î": 112,
+  "ï": 113,
+  "ð": 114,
+  "ñ": 115,
+  "ò": 116,
+  "ó": 117,
+  "ô": 118,
+  "õ": 119,
+  "ö": 120,
+  "÷": 121,
+  "ø": 122,
+  "ù": 123,
+  "ú": 124,
+  "û": 125,
+  "ü": 126,
+  "ý": 127,
+  "þ": 128,
+  "ÿ": 129,
+  "Ā": 130,
+  "ā": 131,
+  "Ă": 132,
+  "ă": 133,
+  "Ą": 134,
+  "ą": 135,
+  "Ć": 136,
+  "ć": 137,
+  "Ĉ": 138,
+  "ĉ": 139,
+  "Ċ": 140,
+  "ċ": 141,
+  "Č": 142,
+  "č": 143,
+  "Ď": 144,
+  "ď": 145,
+  "Đ": 146,
+  "đ": 147,
+  "Ē": 148,
+  "ē": 149,
+  "Ĕ": 150,
+  "ĕ": 151,
+  "Ė": 152,
+  "ė": 153,
+  "Ę": 154,
+  "ę": 155,
+  "Ě": 156,
+  "ě": 157,
+  "Ĝ": 158,
+  "ĝ": 159,
+  "Ğ": 160,
+  "ğ": 161,
+  "Ġ": 162,
+  "ġ": 163,
+  "Ģ": 164,
+  "ģ": 165,
+  "Ĥ": 166,
+  "ĥ": 167,
+  "Ħ": 168,
+  "ħ": 169,
+  "Ĩ": 170,
+  "ĩ": 171,
+  "Ī": 172,
+  "ī": 173,
+  "Ĭ": 174,
+  "ĭ": 175,
+  "Į": 176,
+  "į": 177,
+  "İ": 178,
+  "ı": 179,
+  "IJ": 180,
+  "ij": 181,
+  "Ĵ": 182,
+  "ĵ": 183,
+  "Ķ": 184,
+  "ķ": 185,
+  "ĸ": 186,
+  "Ĺ": 187,
+  "ĺ": 188,
+  "Ļ": 189,
+  "ļ": 190,
+  "Ľ": 191,
+  "ľ": 192,
+  "Ŀ": 193,
+  "ŀ": 194,
+  "Ł": 195,
+  "ł": 196,
+  "Ń": 197,
+  "ń": 198,
+  "Ņ": 199,
+  "ņ": 200,
+  "Ň": 201,
+  "ň": 202,
+  "ʼn": 203,
+  "Ŋ": 204,
+  "ŋ": 205,
+  "Ō": 206,
+  "ō": 207,
+  "Ŏ": 208,
+  "ŏ": 209,
+  "Ő": 210,
+  "ő": 211,
+  "Œ": 212,
+  "œ": 213,
+  "Ŕ": 214,
+  "ŕ": 215,
+  "Ŗ": 216,
+  "ŗ": 217,
+  "Ř": 218,
+  "ř": 219,
+  "Ś": 220,
+  "ś": 221,
+  "Ŝ": 222,
+  "ŝ": 223,
+  "Ş": 224,
+  "ş": 225,
+  "Š": 226,
+  "š": 227,
+  "Ţ": 228,
+  "ţ": 229,
+  "Ť": 230,
+  "ť": 231,
+  "Ŧ": 232,
+  "ŧ": 233,
+  "Ũ": 234,
+  "ũ": 235,
+  "Ū": 236,
+  "ū": 237,
+  "Ŭ": 238,
+  "ŭ": 239,
+  "Ů": 240,
+  "ů": 241,
+  "Ű": 242,
+  "ű": 243,
+  "Ų": 244,
+  "ų": 245,
+  "Ŵ": 246,
+  "ŵ": 247,
+  "Ŷ": 248,
+  "ŷ": 249,
+  "Ÿ": 250,
+  "Ź": 251,
+  "ź": 252,
+  "Ż": 253,
+  "ż": 254,
+  "Ž": 255,
+  "ž": 256,
+  "ſ": 257,
+  "  ": 258,
+  "    ": 259,
+  "        ": 260,
+  "Š        ": 261,
+  "éî": 262,
+  "åî": 263,
+  "Š    ": 264,
+  "óå": 265,
+  "åò": 266,
+  "½ ": 267,
+  "áô": 268,
+  "ôï": 269,
+  " ½ ": 270,
+  "¬ ": 271,
+  "áò": 272,
+  "äå": 273,
+  "ïò": 274,
+  "ìæ": 275,
+  "óåìæ": 276,
+  "Š            ": 277,
+  "ôå": 278,
+  "¢¢": 279,
+  "òå": 280,
+  "áì": 281,
+  "óåìæ®": 282,
+  "ëåî": 283,
+  "ïî": 284,
+  "ãè": 285,
+  "óô": 286,
+  "ôïëåî": 287,
+  "ìï": 288,
+  "éú": 289,
+  "ôé": 290,
+  "©Š        ": 291,
+  "ìå": 292,
+  "æ ": 293,
+  "ô¨": 294,
+  "º ": 295,
+  "©Š    ": 296,
+  "áä": 297,
+  "ó®": 298,
+  "äé": 299,
+  "ºŠ        ": 300,
+  "õô": 301,
+  "íð": 302,
+  "ô ": 303,
+  "áòç": 304,
+  "òá": 305,
+  "ðò": 306,
+  "áâ": 307,
+  "ïäå": 308,
+  "¬Š        ": 309,
+  "¢¢¢": 310,
+  "éä": 311,
+  "ó ": 312,
+  "íá": 313,
+  "éîç": 314,
+  "ðáò": 315,
+  "çå": 316,
+  "ðå": 317,
+  "éúåò": 318,
+  "áôá": 319,
+  "ôåø": 320,
+  "ãå": 321,
+  "ó ½ ": 322,
+  "áí": 323,
+  "ìåî": 324,
+  "õí": 325,
+  "Š��": 326,
+  "áôå": 327,
+  "éæ ": 328,
+  "ïã": 329,
+  "éô": 330,
+  "öïã": 331,
+  "öïãáâ": 332,
+  "£ ": 333,
+  "ßéä": 334,
+  "ôåøô": 335,
+  "ïäåì": 336,
+  "æïò": 337,
+  "éîô": 338,
+  "éîô¨": 339,
+  "äß": 340,
+  "äáôá": 341,
+  "áòçó®": 342,
+  "ðõô": 343,
+  "òãè": 344,
+  "©ºŠ        ": 345,
+  "©Š    Š    ": 346,
+  "ôïòãè": 347,
+  "æé": 348,
+  "íïäåì": 349,
+  "ßôïëåî": 350,
+  "ðòéîô¨": 351,
+  "òï": 352,
+  "ôïòãè®": 353,
+  "äåæ ": 354,
+  "íáø": 355,
+  "éî ": 356,
+  "õå": 357,
+  "åì": 358,
+  "ôïëåîéúåò": 359,
+  "õò": 360,
+  "öáì": 361,
+  "éúå": 362,
+  "ä ": 363,
+  "ßó": 364,
+  "ðáòáí": 365,
+  "ßß": 366,
+  "æïò ": 367,
+  "© ": 368,
+  "ïõô": 369,
+  "äéí": 370,
+  "ºŠ            ": 371,
+  "óôò": 372,
+  "ôéïî": 373,
+  "©Š            ": 374,
+  "öé": 375,
+  "æ¢": 376,
+  "óåô": 377,
+  "Š        óåìæ®": 378,
+  "ôòá": 379,
+  "õì": 380,
+  "õî": 381,
+  "áî": 382,
+  "íáøß": 383,
+  "÷ïò": 384,
+  "ìïç": 385,
+  "ôåò": 386,
+  "î ": 387,
+  "§¬ ": 388,
+  "òåô": 389,
+  "òåôõò": 390,
+  "§º ": 391,
+  "òåôõòî ": 392,
+  "¬Š    ": 393,
+  "ðá": 394,
+  "ðòï": 395,
+  "éîç ": 396,
+  "Š    Š    ": 397,
+  " éî ": 398,
+  "ôòáéî": 399,
+  "òáî": 400,
+  "öéãå": 401,
+  "            ": 402,
+  "èå": 403,
+  "ðåî": 404,
+  "ôù": 405,
+  "ðï": 406,
+  "ßë": 407,
+  "ðòéîô¨æ¢": 408,
+  "ðõôßéä": 409,
+  "äåöéãå": 410,
+  "ãï": 411,
+  "áôè": 412,
+  "öå": 413,
+  "ãïî": 414,
+  "éíð": 415,
+  "ó¨": 416,
+  "ìïó": 417,
+  "ºŠ    ": 418,
+  "©Š        Š        ": 419,
+  "îõí": 420,
+  "óÛ": 421,
+  "§Ý": 422,
+  "ôéí": 423,
+  "éíðïò": 424,
+  "éíðïòô ": 425,
+  "ŠŠŠ": 426,
+  "åä": 427,
+  "ôïð": 428,
+  "îå": 429,
+  "óåò": 430,
+  "èåáä": 431,
+  "éîðõôßéä": 432,
+  "åîåò": 433,
+  "í ": 434,
+  "°°": 435,
+  "ùôå": 436,
+  "ìïáä": 437,
+  "ô¨§": 438,
+  "éã": 439,
+  "íåò": 440,
+  "éóô": 441,
+  "®¢¢¢": 442,
+  "áôé": 443,
+  "éôè": 444,
+  "ßóéúå": 445,
+  "ºŠ    ¢¢¢": 446,
+  "ä¨": 447,
+  "÷éôè": 448,
+  "õìô": 449,
+  "ª ": 450,
+  "« ": 451,
+  "óß": 452,
+  "æéç": 453,
+  "¬Š            ": 454,
+  "ý¢": 455,
+  "åø": 456,
+  "ìïçç": 457,
+  "ôåð": 458,
+  "éç": 459,
+  "áíð": 460,
+  "ðáôè": 461,
+  "áääß": 462,
+  "ìåî¨": 463,
+  "ôè": 464,
+  "æòï": 465,
+  "ìá": 466,
+  "ßðáòáí": 467,
+  "îå÷": 468,
+  "ðáòóåò": 469,
+  "æá": 470,
+  "íåòçå": 471,
+  "ãïäå": 472,
+  "æòïí ": 473,
+  "º éîô": 474,
+  "ïîå": 475,
+  "­ ": 476,
+  "¬Š                ": 477,
+  "çåîåò": 478,
+  "õíåî": 479,
+  "äåæá": 480,
+  "äåæáõìô": 481,
+  "÷ïòä": 482,
+  "ðòïíð": 483,
+  "áôéïî": 484,
+  "ó½": 485,
+  "ïõôðõô": 486,
+  "ïð": 487,
+  "ðáòóåò®": 488,
+  "æéìå": 489,
+  "ôåî": 490,
+  "ßäé": 491,
+  "ìåîç": 492,
+  "ìåîçôè": 493,
+  "âùôå": 494,
+  "©ºŠ    ¢¢¢": 495,
+  "óåì欠": 496,
+  "óåñ": 497,
+  "óïò": 498,
+  "öïãáâßóéúå": 499,
+  "ãìå": 500,
+  "ãèå": 501,
+  "ðáòóåò®áääß": 502,
+  "ðáòóåò®áääßáòç": 503,
+  "ðáòóåò®áääßáòçõíåî": 504,
+  "ðáòóåò®áääßáòçõíåîô¨§": 505,
+  "ðáòóåò®áääßáòçõíåîô¨§­": 506,
+  "ðáòóåò®áääßáòçõíåîô¨§­­": 507,
+  "õð": 508,
+  "ãé": 509,
+  "ôß": 510,
+  "Îïîå": 511,
+  "ßèåáä": 512,
+  "ãë": 513,
+  "÷éôè ": 514,
+  "ãïîæéç": 515,
+  "®ó": 516,
+  "õîë": 517,
+  "ìïççåò": 518,
+  "äåæáõìô½": 519,
+  "ôï ": 520,
+  "֌": 521,
+  "öáìõå": 522,
+  "ºŠ                ": 523,
+  "©Š    Š    £ ": 524,
+  "äáôáóåô": 525,
+  "äõ": 526,
+  "Š    Š    äåæ ": 527,
+  "æìï": 528,
+  "æìïáô": 529,
+  "òõå": 530,
+  " « ": 531,
+  "§¬ ôù": 532,
+  "§¬ ôùðå": 533,
+  "§¬ ôùðå½": 534,
+  "©Š    ðáòóåò®áääßáòçõíåîô¨§­­": 535,
+  "ðòéîô¨¢": 536,
+  "îî": 537,
+  "÷åéç": 538,
+  "÷åéçè": 539,
+  "¯ ": 540,
+  "óã": 541,
+  "âáô": 542,
+  "©Š                ": 543,
+  "ŠŠŠäåæ ": 544,
+  "¬ äåæáõìô½": 545,
+  "ðåãé": 546,
+  "íïäåì®": 547,
+  "áíå": 548,
+  "íâ": 549,
+  "áð": 550,
+  "çåîåòáôå": 551,
+  "áòôé": 552,
+  "îî®": 553,
+  "ïòí": 554,
+  "åò¨": 555,
+  " ª ": 556,
+  "Š        Š        ": 557,
+  "ùß": 558,
+  " ½ óåìæ®": 559,
+  "Š                ": 560,
+  "ßôïëåîó": 561,
+  "åòáô": 562,
+  "½½ ": 563,
+  "åóô": 564,
+  "óôòé": 565,
+  "áòôéãìå": 566,
+  "óðåãé": 567,
+  "õòå": 568,
+  "ù ": 569,
+  "®°": 570,
+  "ãá": 571,
+  "ãèõîë": 572,
+  "óë": 573,
+  "îï": 574,
+  "ìå ": 575,
+  "ãèåãë": 576,
+  "ãèåãëðï": 577,
+  "ßìïó": 578,
+  "ïðôéí": 579,
+  "ôïëåîéúåò®": 580,
+  "ßߨ": 581,
+  "Ôòõå": 582,
+  "ó¬ ": 583,
+  "éîæ": 584,
+  "ßôïëåîßéä": 585,
+  "Äáôá": 586,
+  "òáîë": 587,
+  "óðåãéáì": 588,
+  "åôåò": 589,
+  "º éîô ½ ": 590,
+  "íáóë": 591,
+  "óÛ§": 592,
+  "®®": 593,
+  "º û": 594,
+  "éë": 595,
+  "ïðôéíéúåò": 596,
+  "ý ": 597,
+  "éîéô": 598,
+  "÷åéçèô": 599,
+  " ½ °": 600,
+  "áöå": 601,
+  "åô": 602,
+  "áôå ": 603,
+  "±Ý": 604,
+  "áîä ": 605,
+  " ½½ ": 606,
+  "Ôï": 607,
+  "Û§": 608,
+  "íáøßìåîçôè": 609,
+  "Öéã": 610,
+  "ÖéãÁ": 611,
+  "ÖéãÁÉ": 612,
+  "©Š        óåìæ®": 613,
+  "ïó": 614,
+  "óåñß": 615,
+  "óåñßìåî": 616,
+  "éëé": 617,
+  "éò": 618,
+  "âùôåß": 619,
+  "ðòïíðô": 620,
+  "éôå": 621,
+  "ÐÅ": 622,
+  "òåó": 623,
+  "íï": 624,
+  "âáôãè": 625,
+  "©Š            Š            ": 626,
+  "á ": 627,
+  "Ôïëåî": 628,
+  "áã": 629,
+  "ìáâ": 630,
+  "Üî": 631,
+  "áó ": 632,
+  "©®": 633,
+  "ßäéí": 634,
+  "ðáóô": 635,
+  "ôéïîß": 636,
+  "éîæï": 637,
+  "çåôß": 638,
+  "åìóå": 639,
+  "ôéíå": 640,
+  "½§": 641,
+  "ðáéò": 642,
+  "÷áò": 643,
+  "òåñ": 644,
+  "âõ": 645,
+  "èá": 646,
+  "äòï": 647,
+  "ôáì": 648,
+  "¬Š            §": 649,
+  "ôåíð": 650,
+  "ÂÐÅ": 651,
+  "ãèåãëðïéîô": 652,
+  "îõíß": 653,
+  "ìïççåò®": 654,
+  "äï": 655,
+  "óåìæ®öïãáâ": 656,
+  "åîãïäå": 657,
+  "©Š    Š    äåæ ": 658,
+  " ½ ôïòãè®": 659,
+  "éî¨": 660,
+  "Ý ½ ": 661,
+  "ðáóôßë": 662,
+  "«½ ": 663,
+  "áððåî": 664,
+  "áððåîä¨": 665,
+  "çò": 666,
+  "åòáôõòå": 667,
+  "¾ ": 668,
+  "®¢¢¢Š    ": 669,
+  "îáíå": 670,
+  "ìé": 671,
+  "ìåò": 672,
+  "Ôïëåîéúåò": 673,
+  "ìïã": 674,
+  "äá": 675,
+  "óãáì": 676,
+  "óãáìåò": 677,
+  "óôåð": 678,
+  "áö": 679,
+  "º óôò": 680,
+  "ãïä": 681,
+  "ðôéïî": 682,
+  "ßßéîéô": 683,
+  "ßßéîéôßߨ": 684,
+  "åùß": 685,
+  "åùßöáìõå": 686,
+  "ôïôáì": 687,
+  "©ºŠ            ": 688,
+  "éîðõôßéäó": 689,
+  "ó ½ Û": 690,
+  "ôåíðåòáôõòå": 691,
+  "Ìï": 692,
+  "éëéðå": 693,
+  "éëéðåäé": 694,
+  "Éî": 695,
+  "óåß": 696,
+  "ôéîç": 697,
+  "Šæòïí ": 698,
+  "çå¨": 699,
+  "äòïð": 700,
+  "òåðå": 701,
+  "îïòí": 702,
+  "ìïçéô": 703,
+  "óéúå": 704,
+  "çòáä": 705,
+  "íáøßîå÷": 706,
+  "ßð": 707,
+  "ðåîáì": 708,
+  "ðåîáìôù": 709,
+  "óôòéâõ": 710,
+  "óôòéâõôå": 711,
+  "ãõ": 712,
+  "óáíð": 713,
+  "󮢢¢": 714,
+  "ôåøô ½ ": 715,
+  "áîä": 716,
+  "óåôôéîç": 717,
+  "ãô": 718,
+  "ðáòáíåôåò": 719,
+  "äòïðïõô": 720,
+  "îïô ": 721,
+  "îõíßðáòáí": 722,
+  "¨§": 723,
+  "Š            Š            ": 724,
+  "©ºŠ                ": 725,
+  "×éëéðåäé": 726,
+  "ìïáäåò": 727,
+  "ßóôåð": 728,
+  "ìáâåì": 729,
+  "âåóô": 730,
+  "éîæï¨": 731,
+  "äõìå": 732,
+  "ùåò": 733,
+  "íâåä": 734,
+  "±°°": 735,
+  "¨óåì欠": 736,
+  "ðáóôßëåùßöáìõå": 737,
+  "º®": 738,
+  "Ãòå": 739,
+  "ôò": 740,
+  "æïò é": 741,
+  "òåðåôé": 742,
+  "ôåøô ": 743,
+  "ãõäá": 744,
+  "éôåí": 745,
+  "¬ §": 746,
+  "ßäéò": 747,
+  "öåò": 748,
+  "ßðáôè": 749,
+  "¢©Š    ": 750,
+  "ãïäåò": 751,
+  "óó ": 752,
+  "îßèåáä": 753,
+  "îßë": 754,
+  "îßëö": 755,
+  "îßëößèåáä": 756,
+  "îß": 757,
+  "éîå": 758,
+  "éó ": 759,
+  "äåî": 760,
+  "áôéïî ": 761,
+  "ïì": 762,
+  "󊠠  ": 763,
+  "Ìéóô": 764,
+  "òáîçå¨": 765,
+  "¬ äåöéãå": 766,
+  "ôïðßë": 767,
+  "ôïðßð": 768,
+  "ïóßôïëåîßéä": 769,
+  "öåì": 770,
+  "ôïß": 771,
+  "Ìïáä": 772,
+  "ìò": 773,
+  "÷ïòì": 774,
+  "áâìå": 775,
+  "ôòáéîß": 776,
+  "óôáò": 777,
+  "ìïççåò®éîæï¨": 778,
+  "äåãá": 779,
+  "óðåãéáìßôïëåî": 780,
+  "®ê": 781,
+  " ­ ": 782,
+  "Û±Ý": 783,
+  "óôáôå": 784,
+  "óôáôåßäé": 785,
+  "áçå": 786,
+  "Šéíðïòô ": 787,
+  "óº ": 788,
+  "±å": 789,
+  "äéîç": 790,
+  "èéä": 791,
+  "èéääåî": 792,
+  " éî òáîçå¨": 793,
+  "ìéóô": 794,
+  "°Ý": 795,
+  "ðõ": 796,
+  "©ŠŠŠäåæ ": 797,
+  " ½½ °": 798,
+  "éæ áòçó®": 799,
+  "ôåîóïò": 800,
+  "§¬ ôùðå½éîô": 801,
+  "§¬ ôùðå½éîô¬ äåæáõìô½": 802,
+  "äåãáù": 803,
+  "½áòçó®": 804,
+  "îå÷ß": 805,
+  "íåòçåó": 806,
+  "Ïðôéïî": 807,
+  "Ïðôéïîáì": 808,
+  "ŠŠŠã": 809,
+  "ŠŠŠãìá": 810,
+  "ŠŠŠãìáóó ": 811,
+  "Š    Š    äåæ ßßéîéôßߨ": 812,
+  "­±": 813,
+  "óõí": 814,
+  "Æáì": 815,
+  "Æáìóå": 816,
+  "èáðå": 817,
+  "³²": 818,
+  "ìáùåò": 819,
+  "ãïîæéç®": 820,
+  "ó ½ ÛÝ": 821,
+  "ó®áððåîä¨": 822,
+  "åîô": 823,
+  " áîä ": 824,
+  "Ôòá": 825,
+  " éíðïòô ": 826,
+  "áöç": 827,
+  " §": 828,
+  " ½ û": 829,
+  "é « ": 830,
+  "®êï": 831,
+  "®êïéî¨": 832,
+  "òïò": 833,
+  "óåôôéîçóÛ§": 834,
+  "çåîåòáôåäß": 835,
+  " ½ îî®": 836,
+  "äéí½": 837,
+  "íáøßóåñßìåî": 838,
+  "âá": 839,
+  "¨óåìæ®": 840,
+  "ó ½ ôïòãè®": 841,
+  "ãèåä": 842,
+  "áôôåî": 843,
+  "èéääåîßäéí": 844,
+  "Ãïî": 845,
+  "®¢¢¢Š        ": 846,
+  "Ãòåáôå ": 847,
+  "®ôï": 848,
+  "íáøßîå÷ßôïëåîó": 849,
+  "òåðåôéôéïîß": 850,
+  "òåðåôéôéïîßðåîáìôù": 851,
+  "åïóßôïëåîßéä": 852,
+  "ôïëåîßéä": 853,
+  "óßôïß": 854,
+  "íïöå": 855,
+  "áôôåò": 856,
+  "© ­": 857,
+  "© ­¾ ": 858,
+  "ÂÐÅÔïëåîéúåò": 859,
+  "ìåß": 860,
+  "óáöå": 861,
+  "ïó®": 862,
+  "÷ïòìäß": 863,
+  "÷ïòìäßóéúå": 864,
+  "ãõäá®": 865,
+  "¬ áòçó®": 866,
+  "óôáòô": 867,
+  "ðáä": 868,
+  "ãô¨": 869,
+  "é ": 870,
+  "èáîä": 871,
+  "ðáçå": 872,
+  "Åò": 873,
+  "Åòòïò": 874,
+  "áì ": 875,
+  "󊠠      óåìæ®": 876,
+  "íâåääéîç": 877,
+  "º æìïáô": 878,
+  "æòåñ": 879,
+  "éîß": 880,
+  "󊠠      ": 881,
+  "âé": 882,
+  "éîôåò": 883,
+  "ó§Ý": 884,
+  "åô ": 885,
+  "æïò é éî òáîçå¨": 886,
+  "óåô¨": 887,
+  "éîäé": 888,
+  "®çå": 889,
+  "öáìßìïó": 890,
+  "ðô ": 891,
+  "ºŠ                    ": 892,
+  "ïõôðõôßäéò": 893,
+  "§©Š    ðáòóåò®áääßáòçõíåîô¨§­­": 894,
+  "×éëéðåäéá ": 895,
+  "ÌéóôÛ": 896,
+  "íáò": 897,
+  "èáîäìåò": 898,
+  "¢ ª ": 899,
+  "Íïäåì": 900,
+  "íå": 901,
+  "ëå": 902,
+  "ó© ": 903,
+  "ææ": 904,
+  "¬ º": 905,
+  "ßå": 906,
+  "ó ½ óåìæ®": 907,
+  "Š        Š        £ ": 908,
+  "©ºŠ        ¢¢¢": 909,
+  "ãïõî": 910,
+  "íïäåì ": 911,
+  "ó¢": 912,
+  "áìß": 913,
+  "½Ôòõå": 914,
+  "ìéóô¨": 915,
+  "éîäéãå": 916,
+  "®®®": 917,
+  "áìì": 918,
+  "ãòå": 919,
+  "©Š    ðòéîô¨æ¢": 920,
+  "äéóôòéâõôå": 921,
+  "äéóô": 922,
+  "ãïò": 923,
+  "¨íïäåì": 924,
+  "ðïãè": 925,
+  "åøãå": 926,
+  "åøãåðô ": 927,
+  "ïðåî": 928,
+  "óù": 929,
+  "ºŠ        ¢¢¢": 930,
+  "äåãïäå": 931,
+  "ãïîô": 932,
+  "¬Š                §": 933,
+  "åìð": 934,
+  "ÖéãÁÉ ": 935,
+  "ŠŠéíðïòô ": 936,
+  "÷áòä¨": 937,
+  "±®°": 938,
+  " ¯ ": 939,
+  "äéí¬ ": 940,
+  "ó©": 941,
+  "Š        óåìæ": 942,
+  "Š        óåì權        ": 943,
+  "òåð": 944,
+  "ôïòãè®Ô": 945,
+  "ôïòãè®Ôåî": 946,
+  "ôïòãè®Ôåîóïò": 947,
+  "ÏðôéïîáìÛ": 948,
+  "¨Û": 949,
+  "©Š        Š        £ ": 950,
+  "°°°": 951,
+  "©ºŠ        óåìæ®": 952,
+  "󺊠           ": 953,
+  "¯ ±å": 954,
+  "º®²": 955,
+  "íïäõìå": 956,
+  "°®°": 957,
+  "󧺠": 958,
+  "îïß": 959,
+  "Š            éæ ": 960,
+  "ë ": 961,
+  "¼ ": 962,
+  "©Š            Š            £ ": 963,
+  "Ìå": 964,
+  "óåô ": 965,
+  "Äáôáóåô": 966,
+  "ãïòðõ": 967,
+  "éóß": 968,
+  "ôòù": 969,
+  "¬Š            ��       ": 970,
+  "åøéóô": 971,
+  "Ìïáä ": 972,
+  "ôïëåîéúåò ": 973,
+  "§© ": 974,
+  "éôåíó¨": 975,
+  "§ÝŠ        óåìæ®": 976,
+  "òáîäï": 977,
+  "¢ ª ¶": 978,
+  "¢ ª ¶°": 979,
+  "èåìð": 980,
+  "èåìð½§": 981,
+  "ñõ": 982,
+  "®¢¢¢Š    Š    äåæ ßßéîéôßߨ": 983,
+  "ôéïî ": 984,
+  "¬ ë": 985,
+  "èåáäßäéí": 986,
+  "Ìéîå": 987,
+  "Ìéîåáò": 988,
+  "âéá": 989,
+  "áôô": 990,
+  "¨ø": 991,
+  "Òå": 992,
+  "áôôåîôéïîß": 993,
+  "º®²æ": 994,
+  "º®²æý": 995,
+  "óèáðå": 996,
+  "¨éîðõôßéä": 997,
+  "Š            §": 998,
+  "¬Š        ý": 999,
+  "åöáì": 1000,
+  " ôïëåî": 1001,
+  "éîäéãåóßôïß": 1002,
+  "éîäéãåóßôïßòå": 1003,
+  "éîäéãåóßôïßòåíïöå": 1004,
+  "óïòôå": 1005,
+  "óïòôåäß": 1006,
+  "âòå": 1007,
+  "âòåá": 1008,
+  "âòåáë": 1009,
+  "íáéî": 1010,
+  "Ôòáéî": 1011,
+  "Âùôå": 1012,
+  "Ìåöåì": 1013,
+  "éìå": 1014,
+  "ìïáäß": 1015,
+  "ãèåäõì": 1016,
+  "åìó床        ": 1017,
+  "äéóô®": 1018,
+  "©Š    Š    òåôõòî ": 1019,
+  "òáîë ½½ °": 1020,
+  "ìïççåò®éîæï¨æ¢": 1021,
+  "©Š                    ": 1022,
+  "ãïíð": 1023,
+  "±°": 1024,
+  "¨áòçó®": 1025,
+  "÷éôè ïðåî": 1026,
+  "÷éôè ïðåî¨": 1027,
+  "§© áó ": 1028,
+  "§© áó æ": 1029,
+  "ðìé": 1030,
+  "¼ü": 1031,
+  "ü¾": 1032,
+  "Äáôáóåô¨": 1033,
+  "îå÷ß÷ïòä": 1034,
+  "ãïîôéî": 1035,
+  "ãïîôéîõå": 1036,
+  "äáôáÛ§": 1037,
+  "åîãïäåä": 1038,
+  "íéîß": 1039,
+  "åôãè": 1040,
+  "¢©Š    ðòéîô¨¢": 1041,
+  "åîãè": 1042,
+  "åîãèíáò": 1043,
+  "áììïã": 1044,
+  "áììïãáôå": 1045,
+  " ðáòáíåôåò": 1046,
+  "äåãïäåò": 1047,
+  "ôòáî": 1048,
+  "ôòáîó": 1049,
+  "çõ": 1050,
+  "¬ äéí½": 1051,
+  "áôåß": 1052,
+  "¬ óåñßìåî": 1053,
+  "åîãå": 1054,
+  "º æìïáô ½ °": 1055,
+  "ó½Æáìóå": 1056,
+  "Ý ½ Îïîå": 1057,
+  "´°": 1058,
+  "¯ ±å¹": 1059,
+  "©ºŠ        éæ ": 1060,
+  "ð®": 1061,
+  "֏": 1062,
+  "éÝ": 1063,
+  "çòáä¨": 1064,
+  "Çåîåò": 1065,
+  "©ºŠ                    ": 1066,
+  "ð ": 1067,
+  "Ðáôè": 1068,
+  "Ôåø": 1069,
+  "ôïòãè®ãõäá®": 1070,
+  "õóåß": 1071,
+  "ôòáéî¨": 1072,
+  "öáìß": 1073,
+  "éôåòáô": 1074,
+  "éôåòáôïò": 1075,
+  "åðïãè": 1076,
+  "¾½ ": 1077,
+  "½äåöéãå": 1078,
+  "§¬ ôùðå½æìïáô": 1079,
+  "§¬ ôùðå½æìïáô¬ äåæáõìô½": 1080,
+  "§¬ ôùðå½óôò": 1081,
+  "ôïëåîéúåò ½ ": 1082,
+  "®óðìé": 1083,
+  "©Ý": 1084,
+  "óïî": 1085,
+  "öïãáâ ½ û": 1086,
+  "óùíâ": 1087,
+  "óùíâïì": 1088,
+  "󮢢¢Š        ": 1089,
+  "õòò": 1090,
+  "ôåøô®": 1091,
+  "ìåî¨óåìæ®öïãáâ": 1092,
+  "é « ±": 1093,
+  "óðåãéáìßôïëåîó": 1094,
+  "èå ": 1095,
+  "éîç®": 1096,
+  "ðòéîô¨æ¢Üî": 1097,
+  "òáîäïí": 1098,
+  "óº": 1099,
+  "ó÷éôè": 1100,
+  "óôáôåßäéãô¨": 1101,
+  "û§": 1102,
+  "óååä": 1103,
+  "áãôé": 1104,
+  "áãôéöå": 1105,
+  "ðòïíðôß": 1106,
+  "ìù ": 1107,
+  "ìáî": 1108,
+  "ìáîçõ": 1109,
+  "ìáîçõáçå": 1110,
+  "¨© ": 1111,
+  "±¬ ": 1112,
+  "óéî": 1113,
+  "Š        òåôõòî ": 1114,
+  "åîô ": 1115,
+  " ½ îî®Ìéîåáò": 1116,
+  " ½ îî®Ìéîåáò¨": 1117,
+  "óåìæ®èåáäßäéí": 1118,
+  "âéáó½Æáìóå": 1119,
+  "©Š        Š        éæ ": 1120,
+  " æïò ": 1121,
+  "éì": 1122,
+  "ðòå": 1123,
+  "Ãïîæéç": 1124,
+  "îßìáùåò": 1125,
+  "ôéå": 1126,
+  "âï": 1127,
+  "éí": 1128,
+  "ëåù": 1129,
+  "éôé": 1130,
+  "áìéúå": 1131,
+  "ûŠ            §": 1132,
+  "ôïòãè®îïß": 1133,
+  "ôïòãè®îïßçòáä¨": 1134,
+  "Û°Ý": 1135,
+  "Š                    ": 1136,
+  "öïãáâßóéúå½": 1137,
+  "Ôåóô": 1138,
+  "ôòáéîéîç ": 1139,
+  "ÄÐ": 1140,
+  "õôé": 1141,
+  "¬Š": 1142,
+  "ìòßó": 1143,
+  "ìòßóãèåäõì": 1144,
+  "ôòáéîéîç": 1145,
+  "®çåô¨§": 1146,
+  "©Š    åìó床        ": 1147,
+  "õóåßáíð": 1148,
+  "âáôãèÛ§": 1149,
+  "¨äåöéãå": 1150,
+  "áòçó": 1151,
+  "éìå ": 1152,
+  "Éôåò": 1153,
+  "ûáòçó®": 1154,
+  "éæ ��òçó®òáîë ½½ °": 1155,
+  "Óáöå": 1156,
+  "âáôãèå": 1157,
+  "Š            Š            éæ ": 1158,
+  "òåäõ": 1159,
+  "òåäõãå": 1160,
+  "ìåáò": 1161,
+  "íõð": 1162,
+  "§¬ ôùðå½óôò¬ äåæáõìô½": 1163,
+  "ïæ ": 1164,
+  "®óðìéô¨": 1165,
+  "÷ïòä ": 1166,
+  " ôåøô ": 1167,
+  "óº ÌéóôÛ": 1168,
+  "æïòí": 1169,
+  "ãèáò": 1170,
+  " ôï ": 1171,
+  "§§": 1172,
+  "ä û": 1173,
+  "ôåøô©Š        ": 1174,
+  "§º óåìæ®": 1175,
+  "ôï û": 1176,
+  "åîãïäå¨": 1177,
+  " ½ Û": 1178,
+  "áôôåîôéïîßíáóë": 1179,
+  "âùôåßåî": 1180,
+  "âùôåßåîãïäåò": 1181,
+  "óï": 1182,
+  "æåôãè": 1183,
+  "ó ½ ôïòãè®ôåîóïò": 1184,
+  "ìåî¨óåìæ®": 1185,
+  "ßéäø": 1186,
+  "֔": 1187,
+  "ßæéìå": 1188,
+  "óôáôåßäéãô": 1189,
+  "¬Š        §": 1190,
+  "ôòáéîáâìå": 1191,
+  "åîãèíáòë": 1192,
+  "óôáòôó÷éôè": 1193,
+  "ïõôðõôßéä": 1194,
+  "îî®Í": 1195,
+  "îî®Íï": 1196,
+  "îî®Íïäõìå": 1197,
+  "óõð": 1198,
+  "æïò÷áòä¨": 1199,
+  "©ºŠ        òåôõòî ": 1200,
+  "֬": 1201,
+  "ôáò": 1202,
+  "ñ¬ ë": 1203,
+  "Ûº": 1204,
+  "º éîô¬Š        ": 1205,
+  "©Š        Š        óåìæ®": 1206,
+  "ðïõ": 1207,
+  "ðïõô¨": 1208,
+  "¬ óåìæ®": 1209,
+  "éó îïô ": 1210,
+  "éó îïô Îïîå": 1211,
+  "¬ ö": 1212,
+  "ßéîôåò": 1213,
+  " ÷éôè ": 1214,
+  "©¬ ": 1215,
+  "º éîô ½ ³²": 1216,
+  "ðòïð": 1217,
+  "îßðáòáí": 1218,
+  "  £ ": 1219,
+  "ó «½ ": 1220,
+  "󩊠       ": 1221,
+  "©Š        ðòéîô¨æ¢": 1222,
+  "éôéáìéúå": 1223,
+  "½°®°": 1224,
+  "©Š            éæ ": 1225,
+  " éî óåìæ®": 1226,
+  "¨©Š        ": 1227,
+  "éîðõôßéäó®": 1228,
+  "äåø": 1229,
+  "âáôãèßóéúå": 1230,
+  "ïõôðõôóÛ§": 1231,
+  "Çåô ": 1232,
+  "®ôïìéóô¨": 1233,
+  "®ôïìéóô¨©": 1234,
+  "îåø": 1235,
+  "Š    íïäåì": 1236,
+  "Ôòáéîéîç ": 1237,
+  "Šæòïí ôïòãè®": 1238,
+  "ì ": 1239,
+  "ðß": 1240,
+  "ÂùôåÌåöåì": 1241,
+  "ÂùôåÌåöåìÂÐÅÔïëåîéúåò": 1242,
+  "òïî": 1243,
+  "ìïãáìß": 1244,
+  "ìïãáìßòáîë": 1245,
+  "äáôáß": 1246,
+  "íåí": 1247,
+  "íåíïò": 1248,
+  "ôòáéîßìïáäåò": 1249,
+  "óôåð ": 1250,
+  "óÛ°Ý": 1251,
+  "öáìßìïóó": 1252,
+  "¬Š                            ": 1253,
+  "ý¢©Š                ": 1254,
+  "ïî ": 1255,
+  "ßìïóó ½ ": 1256,
+  "îõíßâáôãèå": 1257,
+  "÷éôè ôïòãè®îïßçòáä¨": 1258,
+  "󧬠ôùðå½éîô¬ äåæáõìô½": 1259,
+  "ìåáòî": 1260,
+  "÷áòíõð": 1261,
+  "°®": 1262,
+  "äéò": 1263,
+  "áãôéïî": 1264,
+  "óáíðìåß": 1265,
+  "¼üåî": 1266,
+  "¼üåîäï": 1267,
+  "¼üåîäïæ": 1268,
+  "¼üåîäïæôåøô": 1269,
+  "¼üåîäïæôåøôü¾": 1270,
+  "óôòéð": 1271,
+  "óôòéð¨": 1272,
+  "¢©Š    Š    ": 1273,
+  "Åî": 1274,
+  "ðáäßôïëåîßéä": 1275,
+  "ãõòò": 1276,
+  "ò§": 1277,
+  "Š        æïò ": 1278,
+  "ôåøôº óôò": 1279,
+  "óôòÝ": 1280,
+  "ÂÐÅ ": 1281,
+  "©ý": 1282,
+  "ó®çå": 1283,
+  " ½ óåìæ®ß": 1284,
+  "äßôïëåî": 1285,
+  "äõíð": 1286,
+  " ½ óåìæ®óðåãéáìßôïëåî": 1287,
+  "®áððåîä¨": 1288,
+  "âùôåßäåãïäåò": 1289,
+  "ïõôðõôßðáôè": 1290,
+  "ôèå ": 1291,
+  "󮢢¢Š    ": 1292,
+  "ôïëåîéúåò®äåãïäå": 1293,
+  "õòì": 1294,
+  "§º §": 1295,
+  "§¬Š                §": 1296,
+  "òåóð": 1297,
+  "òåóðïî": 1298,
+  "òåóðïîóå": 1299,
+  "æéìåî": 1300,
+  "æéìåîáíå": 1301,
+  "ðòéîô¨¢Üî": 1302,
+  "ìïççéîç®": 1303,
+  "Éîôåò": 1304,
+  "¢©Š    ðòéîô¨¢  ": 1305,
+  "¢©Š    ðòéîô¨¢  ¯": 1306,
+  "ðòïíðô ": 1307,
+  "¢©Š                ": 1308,
+  "¢¢¢Š": 1309,
+  "¢¢¢ŠÖéãÁÉ ": 1310,
+  "æïòíåò": 1311,
+  "Š¢¢¢": 1312,
+  "Š¢¢¢ŠŠéíðïòô ": 1313,
+  "ðéîç ": 1314,
+  "õðìå": 1315,
+  "ãôéïî": 1316,
+  "Îïòí": 1317,
+  "îî®Íïäõì婺Š    ¢¢¢": 1318,
+  "Òï": 1319,
+  "®¢¢¢Š    Š    äåæ ßßéîéôßߨóåì欠": 1320,
+  "åð": 1321,
+  "©ºŠ        óõð": 1322,
+  "©ºŠ        ó��ðåò¨": 1323,
+  "©ºŠ        óõðåò¨©®": 1324,
+  "©ºŠ        óõðåò¨©®ßßéîéôßߨ": 1325,
+  "©ºŠ        óõðåò¨©®ßßéîéôßߨ©Š        óåìæ®": 1326,
+  "Ðáò": 1327,
+  "óõí¨": 1328,
+  "åòù": 1329,
+  "¬ âéáó½Æáìóå": 1330,
+  "º ÏðôéïîáìÛ": 1331,
+  "âó": 1332,
+  "âóú": 1333,
+  "âóú¬ óåñßìåî": 1334,
+  "öéå": 1335,
+  "öéå÷¨": 1336,
+  "çòï": 1337,
+  "íõì": 1338,
+  "íáø¨": 1339,
+  "£ Á": 1340,
+  "ÖéãÁÉÃïîæéç": 1341,
+  "ß÷åéçèô": 1342,
+  "âïïì": 1343,
+  " ª óåìæ®": 1344,
+  "ÖéãÁÉÍïäåì": 1345,
+  "ßåíâåääéîç": 1346,
+  "ìù": 1347,
+  "ôïôáìßðáòáí": 1348,
+  "íïäõìå®": 1349,
+  "ðáòáíåôåòó¨": 1350,
+  "ôáòç": 1351,
+  "ôáòçåô": 1352,
+  "õó": 1353,
+  "å¨": 1354,
+  "ìïóó": 1355,
+  "ôïòå": 1356,
+  "Ôïð": 1357,
+  "Ôïð­": 1358,
+  "óïòôåäßéîäéãåóßôïßòåíïöå": 1359,
+  "çåîåòáôåä": 1360,
+  "ãòåáôåß": 1361,
+  "öéãá": 1362,
+  "öéãáé": 1363,
+  "öéãáéß": 1364,
+  "â¨": 1365,
+  "ŠŠŠéæ ": 1366,
+  "ŠŠŠéæ ßß": 1367,
+  "ŠŠŠéæ ßßîáíå": 1368,
+  "ŠŠŠéæ ßßîáíåßß": 1369,
+  "ŠŠŠéæ ßßîáíåßß ½½ ": 1370,
+  "ßßíáéî": 1371,
+  "ßßíáéîßß": 1372,
+  "Ôåóô ": 1373,
+  "©Š": 1374,
+  "©Š<": 1375,
+  "©Š<|": 1376,
+  "©Š<|e": 1377,
+  "©Š<|en": 1378,
+  "©Š<|end": 1379,
+  "©Š<|endo": 1380,
+  "©Š<|endof": 1381,
+  "©Š<|endoft": 1382,
+  "©Š<|endofte": 1383,
+  "©Š<|endoftex": 1384,
+  "©Š<|endoftext": 1385,
+  "©Š<|endoftext|": 1386,
+  "©Š<|endoftext|>": 1387,
+  "äéóôòéâõôåä ": 1388,
+  "äéóôòéâõôåä": 1389,
+  "õôéì": 1390,
+  "×éëéðåäéá": 1391,
+  "óáöåß": 1392,
+  "ìòßóãèåäõìåò": 1393,
+  "§ éî ": 1394,
+  "ãåó": 1395,
+  "ãìåáî": 1396,
+  "óè": 1397,
+  "ïðôéíéúåò¬ ": 1398,
+  "ó ½ âáôãèÛ§": 1399,
+  "®ôï¨äåöéãå": 1400,
+  "Š    íïäå쬊    ": 1401,
+  "¬Š©ºŠ    ¢¢¢": 1402,
+  "÷èéìå ": 1403,
+  "«½ ±": 1404,
+  "¥ ": 1405,
+  "åý¢": 1406,
+  "Öáì": 1407,
+  "Óáöå ": 1408,
+  "ãèåãëðïéîô¨": 1409,
+  "áòçó®ïõôðõôßäéò": 1410,
+  "öáìéä": 1411,
+  "¬ äåöéãå½äåöéãå": 1412,
+  "§¬ ôùðå½óôò¬ äåæáõìô½§": 1413,
+  "±°°°": 1414,
+  "§ ": 1415,
+  "Äå": 1416,
+  "áöá": 1417,
+  "áöáé": 1418,
+  "áöáéìáâ": 1419,
+  "ïó®ðáôè": 1420,
+  "ôïëåîéúåòßðáôè": 1421,
+  "Ôïëåîéúåò ": 1422,
+  "¢©Š        ": 1423,
+  "ìïççåò®éîæ諸": 1424,
+  "íïäåì ½ ": 1425,
+  "äáôáóåô ½ ": 1426,
+  "êóïî": 1427,
+  "§º °": 1428,
+  "¼¯": 1429,
+  " ½ °Š        óåìæ®": 1430,
+  "®éôåíó¨": 1431,
+  "Š    Š    äåæ ß": 1432,
+  "âéç": 1433,
+  "âéçòá": 1434,
+  "ðáôôåò": 1435,
+  "ðáôôåòî": 1436,
+  "öïãáâÛ": 1437,
+  "ôåøôº óôò© ­¾ ": 1438,
+  "ü§": 1439,
+  "®óðìéô¨©": 1440,
+  " ½ Ôòõå": 1441,
+  "ÉÄ": 1442,
+  "ìáãå": 1443,
+  "ó ½ äáôáÛ§": 1444,
+  "åìó床            ": 1445,
+  "ìåöåì": 1446,
+  "²µ": 1447,
+  "²µ¶": 1448,
+  "õîéã": 1449,
+  "õîéãïäå": 1450,
+  "÷ïòäÛ": 1451,
+  "۱ݩŠ                    ": 1452,
+  " ½ ¢": 1453,
+  "ôåóô": 1454,
+  "ôïëåîéúåò®åîãïäå¨": 1455,
+  "ßìåîçôè": 1456,
+  "éîðõôßéäó ½ ôïòãè®ôåîóïò": 1457,
+  "ºÝ": 1458,
+  "÷îìïáä": 1459,
+  "ó ­ ": 1460,
+  "Èáîä": 1461,
+  "äåãáùßðáòáí": 1462,
+  "îõíßðáòáíó ": 1463,
+  "½¢ ª ¶°": 1464,
+  "§º": 1465,
+  "ðòéîô¨¢­": 1466,
+  "Óåô ": 1467,
+  "âåîãèíáòë": 1468,
+  "äßç": 1469,
+  "äßçâ": 1470,
+  "çåîåòáôéïî ": 1471,
+  "ôïð­": 1472,
+  "ãïîôéîõ加           Š            éæ ": 1473,
+  "íáøßîå÷ßôïëåî": 1474,
+  "íáøßîå÷ßôïëåîó½": 1475,
+  "çåîåòáôåäßôåøô ½ ": 1476,
+  "óÛ°Ý®ôïìéóô¨©": 1477,
+  "ðòïíðôßôåøô": 1478,
+  "¬ èåìð½§": 1479,
+  "Íïäåì ": 1480,
+  " ðáòáíåôåò": 1481,
+  "íáôè": 1482,
+  "ôïòãè®îî®": 1483,
+  "ÒÍ": 1484,
+  "ÒÍÓ": 1485,
+  "¨îî®Íïäõì婺Š    ¢¢¢": 1486,
+  "Åíâåääéîç": 1487,
+  "²¬ ": 1488,
+  "âáóå": 1489,
+  "¢¬ ": 1490,
+  "åíâ": 1491,
+  "ãáô": 1492,
+  "¬ ºÝ": 1493,
+  "©Š        òåôõòî ": 1494,
+  "©ŠŠŠãìáóó ": 1495,
+  "ôôåî": 1496,
+  "© æïò ": 1497,
+  "îßèåáäó": 1498,
+  "îßëößèåáäó": 1499,
+  " ½ îî®Ìéîåáò¨äéí¬ ": 1500,
+  "ó ª ": 1501,
+  "¬ âéáó½Æáìó婊        óåìæ®": 1502,
+  "Ý ½ Îïî嬊    ": 1503,
+  "®óèáðå": 1504,
+  "ôòáîóðï": 1505,
+  "ôòáîóðïóå": 1506,
+  "ôòáîóðïóå¨": 1507,
+  "ïòå": 1508,
+  "©Š        Š        òåôõòî ": 1509,
+  "öïãáâßóéú庠éîô ½ ³²": 1510,
+  "öïãáâßóéú庠éîô ½ ³²°°°": 1511,
+  "öïãáâßóéúå ½ ": 1512,
+  "ãïõîô": 1513,
+  " ª óåìæ®äéí": 1514,
+  "ôá": 1515,
+  "ó æïò ": 1516,
+  "¨Š                ": 1517,
+  "¯ ±å¹º®²æý": 1518,
+  "ó¨óåì欠": 1519,
+  "éîðõôßéäó®óèáðå": 1520,
+  "õîó": 1521,
+  "æïò é¬ ": 1522,
+  " éî åî": 1523,
+  " éî åîõí": 1524,
+  " éî åîõíåò": 1525,
+  " éî åîõíåòáôå": 1526,
+  "ìïóó ½ ": 1527,
+  "º éîô ½ µ": 1528,
+  "º æìïáô ½ °®": 1529,
+  "çòåó": 1530,
+  "ïõôðõôó ½ ": 1531,
+  "¨éîðõôßéäó¬ ": 1532,
+  "ôéïî ðåîáìôù": 1533,
+  "ìïçéôó¬ ": 1534,
+  "îõ": 1535,
+  "ßóáíð": 1536,
+  "ãòåáôåßöéãáéß": 1537,
+  "ãòåáôåßöéãáéßµ": 1538,
+  "±²": 1539,
+  "ý¢©Š    ðòéîô¨æ¢": 1540,
+  "ðô": 1541,
+  "÷òá": 1542,
+  "Óáíð": 1543,
+  "Æéìå": 1544,
+  "çåôßìïççåò": 1545,
+  "®¢¢¢Š    éæ ": 1546,
+  "ïó®åî": 1547,
+  "ïó®åîöé": 1548,
+  "ïó®åîöéòïî": 1549,
+  " ½ éîô¨": 1550,
+  "ðòïãåó": 1551,
+  "çòïõð": 1552,
+  "äáôá ": 1553,
+  "óáíðìåò": 1554,
+  "ìåò¨": 1555,
+  "¨íïäå쬠": 1556,
+  "¬Š    äåöéãå": 1557,
+  "Íá": 1558,
+  "®óåô": 1559,
+  " û": 1560,
+  "ü ": 1561,
+  "æý ": 1562,
+  "Š                        ": 1563,
+  "󺊠               ": 1564,
+  "ßôéíå": 1565,
+  "ìáâåìó": 1566,
+  "öåòáçå": 1567,
+  "áìì ": 1568,
+  "§¬ ôùðå½æìïáô¬ äåæáõìô½°®": 1569,
+  "âåô": 1570,
+  "âåôá": 1571,
+  "§¬ áãôéïî": 1572,
+  "§¬ áãôéï": 1573,
+  "§¬ áãôéïó": 1574,
+  "§¬ áãôéïóôïòå": 1575,
+  "§¬ áãôéïóôïòåß": 1576,
+  "§¬ áãôéïóôïòåßô": 1577,
+  "§¬ áãôéïóôïòåßôòõå": 1578,
+  "§¬ áãôéïóôïòåßôòõ姬 ": 1579,
+  "ãïíðéìå": 1580,
+  "ðáòåîô": 1581,
+  "¬ åøéóô": 1582,
+  "¬ åøéóôß": 1583,
+  "¬ åøéóôßï": 1584,
+  "¬ åøéóôßïë": 1585,
+  "¬ åøéóôßïë½Ôòõå": 1586,
+  "¨æ": 1587,
+  "áöáéìáâìå": 1588,
+  "óáíðìå ": 1589,
+  "îõíßáòôéãìå": 1590,
+  "¬ §ò": 1591,
+  "éî ôåøô": 1592,
+  "Ãï": 1593,
+  "òåáí": 1594,
+  "×éëéðåäéáÄáôáóåô¨": 1595,
+  "ó¨íïäåì": 1596,
+  "æéî": 1597,
+  "ðéã": 1598,
+  "ðéãë": 1599,
+  "ðéãëìå": 1600,
+  "äåæáõìôäé": 1601,
+  "Äéã": 1602,
+  "Äéãô": 1603,
+  "óôáô": 1604,
+  " ½ òå": 1605,
+  " §®êïéî¨": 1606,
+  "¨¿": 1607,
+  "¡Ü": 1608,
+  "¡ÜÓ": 1609,
+  "¡ÜÓ©": 1610,
+  "÷ïòä éî ": 1611,
+  "®óõ": 1612,
+  "ßôïëåîéúå": 1613,
+  "áòù ": 1614,
+  "ó®éôåíó¨": 1615,
+  "Ãïîöåò": 1616,
+  "Ãïîöåòô ": 1617,
+  "§§®êïéî¨": 1618,
+  "¨é « ±": 1619,
+  "ðòéîô¨æ¢  ": 1620,
+  "ý¢©Š    Š    äåæ ": 1621,
+  "éîôÝ": 1622,
+  "Š            æïò ": 1623,
+  "óôòºŠ        ¢¢¢": 1624,
+  "òåöåò": 1625,
+  "ðáô躠óôò": 1626,
+  "¬ §÷": 1627,
+  "⧩ áó æ": 1628,
+  " ½ óåìæ®óðåãéáìßôïëåîóÛ§": 1629,
+  "ìïá": 1630,
+  "ìïáäå": 1631,
+  "ìåî¨óåñ": 1632,
+  " ½ ÛÝ": 1633,
+  "Š    Š    äåæ ßß": 1634,
+  "ßߨóåìæ": 1635,
+  "­ìåöåì": 1636,
+  "­ìåöåì ": 1637,
+  "­¸": 1638,
+  "âùôåßôåøô": 1639,
+  "¢¬Š        ": 1640,
+  "¢¬Š        ¢": 1641,
+  "éîå ": 1642,
+  "¢Š    ": 1643,
+  "òåñõå": 1644,
+  " ×éëéðåäéá ": 1645,
+  "âáóåß": 1646,
+  "âáóåßõòì": 1647,
+  "ßäáôá": 1648,
+  "Åø": 1649,
+  "ôéô": 1650,
+  "ôéôìå": 1651,
+  "¨ãèõîë": 1652,
+  "äáôáóåôßéäø": 1653,
+  "äáôáßäéò": 1654,
+  "÷éëé": 1655,
+  "×å": 1656,
+  "Ôåøô ": 1657,
+  "òåñõ": 1658,
+  "òåñõé": 1659,
+  "òåñõéòå": 1660,
+  "ãïîóï": 1661,
+  "Æïòí": 1662,
+  "ìïáäßóôáôåßäéãô¨": 1663,
+  " « ¢": 1664,
+  "¼´°": 1665,
+  "¼´°ý ": 1666,
+  "¾±": 1667,
+  "¾±µ": 1668,
+  "¨óååä": 1669,
+  "òåóåò": 1670,
+  "òåóåòöå": 1671,
+  "Éîôåòáãôéöå": 1672,
+  "ãïîôéîõ加           Š            éæ ðòïíðô": 1673,
+  "ãïîôéîõ加           Š            éæ ðòïíðô®": 1674,
+  "ãïîôéîõ加           Š            éæ ðòïíðô®óôáòôó÷éôè": 1675,
+  "ãïîôéîõ加           Š            éæ ðòïíðô®óôáòôó÷éô訧": 1676,
+  "ãïîôéîõ加           Š            éæ ðòïíðô®óôáòôó÷éô訧¯": 1677,
+  " §©ºŠ                ": 1678,
+  " §©ºŠ                ôòù": 1679,
+  " §©ºŠ                ôòùºŠ                    ": 1680,
+  " §©ºŠ                ôòùºŠ                    óåôôéîçóÛ§": 1681,
+  "ðòïíðô®óðìéô¨©": 1682,
+  "ðòïíðô®óðìéô¨©Û±Ý©Š                    ": 1683,
+  "ðòïíðô®óðìéô¨©Û±Ý©Š                    ðòéîô¨æ¢": 1684,
+  "óåô ôï û": 1685,
+  "óåô ôï ûóåôôéîçóÛ§": 1686,
+  "§Ýý¢©Š                ": 1687,
+  "§Ýý¢©Š                åøãåðô ": 1688,
+  "§Ýý¢©Š                åøãåðô ¨": 1689,
+  "§Ýý¢©Š                åøãåðô ¨Öáì": 1690,
+  "§Ýý¢©Š                åøãåðô ¨Öáìõå": 1691,
+  "§Ýý¢©Š                åøãåðô ¨ÖáìõåÅòòïò": 1692,
+  "§Ýý¢©Š                åøãåðô ¨ÖáìõåÅòòïò¬ ": 1693,
+  "§Ýý¢©Š                åøãåðô ¨ÖáìõåÅòòïò¬ Éî": 1694,
+  "§Ýý¢©Š                åøãåðô ¨ÖáìõåÅòòïò¬ Éîäåø": 1695,
+  "§Ýý¢©Š                åøãåðô ¨ÖáìõåÅòòïò¬ ÉîäåøÅòòïò": 1696,
+  "§Ýý¢©Š                åøãåðô ¨ÖáìõåÅòòïò¬ ÉîäåøÅòòïò©ºŠ                    ": 1697,
+  "§Ýý¢©Š                åøãåðô ¨ÖáìõåÅòòïò¬ ÉîäåøÅòòïò©ºŠ                    ðòéîô¨¢": 1698,
+  "§Ýý¢©Š                åøãåðô ¨ÖáìõåÅòòïò¬ ÉîäåøÅòòïò©ºŠ                    ðòéîô¨¢Éî": 1699,
+  "§Ýý¢©Š                åøãåðô ¨ÖáìõåÅòòïò¬ ÉîäåøÅòòïò©ºŠ                    ðòéîô¨¢Éîöáìéä": 1700,
+  "§Ýý¢©Š                åøãåðô ¨ÖáìõåÅòòïò¬ ÉîäåøÅòòïò©ºŠ                    ðòéîô¨¢Éîöáìéä ": 1701,
+  "öáìõ墩Š                ": 1702,
+  "íïäåì®çåîåòáôå": 1703,
+  "§Ý¬Š                    ": 1704,
+  "çåîåòáôåäßôåøô": 1705,
+  "îõíßôïëåîó": 1706,
+  "ôòáîóæïòíåò": 1707,
+  "Šæòïí ôù": 1708,
+  "Šæòïí ôùðéîç ": 1709,
+  "Šæòïí ôùðéîç éíðïòô ": 1710,
+  "ŠŠéíðïòô ôïòãè": 1711,
+  "ÒÍÓÎïòí": 1712,
+  "å ": 1713,
+  "áìéú": 1714,
+  "º éîô¬ ": 1715,
+  "ôïòãè®ïîå": 1716,
+  "²©®": 1717,
+  "© « ": 1718,
+  "¸±": 1719,
+  "¸±¹": 1720,
+  "º æìïáô ½ ": 1721,
+  " ½ äéí": 1722,
+  "éîö": 1723,
+  "éîöß": 1724,
+  "éîößæòåñ": 1725,
+  "°¬ ": 1726,
+  "åòß": 1727,
+  "¬ äéí½­±": 1728,
+  "ãáãèåä": 1729,
+  " ½ ø": 1730,
+  "áðð": 1731,
+  "ßåíâåä": 1732,
+  "© ª ": 1733,
+  "õðå": 1734,
+  "®¢¢¢Š    Š    äåæ ßßéîéôßߨŠ        óåì權        ": 1735,
+  "äòïðïõôº æìïáô ½ °": 1736,
+  "äòïðïõôº æìïáô ½ °®°": 1737,
+  "ó ª óåìæ®èåáäßäéí": 1738,
+  "¬ âéáó½Æáìó婊        óåìæ®÷": 1739,
+  "áôôîß": 1740,
+  "äòïðïõô ½ îî®": 1741,
+  "äòïðïõô ½ îî®Ä": 1742,
+  "äòïðïõô ½ îî®Äòï": 1743,
+  "äòïðïõô ½ îî®Äòïðïõô¨": 1744,
+  "òåóéä": 1745,
+  "º ôïòãè®Ôåîóïò": 1746,
+  "º ôïòãè®Ôåîóïò¬Š        ": 1747,
+  " ½ óåìæ®÷": 1748,
+  "©®öéå÷¨": 1749,
+  "©®öéå÷¨âóú¬ óåñßìåî": 1750,
+  "©®öéå÷¨âóú¬ óåñßìåóåìæ®": 1751,
+  "ôòáîóðïó娱¬ ": 1752,
+  " éó îïô Îïîå": 1753,
+  "ö ½ ": 1754,
+  "묠ö": 1755,
+  "áô ": 1756,
+  "óãïòå": 1757,
+  "­±©": 1758,
+  "áôôî": 1759,
+  "Æ®ó": 1760,
+  "¬ ðáóôßëåùßöáìõå": 1761,
+  "Æïò": 1762,
+  "Óéîç": 1763,
+  "Óéîçìå ": 1764,
+  "ðòå­": 1765,
+  "äéí©Š        óåìæ®": 1766,
+  "ä ½ ": 1767,
+  "´°¹": 1768,
+  "´°¹¶": 1769,
+  "¸¬Š        ": 1770,
+  "±´": 1771,
+  "±´³": 1772,
+  "±´³³": 1773,
+  "±´³³¶": 1774,
+  "ôéåß÷åéçèô": 1775,
+  "óé": 1776,
+  "ó ¨": 1777,
+  "ãïîæéç ½ ": 1778,
+  "Š        Š        óåìæ®": 1779,
+  "ôïëåîßåíâåääéîç": 1780,
+  "¬Š                ãïîæéç®": 1781,
+  "éîô ": 1782,
+  "çåôßîõíßðáòáí": 1783,
+  "éîéôéáìéúå": 1784,
+  "åìéæ ": 1785,
+  "îõíåì": 1786,
+  "æìïáô¨§": 1787,
+  "éîæ§": 1788,
+  "óÛéÝ": 1789,
+  " éæ ": 1790,
+  "åìóå ": 1791,
+  "íáøßîå÷ßôïëåîóº éîô ½ ": 1792,
+  "áõ": 1793,
+  "åöáì¨": 1794,
+  "Š            Š            £ ": 1795,
+  "ìïçéôóÛ": 1796,
+  "¡½ ": 1797,
+  "éî犠           éæ ": 1798,
+  " ¾ ": 1799,
+  "Û®®®": 1800,
+  "ðòïâ": 1801,
+  "­±Ý": 1802,
+  "áôôåò¨": 1803,
+  "Š        Š        òåôõòî ": 1804,
+  "©Š    òåôõòî ": 1805,
+  "¨ãïîæéç": 1806,
+  "©ŠŠŠéæ ßßîáíåßß ½½ ": 1807,
+  "¢ßßíáéîßß": 1808,
+  "¢ßßíáéîßߢ": 1809,
+  "¢ßßíáéîßߢºŠ    ": 1810,
+  "¢ßßíáéîßߢºŠ    £ ": 1811,
+  "¢ßßíáéîßߢºŠ    £ Ôåóô ": 1812,
+  "óº û": 1813,
+  "ïõôðõôó ½ íïäåì": 1814,
+  "ïõôðõôóÛ§ìïó": 1815,
+  "ïõôðõôóÛ§ìïóó§Ý": 1816,
+  "ý¢©Š<|endoftext|>": 1817,
+  "òé": 1818,
+  "Äé": 1819,
+  "Äéóôòéâõôå": 1820,
+  "ÆÓ": 1821,
+  "ÆÓÄÐ": 1822,
+  "ÄÄÐ": 1823,
+  "áòçðáò": 1824,
+  "áòçðáòóå": 1825,
+  "æó": 1826,
+  "æóä": 1827,
+  "õôéìó®": 1828,
+  "Ôåøô": 1829,
+  "ÔåøôÆéìå": 1830,
+  "õðßäé": 1831,
+  "õðßäéóôòéâõôå": 1832,
+  "õðßäéóôòéâõôåä¨": 1833,
+  "òáî묠": 1834,
+  "óèõ": 1835,
+  "óèõææ": 1836,
+  "óèõææìå": 1837,
+  "©ºŠ    ¢¢¢Ãòåáôå ": 1838,
+  "íïäåì®ôòáéî¨": 1839,
+  "éîðõôßéäó§Ý": 1840,
+  "ìáâåìó§Ý": 1841,
+  "ïðôéíéúåò®": 1842,
+  "äáôå": 1843,
+  "éôåí¨": 1844,
+  "öáìßìïáäåò": 1845,
+  "¬Š    äåöéã嬊    ": 1846,
+  "âåóôß": 1847,
+  "ôòáéîßéôåòáôïò": 1848,
+  "éôåò¨": 1849,
+  "èáó": 1850,
+  "èáóáô": 1851,
+  "èáóáôôò": 1852,
+  "ôéíå®": 1853,
+  "ôéíå®ôéíå": 1854,
+  "íáøßóôåð": 1855,
+  "îåøô¨": 1856,
+  "Ìïç": 1857,
+  "Ìïçç": 1858,
+  "ý¯": 1859,
+  "óáöåßãèåãëðïéîô¨": 1860,
+  "áòçó®ïõôðõôßäéò ¯ ": 1861,
+  "ðô§": 1862,
+  "©Š                        ": 1863,
+  "󺊠               âòåáë": 1864,
+  "ãïíðìå": 1865,
+  "éî û": 1866,
+  "äáôáßìïáäåò": 1867,
+  "ôïôáìßìïó": 1868,
+  "óÝ": 1869,
+  "íáéî¨": 1870,
+  "Áò": 1871,
+  "Ôòáéî ": 1872,
+  "áòç󊠠  ": 1873,
+  "áòç󊠠  ðáòóåò®áääßáòçõíåîô¨§­­": 1874,
+  "°°°©Š    ðáòóåò®áääßáòçõíåîô¨§­­": 1875,
+  "íáø­": 1876,
+  "²°": 1877,
+  "²°´": 1878,
+  "ìåáòîéîç": 1879,
+  "òáôå": 1880,
+  "òåóõí": 1881,
+  "õóå": 1882,
+  "éóßäéóôòéâõôåä": 1883,
+  "Š    Š    £ ": 1884,
+  "éæ áòçó®òáîë ½½ °ºŠ        ": 1885,
+  "äéò¨": 1886,
+  "ó½Ôòõå": 1887,
+  "ÇÐ": 1888,
+  "ôïòãè®ãõäá®éóß": 1889,
+  "ôïòãè®ãõäá®éóßáöáéìáâìå": 1890,
+  "ãðõ": 1891,
+  "ãðõ§": 1892,
+  "©Š    Š    £ Ìïáä ": 1893,
+  "ôïëåîéúåòŠ    ": 1894,
+  "ïó®ðáôè®": 1895,
+  "ïó®ðáôè®åøéóô": 1896,
+  "ïó®ðáôè®åøéóôó¨": 1897,
+  "Ìïáäéîç ": 1898,
+  "ôïëåîéúåò æòïí ": 1899,
+  "ý¢©Š        ": 1900,
+  "ôïëåîéúåò ½ ÂùôåÌåöåìÂÐÅÔïëåîéúåò": 1901,
+  "ìïáä¨áòçó®": 1902,
+  "áôéîç ": 1903,
+  "æ®": 1904,
+  "®óôòéð¨": 1905,
+  "§©ºŠ        ": 1906,
+  "äáôᬠ": 1907,
+  "äáôáóåô ½ ×éëéðåäéáÄáôáóåô¨": 1908,
+  "¬ íáøßìåîçôè": 1909,
+  "ó¨íïäåì¬��": 1910,
+  " ½ ôïòãè®ãõäá®": 1911,
+  "®®®¢©Š    ": 1912,
+  "öïãáâ ½ ûý": 1913,
+  "¾§º ": 1914,
+  "ßçåôß": 1915,
+  "ßçåôßóôáô": 1916,
+  "ó ïæ ": 1917,
+  "äåæáõìôäéãô¨": 1918,
+  "äåæáõìôäéãô¨éîô": 1919,
+  "äåæáõìôäéãô¨éîô©Š        ": 1920,
+  "äåæáõìôäéãô¨éîô©Š        æïò ": 1921,
+  " éî öïãáâ": 1922,
+  "­ ±": 1923,
+  "é « ±Ý": 1924,
+  "©Ý ": 1925,
+  "íåòçåß": 1926,
+  "íåòçåßöïãáâ": 1927,
+  "¬ öïãáâ": 1928,
+  "âéçòáí": 1929,
+  " ½ òå®": 1930,
+  "򧨿": 1931,
+  "¡ÜÓ©§": 1932,
+  "ôïëåîéúå": 1933,
+  "Üð": 1934,
+  "Üðû": 1935,
+  "Üó": 1936,
+  "ôåøôóº ÌéóôÛ": 1937,
+  "ôåøôóº ÌéóôÛóôòÝ": 1938,
+  "öïãáâ ½ ": 1939,
+  "ôåøô éî ôåøô": 1940,
+  "ìï÷": 1941,
+  "¼¯÷": 1942,
+  "¼¯÷¾": 1943,
+  "÷ïòä ½ ": 1944,
+  "æïòíáô": 1945,
+  "Áä": 1946,
+  "ó ôï ": 1947,
+  "©ºŠ                éæ ": 1948,
+  "îïô éî ": 1949,
+  "îïô éî óåìæ®öïãáâ": 1950,
+  "óåìæ®öïãáâÛ": 1951,
+  "Ý ½ ìåî¨óåìæ®öïãáâ": 1952,
+  "îõíßíåòçåó": 1953,
+  "º âïïì": 1954,
+  "º âïïì ½ Ôòõå": 1955,
+  "Åîãïäå": 1956,
+  " ÉÄ": 1957,
+  "÷ïòäßôïëåî": 1958,
+  "®çåô¨": 1959,
+  "Äåãïäå": 1960,
+  "òåöåòóåß": 1961,
+  "ó ½ Û݊        æïò ": 1962,
+  "¨óåì欠ðáô躠óôò": 1963,
+  "¨óåì欠ðáô躠óôò©ºŠ        ¢¢¢": 1964,
+  "æéì客¢¢Š        ": 1965,
+  "÷éôè ïðåî¨ðáôè": 1966,
+  "⧩ áó æºŠ            ": 1967,
+  "ðéãëìå®": 1968,
+  "äáôá ½ ": 1969,
+  "ìïáä¨æ": 1970,
+  " ½ óåìæ®óðåãéáìßôïëåîóÛ§¼": 1971,
+  "¾§ÝŠ        óåìæ®": 1972,
+  "æòïí û": 1973,
+  "óåñ éî ": 1974,
+  "óåñ éî åîãïäåä": 1975,
+  "íáøßìåî": 1976,
+  "éîðõôßéä󧺠": 1977,
+  "áôôåîôéïîßíáó맺 ": 1978,
+  "âùôåóßôïß": 1979,
+  "âùôåóßôïßõîéãïäå": 1980,
+  "âùôå­ìåöåì ": 1981,
+  "õôæ": 1982,
+  "õôæ­¸": 1983,
+  "ãïòðõó ": 1984,
+  "é ¼ ": 1985,
+  "é ¼ ìåî¨": 1986,
+  "é ¼ ìåî¨÷ïòä": 1987,
+  " áîä ÷ïòäÛ": 1988,
+  "ºŠ                    îå÷ß÷ïòä": 1989,
+  "ºŠ                    îå÷ß÷ïòä®áððåîä¨": 1990,
+  "©Š    ôïëåîéúåò®": 1991,
+  "óáíðìåßôåøô": 1992,
+  "󠽠ۊ        ": 1993,
+  "¬Š    Ý": 1994,
+  "ôïëåîéúåò®äåãïäå¨": 1995,
+  "òåñõåóô": 1996,
+  "Éôåòáâìå": 1997,
+  "ÉôåòáâìåÄáôáóåô": 1998,
+  "íéîßáòôéãìå": 1999
+}