Upload 12 files

src/app.py

@@ -0,0 +1,87 @@
+# src/app.py
+from flask import Flask, request, render_template
+import torch
+from model import TransformerModel
+from utils import load_vocab, tokenize
+import time
+import random
+import os
+
+app = Flask(__name__, template_folder='templates')
+
+# Configuration
+MODEL_PATH = 'models/3ed0k4_model_epoch10.pth'  # Update this path based on the latest model
+VOCAB_PATH = 'vocab.json'
+EMBED_SIZE = 256
+NUM_HEADS = 8
+HIDDEN_DIM = 512
+NUM_LAYERS = 4
+DROPOUT = 0.1
+MAX_LENGTH = 100  # Maximum tokens to generate
+
+# Load vocabulary
+vocab = load_vocab(VOCAB_PATH)
+vocab_size = len(vocab)
+
+# Initialize model
+model = TransformerModel(
+    vocab_size=vocab_size,
+    embed_size=EMBED_SIZE,
+    num_heads=NUM_HEADS,
+    hidden_dim=HIDDEN_DIM,
+    num_layers=NUM_LAYERS,
+    dropout=DROPOUT
+)
+
+# Load model weights
+if not os.path.exists(MODEL_PATH):
+    raise FileNotFoundError(f"Model file not found at {MODEL_PATH}. Please train the model first.")
+model.load_state_dict(torch.load(MODEL_PATH, map_location=torch.device('cpu')))
+model.eval()
+
+def generate_text(prompt, max_length=MAX_LENGTH):
+    tokens = tokenize(prompt)
+    numericalized = [vocab.get(token, vocab['<UNK>']) for token in tokens]
+    input_seq = torch.tensor(numericalized, dtype=torch.long).unsqueeze(0)  # Batch size 1
+
+    generated = numericalized.copy()
+
+    with torch.no_grad():
+        for _ in range(max_length):
+            src_mask = model.generate_square_subsequent_mask(input_seq.size(1)).to(input_seq.device)
+            outputs = model(input_seq, src_mask)
+            next_token_logits = outputs[0, -1, :]
+            next_token = torch.argmax(next_token_logits).item()
+
+            if next_token == vocab['<PAD>']:
+                break
+
+            generated.append(next_token)
+            input_seq = torch.tensor(generated, dtype=torch.long).unsqueeze(0)
+
+    # Convert numerical tokens back to words
+    inv_vocab = {idx: word for word, idx in vocab.items()}
+    generated_tokens = [inv_vocab.get(tok, '<UNK>') for tok in generated]
+    return ' '.join(generated_tokens)
+
+@app.route('/', methods=['GET'])
+def index():
+    return render_template('index.html')
+
+@app.route('/chat', methods=['POST'])
+def chat():
+    message = request.form.get('message')
+    if not message:
+        return render_template('index.html')
+
+    # Simulate thinking delay
+    delay = random.randint(1, 10)
+    print(f"Thinking for {delay} seconds...")
+    time.sleep(delay)
+
+    response = generate_text(message)
+    return render_template('index.html', message=message, response=response)
+
+if __name__ == '__main__':
+    app.run(host='0.0.0.0', port=5000)
+

src/data.py

@@ -0,0 +1,27 @@
+# After saving processed_data.json
+from utils import tokenize, build_vocab, save_vocab
+from utils import load_data
+import json
+
+def prepare_training_data(processed_data, vocab_path='vocab.json'):
+    tokenized_texts = []
+    for entry in processed_data:
+        if isinstance(entry, str):
+            tokens = tokenize(entry)
+            tokenized_texts.append(tokens)
+        elif isinstance(entry, list):
+            for item in entry:
+                if isinstance(item, str):
+                    tokens = tokenize(item)
+                    tokenized_texts.append(tokens)
+    vocab = build_vocab(tokenized_texts)
+    save_vocab(vocab, vocab_path)
+    return tokenized_texts, vocab
+
+if __name__ == "__main__":
+    data = load_data()
+    tokenized_texts, vocab = prepare_training_data(data)
+    # Save tokenized data
+    with open('data/processed/tokenized_data.json', 'w', encoding='utf-8') as f:
+        json.dump(tokenized_texts, f, ensure_ascii=False, indent=4)
+    print("Data processing complete. Tokenized data saved to data/processed/tokenized_data.json")

src/data_processing.py

@@ -0,0 +1,84 @@
+# src/data_processing.py
+import os
+import json
+import csv
+from pdfminer.high_level import extract_text
+import pandas as pd
+from utils import tokenize, build_vocab, save_vocab
+
+def read_txt(file_path):
+    with open(file_path, 'r', encoding='utf-8') as file:
+        return file.read()
+
+def read_pdf(file_path):
+    return extract_text(file_path)
+
+def read_json(file_path):
+    with open(file_path, 'r', encoding='utf-8') as file:
+        return json.load(file)
+
+def read_csv(file_path):
+    df = pd.read_csv(file_path)
+    # Concatenate all text columns into a single string
+    text = ' '.join(df.astype(str).values.flatten())
+    return text
+
+def process_file(file_path):
+    _, ext = os.path.splitext(file_path)
+    ext = ext.lower()
+    if ext == '.txt':
+        return read_txt(file_path)
+    elif ext == '.pdf':
+        return read_pdf(file_path)
+    elif ext == '.json':
+        return read_json(file_path)
+    elif ext == '.csv':
+        return read_csv(file_path)
+    else:
+        print(f"Unsupported file format: {ext}")
+        return None
+
+def load_data(raw_data_dir='data/raw'):
+    all_data = []
+    for root, dirs, files in os.walk(raw_data_dir):
+        for file in files:
+            file_path = os.path.join(root, file)
+            data = process_file(file_path)
+            if data:
+                all_data.append(data)
+    return all_data
+
+def prepare_training_data(processed_data, vocab_path='vocab.json'):
+    tokenized_texts = []
+    for entry in processed_data:
+        if isinstance(entry, str):
+            tokens = tokenize(entry)
+            tokenized_texts.append(tokens)
+        elif isinstance(entry, list):
+            for item in entry:
+                if isinstance(item, str):
+                    tokens = tokenize(item)
+                    tokenized_texts.append(tokens)
+    vocab = build_vocab(tokenized_texts)
+    save_vocab(vocab, vocab_path)
+    return tokenized_texts, vocab
+
+def save_tokenized_data(tokenized_texts, filepath='data/processed/tokenized_data.json'):
+    with open(filepath, 'w', encoding='utf-8') as f:
+        json.dump(tokenized_texts, f, ensure_ascii=False, indent=4)
+
+def save_processed_data(processed_data, filepath='data/processed/processed_data.json'):
+    with open(filepath, 'w', encoding='utf-8') as f:
+        json.dump(processed_data, f, ensure_ascii=False, indent=4)
+
+if __name__ == "__main__":
+    print("Loading raw data...")
+    data = load_data()
+    print(f"Loaded {len(data)} data entries.")
+
+    print("Preparing training data...")
+    tokenized_texts, vocab = prepare_training_data(data)
+    save_tokenized_data(tokenized_texts)
+    save_processed_data(data)
+    print("Data processing complete.")
+    print(f"Vocabulary size: {len(vocab)}")

src/evaluate.py

@@ -0,0 +1,111 @@
+# src/evaluate.py
+import torch
+import torch.nn as nn
+from torch.utils.data import Dataset, DataLoader
+import json
+from model import TransformerModel
+from utils import load_vocab
+from tqdm import tqdm
+import os
+
+class TextDataset(Dataset):
+    def __init__(self, data_path, vocab, seq_length=50):
+        with open(data_path, 'r', encoding='utf-8') as f:
+            self.data = json.load(f)
+        self.vocab = vocab
+        self.seq_length = seq_length
+
+    def __len__(self):
+        return len(self.data)
+
+    def numericalize(self, tokens):
+        return [self.vocab.get(token, self.vocab['<UNK>']) for token in tokens]
+
+    def __getitem__(self, idx):
+        tokens = self.data[idx]
+        numericalized = self.numericalize(tokens)
+        if len(numericalized) < self.seq_length + 1:
+            numericalized += [self.vocab['<PAD>']] * (self.seq_length + 1 - len(numericalized))
+        else:
+            numericalized = numericalized[:self.seq_length + 1]
+        input_seq = torch.tensor(numericalized[:-1], dtype=torch.long)
+        target_seq = torch.tensor(numericalized[1:], dtype=torch.long)
+        return input_seq, target_seq
+
+def collate_fn(batch):
+    inputs, targets = zip(*batch)
+    inputs = torch.stack(inputs)
+    targets = torch.stack(targets)
+    return inputs, targets
+
+def get_dataloader(data_path, vocab, batch_size=64, seq_length=50):
+    dataset = TextDataset(data_path, vocab, seq_length)
+    dataloader = DataLoader(dataset, batch_size=batch_size, shuffle=False, collate_fn=collate_fn)
+    return dataloader
+
+def evaluate_model(config):
+    # Load vocabulary
+    vocab = load_vocab(config['vocab_path'])
+    vocab_size = len(vocab)
+
+    # Initialize model
+    model = TransformerModel(
+        vocab_size=vocab_size,
+        embed_size=config['embed_size'],
+        num_heads=config['num_heads'],
+        hidden_dim=config['hidden_dim'],
+        num_layers=config['num_layers'],
+        dropout=config['dropout']
+    )
+
+    # Load model weights
+    model.load_state_dict(torch.load(config['model_path'], map_location=torch.device('cpu')))
+    model.eval()
+
+    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+    model = model.to(device)
+
+    # Loss function
+    criterion = nn.CrossEntropyLoss(ignore_index=vocab['<PAD>'])
+
+    # DataLoader
+    dataloader = get_dataloader(
+        data_path=config['data_path'],
+        vocab=vocab,
+        batch_size=config['batch_size'],
+        seq_length=config['seq_length']
+    )
+
+    total_loss = 0
+    total_tokens = 0
+
+    with torch.no_grad():
+        for inputs, targets in tqdm(dataloader, desc="Evaluating"):
+            inputs = inputs.to(device)
+            targets = targets.to(device)
+
+            src_mask = model.generate_square_subsequent_mask(inputs.size(1)).to(device)
+            outputs = model(inputs, src_mask)
+            loss = criterion(outputs.view(-1, vocab_size), targets.view(-1))
+            total_loss += loss.item() * inputs.size(0)
+            total_tokens += inputs.size(0)
+
+    average_loss = total_loss / total_tokens
+    perplexity = torch.exp(torch.tensor(average_loss))
+    print(f"Average Loss: {average_loss:.4f}")
+    print(f"Perplexity: {perplexity:.4f}")
+
+if __name__ == "__main__":
+    config = {
+        'vocab_path': 'vocab.json',
+        'data_path': 'data/processed/tokenized_data.json',
+        'model_path': 'models/3ed0k4_model_epoch10.pth',  # Update accordingly
+        'embed_size': 256,
+        'num_heads': 8,
+        'hidden_dim': 512,
+        'num_layers': 4,
+        'dropout': 0.1,
+        'batch_size': 64,
+        'seq_length': 50,
+    }
+    evaluate_model(config)

src/model.py

@@ -0,0 +1,38 @@
+# src/model.py
+import torch
+import torch.nn as nn
+
+class TransformerModel(nn.Module):
+    def __init__(self, vocab_size, embed_size, num_heads, hidden_dim, num_layers, dropout=0.1):
+        super(TransformerModel, self).__init__()
+        self.embed_size = embed_size
+        self.token_embedding = nn.Embedding(vocab_size, embed_size)
+        self.position_embedding = nn.Embedding(5000, embed_size)  # Max sequence length
+
+        encoder_layers = nn.TransformerEncoderLayer(
+            d_model=embed_size,
+            nhead=num_heads,
+            dim_feedforward=hidden_dim,
+            dropout=dropout
+        )
+        self.transformer_encoder = nn.TransformerEncoder(encoder_layers, num_layers=num_layers)
+
+        self.fc_out = nn.Linear(embed_size, vocab_size)
+        self.dropout = nn.Dropout(dropout)
+
+    def forward(self, src, src_mask):
+        batch_size, seq_length = src.size()
+        positions = torch.arange(0, seq_length).unsqueeze(0).repeat(batch_size, 1).to(src.device)
+
+        x = self.token_embedding(src) + self.position_embedding(positions)
+        x = self.dropout(x)
+        x = x.permute(1, 0, 2)  # Transformer expects [seq_length, batch_size, embed_size]
+        transformer_out = self.transformer_encoder(x, src_mask)
+        transformer_out = transformer_out.permute(1, 0, 2)
+        logits = self.fc_out(transformer_out)
+        return logits
+
+    def generate_square_subsequent_mask(self, sz):
+        mask = (torch.triu(torch.ones(sz, sz)) == 1).transpose(0, 1)
+        mask = mask.float().masked_fill(mask == 0, float('-inf')).masked_fill(mask == 1, float(0.0))
+        return mask

src/templates/index.html

@@ -0,0 +1,74 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>Chatbot</title>
+    <style>
+        body {
+            font-family: Arial, sans-serif;
+            text-align: center;
+        }
+
+        .container {
+            width: 50%;
+            margin: 0 auto;
+        }
+
+        .chat-container {
+            display: flex;
+            flex-direction: column;
+            align-items: center;
+            padding: 20px;
+            border: 1px solid #ddd;
+            border-radius: 10px;
+            margin-bottom: 20px;
+        }
+
+        .chat-message {
+            margin-bottom: 10px;
+        }
+
+        .chat-message:first-child {
+            margin-top: 0;
+        }
+
+        .chat-message:last-child {
+            margin-bottom: 0;
+        }
+
+        .chat-user {
+            font-weight: bold;
+        }
+
+        .chat-bot {
+            color: #666;
+        }
+    </style>
+</head>
+<body>
+    <div class="container">
+        <h1>Chatbot</h1>
+        <form action="/chat" method="POST">
+            <input type="text" name="message" placeholder="Type a message..." />
+            <button type="submit">Send</button>
+        </form>
+
+        <div class="chat-container">
+            {% if message %}
+                <div class="chat-message">
+                    <span class="chat-user">You:</span>
+                    {{ message }}
+                </div>
+            {% endif %}
+
+            {% if response %}
+                <div class="chat-message">
+                    <span class="chat-bot">Bot:</span>
+                    {{ response }}
+                </div>
+            {% endif %}
+        </div>
+    </div>
+</body>
+</html>

src/train.py

@@ -0,0 +1,123 @@
+# src/train.py
+import torch
+import torch.nn as nn
+import torch.optim as optim
+from torch.utils.data import Dataset, DataLoader
+import json
+from model import TransformerModel
+from utils import load_vocab, tokenize
+from tqdm import tqdm
+import os
+import subprocess
+
+class TextDataset(Dataset):
+    def __init__(self, data_path, vocab, seq_length=50):
+        with open(data_path, 'r', encoding='utf-8') as f:
+            self.data = json.load(f)
+        self.vocab = vocab
+        self.seq_length = seq_length
+
+    def __len__(self):
+        return len(self.data)
+
+    def numericalize(self, tokens):
+        return [self.vocab.get(token, self.vocab['<UNK>']) for token in tokens]
+
+    def __getitem__(self, idx):
+        tokens = self.data[idx]
+        numericalized = self.numericalize(tokens)
+        if len(numericalized) < self.seq_length + 1:
+            numericalized += [self.vocab['<PAD>']] * (self.seq_length + 1 - len(numericalized))
+        else:
+            numericalized = numericalized[:self.seq_length + 1]
+        input_seq = torch.tensor(numericalized[:-1], dtype=torch.long)
+        target_seq = torch.tensor(numericalized[1:], dtype=torch.long)
+        return input_seq, target_seq
+
+def collate_fn(batch):
+    inputs, targets = zip(*batch)
+    inputs = torch.stack(inputs)
+    targets = torch.stack(targets)
+    return inputs, targets
+
+def get_dataloader(data_path, vocab, batch_size=64, seq_length=50):
+    dataset = TextDataset(data_path, vocab, seq_length)
+    dataloader = DataLoader(dataset, batch_size=batch_size, shuffle=True, collate_fn=collate_fn)
+    return dataloader
+
+def train_model(config):
+    # Check if vocab.json exists
+    if not os.path.exists(config['vocab_path']):
+        print("vocab.json not found. Running data_processing.py...")
+        subprocess.run(['python', 'src/data_processing.py'], check=True)
+
+    # Load vocabulary
+    vocab = load_vocab(config['vocab_path'])
+    vocab_size = len(vocab)
+
+    # Initialize model
+    model = TransformerModel(
+        vocab_size=vocab_size,
+        embed_size=config['embed_size'],
+        num_heads=config['num_heads'],
+        hidden_dim=config['hidden_dim'],
+        num_layers=config['num_layers'],
+        dropout=config['dropout']
+    )
+
+    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+    model = model.to(device)
+
+    # Loss and optimizer
+    criterion = nn.CrossEntropyLoss(ignore_index=vocab['<PAD>'])
+    optimizer = optim.Adam(model.parameters(), lr=config['learning_rate'])
+
+    # DataLoader
+    dataloader = get_dataloader(
+        data_path=config['data_path'],
+        vocab=vocab,
+        batch_size=config['batch_size'],
+        seq_length=config['seq_length']
+    )
+
+    # Training loop
+    model.train()
+    for epoch in range(1, config['epochs'] + 1):
+        epoch_loss = 0
+        progress = tqdm(dataloader, desc=f"Epoch {epoch}/{config['epochs']}")
+        for inputs, targets in progress:
+            inputs = inputs.to(device)
+            targets = targets.to(device)
+
+            optimizer.zero_grad()
+            src_mask = model.generate_square_subsequent_mask(inputs.size(1)).to(device)
+            outputs = model(inputs, src_mask)
+            loss = criterion(outputs.view(-1, vocab_size), targets.view(-1))
+            loss.backward()
+            optimizer.step()
+
+            epoch_loss += loss.item()
+            progress.set_postfix(loss=loss.item())
+        avg_loss = epoch_loss / len(dataloader)
+        print(f"Epoch {epoch} completed. Average Loss: {avg_loss:.4f}")
+
+        # Save model after each epoch
+        os.makedirs('models', exist_ok=True)
+        torch.save(model.state_dict(), f"models/3ed0k4_model_epoch{epoch}.pth")
+        print(f"Model saved at models/3ed0k4_model_epoch{epoch}.pth")
+
+if __name__ == "__main__":
+    config = {
+        'vocab_path': 'vocab.json',
+        'data_path': 'data/processed/tokenized_data.json',
+        'embed_size': 256,
+        'num_heads': 8,
+        'hidden_dim': 512,
+        'num_layers': 4,
+        'dropout': 0.1,
+        'learning_rate': 0.001,
+        'batch_size': 64,
+        'seq_length': 50,
+        'epochs': 10
+    }
+    train_model(config)

src/upload_to_hf.py

@@ -0,0 +1,74 @@
+# src/upload_to_hf.py
+from transformers import PreTrainedTokenizerFast
+import torch
+from model import TransformerModel
+from utils import load_vocab
+import json
+import os
+
+# Configuration
+MODEL_PATH = 'models/3ed0k4_model_epoch10.pth'  # Update this path
+VOCAB_PATH = 'vocab.json'
+TOKENIZER_DIR = 'tokenizer'
+HF_MODEL_REPO = '3ed0k4/3ed0k4'  # Replace with your Hugging Face repo
+
+# Initialize tokenizer
+def init_tokenizer(vocab):
+    tokenizer = PreTrainedTokenizerFast(tokenizer_file=None)
+    tokenizer.add_tokens(list(vocab.keys()))
+    tokenizer.save_pretrained(TOKENIZER_DIR)
+    print(f"Tokenizer saved to {TOKENIZER_DIR}/")
+
+# Prepare model
+def prepare_model(vocab_size, embed_size, num_heads, hidden_dim, num_layers, dropout, model_path):
+    model = TransformerModel(
+        vocab_size=vocab_size,
+        embed_size=embed_size,
+        num_heads=num_heads,
+        hidden_dim=hidden_dim,
+        num_layers=num_layers,
+        dropout=dropout
+    )
+    model.load_state_dict(torch.load(model_path, map_location=torch.device('cpu')))
+    model.eval()
+    # Save model
+    model.save_pretrained('.')  # Saves state_dict; Hugging Face expects more
+    torch.save(model.state_dict(), 'pytorch_model.bin')
+    print("Model weights saved as pytorch_model.bin")
+
+# Create config.json
+def create_config(vocab_size, embed_size, num_heads, hidden_dim, num_layers, dropout):
+    config = {
+        "vocab_size": vocab_size,
+        "embed_size": embed_size,
+        "num_heads": num_heads,
+        "hidden_dim": hidden_dim,
+        "num_layers": num_layers,
+        "dropout": dropout
+    }
+    with open('config.json', 'w') as f:
+        json.dump(config, f, indent=4)
+    print("Config saved as config.json")
+
+if __name__ == "__main__":
+    # Load vocabulary
+    vocab = load_vocab(VOCAB_PATH)
+    vocab_size = len(vocab)
+
+    # Initialize tokenizer
+    init_tokenizer(vocab)
+
+    # Model parameters
+    embed_size = 256
+    num_heads = 8
+    hidden_dim = 512
+    num_layers = 4
+    dropout = 0.1
+
+    # Prepare and save model
+    prepare_model(vocab_size, embed_size, num_heads, hidden_dim, num_layers, dropout, MODEL_PATH)
+
+    # Create config.json
+    create_config(vocab_size, embed_size, num_heads, hidden_dim, num_layers, dropout)
+
+    print("Model preparation for Hugging Face completed.")

src/utils.py

@@ -0,0 +1,39 @@
+# src/utils.py
+import re
+from collections import Counter
+import json
+
+def tokenize(text):
+    """
+    Simple tokenizer that splits text into tokens based on whitespace and punctuation.
+    """
+    tokens = re.findall(r'\b\w+\b', text.lower())
+    return tokens
+
+def build_vocab(tokenized_texts, min_freq=2):
+    """
+    Builds a vocabulary dictionary from tokenized texts.
+    Tokens appearing fewer than `min_freq` times are excluded.
+    """
+    counter = Counter()
+    for tokens in tokenized_texts:
+        counter.update(tokens)
+    vocab = {'<PAD>': 0, '<UNK>': 1}
+    for word, freq in counter.items():
+        if freq >= min_freq:
+            vocab[word] = len(vocab)
+    return vocab
+
+def save_vocab(vocab, filepath='vocab.json'):
+    """
+    Saves the vocabulary dictionary to a JSON file.
+    """
+    with open(filepath, 'w', encoding='utf-8') as f:
+        json.dump(vocab, f, ensure_ascii=False, indent=4)
+
+def load_vocab(filepath='vocab.json'):
+    """
+    Loads the vocabulary dictionary from a JSON file.
+    """
+    with open(filepath, 'r', encoding='utf-8') as f:
+        return json.load(f)