NeuralCode7.py

import math
import random
import time

# CUSTOMIZATION

CONTEXT_WINDOW = 5  
EPOCHS = 500
LR = 0.01

def relu(x):
    return max(0.0, x)

def stable_softmax(x_list):
    if not x_list:
        return []
    m = max(x_list)
    exps = [math.exp(i - m) for i in x_list]
    s = sum(exps)
    if s == 0:
        return [1.0 / len(x_list)] * len(x_list)
    return [e / s for e in exps]

class NeuralNetwork:
    def __init__(self, layer_sizes=None, activation='relu', output_activation='softmax',
                 init_range=0.1, grad_clip=1.0, seed=None, context_window=5):
        if seed is not None:
            random.seed(seed)
        self.layer_sizes = layer_sizes[:] if layer_sizes is not None else None
        self.activation = relu if activation == 'relu' else (lambda x: x)
        self.output_activation = stable_softmax if output_activation == 'softmax' else (lambda x: x)
        self.init_range = float(init_range)
        self.grad_clip = grad_clip
        self.context_window = context_window
        self.weights = []
        self.biases = []
        self.vocab = []
        self.word_to_idx = {}
        self.idx_to_word = {}

    def prepare_data_with_context(self, text):
        words = [w.strip() for w in text.replace('\n', ' ').split(' ') if w.strip()]
        self.vocab = sorted(list(set(words)))
        self.word_to_idx = {w: i for i, w in enumerate(self.vocab)}
        self.idx_to_word = {i: w for w, i in self.word_to_idx.items()}
        
        vocab_size = len(self.vocab)
        X = []
        Y = []
        
        for i in range(len(words) - self.context_window):
            context_words = words[i : i + self.context_window]
            target_word = words[i + self.context_window]
            
            x = [0.0] * vocab_size
            for word in context_words:
                if word in self.word_to_idx:
                    x[self.word_to_idx[word]] = 1.0 
            
            y = [0.0] * vocab_size
            if target_word in self.word_to_idx:
                y[self.word_to_idx[target_word]] = 1.0
            
            X.append(x)
            Y.append(y)
            
        return X, Y

    def initialize_weights(self):
        if self.layer_sizes is None:
            raise ValueError("layer_sizes must be set before initializing weights.")
        if self.weights:
            return
        for i in range(len(self.layer_sizes) - 1):
            in_dim = self.layer_sizes[i]
            out_dim = self.layer_sizes[i + 1]
            W = [[random.uniform(-self.init_range, self.init_range) for _ in range(out_dim)] for _ in range(in_dim)]
            b = [0.0 for _ in range(out_dim)]
            self.weights.append(W)
            self.biases.append(b)

    def forward(self, x):
        a = x[:]
        for i in range(len(self.weights) - 1):
            next_a = []
            W = self.weights[i]
            b = self.biases[i]
            out_dim = len(W[0])
            for j in range(out_dim):
                s = sum(a[k] * W[k][j] for k in range(len(a))) + b[j]
                next_a.append(self.activation(s))
            a = next_a

        W = self.weights[-1]
        b = self.biases[-1]
        out = []
        out_dim = len(W[0])
        for j in range(out_dim):
            s = sum(a[k] * W[k][j] for k in range(len(a))) + b[j]
            out.append(s)
        return self.output_activation(out)

    def train(self, training_data, lr=0.01, epochs=500, verbose_every=50):
        X, Y = self.prepare_data_with_context(training_data)
        if not X:
            raise ValueError("Not enough tokens in training data to create context windows.")

        vocab_size = len(self.vocab)
        if self.layer_sizes is None:
            self.layer_sizes = [vocab_size, 64, vocab_size]
        else:
            self.layer_sizes[0] = vocab_size
            self.layer_sizes[-1] = vocab_size

        self.initialize_weights()

        for epoch in range(epochs):
            total_loss = 0.0
            indices = list(range(len(X)))
            random.shuffle(indices)

            for idx in indices:
                x = X[idx]
                y = Y[idx]

                activations = [x[:]]
                pre_acts = []
                a = x[:]
                
                for i in range(len(self.weights) - 1):
                    W, b = self.weights[i], self.biases[i]
                    z = []
                    out_dim = len(W[0])
                    for j in range(out_dim):
                        s = sum(a[k] * W[k][j] for k in range(len(a))) + b[j]
                        z.append(s)
                    pre_acts.append(z)
                    a = [self.activation(val) for val in z]
                    activations.append(a)

                W, b = self.weights[-1], self.biases[-1]
                z_final = []
                out_dim = len(W[0])
                for j in range(out_dim):
                    s = sum(a[k] * W[k][j] for k in range(len(a))) + b[j]
                    z_final.append(s)
                pre_acts.append(z_final)
                out = self.output_activation(z_final)

                delta = [out[j] - y[j] for j in range(len(y))]

                for i in reversed(range(len(self.weights))):
                    in_act = activations[i]
                    in_dim = len(in_act)
                    out_dim = len(delta)
                    
                    db = delta[:]
                    if self.grad_clip is not None:
                        db = [max(-self.grad_clip, min(self.grad_clip, g)) for g in db]
                    for j in range(len(self.biases[i])):
                        self.biases[i][j] -= lr * db[j]

                    for k in range(in_dim):
                        for j in range(out_dim):
                            grad_w = in_act[k] * delta[j]
                            if self.grad_clip is not None:
                                grad_w = max(-self.grad_clip, min(self.grad_clip, grad_w))
                            self.weights[i][k][j] -= lr * grad_w

                    if i != 0:
                        prev_delta = [0.0] * in_dim
                        for p in range(in_dim):
                            s = sum(self.weights[i][p][j] * delta[j] for j in range(out_dim))
                            if pre_acts[i-1][p] > 0:
                                prev_delta[p] = s
                        delta = prev_delta
            
            if epoch % verbose_every == 0 or epoch == epochs - 1:
                loss = 0.0
                for x_val, y_val in zip(X, Y):
                    p = self.forward(x_val)
                    for j in range(len(y_val)):
                        if y_val[j] > 0:
                            loss -= math.log(p[j] + 1e-12)
                print(f"Epoch {epoch}, Loss: {loss / len(X):.6f}")

    def export_to_python(self, filename):
        lines = []
        lines.append("import math\n")
        lines.append("import time\n\n")
        lines.append("def relu(x):\n    return max(0.0, x)\n\n")
        lines.append("def softmax(x_list):\n")
        lines.append("    if not x_list:\n")
        lines.append("        return []\n")
        lines.append("    m = max(x_list)\n")
        lines.append("    exps = [math.exp(i - m) for i in x_list]\n")
        lines.append("    s = sum(exps)\n")
        lines.append("    if s == 0:\n")
        lines.append("        return [1.0 / len(x_list)] * len(x_list)\n")
        lines.append("    return [e / s for e in exps]\n\n")

        neuron_id = 0
        for layer_idx, (W, b) in enumerate(zip(self.weights, self.biases)):
            in_dim, out_dim = len(W), len(W[0])
            for j in range(out_dim):
                terms = " + ".join([f"{W[i][j]:.8f}*inputs[{i}]" for i in range(in_dim)]) or "0.0"
                b_term = f"{b[j]:.8f}"
                if layer_idx != len(self.weights) - 1:
                    lines.append(f"def neuron_{neuron_id}(inputs):\n    return relu({terms} + {b_term})\n\n")
                else:
                    lines.append(f"def neuron_{neuron_id}(inputs):\n    return {terms} + {b_term}\n\n")
                neuron_id += 1

        neuron_counter = 0
        for layer_idx, (W, b) in enumerate(zip(self.weights, self.biases)):
            out_dim = len(W[0])
            lines.append(f"def layer_{layer_idx}(inputs):\n")
            inner = ", ".join([f"neuron_{neuron_counter + j}(inputs)" for j in range(out_dim)])
            lines.append(f"    return [{inner}]\n\n")
            neuron_counter += out_dim

        lines.append("def predict(inputs):\n")
        lines.append("    a = inputs\n")
        for i in range(len(self.weights)):
            lines.append(f"    a = layer_{i}(a)\n")
        lines.append("    return softmax(a)\n\n")

        lines.append(f"vocab = {self.vocab}\n")
        lines.append(f"word_to_idx = {{w: i for i, w in enumerate(vocab)}}\n")
        lines.append(f"context_window = {self.context_window}\n\n")

        lines.append("if __name__ == '__main__':\n")
        lines.append("    print('Interactive multi-word text completion.')\n")
        lines.append("    print(f'Model context window: {context_window} words. Type text or empty to exit.')\n")
        lines.append("    while True:\n")
        lines.append("        inp = input('> ').strip()\n")
        lines.append("        if not inp:\n")
        lines.append("            break\n")
        lines.append("        words = [w.strip() for w in inp.split(' ') if w.strip()]\n")
        lines.append("        generated_words = words[:]\n")
        lines.append("        print('Input:', ' '.join(generated_words), end='', flush=True)\n")
        lines.append("        for _ in range(20):\n")
        lines.append("            context = generated_words[-context_window:]\n")
        lines.append("            x = [0.0] * len(vocab)\n")
        lines.append("            for word in context:\n")
        lines.append("                if word in word_to_idx:\n")
        lines.append("                    x[word_to_idx[word]] = 1.0\n")
        lines.append("            out = predict(x)\n")
        lines.append("            idx = out.index(max(out))\n")
        lines.append("            next_word = vocab[idx]\n")
        lines.append("            if next_word == '<|endoftext|>': break\n")
        lines.append("            generated_words.append(next_word)\n")
        lines.append("            print(' ' + next_word, end='', flush=True)\n")
        lines.append("            time.sleep(0.1)\n")
        lines.append("        print('\\n')\n")

        with open(filename, "w") as f:
            f.writelines(lines)
        print(f"Exported network to {filename}")

    @staticmethod
    def load_network(filename):
        ns = {"__name__": "__loaded_model__"}
        with open(filename, "r") as f:
            code = f.read()
        exec(code, ns)
        class ModelWrapper:
            def __init__(self, ns):
                self.ns = ns
                self.vocab = ns.get("vocab", [])
                self.word_to_idx = ns.get("word_to_idx", {})
                self.context_window = ns.get("context_window", 5)

            def complete(self, input_text, max_new_words=20):
                words = [w.strip() for w in input_text.strip().split(' ') if w.strip()]
                generated = words[:]
                for _ in range(max_new_words):
                    context = generated[-self.context_window:]
                    x = [0.0] * len(self.vocab)
                    for word in context:
                        if word in self.word_to_idx:
                            x[self.word_to_idx[word]] = 1.0
                    
                    out = self.ns["predict"](x)
                    idx = out.index(max(out))
                    next_word = self.vocab[idx]
                    
                    if next_word == '<|endoftext|>':
                        break
                    generated.append(next_word)
                return ' '.join(generated)

        return ModelWrapper(ns)


if __name__ == "__main__":
    sample_text = """
user: hi
ai: Hello! How can I help you today?
<|endoftext|>
user: hi
ai: Hi! What can I do for you today?
<|endoftext|>
user: hello
ai: Hello! How can I help you today?
<|endoftext|>
user: hey
ai: Hi! What can I do for you today?
<|endoftext|>
user: How's your day going?
ai: It's been great! Thanks for asking! How about yours?
<|endoftext|>
user: What's new with you?
ai: Not much, just here and ready to help! What's new with you?
<|endoftext|>
user: What can you do?
ai: I can help you with a variety of tasks. What's on your mind?
<|endoftext|>
user: Tell me a joke.
ai: Why did the scarecrow win an award? Because he was outstanding in his field!
<|endoftext|>
"""
    nn = NeuralNetwork(context_window=CONTEXT_WINDOW, seed=42)
    nn.train(training_data=sample_text, lr=LR, epochs=EPOCHS, verbose_every=100)
    nn.export_to_python("exported_model.py")

    model = NeuralNetwork.load_network("exported_model.py")
    print("\n--- Testing loaded model ---")
    print(f"Vocabulary size: {len(model.vocab)}")
    
    test_inputs = ["user: hi", "user: What's new", "ai: It's been"]
    for test_input in test_inputs:
        completion = model.complete(test_input, max_new_words=10)
        print(f"Input: '{test_input}'\nOutput: '{completion}'\n")