раздели нагрузку скрипта на все ядра cpu const fs = require('fs').promises;

// Функции активации и их производные
function relu(x) {
    return x > 0 ? x : 0;
}

function reluDerivative(x) {
    return x > 0 ? 1 : 0;
}

function softmax(arr) {
    const max = Math.max(...arr);
    const exps = arr.map(x => Math.exp(x - max));
    const sum = exps.reduce((a, b) => a + b, 0);
    return exps.map(x => x / sum);
}

// Функция для токенизации предложения
function tokenize(sentence) {
    return sentence
        .toLowerCase()
        .replace(/[.,!?]/g, '') // Удаление знаков препинания
        .split(' ')
        .filter(word => word);
}

// Функция для создания TF-IDF векторов
class TfidfVectorizer {
    constructor(vocabulary) {
        this.vocabulary = vocabulary;
        this.idf = new Array(vocabulary.length).fill(0);
    }

    fit(docs) {
        const docCount = docs.length;
        const termDocCounts = new Array(this.vocabulary.length).fill(0);

        docs.forEach(doc => {
            const tokens = new Set(tokenize(doc));
            tokens.forEach(word => {
                let index = this.vocabulary.indexOf(word);
                if (index !== -1) termDocCounts[index]++;
            });
        });

        this.idf = this.vocabulary.map((word, idx) => {
            return Math.log((1 + docCount) / (1 + termDocCounts[idx])) + 1;
        });
    }

    transform(doc) {
        let tf = new Array(this.vocabulary.length).fill(0);
        tokenize(doc).forEach(word => {
            let index = this.vocabulary.indexOf(word);
            if (index !== -1) {
                tf[index]++;
            }
        });
        let tfidf = tf.map((termFreq, idx) => termFreq * this.idf[idx]);
        return tfidf;
    }
}

// Инициализация нейронной сети с Adam-оптимизатором
class NeuralNetwork {
    constructor(input_nodes, hidden_nodes, output_nodes) {
        this.input_nodes = input_nodes;
        this.hidden_nodes = hidden_nodes;
        this.output_nodes = output_nodes;

        this.weights_ih = this.initWeights(this.hidden_nodes, this.input_nodes);
        this.weights_ho = this.initWeights(this.output_nodes, this.hidden_nodes);

        this.bias_h = new Array(this.hidden_nodes).fill(0).map(() => Math.random() * 2 - 1);
        this.bias_o = new Array(this.output_nodes).fill(0).map(() => Math.random() * 2 - 1);

        this.learningRate = 0.001;
        this.beta1 = 0.9;
        this.beta2 = 0.999;
        this.epsilon = 1e-8;

        this.m_ih = this.initZeroes(this.hidden_nodes, this.input_nodes);
        this.v_ih = this.initZeroes(this.hidden_nodes, this.input_nodes);
        this.m_ho = this.initZeroes(this.output_nodes, this.hidden_nodes);
        this.v_ho = this.initZeroes(this.output_nodes, this.hidden_nodes);
    }

    initWeights(rows, cols) {
        return new Array(rows).fill(0).map(() =>
            new Array(cols).fill(0).map(() => (Math.random() * 2 - 1) * Math.sqrt(2 / (rows + cols)))
        );
    }

    initZeroes(rows, cols) {
        return new Array(rows).fill(0).map(() => new Array(cols).fill(0));
    }

    feedforward(input_array) {
        let hidden = this.weights_ih.map((weights, i) =>
            weights.reduce((sum, w, j) => sum + w * input_array[j], this.bias_h[i])
        ).map(x => relu(x));

        let output = this.weights_ho.map((weights, i) =>
            weights.reduce((sum, w, j) => sum + w * hidden[j], this.bias_o[i])
        );

        output = softmax(output);

        return output;
    }

    train(input_array, target_array, t) {
        let hidden_inputs = this.weights_ih.map((weights, i) =>
            weights.reduce((sum, w, j) => sum + w * input_array[j], this.bias_h[i])
        );

        let hidden_outputs = hidden_inputs.map(x => relu(x));

        let final_inputs = this.weights_ho.map((weights, i) =>
            weights.reduce((sum, w, j) => sum + w * hidden_outputs[j], this.bias_o[i])
        );

        let final_outputs = softmax(final_inputs);

        let output_errors = final_outputs.map((output, i) => final_outputs[i] - target_array[i]);

        let hidden_errors = new Array(this.hidden_nodes).fill(0);

        for (let i = 0; i < this.output_nodes; i++) {
            for (let j = 0; j < this.hidden_nodes; j++) {
                hidden_errors[j] += this.weights_ho[i][j] * output_errors[i];
            }
        }

        let gradients_ho = this.initZeroes(this.output_nodes, this.hidden_nodes);
        for (let i = 0; i < this.output_nodes; i++) {
            for (let j = 0; j < this.hidden_nodes; j++) {
                gradients_ho[i][j] = output_errors[i] * hidden_outputs[j];
            }
            this.bias_o[i] -= this.learningRate * output_errors[i];
        }

        let gradients_ih = this.initZeroes(this.hidden_nodes, this.input_nodes);
        for (let i = 0; i < this.hidden_nodes; i++) {
            let hidden_gradient = hidden_errors[i] * reluDerivative(hidden_inputs[i]);
            for (let j = 0; j < this.input_nodes; j++) {
                gradients_ih[i][j] = hidden_gradient * input_array[j];
            }
            this.bias_h[i] -= this.learningRate * hidden_gradient;
        }

        this.updateWeights(this.weights_ho, gradients_ho, this.m_ho, this.v_ho, t);
        this.updateWeights(this.weights_ih, gradients_ih, this.m_ih, this.v_ih, t);
    }

    updateWeights(weights, gradients, m, v, t) {
        for (let i = 0; i < weights.length; i++) {
            for (let j = 0; j < weights[i].length; j++) {
                m[i][j] = this.beta1 * m[i][j] + (1 - this.beta1) * gradients[i][j];
                v[i][j] = this.beta2 * v[i][j] + (1 - this.beta2) * gradients[i][j] ** 2;

                let m_hat = m[i][j] / (1 - this.beta1 ** t);
                let v_hat = v[i][j] / (1 - this.beta2 ** t);

                weights[i][j] -= this.learningRate * m_hat / (Math.sqrt(v_hat) + this.epsilon);
            }
        }
    }
}

// Функция перемешивания массива
function shuffleArray(array) {
    for (let i = array.length - 1; i > 0; i--) {
        const j = Math.floor(Math.random() * (i + 1));
        [array[i], array[j]] = [array[j], array[i]];
    }
}

// Основная часть программы
(async () => {
    try {
        const data = await fs.readFile('bd.json', 'utf8');
        const jsonData = JSON.parse(data);

        const training_data = jsonData.map(item => ({
            question: item.question,
            answer: item.answer
        }));

        const uniqueAnswers = [...new Set(training_data.map(pair => pair.answer))];

        const answerToIndex = {};
        const indexToAnswer = {};
        uniqueAnswers.forEach((answer, index) => {
            answerToIndex[answer] = index;
            indexToAnswer[index] = answer;
        });

        let vocabularySet = new Set();
        training_data.forEach(pair => {
            tokenize(pair.question).forEach(word => {
                vocabularySet.add(word);
            });
        });
        const vocabulary = [...vocabularySet];

        const vectorizer = new TfidfVectorizer(vocabulary);
        const all_questions = training_data.map(pair => pair.question);
        vectorizer.fit(all_questions);

        function prepareData(data, vectorizer) {
            let inputs = [];
            let targets = [];
            data.forEach(pair => {
                let input = vectorizer.transform(pair.question);
                inputs.push(input);

                let targetIndex = answerToIndex[pair.answer];
                let target = new Array(uniqueAnswers.length).fill(0);
                target[targetIndex] = 1;
                targets.push(target);
            });
            return { inputs, targets };
        }

        let prepared = prepareData(training_data, vectorizer);

        let nn = new NeuralNetwork(vectorizer.vocabulary.length, 64, uniqueAnswers.length);

        function trainNetwork(nn, trainingInputs, trainingOutputs, epochs) {
            for (let epoch = 0; epoch < epochs; epoch++) {
                let indices = [...Array(trainingInputs.length).keys()];
                shuffleArray(indices);

                for (let i = 0; i < indices.length; i++) {
                    let index = indices[i];
                    nn.train(trainingInputs[index], trainingOutputs[index], epoch * trainingInputs.length + i + 1);
                }

                if ((epoch + 1) % 10 === 0 || epoch === 0) {
                    let loss = 0;
                    for (let i = 0; i < trainingInputs.length; i++) {
                        let output = nn.feedforward(trainingInputs[i]);
                        let targetIndex = trainingOutputs[i].indexOf(1);
                        loss -= Math.log(output[targetIndex] + 1e-15);
                    }
                    let accuracy = evaluateAccuracy(nn, trainingInputs, trainingOutputs);
                    console.log(`Эпоха ${epoch + 1}/${epochs}, Потеря: ${loss.toFixed(4)}, Точность: ${accuracy.toFixed(2)}%`);
                }
            }
        }

        function evaluateAccuracy(nn, inputs, outputs) {
            let correct = 0;
            for (let i = 0; i < inputs.length; i++) {
                let prediction = nn.feedforward(inputs[i]);
                let predictedIndex = prediction.indexOf(Math.max(...prediction));
                let actualIndex = outputs[i].indexOf(1);
                if (predictedIndex === actualIndex) {
                    correct++;
                }
            }
            return (correct / inputs.length) * 100;
        }

        let epochs = 500;
        trainNetwork(nn, prepared.inputs, prepared.targets, epochs);

        function getAnswer(question) {
            let input = vectorizer.transform(question);
            input = input.map(x => Number(x));
            let output = nn.feedforward(input);
            let predictedIndex = output.indexOf(Math.max(...output));
            let answer = indexToAnswer[predictedIndex] || "Извините, я не понял ваш вопрос.";
            return answer;
        }

        const test_questions = [
            "привет",
            "как дела",
            "что тебя зовут",
            "как тебя зовут",
            "рад тебя видеть",
            "пока",
            "до свидания",
            "добрый день",
            "приветствую",
            "до встречи",
            "как твои дела",
            "что за фигня",
            "спасибо",
            "пожалуйста",
            "как работает твой мозг",
            "расскажи шутку",
            "что ты умеешь",
            "сколько тебе лет",
            "где ты находишься",
            "расскажи анекдот"
        ];

        test_questions.forEach(q => {
            let ans = getAnswer(q);
            console.log(`Вопрос: "${q}" => Ответ: "${ans}"`);
        });

    } catch (error) {
        console.error("Ошибка:", error);
    }
})();