package main

import (
"bufio"
"encoding/csv"
"encoding/gob"
"fmt"
"io/ioutil"
"log"
"math"
"os"
"path/filepath"
"sort"
"strings"


    "gorgonia.org/gorgonia"
    "gorgonia.org/tensor"
)

// ParsedData представляет структуру для входных и выходных данных
type ParsedData struct {
Input string
Output string
}

// Константы для ограничения количества сообщений и размера словаря
const (
maxTrainMessages = 9900
maxVocabSize = 5000000
numEpochs = 10
batchSize = 150
)

// LoadFiles загружает файлы из указанной директории
func LoadFiles(dirname string, maxMessages int) ([]ParsedData, error) {
var data []ParsedData
err := filepath.WalkDir(dirname, func(path string, d os.DirEntry, err error) error {
if err != nil {
return err
}
if d.IsDir() || len(data) >= maxMessages {
return nil
}


            if strings.HasSuffix(d.Name(), ".csv") {
                    f, err := os.Open(path)
                    if err != nil {
                            return err
                    }
                    defer f.Close()

                    r := csv.NewReader(f)
                    records, err := r.ReadAll()
                    if err != nil {
                            return err
                    }

                    for i := 1; i < len(records) && len(data) < maxMessages; i++ {
                            record := records[i]
                            if len(record) >= 2 {
                                    data = append(data, ParsedData{
                                            Input:  record[0],
                                            Output: record[1],
                                    })
                            }
                    }
            } else if strings.HasSuffix(d.Name(), ".txt") {
                    content, err := ioutil.ReadFile(path)
                    if err != nil {
                            return err
                    }
                    lines := strings.Split(string(content), "\n")
                    for _, line := range lines {
                            if len(data) >= maxMessages {
                                    break
                            }
                            parts := strings.Split(line, ":")
                            if len(parts) > 1 {
                                    data = append(data, ParsedData{
                                            Input:  strings.TrimSpace(parts[0]),
                                            Output: strings.TrimSpace(parts[1]),
                                    })
                            }
                    }
            } else if strings.HasSuffix(d.Name(), ".dt") {
                    content, err := ioutil.ReadFile(path)
                    if err != nil {
                            return err
                    }
                    lines := strings.Split(string(content), "\n")
                    for _, line := range lines {
                            if len(data) >= maxMessages {
                                    break
                            }
                            parts := strings.Split(line, ":")
                            if len(parts) > 1 {
                                    data = append(data, ParsedData{
                                            Input:  strings.TrimSpace(parts[0]),
                                            Output: strings.TrimSpace(parts[1]),
                                    })
                            }
                    }
            }
            return nil
    })
    return data, err
}

func Tokenize(text string) []string {
return strings.Fields(strings.TrimSpace(text))
}

func BuildVocab(data []ParsedData) map[string]int {
vocab := make(map[string]int)
wordFreq := make(map[string]int)


    for _, item := range data {
            tokens := append(Tokenize(item.Input), Tokenize(item.Output)...)
            for _, token := range tokens {
                    wordFreq[token]++
            }
    }

    type kv struct {
            Key   string
            Value int
    }

    var freqs []kv
    for k, v := range wordFreq {
            freqs = append(freqs, kv{k, v})
    }

    sort.Slice(freqs, func(i, j int) bool {
            return freqs[i].Value > freqs[j].Value
    })

    if len(freqs) > maxVocabSize {
            freqs = freqs[:maxVocabSize]
    }

    for i, kv := range freqs {
            vocab[kv.Key] = i
    }
    return vocab
}

func Vectorize(tokens []string, vocab map[string]int) []float32 {
vector := make([]float32, len(vocab))
for _, token := range tokens {
if idx, exists := vocab[token]; exists {
vector[idx] += 1
}
}
return vector
}

// Сохранение и загрузка весов модели
func SaveWeights(weights *tensor.Dense, filename string) error {
f, err := os.Create(filename)
if err != nil {
return err
}
defer f.Close()


    encoder := gob.NewEncoder(f)
    return encoder.Encode(weights)
}

func LoadWeights(filename string) (*tensor.Dense, error) {
f, err := os.Open(filename)
if err != nil {
return nil, err
}
defer f.Close()


    var weights tensor.Dense
    decoder := gob.NewDecoder(f)
    if err := decoder.Decode(&weights); err != nil {
            return nil, err
    }
    return &weights, nil
}

// Сохранение и загрузка словаря
func SaveVocabulary(vocab map[string]int, filename string) error {
f, err := os.Create(filename)
if err != nil {
return err
}
defer f.Close()


    encoder := gob.NewEncoder(f)
    return encoder.Encode(vocab)
}

func LoadVocabulary(filename string) (map[string]int, error) {
f, err := os.Open(filename)
if err != nil {
return nil, err
}
defer f.Close()


    var vocab map[string]int
    decoder := gob.NewDecoder(f)
    if err := decoder.Decode(&vocab); err != nil {
            return nil, err
    }
    return vocab, nil
}

func trainModel(trainDir string, maxTrainMessages int) {
parsedData, err := LoadFiles(trainDir, maxTrainMessages)
if err != nil {
log.Fatalf("Error loading training files: %v", err)
}


    fmt.Printf("Loaded %d training records\n", len(parsedData))

    vocab := BuildVocab(parsedData)

    // Create models directory if it doesn't exist
    if err := os.MkdirAll("models", os.ModePerm); err != nil {
            log.Fatalf("Error creating models directory: %v", err)
    }

    // Save vocabulary to models directory
    if err := SaveVocabulary(vocab, "models/vocab.gob"); err != nil {
            log.Fatalf("Error saving vocabulary: %v", err)
    }

    totalSamples := len(parsedData)

    g := gorgonia.NewGraph()

    inputSize := len(vocab)
    w := gorgonia.NewMatrix(g, tensor.Float32, gorgonia.WithShape(inputSize, 1), gorgonia.WithInit(gorgonia.GlorotN(1.0)))

    x := gorgonia.NewMatrix(g, tensor.Float32, gorgonia.WithShape(batchSize, inputSize), gorgonia.WithName("x"))
    y := gorgonia.NewMatrix(g, tensor.Float32, gorgonia.WithShape(batchSize, 1), gorgonia.WithName("y"))

    yhat := gorgonia.Must(gorgonia.Mul(x, w))
    yhatSig := gorgonia.Must(gorgonia.Sigmoid(yhat)) // Apply sigmoid

    // Binary cross-entropy loss
    losses := gorgonia.Must(gorgonia.Sub(yhatSig, y))
    squaredLoss := gorgonia.Must(gorgonia.Square(losses))
    loss := gorgonia.Must(gorgonia.Mean(squaredLoss))

    // Create gradients
    _, err = gorgonia.Grad(loss, w)
    if err != nil {
            log.Fatalf("Failed to create gradients: %v", err)
    }

    // Initialize optimizer
    optimizer := gorgonia.NewAdamSolver(gorgonia.WithLearnRate(0.001))

    // Create the machine
    machine := gorgonia.NewTapeMachine(g, gorgonia.BindDualValues(w))
    defer machine.Close()

    for epoch := 0; epoch < numEpochs; epoch++ {
            var epochLossTotal float64
            barWidth := 50 // Width of the progress bar

            // Initialize progress bar
            fmt.Printf("Epoch %d/%d\n[", epoch+1, numEpochs)
            for i := 0; i < barWidth; i++ {
                    fmt.Print(" ")
            }
            fmt.Print("] 0%")
            fmt.Print("\r[") // Move cursor back to start of the progress bar

            for start := 0; start < totalSamples; start += batchSize {
                    end := start + batchSize
                    batchSizeActual := batchSize

                    if end > totalSamples {
                            batchSizeActual = totalSamples - start
                            end = totalSamples
                    }

                    // Initialize data with padding
                    xValData := make([]float32, batchSize*inputSize) // Padded with zeros by default
                    yValData := make([]float32, batchSize)           // Padded with zeros by default

                    for i := start; i < end; i++ {
                            tokens := Tokenize(parsedData[i].Input)
                            inputVec := Vectorize(tokens, vocab)
                            idx := i - start

                            copy(xValData[idx*inputSize:(idx+1)*inputSize], inputVec)

                            // Parse Output as float32
                            if parsedData[i].Output == "1" {
                                    yValData[idx] = 1.0
                            } else {
                                    yValData[idx] = 0.0
                            }
                    }

                    xVal := tensor.New(tensor.WithShape(batchSize, inputSize), tensor.Of(tensor.Float32), tensor.WithBacking(xValData))
                    yVal := tensor.New(tensor.WithShape(batchSize, 1), tensor.Of(tensor.Float32), tensor.WithBacking(yValData))

                    // Set node values using gorgonia.Let
                    if err := gorgonia.Let(x, xVal); err != nil {
                            log.Fatalf("Error setting x value: %v", err)
                    }
                    if err := gorgonia.Let(y, yVal); err != nil {
                            log.Fatalf("Error setting y value: %v", err)
                    }

                    if err := machine.RunAll(); err != nil {
                            log.Fatalf("Machine run error during training: %v", err)
                    }

                    // Get loss value
                    lossData := loss.Value().Data()
                    lossVal, ok := lossData.(float32)
                    if !ok {
                            log.Fatalf("Unexpected type for loss value: %T", lossData)
                    }

                    epochLossTotal += float64(lossVal) * float64(batchSizeActual)

                    // Update weights
                    nodes := gorgonia.Nodes{w}
                    err = optimizer.Step(gorgonia.NodesToValueGrads(nodes))
                    if err != nil {
                            log.Fatalf("Error during optimizer step: %v", err)
                    }

                    // Update progress bar
                    currentBatch := start + batchSize
                    if currentBatch > totalSamples {
                            currentBatch = totalSamples
                    }
                    percent := float64(currentBatch) / float64(totalSamples)
                    filledLength := int(float64(barWidth) * percent) // Convert barWidth to float64 before multiplication

                    // Move cursor to the progress bar start
                    fmt.Printf("\r[") // Reset to the start of the progress bar line
                    for i := 0; i < filledLength; i++ {
                            fmt.Print("=")
                    }
                    for i := filledLength; i < barWidth; i++ {
                            fmt.Print(" ")
                    }
                    fmt.Printf("] %3.0f%%", percent*100)

                    machine.Reset()
            }

            averageLoss := epochLossTotal / float64(totalSamples)
            // Move to the next line after progress bar completes
            fmt.Printf("\r[")
            for i := 0; i < barWidth; i++ {
                    fmt.Print("=")
            }
            fmt.Printf("] 100%% - Average Loss: %.4f\n", averageLoss)
    }

    fmt.Println("Model training complete")

    // Save weights to models directory
    if err := SaveWeights(w.Value().(*tensor.Dense), "models/model_weights.gob"); err != nil {
            log.Fatalf("Error saving model weights: %v", err)
    } else {
            fmt.Println("Model weights saved successfully.")
    }
}

// interactWithModel позволяет взаимодействовать с обученной моделью
func interactWithModel() {
weightsTensor, err := LoadWeights("models/model_weights.gob")
if err != nil {
log.Fatalf("Error loading model weights: %v", err)
}


    vocab, err := LoadVocabulary("models/vocab.gob")
    if err != nil {
            log.Fatalf("Error loading vocabulary: %v", err)
    }

    inputSize := len(vocab)
    reader := bufio.NewReader(os.Stdin)

    // Создаем граф для предсказания
    g := gorgonia.NewGraph()
    w := gorgonia.NewMatrix(g, tensor.Float32, gorgonia.WithShape(inputSize, 1), gorgonia.WithValue(weightsTensor))

    x := gorgonia.NewMatrix(g, tensor.Float32, gorgonia.WithShape(1, inputSize))
    yhat := gorgonia.Must(gorgonia.Mul(x, w))
    yhatSig := gorgonia.Must(gorgonia.Sigmoid(yhat)) // Применяем сигмоиду

    machine := gorgonia.NewTapeMachine(g, gorgonia.BindDualValues(w))
    defer machine.Close()

    for {
            fmt.Print("Ask a question (or type 'exit' to quit): ")
            text, _ := reader.ReadString('\n')
            text = strings.TrimSpace(text)

            if text == "exit" {
                    break
            }

            tokens := Tokenize(text)
            inputVec := Vectorize(tokens, vocab)

            xValData := make([]float32, inputSize)
            copy(xValData, inputVec)
            xVal := tensor.New(tensor.WithShape(1, inputSize), tensor.Of(tensor.Float32), tensor.WithBacking(xValData))

            // Устанавливаем значение узла x с помощью gorgonia.Let
            if err := gorgonia.Let(x, xVal); err != nil {
                    log.Fatalf("Error setting x value: %v", err)
            }

            if err := machine.RunAll(); err != nil {
                    log.Fatalf("Machine run error during interaction: %v", err)
            }

            predictionsVal := yhatSig.Value().Data().([]float32)
            predictedProbability := predictionsVal[0]
            predictedOutput := 0
            if predictedProbability >= 0.5 {
                    predictedOutput = 1
            }

            var response string
            if predictedOutput == 1 {
                    response = "Positive Response"
            } else {
                    response = "Negative Response"
            }

            fmt.Printf("Generated Response: %s (Probability: %.4f)\n", response, predictedProbability)
            machine.Reset()
    }
}

// validateModel выполняет валидацию модели на данных из validationDir
func validateModel(validationDir string) {
weightsTensor, err := LoadWeights("models/model_weights.gob")
if err != nil {
log.Fatalf("Error loading model weights: %v", err)
}


    vocab, err := LoadVocabulary("models/vocab.gob")
    if err != nil {
            log.Fatalf("Error loading vocabulary: %v", err)
    }

    validationData, err := LoadFiles(validationDir, maxTrainMessages) // Используйте подходящее ограничение
    if err != nil {
            log.Fatalf("Error loading validation files: %v", err)
    }

    fmt.Printf("Loaded %d validation records\n", len(validationData))

    g := gorgonia.NewGraph()
    w := gorgonia.NewMatrix(g, tensor.Float32, gorgonia.WithShape(len(vocab), 1), gorgonia.WithValue(weightsTensor))

    x := gorgonia.NewMatrix(g, tensor.Float32, gorgonia.WithShape(1, len(vocab)))
    y := gorgonia.NewMatrix(g, tensor.Float32, gorgonia.WithShape(1, 1))

    yhat := gorgonia.Must(gorgonia.Mul(x, w))
    yhatSig := gorgonia.Must(gorgonia.Sigmoid(yhat)) // Применяем сигмоиду

    // Валидация не требует градиентов, поэтому не нужно вызывать Grad
    machine := gorgonia.NewTapeMachine(g, gorgonia.BindDualValues(w))
    defer machine.Close()

    var totalLoss float64
    correct := 0
    total := len(validationData)

    for idx, item := range validationData {
            inputTokens := Tokenize(item.Input)
            inputVec := Vectorize(inputTokens, vocab)

            xValData := make([]float32, len(vocab))
            copy(xValData, inputVec)
            xVal := tensor.New(tensor.WithShape(1, len(vocab)), tensor.Of(tensor.Float32), tensor.WithBacking(xValData))

            // Парсим Output как float32
            var expectedOutput float32
            if item.Output == "1" {
                    expectedOutput = 1.0
            } else {
                    expectedOutput = 0.0
            }

            yValData := []float32{expectedOutput}
            yVal := tensor.New(tensor.WithShape(1, 1), tensor.Of(tensor.Float32), tensor.WithBacking(yValData))

            // Устанавливаем значения узлов x и y с помощью gorgonia.Let
            if err := gorgonia.Let(x, xVal); err != nil {
                    log.Fatalf("Error setting x value: %v", err)
            }
            if err := gorgonia.Let(y, yVal); err != nil {
                    log.Fatalf("Error setting y value: %v", err)
            }

            if err := machine.RunAll(); err != nil {
                    log.Fatalf("Machine run error during validation: %v", err)
            }

            predictionsVal := yhatSig.Value().Data().([]float32)
            predictedProbability := predictionsVal[0]
            predictedOutput := 0
            if predictedProbability >= 0.5 {
                    predictedOutput = 1
            }

            // Вычисляем бинарную кросс-энтропию для текущего примера
            epsilon := float32(1e-7) // Для предотвращения log(0)
            p := float32(math.Min(math.Max(float64(predictedProbability), float64(epsilon)), 1-1e-7))
            lossVal := -expectedOutput*float32(math.Log(float64(p))) - (1-expectedOutput)*float32(math.Log(float64(1-p)))
            totalLoss += float64(lossVal)

            if predictedOutput == int(expectedOutput) {
                    correct++
            }

            machine.Reset()

            // Опционально: вывод прогресса
            if (idx+1)%1000 == 0 || idx+1 == total {
                    fmt.Printf("Validated %d/%d examples\n", idx+1, total)
            }
    }

    averageLoss := totalLoss / float64(total)
    accuracy := float64(correct) / float64(total) * 100
    fmt.Printf("Validation Loss: %.4f\n", averageLoss)
    fmt.Printf("Validation Accuracy: %.2f%%\n", accuracy)
}

// main - точка входа программы
func main() {
fmt.Print("Enter mode of operation (t for training, v for validation, d for dialogue, deb for debug): ")
scanner := bufio.NewScanner(os.Stdin)
scanner.Scan()
mode := scanner.Text()


    switch mode {
    case "t":
            trainModel("fgothi", maxTrainMessages)
    case "v":
            validateModel("fgotbye")
    case "d":
            interactWithModel()
    default:
            fmt.Println("Unknown mode. Use 't' for training, 'v' for validation, 'd' for dialogue, or 'deb' for debug.")
    }
}

сделай так чтобы ИИ выдавал не позитив респонс а выдавал внятно составленный ответ созданный на основе обучений чтобы он выдавал внятный текст то етсь привет -> внятный ответ от ИИ

Enter mode of operation (t for training, v for validation, d for dialogue, deb for debug): d
Ask a question (or type 'exit' to quit): g
Generated Response: Positive Response (Probability: 0.5000)
Ask a question (or type 'exit' to quit): e
Generated Response: Positive Response (Probability: 0.5000)
Ask a question (or type 'exit' to quit): e
Generated Response: Positive Response (Probability: 0.5000)
Ask a question (or type 'exit' to quit): wq
Generated Response: Positive Response (Probability: 0.5000)
Ask a question (or type 'exit' to quit): ddw
Generated Response: Positive Response (Probability: 0.5000)
Ask a question (or type 'exit' to quit): qwdkqplfmqwoplkfjnmpqweofjqweopfjqweopfqejfopqjopfqejfopqeopfjwerkopgfwegplwernmgklwenm
Generated Response: Positive Response (Probability: 0.5000)

сделай так чтобы ИИ выдавал не позитив респонс а выдавал внятно составленный ответ созданный на основе обучений чтобы он выдавал внятный текст то етсь привет -> внятный ответ от ИИ

напиши весь код сам целиком используй только то что существует в методах и функциях сам пиши все весь код со всем сам

если нужен OneShot то напиши кго сам ведьв gorgonia его нету

не удаляй то что уже есть код не должен сатть меньше