package main

import (
        "bufio"
        "encoding/csv"
        "encoding/gob"
        "fmt"
        "io/ioutil"
        "log"
        "math"
        "math/rand"
        "os"
        "path/filepath"
        "sort"
        "strings"
        "time"

        "gorgonia.org/gorgonia"
        "gorgonia.org/tensor"
)

// ParsedData represents the structure for input and output data
type ParsedData struct {
        Input  string
        Output string
}

// Constants to limit the number of messages and vocabulary size
const (
        maxTrainMessages = 9900
        maxVocabSize     = 5000000
)

// LoadFiles loads files from the specified directory
func LoadFiles(dirname string, maxMessages int) ([]ParsedData, error) {
        var data []ParsedData
        err := filepath.WalkDir(dirname, func(path string, d os.DirEntry, err error) error {
                if err != nil {
                        return err
                }
                if d.IsDir() || len(data) >= maxMessages {
                        return nil
                }

                if strings.HasSuffix(d.Name(), ".csv") {
                        f, err := os.Open(path)
                        if err != nil {
                                return err
                        }
                        defer f.Close()

                        r := csv.NewReader(f)
                        records, err := r.ReadAll()
                        if err != nil {
                                return err
                        }

                        for i := 1; i < len(records) && len(data) < maxMessages; i++ { // Assume first line is header
                                record := records[i]
                                if len(record) >= 2 {
                                        data = append(data, ParsedData{
                                                Input:  record[0],
                                                Output: record[1],
                                        })
                                }
                        }
                } else if strings.HasSuffix(d.Name(), ".txt") {
                        content, err := ioutil.ReadFile(path)
                        if err != nil {
                                return err
                        }
                        lines := strings.Split(string(content), "\n")
                        for _, line := range lines {
                                if len(data) >= maxMessages {
                                        break
                                }
                                parts := strings.Split(line, ":")
                                if len(parts) > 1 {
                                        data = append(data, ParsedData{
                                                Input:  strings.TrimSpace(parts[0]),
                                                Output: strings.TrimSpace(parts[1]),
                                        })
                                }
                        }
                }
                return nil
        })
        return data, err
}

// Tokenize splits text into tokens
func Tokenize(text string) []string {
        return strings.Fields(strings.TrimSpace(text))
}

// BuildVocab builds a vocabulary from the dataset
func BuildVocab(data []ParsedData) map[string]int {
        vocab := make(map[string]int)
        wordFreq := make(map[string]int)

        for _, item := range data {
                tokens := append(Tokenize(item.Input), Tokenize(item.Output)...)
                for _, token := range tokens {
                        wordFreq[token]++
                }
        }

        type kv struct {
                Key   string
                Value int
        }

        var freqs []kv
        for k, v := range wordFreq {
                freqs = append(freqs, kv{k, v})
        }

        sort.Slice(freqs, func(i, j int) bool {
                return freqs[i].Value > freqs[j].Value
        })

        if len(freqs) > maxVocabSize {
                freqs = freqs[:maxVocabSize]
        }

        // Reserve index 0 for padding and 1 for unknown tokens
        vocab["<PAD>"] = 0
        vocab["<UNK>"] = 1
        for i, kv := range freqs {
                vocab[kv.Key] = i + 2
        }
        return vocab
}

// Vectorize converts tokens into a feature vector
func Vectorize(tokens []string, vocab map[string]int) []float32 {
        vector := make([]float32, len(vocab))
        for _, token := range tokens {
                if idx, exists := vocab[token]; exists {
                        vector[idx] += 1
                } else {
                        vector[vocab["<UNK>"]] += 1
                }
        }
        return vector
}

// SaveWeights saves model weights to a file
func SaveWeights(weights *tensor.Dense, filename string) error {
        f, err := os.Create(filename)
        if err != nil {
                return err
        }
        defer f.Close()

        encoder := gob.NewEncoder(f)
        return encoder.Encode(weights)
}

// LoadWeights loads model weights from a file
func LoadWeights(filename string) (*tensor.Dense, error) {
        f, err := os.Open(filename)
        if err != nil {
                return nil, err
        }
        defer f.Close()

        var weights tensor.Dense
        decoder := gob.NewDecoder(f)
        if err := decoder.Decode(&weights); err != nil {
                return nil, err
        }
        return &weights, nil
}

// SaveVocabulary saves the vocabulary to a file
func SaveVocabulary(vocab map[string]int, filename string) error {
        f, err := os.Create(filename)
        if err != nil {
                return err
        }
        defer f.Close()

        encoder := gob.NewEncoder(f)
        return encoder.Encode(vocab)
}

// LoadVocabulary loads the vocabulary from a file
func LoadVocabulary(filename string) (map[string]int, error) {
        f, err := os.Open(filename)
        if err != nil {
                return nil, err
        }
        defer f.Close()

        var vocab map[string]int
        decoder := gob.NewDecoder(f)
        if err := decoder.Decode(&vocab); err != nil {
                return nil, err
        }
        return vocab, nil
}

// cosineSimilarity calculates the cosine similarity between two vectors
func cosineSimilarity(a, b []float32) float64 {
        if len(a) != len(b) {
                return 0
        }
        var dot, normA, normB float64
        for i := 0; i < len(a); i++ {
                dot += float64(a[i] * b[i])
                normA += float64(a[i] * a[i])
                normB += float64(b[i] * b[i])
        }
        if normA == 0 || normB == 0 {
                return 0
        }
        return dot / (math.Sqrt(normA) * math.Sqrt(normB))
}

// Shuffle shuffles a slice of ParsedData
func Shuffle(data []ParsedData) {
        rand.Seed(time.Now().UnixNano())
        rand.Shuffle(len(data), func(i, j int) {
                data[i], data[j] = data[j], data[i]
        })
}

// trainModel trains the model by preparing data vectors
func trainModel(trainDir string, maxTrainMessages int) ([]ParsedData, map[string]int, error) {
        parsedData, err := LoadFiles(trainDir, maxTrainMessages)
        if err != nil {
                return nil, nil, fmt.Errorf("error loading training files: %v", err)
        }

        fmt.Printf("Loaded %d training records\n", len(parsedData))

        vocab := BuildVocab(parsedData)

        // Shuffle the data to ensure randomness
        Shuffle(parsedData)

        // Create models directory if it doesn't exist
        if err := os.MkdirAll("models", os.ModePerm); err != nil {
                return nil, nil, fmt.Errorf("error creating models directory: %v", err)
        }

        // Save vocabulary to models directory
        if err := SaveVocabulary(vocab, "models/vocab.gob"); err != nil {
                return nil, nil, fmt.Errorf("error saving vocabulary: %v", err)
        }

        // Optionally, preprocess and save input-output vectors for faster retrieval
        // Here, we skip this step for simplicity

        fmt.Println("Training data is ready.")
        return parsedData, vocab, nil
}

// interactWithModel allows interaction with the trained model
func interactWithModel(parsedData []ParsedData, vocab map[string]int) {
        reader := bufio.NewReader(os.Stdin)

        fmt.Println("You can start chatting with the AI. Type 'exit' to quit.")

        for {
                fmt.Print("You: ")
                text, err := reader.ReadString('\n')
                if err != nil {
                        log.Fatalf("Error reading input: %v", err)
                }
                text = strings.TrimSpace(text)

                if strings.ToLower(text) == "exit" {
                        fmt.Println("Goodbye!")
                        break
                }

                tokens := Tokenize(text)
                inputVec := Vectorize(tokens, vocab)

                // Find the most similar input in the training data
                var bestMatch *ParsedData
                highestSim := -1.0

                for _, data := range parsedData {
                        responseTokens := Tokenize(data.Input)
                        responseVec := Vectorize(responseTokens, vocab)
                        sim := cosineSimilarity(inputVec, responseVec)
                        if sim > highestSim {
                                highestSim = sim
                                bestMatch = &data
                        }
                }

                // If a good match is found, respond with the corresponding output
                if bestMatch != nil && highestSim > 0.1 { // Threshold can be adjusted
                        fmt.Printf("AI: %s\n", bestMatch.Output)
                } else {
                        // Default response if no good match is found
                        defaultResponses := []string{
                                "I'm not sure I understand. Could you please elaborate?",
                                "Can you tell me more about that?",
                                "Interesting! Let's discuss further.",
                                "Could you clarify that for me?",
                                "That's fascinating. What else can you share?",
                        }
                        fmt.Printf("AI: %s\n", defaultResponses[rand.Intn(len(defaultResponses))])
                }
        }
}

// validateModel performs model validation on data from validationDir
func validateModel(validationDir string, vocab map[string]int) {
        // For retrieval-based approach, validation can measure how often the most similar match has the correct or expected response
        // Here, we implement a simple validation by checking if the top match's output is as expected

        validationData, err := LoadFiles(validationDir, maxTrainMessages) // Use appropriate limit
        if err != nil {
                log.Fatalf("Error loading validation files: %v", err)
        }

        fmt.Printf("Loaded %d validation records\n", len(validationData))

        var correct int
        var total float64

        for _, item := range validationData {
                inputTokens := Tokenize(item.Input)
                inputVec := Vectorize(inputTokens, vocab)

                // Find the most similar input in the training data
                var bestMatch *ParsedData
                highestSim := -1.0

                for _, data := range validationData {
                        responseTokens := Tokenize(data.Input)
                        responseVec := Vectorize(responseTokens, vocab)
                        sim := cosineSimilarity(inputVec, responseVec)
                        if sim > highestSim {
                                highestSim = sim
                                bestMatch = &data
                        }
                }

                if bestMatch != nil && highestSim > 0.1 { // Threshold can be adjusted
                        if strings.ToLower(bestMatch.Output) == strings.ToLower(item.Output) {
                                correct++
                        }
                }
                total++
        }

        accuracy := (float64(correct) / total) * 100
        fmt.Printf("Validation Accuracy: %.2f%%\n", accuracy)
}

// main is the entry point of the program
func main() {
        fmt.Print("Enter mode of operation (t for training, v for validation, d for dialogue): ")
        scanner := bufio.NewScanner(os.Stdin)
        scanner.Scan()
        mode := scanner.Text()

        switch mode {
        case "t":
                parsedData, vocab, err := trainModel("train", maxTrainMessages)
                if err != nil {
                        log.Fatalf("Training failed: %v", err)
                }
                // Optionally, save the prepared data for faster access during interaction
                // Here, we skip saving for simplicity
                fmt.Println("Training completed successfully.")

        case "v":
                vocab, err := LoadVocabulary("models/vocab.gob")
                if err != nil {
                        log.Fatalf("Error loading vocabulary: %v", err)
                }
                validateModel("valid", vocab)

        case "d":
                vocab, err := LoadVocabulary("models/vocab.gob")
                if err != nil {
                        log.Fatalf("Error loading vocabulary: %v", err)
                }
                parsedData, err := LoadFiles("train", maxTrainMessages)
                if err != nil {
                        log.Fatalf("Error loading training data for interaction: %v", err)
                }
                interactWithModel(parsedData, vocab)

        default:
                fmt.Println("Unknown mode. Use 't' for training, 'v' for validation, or 'd' for dialogue.")
        }
}

сделай так чтобы ИИ делал нормальный ответ в d при диалоге с ним чтобы он составлял ответ на вопрос используя ДАННЫЕ ОБУЧЕНИЯ И САМ ПИСАЛ ВСЕ ПО БУКВЕ И СОСТАВЛЯЛ ВНЯТНОЕ НОРАЛЬНОЕ ПРЕДЛОЖЕНИЕ

например я пишу ему hi а он должен дать развернутый ответ типо hello i am AI по базе обучения сам ссотавить ответ

скрипт не должекн быть маленьким напиши его цилеком и полностью

ИИ должен сам составлять ответ на запрос скрипт не должен стать меньше