package main

import (
	"bufio"
	"fmt"
	"log"
	"os"
	"regexp"
	"strings"

	"gorgonia.org/gorgonia"
	"gorgonia.org/tensor"
)

// Tokenizer structure to hold tokens and their indices
type Tokenizer struct {
	tokens map[string]int
	index  int
}

// Create a new Tokenizer
func NewTokenizer() *Tokenizer {
	return &Tokenizer{
		tokens: make(map[string]int),
		index:  0,
	}
}

// Tokenize a string of code into numeric tokens
func (t *Tokenizer) Tokenize(code string) []int {
	var result []int
	re := regexp.MustCompile(`\w+|\S`) // Regular expression to find tokens

	matches := re.FindAllString(code, -1)

	for _, match := range matches {
		if _, exists := t.tokens[match]; !exists {
			t.tokens[match] = t.index
			t.index++
		}
		result = append(result, t.tokens[match])
	}

	return result
}

// Read codes from a file and return them as slices of strings
func readCodesFromFile(filename string) ([]string, error) {
	var codes []string
	file, err := os.Open(filename)
	if err != nil {
		return nil, err
	}
	defer file.Close()

	scanner := bufio.NewScanner(file)
	var currentCode strings.Builder

	for scanner.Scan() {
		line := scanner.Text()
		if strings.HasPrefix(line, "```") && currentCode.Len() > 0 {
			codes = append(codes, currentCode.String())
			currentCode.Reset()
		} else if !strings.HasPrefix(line, "```") {
			currentCode.WriteString(line + "\n")
		}
	}

	if currentCode.Len() > 0 {
		codes = append(codes, currentCode.String())
	}

	return codes, scanner.Err()
}

// Preprocess codes into numeric format using the tokenizer
func preprocessCodes(codes []string) ([]int, *Tokenizer) {
	tokenizer := NewTokenizer()
	var allTokens []int

	for _, code := range codes {
		tokens := tokenizer.Tokenize(code)
		allTokens = append(allTokens, tokens...)
	}

	return allTokens, tokenizer
}

func main() {
	codes, err := readCodesFromFile("codes.txt")
	if err != nil {
		log.Fatal(err)
	}

	allTokens, tokenizer := preprocessCodes(codes)

	fmt.Println("Словарь токенов:", tokenizer.tokens)

	g := gorgonia.NewGraph()

	inputs := tensor.New(tensor.WithShape(len(allTokens), 1), tensor.WithBacking(allTokens))
	x := gorgonia.NewMatrix(g, tensor.Int, gorgonia.WithShape(len(allTokens), 1), gorgonia.WithValue(inputs))

	w := gorgonia.NewMatrix(g, tensor.Float32, gorgonia.WithShape(1, 1), gorgonia.WithName("w"), gorgonia.WithInit(gorgonia.GlorotU(1)))
	b := gorgonia.NewScalar(g, tensor.Float32, gorgonia.WithName("b"), gorgonia.WithInit(gorgonia.Zeroes()))

	z := gorgonia.Must(gorgonia.Add(gorgonia.Must(gorgonia.Mul(x, w)), b))

	loss := gorgonia.Must(gorgonia.Mean(gorgonia.Must(gorgonia.Square(gorgonia.Must(gorgonia.Sub(x, z))))))

	grads, err := gorgonia.Grad(loss, w, b)
	if err != nil {
		log.Fatal(err)
	}

	vmachine := gorgonia.NewTapeMachine(g)

	for i := 0; i < 100; i++ { // Train for several epochs
		if err = vmachine.RunAll(); err != nil {
			log.Fatal(err)
		}

		// Update weights and biases using gradients and learning rate
		learningRate := float32(0.01)

		// Get gradients
		gradWVal, err := grads[0].Value().(*tensor.Dense).MulScalar(learningRate, false) // Add 'false' for leftTensor
		if err != nil {
			log.Fatal(err)
		}

		gradBVal, err := grads[1].Value().(*tensor.Dense).MulScalar(learningRate, false) // Add 'false' for leftTensor
		if err != nil {
			log.Fatal(err)
		}

		// Update weights and biases
		wVal, err := w.Value().(*tensor.Dense).Sub(gradWVal)
		if err != nil {
			log.Fatal(err)
		}

		bVal, err := b.Value().(*tensor.Dense).Sub(gradBVal)
		if err != nil {
			log.Fatal(err)
		}

		// Create new nodes for updated weights and biases
		w = gorgonia.NewMatrix(g, tensor.Float32, gorgonia.WithShape(1, 1), gorgonia.WithValue(wVal))
		b = gorgonia.NewScalar(g, tensor.Float32, gorgonia.WithValue(bVal))

		vmachine.Reset()

		fmt.Printf("Epoch %d: Loss = %v\n", i+1, loss.Value())
	}

	fmt.Println("Обученные веса:", w.Value())
	fmt.Println("Обученное смещение:", b.Value())

	// Example code generation (printing the token dictionary)
	fmt.Println("Генерация нового кода (пример):")
	for tokenStr, tokenID := range tokenizer.tokens {
		fmt.Printf("Токен: %s -> ID: %d\n", tokenStr, tokenID)
	}
}


Словарь токенов: map[":7 (:6 ):8 +:98 ,:15 -:21 .:36 0:66 1:45 ::4 ;:75 =:2 >:22 ?:20 Async:107 BindableEvent:81 Connect:95 Cooldown:58 CooldownClass:56 Disconnect:100 Event:103 Fire:99 GetService:5 Heartbeat:94 HttpService:1 Instance:90 LocalPlayer:52 Module:101 OnTimeOut:82 OneShot:78 Out:106 Parent:51 ParentFrame:35 Paused:79 PlayerGui:53 Players:10 ReleaseTimer:86 Resume:84 RunService:9 Start:83 Stop:85 Tick:80 Time:105 Timer:23ing:47 _step:40 _tick:64 _time:39 _timerLabel:46 boolean:62 callback:28 class:91 con:26 cooldown:65 deltaTime:96 end:55 events:89 example:30 export:73 false:48 finish:27 function:42 game:3 get:68 getCooldownTime:59 getDuration:67 getTick:60 if:71 isRecharged:61 local:0 new:16 nil:50 not:97 number:18 or:44 print:104 recharged:69 require:102 return:54 script:31 seconds:17 self:43 setmetatable:38 start:24 step:19 stop:25 then:72 tick:70 time:57 timerLabel:34 true:88 type:11 typeof:37 update:63 vars:87 {:13 }:29]
panic: Type inference error. Op: A × B. Children: [Matrix int, Matrix float32], OpType:Matrix a → Matrix a → Matrix a: Unable to unify while inferring type of A × B: Unification Fail: float32 ~ int cannot be unified


goroutine 1 [running]:
gorgoniaC:/Users/79685/go/pkg/mod/gorgonia.org/gorgonia@v0.9.18/gorgonia.go:19
main.maiD:/JBProjects/GOLAND/AITEST/main.go:107 +0xe18
Answer