package main

import (
	"bufio"
	"fmt"
	"log"
	"os"
	"regexp"
	"strings"

	"gorgonia.org/gorgonia"
	"gorgonia.org/tensor"
)

// Tokenizer structure to hold tokens and their indices
type Tokenizer struct {
	tokens map[string]int
	index  int
}

// Create a new Tokenizer
func NewTokenizer() *Tokenizer {
	return &Tokenizer{
		tokens: make(map[string]int),
		index:  0,
	}
}

// Tokenize a string of code into numeric tokens
func (t *Tokenizer) Tokenize(code string) []int {
	var result []int
	re := regexp.MustCompile(`\w+|\S`) // Regular expression to find tokens

	matches := re.FindAllString(code, -1)

	for _, match := range matches {
		if _, exists := t.tokens[match]; !exists {
			t.tokens[match] = t.index
			t.index++
		}
		result = append(result, t.tokens[match])
	}

	return result
}

// Read codes from a file and return them as slices of strings
func readCodesFromFile(filename string) ([]string, error) {
	var codes []string
	file, err := os.Open(filename)
	if err != nil {
		return nil, err
	}
	defer file.Close()

	scanner := bufio.NewScanner(file)
	var currentCode strings.Builder

	for scanner.Scan() {
		line := scanner.Text()
		if strings.HasPrefix(line, "```") && currentCode.Len() > 0 {
			codes = append(codes, currentCode.String())
			currentCode.Reset()
		} else if !strings.HasPrefix(line, "```") {
			currentCode.WriteString(line + "\n")
		}
	}

	if currentCode.Len() > 0 {
		codes = append(codes, currentCode.String())
	}

	return codes, scanner.Err()
}

// Preprocess codes into numeric format using the tokenizer
func preprocessCodes(codes []string) ([]int, *Tokenizer) {
	tokenizer := NewTokenizer()
	var allTokens []int

	for _, code := range codes {
		tokens := tokenizer.Tokenize(code)
		allTokens = append(allTokens, tokens...)
	}

	return allTokens, tokenizer
}

func main() {
	codes, err := readCodesFromFile("codes.txt")
	if err != nil {
		log.Fatal(err)
	}

	allTokens, tokenizer := preprocessCodes(codes)

	fmt.Println("Словарь токенов:", tokenizer.tokens)

	g := gorgonia.NewGraph()

	inputs := tensor.New(tensor.WithShape(len(allTokens), 1), tensor.WithBacking(allTokens))
	x := gorgonia.NewMatrix(g, tensor.Int, gorgonia.WithShape(len(allTokens), 1), gorgonia.WithValue(inputs))

	w := gorgonia.NewMatrix(g, tensor.Float32, gorgonia.WithShape(1, 1), gorgonia.WithName("w"), gorgonia.WithInit(gorgonia.GlorotU(1)))
	b := gorgonia.NewScalar(g, tensor.Float32, gorgonia.WithName("b"), gorgonia.WithInit(gorgonia.Zeroes()))

	z := gorgonia.Must(gorgonia.Add(gorgonia.Must(gorgonia.Mul(x, w)), b))

	loss := gorgonia.Must(gorgonia.Mean(gorgonia.Must(gorgonia.Square(gorgonia.Must(gorgonia.Sub(x, z))))))

	grads, err := gorgonia.Grad(loss, w, b)
	if err != nil {
		log.Fatal(err)
	}

	vmachine := gorgonia.NewTapeMachine(g)

	for i := 0; i < 100; i++ { // Train for several epochs
		if err = vmachine.RunAll(); err != nil {
			log.Fatal(err)
		}

		// Update weights and biases using gradients and learning rate
		learningRate := float32(0.01)

		// Get gradients
		gradWVal, err := grads[0].Value().(*tensor.Dense).MulScalar(learningRate, false) // Add 'false' for leftTensor
		if err != nil {
			log.Fatal(err)
		}

		gradBVal, err := grads[1].Value().(*tensor.Dense).MulScalar(learningRate, false) // Add 'false' for leftTensor
		if err != nil {
			log.Fatal(err)
		}

		// Update weights and biases
		wVal, err := w.Value().(*tensor.Dense).Sub(gradWVal)
		if err != nil {
			log.Fatal(err)
		}

		bVal, err := b.Value().(*tensor.Dense).Sub(gradBVal)
		if err != nil {
			log.Fatal(err)
		}

		// Create new nodes for updated weights and biases
		w = gorgonia.NewMatrix(g, tensor.Float32, gorgonia.WithShape(1, 1), gorgonia.WithValue(wVal))
		b = gorgonia.NewScalar(g, tensor.Float32, gorgonia.WithValue(bVal))

		vmachine.Reset()

		fmt.Printf("Epoch %d: Loss = %v\n", i+1, loss.Value())
	}

	fmt.Println("Обученные веса:", w.Value())
	fmt.Println("Обученное смещение:", b.Value())

	// Example code generation (printing the token dictionary)
	fmt.Println("Генерация нового кода (пример):")
	for tokenStr, tokenID := range tokenizer.tokens {
		fmt.Printf("Токен: %s -> ID: %d\n", tokenStr, tokenID)
	}
}



rt that it's safe against the go1.22 runtime. If you want to risk it, run with environment variable ASSUME_NO_MOVING_GC_UNSAFE_RISK_IT_WITH=go1.22 set. Notably, if go1.22 adds a moving garbage collector, this program is unsafe to use.




goroutine 1 [running]:
go4.org/C:/Users/79685/go/pkg/mod/go4.org/unsafe/assume-no-moving-gc@v0.0.0-20220617031537-928513b29760/untested.go:25 +0x1da