tokenizer/tokenizer_config.json

{
  "vocab_size": 32000,
  "min_freq": 2,
  "special_tokens": {
    "<pad>": 0,
    "<unk>": 1,
    "<bos>": 2,
    "<eos>": 3,
    "<system>": 4,
    "<user>": 5,
    "<assistant>": 6,
    "<|endoftext|>": 7,
    "<|newline|>": 8,
    "<|tab|>": 9,
    "<|code|>": 10,
    "<|/code|>": 11,
    "<|math|>": 12,
    "<|/math|>": 13
  },
  "technical_terms": [
    "protocol",
    "backpropagation",
    "object",
    "async",
    "validation",
    "test",
    "await",
    "boolean",
    "hypothesis",
    "f1score",
    "string",
    "html",
    "polynomial",
    "neuron",
    "class",
    "loss",
    "import",
    "promise",
    "graphql",
    "export",
    "authorization",
    "proof",
    "theorem",
    "variable",
    "authentication",
    "https",
    "regression",
    "algorithm",
    "security",
    "architecture",
    "function",
    "accuracy",
    "statistics",
    "precision",
    "css",
    "weight",
    "python",
    "sql",
    "attention",
    "float",
    "encryption",
    "docker",
    "gradient",
    "datatype",
    "transformer",
    "java",
    "formula",
    "equation",
    "network",
    "vector",
    "client",
    "callback",
    "integer",
    "http",
    "database",
    "layer",
    "javascript",
    "bias",
    "cpp",
    "matrix",
    "embedding",
    "microservice",
    "rest",
    "derivative",
    "recall",
    "tokenization",
    "xml",
    "scalability",
    "kubernetes",
    "model",
    "return",
    "neural",
    "parameter",
    "server",
    "probability",
    "deployment",
    "array",
    "api",
    "integral",
    "training",
    "forward",
    "correlation",
    "method",
    "json",
    "performance"
  ]
}