api/server/routes/tokenizer.js

const express = require('express');
const router = express.Router();
const { Tiktoken } = require('@dqbd/tiktoken/lite');
const { load } = require('@dqbd/tiktoken/load');
const registry = require('@dqbd/tiktoken/registry.json');
const models = require('@dqbd/tiktoken/model_to_encoding.json');
const requireJwtAuth = require('../../middleware/requireJwtAuth');

router.post('/', requireJwtAuth, async (req, res) => {
  try {
    const { arg } = req.body;

    // console.log('context:', arg, req.body);
    // console.log(typeof req.body === 'object' ? { ...req.body, ...req.query } : req.query);
    const model = await load(registry[models['gpt-3.5-turbo']]);
    const encoder = new Tiktoken(model.bpe_ranks, model.special_tokens, model.pat_str);
    const tokens = encoder.encode(arg.text);
    encoder.free();
    res.send({ count: tokens.length });
  } catch (e) {
    console.error(e);
    res.status(500).send(e.message);
  }
});

module.exports = router;