Repository: xenova/transformers.js
File: tests/models/bert/test_tokenization_bert.js (JavaScript)

Code before:
    MIXED_CASE_WITH_ACCENTS: {
      text: BERT_TEST_STRINGS.MIXED_CASE_WITH_ACCENTS,
      tokens: ["H", "\u00e4", "L", "L", "o</w>", "!</w>", "ho", "w</w>", "Ar", "e</w>", "yo", "U</w>", "?</w>"],
      ids: [0, 44, 158, 48, 48, 1007, 1725, 3145, 1019, 2921, 1015, 13908, 1041, 1550, 2],
      decoded: "
H\u00e4LLo! how Are yoU? 
",
    },
  },
  "Xenova/ernie-gram-zh": {
    CURRENCY: {
      text: BASE_TEST_STRINGS.CURRENCY,
      tokens: ["test", "$", "1", "r2", "#", "3", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "test"],
      ids: [1, 6943, 18005, 208, 6847, 9474, 284, 18017, 18017, 18017, 18017, 18017, 18017, 6943, 2],
      decoded: "[CLS] test $ 1 r2 # 3 [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] test [SEP]",
    },
  },
  // `model.type` field missing in tokenizer.json
  "google-bert/bert-base-cased": {
    CHINESE_LATIN_MIXED: {
      text: BERT_TEST_STRINGS.CHINESE_LATIN_MIXED,
      tokens: ["ah", "[UNK]", "[UNK]", "z", "##z"],
      ids: [101, 18257, 100, 100, 195, 1584, 102],
      decoded: "[CLS] ah [UNK] [UNK] zz [SEP]",
    },
  },
  // NFD normalizer
  "alexneakameni/language_detection": {
    DEFAULT_EXAMPLE: {
      text: NORMALIZATION_TEST_STRINGS.DEFAULT_EXAMPLE,
      tokens: ["ame", "##lie", "|", "ame", "##lie"],
      ids: [1, 21947, 31933, 70, 21947, 31933, 2],
      decoded: "[CLS] amelie | amelie [SEP]",
    },
    CANONICAL_EQUIVALENCE_NORMALIZATION: {
      text: NORMALIZATION_TEST_STRINGS.CANONICAL_EQUIVALENCE_NORMALIZATION,
      tokens: ["n", "|", "n"],
      ids: [1, 56, 70, 56, 2],
      decoded: "[CLS] n | n [SEP]",
    },
    COMPATIBILITY_NORMALIZATION: {
      text: NORMALIZATION_TEST_STRINGS.COMPATIBILITY_NORMALIZATION,
      tokens: ["[UNK]", "|", "ff"],
      ids: [1, 0, 70, 40133, 2],
      decoded: "[CLS] [UNK] | ff [SEP]",
    },
    COMBINED_EXAMPLE: {
      text: NORMALIZATION_TEST_STRINGS.COMBINED_EXAMPLE,
      tokens: ["ſ", "|", "ſ", "|", "ſ", "|", "s", "|", "s"],
      ids: [1, 121, 70, 121, 70, 121, 70, 61, 70, 61, 2],
      decoded: "[CLS] ſ | ſ | ſ | s | s [SEP]",
    },
  },

Review comment (suggestion):
```suggestion
  "onnx-community/language_detection-ONNX": {
```

Update model ID

Code after:
    MIXED_CASE_WITH_ACCENTS: {
      text: BERT_TEST_STRINGS.MIXED_CASE_WITH_ACCENTS,
      tokens: ["H", "\u00e4", "L", "L", "o</w>", "!</w>", "ho", "w</w>", "Ar", "e</w>", "yo", "U</w>", "?</w>"],
      ids: [0, 44, 158, 48, 48, 1007, 1725, 3145, 1019, 2921, 1015, 13908, 1041, 1550, 2],
      decoded: "
H\u00e4LLo! how Are yoU? 
",
    },
  },
  "Xenova/ernie-gram-zh": {
    CURRENCY: {
      text: BASE_TEST_STRINGS.CURRENCY,
      tokens: ["test", "$", "1", "r2", "#", "3", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "test"],
      ids: [1, 6943, 18005, 208, 6847, 9474, 284, 18017, 18017, 18017, 18017, 18017, 18017, 6943, 2],
      decoded: "[CLS] test $ 1 r2 # 3 [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] test [SEP]",
    },
  },
  // `model.type` field missing in tokenizer.json
  "google-bert/bert-base-cased": {
    CHINESE_LATIN_MIXED: {
      text: BERT_TEST_STRINGS.CHINESE_LATIN_MIXED,
      tokens: ["ah", "[UNK]", "[UNK]", "z", "##z"],
      ids: [101, 18257, 100, 100, 195, 1584, 102],
      decoded: "[CLS] ah [UNK] [UNK] zz [SEP]",
    },
  },
  // NFD normalizer
  "onnx-community/language_detection-ONNX": {
    DEFAULT_EXAMPLE: {
      text: NORMALIZATION_TEST_STRINGS.DEFAULT_EXAMPLE,
      tokens: ["ame", "##lie", "|", "ame", "##lie"],
      ids: [1, 21947, 31933, 70, 21947, 31933, 2],
      decoded: "[CLS] amelie | amelie [SEP]",
    },
    CANONICAL_EQUIVALENCE_NORMALIZATION: {
      text: NORMALIZATION_TEST_STRINGS.CANONICAL_EQUIVALENCE_NORMALIZATION,
      tokens: ["n", "|", "n"],
      ids: [1, 56, 70, 56, 2],
      decoded: "[CLS] n | n [SEP]",
    },
    COMPATIBILITY_NORMALIZATION: {
      text: NORMALIZATION_TEST_STRINGS.COMPATIBILITY_NORMALIZATION,
      tokens: ["[UNK]", "|", "ff"],
      ids: [1, 0, 70, 40133, 2],
      decoded: "[CLS] [UNK] | ff [SEP]",
    },
    COMBINED_EXAMPLE: {
      text: NORMALIZATION_TEST_STRINGS.COMBINED_EXAMPLE,
      tokens: ["ſ", "|", "ſ", "|", "ſ", "|", "s", "|", "s"],
      ids: [1, 121, 70, 121, 70, 121, 70, 61, 70, 61, 2],
      decoded: "[CLS] ſ | ſ | ſ | s | s [SEP]",
    },
  },