Repository: dotnet/machinelearning (2025-08-09T23:29:27Z)

Issue:
Incorporate new harmony tiktoken vocabulary
The recently released OSS openai models come with a new vocabulary. The TiktokenTokenizer should be updated accordingly.

https://github.com/openai/tiktoken/commit/3591ff175d6a80efbe4fcc7f0e219ddd4b8c52f1

cc: @tarekgh


Discussion:
Interesting, I logged issue in tiktoken repo https://github.com/openai/tiktoken/issues/421 thought the models are using the same `4o` vocabs according to https://huggingface.co/blog/welcome-openai-gpt-oss. I'll follow up.
Looks it is using the same `4o` vocab `o200k_base`, special tokens could be different though.
> special tokens could be different though

Yes, it uses the same base, but has a bunch of special tokens as outlined in the commit I linked to in the opening post.



Patch:
diff --git a/src/Microsoft.ML.Tokenizers/Model/TiktokenTokenizer.cs b/src/Microsoft.ML.Tokenizers/Model/TiktokenTokenizer.cs
index 69cc3dfb9c..0b9e64cec9 100644
--- a/src/Microsoft.ML.Tokenizers/Model/TiktokenTokenizer.cs
+++ b/src/Microsoft.ML.Tokenizers/Model/TiktokenTokenizer.cs
@@ -1013,6 +1013,15 @@ public override OperationStatus Decode(IEnumerable<int> ids, Span<char> destinat
         private const string IMStart = "
";
         private const string IMEnd = "
";
         private const string IMSep = "
";
+        private const string StartOfText = "
";
+        private const string Return = "
";
+        private const string Constrain = "
";
+        private const string Channel = "
";
+        private const string Start = "
";
+        private const string End = "
";
+        private const string Message = "
";
+        private const string Call = "
";
+        private const string ReservedPrefix = "<|reserved_";
 
         private enum ModelEncoding
         {
@@ -1022,40 +1031,69 @@ private enum ModelEncoding
             P50kEdit,
             R50kBase,
             GPT2,
-            O200kBase
+            O200kBase,
+            O200kHarmony
         }
 
         private const string Phi4ModelName = "phi-4";
 
         private static readonly (string Prefix, ModelEncoding Encoding)[] _modelPrefixToEncoding =
                                                             [
-                                                                // chat
                                                                 ( "o1-", ModelEncoding.O200kBase ),       // e.g. o1-mini
                                                                 ( "o3-", ModelEncoding.O200kBase ),       // e.g. o3-mini
+                                                                ( "o4-mini-", ModelEncoding.O200kBase ),  // e.g. o4-mini
+
+                                                                // chat
+                                                                ( "gpt-5-", ModelEncoding.O200kBase),
                                                                 ( "gpt-4.1-", ModelEncoding.O200kBase),   // e.g., gpt-4.1-mini
+                                                                ( "gpt-4.5-", ModelEncoding.O200kBase),   // e.g., gpt-4.5
                                                                 ( "gpt-4o-", ModelEncoding.O200kBase),    // e.g., gpt-4o-2024-05-13
+                                                                ( "chatgpt-4o-", ModelEncoding.O200kBase),
                                                                 ( "gpt-4-", ModelEncoding.Cl100kBase),    // e.g., gpt-4-0314, etc., plus gpt-4-32k
                                                                 ( "gpt-3.5-", ModelEncoding.Cl100kBase),  // e.g, gpt-3.5-turbo-0301, -0401, etc.
-                                                                ( "gpt-35-", ModelEncoding.Cl100kBase )   // Azure deployment name
+                                                                ( "gpt-35-", ModelEncoding.Cl100kBase ),  // Azure deployment name
+                                                                ( "gpt-oss-", ModelEncoding.O200kHarmony ),
+
+                                                                // fine-tuned
+                                                                ( "ft:gpt-4o", ModelEncoding.O200kBase ),
+                                                                ( "ft:gpt-4", ModelEncoding.Cl100kBase ),
+                                                                ( "ft:gpt-3.5-turbo", ModelEncoding.Cl100kBase ),
+                                                                ( "ft:davinci-002", ModelEncoding.Cl100kBase ),
+                                                                ( "ft:babbage-002", ModelEncoding.Cl100kBase ),
                                                             ];
 
         private static readonly Dictionary<string, ModelEncoding> _modelToEncoding =
                                                             new Dictionary<string, ModelEncoding>(StringComparer.OrdinalIgnoreCase)
                                                             {
-                                                                // chat
-                                                                { "gpt-4o", ModelEncoding.O200kBase },
+                                                                // reasoning
                                                                 { "o1", ModelEncoding.O200kBase },
                                                                 { "o3", ModelEncoding.O200kBase },
                                                                 { "o4-mini", ModelEncoding.O200kBase },
+
+                                                                // chat
+                                                                { "gpt-5", ModelEncoding.O200kBase },
                                                                 { "gpt-4.1", ModelEncoding.O200kBase },
+                                                                { "gpt-4o", ModelEncoding.O200kBase },
                                                                 { "gpt-4", ModelEncoding.Cl100kBase },
                                                                 { "gpt-3.5-turbo", ModelEncoding.Cl100kBase },
+                                                                { "gpt-3.5", ModelEncoding.Cl100kBase },
                                                                 { "gpt-3.5-turbo-16k", ModelEncoding.Cl100kBase },
                                                                 { "gpt-35", ModelEncoding.Cl100kBase },           // Azure deployment name
                                                                 { "gpt-35-turbo", ModelEncoding.Cl100kBase },     // Azure deployment name
                                                                 { "gpt-35-turbo-16k", ModelEncoding.Cl100kBase }, // Azure deployment name
 
-                                                                // text
+                                                                // Base
+                                                                { "davinci-002", ModelEncoding.Cl100kBase },
+                                                                { "babbage-002", ModelEncoding.Cl100kBase },
+
+                                                                // embeddings
+                                                                // https://platform.openai.com/docs/guides/embeddings/what-are-embeddings
+                                                                { "text-embedding-ada-002", ModelEncoding.Cl100kBase },
+                                                                { "text-embedding-3-small", ModelEncoding.Cl100kBase },
+                                                                { "text-embedding-3-large", ModelEncoding.Cl100kBase },
+
+                                                                // DEPRECATED MODELS
+                                                                // text (DEPRECATED)
                                                                 { "text-davinci-003", ModelEncoding.P50kBase },
                                                                 { "text-davinci-002", ModelEncoding.P50kBase },
                                                                 { "text-davinci-001", ModelEncoding.R50kBase },
@@ -1067,7 +1105,7 @@ private static readonly (string Prefix, ModelEncoding Encoding)[] _modelPrefixTo
                                                                 { "babbage", ModelEncoding.R50kBase },
                                                                 { "ada", ModelEncoding.R50kBase },
 
-                                                                // code
+                                                                // code (DEPRECATED)
                                                                 { "code-davinci-002", ModelEncoding.P50kBase },
                                                                 { "code-davinci-001", ModelEncoding.P50kBase },
                                                                 { "code-cushman-002", ModelEncoding.P50kBase },
@@ -1075,17 +1113,12 @@ private static readonly (string Prefix, ModelEncoding Encoding)[] _modelPrefixTo
                                                                 { "davinci-codex", ModelEncoding.P50kBase },
                                                                 { "cushman-codex", ModelEncoding.P50kBase },
 
-                                                                // edit
+                                                                // edit (DEPRECATED)
                                                                 { "text-davinci-edit-001", ModelEncoding.P50kEdit },
                                                                 { "code-davinci-edit-001", ModelEncoding.P50kEdit },
 
-                                                                // embeddings
-                                                                // https://platform.openai.com/docs/guides/embeddings/what-are-embeddings
-                                                                { "text-embedding-ada-002", ModelEncoding.Cl100kBase },
-                                                                { "text-embedding-3-small", ModelEncoding.Cl100kBase },
-                                                                { "text-embedding-3-large", ModelEncoding.Cl100kBase },
 
-                                                                // old embeddings
+                                                                // old embeddings (DEPRECATED)
                                                                 { "text-similarity-davinci-001", ModelEncoding.R50kBase },
                                                                 { "text-similarity-curie-001", ModelEncoding.R50kBase },
                                                                 { "text-similarity-babbage-001", ModelEncoding.R50kBase },
@@ -1099,6 +1132,7 @@ private static readonly (string Prefix, ModelEncoding Encoding)[] _modelPrefixTo
 
                                                                 // open source
                                                                 { "gpt2", ModelEncoding.GPT2 },
+                                                                { "gpt-2", ModelEncoding.GPT2 },
 
                                                                 // phi-4
                                                                 { Phi4ModelName, ModelEncoding.Cl100kBase },
@@ -1126,6 +1160,32 @@ private static ModelEncoding GetModelEncoding(string modelName)
             return encoder;
         }
 
+        private static Dictionary<string, int> CreateHarmonyEncodingSpecialTokens() =>
+            new Dictionary<string, int>
+            {
+                { StartOfText,                  199998 },
+                { EndOfText,                    199999 },
+                { $"{ReservedPrefix}200000|>",  200000 },
+                { $"{ReservedPrefix}200001|>",  200001 },
+                { Return,                       200002 },
+                { Constrain,                    200003 },
+                { $"{ReservedPrefix}200004|>",  200004 },
+                { Channel,                      200005 },
+                { Start,                        200006 },
+                { End,                          200007 },
+                { Message,                      200008 },
+                { $"{ReservedPrefix}200009|>",  200009 },
+                { $"{ReservedPrefix}200010|>",  200010 },
+                { $"{ReservedPrefix}200011|>",  200011 },
+                { Call,                         200012 },
+                { $"{ReservedPrefix}200013|>",  200013 },
+                { $"{ReservedPrefix}200014|>",  200014 },
+                { $"{ReservedPrefix}200015|>",  200015 },
+                { $"{ReservedPrefix}200016|>",  200016 },
+                { $"{ReservedPrefix}200017|>",  200017 },
+                { EndOfPrompt,                  200018 },
+            };
+
         private static (Dictionary<string, int> SpecialTokens, Regex Regex, string VocabFile, Type? DataType, string PackageName) GetTiktokenConfigurations(string modelName) => GetTiktokenConfigurations(GetModelEncoding(modelName), modelName);
 
         private static (Dictionary<string, int> SpecialTokens, Regex Regex, string VocabFile, Type? DataType, string PackageName) GetTiktokenConfigurations(ModelEncoding modelEncoding, string? modelName = null)
@@ -1157,6 +1217,9 @@ private static (Dictionary<string, int> SpecialTokens, Regex Regex, string Vocab
                 case ModelEncoding.R50kBase:
                     return (new Dictionary<string, int> { { EndOfText, 50256 } }, P50kBaseRegex(), R50RanksFile, Type.GetType(R50kBaseTypeName), R50kBasePackageName);
 
+                case ModelEncoding.O200kHarmony:
+                    return (CreateHarmonyEncodingSpecialTokens(), O200kBaseRegex(), O200kBaseFile, Type.GetType(O200kBaseTypeName), O200kBasePackageName);
+
                 default:
                     throw new NotSupportedException($"The model '{modelName ?? modelEncoding.ToString()}' is not supported.");
             }
@@ -1179,6 +1242,7 @@ private static (Dictionary<string, int> SpecialTokens, Regex Regex, string Vocab
         internal const string P50kEditEncodingName = "p50k_edit";
         internal const string R50kBaseEncodingName = "r50k_base";
         internal const string O200kBaseEncodingName = "o200k_base";
+        internal const string O200kHarmonyEncodingName = "o200k_harmony";
 
         internal const string Cl100kBasePackageName = "Microsoft.ML.Tokenizers.Data.Cl100kBase";
         internal const string Gpt2PackageName = "Microsoft.ML.Tokenizers.Data.Gpt2";
@@ -1474,6 +1538,10 @@ public static TiktokenTokenizer CreateForEncoding(string encodingName, IReadOnly
             {
                 modelEncoding = ModelEncoding.O200kBase;
             }
+            else if (encodingName.Equals(O200kHarmonyEncodingName, StringComparison.OrdinalIgnoreCase))
+            {
+                modelEncoding = ModelEncoding.O200kHarmony;
+            }
             else if (encodingName.Equals(P50kBaseEncodingName, StringComparison.OrdinalIgnoreCase))
             {
                 modelEncoding = ModelEncoding.P50kBase;


Test patch:
diff --git a/test/Microsoft.ML.Tokenizers.Tests/TiktokenTests.cs b/test/Microsoft.ML.Tokenizers.Tests/TiktokenTests.cs
index 1ace27f1fb..e7a0bf5acc 100644
--- a/test/Microsoft.ML.Tokenizers.Tests/TiktokenTests.cs
+++ b/test/Microsoft.ML.Tokenizers.Tests/TiktokenTests.cs
@@ -35,7 +35,9 @@ public class TiktokenTests
         public static Tokenizer R50kBase { get; } = TiktokenTokenizer.CreateForModel("ada");
         public static Tokenizer P50kEdit { get; } = TiktokenTokenizer.CreateForModel("text-davinci-edit-001");
         public static Tokenizer GPT4o { get; } = TiktokenTokenizer.CreateForModel("gpt-4o");
+        public static Tokenizer GPT5 { get; } = TiktokenTokenizer.CreateForModel("gpt-5");
         public static Tokenizer Phi4 { get; } = TiktokenTokenizer.CreateForModel("phi-4");
+        public static TiktokenTokenizer GptOss { get; } = TiktokenTokenizer.CreateForModel("gpt-oss-20b");
 
         [Fact]
         public async Task TestTokenizerCreation()
@@ -282,40 +284,43 @@ public void TestEncode5()
         }
 
         [Fact]
-        public void TestEncodeGpt4o()
+        public void TestEncodeO200kBaseEncoding()
         {
-            string text = ReadAndSanitizeFile("./Data/lib.rs.txt");
-            IReadOnlyList<int> encoded = GPT4o.EncodeToIds(text);
-            int idsCount = GPT4o.CountTokens(text);
+            foreach (TiktokenTokenizer tokenizer in new[] { GPT4o, GptOss, GPT5 })
+            {
+                string text = ReadAndSanitizeFile("./Data/lib.rs.txt");
+                IReadOnlyList<int> encoded = tokenizer.EncodeToIds(text);
+                int idsCount = tokenizer.CountTokens(text);
 
-            Assert.Equal(5609, encoded.Count);
-            Assert.Equal(encoded.Count, idsCount);
+                Assert.Equal(5609, encoded.Count);
+                Assert.Equal(encoded.Count, idsCount);
 
-            using (Stream stream = File.OpenRead("./Data/tokens_gpt4o.json"))
-            {
-                int[]? expected = JsonSerializer.Deserialize<int[]>(stream) as int[];
-                Assert.Equal(expected!, encoded);
-            }
+                using (Stream stream = File.OpenRead("./Data/tokens_gpt4o.json"))
+                {
+                    int[]? expected = JsonSerializer.Deserialize<int[]>(stream) as int[];
+                    Assert.Equal(expected!, encoded);
+                }
 
-            Assert.Equal(text, GPT4o.Decode(encoded));
-            TestDecodingWithSpan((GPT4o as TiktokenTokenizer)!, encoded.ToArray(), text);
+                Assert.Equal(text, tokenizer.Decode(encoded));
+                TestDecodingWithSpan(tokenizer, encoded.ToArray(), text);
 
-            text = "
Hello ⭐ World
";
+                text = "
Hello ⭐ World
";
 
-            encoded = GPT4o.EncodeToIds(text);
-            idsCount = GPT4o.CountTokens(text);
-            Assert.Equal(new List<int>() { 199999, 13225, 161181, 5922, 200018 }, encoded);
-            Assert.Equal(text, GPT4o.Decode(encoded));
-            TestDecodingWithSpan((GPT4o as TiktokenTokenizer)!, encoded.ToArray(), text);
+                encoded = tokenizer.EncodeToIds(text);
+                idsCount = tokenizer.CountTokens(text);
+                Assert.Equal(new List<int>() { 199999, 13225, 161181, 5922, 200018 }, encoded);
+                Assert.Equal(text, tokenizer.Decode(encoded));
+                TestDecodingWithSpan(tokenizer, encoded.ToArray(), text);
 
-            IReadOnlyList<EncodedToken> result = GPT4o.EncodeToTokens(text, out string? normalizedText);
+                IReadOnlyList<EncodedToken> result = tokenizer.EncodeToTokens(text, out string? normalizedText);
 
-            Assert.Equal(encoded, result.Select(token => token.Id).ToArray());
-            Assert.Equal(encoded.Count, idsCount);
-            Assert.Equal(new string[] { "
", "Hello", " ⭐", " World", "
" }, result.Select(token => token.Value).ToArray());
-            Assert.Equal(new List<(int, int)> { (0, 13), (13, 5), (18, 2), (20, 6), (26, 15) }, result.Select(token => (token.Offset.Start.Value, token.Offset.End.Value - token.Offset.Start.Value)).ToArray());
+                Assert.Equal(encoded, result.Select(token => token.Id).ToArray());
+                Assert.Equal(encoded.Count, idsCount);
+                Assert.Equal(new string[] { "
", "Hello", " ⭐", " World", "
" }, result.Select(token => token.Value).ToArray());
+                Assert.Equal(new List<(int, int)> { (0, 13), (13, 5), (18, 2), (20, 6), (26, 15) }, result.Select(token => (token.Offset.Start.Value, token.Offset.End.Value - token.Offset.Start.Value)).ToArray());
 
-            TokenizerTests.TestTokenLimits(GPT4o);
+                TokenizerTests.TestTokenLimits(tokenizer);
+            }
         }
 
         [Fact]
@@ -398,16 +403,22 @@ public void TestEncodeR50kBase()
         [InlineData("o1")]
         [InlineData("o1-")]
         [InlineData("o1-mini")]
+        [InlineData("o4-mini-")]
         [InlineData("o3")]
         [InlineData("o3-")]
         [InlineData("o3-mini")]
         [InlineData("o4-mini")]
         [InlineData("gpt-4.1")]
         [InlineData("gpt-4.1-mini")]
+        [InlineData("gpt-4.5-")]
         [InlineData("gpt-4o")]
         [InlineData("gpt-4o-")]
+        [InlineData("gpt-5")]
+        [InlineData("gpt-5-chat")]
+        [InlineData("chatgpt-4o-")]
         [InlineData("gpt-4")]
         [InlineData("gpt-4-")]
+        [InlineData("gpt-3.5")]
         [InlineData("gpt-3.5-")]
         [InlineData("gpt-3.5-turbo")]
         [InlineData("gpt-3.5-turbo-")]
@@ -424,8 +435,10 @@ public void TestEncodeR50kBase()
         [InlineData("text-babbage-001")]
         [InlineData("text-ada-001")]
         [InlineData("davinci")]
+        [InlineData("davinci-002")]
         [InlineData("curie")]
         [InlineData("babbage")]
+        [InlineData("babbage-002")]
         [InlineData("ada")]
         [InlineData("code-davinci-002")]
         [InlineData("code-davinci-001")]
@@ -449,7 +462,16 @@ public void TestEncodeR50kBase()
         [InlineData("code-search-babbage-code-001")]
         [InlineData("code-search-ada-code-001")]
         [InlineData("gpt2")]
+        [InlineData("gpt-2")]
         [InlineData("phi-4")]
+        [InlineData("gpt-oss-")]
+        [InlineData("gpt-oss-120b")]
+        [InlineData("gpt-oss-20b")]
+        [InlineData("ft:gpt-4o")]
+        [InlineData("ft:gpt-4")]
+        [InlineData("ft:gpt-3.5-turbo")]
+        [InlineData("ft:davinci-002")]
+        [InlineData("ft:babbage-002")]
         public void TestAllSupportedModelNames(string modelName)
         {
             Tokenizer tokenizer = TiktokenTokenizer.CreateForModel(modelName);
@@ -463,6 +485,7 @@ public void TestAllSupportedModelNames(string modelName)
         [InlineData("p50k_edit")]
         [InlineData("cl100k_base")]
         [InlineData("o200k_base")]
+        [InlineData("o200k_harmony")]
         public void TestAllSupportedEncodingNames(string encodingName)
         {
             Tokenizer tokenizer = TiktokenTokenizer.CreateForEncoding(encodingName);
@@ -476,6 +499,7 @@ public void TestAllSupportedEncodingNames(string encodingName)
                 "p50k_edit" => "text-davinci-edit-001",
                 "cl100k_base" => "gpt-4",
                 "o200k_base" => "gpt-4o",
+                "o200k_harmony" => "gpt-oss-120b",
                 _ => throw new ArgumentException("Invalid encoding name"),
             };
 
@@ -502,11 +526,13 @@ public void TestEncodingNamesNegativeCases()
             Assert.Throws<ArgumentException>(() => TiktokenTokenizer.CreateForEncoding("p50k_edit_"));
             Assert.Throws<ArgumentException>(() => TiktokenTokenizer.CreateForEncoding("cl100k_base_"));
             Assert.Throws<ArgumentException>(() => TiktokenTokenizer.CreateForEncoding("o200k_base_"));
+            Assert.Throws<ArgumentException>(() => TiktokenTokenizer.CreateForEncoding("o200k_harmony_"));
         }
 
         [InlineData("gpt-4")]
         [InlineData("gpt-4.1")]
         [InlineData("gpt-4o")]
+        [InlineData("gpt-5")]
         [InlineData("o1")]
         [InlineData("o3")]
         [InlineData("o4-mini")]
@@ -514,6 +540,7 @@ public void TestEncodingNamesNegativeCases()
         [InlineData("text-curie-001")]
         [InlineData("text-davinci-edit-001")]
         [InlineData("phi-4")]
+        [InlineData("gpt-oss-20b")]
         [ConditionalTheory(typeof(RemoteExecutor), nameof(RemoteExecutor.IsSupported))]
         public void TestCreationUsingModel(string modelName)
         {
@@ -757,6 +784,53 @@ public void TestPhi4SpecialCases()
             Assert.Equal(text, Phi4.Decode(encoded));
         }
 
+        [Fact]
+        public void TestOss()
+        {
+            Assert.Equal(
+                new Dictionary<string, int>
+                {
+                    { "
",     199998 },
+                    { "
",       199999 },
+                    { "
", 200000 },
+                    { "
", 200001 },
+                    { "
",          200002 },
+                    { "
",       200003 },
+                    { "
", 200004 },
+                    { "
",         200005 },
+                    { "
",           200006 },
+                    { "
",             200007 },
+                    { "
",         200008 },
+                    { "
", 200009 },
+                    { "
", 200010 },
+                    { "
", 200011 },
+                    { "
",            200012 },
+                    { "
", 200013 },
+                    { "
", 200014 },
+                    { "
", 200015 },
+                    { "
", 200016 },
+                    { "
", 200017 },
+                    { "
",     200018 },
+                }, GptOss.SpecialTokens);
+
+            string text = "


Hello World

";
+
+            IReadOnlyList<int> ids = GptOss.EncodeToIds(text);
+
+            Assert.Equal(
+                new List<int> { 199998, 200006, 200008, 13225, 5922, 200007, 199999 },
+                ids);
+            Assert.Equal(text, GptOss.Decode(ids));
+
+            Assert.Equal(new string[] { "
", "
", "
", "Hello", " World", "
", "
" },
+                GptOss.EncodeToTokens(text, out _).Select(t => t.Value).ToArray());
+
+            Assert.Equal(new List<(int, int)> { (0, 15), (15, 24), (24, 35), (35, 40), (40, 46), (46, 53), (53, 66) },
+                GptOss.EncodeToTokens(text, out _).Select(t => (t.Offset.Start.Value, t.Offset.End.Value)).ToList());
+
+            Assert.Equal(ids, GptOss.EncodeToTokens(text, out _).Select(t => t.Id).ToList());
+        }
+
         // We are not exposing the Encoder, Decoder, or Vocabulary so far. For now, use reflection to test it.
         private static IReadOnlyDictionary<ReadOnlyMemory<byte>, int>? GetEncoder(TiktokenTokenizer tiktoken)
             => typeof(TiktokenTokenizer).GetProperty("Encoder", BindingFlags.Instance | BindingFlags.NonPublic)?.GetValue(tiktoken) as IReadOnlyDictionary<ReadOnlyMemory<byte>, int>;