Repository: deepjavalibrary/djl (java)

Issue:
Returned character token spans are not correct for some inputs
## Description
For some inputs the character spans returned by `Encoding.getCharTokenSpans()` does not correspond to the correct character span in the input string.

### Expected Behavior
`Encoding.getCharTokenSpans()` should return the correct character span, and it should return the same character spans as when using the same tokenizer in Python.

### Error Message
No error message.

### Steps to reproduce
The error can be reproduced with this short Scala snippet
```
    val someText = "just some string including a £9.00 price"
    val tokenizer = HuggingFaceTokenizer.newInstance("distilbert-base-uncased")
    val encodedText = tokenizer.encode(someText)

    val charSpans = encodedText.getCharTokenSpans()
    val tokens = encodedText.getTokens()
    for (i <- 1 until charSpans.length) {
        println(s"$token: ${tokens(i)}, orig string: ${someText.substring(charSpans(i).getStart, charSpans(i).getEnd)}, span start=${charSpans(i).getStart}, span end=${charSpans(i).getEnd}")
    }
```
This script has the following output
```
token: just, orig string: just, span start=0, span end=4
token: some, orig string: some, span start=5, span end=9
token: string, orig string: string, span start=10, span end=16
token: including, orig string: including, span start=17, span end=26
token: a, orig string: a, span start=27, span end=28
token: £, orig string: £9, span start=29, span end=31
token: ##9, orig string: ., span start=31, span end=32
token: ., orig string: 0, span start=32, span end=33
token: 00, orig string: 0 , span start=33, span end=35
```
So for some reason the 6th token ("£") gets a character span of two characters, and then the rest of the spans are off by one.

Running this very similar Python script
```
tokenizer = AutoTokenizer.from_pretrained("distilbert-base-uncased")

some_string = "just some string including a £9.00 price"
encoded_input = tokenizer(some_string, return_offsets_mapping=True)

tokens = tokenizer.convert_ids_to_tokens(encoded_input["input_ids"])
for token, span in zip(tokens[1:-1], encoded_input["offset_mapping"][1:-1]):
    print(f"token: {token}, orig string: {some_string[span[0]:span[1]]}, span start={span[0]}, span end={span[1]}")
```
gives the following output
```
token: just, orig string: just, span start=0, span end=4
token: some, orig string: some, span start=5, span end=9
token: string, orig string: string, span start=10, span end=16
token: including, orig string: including, span start=17, span end=26
token: a, orig string: a, span start=27, span end=28
token: £, orig string: £, span start=29, span end=30
token: ##9, orig string: 9, span start=30, span end=31
token: ., orig string: ., span start=31, span end=32
token: 00, orig string: 00, span start=32, span end=34
token: price, orig string: price, span start=35, span end=40
```
And the 6th token ("£") gets a character span of one characters, as expected.

## Environment Info

"ai.djl.huggingface:tokenizers:0.19.0"


Pull request description:
[Tokenizer] fix char offset
## Description ##

Fixed char offset issues.
fix #2112 


Patch:
diff --git a/examples/src/main/java/ai/djl/examples/inference/timeseries/AirPassengersDeepAR.java b/examples/src/main/java/ai/djl/examples/inference/timeseries/AirPassengersDeepAR.java
index c178e1a94..5ee5fd1a2 100644
--- a/examples/src/main/java/ai/djl/examples/inference/timeseries/AirPassengersDeepAR.java
+++ b/examples/src/main/java/ai/djl/examples/inference/timeseries/AirPassengersDeepAR.java
@@ -77,14 +77,14 @@ public final class AirPassengersDeepAR {
         try (ZooModel<TimeSeriesData, Forecast> model = criteria.loadModel();
                 Predictor<TimeSeriesData, Forecast> predictor = model.newPredictor();
                 NDManager manager = NDManager.newBaseManager(null, "MXNet")) {
-            TimeSeriesData data = getTimeSeriesData(manager, new URL(url));
+            TimeSeriesData input = getTimeSeriesData(manager, new URL(url));
 
             // save data for plotting
-            NDArray target = data.get(FieldName.TARGET);
+            NDArray target = input.get(FieldName.TARGET);
             target.setName("target");
             saveNDArray(target);
 
-            Forecast forecast = predictor.predict(data);
+            Forecast forecast = predictor.predict(input);
 
             // save data for plotting. Please see the corresponding python script from
             // https://gist.github.com/Carkham/a5162c9298bc51fec648a458a3437008
diff --git a/examples/src/main/java/ai/djl/examples/inference/timeseries/M5ForecastingDeepAR.java b/examples/src/main/java/ai/djl/examples/inference/timeseries/M5ForecastingDeepAR.java
index d43c16e79..46346d08a 100644
--- a/examples/src/main/java/ai/djl/examples/inference/timeseries/M5ForecastingDeepAR.java
+++ b/examples/src/main/java/ai/djl/examples/inference/timeseries/M5ForecastingDeepAR.java
@@ -13,7 +13,9 @@
 
 package ai.djl.examples.inference.timeseries;
 
+import ai.djl.Application;
 import ai.djl.ModelException;
+import ai.djl.basicdataset.BasicDatasets;
 import ai.djl.basicdataset.tabular.utils.DynamicBuffer;
 import ai.djl.basicdataset.tabular.utils.Feature;
 import ai.djl.inference.Predictor;
@@ -22,9 +24,13 @@ import ai.djl.ndarray.NDArrays;
 import ai.djl.ndarray.NDList;
 import ai.djl.ndarray.NDManager;
 import ai.djl.ndarray.types.Shape;
+import ai.djl.repository.Artifact;
+import ai.djl.repository.MRL;
+import ai.djl.repository.Repository;
 import ai.djl.repository.zoo.Criteria;
 import ai.djl.repository.zoo.ZooModel;
 import ai.djl.timeseries.Forecast;
+import ai.djl.timeseries.SampleForecast;
 import ai.djl.timeseries.TimeSeriesData;
 import ai.djl.timeseries.dataset.FieldName;
 import ai.djl.training.util.ProgressBar;
@@ -41,10 +47,12 @@ import org.slf4j.LoggerFactory;
 import java.io.BufferedInputStream;
 import java.io.IOException;
 import java.io.InputStreamReader;
+import java.io.OutputStream;
 import java.io.Reader;
 import java.net.URL;
 import java.nio.FloatBuffer;
 import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.time.LocalDateTime;
@@ -70,17 +78,22 @@ public final class M5ForecastingDeepAR {
 
     public static Map<String, Float> predict()
             throws IOException, TranslateException, ModelException {
-        // M5 Forecasting - Accuracy dataset requires manual download
-        String pathToData = "/Desktop/m5example/m5-forecasting-accuracy";
-        Path m5ForecastFile = Paths.get(System.getProperty("user.home") + pathToData);
         NDManager manager = NDManager.newBaseManager(null, "MXNet");
-        M5Dataset dataset = M5Dataset.builder().setManager(manager).setRoot(m5ForecastFile).build();
 
+        // To use local dataset, users can load data as follows
+        // Repository repository = Repository.newInstance("local_dataset",
+        // Paths.get("rootPath/m5-forecasting-accuracy"));
+        // Then add the setting `.optRepository(repository)` to the builder below
+        M5Dataset dataset = M5Dataset.builder().setManager(manager).build();
+
+        // The modelUrl can be replaced by local model path. E.g.,
+        // String modelUrl = "rootPath/deepar.zip";
+        String modelUrl = "djl://ai.djl.mxnet/deepar/0.0.1/m5forecast";
         int predictionLength = 4;
         Criteria<TimeSeriesData, Forecast> criteria =
                 Criteria.builder()
                         .setTypes(TimeSeriesData.class, Forecast.class)
-                        .optModelUrls("djl://ai.djl.mxnet/deepar/0.0.1/m5forecast")
+                        .optModelUrls(modelUrl)
                         .optEngine("MXNet")
                         .optTranslatorFactory(new DeferredTranslatorFactory())
                         .optArgument("prediction_length", predictionLength)
@@ -104,15 +117,19 @@ public final class M5ForecastingDeepAR {
                 input.setStartTime(LocalDateTime.parse("2011-01-29T00:00"));
                 input.setField(FieldName.TARGET, pastTarget);
                 Forecast forecast = predictor.predict(input);
-                // Here we focus on the metric Weighted Root Mean Squared Scaled Error (RMSSE) same
-                // as
+                // We focus on the metric Weighted Root Mean Squared Scaled Error (RMSSE) same as
                 // https://www.kaggle.com/competitions/m5-forecasting-accuracy/overview/evaluation
                 // The error is not small compared to the data values (sale amount). This is because
                 // The model is trained on a sparse data with many zeros. This will be improved by
-                // aggregating/coarse graining the data which will appear in the next PR.
-                // TODO: coarse graining the data.
+                // aggregating/coarse graining the data. See https://github.com/Carkham/m5_blog
                 evaluator.aggregateMetrics(evaluator.getMetricsPerTs(gt, pastTarget, forecast));
                 progress.increment(1);
+
+                // save data for plotting. Please see the corresponding python script from
+                // https://gist.github.com/Carkham/a5162c9298bc51fec648a458a3437008
+                NDArray samples = ((SampleForecast) forecast).getSortedSamples();
+                samples.setName("samples");
+                saveNDArray(samples);
             }
 
             manager.close();
@@ -120,6 +137,13 @@ public final class M5ForecastingDeepAR {
         }
     }
 
+    private static void saveNDArray(NDArray array) throws IOException {
+        Path path = Paths.get("build").resolve(array.getName() + ".npz");
+        try (OutputStream os = Files.newOutputStream(path)) {
+            new NDList(new NDList(array)).encode(os, true);
+        }
+    }
+
     /**
      * M5 Forecasting - Accuracy from <a
      * href="https://www.kaggle.com/competitions/m5-forecasting-accuracy">https://www.kaggle.com/competitions/m5-forecasting-accuracy</a>
@@ -146,7 +170,14 @@ public final class M5ForecastingDeepAR {
         }
 
         private void prepare(Builder builder) throws IOException {
-            URL csvUrl = builder.root.resolve("weekly_sales_train_evaluation.csv").toUri().toURL();
+            MRL mrl = builder.getMrl();
+            Artifact artifact = mrl.getDefaultArtifact();
+            mrl.prepare(artifact, null);
+
+            Path root = mrl.getRepository().getResourceDirectory(artifact);
+            Path csvFile = root.resolve("weekly_sales_train_evaluation.csv");
+
+            URL csvUrl = csvFile.toUri().toURL();
             try (Reader reader =
                     new InputStreamReader(
                             new BufferedInputStream(csvUrl.openStream()), StandardCharsets.UTF_8)) {
@@ -197,9 +228,17 @@ public final class M5ForecastingDeepAR {
             NDManager manager;
             List<Feature> target;
             CSVFormat csvFormat;
-            Path root;
+
+            Repository repository;
+            String groupId;
+            String artifactId;
+            String version;
 
             Builder() {
+                repository = BasicDatasets.REPOSITORY;
+                groupId = BasicDatasets.GROUP_ID;
+                artifactId = "m5forecast-unittest";
+                version = "1.0";
                 csvFormat =
                         CSVFormat.DEFAULT
                                 .builder()
@@ -214,8 +253,8 @@ public final class M5ForecastingDeepAR {
                 }
             }
 
-            public Builder setRoot(Path root) {
-                this.root = root;
+            public Builder optRepository(Repository repository) {
+                this.repository = repository;
                 return this;
             }
 
@@ -227,6 +266,10 @@ public final class M5ForecastingDeepAR {
             public M5Dataset build() {
                 return new M5Dataset(this);
             }
+
+            MRL getMrl() {
+                return repository.dataset(Application.Tabular.ANY, groupId, artifactId, version);
+            }
         }
     }
 
diff --git a/examples/src/main/java/ai/djl/examples/training/TrainTimeSeries.java b/examples/src/main/java/ai/djl/examples/training/TrainTimeSeries.java
index 65c926c16..e0143ed52 100644
--- a/examples/src/main/java/ai/djl/examples/training/TrainTimeSeries.java
+++ b/examples/src/main/java/ai/djl/examples/training/TrainTimeSeries.java
@@ -15,6 +15,7 @@ package ai.djl.examples.training;
 
 import ai.djl.Model;
 import ai.djl.ModelException;
+import ai.djl.basicdataset.BasicDatasets;
 import ai.djl.basicdataset.tabular.utils.Feature;
 import ai.djl.engine.Engine;
 import ai.djl.examples.inference.timeseries.M5ForecastingDeepAR;
@@ -26,7 +27,6 @@ import ai.djl.ndarray.NDList;
 import ai.djl.ndarray.NDManager;
 import ai.djl.ndarray.types.Shape;
 import ai.djl.nn.Parameter;
-import ai.djl.repository.Repository;
 import ai.djl.timeseries.Forecast;
 import ai.djl.timeseries.TimeSeriesData;
 import ai.djl.timeseries.dataset.FieldName;
@@ -83,13 +83,6 @@ public final class TrainTimeSeries {
     }
 
     public static TrainingResult runExample(String[] args) throws IOException, TranslateException {
-        // use data path to create a custom repository
-        Repository repository =
-                Repository.newInstance(
-                        "test",
-                        Paths.get(
-                                System.getProperty("user.home")
-                                        + "/Desktop/m5-forecasting-accuracy"));
 
         Arguments arguments = new Arguments().parseArgs(args);
         try (Model model = Model.newInstance("deepar")) {
@@ -106,8 +99,7 @@ public final class TrainTimeSeries {
             int contextLength = trainingNetwork.getContextLength();
 
             M5Forecast trainSet =
-                    getDataset(
-                            trainingTransformation, repository, contextLength, Dataset.Usage.TRAIN);
+                    getDataset(trainingTransformation, contextLength, Dataset.Usage.TRAIN);
 
             try (Trainer trainer = model.newTrainer(config)) {
                 trainer.setMetrics(new Metrics());
@@ -144,13 +136,6 @@ public final class TrainTimeSeries {
 
     public static Map<String, Float> predict(String outputDir)
             throws IOException, TranslateException, ModelException {
-        Repository repository =
-                Repository.newInstance(
-                        "test",
-                        Paths.get(
-                                System.getProperty("user.home")
-                                        + "/Desktop/m5-forecasting-accuracy"));
-
         try (Model model = Model.newInstance("deepar")) {
             DeepARNetwork predictionNetwork = getDeepARModel(new NegativeBinomialOutput(), false);
             model.setBlock(predictionNetwork);
@@ -159,7 +144,6 @@ public final class TrainTimeSeries {
             M5Forecast testSet =
                     getDataset(
                             new ArrayList<>(),
-                            repository,
                             predictionNetwork.getContextLength(),
                             Dataset.Usage.TEST);
 
@@ -262,17 +246,16 @@ public final class TrainTimeSeries {
     }
 
     private static M5Forecast getDataset(
-            List<TimeSeriesTransform> transformation,
-            Repository repository,
-            int contextLength,
-            Dataset.Usage usage)
+            List<TimeSeriesTransform> transformation, int contextLength, Dataset.Usage usage)
             throws IOException {
         // In order to create a TimeSeriesDataset, you must specify the transformation of the data
         // preprocessing
         M5Forecast.Builder builder =
                 M5Forecast.builder()
                         .optUsage(usage)
-                        .optRepository(repository)
+                        .optRepository(BasicDatasets.REPOSITORY)
+                        .optGroupId(BasicDatasets.GROUP_ID)
+                        .optArtifactId("m5forecast-unittest")
                         .setTransformation(transformation)
                         .setContextLength(contextLength)
                         .setSampling(32, usage == Dataset.Usage.TRAIN);
diff --git a/examples/src/main/java/ai/djl/examples/training/transferlearning/TransferFreshFruit.java b/examples/src/main/java/ai/djl/examples/training/transferlearning/TransferFreshFruit.java
index 79c730abf..2f2b85c8e 100644
--- a/examples/src/main/java/ai/djl/examples/training/transferlearning/TransferFreshFruit.java
+++ b/examples/src/main/java/ai/djl/examples/training/transferlearning/TransferFreshFruit.java
@@ -131,13 +131,10 @@ public final class TransferFreshFruit {
         float[] mean = {0.485f, 0.456f, 0.406f};
         float[] std = {0.229f, 0.224f, 0.225f};
 
-        // If users want to use local repository, then the dataset can be loaded as follows
-        // Repository repository = Repository.newInstance("banana", Paths.get(LOCAL_FOLDER/{train OR
-        // test}));
-        // FruitsFreshAndRotten dataset =
-        //        FruitsFreshAndRotten.builder()
-        //                            .optRepository(repository)
-        //                            .build()
+        // To use local dataset, users can load it as follows
+        // Repository repository = Repository.newInstance("banana",
+        // Paths.get("local_data_root/banana/train"));
+        // Then add the setting `.optRepository(repository)` to the builder below
         FruitsFreshAndRotten dataset =
                 FruitsFreshAndRotten.builder()
                         .optUsage(usage)
diff --git a/extensions/tokenizers/rust/src/lib.rs b/extensions/tokenizers/rust/src/lib.rs
index cda085df2..92038b222 100644
--- a/extensions/tokenizers/rust/src/lib.rs
+++ b/extensions/tokenizers/rust/src/lib.rs
@@ -94,7 +94,7 @@ pub extern "system" fn Java_ai_djl_huggingface_tokenizers_jni_TokenizersLibrary_
 
     let input_sequence = tk::InputSequence::from(sequence);
     let encoded_input = EncodeInput::Single(input_sequence);
-    let encoding = tokenizer.encode(encoded_input, add_special_tokens == JNI_TRUE);
+    let encoding = tokenizer.encode_char_offsets(encoded_input, add_special_tokens == JNI_TRUE);
 
     match encoding {
         Ok(output) => to_handle(output),
@@ -127,7 +127,7 @@ pub extern "system" fn Java_ai_djl_huggingface_tokenizers_jni_TokenizersLibrary_
     let input_sequence1 = tk::InputSequence::from(sequence1);
     let input_sequence2 = tk::InputSequence::from(sequence2);
     let encoded_input = EncodeInput::Dual(input_sequence1, input_sequence2);
-    let encoding = tokenizer.encode(encoded_input, add_special_tokens == JNI_TRUE);
+    let encoding = tokenizer.encode_char_offsets(encoded_input, add_special_tokens == JNI_TRUE);
 
     match encoding {
         Ok(output) => to_handle(output),
@@ -160,7 +160,7 @@ pub extern "system" fn Java_ai_djl_huggingface_tokenizers_jni_TokenizersLibrary_
 
     let input_sequence = tk::InputSequence::from(array);
     let encoded_input = EncodeInput::from(input_sequence);
-    let encoding = tokenizer.encode(encoded_input, add_special_tokens == JNI_TRUE);
+    let encoding = tokenizer.encode_char_offsets(encoded_input, add_special_tokens == JNI_TRUE);
 
     match encoding {
         Ok(output) => to_handle(output),
@@ -192,7 +192,7 @@ pub extern "system" fn Java_ai_djl_huggingface_tokenizers_jni_TokenizersLibrary_
     }
 
     let encodings = tokenizer
-        .encode_batch(array, add_special_tokens == JNI_TRUE)
+        .encode_batch_char_offsets(array, add_special_tokens == JNI_TRUE)
         .unwrap();
     let handles = encodings
         .into_iter()


Test patch:
diff --git a/examples/src/test/java/ai/djl/examples/inference/TimeSeriesTest.java b/examples/src/test/java/ai/djl/examples/inference/TimeSeriesTest.java
new file mode 100644
index 000000000..f6b315cee
--- /dev/null
+++ b/examples/src/test/java/ai/djl/examples/inference/TimeSeriesTest.java
@@ -0,0 +1,65 @@
+/*
+ * Copyright 2022 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance
+ * with the License. A copy of the License is located at
+ *
+ * http://aws.amazon.com/apache2.0/
+ *
+ * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
+ * OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
+ */
+
+package ai.djl.examples.inference;
+
+import ai.djl.ModelException;
+import ai.djl.examples.inference.timeseries.AirPassengersDeepAR;
+import ai.djl.examples.inference.timeseries.M5ForecastingDeepAR;
+import ai.djl.testing.TestRequirements;
+import ai.djl.translate.TranslateException;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+import java.io.IOException;
+import java.util.Map;
+
+public class TimeSeriesTest {
+
+    private static final Logger logger = LoggerFactory.getLogger(TimeSeriesTest.class);
+
+    @Test
+    public void testM5Forecasting() throws ModelException, TranslateException, IOException {
+        TestRequirements.engine("MXNet");
+
+        Map<String, Float> result = M5ForecastingDeepAR.predict();
+
+        String[] metricNames =
+                new String[] {
+                    "RMSSE",
+                    "MSE",
+                    "abs_error",
+                    "abs_target_sum",
+                    "abs_target_mean",
+                    "MAPE",
+                    "sMAPE",
+                    "ND"
+                };
+        for (String metricName : metricNames) {
+            Assert.assertTrue(result.containsKey(metricName));
+        }
+    }
+
+    @Test
+    public void testAirPassenger() throws ModelException, TranslateException, IOException {
+        TestRequirements.engine("MXNet");
+
+        float[] result = AirPassengersDeepAR.predict();
+        logger.info("{}", result);
+
+        Assert.assertEquals(result.length, 12);
+    }
+}
diff --git a/examples/src/test/java/ai/djl/examples/training/TrainTimeSeriesTest.java b/examples/src/test/java/ai/djl/examples/training/TrainTimeSeriesTest.java
new file mode 100644
index 000000000..b92c1bb82
--- /dev/null
+++ b/examples/src/test/java/ai/djl/examples/training/TrainTimeSeriesTest.java
@@ -0,0 +1,37 @@
+/*
+ * Copyright 2022 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance
+ * with the License. A copy of the License is located at
+ *
+ * http://aws.amazon.com/apache2.0/
+ *
+ * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
+ * OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
+ */
+
+package ai.djl.examples.training;
+
+import ai.djl.testing.TestRequirements;
+import ai.djl.training.TrainingResult;
+import ai.djl.translate.TranslateException;
+
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+import java.io.IOException;
+
+public class TrainTimeSeriesTest {
+
+    @Test
+    public void testTrainTimeSeries() throws TranslateException, IOException {
+        TestRequirements.engine("MXNet");
+
+        String[] args = new String[] {"-g", "1", "-e", "5", "-b", "32"};
+        TrainingResult result = TrainTimeSeries.runExample(args);
+        Assert.assertNotNull(result);
+        float loss = result.getTrainLoss();
+        Assert.assertTrue(loss < 10f, "Loss: " + loss);
+    }
+}
diff --git a/extensions/tokenizers/src/test/java/ai/djl/huggingface/tokenizers/HuggingFaceTokenizerTest.java b/extensions/tokenizers/src/test/java/ai/djl/huggingface/tokenizers/HuggingFaceTokenizerTest.java
index fbb6dabf8..2413e24e1 100644
--- a/extensions/tokenizers/src/test/java/ai/djl/huggingface/tokenizers/HuggingFaceTokenizerTest.java
+++ b/extensions/tokenizers/src/test/java/ai/djl/huggingface/tokenizers/HuggingFaceTokenizerTest.java
@@ -67,8 +67,8 @@ public class HuggingFaceTokenizerTest {
                 new CharSpan(14, 17),
                 new CharSpan(18, 21),
                 new CharSpan(22, 25),
-                new CharSpan(26, 30),
-                new CharSpan(31, 32),
+                new CharSpan(26, 27),
+                new CharSpan(28, 29),
                 null
             };
             int expectedLength = charSpansExpected.length;
@@ -389,4 +389,29 @@ public class HuggingFaceTokenizerTest {
             Assert.assertEquals(encoding.getIds().length, 8);
         }
     }
+
+    @Test
+    public void testSpecialTokenHandling() throws IOException {
+        try (HuggingFaceTokenizer tokenizer =
+                HuggingFaceTokenizer.builder()
+                        .optTokenizerName("distilbert-base-uncased")
+                        .build()) {
+            String someText = "¥$9";
+            Encoding encodedText = tokenizer.encode(someText);
+
+            CharSpan[] expected =
+                    new CharSpan[] {
+                        new CharSpan(-1, -1),
+                        new CharSpan(0, 1),
+                        new CharSpan(1, 2),
+                        new CharSpan(2, 3),
+                        new CharSpan(-1, -1)
+                    };
+            CharSpan[] charSpans = encodedText.getCharTokenSpans();
+            for (int i = 1; i < charSpans.length - 1; i++) {
+                Assert.assertEquals(expected[i].getStart(), charSpans[i].getStart());
+                Assert.assertEquals(expected[i].getEnd(), charSpans[i].getEnd());
+            }
+        }
+    }
 }