Repository: deepjavalibrary/djl (java)

Issue:
NPE in SpTokenizer initialization
## Description
If `SpTokenizer` is initialized just by passing a folder path (as specified in the Javadoc), it crashes with an NPE because code after that assumes the prefix to be non-null. See [here](https://github.com/deepjavalibrary/djl/blob/8749e4188e414a22d7de5a78bde336de3ffda756/extensions/sentencepiece/src/main/java/ai/djl/sentencepiece/SpTokenizer.java#L38) and then [here](https://github.com/deepjavalibrary/djl/blob/8749e4188e414a22d7de5a78bde336de3ffda756/extensions/sentencepiece/src/main/java/ai/djl/sentencepiece/SpTokenizer.java#L113)

### Expected Behavior
That constructor documentation should specify that only a file path can be past, which should be validated upfront.

### Error Message
Exception in thread "main" java.lang.NullPointerException
	at java.base/sun.nio.fs.UnixPath.normalizeAndCheck(UnixPath.java:75)
	at java.base/sun.nio.fs.UnixPath.<init>(UnixPath.java:69)
	at java.base/sun.nio.fs.UnixFileSystem.getPath(UnixFileSystem.java:279)
	at java.base/java.nio.file.Path.resolve(Path.java:515)
	at ai.djl.sentencepiece.SpTokenizer.findModelFile(SpTokenizer.java:113)
	at ai.djl.sentencepiece.SpTokenizer.loadModel(SpTokenizer.java:96)
	at ai.djl.sentencepiece.SpTokenizer.<init>(SpTokenizer.java:50)
	at ai.djl.sentencepiece.SpTokenizer.<init>(SpTokenizer.java:38)

## How to Reproduce?
In Kotlin:
```kotlin
    val modelDir = Paths.get("name of a directory")
    val tokenizer = SpTokenizer(modelDir)
```


Pull request:
[sentencepiece] Fixes #1491, fix NPE bug
Change-Id: Ib86efb38def0bcb21862c5493683395071eb3592

## Description ##

Fixes #1491 

- If this change is a backward incompatible change, why must this change be made?
- Interesting edge cases to note here


Patch:
diff --git a/extensions/sentencepiece/src/main/java/ai/djl/sentencepiece/SpTokenizer.java b/extensions/sentencepiece/src/main/java/ai/djl/sentencepiece/SpTokenizer.java
index 613fcced4..37b39bba0 100644
--- a/extensions/sentencepiece/src/main/java/ai/djl/sentencepiece/SpTokenizer.java
+++ b/extensions/sentencepiece/src/main/java/ai/djl/sentencepiece/SpTokenizer.java
@@ -93,13 +93,13 @@ public class SpTokenizer implements Tokenizer, AutoCloseable {
                     "Model path doesn't exist: " + modelPath.toAbsolutePath());
         }
         Path modelDir = modelPath.toAbsolutePath();
+        if (prefix == null || prefix.isEmpty()) {
+            prefix = modelDir.toFile().getName();
+        }
         Path modelFile = findModelFile(modelDir, prefix);
         if (modelFile == null) {
             // TODO: support proto and IOStream model
-            modelFile = findModelFile(modelDir, modelDir.toFile().getName());
-            if (modelFile == null) {
-                throw new FileNotFoundException("No .model found in : " + modelPath);
-            }
+            throw new FileNotFoundException("No .model found in : " + modelPath);
         }
 
         String modelFilePath = modelFile.toString();


Test patch:
diff --git a/extensions/sentencepiece/src/test/java/ai/djl/sentencepiece/SpTokenizerTest.java b/extensions/sentencepiece/src/test/java/ai/djl/sentencepiece/SpTokenizerTest.java
index 155290e37..9838db194 100644
--- a/extensions/sentencepiece/src/test/java/ai/djl/sentencepiece/SpTokenizerTest.java
+++ b/extensions/sentencepiece/src/test/java/ai/djl/sentencepiece/SpTokenizerTest.java
@@ -29,17 +29,17 @@ public class SpTokenizerTest {
 
     @BeforeTest
     public void downloadModel() throws IOException {
-        Path modelFile = Paths.get("build/test/models/sententpiece_test_model.model");
+        Path modelFile = Paths.get("build/test/sp_model/sp_model.model");
         if (Files.notExists(modelFile)) {
             DownloadUtils.download(
                     "https://resources.djl.ai/test-models/sententpiece_test_model.model",
-                    "build/test/models/sententpiece_test_model.model");
+                    "build/test/sp_model/sp_model.model");
         }
     }
 
     @Test
     public void testLoadFromBytes() throws IOException {
-        Path modelPath = Paths.get("build/test/models/sententpiece_test_model.model");
+        Path modelPath = Paths.get("build/test/sp_model/sp_model.model");
         byte[] bytes = Files.readAllBytes(modelPath);
         try (SpTokenizer tokenizer = new SpTokenizer(bytes)) {
             String original = "Hello World";
@@ -55,7 +55,7 @@ public class SpTokenizerTest {
     public void testTokenize() throws IOException {
         TestRequirements.notWindows();
 
-        Path modelPath = Paths.get("build/test/models/sententpiece_test_model.model");
+        Path modelPath = Paths.get("build/test/sp_model");
         try (SpTokenizer tokenizer = new SpTokenizer(modelPath)) {
             String original = "Hello World";
             List<String> tokens = tokenizer.tokenize(original);
@@ -71,7 +71,7 @@ public class SpTokenizerTest {
     public void testUtf16Tokenize() throws IOException {
         TestRequirements.notWindows();
 
-        Path modelPath = Paths.get("build/test/models/sententpiece_test_model.model");
+        Path modelPath = Paths.get("build/test/sp_model/sp_model.model");
         try (SpTokenizer tokenizer = new SpTokenizer(modelPath)) {
             String original = "\uD83D\uDC4B\uD83D\uDC4B";
             List<String> tokens = tokenizer.tokenize(original);
@@ -84,8 +84,8 @@ public class SpTokenizerTest {
     public void testEncodeDecode() throws IOException {
         TestRequirements.notWindows();
 
-        Path modelPath = Paths.get("build/test/models");
-        String prefix = "sententpiece_test_model";
+        Path modelPath = Paths.get("build/test/sp_model");
+        String prefix = "sp_model";
         try (SpTokenizer tokenizer = new SpTokenizer(modelPath, prefix)) {
             String original = "Hello World";
             SpProcessor processor = tokenizer.getProcessor();
@@ -96,4 +96,24 @@ public class SpTokenizerTest {
             Assert.assertEquals(recovered, original);
         }
     }
+
+    @Test
+    public void testModelNotFound() throws IOException {
+        TestRequirements.notWindows();
+
+        Assert.assertThrows(
+                () -> {
+                    new SpTokenizer(Paths.get("build/test/non-exists"));
+                });
+
+        Assert.assertThrows(
+                () -> {
+                    new SpTokenizer(Paths.get("build/test/sp_model"), "non-exists.model");
+                });
+
+        Assert.assertThrows(
+                () -> {
+                    new SpTokenizer(Paths.get("build/test/sp_model"), "non-exists");
+                });
+    }
 }