nebius/SWE-rebench-V2-PRs huggingface dataset:
Repository: deepjavalibrary/djl
Pull request #2657 (2023-06-15 15:22:02)

[api] Masks sensitive env vars in debug print out
## Description ##

Brief description of what this PR is about

- If this change is a backward incompatible change, why must this change be made?
- Interesting edge cases to note here



Patch:
diff --git a/api/src/main/java/ai/djl/engine/Engine.java b/api/src/main/java/ai/djl/engine/Engine.java
index 3b29a9295..8a1fc8871 100644
--- a/api/src/main/java/ai/djl/engine/Engine.java
+++ b/api/src/main/java/ai/djl/engine/Engine.java
@@ -39,6 +39,7 @@ import java.util.Properties;
 import java.util.ServiceLoader;
 import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.regex.Pattern;
 
 /**
  * The {@code Engine} interface is the base of the provided implementation for DJL.
@@ -59,6 +60,8 @@ public abstract class Engine {
     private static final Map<String, EngineProvider> ALL_ENGINES = new ConcurrentHashMap<>();
 
     private static final String DEFAULT_ENGINE = initEngine();
+    private static final Pattern PATTERN =
+            Pattern.compile("KEY|TOKEN|PASSWORD", Pattern.CASE_INSENSITIVE);
 
     private Device defaultDevice;
 
@@ -372,11 +375,11 @@ public abstract class Engine {
     @SuppressWarnings("PMD.SystemPrintln")
     public static void debugEnvironment() {
         System.out.println("----------- System Properties -----------");
-        System.getProperties().forEach((k, v) -> System.out.println(k + ": " + v));
+        System.getProperties().forEach((k, v) -> print((String) k, v));
 
         System.out.println();
         System.out.println("--------- Environment Variables ---------");
-        Utils.getenv().forEach((k, v) -> System.out.println(k + ": " + v));
+        Utils.getenv().forEach(Engine::print);
 
         System.out.println();
         System.out.println("-------------- Directories --------------");
@@ -422,4 +425,11 @@ public abstract class Engine {
             System.out.println(provider.getEngineName() + ": " + provider.getEngineRank());
         }
     }
+
+    private static void print(String key, Object value) {
+        if (PATTERN.matcher(key).find()) {
+            value = "*********";
+        }
+        System.out.println(key + ": " + value); // NOPMD
+    }
 }


Test patch:
diff --git a/api/src/test/java/ai/djl/DeviceTest.java b/api/src/test/java/ai/djl/DeviceTest.java
index 4d61a8bb3..92a0474c6 100644
--- a/api/src/test/java/ai/djl/DeviceTest.java
+++ b/api/src/test/java/ai/djl/DeviceTest.java
@@ -35,6 +35,7 @@ public class DeviceTest {
         Device dev = Device.of("myDevice", 1);
         Assert.assertEquals(dev.getDeviceType(), "myDevice");
 
+        System.setProperty("test_key", "test");
         Engine.debugEnvironment();
     }
Repository: deepjavalibrary/djl
Pull request #2804 (2023-10-12 00:30:07)

[api] Fixed NDList decode numpy file bug
## Description ##

Brief description of what this PR is about

- If this change is a backward incompatible change, why must this change be made?
- Interesting edge cases to note here



Patch:
diff --git a/api/src/main/java/ai/djl/ndarray/NDList.java b/api/src/main/java/ai/djl/ndarray/NDList.java
index e48c243a3..f0069d3f3 100644
--- a/api/src/main/java/ai/djl/ndarray/NDList.java
+++ b/api/src/main/java/ai/djl/ndarray/NDList.java
@@ -100,12 +100,12 @@ public class NDList extends ArrayList<NDArray> implements NDResource, BytesSuppl
         try {
             if (byteArray[0] == 'P' && byteArray[1] == 'K') {
                 return decodeNumpy(manager, new ByteArrayInputStream(byteArray));
-            } else if (byteArray[0] == (byte) 0x39
+            } else if (byteArray[0] == (byte) 0x93
                     && byteArray[1] == 'N'
                     && byteArray[2] == 'U'
                     && byteArray[3] == 'M') {
                 return new NDList(
-                        NDSerializer.decode(manager, new ByteArrayInputStream(byteArray)));
+                        NDSerializer.decodeNumpy(manager, new ByteArrayInputStream(byteArray)));
             } else if (byteArray[8] == '{') {
                 return decodeSafetensors(manager, new ByteArrayInputStream(byteArray));
             }
@@ -144,11 +144,11 @@ public class NDList extends ArrayList<NDArray> implements NDResource, BytesSuppl
             if (magic[0] == 'P' && magic[1] == 'K') {
                 // assume this is npz file
                 return decodeNumpy(manager, pis);
-            } else if (magic[0] == (byte) 0x39
+            } else if (magic[0] == (byte) 0x93
                     && magic[1] == 'N'
                     && magic[2] == 'U'
                     && magic[3] == 'M') {
-                return new NDList(NDSerializer.decode(manager, pis));
+                return new NDList(NDSerializer.decodeNumpy(manager, pis));
             } else if (magic[8] == '{') {
                 return decodeSafetensors(manager, pis);
             }


Test patch:
diff --git a/api/src/test/java/ai/djl/ndarray/NDSerializerTest.java b/api/src/test/java/ai/djl/ndarray/NDSerializerTest.java
index 0e38c2d8b..e89f22442 100644
--- a/api/src/test/java/ai/djl/ndarray/NDSerializerTest.java
+++ b/api/src/test/java/ai/djl/ndarray/NDSerializerTest.java
@@ -107,7 +107,7 @@ public class NDSerializerTest {
 
     private static NDArray decode(NDManager manager, byte[] data) throws IOException {
         try (ByteArrayInputStream bis = new ByteArrayInputStream(data)) {
-            return NDSerializer.decodeNumpy(manager, bis);
+            return NDList.decode(manager, bis).get(0);
         }
     }
Repository: deepjavalibrary/djl
Pull request #2994 (2024-02-15 17:21:49)

[docs] Update Huggingface tokenizer cache directory document
## Description ##

Brief description of what this PR is about

- If this change is a backward incompatible change, why must this change be made?
- Interesting edge cases to note here



Patch:
diff --git a/docs/development/cache_management.md b/docs/development/cache_management.md
index b0b56460e..2bdacb9a6 100644
--- a/docs/development/cache_management.md
+++ b/docs/development/cache_management.md
@@ -30,10 +30,10 @@ ONNXRuntime will extract native libraries into system default temporary-file dir
 
 ### Huggingface tokenizer
 
-If the `TOKENIZERS_CACHE` environment variable is set, Huggingface tokenizer will store cache files in it.
+If the `HF_HOME` or `HF_HUB_CACHE` environment variable is set, Huggingface tokenizer will store cache files in it.
 It is the responsibility of the user to make sure this path is correct. Otherwise, we try to use
 the default cache directory as defined for each OS:
 
-- macOS: `/Users/{user}/Library/Caches/huggingface/tokenizers`
-- linux: `/home/{user}/.cache/huggingface/tokenizers`
-- windows: `C:\Users\{user}\AppData\Local\huggingface\tokenizers`
+- macOS: `/Users/{user}/.cache/huggingface/hub`
+- linux: `/home/{user}/.cache/huggingface/hub`
+- windows: `C:\Users\{user}\.cache\huggingface\hub`


Test patch:
diff --git a/basicdataset/src/test/resources/mlrepo/dataset/nlp/ai/djl/basicdataset/wikitext-2/metadata.json b/basicdataset/src/test/resources/mlrepo/dataset/nlp/ai/djl/basicdataset/wikitext-2/metadata.json
index f9c64dc80..1f31ac3af 100644
--- a/basicdataset/src/test/resources/mlrepo/dataset/nlp/ai/djl/basicdataset/wikitext-2/metadata.json
+++ b/basicdataset/src/test/resources/mlrepo/dataset/nlp/ai/djl/basicdataset/wikitext-2/metadata.json
@@ -20,10 +20,10 @@
       "name": "wikitext-2",
       "files": {
         "wikitext-2": {
-          "uri": "https://s3.amazonaws.com/research.metamind.io/wikitext/wikitext-2-v1.zip",
-          "sha1Hash": "3c914d17d80b1459be871a5039ac23e752a53cbe",
+          "uri": "https://mlrepo.djl.ai/dataset/nlp/ai/djl/basicdataset/wikitext-2/1.0/wikitext-2-v1.zip",
+          "sha1Hash": "46965bdeca1d8165e688598752ca467bb5bee018",
           "name": "",
-          "size": 4475746
+          "size": 4475596
         }
       }
     }
Repository: deepjavalibrary/djl
Pull request #3228 (2024-05-30 22:58:16)

[api] Fix unitest in GPU docker running on CPU case
## Description ##

Brief description of what this PR is about

- If this change is a backward incompatible change, why must this change be made?
- Interesting edge cases to note here



Patch:
diff --git a/api/src/main/java/ai/djl/util/cuda/CudaUtils.java b/api/src/main/java/ai/djl/util/cuda/CudaUtils.java
index 1de074ea6..859596a6b 100644
--- a/api/src/main/java/ai/djl/util/cuda/CudaUtils.java
+++ b/api/src/main/java/ai/djl/util/cuda/CudaUtils.java
@@ -61,7 +61,11 @@ public final class CudaUtils {
             if (gpuInfo == null) {
                 gpuInfo = execute(-1); // NOPMD
             }
-            return Integer.parseInt(gpuInfo[0]);
+            try {
+                return Integer.parseInt(gpuInfo[0]);
+            } catch (NumberFormatException e) {
+                return 0;
+            }
         }
 
         if (LIB == null) {
diff --git a/extensions/tokenizers/rust/src/models/mod.rs b/extensions/tokenizers/rust/src/models/mod.rs
index a4db801bb..b915f9229 100644
--- a/extensions/tokenizers/rust/src/models/mod.rs
+++ b/extensions/tokenizers/rust/src/models/mod.rs
@@ -124,7 +124,8 @@ pub extern "system" fn Java_ai_djl_engine_rust_RustLibrary_loadModel<'local>(
 
 #[no_mangle]
 pub extern "system" fn Java_ai_djl_engine_rust_RustLibrary_deleteModel<'local>(
-    _env: JObject,
+    _: JNIEnv,
+    _: JObject,
     handle: jlong,
 ) {
     drop_handle::<Box<dyn Model>>(handle);


Test patch:
diff --git a/api/src/test/java/ai/djl/util/cuda/CudaUtilsTest.java b/api/src/test/java/ai/djl/util/cuda/CudaUtilsTest.java
index e09417fa5..df135938a 100644
--- a/api/src/test/java/ai/djl/util/cuda/CudaUtilsTest.java
+++ b/api/src/test/java/ai/djl/util/cuda/CudaUtilsTest.java
@@ -13,6 +13,7 @@
 package ai.djl.util.cuda;
 
 import ai.djl.Device;
+import ai.djl.engine.EngineException;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -30,17 +31,13 @@ public class CudaUtilsTest {
         if (!CudaUtils.hasCuda()) {
             try {
                 CudaUtils.getCudaVersionString();
-            } catch (IllegalStateException | IllegalArgumentException ignore) {
+            } catch (IllegalStateException | IllegalArgumentException | EngineException ignore) {
                 // ignore exception
             }
             Assert.assertThrows(() -> CudaUtils.getComputeCapability(0));
             Assert.assertThrows(() -> CudaUtils.getGpuMemory(Device.gpu()));
             return;
         }
-        // Possible to have CUDA and not have a GPU.
-        if (CudaUtils.getGpuCount() == 0) {
-            return;
-        }
 
         String cudaVersion = CudaUtils.getCudaVersionString();
         String smVersion = CudaUtils.getComputeCapability(0);
Repository: deepjavalibrary/djl
Pull request #3381 (2024-08-02 20:24:50)

Refactored Identifiers
## Description ##

Refactored certain identifiers in the code - expanded certain variable names according to its usage in the code to make them more descriptive. Meanwhile, changed "testOffLine" to "testOffline" as "Offline" is a single word, to conform to Java naming conventions.

- If this change is a backward incompatible change, why must this change be made?
- Interesting edge cases to note here



Patch:
diff --git a/model-zoo/src/main/java/ai/djl/basicmodelzoo/basic/Mlp.java b/model-zoo/src/main/java/ai/djl/basicmodelzoo/basic/Mlp.java
index 2869e42f5..642b48364 100644
--- a/model-zoo/src/main/java/ai/djl/basicmodelzoo/basic/Mlp.java
+++ b/model-zoo/src/main/java/ai/djl/basicmodelzoo/basic/Mlp.java
@@ -59,8 +59,8 @@ public class Mlp extends SequentialBlock {
     @SuppressWarnings("this-escape")
     public Mlp(int input, int output, int[] hidden, Function<NDList, NDList> activation) {
         add(Blocks.batchFlattenBlock(input));
-        for (int hiddenSize : hidden) {
-            add(Linear.builder().setUnits(hiddenSize).build());
+        for (int hiddenLayerSize : hidden) {
+            add(Linear.builder().setUnits(hiddenLayerSize).build());
             add(activation);
         }
 


Test patch:
diff --git a/extensions/tokenizers/src/test/java/ai/djl/huggingface/zoo/ModelZooTest.java b/extensions/tokenizers/src/test/java/ai/djl/huggingface/zoo/ModelZooTest.java
index 4605702a1..6ea38ddb0 100644
--- a/extensions/tokenizers/src/test/java/ai/djl/huggingface/zoo/ModelZooTest.java
+++ b/extensions/tokenizers/src/test/java/ai/djl/huggingface/zoo/ModelZooTest.java
@@ -55,8 +55,8 @@ public class ModelZooTest {
         try (ZooModel<QAInput, String> model = criteria.loadModel();
                 Predictor<QAInput, String> predictor = model.newPredictor()) {
             QAInput input = new QAInput(question, paragraph);
-            String res = predictor.predict(input);
-            Assert.assertEquals(res, "december 2004");
+            String result = predictor.predict(input);
+            Assert.assertEquals(result, "december 2004");
         }
     }
 
@@ -101,7 +101,7 @@ public class ModelZooTest {
     }
 
     @Test
-    public void testOffLine() {
+    public void testOffline() {
         System.setProperty("DJL_CACHE_DIR", "build/cache");
         System.setProperty("ai.djl.offline", "true");
         try {
Repository: deepjavalibrary/djl
Pull request #3485 (2024-09-27 16:34:30)

[mxnet] Fixes build error on JDK 22
## Description ##

Brief description of what this PR is about

- If this change is a backward incompatible change, why must this change be made?
- Interesting edge cases to note here



Patch:
diff --git a/engines/mxnet/jnarator/build.gradle.kts b/engines/mxnet/jnarator/build.gradle.kts
index 3f63a179a..4f3ddbcbb 100644
--- a/engines/mxnet/jnarator/build.gradle.kts
+++ b/engines/mxnet/jnarator/build.gradle.kts
@@ -19,6 +19,14 @@ tasks {
     checkstyleMain { exclude("ai/djl/mxnet/jnarator/parser/*") }
     pmdMain { exclude("ai/djl/mxnet/jnarator/parser/*") }
 
+    compileJava {
+        options.apply {
+            release = 8
+            encoding = "UTF-8"
+            compilerArgs = listOf("-proc:none", "-Xlint:all,-options,-static")
+        }
+    }
+
     jar {
         dependsOn(generateGrammarSource)
         manifest {
@@ -39,5 +47,4 @@ tasks {
     generateTestGrammarSource {
         dependsOn(verifyJava)
     }
-
-}
\ No newline at end of file
+}


Test patch:
diff --git a/api/src/test/java/ai/djl/util/SecurityManagerTest.java b/api/src/test/java/ai/djl/util/SecurityManagerTest.java
index fbe09975d..322aed734 100644
--- a/api/src/test/java/ai/djl/util/SecurityManagerTest.java
+++ b/api/src/test/java/ai/djl/util/SecurityManagerTest.java
@@ -13,6 +13,7 @@
 package ai.djl.util;
 
 import org.testng.Assert;
+import org.testng.SkipException;
 import org.testng.annotations.AfterTest;
 import org.testng.annotations.BeforeTest;
 import org.testng.annotations.Test;
@@ -25,6 +26,9 @@ public class SecurityManagerTest {
 
     @BeforeTest
     public void setUp() {
+        if (Runtime.version().version().get(0) > 17) {
+            throw new SkipException("Skip SecurityManagerTest for JDK 19+");
+        }
         originalSM = System.getSecurityManager();
     }
Repository: deepjavalibrary/djl
Pull request #3498 (2024-10-06 03:11:50)

[api] Makes Sam2 input consistent with other CV model
## Description ##

Brief description of what this PR is about

- If this change is a backward incompatible change, why must this change be made?
- Interesting edge cases to note here



Patch:
diff --git a/api/src/main/java/ai/djl/modality/cv/output/CategoryMask.java b/api/src/main/java/ai/djl/modality/cv/output/CategoryMask.java
index c0aea00b8..394a7461a 100644
--- a/api/src/main/java/ai/djl/modality/cv/output/CategoryMask.java
+++ b/api/src/main/java/ai/djl/modality/cv/output/CategoryMask.java
@@ -34,7 +34,9 @@ public class CategoryMask implements JsonSerializable {
 
     private static final int COLOR_BLACK = 0xFF000000;
 
+    @SuppressWarnings("serial")
     private List<String> classes;
+
     private int[][] mask;
 
     /**
diff --git a/api/src/main/java/ai/djl/modality/cv/translator/Sam2Translator.java b/api/src/main/java/ai/djl/modality/cv/translator/Sam2Translator.java
index 751019c76..089d071c6 100644
--- a/api/src/main/java/ai/djl/modality/cv/translator/Sam2Translator.java
+++ b/api/src/main/java/ai/djl/modality/cv/translator/Sam2Translator.java
@@ -39,6 +39,8 @@ import ai.djl.translate.Translator;
 import ai.djl.translate.TranslatorContext;
 import ai.djl.util.JsonUtils;
 
+import com.google.gson.annotations.SerializedName;
+
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
@@ -340,7 +342,7 @@ public class Sam2Translator implements NoBatchifyTranslator<Sam2Input, DetectedO
         public static Sam2Input fromJson(String input) throws IOException {
             Prompt prompt = JsonUtils.GSON.fromJson(input, Prompt.class);
             if (prompt.image == null) {
-                throw new IllegalArgumentException("Missing image value");
+                throw new IllegalArgumentException("Missing image_url value");
             }
             if (prompt.prompt == null || prompt.prompt.length == 0) {
                 throw new IllegalArgumentException("Missing prompt value");
@@ -477,7 +479,10 @@ public class Sam2Translator implements NoBatchifyTranslator<Sam2Input, DetectedO
         }
 
         private static final class Prompt {
+
+            @SerializedName("image_url")
             String image;
+
             Location[] prompt;
             boolean visualize;
 


Test patch:
diff --git a/api/src/test/java/ai/djl/modality/cv/translator/Sam2InputTest.java b/api/src/test/java/ai/djl/modality/cv/translator/Sam2InputTest.java
index a742e92d4..417896049 100644
--- a/api/src/test/java/ai/djl/modality/cv/translator/Sam2InputTest.java
+++ b/api/src/test/java/ai/djl/modality/cv/translator/Sam2InputTest.java
@@ -30,7 +30,7 @@ public class Sam2InputTest {
         Path file = Paths.get("../examples/src/test/resources/kitten.jpg");
         Image img = ImageFactory.getInstance().fromFile(file);
         String json =
-                "{\"image\": \""
+                "{\"image_url\": \""
                         + file.toUri().toURL()
                         + "\",\n"
                         + "\"visualize\": true,\n"
Repository: deepjavalibrary/djl
Pull request #3515 (2024-11-11 21:28:27)

[tokenizer] Updates tokenizer to 0.20.3 in libs.versions.toml
## Description ##

Brief description of what this PR is about

- If this change is a backward incompatible change, why must this change be made?
- Interesting edge cases to note here



Patch:
diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml
index 990bad2da..509e4adf1 100644
--- a/gradle/libs.versions.toml
+++ b/gradle/libs.versions.toml
@@ -14,7 +14,7 @@ lightgbm = "3.2.110"
 tensorrt = "8.4.1"
 fasttext = "0.9.2"
 sentencepiece = "0.2.0"
-tokenizers = "0.20.0"
+tokenizers = "0.20.3"
 
 slf4j = "2.0.16"
 log4j_slf4j = "2.24.0"


Test patch:
diff --git a/extensions/tokenizers/src/test/java/ai/djl/huggingface/tokenizers/HuggingFaceTokenizerTest.java b/extensions/tokenizers/src/test/java/ai/djl/huggingface/tokenizers/HuggingFaceTokenizerTest.java
index 27fb1056b..4e11fee74 100644
--- a/extensions/tokenizers/src/test/java/ai/djl/huggingface/tokenizers/HuggingFaceTokenizerTest.java
+++ b/extensions/tokenizers/src/test/java/ai/djl/huggingface/tokenizers/HuggingFaceTokenizerTest.java
@@ -39,7 +39,7 @@ public class HuggingFaceTokenizerTest {
     public void testVersion() {
         try (HuggingFaceTokenizer tokenizer = HuggingFaceTokenizer.newInstance("bert-base-cased")) {
             String djlVersion = Engine.getDjlVersion();
-            Assert.assertEquals(tokenizer.getVersion(), "0.20.0-" + djlVersion);
+            Assert.assertEquals(tokenizer.getVersion(), "0.20.3-" + djlVersion);
         }
     }