Repository: ray-project/ray

Issue:
Make the multi-worker feature for Java worker experimental
Right now, the multi-worker feature for Java worker is enabled by default, but the `ActorHandle::kill()` API doesn't work well if multi-worker is enabled because it will kill the whole process instead of one worker in the process.

To avoid complaints from Java users, we should disable the multi-worker feature by default, but we still enable it in unit test.


Discussion:
#7291

Resolving patch:
diff --git a/python/ray/job_config.py b/python/ray/job_config.py
--- a/python/ray/job_config.py
+++ b/python/ray/job_config.py
@@ -15,7 +15,7 @@ class JobConfig:
     def __init__(
             self,
             worker_env=None,
-            num_java_workers_per_process=10,
+            num_java_workers_per_process=1,
             jvm_options=None,
     ):
         if worker_env is None:


Test patch:
diff --git a/java/test.sh b/java/test.sh
--- a/java/test.sh
+++ b/java/test.sh
@@ -33,15 +33,18 @@ bazel build //java:gen_maven_deps
 echo "Build test jar."
 bazel build //java:all_tests_deploy.jar
 
+# Enable multi-worker feature in Java test
+TEST_ARGS=(-Dray.raylet.config.num_workers_per_process_java=10 -Dray.job.num-java-workers-per-process=10)
+
 echo "Running tests under cluster mode."
 # TODO(hchen): Ideally, we should use the following bazel command to run Java tests. However, if there're skipped tests,
 # TestNG will exit with code 2. And bazel treats it as test failure.
 # bazel test //java:all_tests --action_env=ENABLE_MULTI_LANGUAGE_TESTS=1 --config=ci || cluster_exit_code=$?
-ENABLE_MULTI_LANGUAGE_TESTS=1 run_testng java -cp "$ROOT_DIR"/../bazel-bin/java/all_tests_deploy.jar org.testng.TestNG -d /tmp/ray_java_test_output "$ROOT_DIR"/testng.xml
+ENABLE_MULTI_LANGUAGE_TESTS=1 run_testng java -cp "$ROOT_DIR"/../bazel-bin/java/all_tests_deploy.jar "${TEST_ARGS[@]}" org.testng.TestNG -d /tmp/ray_java_test_output "$ROOT_DIR"/testng.xml
 
 echo "Running tests under single-process mode."
 # bazel test //java:all_tests --jvmopt="-Dray.run-mode=SINGLE_PROCESS" --config=ci || single_exit_code=$?
-run_testng java -Dray.run-mode="SINGLE_PROCESS" -cp "$ROOT_DIR"/../bazel-bin/java/all_tests_deploy.jar org.testng.TestNG -d /tmp/ray_java_test_output "$ROOT_DIR"/testng.xml
+run_testng java -Dray.run-mode="SINGLE_PROCESS" -cp "$ROOT_DIR"/../bazel-bin/java/all_tests_deploy.jar "${TEST_ARGS[@]}" org.testng.TestNG -d /tmp/ray_java_test_output "$ROOT_DIR"/testng.xml
 
 popd
 
diff --git a/java/test/src/main/java/io/ray/test/ClassLoaderTest.java b/java/test/src/main/java/io/ray/test/ClassLoaderTest.java
--- a/java/test/src/main/java/io/ray/test/ClassLoaderTest.java
+++ b/java/test/src/main/java/io/ray/test/ClassLoaderTest.java
@@ -40,7 +40,7 @@ public void tearDown() {
 
   @Test(groups = {"cluster"})
   public void testClassLoaderInMultiThreading() throws Exception {
-    Assert.assertTrue(TestUtils.getRuntime().getRayConfig().numWorkersPerProcess > 1);
+    Assert.assertTrue(TestUtils.getNumWorkersPerProcess() > 1);
 
     final String jobResourcePath = resourcePath + "/" + Ray.getRuntimeContext().getCurrentJobId();
     File jobResourceDir = new File(jobResourcePath);
diff --git a/java/test/src/main/java/io/ray/test/ExitActorTest.java b/java/test/src/main/java/io/ray/test/ExitActorTest.java
--- a/java/test/src/main/java/io/ray/test/ExitActorTest.java
+++ b/java/test/src/main/java/io/ray/test/ExitActorTest.java
@@ -73,7 +73,7 @@ public void testExitActor() throws IOException, InterruptedException {
   }
 
   public void testExitActorInMultiWorker() {
-    Assert.assertTrue(TestUtils.getRuntime().getRayConfig().numWorkersPerProcess > 1);
+    Assert.assertTrue(TestUtils.getNumWorkersPerProcess() > 1);
     ActorHandle<ExitingActor> actor1 = Ray.actor(ExitingActor::new)
         .setMaxRestarts(10000).remote();
     int pid = actor1.task(ExitingActor::getPid).remote().get();
diff --git a/java/test/src/main/java/io/ray/test/FailureTest.java b/java/test/src/main/java/io/ray/test/FailureTest.java
--- a/java/test/src/main/java/io/ray/test/FailureTest.java
+++ b/java/test/src/main/java/io/ray/test/FailureTest.java
@@ -21,17 +21,20 @@ public class FailureTest extends BaseTest {
 
   private static final String EXCEPTION_MESSAGE = "Oops";
 
+  private String oldNumWorkersPerProcess;
+
   @BeforeClass
   public void setUp() {
     // This is needed by `testGetThrowsQuicklyWhenFoundException`.
     // Set one worker per process. Otherwise, if `badFunc2` and `slowFunc` run in the same
     // process, `sleep` will delay `System.exit`.
+    oldNumWorkersPerProcess = System.getProperty("ray.raylet.config.num_workers_per_process_java");
     System.setProperty("ray.raylet.config.num_workers_per_process_java", "1");
   }
 
   @AfterClass
   public void tearDown() {
-    System.clearProperty("ray.raylet.config.num_workers_per_process_java");
+    System.setProperty("ray.raylet.config.num_workers_per_process_java", oldNumWorkersPerProcess);
   }
 
   public static int badFunc() {
diff --git a/java/test/src/main/java/io/ray/test/JobConfigTest.java b/java/test/src/main/java/io/ray/test/JobConfigTest.java
--- a/java/test/src/main/java/io/ray/test/JobConfigTest.java
+++ b/java/test/src/main/java/io/ray/test/JobConfigTest.java
@@ -1,7 +1,6 @@
 package io.ray.test;
 
 import io.ray.api.ActorHandle;
-import io.ray.api.ObjectRef;
 import io.ray.api.Ray;
 import org.testng.Assert;
 import org.testng.annotations.AfterClass;
@@ -11,9 +10,12 @@
 @Test(groups = {"cluster"})
 public class JobConfigTest extends BaseTest {
 
+  private String oldNumWorkersPerProcess;
+
   @BeforeClass
   public void setupJobConfig() {
     System.setProperty("ray.raylet.config.enable_multi_tenancy", "true");
+    oldNumWorkersPerProcess = System.getProperty("ray.job.num-java-workers-per-process");
     System.setProperty("ray.job.num-java-workers-per-process", "3");
     System.setProperty("ray.job.jvm-options.0", "-DX=999");
     System.setProperty("ray.job.jvm-options.1", "-DY=998");
@@ -24,7 +26,7 @@ public void setupJobConfig() {
   @AfterClass
   public void tearDownJobConfig() {
     System.clearProperty("ray.raylet.config.enable_multi_tenancy");
-    System.clearProperty("ray.job.num-java-workers-per-process");
+    System.setProperty("ray.job.num-java-workers-per-process", oldNumWorkersPerProcess);
     System.clearProperty("ray.job.jvm-options.0");
     System.clearProperty("ray.job.jvm-options.1");
     System.clearProperty("ray.job.worker-env.foo1");
@@ -39,16 +41,8 @@ public static String getEnvVariable(String key) {
     return System.getenv(key);
   }
 
-  public static Integer getWorkersNum() {
-    return TestUtils.getRuntime().getRayConfig().numWorkersPerProcess;
-  }
-
   public static class MyActor {
 
-    public Integer getWorkersNum() {
-      return TestUtils.getRuntime().getRayConfig().numWorkersPerProcess;
-    }
-
     public String getJvmOptions(String propertyName) {
       return System.getProperty(propertyName);
     }
@@ -68,9 +62,8 @@ public void testWorkerEnvVariable() {
     Assert.assertEquals("bar2", Ray.task(JobConfigTest::getEnvVariable, "foo2").remote().get());
   }
 
-  public void testNumJavaWorkerPerProcess() {
-    ObjectRef<Integer> obj = Ray.task(JobConfigTest::getWorkersNum).remote();
-    Assert.assertEquals(3, (int) obj.get());
+  public void testNumJavaWorkersPerProcess() {
+    Assert.assertEquals(TestUtils.getNumWorkersPerProcess(), 3);
   }
 
 
@@ -84,9 +77,5 @@ public void testInActor() {
     // test worker env variables
     Assert.assertEquals("bar1", Ray.task(MyActor::getEnvVariable, "foo1").remote().get());
     Assert.assertEquals("bar2", Ray.task(MyActor::getEnvVariable, "foo2").remote().get());
-
-    //  test workers number.
-    ObjectRef<Integer> obj2 = actor.task(MyActor::getWorkersNum).remote();
-    Assert.assertEquals(3, (int) obj2.get());
   }
 }
diff --git a/java/test/src/main/java/io/ray/test/KillActorTest.java b/java/test/src/main/java/io/ray/test/KillActorTest.java
--- a/java/test/src/main/java/io/ray/test/KillActorTest.java
+++ b/java/test/src/main/java/io/ray/test/KillActorTest.java
@@ -14,14 +14,17 @@
 @Test(groups = {"cluster"})
 public class KillActorTest extends BaseTest {
 
+  private String oldNumWorkersPerProcess;
+
   @BeforeClass
   public void setUp() {
+    oldNumWorkersPerProcess = System.getProperty("ray.raylet.config.num_workers_per_process_java");
     System.setProperty("ray.raylet.config.num_workers_per_process_java", "1");
   }
 
   @AfterClass
   public void tearDown() {
-    System.clearProperty("ray.raylet.config.num_workers_per_process_java");
+    System.setProperty("ray.raylet.config.num_workers_per_process_java", oldNumWorkersPerProcess);
   }
 
   public static class HangActor {
diff --git a/java/test/src/main/java/io/ray/test/MultiThreadingTest.java b/java/test/src/main/java/io/ray/test/MultiThreadingTest.java
--- a/java/test/src/main/java/io/ray/test/MultiThreadingTest.java
+++ b/java/test/src/main/java/io/ray/test/MultiThreadingTest.java
@@ -19,7 +19,7 @@
 import org.testng.Assert;
 import org.testng.annotations.Test;
 
-@Test
+@Test(groups = {"cluster"})
 public class MultiThreadingTest extends BaseTest {
 
   private static final Logger LOGGER = LoggerFactory.getLogger(MultiThreadingTest.class);
@@ -221,11 +221,6 @@ static boolean testMissingWrapRunnable() throws InterruptedException {
     return true;
   }
 
-  @Test
-  public void testMissingWrapRunnableInDriver() throws InterruptedException {
-    testMissingWrapRunnable();
-  }
-
   @Test
   public void testMissingWrapRunnableInWorker() {
     Ray.task(MultiThreadingTest::testMissingWrapRunnable).remote().get();
diff --git a/java/test/src/main/java/io/ray/test/TestUtils.java b/java/test/src/main/java/io/ray/test/TestUtils.java
--- a/java/test/src/main/java/io/ray/test/TestUtils.java
+++ b/java/test/src/main/java/io/ray/test/TestUtils.java
@@ -3,6 +3,7 @@
 import com.google.common.base.Preconditions;
 import io.ray.api.ObjectRef;
 import io.ray.api.Ray;
+import io.ray.runtime.AbstractRayRuntime;
 import io.ray.runtime.RayRuntimeInternal;
 import io.ray.runtime.RayRuntimeProxy;
 import io.ray.runtime.config.RunMode;
@@ -83,8 +84,19 @@ public static RayRuntimeInternal getRuntime() {
   }
 
   public static RayRuntimeInternal getUnderlyingRuntime() {
+    if (Ray.internal() instanceof AbstractRayRuntime) {
+      return (RayRuntimeInternal) Ray.internal();
+    }
     RayRuntimeProxy proxy = (RayRuntimeProxy) (java.lang.reflect.Proxy
         .getInvocationHandler(Ray.internal()));
     return proxy.getRuntimeObject();
   }
+
+  private static int getNumWorkersPerProcessRemoteFunction() {
+    return TestUtils.getRuntime().getRayConfig().numWorkersPerProcess;
+  }
+
+  public static int getNumWorkersPerProcess() {
+    return Ray.task(TestUtils::getNumWorkersPerProcessRemoteFunction).remote().get();
+  }
 }