Repository: apache/shenyu (Java)

Issue:
[Task] optimize UpstreamCheckService
### Description

In the k8s environment, due to the ip change after the client container is restarted, the UpstreamCheckService health check is easy to fail. Due to the use of single-thread health check, it is easy to slow down the health check of other nodes, and eventually cause the OOM of shenyu-admin.

This task needs you to use muti-thread to optimize `org.apache.shenyu.admin.service.impl.UpstreamCheckService#scheduled`, you can refer to the implementation of `org.apache.shenyu.loadbalancer.cache.UpstreamCacheManager`.

Please remember to allow users to configure the number of threads, maybe you should also do some modify to `org.apache.shenyu.loadbalancer.cache.UpstreamCheckTask` to support config.

Good luck !

### Task List

- [ ] optimize UpstreamCheckService;
- [ ] support config the number of threads in `UpstreamCheckTask`;


Discussion:
pls assign to me and I'll try it.



Patch:
diff --git a/shenyu-admin/src/main/java/org/apache/shenyu/admin/service/impl/UpstreamCheckService.java b/shenyu-admin/src/main/java/org/apache/shenyu/admin/service/impl/UpstreamCheckService.java
index c25f630d0498..4cb1a81c25d4 100644
--- a/shenyu-admin/src/main/java/org/apache/shenyu/admin/service/impl/UpstreamCheckService.java
+++ b/shenyu-admin/src/main/java/org/apache/shenyu/admin/service/impl/UpstreamCheckService.java
@@ -65,9 +65,11 @@
 import java.util.Optional;
 import java.util.Properties;
 import java.util.Set;
+import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.CopyOnWriteArrayList;
 import java.util.concurrent.ScheduledFuture;
 import java.util.concurrent.ScheduledThreadPoolExecutor;
+import java.util.concurrent.ThreadFactory;
 import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
 
@@ -97,6 +99,8 @@ public class UpstreamCheckService {
 
     private final boolean checked;
 
+    private final Integer scheduledThreads;
+
     private final SelectorMapper selectorMapper;
 
     private final ApplicationEventPublisher eventPublisher;
@@ -111,6 +115,10 @@ public class UpstreamCheckService {
 
     private ScheduledFuture<?> scheduledFuture;
 
+    private ScheduledThreadPoolExecutor invokeExecutor;
+
+    private final List<CompletableFuture<Void>> futures = Lists.newArrayList();
+
     /**
      * Instantiates a new Upstream check service.
      *
@@ -134,6 +142,7 @@ public UpstreamCheckService(final SelectorMapper selectorMapper,
         this.converterFactor = converterFactor;
         Properties props = shenyuRegisterCenterConfig.getProps();
         this.checked = Boolean.parseBoolean(props.getProperty(Constants.IS_CHECKED, Constants.DEFAULT_CHECK_VALUE));
+        this.scheduledThreads = Integer.parseInt(props.getProperty(Constants.ZOMBIE_CHECK_THREADS, Constants.ZOMBIE_CHECK_THREADS_VALUE));
         this.zombieCheckTimes = Integer.parseInt(props.getProperty(Constants.ZOMBIE_CHECK_TIMES, Constants.ZOMBIE_CHECK_TIMES_VALUE));
         this.scheduledTime = Integer.parseInt(props.getProperty(Constants.SCHEDULED_TIME, Constants.SCHEDULED_TIME_VALUE));
         this.registerType = shenyuRegisterCenterConfig.getRegisterType();
@@ -151,6 +160,9 @@ public void setup() {
             this.fetchUpstreamData();
             executor = new ScheduledThreadPoolExecutor(1, ShenyuThreadFactory.create("scheduled-upstream-task", false));
             scheduledFuture = executor.scheduleWithFixedDelay(this::scheduled, 10, scheduledTime, TimeUnit.SECONDS);
+
+            ThreadFactory requestFactory = ShenyuThreadFactory.create("upstream-health-check-request", true);
+            invokeExecutor = new ScheduledThreadPoolExecutor(this.scheduledThreads, requestFactory);
         }
     }
 
@@ -203,7 +215,7 @@ public boolean submit(final String selectorId, final CommonUpstream commonUpstre
         executor.execute(() -> updateHandler(selectorId, upstreams, upstreams));
         return true;
     }
-    
+
     /**
      * If the health check passes, the service will be added to
      * the normal service list; if the health check fails, the service
@@ -213,7 +225,7 @@ public boolean submit(final String selectorId, final CommonUpstream commonUpstre
      * that do not register with the gateway by listening to
      * {@link org.springframework.context.event.ContextRefreshedEvent},
      * which will cause some problems,
-     * check https://github.com/apache/shenyu/issues/3484 for more details.
+     * check <a href="https://github.com/apache/shenyu/issues/3484">...</a> for more details.
      *
      * @param selectorId     the selector id
      * @param commonUpstream the common upstream
@@ -244,18 +256,37 @@ public void replace(final String selectorId, final List<CommonUpstream> commonUp
 
     private void scheduled() {
         try {
-            if (!ZOMBIE_SET.isEmpty()) {
-                ZOMBIE_SET.parallelStream().forEach(this::checkZombie);
-            }
-            if (!UPSTREAM_MAP.isEmpty()) {
-                UPSTREAM_MAP.forEach(this::check);
-            }
+            doCheck();
+            waitFinish();
         } catch (Exception e) {
             LOG.error("upstream scheduled check error -------- ", e);
         }
     }
 
+    private void doCheck() {
+        // check zombie
+        if (!ZOMBIE_SET.isEmpty()) {
+            ZOMBIE_SET.forEach(this::checkZombie);
+        }
+        // check up
+        if (!UPSTREAM_MAP.isEmpty()) {
+            UPSTREAM_MAP.forEach(this::check);
+        }
+    }
+
+    private void waitFinish() {
+        // wait all check success
+        CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])).join();
+        // clear, for next time
+        futures.clear();
+    }
+
     private void checkZombie(final ZombieUpstream zombieUpstream) {
+        CompletableFuture<Void> future = CompletableFuture.runAsync(() -> checkZombie0(zombieUpstream), invokeExecutor);
+        futures.add(future);
+    }
+
+    private void checkZombie0(final ZombieUpstream zombieUpstream) {
         ZOMBIE_SET.remove(zombieUpstream);
         String selectorId = zombieUpstream.getSelectorId();
         CommonUpstream commonUpstream = zombieUpstream.getCommonUpstream();
@@ -277,23 +308,36 @@ private void checkZombie(final ZombieUpstream zombieUpstream) {
     }
 
     private void check(final String selectorId, final List<CommonUpstream> upstreamList) {
-        List<CommonUpstream> successList = Lists.newArrayListWithCapacity(upstreamList.size());
+        final List<CompletableFuture<CommonUpstream>> checkFutures = new ArrayList<>(upstreamList.size());
         for (CommonUpstream commonUpstream : upstreamList) {
-            final boolean pass = UpstreamCheckUtils.checkUrl(commonUpstream.getUpstreamUrl());
-            if (pass) {
-                if (!commonUpstream.isStatus()) {
-                    commonUpstream.setTimestamp(System.currentTimeMillis());
-                    commonUpstream.setStatus(true);
-                    LOG.info("UpstreamCacheManager check success the url: {}, host: {} ", commonUpstream.getUpstreamUrl(), commonUpstream.getUpstreamHost());
+            checkFutures.add(CompletableFuture.supplyAsync(() -> {
+                final boolean pass = UpstreamCheckUtils.checkUrl(commonUpstream.getUpstreamUrl());
+                if (pass) {
+                    if (!commonUpstream.isStatus()) {
+                        commonUpstream.setTimestamp(System.currentTimeMillis());
+                        commonUpstream.setStatus(true);
+                        LOG.info("UpstreamCacheManager check success the url: {}, host: {} ", commonUpstream.getUpstreamUrl(), commonUpstream.getUpstreamHost());
+                    }
+                    return commonUpstream;
+                } else {
+                    commonUpstream.setStatus(false);
+                    ZOMBIE_SET.add(ZombieUpstream.transform(commonUpstream, zombieCheckTimes, selectorId));
+                    LOG.error("check the url={} is fail ", commonUpstream.getUpstreamUrl());
                 }
-                successList.add(commonUpstream);
-            } else {
-                commonUpstream.setStatus(false);
-                ZOMBIE_SET.add(ZombieUpstream.transform(commonUpstream, zombieCheckTimes, selectorId));
-                LOG.error("check the url={} is fail ", commonUpstream.getUpstreamUrl());
-            }
+                return null;
+            }).exceptionally(ex -> {
+                LOG.error("An exception occurred during the check of url {}: {}", commonUpstream.getUpstreamUrl(), ex);
+                return null;
+            }));
         }
-        updateHandler(selectorId, upstreamList, successList);
+
+        this.futures.add(CompletableFuture.runAsync(() -> {
+            List<CommonUpstream> successList = checkFutures.stream()
+                    .map(CompletableFuture::join)
+                    .filter(Objects::nonNull)
+                    .collect(Collectors.toList());
+            updateHandler(selectorId, upstreamList, successList);
+        }));
     }
 
     private void updateHandler(final String selectorId, final List<CommonUpstream> upstreamList, final List<CommonUpstream> successList) {
@@ -362,7 +406,7 @@ public void fetchUpstreamData() {
                     }
                 });
     }
-    
+
     /**
      * listen {@link SelectorCreatedEvent} add data permission.
      *
@@ -376,7 +420,7 @@ public void onSelectorCreated(final SelectorCreatedEvent event) {
             replace(event.getSelector().getId(), CommonUpstreamUtils.convertCommonUpstreamList(existDivideUpstreams));
         }
     }
-    
+
     /**
      * listen {@link SelectorCreatedEvent} add data permission.
      *
diff --git a/shenyu-admin/src/main/resources/application.yml b/shenyu-admin/src/main/resources/application.yml
index b041451341c7..d2388e005a97 100755
--- a/shenyu-admin/src/main/resources/application.yml
+++ b/shenyu-admin/src/main/resources/application.yml
@@ -42,6 +42,7 @@ shenyu:
       sessionTimeout: 5000
       connectionTimeout: 2000
       checked: true
+      zombieCheckThreads: 10
       zombieCheckTimes: 5
       scheduledTime: 10
       nacosNameSpace: ShenyuRegisterCenter
diff --git a/shenyu-bootstrap/src/main/resources/application.yml b/shenyu-bootstrap/src/main/resources/application.yml
index 030c4015191a..f90973d6fa1e 100644
--- a/shenyu-bootstrap/src/main/resources/application.yml
+++ b/shenyu-bootstrap/src/main/resources/application.yml
@@ -238,6 +238,7 @@ shenyu:
     threads: 16
   upstreamCheck:
     enabled: false
+    poolSize: 10
     timeout: 3000
     healthyThreshold: 1
     unhealthyThreshold: 1
diff --git a/shenyu-common/src/main/java/org/apache/shenyu/common/config/ShenyuConfig.java b/shenyu-common/src/main/java/org/apache/shenyu/common/config/ShenyuConfig.java
index f0d3631daf5d..3f1bd8b681fa 100644
--- a/shenyu-common/src/main/java/org/apache/shenyu/common/config/ShenyuConfig.java
+++ b/shenyu-common/src/main/java/org/apache/shenyu/common/config/ShenyuConfig.java
@@ -794,6 +794,8 @@ public void setLocal(final boolean local) {
     public static class UpstreamCheck {
     
         private boolean enabled;
+
+        private Integer poolSize = 10;
         
         private Integer timeout = 3000;
         
@@ -824,7 +826,25 @@ public boolean getEnabled() {
         public void setEnabled(final boolean enabled) {
             this.enabled = enabled;
         }
-    
+
+        /**
+         * get checkThreadPoolSize.
+         *
+         * @return checkThreadPoolSize
+         */
+        public Integer getPoolSize() {
+            return poolSize;
+        }
+
+        /**
+         * set checkThreadPoolSize.
+         *
+         * @param poolSize checkThreadPoolSize
+         */
+        public void setPoolSize(final Integer poolSize) {
+            this.poolSize = poolSize;
+        }
+
         /**
          * Gets timeout.
          *
diff --git a/shenyu-common/src/main/java/org/apache/shenyu/common/constant/Constants.java b/shenyu-common/src/main/java/org/apache/shenyu/common/constant/Constants.java
index 94c487ff9fcb..68c933077f11 100644
--- a/shenyu-common/src/main/java/org/apache/shenyu/common/constant/Constants.java
+++ b/shenyu-common/src/main/java/org/apache/shenyu/common/constant/Constants.java
@@ -437,6 +437,16 @@ public interface Constants {
      */
     String DEFAULT_CHECK_VALUE = "false";
 
+    /**
+     * zombie check threads.
+     */
+    String ZOMBIE_CHECK_THREADS = "zombieCheckThreads";
+
+    /**
+     * default zombie check threads value.
+     */
+    String ZOMBIE_CHECK_THREADS_VALUE = "10";
+
     /**
      * zombie check times.
      */
diff --git a/shenyu-common/src/main/java/org/apache/shenyu/common/dto/convert/selector/ZombieUpstream.java b/shenyu-common/src/main/java/org/apache/shenyu/common/dto/convert/selector/ZombieUpstream.java
index dfba6bdfdcff..4ef84c51836f 100644
--- a/shenyu-common/src/main/java/org/apache/shenyu/common/dto/convert/selector/ZombieUpstream.java
+++ b/shenyu-common/src/main/java/org/apache/shenyu/common/dto/convert/selector/ZombieUpstream.java
@@ -130,7 +130,6 @@ public boolean equals(final Object o) {
         }
         ZombieUpstream that = (ZombieUpstream) o;
         return new EqualsBuilder()
-                .append(zombieCheckTimes, that.zombieCheckTimes)
                 .append(commonUpstream, that.commonUpstream)
                 .append(selectorId, that.selectorId)
                 .isEquals();
@@ -140,7 +139,6 @@ public boolean equals(final Object o) {
     public int hashCode() {
         return new HashCodeBuilder(17, 37)
                 .append(commonUpstream)
-                .append(zombieCheckTimes)
                 .append(selectorId)
                 .toHashCode();
     }
diff --git a/shenyu-loadbalancer/src/main/java/org/apache/shenyu/loadbalancer/cache/UpstreamCacheManager.java b/shenyu-loadbalancer/src/main/java/org/apache/shenyu/loadbalancer/cache/UpstreamCacheManager.java
index 899ae24dbd00..630b9f15588f 100644
--- a/shenyu-loadbalancer/src/main/java/org/apache/shenyu/loadbalancer/cache/UpstreamCacheManager.java
+++ b/shenyu-loadbalancer/src/main/java/org/apache/shenyu/loadbalancer/cache/UpstreamCacheManager.java
@@ -51,6 +51,8 @@ public final class UpstreamCacheManager {
      */
     private Boolean checkEnable;
 
+    private int poolSize;
+
     private int checkTimeout;
 
     private int checkInterval;
@@ -74,6 +76,7 @@ private void initHealthCheck() {
         ShenyuConfig shenyuConfig = Optional.ofNullable(Singleton.INST.get(ShenyuConfig.class)).orElse(new ShenyuConfig());
         UpstreamCheck upstreamCheck = shenyuConfig.getUpstreamCheck();
         checkEnable = upstreamCheck.getEnabled();
+        poolSize = upstreamCheck.getPoolSize();
         checkTimeout = upstreamCheck.getTimeout();
         healthyThreshold = upstreamCheck.getHealthyThreshold();
         unhealthyThreshold = upstreamCheck.getUnhealthyThreshold();
@@ -86,6 +89,7 @@ private void initHealthCheck() {
 
     private void createTask() {
         task = new UpstreamCheckTask(checkInterval);
+        task.setPoolSize(poolSize);
         task.setCheckTimeout(checkTimeout);
         task.setHealthyThreshold(healthyThreshold);
         task.setUnhealthyThreshold(unhealthyThreshold);
diff --git a/shenyu-loadbalancer/src/main/java/org/apache/shenyu/loadbalancer/cache/UpstreamCheckTask.java b/shenyu-loadbalancer/src/main/java/org/apache/shenyu/loadbalancer/cache/UpstreamCheckTask.java
index 46b9ddc76e75..e604a22ef3f4 100644
--- a/shenyu-loadbalancer/src/main/java/org/apache/shenyu/loadbalancer/cache/UpstreamCheckTask.java
+++ b/shenyu-loadbalancer/src/main/java/org/apache/shenyu/loadbalancer/cache/UpstreamCheckTask.java
@@ -64,6 +64,8 @@ public final class UpstreamCheckTask implements Runnable {
 
     private ExecutorService executor;
 
+    private int poolSize;
+
     private int checkTimeout = 3000;
 
     private int healthyThreshold = 1;
@@ -99,7 +101,7 @@ public void schedule() {
 
         // executor for async request, avoid request block health check thread
         ThreadFactory requestFactory = ShenyuThreadFactory.create("upstream-health-check-request", true);
-        executor = new ScheduledThreadPoolExecutor(10, requestFactory);
+        executor = new ScheduledThreadPoolExecutor(poolSize, requestFactory);
     }
     
     /**
@@ -110,7 +112,25 @@ public void schedule() {
     public void setCheckTimeout(final int checkTimeout) {
         this.checkTimeout = checkTimeout;
     }
-    
+
+    /**
+     * get checkThreadPoolSize.
+     *
+     * @return checkThreadPoolSize
+     */
+    public int getPoolSize() {
+        return poolSize;
+    }
+
+    /**
+     * set checkThreadPoolSize.
+     *
+     * @param poolSize checkThreadPoolSize
+     */
+    public void setPoolSize(final int poolSize) {
+        this.poolSize = poolSize;
+    }
+
     /**
      * Set healthy threshold.
      *


Test patch:
diff --git a/shenyu-common/src/test/java/org/apache/shenyu/common/config/ShenyuConfigTest.java b/shenyu-common/src/test/java/org/apache/shenyu/common/config/ShenyuConfigTest.java
index cce15adc49c7..9ff90c729555 100644
--- a/shenyu-common/src/test/java/org/apache/shenyu/common/config/ShenyuConfigTest.java
+++ b/shenyu-common/src/test/java/org/apache/shenyu/common/config/ShenyuConfigTest.java
@@ -80,6 +80,7 @@ private void notEmptyElements(final Object... objects) {
     public void testUpstreamCheck() {
         ShenyuConfig.UpstreamCheck upstreamCheck = config.getUpstreamCheck();
         upstreamCheck.setEnabled(false);
+        upstreamCheck.setPoolSize(10);
         upstreamCheck.setHealthyThreshold(4);
         upstreamCheck.setTimeout(10);
         upstreamCheck.setInterval(5);
@@ -87,7 +88,7 @@ public void testUpstreamCheck() {
         upstreamCheck.setPrintEnabled(false);
         upstreamCheck.setPrintInterval(5);
 
-        notEmptyElements(upstreamCheck.getEnabled(), upstreamCheck.getHealthyThreshold(), upstreamCheck.getTimeout(),
+        notEmptyElements(upstreamCheck.getEnabled(), upstreamCheck.getPoolSize(), upstreamCheck.getHealthyThreshold(), upstreamCheck.getTimeout(),
                 upstreamCheck.getInterval(), upstreamCheck.getUnhealthyThreshold(), upstreamCheck.getPrintInterval(), upstreamCheck.getPrintEnabled());
     }