dashboard: support automatic/manual removal of unhealthy machines and disconnected applications (#168)

* Support removing unhealthy machine manually
* Auto removal support for dead machine
* Auto hidden for app if one have no healthy machine after some time(configurable)
* Auto removal for app if hidden status lasts(configurable).

sentinel-dashboard/Sentinel_Dashboard_Feature.md

@@ -47,7 +47,35 @@ Sentinel 提供了多种规则来保护系统的不同部分。流量控制规
 本控制台只是用于演示 Sentinel 的基本能力和工作流程，并没有依赖生产环境中所必需的组件，比如**持久化的后端数据库、可靠的配置中心**等。
 目前 Sentinel 采用内存态的方式存储监控和规则数据，监控最长存储时间为 5 分钟，控制台重启后数据丢失。
 
+## 3. 配置项
+
+控制台的一些特性可以通过配置项来进行配置，配置项主要有两个来源：`System.getProperty()`和`System.getenv()`，同时存在时后者可以覆盖前者。  
+> 环境变量因为不支持`.`所以需要将其更换为`_`。
+
+项 | 类型 | 默认值 | 最小值 | 描述
+--- | --- | --- | --- | ---
+sentinel.dashboard.app.hideAppNoMachineMillis | Integer | 0 | 60000 | 是否隐藏无健康节点的应用，距离最近一次主机心跳时间的毫秒数，默认关闭
+sentinel.dashboard.removeAppNoMachineMillis | Integer | 0 | 120000 | 是否自动删除无健康节点的应用，距离最近一次其下节点的心跳时间毫秒数，默认关闭
+sentinel.dashboard.unhealthyMachineMillis | Integer | 60000 | 30000 | 主机失联判定，不可关闭
+sentinel.dashboard.autoRemoveMachineMillis | Integer | 0 | 300000 | 距离最近心跳时间超过指定时间是否自动删除失联节点，默认关闭
+
+配置示例：
+
+命令行
+```shell
+java -Dsentinel.dashboard.app.hideAppNoMachineMillis=60000
+```
+java
+```java
+System.setProperty("sentinel.dashboard.app.hideAppNoMachineMillis", "60000");
+```
+环境变量
+```shell
+sentinel_dashboard_app_hideAppNoMachineMillis=60000
+```
+
 更多：
 
 - [Sentinel 控制台启动和客户端接入](./README.md)
-- [控制台 Wiki](https://github.com/alibaba/Sentinel/wiki/%E6%8E%A7%E5%88%B6%E5%8F%B0)
+- [控制台 Wiki](https://github.com/alibaba/Sentinel/wiki/%E6%8E%A7%E5%88%B6%E5%8F%B0)
+

sentinel-dashboard/pom.xml

@@ -23,7 +23,6 @@
         <dependency>
             <groupId>com.alibaba.csp</groupId>
             <artifactId>sentinel-core</artifactId>
-            <version>${project.version}</version>
         </dependency>
         <dependency>
             <groupId>com.alibaba.csp</groupId>
@@ -33,7 +32,6 @@
         <dependency>
             <groupId>com.alibaba.csp</groupId>
             <artifactId>sentinel-transport-simple-http</artifactId>
-            <version>${project.version}</version>
         </dependency>
         <dependency>
             <groupId>com.alibaba.csp</groupId>
@@ -118,7 +116,12 @@
             <version>${apollo.openapi.version}</version>
             <scope>test</scope>
         </dependency>
-
+        <dependency>
+        	<groupId>com.github.stefanbirkner</groupId>
+        	<artifactId>system-rules</artifactId>
+        	<version>1.16.1</version>
+        	<scope>test</scope>
+        </dependency>
     </dependencies>
 
     <build>

sentinel-dashboard/src/main/java/com/alibaba/csp/sentinel/dashboard/config/DashboardConfig.java

@@ -0,0 +1,103 @@
+/*
+ * Copyright 1999-2018 Alibaba Group Holding Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alibaba.csp.sentinel.dashboard.config;
+
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang.math.NumberUtils;
+import org.springframework.lang.NonNull;
+
+/**
+ * Dashboard config support
+ * <p>
+ * Dashboard supports configuration loading by several ways by order:<br>
+ * 1. System.properties<br>
+ * 2. Env
+ * 
+ * @author jason
+ * @since 1.5.0
+ *
+ */
+public class DashboardConfig {
+    /**
+     * hide app in sidebar when it had no healthy machine after specific period in millis
+     */
+    public static final String CONFIG_HIDE_APP_NO_MACHINE_MILLIS = "sentinel.dashboard.app.hideAppNoMachineMillis";
+    /**
+     * remove app when it had no healthy machine after specific period in millis
+     */
+    public static final String CONFIG_REMOVE_APP_NO_MACHINE_MILLIS = "sentinel.dashboard.removeAppNoMachineMillis";
+    /**
+     * unhealthy millis
+     */
+    public static final String CONFIG_UNHEALTHY_MACHINE_MILLIS = "sentinel.dashboard.unhealthyMachineMillis";
+    /**
+     * auto remove unhealthy machine after specific period in millis
+     */
+    public static final String CONFIG_AUTO_REMOVE_MACHINE_MILLIS = "sentinel.dashboard.autoRemoveMachineMillis";
+    private static final ConcurrentMap<String, Object> cacheMap = new ConcurrentHashMap<>();
+    
+    @NonNull
+    private static String getConfig(String name) {
+        // env
+        String val = System.getenv(name);
+        if (StringUtils.isNotEmpty(val)) {
+            return val;
+        }
+        // properties
+        val = System.getProperty(name);
+        if (StringUtils.isNotEmpty(val)) {
+            return val;
+        }
+        return "";
+    }
+    
+    protected static int getConfigInt(String name, int defaultVal, int minVal) {
+        if (cacheMap.containsKey(name)) {
+            return (int)cacheMap.get(name);
+        }
+        int val = NumberUtils.toInt(getConfig(name));
+        if (val == 0) {
+            val = defaultVal;
+        } else if (val < minVal) {
+            val = minVal;
+        }
+        cacheMap.put(name, val);
+        return val;
+    }
+    
+    public static int getHideAppNoMachineMillis() {
+        return getConfigInt(CONFIG_HIDE_APP_NO_MACHINE_MILLIS, 0, 60000);
+    }
+    
+    public static int getRemoveAppNoMachineMillis() {
+        return getConfigInt(CONFIG_REMOVE_APP_NO_MACHINE_MILLIS, 0, 120000);
+    }
+    
+    public static int getAutoRemoveMachineMillis() {
+        return getConfigInt(CONFIG_AUTO_REMOVE_MACHINE_MILLIS, 0, 300000);
+    }
+    
+    public static int getUnhealthyMachineMillis() {
+        return getConfigInt(CONFIG_UNHEALTHY_MACHINE_MILLIS, 60000, 30000);
+    }
+    
+    public static void clearCache() {
+        cacheMap.clear();
+    }
+}

sentinel-dashboard/src/main/java/com/alibaba/csp/sentinel/dashboard/controller/AppController.java

@@ -32,6 +32,7 @@ import org.springframework.http.MediaType;
 import org.springframework.stereotype.Controller;
 import org.springframework.web.bind.annotation.PathVariable;
 import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestParam;
 import org.springframework.web.bind.annotation.ResponseBody;
 
 /**
@@ -79,4 +80,21 @@ public class AppController {
         });
         return Result.ofSuccess(MachineInfoVo.fromMachineInfoList(list));
     }
+    
+    @ResponseBody
+    @RequestMapping(value = "/{app}/machine/remove.json")
+    Result<String> removeMachineById(
+            @PathVariable("app") String app,
+            @RequestParam(name = "ip") String ip,
+            @RequestParam(name = "port") int port) {
+        AppInfo appInfo = appManagement.getDetailApp(app);
+        if (appInfo == null) {
+            return Result.ofSuccess(null);
+        }
+        if (appManagement.removeMachine(app, ip, port)) {
+            return Result.ofSuccessMsg("success");
+        } else {
+            return Result.ofFail(1, "remove failed");
+        }
+    }
 }

sentinel-dashboard/src/main/java/com/alibaba/csp/sentinel/dashboard/controller/MachineRegistryController.java

@@ -15,8 +15,6 @@
  */
 package com.alibaba.csp.sentinel.dashboard.controller;
 
-import java.util.Date;
-
 import com.alibaba.csp.sentinel.dashboard.discovery.AppManagement;
 import com.alibaba.csp.sentinel.util.StringUtil;
 
@@ -57,14 +55,15 @@ public class MachineRegistryController {
             return Result.ofFail(-1, "your port not set yet");
         }
         String sentinelVersion = StringUtil.isEmpty(v) ? "unknown" : v;
-        long timestamp = version == null ? System.currentTimeMillis() : version;
+        version = version == null ? System.currentTimeMillis() : version;
         try {
             MachineInfo machineInfo = new MachineInfo();
             machineInfo.setApp(app);
             machineInfo.setHostname(hostname);
             machineInfo.setIp(ip);
             machineInfo.setPort(port);
-            machineInfo.setTimestamp(new Date(timestamp));
+            machineInfo.setHeartbeatVersion(version);
+            machineInfo.setLastHeatbeat(System.currentTimeMillis());
             machineInfo.setVersion(sentinelVersion);
             appManagement.addMachine(machineInfo);
             return Result.ofSuccessMsg("success");

sentinel-dashboard/src/main/java/com/alibaba/csp/sentinel/dashboard/datasource/entity/MachineEntity.java

@@ -103,7 +103,8 @@ public class MachineEntity {
         machineInfo.setHostname(hostname);
         machineInfo.setIp(ip);
         machineInfo.setPort(port);
-        machineInfo.setTimestamp(timestamp);
+        machineInfo.setLastHeatbeat(timestamp.getTime());
+        machineInfo.setHeartbeatVersion(timestamp.getTime());
 
         return machineInfo;
     }

sentinel-dashboard/src/main/java/com/alibaba/csp/sentinel/dashboard/discovery/AppInfo.java

@@ -15,12 +15,29 @@
  */
 package com.alibaba.csp.sentinel.dashboard.discovery;
 
+import java.util.Comparator;
 import java.util.HashSet;
 import java.util.Optional;
+import java.util.Iterator;
 import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
 
+import com.alibaba.csp.sentinel.dashboard.config.DashboardConfig;
+
 public class AppInfo {
+    private static final Comparator<MachineInfo> COMPARATOR_BY_MACHINE_HEARTBEAT_DESC = new Comparator<MachineInfo>() {
+
+        @Override
+        public int compare(MachineInfo o1, MachineInfo o2) {
+            if (o1.getLastHeatbeat() < o2.getLastHeatbeat()) {
+                return -1;
+            }
+            if (o1.getLastHeatbeat() > o2.getLastHeatbeat()) {
+                return 1;
+            }
+            return 0;
+        }
+    };
 
     private String app = "";
 
@@ -59,10 +76,58 @@ public class AppInfo {
         machines.remove(machineInfo);
         return machines.add(machineInfo);
     }
-
+    
+    public synchronized boolean removeMachine(String ip, int port) {
+        Iterator<MachineInfo> it = machines.iterator();
+        while (it.hasNext()) {
+            MachineInfo machine = it.next();
+            if (machine.getIp().equals(ip) && machine.getPort() == port) {
+                it.remove();
+                return true;
+            }
+        }
+        return false;
+    }
+    
     public Optional<MachineInfo> getMachine(String ip, int port) {
         return machines.stream()
             .filter(e -> e.getIp().equals(ip) && e.getPort().equals(port))
             .findFirst();
     }
+    
+    private boolean heartbeatJudge(int threshold) {
+        if (machines.size() == 0) {
+            return false;
+        }
+        if (threshold > 0) {
+            long healthyCount = machines.stream()
+                    .filter(m -> m.isHealthy())
+                    .count();
+            if (healthyCount == 0) {
+                // no machine
+                long recentHeartBeat = machines.stream()
+                        .max(COMPARATOR_BY_MACHINE_HEARTBEAT_DESC).get().getLastHeatbeat();
+                return System.currentTimeMillis() - recentHeartBeat < threshold;
+            }
+        }
+        return true;
+    }
+    
+    /**
+     * having no healthy machine and should not be displayed
+     * 
+     * @return
+     */
+    public boolean isShown() {
+        return heartbeatJudge(DashboardConfig.getHideAppNoMachineMillis());
+    }
+    
+    /**
+     * having no healthy machine and should be removed
+     * 
+     * @return
+     */
+    public boolean isDead() {
+        return !heartbeatJudge(DashboardConfig.getRemoveAppNoMachineMillis());
+    }
 }

sentinel-dashboard/src/main/java/com/alibaba/csp/sentinel/dashboard/discovery/AppManagement.java

@@ -52,6 +52,11 @@ public class AppManagement implements MachineDiscovery {
     public long addMachine(MachineInfo machineInfo) {
         return machineDiscovery.addMachine(machineInfo);
     }
+    
+    @Override
+    public boolean removeMachine(String app, String ip, int port) {
+        return machineDiscovery.removeMachine(app, ip, port);
+    }
 
     @Override
     public List<String> getAppNames() {
@@ -62,5 +67,10 @@ public class AppManagement implements MachineDiscovery {
     public AppInfo getDetailApp(String app) {
         return machineDiscovery.getDetailApp(app);
     }
+    
+    @Override
+    public void removeApp(String app) {
+        machineDiscovery.removeApp(app);
+    }
 
 }

sentinel-dashboard/src/main/java/com/alibaba/csp/sentinel/dashboard/discovery/MachineDiscovery.java

@@ -20,7 +20,6 @@ import java.util.Set;
 
 public interface MachineDiscovery {
 
-    long MAX_CLIENT_LIVE_TIME_MS = 1000 * 60 * 5;
     String UNKNOWN_APP_NAME = "CLUSTER_NOT_STARTED";
 
     List<String> getAppNames();
@@ -28,6 +27,10 @@ public interface MachineDiscovery {
     Set<AppInfo> getBriefApps();
 
     AppInfo getDetailApp(String app);
+    
+    void removeApp(String app);
 
     long addMachine(MachineInfo machineInfo);
+    
+    boolean removeMachine(String app, String ip, int port);
 }

sentinel-dashboard/src/main/java/com/alibaba/csp/sentinel/dashboard/discovery/MachineInfo.java

@@ -15,9 +15,9 @@
  */
 package com.alibaba.csp.sentinel.dashboard.discovery;
 
-import java.util.Date;
 import java.util.Objects;
 
+import com.alibaba.csp.sentinel.dashboard.config.DashboardConfig;
 import com.alibaba.csp.sentinel.util.StringUtil;
 
 public class MachineInfo implements Comparable<MachineInfo> {
@@ -26,7 +26,8 @@ public class MachineInfo implements Comparable<MachineInfo> {
     private String hostname = "";
     private String ip = "";
     private Integer port = -1;
-    private Date timestamp;
+    private long lastHeatbeat;
+    private long heartbeatVersion;
 
     /**
      * Indicates the version of Sentinel client (since 0.2.0).
@@ -77,12 +78,12 @@ public class MachineInfo implements Comparable<MachineInfo> {
         this.ip = ip;
     }
 
-    public Date getTimestamp() {
-        return timestamp;
+    public long getHeartbeatVersion() {
+        return heartbeatVersion;
     }
-
-    public void setTimestamp(Date timestamp) {
-        this.timestamp = timestamp;
+    
+    public void setHeartbeatVersion(long heartbeatVersion) {
+        this.heartbeatVersion = heartbeatVersion;
     }
 
     public String getVersion() {
@@ -93,6 +94,32 @@ public class MachineInfo implements Comparable<MachineInfo> {
         this.version = version;
         return this;
     }
+    
+    public boolean isHealthy() {
+        long delta = System.currentTimeMillis() - lastHeatbeat;
+        return delta < DashboardConfig.getUnhealthyMachineMillis();
+    }
+    
+    /**
+     * whether dead should be removed
+     * 
+     * @return
+     */
+    public boolean isDead() {
+        if (DashboardConfig.getAutoRemoveMachineMillis() > 0) {
+            long delta = System.currentTimeMillis() - lastHeatbeat;
+            return delta > DashboardConfig.getAutoRemoveMachineMillis();
+        }
+        return false;
+    }
+    
+    public long getLastHeatbeat() {
+        return lastHeatbeat;
+    }
+    
+    public void setLastHeatbeat(long lastHeatbeat) {
+        this.lastHeatbeat = lastHeatbeat;
+    }
 
     @Override
     public int compareTo(MachineInfo o) {
@@ -110,14 +137,16 @@ public class MachineInfo implements Comparable<MachineInfo> {
 
     @Override
     public String toString() {
-        return "MachineInfo{" +
-            "app='" + app + '\'' +
-            ", hostname='" + hostname + '\'' +
-            ", ip='" + ip + '\'' +
-            ", port=" + port +
-            ", timestamp=" + timestamp +
-            ", version='" + version + '\'' +
-            '}';
+        return new StringBuilder("MachineInfo {")
+            .append("app='").append(app).append('\'')
+            .append(", hostname='").append(hostname).append('\'')
+            .append(", ip='").append(ip).append('\'')
+            .append(", port=").append(port)
+            .append(", heartbeatVersion=").append(heartbeatVersion)
+            .append(", lastHeartbeat=").append(lastHeatbeat)
+            .append(", version='").append(version).append('\'')
+            .append(", healthy=").append(isHealthy())
+            .append('}').toString();
     }
 
     @Override

sentinel-dashboard/src/main/java/com/alibaba/csp/sentinel/dashboard/discovery/SimpleMachineDiscovery.java

@@ -36,6 +36,15 @@ public class SimpleMachineDiscovery implements MachineDiscovery {
         appInfo.addMachine(machineInfo);
         return 1;
     }
+    
+    @Override
+    public boolean removeMachine(String app, String ip, int port) {
+        AppInfo appInfo = apps.get(app);
+        if (appInfo != null) {
+            return appInfo.removeMachine(ip, port);
+        }
+        return false;
+    }
 
     @Override
     public List<String> getAppNames() {
@@ -52,4 +61,9 @@ public class SimpleMachineDiscovery implements MachineDiscovery {
         return new HashSet<>(apps.values());
     }
 
+    @Override
+    public void removeApp(String app) {
+        apps.remove(app);
+    }
+
 }

sentinel-dashboard/src/main/java/com/alibaba/csp/sentinel/dashboard/domain/vo/MachineInfoVo.java

@@ -16,10 +16,8 @@
 package com.alibaba.csp.sentinel.dashboard.domain.vo;
 
 import java.util.ArrayList;
-import java.util.Date;
 import java.util.List;
 
-import com.alibaba.csp.sentinel.dashboard.discovery.MachineDiscovery;
 import com.alibaba.csp.sentinel.dashboard.discovery.MachineInfo;
 
 /**
@@ -30,9 +28,10 @@ public class MachineInfoVo {
     private String app;
     private String hostname;
     private String ip;
-    private Integer port;
-    private Date timestamp;
-    private boolean health;
+    private int port;
+    private long heartbeatVersion;
+    private long lastHeartbeat;
+    private boolean healthy;
 
     private String version;
 
@@ -50,11 +49,10 @@ public class MachineInfoVo {
         vo.setHostname(machine.getHostname());
         vo.setIp(machine.getIp());
         vo.setPort(machine.getPort());
-        vo.setTimestamp(machine.getTimestamp());
+        vo.setLastHeartbeat(machine.getLastHeatbeat());
+        vo.setHeartbeatVersion(machine.getHeartbeatVersion());
         vo.setVersion(machine.getVersion());
-        if (System.currentTimeMillis() - machine.getTimestamp().getTime() < MachineDiscovery.MAX_CLIENT_LIVE_TIME_MS) {
-            vo.setHealth(true);
-        }
+        vo.setHealthy(machine.isHealthy());
         return vo;
     }
 
@@ -82,20 +80,28 @@ public class MachineInfoVo {
         this.ip = ip;
     }
 
-    public Integer getPort() {
+    public int getPort() {
         return port;
     }
 
-    public void setPort(Integer port) {
+    public void setPort(int port) {
         this.port = port;
     }
 
-    public Date getTimestamp() {
-        return timestamp;
+    public long getLastHeartbeat() {
+        return lastHeartbeat;
     }
-
-    public void setTimestamp(Date timestamp) {
-        this.timestamp = timestamp;
+    
+    public void setLastHeartbeat(long lastHeartbeat) {
+        this.lastHeartbeat = lastHeartbeat;
+    }
+    
+    public void setHeartbeatVersion(long heartbeatVersion) {
+        this.heartbeatVersion = heartbeatVersion;
+    }
+    
+    public long getHeartbeatVersion() {
+        return heartbeatVersion;
     }
 
     public String getVersion() {
@@ -107,11 +113,11 @@ public class MachineInfoVo {
         return this;
     }
 
-    public boolean isHealth() {
-        return health;
+    public boolean isHealthy() {
+        return healthy;
     }
 
-    public void setHealth(boolean health) {
-        this.health = health;
+    public void setHealthy(boolean healthy) {
+        this.healthy = healthy;
     }
 }

sentinel-dashboard/src/main/java/com/alibaba/csp/sentinel/dashboard/metric/MetricFetcher.java

@@ -37,13 +37,13 @@ import java.util.concurrent.atomic.AtomicLong;
 import com.alibaba.csp.sentinel.concurrent.NamedThreadFactory;
 import com.alibaba.csp.sentinel.config.SentinelConfig;
 import com.alibaba.csp.sentinel.dashboard.datasource.entity.MetricEntity;
+import com.alibaba.csp.sentinel.dashboard.discovery.AppInfo;
 import com.alibaba.csp.sentinel.dashboard.discovery.AppManagement;
 import com.alibaba.csp.sentinel.dashboard.discovery.MachineInfo;
 import com.alibaba.csp.sentinel.node.metric.MetricNode;
 import com.alibaba.csp.sentinel.util.StringUtil;
 
 import com.alibaba.csp.sentinel.dashboard.repository.metric.MetricsRepository;
-import com.alibaba.csp.sentinel.dashboard.util.MachineUtils;
 import org.apache.http.HttpResponse;
 import org.apache.http.client.methods.HttpGet;
 import org.apache.http.concurrent.FutureCallback;
@@ -168,14 +168,21 @@ public class MetricFetcher {
         if (maxWaitSeconds <= 0) {
             throw new IllegalArgumentException("maxWaitSeconds must > 0, but " + maxWaitSeconds);
         }
-        Set<MachineInfo> machines = appManagement.getDetailApp(app).getMachines();
+        AppInfo appInfo = appManagement.getDetailApp(app);
+        // auto remove for app
+        if (appInfo.isDead()) {
+            logger.info("Dead app removed: {}", app);
+            appManagement.removeApp(app);
+            return;
+        }
+        Set<MachineInfo> machines = appInfo.getMachines();
         logger.debug("enter fetchOnce(" + app + "), machines.size()=" + machines.size()
             + ", time intervalMs [" + startTime + ", " + endTime + "]");
         if (machines.isEmpty()) {
             return;
         }
         final String msg = "fetch";
-        AtomicLong dead = new AtomicLong();
+        AtomicLong unhealthy = new AtomicLong();
         final AtomicLong success = new AtomicLong();
         final AtomicLong fail = new AtomicLong();
 
@@ -184,10 +191,15 @@ public class MetricFetcher {
         final Map<String, MetricEntity> metricMap = new ConcurrentHashMap<>(16);
         final CountDownLatch latch = new CountDownLatch(machines.size());
         for (final MachineInfo machine : machines) {
-            // dead
-            if (System.currentTimeMillis() - machine.getTimestamp().getTime() > MachineUtils.getMaxClientTimeout()) {
+            // auto remove
+            if (machine.isDead()) {
+                appManagement.getDetailApp(app).removeMachine(machine.getIp(), machine.getPort());
+                logger.info("Dead machine removed: {}:{} of {}", machine.getIp(), machine.getPort(), app);
+                continue;
+            }
+            if (!machine.isHealthy()) {
                 latch.countDown();
-                dead.incrementAndGet();
+                unhealthy.incrementAndGet();
                 continue;
             }
             final String url = "http://" + machine.getIp() + ":" + machine.getPort() + "/" + METRIC_URL_PATH

sentinel-dashboard/src/main/java/com/alibaba/csp/sentinel/dashboard/rule/FlowRuleApiProvider.java

@@ -22,7 +22,6 @@ import java.util.stream.Collectors;
 import com.alibaba.csp.sentinel.dashboard.client.SentinelApiClient;
 import com.alibaba.csp.sentinel.dashboard.discovery.AppManagement;
 import com.alibaba.csp.sentinel.dashboard.discovery.MachineInfo;
-import com.alibaba.csp.sentinel.dashboard.util.MachineUtils;
 import com.alibaba.csp.sentinel.util.StringUtil;
 
 import com.alibaba.csp.sentinel.dashboard.datasource.entity.rule.FlowRuleEntity;
@@ -48,11 +47,11 @@ public class FlowRuleApiProvider implements DynamicRuleProvider<List<FlowRuleEnt
         }
         List<MachineInfo> list = appManagement.getDetailApp(appName).getMachines()
             .stream()
-            .filter(MachineUtils::isMachineHealth)
+            .filter(e -> e.isHealthy())
             .sorted((e1, e2) -> {
-                if (e1.getTimestamp().before(e2.getTimestamp())) {
+                if (e1.getLastHeatbeat() < e2.getLastHeatbeat()) {
                     return 1;
-                } else if (e1.getTimestamp().after(e2.getTimestamp())) {
+                } else if (e1.getLastHeatbeat() > e2.getLastHeatbeat()) {
                     return -1;
                 } else {
                     return 0;

sentinel-dashboard/src/main/java/com/alibaba/csp/sentinel/dashboard/rule/FlowRuleApiPublisher.java

@@ -24,7 +24,6 @@ import com.alibaba.csp.sentinel.dashboard.discovery.MachineInfo;
 import com.alibaba.csp.sentinel.util.StringUtil;
 
 import com.alibaba.csp.sentinel.dashboard.datasource.entity.rule.FlowRuleEntity;
-import com.alibaba.csp.sentinel.dashboard.util.MachineUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Component;
 
@@ -51,7 +50,7 @@ public class FlowRuleApiPublisher implements DynamicRulePublisher<List<FlowRuleE
         Set<MachineInfo> set = appManagement.getDetailApp(app).getMachines();
 
         for (MachineInfo machine : set) {
-            if (!MachineUtils.isMachineHealth(machine)) {
+            if (!machine.isHealthy()) {
                 continue;
             }
             // TODO: parse the results

sentinel-dashboard/src/main/java/com/alibaba/csp/sentinel/dashboard/service/ClusterConfigService.java

@@ -38,7 +38,6 @@ import com.alibaba.csp.sentinel.dashboard.domain.cluster.state.ClusterUniversalS
 import com.alibaba.csp.sentinel.dashboard.domain.cluster.config.ClusterClientConfig;
 import com.alibaba.csp.sentinel.dashboard.domain.cluster.config.ServerFlowConfig;
 import com.alibaba.csp.sentinel.dashboard.domain.cluster.config.ServerTransportConfig;
-import com.alibaba.csp.sentinel.dashboard.util.MachineUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 
@@ -111,7 +110,7 @@ public class ClusterConfigService {
         }
 
         List<CompletableFuture<ClusterUniversalStatePairVO>> futures = appInfo.getMachines().stream()
-            .filter(MachineUtils::isMachineHealth)
+            .filter(e -> e.isHealthy())
             .map(machine -> getClusterUniversalState(app, machine.getIp(), machine.getPort())
                 .thenApply(e -> new ClusterUniversalStatePairVO(machine.getIp(), machine.getPort(), e)))
             .collect(Collectors.toList());
@@ -129,7 +128,7 @@ public class ClusterConfigService {
         }
 
         boolean machineOk = appInfo.getMachines().stream()
-            .filter(MachineUtils::isMachineHealth)
+            .filter(e -> e.isHealthy())
             .map(e -> e.getIp() + '@' + e.getPort())
             .anyMatch(e -> e.equals(machineId));
         if (!machineOk) {

sentinel-dashboard/src/main/java/com/alibaba/csp/sentinel/dashboard/util/MachineUtils.java

@@ -17,7 +17,6 @@ package com.alibaba.csp.sentinel.dashboard.util;
 
 import java.util.Optional;
 
-import com.alibaba.csp.sentinel.dashboard.discovery.MachineInfo;
 import com.alibaba.csp.sentinel.util.StringUtil;
 import com.alibaba.csp.sentinel.util.function.Tuple2;
 
@@ -25,13 +24,6 @@ import com.alibaba.csp.sentinel.util.function.Tuple2;
  * @author Eric Zhao
  */
 public final class MachineUtils {
-
-    public static final long DEFAULT_MAX_CLIENT_PING_TIMEOUT = 60 * 1000;
-
-    public static long getMaxClientTimeout() {
-        return DEFAULT_MAX_CLIENT_PING_TIMEOUT;
-    }
-
     public static Optional<Integer> parseCommandPort(String machineIp) {
         try {
             if (!machineIp.contains("@")) {
@@ -61,11 +53,4 @@ public final class MachineUtils {
             return Optional.empty();
         }
     }
-
-    public static boolean isMachineHealth(MachineInfo machine) {
-        if (machine == null) {
-            return false;
-        }
-        return System.currentTimeMillis() - machine.getTimestamp().getTime() < getMaxClientTimeout();
-    }
 }

sentinel-dashboard/src/main/webapp/resources/app/scripts/controllers/machine.js

@@ -19,27 +19,47 @@ app.controller('MachineCtl', ['$scope', '$stateParams', 'MachineService',
       $scope.reverse = ($scope.propertyName === propertyName) ? !$scope.reverse : false;
       $scope.propertyName = propertyName;
     };
-
-    MachineService.getAppMachines($scope.app).success(
-      function (data) {
-        // console.log('get machines: ' + data.data[0].hostname)
-        if (data.code == 0 && data.data) {
-          $scope.machines = data.data;
-          var health = 0;
-          $scope.machines.forEach(function (item) {
-            if (item.health) {
-              health++;
-            }
-            if (!item.hostname) {
-              item.hostname = '未知'
-            }
-          })
-          $scope.healthCount = health;
-          $scope.machinesPageConfig.totalCount = $scope.machines.length;
-        } else {
-          $scope.machines = [];
-          $scope.healthCount = 0;
+    
+    $scope.reloadMachines = function() {
+      MachineService.getAppMachines($scope.app).success(
+        function (data) {
+          // console.log('get machines: ' + data.data[0].hostname)
+          if (data.code == 0 && data.data) {
+            $scope.machines = data.data;
+            var healthy = 0;
+            $scope.machines.forEach(function (item) {
+              if (item.healthy) {
+                  healthy++;
+              }
+              if (!item.hostname) {
+                item.hostname = '未知'
+              }
+            })
+            $scope.healthyCount = healthy;
+            $scope.machinesPageConfig.totalCount = $scope.machines.length;
+          } else {
+            $scope.machines = [];
+            $scope.healthyCount = 0;
+          }
         }
+      );
+    };
+    
+    $scope.removeMachine = function(ip, port) {
+      if (!confirm("confirm to remove machine [" + ip + ":" + port + "]?")) {
+        return;
       }
-    );
+      MachineService.removeAppMachine($scope.app, ip, port).success(
+        function(data) {
+          if (data.code == 0) {
+            $scope.reloadMachines();
+          } else {
+            alert("remove failed");
+          }
+        }
+      );
+    };
+    
+    $scope.reloadMachines();
+    
   }]);

sentinel-dashboard/src/main/webapp/resources/app/scripts/directives/sidebar/sidebar.html

@@ -19,7 +19,7 @@
         <a href="javascript:void(0);" ng-click="click($event)" collapse="{{collpaseall == 1}}" style="font-size: 16px;word-break: break-word;">
           &nbsp;{{entry.app}}
           <span class="fa arrow"></span>
-          <span class="arrow">({{entry.heathCount}}/{{entry.machines.length}})</span>
+          <span class="arrow">({{entry.heathyCount}}/{{entry.machines.length}})</span>
         </a>
 
         <!--<ul class="nav nav-second-level" collapse="{{entry.active}}" style="display: none;">-->

sentinel-dashboard/src/main/webapp/resources/app/scripts/directives/sidebar/sidebar.js

@@ -21,20 +21,19 @@ angular.module('sentinelDashboardApp')
           function (data) {
             if (data.code === 0) {
               let initHashApp = $location.path().split('/')[3];
-              let currTime = moment(new Date()).utc().add(-1000*60*5).format('YYYY-MM-DDTHH:mm:ss')
               $scope.apps = data.data;
               $scope.apps = $scope.apps.map(function (item) {
                 if (item.app === initHashApp) {
                   item.active = true;
                 }
-                var heathCount = 0;
+                var heathyCount = 0;
                 for (var i in item.machines) {
-                  if (item.machines[i].timestamp>currTime) {
-                    heathCount++;
+                  if (item.machines[i].healthy) {
+                      heathyCount++;
                   }
                 }
-                item.heathCount = heathCount;
-                if (heathCount>0) {
+                item.heathyCount = heathyCount;
+                if (item.shown) {
                   return item;
                 }
               });

sentinel-dashboard/src/main/webapp/resources/app/scripts/services/machineservice.js

@@ -1,10 +1,23 @@
 var app = angular.module('sentinelDashboardApp');
 
-app.service('MachineService', ['$http', function ($http) {
+app.service('MachineService', ['$http', '$httpParamSerializerJQLike', function ($http, $httpParamSerializerJQLike) {
   this.getAppMachines = function (app) {
     return $http({
       url: 'app/' + app + '/machines.json',
       method: 'GET'
     });
   };
+  this.removeAppMachine = function (app, ip, port) {
+    return $http({
+      url: 'app/' + app + '/machine/remove.json',
+      method: 'POST',
+      headers: {
+        "Content-type": 'application/x-www-form-urlencoded; charset=UTF-8'
+      },
+      data: $httpParamSerializerJQLike({
+        ip: ip,
+        port: port
+      })
+    });
+  };
 }]);

sentinel-dashboard/src/main/webapp/resources/app/views/machine.html

@@ -3,8 +3,8 @@
 <!--<h2 class="page-header">{{app}} &nbsp;&nbsp;&nbsp;</h2>-->
 <!--<div>-->
 <!--<span>实例总数 {{machines.length}}</span>,-->
-<!--<span style="color: green"> 健康 {{healthCount}}</span>,-->
-<!--<span style="color: red"> 失联 {{machines.length-healthCount}}</span>。-->
+<!--<span style="color: green"> 健康 {{healthyCount}}</span>,-->
+<!--<span style="color: red"> 失联 {{machines.length-healthyCount}}</span>。-->
 
 <!--</div>-->
 <!--</div>-->
@@ -16,7 +16,7 @@
 </div>
 
 <!--<div>-->
-<!--<span>实例总数 {{machines.length}}, 健康 {{healthCount}}, 失联 {{machines.length-healthCount}}</span>-->
+<!--<span>实例总数 {{machines.length}}, 健康 {{healthyCount}}, 失联 {{machines.length-healthyCount}}</span>-->
 <!--</div>-->
 
 <div class="separator"></div>
@@ -26,7 +26,7 @@
       <div class="card">
         <div class="inputs-header">
           <span class="brand" style="font-size: 13px;">机器列表</span>
-          <span>实例总数 {{machines.length}}, 健康 {{healthCount}}, 失联 {{machines.length-healthCount}}.</span>
+          <span>实例总数 {{machines.length}}, 健康 {{healthyCount}}, 失联 {{machines.length-healthyCount}}.</span>
           <!--<button class="btn btn-danger" style="float: right;margin-right: 10px;height: 30px;font-size: 12px;" ng-click="addNewApp()">全部禁用</button>-->
           <!--<button class="btn btn-danger" style="float: right; margin-right: 10px; height: 30px;font-size: 12px;margin-left: 10px;" ng-click="addNewApp()">全部启用</button>-->
           <input class="form-control witdh-300" placeholder="关键字" ng-model="searchKey">
@@ -43,6 +43,7 @@
                 <td>Sentinel 客户端版本</td>
                 <td>健康状态</td>
                 <td>心跳时间</td>
+                <td>操作</td>
               </tr>
             </thead>
             <tbody>
@@ -52,10 +53,12 @@
                 <td style="word-wrap:break-word;word-break:break-all;">{{entry.ip}}</td>
                 <td> {{entry.port}} </td>
                 <td> {{entry.version}} </td>
-                <td ng-if="entry.health">健康</td>
-                <td ng-if="!entry.health" style="color: red">失联</td>
-                <td>{{entry.timestamp | date: 'yyyy/MM/dd HH:mm:ss'}}</td>
-                <!--<td ng-if="!entry.health" style="color: grey">{{entry.timestamp | date: 'yyyy/MM/dd HH:mm:ss'}}</td>-->
+                <td ng-if="entry.healthy">健康</td>
+                <td ng-if="!entry.healthy" style="color: red">失联</td>
+                <td>{{entry.lastHeartbeat | date: 'yyyy/MM/dd HH:mm:ss'}}</td>
+                <td>
+                    <button ng-if="!entry.healthy" class="btn btn-xs btn-default" style="height: 25px; font-size: 12px;" ng-click="removeMachine(entry.ip, entry.port)">REMOVE</button>
+                </td>
               </tr>
             </tbody>
           </table>

sentinel-dashboard/src/test/java/com/alibaba/csp/sentinel/dashboard/config/DashboardConfigTest.java

@@ -0,0 +1,80 @@
+/*
+ * Copyright 1999-2018 Alibaba Group Holding Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alibaba.csp.sentinel.dashboard.config;
+
+import static org.junit.Assert.assertEquals;
+
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.contrib.java.lang.system.EnvironmentVariables;
+
+public class DashboardConfigTest {
+    @Rule
+    public final EnvironmentVariables environmentVariables = new EnvironmentVariables();
+    
+    @Test
+    public void testGetConfigInt() {
+        // skip cache
+        
+        // default value
+        assertEquals(0, DashboardConfig.getConfigInt("t", 0, 10));
+        DashboardConfig.clearCache();
+        assertEquals(1, DashboardConfig.getConfigInt("t", 1, 10));
+        
+        // property, wrong format
+        System.setProperty("t", "asdf");
+        DashboardConfig.clearCache();
+        assertEquals(0, DashboardConfig.getConfigInt("t", 0, 10));
+        System.setProperty("t", "");
+        DashboardConfig.clearCache();
+        assertEquals(0, DashboardConfig.getConfigInt("t", 0, 10));
+        
+        // min value
+        System.setProperty("t", "2");
+        DashboardConfig.clearCache();
+        assertEquals(2, DashboardConfig.getConfigInt("t", 0, 1));
+        DashboardConfig.clearCache();
+        assertEquals(10, DashboardConfig.getConfigInt("t", 0, 10));
+        DashboardConfig.clearCache();
+        assertEquals(2, DashboardConfig.getConfigInt("t", 0, -1));
+        
+        // env
+        environmentVariables.set("t", "20");
+        DashboardConfig.clearCache();
+        assertEquals(20, DashboardConfig.getConfigInt("t", 0, 10));
+        
+        // wrong format env var, but it will override property
+        environmentVariables.set("t", "20dddd");
+        DashboardConfig.clearCache();
+        assertEquals(0, DashboardConfig.getConfigInt("t", 0, 10));
+        
+        // clear env, it will take property
+        environmentVariables.set("t", "");
+        DashboardConfig.clearCache();
+        assertEquals(10, DashboardConfig.getConfigInt("t", 0, 10));
+        DashboardConfig.clearCache();
+        assertEquals(2, DashboardConfig.getConfigInt("t", 0, 1));
+        
+        // enable cache
+        System.setProperty("t", "666");
+        DashboardConfig.clearCache();
+        assertEquals(666, DashboardConfig.getConfigInt("t", 0, 1));
+        System.setProperty("t", "777");
+        assertEquals(666, DashboardConfig.getConfigInt("t", 0, 1));
+        System.setProperty("t", "555");
+        assertEquals(666, DashboardConfig.getConfigInt("t", 0, 1));
+    }
+}

sentinel-dashboard/src/test/java/com/alibaba/csp/sentinel/dashboard/discovery/AppInfoTest.java

@@ -20,6 +20,8 @@ import java.util.Set;
 
 import org.junit.Test;
 
+import com.alibaba.csp.sentinel.dashboard.config.DashboardConfig;
+
 import static org.junit.Assert.*;
 
 public class AppInfoTest {
@@ -65,4 +67,91 @@ public class AppInfoTest {
         return machine;
     }
 
-}
+    @Test
+    public void addRemoveMachineTest() {
+        AppInfo appInfo = new AppInfo("default");
+        assertEquals("default", appInfo.getApp());
+        assertEquals(0, appInfo.getMachines().size());
+        //add one
+        {
+            MachineInfo machineInfo = new MachineInfo();
+            machineInfo.setApp("default");
+            machineInfo.setHostname("bogon");
+            machineInfo.setIp("127.0.0.1");
+            machineInfo.setPort(3389);
+            machineInfo.setLastHeatbeat(System.currentTimeMillis());
+            machineInfo.setHeartbeatVersion(1);
+            machineInfo.setVersion("0.4.1");
+            appInfo.addMachine(machineInfo);
+        }
+        assertEquals(1, appInfo.getMachines().size());
+        //add duplicated one
+        {
+            MachineInfo machineInfo = new MachineInfo();
+            machineInfo.setApp("default");
+            machineInfo.setHostname("bogon");
+            machineInfo.setIp("127.0.0.1");
+            machineInfo.setPort(3389);
+            machineInfo.setLastHeatbeat(System.currentTimeMillis());
+            machineInfo.setHeartbeatVersion(1);
+            machineInfo.setVersion("0.4.2");
+            appInfo.addMachine(machineInfo);
+        }
+        assertEquals(1, appInfo.getMachines().size());
+        //add different one
+        {
+            MachineInfo machineInfo = new MachineInfo();
+            machineInfo.setApp("default");
+            machineInfo.setHostname("bogon");
+            machineInfo.setIp("127.0.0.1");
+            machineInfo.setPort(3390);
+            machineInfo.setLastHeatbeat(System.currentTimeMillis());
+            machineInfo.setHeartbeatVersion(1);
+            machineInfo.setVersion("0.4.3");
+            appInfo.addMachine(machineInfo);
+        }
+        assertEquals(2, appInfo.getMachines().size());
+        appInfo.removeMachine("127.0.0.1", 3389);
+        assertEquals(1, appInfo.getMachines().size());
+        appInfo.removeMachine("127.0.0.1", 3390);
+        assertEquals(0, appInfo.getMachines().size());
+    }
+    
+    @Test
+    public void testHealthyAndDead() {
+        System.setProperty(DashboardConfig.CONFIG_HIDE_APP_NO_MACHINE_MILLIS, "60000");
+        System.setProperty(DashboardConfig.CONFIG_REMOVE_APP_NO_MACHINE_MILLIS, "600000");
+        DashboardConfig.clearCache();
+        String appName = "default";
+        AppInfo appInfo = new AppInfo();
+        appInfo.setApp(appName);
+        {
+            MachineInfo machineInfo = MachineInfo.of(appName, "127.0.0.1", 8801);
+            machineInfo.setHeartbeatVersion(1);
+            machineInfo.setLastHeatbeat(System.currentTimeMillis());
+            appInfo.addMachine(machineInfo);
+        }
+        assertTrue(appInfo.isShown());
+        assertFalse(appInfo.isDead());
+        
+        {
+            MachineInfo machineInfo = MachineInfo.of(appName, "127.0.0.1", 8801);
+            machineInfo.setHeartbeatVersion(1);
+            machineInfo.setLastHeatbeat(System.currentTimeMillis() - 70000);
+            appInfo.addMachine(machineInfo);
+        }
+        assertFalse(appInfo.isShown());
+        assertFalse(appInfo.isDead());
+        
+        {
+            MachineInfo machineInfo = MachineInfo.of(appName, "127.0.0.1", 8801);
+            machineInfo.setHeartbeatVersion(1);
+            machineInfo.setLastHeatbeat(System.currentTimeMillis() - 700000);
+            appInfo.addMachine(machineInfo);
+        }
+        assertFalse(appInfo.isShown());
+        assertTrue(appInfo.isDead());
+    }
+
+}
+

sentinel-dashboard/src/test/java/com/alibaba/csp/sentinel/dashboard/discovery/MachineInfoTest.java

@@ -0,0 +1,44 @@
+/*
+ * Copyright 1999-2018 Alibaba Group Holding Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alibaba.csp.sentinel.dashboard.discovery;
+
+import static org.junit.Assert.*;
+
+import org.junit.Test;
+
+import com.alibaba.csp.sentinel.dashboard.config.DashboardConfig;
+
+public class MachineInfoTest {
+    @Test
+    public void testHealthyAndDead() {
+        System.setProperty(DashboardConfig.CONFIG_UNHEALTHY_MACHINE_MILLIS, "60000");
+        System.setProperty(DashboardConfig.CONFIG_AUTO_REMOVE_MACHINE_MILLIS, "600000");
+        DashboardConfig.clearCache();
+        MachineInfo machineInfo = new MachineInfo();
+        machineInfo.setHeartbeatVersion(1);
+        machineInfo.setLastHeatbeat(System.currentTimeMillis() - 10000);
+        assertTrue(machineInfo.isHealthy());
+        assertFalse(machineInfo.isDead());
+        
+        machineInfo.setLastHeatbeat(System.currentTimeMillis() - 100000);
+        assertFalse(machineInfo.isHealthy());
+        assertFalse(machineInfo.isDead());
+        
+        machineInfo.setLastHeatbeat(System.currentTimeMillis() - 1000000);
+        assertFalse(machineInfo.isHealthy());
+        assertTrue(machineInfo.isDead());
+    }
+}