- The BBR period for maxCpuUsage strategy is deprecated. Now Sentinel will just check the CPU usage value, which could avert surge of CPU usage quickly - Improve SystemRuleManager and SystemStatusListener - Add some test cases Signed-off-by: Eric Zhao <sczyh16@gmail.com>master
@@ -63,7 +63,7 @@ import com.alibaba.csp.sentinel.slots.block.BlockException; | |||
* @author jialiang.linjl | |||
* @author leyou | |||
*/ | |||
public class SystemRuleManager { | |||
public final class SystemRuleManager { | |||
private static volatile double highestSystemLoad = Double.MAX_VALUE; | |||
/** | |||
@@ -95,7 +95,7 @@ public class SystemRuleManager { | |||
static { | |||
checkSystemStatus.set(false); | |||
statusListener = new SystemStatusListener(); | |||
scheduler.scheduleAtFixedRate(statusListener, 5, 1, TimeUnit.SECONDS); | |||
scheduler.scheduleAtFixedRate(statusListener, 0, 1, TimeUnit.SECONDS); | |||
currentProperty.addListener(listener); | |||
} | |||
@@ -107,6 +107,7 @@ public class SystemRuleManager { | |||
*/ | |||
public static void register2Property(SentinelProperty<List<SystemRule>> property) { | |||
synchronized (listener) { | |||
RecordLog.info("[SystemRuleManager] Registering new property to system rule manager"); | |||
currentProperty.removeListener(listener); | |||
property.addListener(listener); | |||
currentProperty = property; | |||
@@ -167,26 +168,22 @@ public class SystemRuleManager { | |||
return result; | |||
} | |||
public static double getQps() { | |||
public static double getInboundQpsThreshold() { | |||
return qps; | |||
} | |||
public static void setQps(double qps) { | |||
SystemRuleManager.qps = qps; | |||
} | |||
public static long getMaxRt() { | |||
public static long getRtThreshold() { | |||
return maxRt; | |||
} | |||
public static long getMaxThread() { | |||
public static long getMaxThreadThreshold() { | |||
return maxThread; | |||
} | |||
static class SystemPropertyListener extends SimplePropertyListener<List<SystemRule>> { | |||
@Override | |||
public void configUpdate(List<SystemRule> rules) { | |||
public synchronized void configUpdate(List<SystemRule> rules) { | |||
restoreSetting(); | |||
// systemRules = rules; | |||
if (rules != null && rules.size() >= 1) { | |||
@@ -234,14 +231,10 @@ public class SystemRuleManager { | |||
return checkSystemStatus.get(); | |||
} | |||
public static double getHighestSystemLoad() { | |||
public static double getSystemLoadThreshold() { | |||
return highestSystemLoad; | |||
} | |||
public static void setHighestSystemLoad(double highestSystemLoad) { | |||
SystemRuleManager.highestSystemLoad = highestSystemLoad; | |||
} | |||
public static double getCpuUsageThreshold() { | |||
return highestCpuUsage; | |||
} | |||
@@ -257,9 +250,14 @@ public class SystemRuleManager { | |||
} | |||
if (rule.getHighestCpuUsage() >= 0) { | |||
highestCpuUsage = Math.min(highestCpuUsage, rule.getHighestCpuUsage()); | |||
highestCpuUsageIsSet = true; | |||
checkStatus = true; | |||
if (rule.getHighestCpuUsage() > 1) { | |||
RecordLog.warn(String.format("[SystemRuleManager] Ignoring invalid SystemRule: " | |||
+ "highestCpuUsage %.3f > 1", rule.getHighestCpuUsage())); | |||
} else { | |||
highestCpuUsage = Math.min(highestCpuUsage, rule.getHighestCpuUsage()); | |||
highestCpuUsageIsSet = true; | |||
checkStatus = true; | |||
} | |||
} | |||
if (rule.getAvgRt() >= 0) { | |||
@@ -290,6 +288,9 @@ public class SystemRuleManager { | |||
* @throws BlockException when any system rule's threshold is exceeded. | |||
*/ | |||
public static void checkSystem(ResourceWrapper resourceWrapper) throws BlockException { | |||
if (resourceWrapper == null) { | |||
return; | |||
} | |||
// Ensure the checking switch is on. | |||
if (!checkSystemStatus.get()) { | |||
return; | |||
@@ -326,9 +327,7 @@ public class SystemRuleManager { | |||
// cpu usage | |||
if (highestCpuUsageIsSet && getCurrentCpuUsage() > highestCpuUsage) { | |||
if (!checkBbr(currentThread)) { | |||
throw new SystemBlockException(resourceWrapper.getName(), "cpu"); | |||
} | |||
throw new SystemBlockException(resourceWrapper.getName(), "cpu"); | |||
} | |||
} | |||
@@ -347,4 +346,4 @@ public class SystemRuleManager { | |||
public static double getCurrentCpuUsage() { | |||
return statusListener.getCpuUsage(); | |||
} | |||
} | |||
} |
@@ -46,32 +46,35 @@ public class SystemStatusListener implements Runnable { | |||
try { | |||
OperatingSystemMXBean osBean = ManagementFactory.getPlatformMXBean(OperatingSystemMXBean.class); | |||
currentLoad = osBean.getSystemLoadAverage(); | |||
/** | |||
/* | |||
* Java Doc copied from {@link OperatingSystemMXBean#getSystemCpuLoad()}:</br> | |||
* Returns the "recent cpu usage" for the whole system. This value is a double in the [0.0,1.0] interval. | |||
* A value of 0.0 means that all CPUs were idle during the recent period of time observed, while a value | |||
* of 1.0 means that all CPUs were actively running 100% of the time during the recent period being | |||
* observed. All values betweens 0.0 and 1.0 are possible depending of the activities going on in the | |||
* observed. All values between 0.0 and 1.0 are possible depending of the activities going on in the | |||
* system. If the system recent cpu usage is not available, the method returns a negative value. | |||
*/ | |||
currentCpuUsage = osBean.getSystemCpuLoad(); | |||
StringBuilder sb = new StringBuilder(); | |||
if (currentLoad > SystemRuleManager.getHighestSystemLoad()) { | |||
sb.append("load:").append(currentLoad).append(";"); | |||
sb.append("cpu:").append(currentCpuUsage).append(";"); | |||
sb.append("qps:").append(Constants.ENTRY_NODE.passQps()).append(";"); | |||
sb.append("rt:").append(Constants.ENTRY_NODE.avgRt()).append(";"); | |||
sb.append("thread:").append(Constants.ENTRY_NODE.curThreadNum()).append(";"); | |||
sb.append("success:").append(Constants.ENTRY_NODE.successQps()).append(";"); | |||
sb.append("minRt:").append(Constants.ENTRY_NODE.minRt()).append(";"); | |||
sb.append("maxSuccess:").append(Constants.ENTRY_NODE.maxSuccessQps()).append(";"); | |||
RecordLog.info(sb.toString()); | |||
if (currentLoad > SystemRuleManager.getSystemLoadThreshold()) { | |||
writeSystemStatusLog(); | |||
} | |||
} catch (Throwable e) { | |||
RecordLog.info("could not get system error ", e); | |||
RecordLog.warn("[SystemStatusListener] Failed to get system metrics from JMX", e); | |||
} | |||
} | |||
private void writeSystemStatusLog() { | |||
StringBuilder sb = new StringBuilder(); | |||
sb.append("Load exceeds the threshold: "); | |||
sb.append("load:").append(String.format("%.4f", currentLoad)).append("; "); | |||
sb.append("cpuUsage:").append(String.format("%.4f", currentCpuUsage)).append("; "); | |||
sb.append("qps:").append(String.format("%.4f", Constants.ENTRY_NODE.passQps())).append("; "); | |||
sb.append("rt:").append(String.format("%.4f", Constants.ENTRY_NODE.avgRt())).append("; "); | |||
sb.append("thread:").append(Constants.ENTRY_NODE.curThreadNum()).append("; "); | |||
sb.append("success:").append(String.format("%.4f", Constants.ENTRY_NODE.successQps())).append("; "); | |||
sb.append("minRt:").append(String.format("%.2f", Constants.ENTRY_NODE.minRt())).append("; "); | |||
sb.append("maxSuccess:").append(String.format("%.2f", Constants.ENTRY_NODE.maxSuccessQps())).append("; "); | |||
RecordLog.info(sb.toString()); | |||
} | |||
} |
@@ -0,0 +1,111 @@ | |||
/* | |||
* Copyright 1999-2018 Alibaba Group Holding Ltd. | |||
* | |||
* Licensed under the Apache License, Version 2.0 (the "License"); | |||
* you may not use this file except in compliance with the License. | |||
* You may obtain a copy of the License at | |||
* | |||
* http://www.apache.org/licenses/LICENSE-2.0 | |||
* | |||
* Unless required by applicable law or agreed to in writing, software | |||
* distributed under the License is distributed on an "AS IS" BASIS, | |||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
* See the License for the specific language governing permissions and | |||
* limitations under the License. | |||
*/ | |||
package com.alibaba.csp.sentinel.slots.system; | |||
import java.util.ArrayList; | |||
import java.util.Arrays; | |||
import java.util.Collections; | |||
import java.util.List; | |||
import com.alibaba.csp.sentinel.EntryType; | |||
import com.alibaba.csp.sentinel.slotchain.StringResourceWrapper; | |||
import com.alibaba.csp.sentinel.slots.block.BlockException; | |||
import org.junit.After; | |||
import org.junit.Before; | |||
import org.junit.Test; | |||
import static org.junit.Assert.*; | |||
/** | |||
* @author Eric Zhao | |||
*/ | |||
public class SystemRuleManagerTest { | |||
@Test | |||
public void testLoadInvalidRules() { | |||
SystemRule rule1 = new SystemRule(); | |||
rule1.setHighestSystemLoad(-0.9d); | |||
SystemRule rule2 = new SystemRule(); | |||
rule2.setHighestCpuUsage(2.7d); | |||
SystemRuleManager.loadRules(Arrays.asList(rule1, rule2)); | |||
assertEquals(0, SystemRuleManager.getRules().size()); | |||
} | |||
@Test | |||
public void testLoadAndGetRules() { | |||
SystemRule rule1 = new SystemRule(); | |||
rule1.setHighestSystemLoad(1.2d); | |||
SystemRule rule2 = new SystemRule(); | |||
rule2.setMaxThread(17); | |||
SystemRule rule3 = new SystemRule(); | |||
rule3.setHighestCpuUsage(0.7d); | |||
SystemRule rule4 = new SystemRule(); | |||
rule4.setQps(1500); | |||
SystemRule rule5 = new SystemRule(); | |||
rule5.setAvgRt(50); | |||
SystemRuleManager.loadRules(Arrays.asList(rule1, rule2, rule3, rule4, rule5)); | |||
assertEquals(1.2d, SystemRuleManager.getSystemLoadThreshold(), 0.01); | |||
assertEquals(17, SystemRuleManager.getMaxThreadThreshold()); | |||
assertEquals(0.7d, SystemRuleManager.getCpuUsageThreshold(), 0.01); | |||
assertEquals(1500, SystemRuleManager.getInboundQpsThreshold(), 0.01); | |||
assertEquals(50, SystemRuleManager.getRtThreshold()); | |||
} | |||
@Test | |||
public void testLoadDuplicateTypeOfRules() { | |||
SystemRule rule1 = new SystemRule(); | |||
rule1.setHighestSystemLoad(1.2d); | |||
SystemRule rule2 = new SystemRule(); | |||
rule2.setHighestSystemLoad(2.3d); | |||
SystemRule rule3 = new SystemRule(); | |||
rule3.setHighestSystemLoad(3.4d); | |||
SystemRuleManager.loadRules(Arrays.asList(rule1, rule2, rule3)); | |||
List<SystemRule> rules = SystemRuleManager.getRules(); | |||
assertEquals(1, rules.size()); | |||
assertEquals(1.2d, rules.get(0).getHighestSystemLoad(), 0.01); | |||
assertEquals(1.2d, SystemRuleManager.getSystemLoadThreshold(), 0.01); | |||
} | |||
@Test | |||
public void testCheckMaxCpuUsageNotBBR() throws Exception { | |||
SystemRule rule1 = new SystemRule(); | |||
rule1.setHighestCpuUsage(0d); | |||
SystemRuleManager.loadRules(Collections.singletonList(rule1)); | |||
// Wait until SystemStatusListener triggered the first CPU usage collecting. | |||
Thread.sleep(1500); | |||
boolean blocked = false; | |||
try { | |||
SystemRuleManager.checkSystem(new StringResourceWrapper("testCheckMaxCpuUsageNotBBR", EntryType.IN)); | |||
} catch (BlockException ex) { | |||
blocked = true; | |||
} | |||
assertTrue("The entry should be blocked under SystemRule maxCpuUsage=0", blocked); | |||
} | |||
@Before | |||
public void setUp() throws Exception { | |||
SystemRuleManager.loadRules(new ArrayList<SystemRule>()); | |||
} | |||
@After | |||
public void tearDown() throws Exception { | |||
SystemRuleManager.loadRules(new ArrayList<SystemRule>()); | |||
} | |||
} |