From 9ae079c152a4784828f3c09a654adc13377a8f19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=8E=E8=B1=AA?= Date: Thu, 11 Oct 2018 16:39:38 +0800 Subject: [PATCH] Support degrade by exception count (#174) * Add a new DegradeRule type, degrade by exception count in the last 60 seconds * Add demo about degrading by exception count --- .../sentinel/slots/block/RuleConstant.java | 11 +- .../slots/block/degrade/DegradeRule.java | 7 +- .../slots/block/degrade/DegradeTest.java | 32 +++- .../degrade/ExceptionCountDegradeDemo.java | 169 ++++++++++++++++++ .../degrade/ExceptionRatioDegradeDemo.java | 11 +- .../sentinel/demo/degrade/RtDegradeDemo.java | 7 +- 6 files changed, 228 insertions(+), 9 deletions(-) create mode 100644 sentinel-demo/sentinel-demo-basic/src/main/java/com/alibaba/csp/sentinel/demo/degrade/ExceptionCountDegradeDemo.java diff --git a/sentinel-core/src/main/java/com/alibaba/csp/sentinel/slots/block/RuleConstant.java b/sentinel-core/src/main/java/com/alibaba/csp/sentinel/slots/block/RuleConstant.java index 4ff26110..51af8871 100755 --- a/sentinel-core/src/main/java/com/alibaba/csp/sentinel/slots/block/RuleConstant.java +++ b/sentinel-core/src/main/java/com/alibaba/csp/sentinel/slots/block/RuleConstant.java @@ -15,6 +15,8 @@ */ package com.alibaba.csp.sentinel.slots.block; +import com.alibaba.csp.sentinel.node.IntervalProperty; + /*** * @author youji.zj * @author jialiang.linjl @@ -25,7 +27,14 @@ public final class RuleConstant { public static final int FLOW_GRADE_QPS = 1; public static final int DEGRADE_GRADE_RT = 0; - public static final int DEGRADE_GRADE_EXCEPTION = 1; + /** + * Degrade by biz exception ratio in the current {@link IntervalProperty#INTERVAL} second(s). + */ + public static final int DEGRADE_GRADE_EXCEPTION_RATIO = 1; + /** + * Degrade by biz exception count in the last 60 seconds. + */ + public static final int DEGRADE_GRADE_EXCEPTION_COUNT = 2; public static final int AUTHORITY_WHITE = 0; public static final int AUTHORITY_BLACK = 1; diff --git a/sentinel-core/src/main/java/com/alibaba/csp/sentinel/slots/block/degrade/DegradeRule.java b/sentinel-core/src/main/java/com/alibaba/csp/sentinel/slots/block/degrade/DegradeRule.java index 4f60634a..571e15c6 100755 --- a/sentinel-core/src/main/java/com/alibaba/csp/sentinel/slots/block/degrade/DegradeRule.java +++ b/sentinel-core/src/main/java/com/alibaba/csp/sentinel/slots/block/degrade/DegradeRule.java @@ -173,7 +173,7 @@ public class DegradeRule extends AbstractRule { if (passCount.incrementAndGet() < RT_MAX_EXCEED_N) { return true; } - } else { + } else if (grade == RuleConstant.DEGRADE_GRADE_EXCEPTION_RATIO) { double exception = clusterNode.exceptionQps(); double success = clusterNode.successQps(); long total = clusterNode.totalQps(); @@ -190,6 +190,11 @@ public class DegradeRule extends AbstractRule { if (exception / success < count) { return true; } + } else if (grade == RuleConstant.DEGRADE_GRADE_EXCEPTION_COUNT) { + double exception = clusterNode.totalException(); + if (exception < count) { + return true; + } } synchronized (lock) { diff --git a/sentinel-core/src/test/java/com/alibaba/csp/sentinel/slots/block/degrade/DegradeTest.java b/sentinel-core/src/test/java/com/alibaba/csp/sentinel/slots/block/degrade/DegradeTest.java index 27400f1b..76346f5d 100755 --- a/sentinel-core/src/test/java/com/alibaba/csp/sentinel/slots/block/degrade/DegradeTest.java +++ b/sentinel-core/src/test/java/com/alibaba/csp/sentinel/slots/block/degrade/DegradeTest.java @@ -82,7 +82,7 @@ public class DegradeTest { rule.setCount(0.15); rule.setResource(key); rule.setTimeWindow(5); - rule.setGrade(RuleConstant.DEGRADE_GRADE_EXCEPTION); + rule.setGrade(RuleConstant.DEGRADE_GRADE_EXCEPTION_RATIO); when(cn.successQps()).thenReturn(8L); @@ -97,4 +97,34 @@ public class DegradeTest { assertTrue(rule.passCheck(context, node, 1)); } + @Test + public void testExceptionCountModeDegrade() throws Throwable { + String key = "test_degrade_exception_count"; + ClusterNode cn = mock(ClusterNode.class); + when(cn.totalException()).thenReturn(10L); + ClusterBuilderSlot.getClusterNodeMap().put(new StringResourceWrapper(key, EntryType.IN), cn); + + Context context = mock(Context.class); + DefaultNode node = mock(DefaultNode.class); + when(node.getClusterNode()).thenReturn(cn); + + DegradeRule rule = new DegradeRule(); + rule.setCount(4); + rule.setResource(key); + rule.setTimeWindow(2); + rule.setGrade(RuleConstant.DEGRADE_GRADE_EXCEPTION_COUNT); + + when(cn.totalException()).thenReturn(4L); + + // Will fail. + assertFalse(rule.passCheck(context, node, 1)); + + // Restore from the degrade timeout. + TimeUnit.SECONDS.sleep(3); + + when(cn.totalException()).thenReturn(0L); + // Will pass. + assertTrue(rule.passCheck(context, node, 1)); + } + } diff --git a/sentinel-demo/sentinel-demo-basic/src/main/java/com/alibaba/csp/sentinel/demo/degrade/ExceptionCountDegradeDemo.java b/sentinel-demo/sentinel-demo-basic/src/main/java/com/alibaba/csp/sentinel/demo/degrade/ExceptionCountDegradeDemo.java new file mode 100644 index 00000000..d0512746 --- /dev/null +++ b/sentinel-demo/sentinel-demo-basic/src/main/java/com/alibaba/csp/sentinel/demo/degrade/ExceptionCountDegradeDemo.java @@ -0,0 +1,169 @@ +package com.alibaba.csp.sentinel.demo.degrade; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +import com.alibaba.csp.sentinel.Entry; +import com.alibaba.csp.sentinel.SphU; +import com.alibaba.csp.sentinel.Tracer; +import com.alibaba.csp.sentinel.slots.block.BlockException; +import com.alibaba.csp.sentinel.slots.block.RuleConstant; +import com.alibaba.csp.sentinel.slots.block.degrade.DegradeRule; +import com.alibaba.csp.sentinel.slots.block.degrade.DegradeRuleManager; +import com.alibaba.csp.sentinel.util.TimeUtil; + +/** + *

+ * Degrade is used when the resources are in an unstable state, these resources + * will be degraded within the next defined time window. There are three ways to + * measure whether a resource is stable or not: + *

+ *

+ *

+ * Note: When degrading by {@link RuleConstant#DEGRADE_GRADE_EXCEPTION_COUNT}, time window + * less than 60 seconds will not work as expected. Because the exception count is + * summed by minute, when a short time window elapsed, the degradation condition + * may still be satisfied. + *

+ * + * @author Carpenter Lee + */ +public class ExceptionCountDegradeDemo { + private static final String KEY = "abc"; + + private static AtomicInteger total = new AtomicInteger(); + private static AtomicInteger pass = new AtomicInteger(); + private static AtomicInteger block = new AtomicInteger(); + private static AtomicInteger bizException = new AtomicInteger(); + + private static volatile boolean stop = false; + private static final int threadCount = 1; + private static int seconds = 60 + 40; + + public static void main(String[] args) throws Exception { + tick(); + initDegradeRule(); + + for (int i = 0; i < threadCount; i++) { + Thread entryThread = new Thread(new Runnable() { + + @Override + public void run() { + int count = 0; + while (true) { + count++; + Entry entry = null; + try { + Thread.sleep(20); + entry = SphU.entry(KEY); + // token acquired, means pass + pass.addAndGet(1); + if (count % 2 == 0) { + // biz code raise an exception. + throw new RuntimeException("throw runtime "); + } + } catch (BlockException e) { + block.addAndGet(1); + } catch (Throwable t) { + bizException.incrementAndGet(); + Tracer.trace(t); + } finally { + total.addAndGet(1); + if (entry != null) { + entry.exit(); + } + } + } + } + + }); + entryThread.setName("working-thread"); + entryThread.start(); + } + + } + + private static void initDegradeRule() { + List rules = new ArrayList(); + DegradeRule rule = new DegradeRule(); + rule.setResource(KEY); + // set limit exception count to 4 + rule.setCount(4); + rule.setGrade(RuleConstant.DEGRADE_GRADE_EXCEPTION_COUNT); + /** + * When degrading by {@link RuleConstant#DEGRADE_GRADE_EXCEPTION_COUNT}, time window + * less than 60 seconds will not work as expected. Because the exception count is + * summed by minute, when a short time window elapsed, the degradation condition + * may still be satisfied. + */ + rule.setTimeWindow(10); + rules.add(rule); + DegradeRuleManager.loadRules(rules); + } + + private static void tick() { + Thread timer = new Thread(new TimerTask()); + timer.setName("sentinel-timer-task"); + timer.start(); + } + + static class TimerTask implements Runnable { + @Override + public void run() { + long start = System.currentTimeMillis(); + System.out.println("begin to statistic!!!"); + long oldTotal = 0; + long oldPass = 0; + long oldBlock = 0; + long oldBizException = 0; + while (!stop) { + try { + TimeUnit.SECONDS.sleep(1); + } catch (InterruptedException e) { + } + long globalTotal = total.get(); + long oneSecondTotal = globalTotal - oldTotal; + oldTotal = globalTotal; + + long globalPass = pass.get(); + long oneSecondPass = globalPass - oldPass; + oldPass = globalPass; + + long globalBlock = block.get(); + long oneSecondBlock = globalBlock - oldBlock; + oldBlock = globalBlock; + + long globalBizException = bizException.get(); + long oneSecondBizException = globalBizException - oldBizException; + oldBizException = globalBizException; + + System.out.println(TimeUtil.currentTimeMillis() + ", oneSecondTotal:" + oneSecondTotal + + ", oneSecondPass:" + oneSecondPass + + ", oneSecondBlock:" + oneSecondBlock + + ", oneSecondBizException:" + oneSecondBizException); + if (seconds-- <= 0) { + stop = true; + } + } + long cost = System.currentTimeMillis() - start; + System.out.println("time cost: " + cost + " ms"); + System.out.println("total:" + total.get() + ", pass:" + pass.get() + + ", block:" + block.get() + ", bizException:" + bizException.get()); + System.exit(0); + } + } +} diff --git a/sentinel-demo/sentinel-demo-basic/src/main/java/com/alibaba/csp/sentinel/demo/degrade/ExceptionRatioDegradeDemo.java b/sentinel-demo/sentinel-demo-basic/src/main/java/com/alibaba/csp/sentinel/demo/degrade/ExceptionRatioDegradeDemo.java index 6139ba76..8007908c 100755 --- a/sentinel-demo/sentinel-demo-basic/src/main/java/com/alibaba/csp/sentinel/demo/degrade/ExceptionRatioDegradeDemo.java +++ b/sentinel-demo/sentinel-demo-basic/src/main/java/com/alibaba/csp/sentinel/demo/degrade/ExceptionRatioDegradeDemo.java @@ -32,13 +32,16 @@ import com.alibaba.csp.sentinel.util.TimeUtil; /** *

* Degrade is used when the resources are in an unstable state, these resources - * will be degraded within the next defined time window. There are two ways to + * will be degraded within the next defined time window. There are three ways to * measure whether a resource is stable or not: *

    *
  • * Exception ratio: When the ratio of exception count per second and the success - * qps exceeds the threshold , access to the resource will be blocked in the - * coming window. + * qps greats than or equals to the threshold, access to the resource will be blocked + * in the coming time window. + *
    • + *
  • + * Exception Count, see {@link ExceptionCountDegradeDemo}. *
    • *
  • * For average response time, see {@link RtDegradeDemo}. @@ -110,7 +113,7 @@ public class ExceptionRatioDegradeDemo { rule.setResource(KEY); // set limit exception ratio to 0.1 rule.setCount(0.1); - rule.setGrade(RuleConstant.DEGRADE_GRADE_EXCEPTION); + rule.setGrade(RuleConstant.DEGRADE_GRADE_EXCEPTION_RATIO); rule.setTimeWindow(10); rules.add(rule); DegradeRuleManager.loadRules(rules); diff --git a/sentinel-demo/sentinel-demo-basic/src/main/java/com/alibaba/csp/sentinel/demo/degrade/RtDegradeDemo.java b/sentinel-demo/sentinel-demo-basic/src/main/java/com/alibaba/csp/sentinel/demo/degrade/RtDegradeDemo.java index 3236a7c6..c04cbf8a 100755 --- a/sentinel-demo/sentinel-demo-basic/src/main/java/com/alibaba/csp/sentinel/demo/degrade/RtDegradeDemo.java +++ b/sentinel-demo/sentinel-demo-basic/src/main/java/com/alibaba/csp/sentinel/demo/degrade/RtDegradeDemo.java @@ -35,8 +35,8 @@ import com.alibaba.csp.sentinel.slots.block.degrade.DegradeRuleManager; *
      *
    • * Average Response Time ('DegradeRule.Grade=RuleContants.DEGRADE_GRADE_RT'): When the - * average RT exceeds the threshold ('count' in 'DegradeRule', ms), the resource - * enters a quasi-degraded state. If the RT of next coming five requests still + * average RT greats than or equals to the threshold ('count' in 'DegradeRule', ms), the + * resource enters a quasi-degraded state. If the RT of next coming five requests still * exceed this threshold, this resource will be downgraded, which means that in * the next time window(Defined in 'timeWindow', s units) all the access to this * resource will be blocked. @@ -44,6 +44,9 @@ import com.alibaba.csp.sentinel.slots.block.degrade.DegradeRuleManager; *
    • * Exception ratio, see {@link ExceptionRatioDegradeDemo}. *
      • + *
    • + * Exception Count, see {@link ExceptionCountDegradeDemo}. + *
      • *
    * *