Support degrade by exception count (#174)

* Add a new DegradeRule type, degrade by exception count in the last 60 seconds
* Add demo about degrading by exception count

sentinel-core/src/main/java/com/alibaba/csp/sentinel/slots/block/RuleConstant.java

@@ -15,6 +15,8 @@
  */
 package com.alibaba.csp.sentinel.slots.block;
 
+import com.alibaba.csp.sentinel.node.IntervalProperty;
+
 /***
  * @author youji.zj
  * @author jialiang.linjl
@@ -25,7 +27,14 @@ public final class RuleConstant {
     public static final int FLOW_GRADE_QPS = 1;
 
     public static final int DEGRADE_GRADE_RT = 0;
-    public static final int DEGRADE_GRADE_EXCEPTION = 1;
+    /**
+     * Degrade by biz exception ratio in the current {@link IntervalProperty#INTERVAL} second(s).
+     */
+    public static final int DEGRADE_GRADE_EXCEPTION_RATIO = 1;
+    /**
+     * Degrade by biz exception count in the last 60 seconds.
+     */
+    public static final int DEGRADE_GRADE_EXCEPTION_COUNT = 2;
 
     public static final int AUTHORITY_WHITE = 0;
     public static final int AUTHORITY_BLACK = 1;

sentinel-core/src/main/java/com/alibaba/csp/sentinel/slots/block/degrade/DegradeRule.java

@@ -173,7 +173,7 @@ public class DegradeRule extends AbstractRule {
             if (passCount.incrementAndGet() < RT_MAX_EXCEED_N) {
                 return true;
             }
-        } else {
+        } else if (grade == RuleConstant.DEGRADE_GRADE_EXCEPTION_RATIO) {
             double exception = clusterNode.exceptionQps();
             double success = clusterNode.successQps();
             long total = clusterNode.totalQps();
@@ -190,6 +190,11 @@ public class DegradeRule extends AbstractRule {
             if (exception / success < count) {
                 return true;
             }
+        } else if (grade == RuleConstant.DEGRADE_GRADE_EXCEPTION_COUNT) {
+            double exception = clusterNode.totalException();
+            if (exception < count) {
+                return true;
+            }
         }
 
         synchronized (lock) {

sentinel-core/src/test/java/com/alibaba/csp/sentinel/slots/block/degrade/DegradeTest.java

@@ -82,7 +82,7 @@ public class DegradeTest {
         rule.setCount(0.15);
         rule.setResource(key);
         rule.setTimeWindow(5);
-        rule.setGrade(RuleConstant.DEGRADE_GRADE_EXCEPTION);
+        rule.setGrade(RuleConstant.DEGRADE_GRADE_EXCEPTION_RATIO);
 
         when(cn.successQps()).thenReturn(8L);
 
@@ -97,4 +97,34 @@ public class DegradeTest {
         assertTrue(rule.passCheck(context, node, 1));
     }
 
+    @Test
+    public void testExceptionCountModeDegrade() throws Throwable {
+        String key = "test_degrade_exception_count";
+        ClusterNode cn = mock(ClusterNode.class);
+        when(cn.totalException()).thenReturn(10L);
+        ClusterBuilderSlot.getClusterNodeMap().put(new StringResourceWrapper(key, EntryType.IN), cn);
+
+        Context context = mock(Context.class);
+        DefaultNode node = mock(DefaultNode.class);
+        when(node.getClusterNode()).thenReturn(cn);
+
+        DegradeRule rule = new DegradeRule();
+        rule.setCount(4);
+        rule.setResource(key);
+        rule.setTimeWindow(2);
+        rule.setGrade(RuleConstant.DEGRADE_GRADE_EXCEPTION_COUNT);
+
+        when(cn.totalException()).thenReturn(4L);
+
+        // Will fail.
+        assertFalse(rule.passCheck(context, node, 1));
+
+        // Restore from the degrade timeout.
+        TimeUnit.SECONDS.sleep(3);
+
+        when(cn.totalException()).thenReturn(0L);
+        // Will pass.
+        assertTrue(rule.passCheck(context, node, 1));
+    }
+
 }

sentinel-demo/sentinel-demo-basic/src/main/java/com/alibaba/csp/sentinel/demo/degrade/ExceptionCountDegradeDemo.java

@@ -0,0 +1,169 @@
+package com.alibaba.csp.sentinel.demo.degrade;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import com.alibaba.csp.sentinel.Entry;
+import com.alibaba.csp.sentinel.SphU;
+import com.alibaba.csp.sentinel.Tracer;
+import com.alibaba.csp.sentinel.slots.block.BlockException;
+import com.alibaba.csp.sentinel.slots.block.RuleConstant;
+import com.alibaba.csp.sentinel.slots.block.degrade.DegradeRule;
+import com.alibaba.csp.sentinel.slots.block.degrade.DegradeRuleManager;
+import com.alibaba.csp.sentinel.util.TimeUtil;
+
+/**
+ * <p>
+ * Degrade is used when the resources are in an unstable state, these resources
+ * will be degraded within the next defined time window. There are three ways to
+ * measure whether a resource is stable or not:
+ * <ul>
+ * <li>
+ * Exception count: When the exception count in the last 60 seconds greats than
+ * or equals to the threshold, access to the resource will be blocked in the
+ * coming time window.
+ * </li>
+ * <li>
+ * Exception ratio, see {@link ExceptionRatioDegradeDemo}.
+ * </li>
+ * <li>
+ * For average response time, see {@link RtDegradeDemo}.
+ * </li>
+ * </ul>
+ * </p>
+ * <p>
+ * Note: When degrading by {@link RuleConstant#DEGRADE_GRADE_EXCEPTION_COUNT}, time window
+ * less than 60 seconds will not work as expected. Because the exception count is
+ * summed by minute, when a short time window elapsed, the degradation condition
+ * may still be satisfied.
+ * </p>
+ *
+ * @author Carpenter Lee
+ */
+public class ExceptionCountDegradeDemo {
+    private static final String KEY = "abc";
+
+    private static AtomicInteger total = new AtomicInteger();
+    private static AtomicInteger pass = new AtomicInteger();
+    private static AtomicInteger block = new AtomicInteger();
+    private static AtomicInteger bizException = new AtomicInteger();
+
+    private static volatile boolean stop = false;
+    private static final int threadCount = 1;
+    private static int seconds = 60 + 40;
+
+    public static void main(String[] args) throws Exception {
+        tick();
+        initDegradeRule();
+
+        for (int i = 0; i < threadCount; i++) {
+            Thread entryThread = new Thread(new Runnable() {
+
+                @Override
+                public void run() {
+                    int count = 0;
+                    while (true) {
+                        count++;
+                        Entry entry = null;
+                        try {
+                            Thread.sleep(20);
+                            entry = SphU.entry(KEY);
+                            // token acquired, means pass
+                            pass.addAndGet(1);
+                            if (count % 2 == 0) {
+                                // biz code raise an exception.
+                                throw new RuntimeException("throw runtime ");
+                            }
+                        } catch (BlockException e) {
+                            block.addAndGet(1);
+                        } catch (Throwable t) {
+                            bizException.incrementAndGet();
+                            Tracer.trace(t);
+                        } finally {
+                            total.addAndGet(1);
+                            if (entry != null) {
+                                entry.exit();
+                            }
+                        }
+                    }
+                }
+
+            });
+            entryThread.setName("working-thread");
+            entryThread.start();
+        }
+
+    }
+
+    private static void initDegradeRule() {
+        List<DegradeRule> rules = new ArrayList<DegradeRule>();
+        DegradeRule rule = new DegradeRule();
+        rule.setResource(KEY);
+        // set limit exception count to 4
+        rule.setCount(4);
+        rule.setGrade(RuleConstant.DEGRADE_GRADE_EXCEPTION_COUNT);
+        /**
+         * When degrading by {@link RuleConstant#DEGRADE_GRADE_EXCEPTION_COUNT}, time window
+         * less than 60 seconds will not work as expected. Because the exception count is
+         * summed by minute, when a short time window elapsed, the degradation condition
+         * may still be satisfied.
+         */
+        rule.setTimeWindow(10);
+        rules.add(rule);
+        DegradeRuleManager.loadRules(rules);
+    }
+
+    private static void tick() {
+        Thread timer = new Thread(new TimerTask());
+        timer.setName("sentinel-timer-task");
+        timer.start();
+    }
+
+    static class TimerTask implements Runnable {
+        @Override
+        public void run() {
+            long start = System.currentTimeMillis();
+            System.out.println("begin to statistic!!!");
+            long oldTotal = 0;
+            long oldPass = 0;
+            long oldBlock = 0;
+            long oldBizException = 0;
+            while (!stop) {
+                try {
+                    TimeUnit.SECONDS.sleep(1);
+                } catch (InterruptedException e) {
+                }
+                long globalTotal = total.get();
+                long oneSecondTotal = globalTotal - oldTotal;
+                oldTotal = globalTotal;
+
+                long globalPass = pass.get();
+                long oneSecondPass = globalPass - oldPass;
+                oldPass = globalPass;
+
+                long globalBlock = block.get();
+                long oneSecondBlock = globalBlock - oldBlock;
+                oldBlock = globalBlock;
+
+                long globalBizException = bizException.get();
+                long oneSecondBizException = globalBizException - oldBizException;
+                oldBizException = globalBizException;
+
+                System.out.println(TimeUtil.currentTimeMillis() + ", oneSecondTotal:" + oneSecondTotal
+                    + ", oneSecondPass:" + oneSecondPass
+                    + ", oneSecondBlock:" + oneSecondBlock
+                    + ", oneSecondBizException:" + oneSecondBizException);
+                if (seconds-- <= 0) {
+                    stop = true;
+                }
+            }
+            long cost = System.currentTimeMillis() - start;
+            System.out.println("time cost: " + cost + " ms");
+            System.out.println("total:" + total.get() + ", pass:" + pass.get()
+                + ", block:" + block.get() + ", bizException:" + bizException.get());
+            System.exit(0);
+        }
+    }
+}

sentinel-demo/sentinel-demo-basic/src/main/java/com/alibaba/csp/sentinel/demo/degrade/ExceptionRatioDegradeDemo.java

@@ -32,13 +32,16 @@ import com.alibaba.csp.sentinel.util.TimeUtil;
 /**
  * <p>
  * Degrade is used when the resources are in an unstable state, these resources
- * will be degraded within the next defined time window. There are two ways to
+ * will be degraded within the next defined time window. There are three ways to
  * measure whether a resource is stable or not:
  * <ul>
  * <li>
  * Exception ratio: When the ratio of exception count per second and the success
- * qps exceeds the threshold , access to the resource will be blocked in the
- * coming window.
+ * qps greats than or equals to the threshold, access to the resource will be blocked
+ * in the coming time window.
+ * </li>
+ * <li>
+ * Exception Count, see {@link ExceptionCountDegradeDemo}.
  * </li>
  * <li>
  * For average response time, see {@link RtDegradeDemo}.
@@ -110,7 +113,7 @@ public class ExceptionRatioDegradeDemo {
         rule.setResource(KEY);
         // set limit exception ratio to 0.1
         rule.setCount(0.1);
-        rule.setGrade(RuleConstant.DEGRADE_GRADE_EXCEPTION);
+        rule.setGrade(RuleConstant.DEGRADE_GRADE_EXCEPTION_RATIO);
         rule.setTimeWindow(10);
         rules.add(rule);
         DegradeRuleManager.loadRules(rules);

sentinel-demo/sentinel-demo-basic/src/main/java/com/alibaba/csp/sentinel/demo/degrade/RtDegradeDemo.java

@@ -35,8 +35,8 @@ import com.alibaba.csp.sentinel.slots.block.degrade.DegradeRuleManager;
  * <ul>
  * <li>
  * Average Response Time ('DegradeRule.Grade=RuleContants.DEGRADE_GRADE_RT'): When the
- * average RT exceeds the threshold ('count' in 'DegradeRule', ms), the resource
- * enters a quasi-degraded state. If the RT of next coming five requests still
+ * average RT greats than or equals to the threshold ('count' in 'DegradeRule', ms), the
+ * resource enters a quasi-degraded state. If the RT of next coming five requests still
  * exceed this threshold, this resource will be downgraded, which means that in
  * the next time window(Defined in 'timeWindow', s units) all the access to this
  * resource will be blocked.
@@ -44,6 +44,9 @@ import com.alibaba.csp.sentinel.slots.block.degrade.DegradeRuleManager;
  * <li>
  * Exception ratio, see {@link ExceptionRatioDegradeDemo}.
  * </li>
+ * <li>
+ * Exception Count, see {@link ExceptionCountDegradeDemo}.
+ * </li>
  * </ul>
  *
  * </p>