Browse Source

Automatically de-duplicate rules when loading rules (#571)

* De-duplicate rules automatically when loading rules
* Update rule managers

Signed-off-by: Eric Zhao <sczyh16@gmail.com>
master
Eric Zhao GitHub 5 years ago
parent
commit
1741da0bab
No known key found for this signature in database GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 138 additions and 83 deletions
  1. +24
    -23
      sentinel-core/src/main/java/com/alibaba/csp/sentinel/slots/block/authority/AuthorityRuleManager.java
  2. +3
    -3
      sentinel-core/src/main/java/com/alibaba/csp/sentinel/slots/block/authority/AuthoritySlot.java
  3. +76
    -31
      sentinel-core/src/main/java/com/alibaba/csp/sentinel/slots/block/degrade/DegradeRuleManager.java
  4. +2
    -0
      sentinel-core/src/main/java/com/alibaba/csp/sentinel/slots/block/flow/FlowRuleManager.java
  5. +15
    -9
      sentinel-core/src/main/java/com/alibaba/csp/sentinel/slots/block/flow/FlowRuleUtil.java
  6. +18
    -17
      sentinel-extension/sentinel-parameter-flow-control/src/main/java/com/alibaba/csp/sentinel/slots/block/flow/param/ParamFlowRuleManager.java

+ 24
- 23
sentinel-core/src/main/java/com/alibaba/csp/sentinel/slots/block/authority/AuthorityRuleManager.java View File

@@ -16,12 +16,15 @@
package com.alibaba.csp.sentinel.slots.block.authority;

import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;

import com.alibaba.csp.sentinel.log.RecordLog;
import com.alibaba.csp.sentinel.slots.block.RuleConstant;
import com.alibaba.csp.sentinel.util.AssertUtil;
import com.alibaba.csp.sentinel.util.StringUtil;
import com.alibaba.csp.sentinel.property.DynamicSentinelProperty;
import com.alibaba.csp.sentinel.property.PropertyListener;
@@ -36,24 +39,22 @@ import com.alibaba.csp.sentinel.property.SentinelProperty;
*/
public final class AuthorityRuleManager {

private static Map<String, List<AuthorityRule>> authorityRules
= new ConcurrentHashMap<String, List<AuthorityRule>>();
private static Map<String, Set<AuthorityRule>> authorityRules = new ConcurrentHashMap<>();

final static RulePropertyListener listener = new RulePropertyListener();

private static SentinelProperty<List<AuthorityRule>> currentProperty
= new DynamicSentinelProperty<List<AuthorityRule>>();
private static final RulePropertyListener LISTENER = new RulePropertyListener();
private static SentinelProperty<List<AuthorityRule>> currentProperty = new DynamicSentinelProperty<>();

static {
currentProperty.addListener(listener);
currentProperty.addListener(LISTENER);
}

public static void register2Property(SentinelProperty<List<AuthorityRule>> property) {
synchronized (listener) {
AssertUtil.notNull(property, "property cannot be null");
synchronized (LISTENER) {
if (currentProperty != null) {
currentProperty.removeListener(listener);
currentProperty.removeListener(LISTENER);
}
property.addListener(listener);
property.addListener(LISTENER);
currentProperty = property;
RecordLog.info("[AuthorityRuleManager] Registering new property to authority rule manager");
}
@@ -78,11 +79,11 @@ public final class AuthorityRuleManager {
* @return a new copy of the rules.
*/
public static List<AuthorityRule> getRules() {
List<AuthorityRule> rules = new ArrayList<AuthorityRule>();
List<AuthorityRule> rules = new ArrayList<>();
if (authorityRules == null) {
return rules;
}
for (Map.Entry<String, List<AuthorityRule>> entry : authorityRules.entrySet()) {
for (Map.Entry<String, Set<AuthorityRule>> entry : authorityRules.entrySet()) {
rules.addAll(entry.getValue());
}
return rules;
@@ -92,7 +93,7 @@ public final class AuthorityRuleManager {

@Override
public void configUpdate(List<AuthorityRule> conf) {
Map<String, List<AuthorityRule>> rules = loadAuthorityConf(conf);
Map<String, Set<AuthorityRule>> rules = loadAuthorityConf(conf);

authorityRules.clear();
if (rules != null) {
@@ -101,8 +102,8 @@ public final class AuthorityRuleManager {
RecordLog.info("[AuthorityRuleManager] Authority rules received: " + authorityRules);
}

private Map<String, List<AuthorityRule>> loadAuthorityConf(List<AuthorityRule> list) {
Map<String, List<AuthorityRule>> newRuleMap = new ConcurrentHashMap<String, List<AuthorityRule>>();
private Map<String, Set<AuthorityRule>> loadAuthorityConf(List<AuthorityRule> list) {
Map<String, Set<AuthorityRule>> newRuleMap = new ConcurrentHashMap<>();

if (list == null || list.isEmpty()) {
return newRuleMap;
@@ -119,12 +120,12 @@ public final class AuthorityRuleManager {
}

String identity = rule.getResource();
List<AuthorityRule> ruleM = newRuleMap.get(identity);
Set<AuthorityRule> ruleSet = newRuleMap.get(identity);
// putIfAbsent
if (ruleM == null) {
ruleM = new ArrayList<AuthorityRule>();
ruleM.add(rule);
newRuleMap.put(identity, ruleM);
if (ruleSet == null) {
ruleSet = new HashSet<>();
ruleSet.add(rule);
newRuleMap.put(identity, ruleSet);
} else {
// One resource should only have at most one authority rule, so just ignore redundant rules.
RecordLog.warn("[AuthorityRuleManager] Ignoring redundant rule: " + rule.toString());
@@ -136,7 +137,7 @@ public final class AuthorityRuleManager {

@Override
public void configLoad(List<AuthorityRule> value) {
Map<String, List<AuthorityRule>> rules = loadAuthorityConf(value);
Map<String, Set<AuthorityRule>> rules = loadAuthorityConf(value);

authorityRules.clear();
if (rules != null) {
@@ -146,11 +147,11 @@ public final class AuthorityRuleManager {
}
}

static Map<String, List<AuthorityRule>> getAuthorityRules() {
static Map<String, Set<AuthorityRule>> getAuthorityRules() {
return authorityRules;
}

static boolean isValidRule(AuthorityRule rule) {
public static boolean isValidRule(AuthorityRule rule) {
return rule != null && !StringUtil.isBlank(rule.getResource())
&& rule.getStrategy() >= 0 && StringUtil.isNotBlank(rule.getLimitApp());
}


+ 3
- 3
sentinel-core/src/main/java/com/alibaba/csp/sentinel/slots/block/authority/AuthoritySlot.java View File

@@ -15,8 +15,8 @@
*/
package com.alibaba.csp.sentinel.slots.block.authority;

import java.util.List;
import java.util.Map;
import java.util.Set;

import com.alibaba.csp.sentinel.context.Context;
import com.alibaba.csp.sentinel.node.DefaultNode;
@@ -45,13 +45,13 @@ public class AuthoritySlot extends AbstractLinkedProcessorSlot<DefaultNode> {
}

void checkBlackWhiteAuthority(ResourceWrapper resource, Context context) throws AuthorityException {
Map<String, List<AuthorityRule>> authorityRules = AuthorityRuleManager.getAuthorityRules();
Map<String, Set<AuthorityRule>> authorityRules = AuthorityRuleManager.getAuthorityRules();

if (authorityRules == null) {
return;
}

List<AuthorityRule> rules = authorityRules.get(resource.getName());
Set<AuthorityRule> rules = authorityRules.get(resource.getName());
if (rules == null) {
return;
}


+ 76
- 31
sentinel-core/src/main/java/com/alibaba/csp/sentinel/slots/block/degrade/DegradeRuleManager.java View File

@@ -16,10 +16,14 @@
package com.alibaba.csp.sentinel.slots.block.degrade;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;

import com.alibaba.csp.sentinel.Constants;
import com.alibaba.csp.sentinel.context.Context;
import com.alibaba.csp.sentinel.log.RecordLog;
import com.alibaba.csp.sentinel.node.DefaultNode;
@@ -29,23 +33,24 @@ import com.alibaba.csp.sentinel.property.SentinelProperty;
import com.alibaba.csp.sentinel.slotchain.ResourceWrapper;
import com.alibaba.csp.sentinel.slots.block.BlockException;
import com.alibaba.csp.sentinel.slots.block.RuleConstant;
import com.alibaba.csp.sentinel.util.AssertUtil;
import com.alibaba.csp.sentinel.util.StringUtil;

/***
/**
* @author youji.zj
* @author jialiang.linjl
* @author Eric Zhao
*/
public class DegradeRuleManager {
public final class DegradeRuleManager {

private static volatile Map<String, List<DegradeRule>> degradeRules
= new ConcurrentHashMap<String, List<DegradeRule>>();
private static final Map<String, Set<DegradeRule>> degradeRules = new ConcurrentHashMap<>();

final static RulePropertyListener listener = new RulePropertyListener();
private static final RulePropertyListener LISTENER = new RulePropertyListener();
private static SentinelProperty<List<DegradeRule>> currentProperty
= new DynamicSentinelProperty<List<DegradeRule>>();
= new DynamicSentinelProperty<>();

static {
currentProperty.addListener(listener);
currentProperty.addListener(LISTENER);
}

/**
@@ -55,21 +60,19 @@ public class DegradeRuleManager {
* @param property the property to listen.
*/
public static void register2Property(SentinelProperty<List<DegradeRule>> property) {
synchronized (listener) {
AssertUtil.notNull(property, "property cannot be null");
synchronized (LISTENER) {
RecordLog.info("[DegradeRuleManager] Registering new property to degrade rule manager");
currentProperty.removeListener(listener);
property.addListener(listener);
currentProperty.removeListener(LISTENER);
property.addListener(LISTENER);
currentProperty = property;
}
}

public static void checkDegrade(ResourceWrapper resource, Context context, DefaultNode node, int count)
throws BlockException {
if (degradeRules == null) {
return;
}

List<DegradeRule> rules = degradeRules.get(resource.getName());
Set<DegradeRule> rules = degradeRules.get(resource.getName());
if (rules == null) {
return;
}
@@ -82,6 +85,9 @@ public class DegradeRuleManager {
}

public static boolean hasConfig(String resource) {
if (resource == null) {
return false;
}
return degradeRules.containsKey(resource);
}

@@ -91,11 +97,8 @@ public class DegradeRuleManager {
* @return a new copy of the rules.
*/
public static List<DegradeRule> getRules() {
List<DegradeRule> rules = new ArrayList<DegradeRule>();
if (degradeRules == null) {
return rules;
}
for (Map.Entry<String, List<DegradeRule>> entry : degradeRules.entrySet()) {
List<DegradeRule> rules = new ArrayList<>();
for (Map.Entry<String, Set<DegradeRule>> entry : degradeRules.entrySet()) {
rules.addAll(entry.getValue());
}
return rules;
@@ -110,7 +113,42 @@ public class DegradeRuleManager {
try {
currentProperty.updateValue(rules);
} catch (Throwable e) {
RecordLog.info(e.getMessage(), e);
RecordLog.warn("[DegradeRuleManager] Unexpected error when loading degrade rules", e);
}
}

/**
* Set degrade rules for provided resource. Former rules of the resource will be replaced.
*
* @param resourceName valid resource name
* @param rules new rule set to load
* @return whether the rules has actually been updated
* @since 1.5.0
*/
public static boolean setRulesForResource(String resourceName, Set<DegradeRule> rules) {
AssertUtil.notEmpty(resourceName, "resourceName cannot be empty");
try {
Map<String, Set<DegradeRule>> newRuleMap = new HashMap<>(degradeRules);
if (rules == null) {
newRuleMap.remove(resourceName);
} else {
Set<DegradeRule> newSet = new HashSet<>();
for (DegradeRule rule : rules) {
if (isValidRule(rule) && resourceName.equals(rule.getResource())) {
newSet.add(rule);
}
}
newRuleMap.put(resourceName, newSet);
}
List<DegradeRule> allRules = new ArrayList<>();
for (Set<DegradeRule> set : newRuleMap.values()) {
allRules.addAll(set);
}
return currentProperty.updateValue(allRules);
} catch (Throwable e) {
RecordLog.warn(
"[DegradeRuleManager] Unexpected error when setting degrade rules for resource: " + resourceName, e);
return false;
}
}

@@ -118,7 +156,7 @@ public class DegradeRuleManager {

@Override
public void configUpdate(List<DegradeRule> conf) {
Map<String, List<DegradeRule>> rules = loadDegradeConf(conf);
Map<String, Set<DegradeRule>> rules = loadDegradeConf(conf);
if (rules != null) {
degradeRules.clear();
degradeRules.putAll(rules);
@@ -128,7 +166,7 @@ public class DegradeRuleManager {

@Override
public void configLoad(List<DegradeRule> conf) {
Map<String, List<DegradeRule>> rules = loadDegradeConf(conf);
Map<String, Set<DegradeRule>> rules = loadDegradeConf(conf);
if (rules != null) {
degradeRules.clear();
degradeRules.putAll(rules);
@@ -136,8 +174,8 @@ public class DegradeRuleManager {
RecordLog.info("[DegradeRuleManager] Degrade rules loaded: " + degradeRules);
}

private Map<String, List<DegradeRule>> loadDegradeConf(List<DegradeRule> list) {
Map<String, List<DegradeRule>> newRuleMap = new ConcurrentHashMap<String, List<DegradeRule>>();
private Map<String, Set<DegradeRule>> loadDegradeConf(List<DegradeRule> list) {
Map<String, Set<DegradeRule>> newRuleMap = new ConcurrentHashMap<>();

if (list == null || list.isEmpty()) {
return newRuleMap;
@@ -145,7 +183,8 @@ public class DegradeRuleManager {

for (DegradeRule rule : list) {
if (!isValidRule(rule)) {
RecordLog.warn("[DegradeRuleManager] Ignoring invalid degrade rule when loading new rules: " + rule);
RecordLog.warn(
"[DegradeRuleManager] Ignoring invalid degrade rule when loading new rules: " + rule);
continue;
}

@@ -154,17 +193,16 @@ public class DegradeRuleManager {
}

String identity = rule.getResource();
List<DegradeRule> ruleM = newRuleMap.get(identity);
if (ruleM == null) {
ruleM = new ArrayList<DegradeRule>();
newRuleMap.put(identity, ruleM);
Set<DegradeRule> ruleSet = newRuleMap.get(identity);
if (ruleSet == null) {
ruleSet = new HashSet<>();
newRuleMap.put(identity, ruleSet);
}
ruleM.add(rule);
ruleSet.add(rule);
}

return newRuleMap;
}

}

public static boolean isValidRule(DegradeRule rule) {
@@ -173,6 +211,13 @@ public class DegradeRuleManager {
if (!baseValid) {
return false;
}
// Warn for RT mode that exceeds the {@code TIME_DROP_VALVE}.
int maxAllowedRt = Constants.TIME_DROP_VALVE;
if (rule.getGrade() == RuleConstant.DEGRADE_GRADE_RT && rule.getCount() > maxAllowedRt) {
RecordLog.warn(String.format("[DegradeRuleManager] WARN: setting large RT threshold (%.1f ms) in RT mode"
+ " will not take effect since it exceeds the max allowed value (%d ms)", rule.getCount(),
maxAllowedRt));
}
// Check exception ratio mode.
if (rule.getGrade() == RuleConstant.DEGRADE_GRADE_EXCEPTION_RATIO && rule.getCount() > 1) {
return false;


+ 2
- 0
sentinel-core/src/main/java/com/alibaba/csp/sentinel/slots/block/flow/FlowRuleManager.java View File

@@ -25,6 +25,7 @@ import java.util.concurrent.TimeUnit;

import com.alibaba.csp.sentinel.concurrent.NamedThreadFactory;
import com.alibaba.csp.sentinel.log.RecordLog;
import com.alibaba.csp.sentinel.util.AssertUtil;
import com.alibaba.csp.sentinel.util.StringUtil;
import com.alibaba.csp.sentinel.node.metric.MetricTimerListener;
import com.alibaba.csp.sentinel.property.DynamicSentinelProperty;
@@ -65,6 +66,7 @@ public class FlowRuleManager {
* @param property the property to listen.
*/
public static void register2Property(SentinelProperty<List<FlowRule>> property) {
AssertUtil.notNull(property, "property cannot be null");
synchronized (LISTENER) {
RecordLog.info("[FlowRuleManager] Registering new property to flow rule manager");
currentProperty.removeListener(LISTENER);


+ 15
- 9
sentinel-core/src/main/java/com/alibaba/csp/sentinel/slots/block/flow/FlowRuleUtil.java View File

@@ -18,8 +18,11 @@ package com.alibaba.csp.sentinel.slots.block.flow;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;

import com.alibaba.csp.sentinel.log.RecordLog;
@@ -85,10 +88,11 @@ public final class FlowRuleUtil {
*/
public static <K> Map<K, List<FlowRule>> buildFlowRuleMap(List<FlowRule> list, Function<FlowRule, K> groupFunction,
Predicate<FlowRule> filter, boolean shouldSort) {
Map<K, List<FlowRule>> newRuleMap = new ConcurrentHashMap<K, List<FlowRule>>();
Map<K, List<FlowRule>> newRuleMap = new ConcurrentHashMap<>();
if (list == null || list.isEmpty()) {
return newRuleMap;
}
Map<K, Set<FlowRule>> tmpMap = new ConcurrentHashMap<>();

for (FlowRule rule : list) {
if (!isValidRule(rule)) {
@@ -109,22 +113,24 @@ public final class FlowRuleUtil {
if (key == null) {
continue;
}
List<FlowRule> flowRules = newRuleMap.get(key);
Set<FlowRule> flowRules = tmpMap.get(key);

if (flowRules == null) {
flowRules = new ArrayList<FlowRule>();
newRuleMap.put(key, flowRules);
// Use hash set here to remove duplicate rules.
flowRules = new HashSet<>();
tmpMap.put(key, flowRules);
}

flowRules.add(rule);
}
if (shouldSort && !newRuleMap.isEmpty()) {
Comparator<FlowRule> comparator = new FlowRuleComparator();
// Sort the rules.
for (List<FlowRule> rules : newRuleMap.values()) {
Comparator<FlowRule> comparator = new FlowRuleComparator();
for (Entry<K, Set<FlowRule>> entries : tmpMap.entrySet()) {
List<FlowRule> rules = new ArrayList<>(entries.getValue());
if (shouldSort) {
// Sort the rules.
Collections.sort(rules, comparator);
}
newRuleMap.put(entries.getKey(), rules);
}

return newRuleMap;


+ 18
- 17
sentinel-extension/sentinel-parameter-flow-control/src/main/java/com/alibaba/csp/sentinel/slots/block/flow/param/ParamFlowRuleManager.java View File

@@ -16,6 +16,7 @@
package com.alibaba.csp.sentinel.slots.block.flow.param;

import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -26,6 +27,7 @@ import com.alibaba.csp.sentinel.property.DynamicSentinelProperty;
import com.alibaba.csp.sentinel.property.PropertyListener;
import com.alibaba.csp.sentinel.property.SentinelProperty;
import com.alibaba.csp.sentinel.slots.block.RuleConstant;
import com.alibaba.csp.sentinel.util.AssertUtil;
import com.alibaba.csp.sentinel.util.StringUtil;

/**
@@ -37,12 +39,10 @@ import com.alibaba.csp.sentinel.util.StringUtil;
*/
public final class ParamFlowRuleManager {

private static final Map<String, List<ParamFlowRule>> paramFlowRules
= new ConcurrentHashMap<String, List<ParamFlowRule>>();
private static final Map<String, Set<ParamFlowRule>> paramFlowRules = new ConcurrentHashMap<>();

private final static RulePropertyListener PROPERTY_LISTENER = new RulePropertyListener();
private static SentinelProperty<List<ParamFlowRule>> currentProperty
= new DynamicSentinelProperty<List<ParamFlowRule>>();
private static SentinelProperty<List<ParamFlowRule>> currentProperty = new DynamicSentinelProperty<>();

static {
currentProperty.addListener(PROPERTY_LISTENER);
@@ -68,6 +68,7 @@ public final class ParamFlowRuleManager {
* @param property the property to listen
*/
public static void register2Property(SentinelProperty<List<ParamFlowRule>> property) {
AssertUtil.notNull(property, "property cannot be null");
synchronized (PROPERTY_LISTENER) {
currentProperty.removeListener(PROPERTY_LISTENER);
property.addListener(PROPERTY_LISTENER);
@@ -77,11 +78,11 @@ public final class ParamFlowRuleManager {
}

public static List<ParamFlowRule> getRulesOfResource(String resourceName) {
return paramFlowRules.get(resourceName);
return new ArrayList<>(paramFlowRules.get(resourceName));
}

public static boolean hasRules(String resourceName) {
List<ParamFlowRule> rules = paramFlowRules.get(resourceName);
Set<ParamFlowRule> rules = paramFlowRules.get(resourceName);
return rules != null && !rules.isEmpty();
}

@@ -91,8 +92,8 @@ public final class ParamFlowRuleManager {
* @return a new copy of the rules.
*/
public static List<ParamFlowRule> getRules() {
List<ParamFlowRule> rules = new ArrayList<ParamFlowRule>();
for (Map.Entry<String, List<ParamFlowRule>> entry : paramFlowRules.entrySet()) {
List<ParamFlowRule> rules = new ArrayList<>();
for (Map.Entry<String, Set<ParamFlowRule>> entry : paramFlowRules.entrySet()) {
rules.addAll(entry.getValue());
}
return rules;
@@ -102,7 +103,7 @@ public final class ParamFlowRuleManager {

@Override
public void configUpdate(List<ParamFlowRule> list) {
Map<String, List<ParamFlowRule>> rules = aggregateHotParamRules(list);
Map<String, Set<ParamFlowRule>> rules = aggregateHotParamRules(list);
if (rules != null) {
paramFlowRules.clear();
paramFlowRules.putAll(rules);
@@ -112,7 +113,7 @@ public final class ParamFlowRuleManager {

@Override
public void configLoad(List<ParamFlowRule> list) {
Map<String, List<ParamFlowRule>> rules = aggregateHotParamRules(list);
Map<String, Set<ParamFlowRule>> rules = aggregateHotParamRules(list);
if (rules != null) {
paramFlowRules.clear();
paramFlowRules.putAll(rules);
@@ -120,8 +121,8 @@ public final class ParamFlowRuleManager {
RecordLog.info("[ParamFlowRuleManager] Hot spot parameter flow rules received: " + paramFlowRules);
}

private Map<String, List<ParamFlowRule>> aggregateHotParamRules(List<ParamFlowRule> list) {
Map<String, List<ParamFlowRule>> newRuleMap = new ConcurrentHashMap<String, List<ParamFlowRule>>();
private Map<String, Set<ParamFlowRule>> aggregateHotParamRules(List<ParamFlowRule> list) {
Map<String, Set<ParamFlowRule>> newRuleMap = new ConcurrentHashMap<>();

if (list == null || list.isEmpty()) {
// No parameter flow rules, so clear all the metrics.
@@ -143,12 +144,12 @@ public final class ParamFlowRuleManager {
ParamFlowRuleUtil.fillExceptionFlowItems(rule);

String resourceName = rule.getResource();
List<ParamFlowRule> ruleList = newRuleMap.get(resourceName);
if (ruleList == null) {
ruleList = new ArrayList<ParamFlowRule>();
newRuleMap.put(resourceName, ruleList);
Set<ParamFlowRule> ruleSet = newRuleMap.get(resourceName);
if (ruleSet == null) {
ruleSet = new HashSet<>();
newRuleMap.put(resourceName, ruleSet);
}
ruleList.add(rule);
ruleSet.add(rule);
}

// Clear unused hot param metrics.


Loading…
Cancel
Save