From b51c3ad13e9d5232e5d274d205e2b0e665e4285e Mon Sep 17 00:00:00 2001 From: Jason Song Date: Wed, 8 Aug 2018 18:53:13 +0800 Subject: [PATCH] Add Apollo data source extension (#46) --- pom.xml | 7 +- sentinel-demo/pom.xml | 3 +- .../sentinel-demo-apollo-datasource/pom.xml | 63 ++++++++ .../apollo/ApolloDataSourceDemo.java | 70 +++++++++ .../demo/datasource/apollo/FlowQpsRunner.java | 139 ++++++++++++++++++ .../src/main/resources/log4j2.xml | 16 ++ sentinel-extension/pom.xml | 2 +- .../sentinel-datasource-apollo/pom.xml | 31 ++++ .../datasource/apollo/ApolloDataSource.java | 94 ++++++++++++ 9 files changed, 422 insertions(+), 3 deletions(-) create mode 100644 sentinel-demo/sentinel-demo-apollo-datasource/pom.xml create mode 100644 sentinel-demo/sentinel-demo-apollo-datasource/src/main/java/com/alibaba/csp/sentinel/demo/datasource/apollo/ApolloDataSourceDemo.java create mode 100644 sentinel-demo/sentinel-demo-apollo-datasource/src/main/java/com/alibaba/csp/sentinel/demo/datasource/apollo/FlowQpsRunner.java create mode 100644 sentinel-demo/sentinel-demo-apollo-datasource/src/main/resources/log4j2.xml create mode 100644 sentinel-extension/sentinel-datasource-apollo/pom.xml create mode 100644 sentinel-extension/sentinel-datasource-apollo/src/main/java/com/alibaba/csp/sentinel/datasource/apollo/ApolloDataSource.java diff --git a/pom.xml b/pom.xml index 9ce69aa8..c00222c6 100755 --- a/pom.xml +++ b/pom.xml @@ -101,6 +101,11 @@ sentinel-datasource-zookeeper ${project.version} + + com.alibaba.csp + sentinel-datasource-apollo + ${project.version} + com.alibaba.csp sentinel-transport-simple-http @@ -235,4 +240,4 @@ - \ No newline at end of file + diff --git a/sentinel-demo/pom.xml b/sentinel-demo/pom.xml index 146331b0..3c400242 100755 --- a/sentinel-demo/pom.xml +++ b/sentinel-demo/pom.xml @@ -19,6 +19,7 @@ sentinel-demo-dubbo sentinel-demo-nacos-datasource sentinel-demo-zookeeper-datasource + sentinel-demo-apollo-datasource sentinel-demo-annotation-spring-aop @@ -29,4 +30,4 @@ - \ No newline at end of file + diff --git a/sentinel-demo/sentinel-demo-apollo-datasource/pom.xml b/sentinel-demo/sentinel-demo-apollo-datasource/pom.xml new file mode 100644 index 00000000..994c5fcc --- /dev/null +++ b/sentinel-demo/sentinel-demo-apollo-datasource/pom.xml @@ -0,0 +1,63 @@ + + + + sentinel-demo + com.alibaba.csp + 0.1.1-SNAPSHOT + + 4.0.0 + + sentinel-demo-apollo-datasource + + + 1.8 + 1.8 + 2.9.1 + 1.7.25 + + + + + + org.slf4j + slf4j-api + ${slf4j.version} + + + org.apache.logging.log4j + log4j-core + ${log4j2.version} + + + org.apache.logging.log4j + log4j-api + ${log4j2.version} + + + org.apache.logging.log4j + log4j-slf4j-impl + ${log4j2.version} + + + + + + + com.alibaba.csp + sentinel-datasource-apollo + + + + com.alibaba + fastjson + + + + org.apache.logging.log4j + log4j-slf4j-impl + + + + diff --git a/sentinel-demo/sentinel-demo-apollo-datasource/src/main/java/com/alibaba/csp/sentinel/demo/datasource/apollo/ApolloDataSourceDemo.java b/sentinel-demo/sentinel-demo-apollo-datasource/src/main/java/com/alibaba/csp/sentinel/demo/datasource/apollo/ApolloDataSourceDemo.java new file mode 100644 index 00000000..5db3d20a --- /dev/null +++ b/sentinel-demo/sentinel-demo-apollo-datasource/src/main/java/com/alibaba/csp/sentinel/demo/datasource/apollo/ApolloDataSourceDemo.java @@ -0,0 +1,70 @@ +package com.alibaba.csp.sentinel.demo.datasource.apollo; + +import com.alibaba.csp.sentinel.datasource.DataSource; +import com.alibaba.csp.sentinel.datasource.apollo.ApolloDataSource; +import com.alibaba.csp.sentinel.slots.block.flow.FlowRule; +import com.alibaba.csp.sentinel.slots.block.flow.FlowRuleManager; +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.TypeReference; +import java.util.List; + +/** + * This demo shows how to use Apollo as the data source of Sentinel rules. + *
+ * You need to first set up data as follows: + *
    + *
  1. Create an application with app id as sentinel-demo in Apollo
    2. + *
  2. + * Create a configuration with key as flowRules and value as follows: + * 
    + *      [
+          {
+            "resource": "TestResource",
+            "controlBehavior": 0,
+            "count": 5.0,
+            "grade": 1,
+            "limitApp": "default",
+            "strategy": 0
+          }
+        ]
+ *
    + *
    3. + *
  3. Publish the application namespace
    4. + *
+ * Then you could start this demo and adjust the rule configuration as you wish. + * The rule changes will take effect in real time. + * + * @author Jason Song + */ +public class ApolloDataSourceDemo { + private static final String KEY = "TestResource"; + + public static void main(String[] args) { + loadRules(); + // Assume we config: resource is `TestResource`, initial QPS threshold is 5. + FlowQpsRunner runner = new FlowQpsRunner(KEY, 1, 100); + runner.simulateTraffic(); + runner.tick(); + } + + private static void loadRules() { + /** + * Set up basic information, only for demo purpose. You may adjust them based on your actual environment. + *
+ * For more information, please refer https://github.com/ctripcorp/apollo + */ + String appId = "sentinel-demo"; + String apolloMetaServerAddress = "http://localhost:8080"; + System.setProperty("app.id", appId); + System.setProperty("apollo.meta", apolloMetaServerAddress); + + String namespaceName = "application"; + String flowRuleKey = "flowRules"; + String defaultFlowRules = "[]"; //it's better to provide a meaningful default value + + DataSource> flowRuleDataSource = new ApolloDataSource<>(namespaceName, flowRuleKey, + defaultFlowRules, source -> JSON.parseObject(source, new TypeReference>() { + })); + FlowRuleManager.register2Property(flowRuleDataSource.getProperty()); + } +} diff --git a/sentinel-demo/sentinel-demo-apollo-datasource/src/main/java/com/alibaba/csp/sentinel/demo/datasource/apollo/FlowQpsRunner.java b/sentinel-demo/sentinel-demo-apollo-datasource/src/main/java/com/alibaba/csp/sentinel/demo/datasource/apollo/FlowQpsRunner.java new file mode 100644 index 00000000..eddc53de --- /dev/null +++ b/sentinel-demo/sentinel-demo-apollo-datasource/src/main/java/com/alibaba/csp/sentinel/demo/datasource/apollo/FlowQpsRunner.java @@ -0,0 +1,139 @@ +/* + * Copyright 1999-2018 Alibaba Group Holding Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.csp.sentinel.demo.datasource.apollo; + +import java.util.Random; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +import com.alibaba.csp.sentinel.Entry; +import com.alibaba.csp.sentinel.SphU; +import com.alibaba.csp.sentinel.slots.block.BlockException; +import com.alibaba.csp.sentinel.util.TimeUtil; + +/** + * Flow QPS runner. + * + * @author Carpenter Lee + * @author Eric Zhao + */ +class FlowQpsRunner { + + private final String resourceName; + private final int threadCount; + private int seconds; + + public FlowQpsRunner(String resourceName, int threadCount, int seconds) { + this.resourceName = resourceName; + this.threadCount = threadCount; + this.seconds = seconds; + } + + private final AtomicInteger pass = new AtomicInteger(); + private final AtomicInteger block = new AtomicInteger(); + private final AtomicInteger total = new AtomicInteger(); + + private volatile boolean stop = false; + + public void simulateTraffic() { + for (int i = 0; i < threadCount; i++) { + Thread t = new Thread(new RunTask()); + t.setName("simulate-traffic-Task"); + t.start(); + } + } + + public void tick() { + Thread timer = new Thread(new TimerTask()); + timer.setName("sentinel-timer-task"); + timer.start(); + } + + final class RunTask implements Runnable { + @Override + public void run() { + while (!stop) { + Entry entry = null; + + try { + entry = SphU.entry(resourceName); + // token acquired, means pass + pass.addAndGet(1); + } catch (BlockException e1) { + block.incrementAndGet(); + } catch (Exception e2) { + // biz exception + } finally { + total.incrementAndGet(); + if (entry != null) { + entry.exit(); + } + } + + Random random2 = new Random(); + try { + TimeUnit.MILLISECONDS.sleep(random2.nextInt(50)); + } catch (InterruptedException e) { + // ignore + } + } + } + } + + final class TimerTask implements Runnable { + @Override + public void run() { + long start = System.currentTimeMillis(); + System.out.println("begin to statistic!!!"); + + long oldTotal = 0; + long oldPass = 0; + long oldBlock = 0; + while (!stop) { + try { + TimeUnit.SECONDS.sleep(1); + } catch (InterruptedException e) { + } + long globalTotal = total.get(); + long oneSecondTotal = globalTotal - oldTotal; + oldTotal = globalTotal; + + long globalPass = pass.get(); + long oneSecondPass = globalPass - oldPass; + oldPass = globalPass; + + long globalBlock = block.get(); + long oneSecondBlock = globalBlock - oldBlock; + oldBlock = globalBlock; + + System.out.println(seconds + " send qps is: " + oneSecondTotal); + System.out.println(TimeUtil.currentTimeMillis() + ", total:" + oneSecondTotal + + ", pass:" + oneSecondPass + + ", block:" + oneSecondBlock); + + if (seconds-- <= 0) { + stop = true; + } + } + + long cost = System.currentTimeMillis() - start; + System.out.println("time cost: " + cost + " ms"); + System.out.println("total:" + total.get() + ", pass:" + pass.get() + + ", block:" + block.get()); + System.exit(0); + } + } +} diff --git a/sentinel-demo/sentinel-demo-apollo-datasource/src/main/resources/log4j2.xml b/sentinel-demo/sentinel-demo-apollo-datasource/src/main/resources/log4j2.xml new file mode 100644 index 00000000..0226a293 --- /dev/null +++ b/sentinel-demo/sentinel-demo-apollo-datasource/src/main/resources/log4j2.xml @@ -0,0 +1,16 @@ + + + + + + + + + + + + + + + + diff --git a/sentinel-extension/pom.xml b/sentinel-extension/pom.xml index 080f3f29..98b30239 100755 --- a/sentinel-extension/pom.xml +++ b/sentinel-extension/pom.xml @@ -15,7 +15,7 @@ sentinel-datasource-extension sentinel-datasource-nacos sentinel-datasource-zookeeper - + sentinel-datasource-apollo sentinel-annotation-aspectj diff --git a/sentinel-extension/sentinel-datasource-apollo/pom.xml b/sentinel-extension/sentinel-datasource-apollo/pom.xml new file mode 100644 index 00000000..171df454 --- /dev/null +++ b/sentinel-extension/sentinel-datasource-apollo/pom.xml @@ -0,0 +1,31 @@ + + + + sentinel-extension + com.alibaba.csp + 0.1.1-SNAPSHOT + + 4.0.0 + + sentinel-datasource-apollo + + + 1.0.0 + + + + + com.alibaba.csp + sentinel-datasource-extension + + + + com.ctrip.framework.apollo + apollo-client + ${apollo.version} + + + + diff --git a/sentinel-extension/sentinel-datasource-apollo/src/main/java/com/alibaba/csp/sentinel/datasource/apollo/ApolloDataSource.java b/sentinel-extension/sentinel-datasource-apollo/src/main/java/com/alibaba/csp/sentinel/datasource/apollo/ApolloDataSource.java new file mode 100644 index 00000000..51561f1d --- /dev/null +++ b/sentinel-extension/sentinel-datasource-apollo/src/main/java/com/alibaba/csp/sentinel/datasource/apollo/ApolloDataSource.java @@ -0,0 +1,94 @@ +package com.alibaba.csp.sentinel.datasource.apollo; + +import com.alibaba.csp.sentinel.datasource.AbstractDataSource; +import com.alibaba.csp.sentinel.datasource.ConfigParser; +import com.alibaba.csp.sentinel.datasource.DataSource; +import com.alibaba.csp.sentinel.log.RecordLog; +import com.ctrip.framework.apollo.Config; +import com.ctrip.framework.apollo.ConfigChangeListener; +import com.ctrip.framework.apollo.ConfigService; +import com.ctrip.framework.apollo.model.ConfigChange; +import com.ctrip.framework.apollo.model.ConfigChangeEvent; +import com.google.common.base.Preconditions; +import com.google.common.base.Strings; +import com.google.common.collect.Sets; + +/** + * A {@link DataSource} with Apollo as its configuration source. + *
+ * When the rule is changed in Apollo, it will take effect in real time. + * + * @author Jason Song + */ +public class ApolloDataSource extends AbstractDataSource { + + private final Config config; + private final String flowRulesKey; + private final String defaultFlowRuleValue; + + /** + * Constructs the Apollo data source + * + * @param namespaceName the namespace name in Apollo, should not be null or empty + * @param flowRulesKey the flow rules key in the namespace, should not be null or empty + * @param defaultFlowRuleValue the default flow rules value when the flow rules key is not found or any error occurred + * @param parser the parser to transform string configuration to actual flow rules + */ + public ApolloDataSource(String namespaceName, String flowRulesKey, String defaultFlowRuleValue, + ConfigParser parser) { + super(parser); + + Preconditions.checkArgument(!Strings.isNullOrEmpty(namespaceName), "Namespace name could not be null or empty"); + Preconditions.checkArgument(!Strings.isNullOrEmpty(flowRulesKey), "FlowRuleKey could not be null or empty!"); + + this.flowRulesKey = flowRulesKey; + this.defaultFlowRuleValue = defaultFlowRuleValue; + + this.config = ConfigService.getConfig(namespaceName); + + initialize(); + + RecordLog.info(String.format("Initialized rule for namespace: %s, flow rules key: %s", namespaceName, flowRulesKey)); + } + + private void initialize() { + initializeConfigChangeListener(); + loadAndUpdateRules(); + } + + private void loadAndUpdateRules() { + try { + T newValue = loadConfig(); + if (newValue == null) { + RecordLog.warn("[ApolloDataSource] WARN: rule config is null, you may have to check your data source"); + } + getProperty().updateValue(newValue); + } catch (Throwable ex) { + RecordLog.warn("[ApolloDataSource] Error when loading rule config", ex); + } + } + + private void initializeConfigChangeListener() { + config.addChangeListener(new ConfigChangeListener() { + @Override + public void onChange(ConfigChangeEvent changeEvent) { + ConfigChange change = changeEvent.getChange(flowRulesKey); + //change is never null because the listener will only notify for this key + if (change != null) { + RecordLog.info("[ApolloDataSource] Received config changes: " + change.toString()); + } + loadAndUpdateRules(); + } + }, Sets.newHashSet(flowRulesKey)); + } + + @Override + public String readSource() throws Exception { + return config.getProperty(flowRulesKey, defaultFlowRuleValue); + } + + @Override + public void close() throws Exception { + // nothing to destroy + } +}