From 02cf5e5639b957ef3c896c1db172c99a1c6b9380 Mon Sep 17 00:00:00 2001 From: Eric Zhao Date: Thu, 6 Sep 2018 18:59:31 +0800 Subject: [PATCH] Add MetricsRepository interface and reuse original in-memory implementation (#126) - Abstract a universal `MetricsRepository` interface so that users can implement their own metrics persistence. - Reuse original in-memory implementation (`InMemoryMetricsRepository`) as the default repository. Signed-off-by: Eric Zhao --- .../dashboard/metric/MetricFetcher.java | 4 +- .../metric/InMemoryMetricsRepository.java} | 104 ++++++++++-------- .../repository/metric/MetricsRepository.java | 60 ++++++++++ .../dashboard/view/MetricController.java | 10 +- 4 files changed, 125 insertions(+), 53 deletions(-) rename sentinel-dashboard/src/main/java/com/taobao/csp/sentinel/dashboard/{inmem/InMemMetricStore.java => repository/metric/InMemoryMetricsRepository.java} (54%) mode change 100755 => 100644 create mode 100644 sentinel-dashboard/src/main/java/com/taobao/csp/sentinel/dashboard/repository/metric/MetricsRepository.java diff --git a/sentinel-dashboard/src/main/java/com/taobao/csp/sentinel/dashboard/metric/MetricFetcher.java b/sentinel-dashboard/src/main/java/com/taobao/csp/sentinel/dashboard/metric/MetricFetcher.java index 630c66ef..ba93100f 100755 --- a/sentinel-dashboard/src/main/java/com/taobao/csp/sentinel/dashboard/metric/MetricFetcher.java +++ b/sentinel-dashboard/src/main/java/com/taobao/csp/sentinel/dashboard/metric/MetricFetcher.java @@ -40,7 +40,7 @@ import com.alibaba.csp.sentinel.util.StringUtil; import com.taobao.csp.sentinel.dashboard.datasource.entity.MetricEntity; import com.taobao.csp.sentinel.dashboard.discovery.AppManagement; import com.taobao.csp.sentinel.dashboard.discovery.MachineInfo; -import com.taobao.csp.sentinel.dashboard.inmem.InMemMetricStore; +import com.taobao.csp.sentinel.dashboard.repository.metric.MetricsRepository; import org.apache.http.HttpResponse; import org.apache.http.client.methods.HttpGet; import org.apache.http.concurrent.FutureCallback; @@ -78,7 +78,7 @@ public class MetricFetcher { private Map appLastFetchTime = new ConcurrentHashMap<>(); @Autowired - private InMemMetricStore metricStore; + private MetricsRepository metricStore; @Autowired private AppManagement appManagement; diff --git a/sentinel-dashboard/src/main/java/com/taobao/csp/sentinel/dashboard/inmem/InMemMetricStore.java b/sentinel-dashboard/src/main/java/com/taobao/csp/sentinel/dashboard/repository/metric/InMemoryMetricsRepository.java old mode 100755 new mode 100644 similarity index 54% rename from sentinel-dashboard/src/main/java/com/taobao/csp/sentinel/dashboard/inmem/InMemMetricStore.java rename to sentinel-dashboard/src/main/java/com/taobao/csp/sentinel/dashboard/repository/metric/InMemoryMetricsRepository.java index 59409f58..29f42770 --- a/sentinel-dashboard/src/main/java/com/taobao/csp/sentinel/dashboard/inmem/InMemMetricStore.java +++ b/sentinel-dashboard/src/main/java/com/taobao/csp/sentinel/dashboard/repository/metric/InMemoryMetricsRepository.java @@ -13,66 +13,77 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.taobao.csp.sentinel.dashboard.inmem; +package com.taobao.csp.sentinel.dashboard.repository.metric; import java.util.ArrayList; import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; +import com.alibaba.csp.sentinel.util.StringUtil; + import com.taobao.csp.sentinel.dashboard.datasource.entity.MetricEntity; import org.springframework.stereotype.Component; /** - * Store metrics in memory. + * Caches metrics data in a period of time in memory. * - * @author leyou + * @author Carpenter Lee + * @author Eric Zhao */ @Component -public class InMemMetricStore { - public static final long MAX_METRIC_LIVE_TIME_MS = 1000 * 60 * 5; +public class InMemoryMetricsRepository implements MetricsRepository { + + private static final long MAX_METRIC_LIVE_TIME_MS = 1000 * 60 * 5; + /** * {@code app -> resource -> timestamp -> metric} */ private Map>> allMetrics = new ConcurrentHashMap<>(); - /** - * Save all metrics in memory. Metric older than {@link #MAX_METRIC_LIVE_TIME_MS} will be removed. - * - * @param metrics metrics to be saved. - */ + @Override + public synchronized void save(MetricEntity entity) { + if (entity == null || StringUtil.isBlank(entity.getApp())) { + return; + } + allMetrics.computeIfAbsent(entity.getApp(), e -> new HashMap<>(16)) + .computeIfAbsent(entity.getResource(), e -> new LinkedHashMap() { + @Override + protected boolean removeEldestEntry(Entry eldest) { + // Metric older than {@link #MAX_METRIC_LIVE_TIME_MS} will be removed. + return eldest.getKey() < System.currentTimeMillis() - MAX_METRIC_LIVE_TIME_MS; + } + }).put(entity.getTimestamp().getTime(), entity); + } + + @Override public synchronized void saveAll(Iterable metrics) { if (metrics == null) { return; } - for (MetricEntity entity : metrics) { - allMetrics.computeIfAbsent(entity.getApp(), e -> new HashMap<>(16)) - .computeIfAbsent(entity.getResource(), e -> new LinkedHashMap() { - @Override - protected boolean removeEldestEntry(Map.Entry eldest) { - return eldest.getKey() < System.currentTimeMillis() - MAX_METRIC_LIVE_TIME_MS; - } - }).put(entity.getTimestamp().getTime(), entity); - } + metrics.forEach(this::save); } - public synchronized List queryByAppAndResouce(String app, - String resource, - long startTime, - long endTime) { + @Override + public synchronized List queryByAppAndResourceBetween(String app, String resource, + long startTime, long endTime) { List results = new ArrayList<>(); - Map> resouceMap = allMetrics.get(app); - if (resouceMap == null) { + if (StringUtil.isBlank(app)) { return results; } - LinkedHashMap metricsMap = resouceMap.get(resource); + Map> resourceMap = allMetrics.get(app); + if (resourceMap == null) { + return results; + } + LinkedHashMap metricsMap = resourceMap.get(resource); if (metricsMap == null) { return results; } - for (Map.Entry entry : metricsMap.entrySet()) { + for (Entry entry : metricsMap.entrySet()) { if (entry.getKey() >= startTime && entry.getKey() <= endTime) { results.add(entry.getValue()); } @@ -80,14 +91,12 @@ public class InMemMetricStore { return results; } - /** - * Find resources of App order by last minute b_qps desc - * - * @param app app name - * @return Resources list, order by last minute b_qps desc. - */ - public synchronized List findResourcesOfApp(String app) { + @Override + public List listResourcesOfApp(String app) { List results = new ArrayList<>(); + if (StringUtil.isBlank(app)) { + return results; + } // resource -> timestamp -> metric Map> resourceMap = allMetrics.get(app); if (resourceMap == null) { @@ -96,8 +105,8 @@ public class InMemMetricStore { final long minTimeMs = System.currentTimeMillis() - 1000 * 60; Map resourceCount = new HashMap<>(32); - for (Map.Entry> resourceMetrics : resourceMap.entrySet()) { - for (Map.Entry metrics : resourceMetrics.getValue().entrySet()) { + for (Entry> resourceMetrics : resourceMap.entrySet()) { + for (Entry metrics : resourceMetrics.getValue().entrySet()) { if (metrics.getKey() < minTimeMs) { continue; } @@ -114,16 +123,19 @@ public class InMemMetricStore { } } } - return resourceCount.entrySet().stream().sorted((o1, o2) -> { - MetricEntity e1 = o1.getValue(); - MetricEntity e2 = o2.getValue(); - int t = e2.getBlockedQps().compareTo(e1.getBlockedQps()); - if (t != 0) { - return t; - } - return e2.getPassedQps().compareTo(e1.getPassedQps()); - }).map(e -> e.getKey()) + // Order by last minute b_qps DESC. + return resourceCount.entrySet() + .stream() + .sorted((o1, o2) -> { + MetricEntity e1 = o1.getValue(); + MetricEntity e2 = o2.getValue(); + int t = e2.getBlockedQps().compareTo(e1.getBlockedQps()); + if (t != 0) { + return t; + } + return e2.getPassedQps().compareTo(e1.getPassedQps()); + }) + .map(Entry::getKey) .collect(Collectors.toList()); } - } diff --git a/sentinel-dashboard/src/main/java/com/taobao/csp/sentinel/dashboard/repository/metric/MetricsRepository.java b/sentinel-dashboard/src/main/java/com/taobao/csp/sentinel/dashboard/repository/metric/MetricsRepository.java new file mode 100644 index 00000000..5840dd3f --- /dev/null +++ b/sentinel-dashboard/src/main/java/com/taobao/csp/sentinel/dashboard/repository/metric/MetricsRepository.java @@ -0,0 +1,60 @@ +/* + * Copyright 1999-2018 Alibaba Group Holding Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.taobao.csp.sentinel.dashboard.repository.metric; + +import java.util.List; + +/** + * Repository interface for aggregated metrics data. + * + * @param type of metrics + * @author Eric Zhao + */ +public interface MetricsRepository { + + /** + * Save the metric to the storage repository. + * + * @param metric metric data to save + */ + void save(T metric); + + /** + * Save all metrics to the storage repository. + * + * @param metrics metrics to save + */ + void saveAll(Iterable metrics); + + /** + * Get all metrics by {@code appName} and {@code resourceName} between a period of time. + * + * @param app application name for Sentinel + * @param resource resource name + * @param startTime start timestamp + * @param endTime end timestamp + * @return all metrics in query conditions + */ + List queryByAppAndResourceBetween(String app, String resource, long startTime, long endTime); + + /** + * List resource name of provided application name. + * + * @param app application name + * @return list of resources + */ + List listResourcesOfApp(String app); +} diff --git a/sentinel-dashboard/src/main/java/com/taobao/csp/sentinel/dashboard/view/MetricController.java b/sentinel-dashboard/src/main/java/com/taobao/csp/sentinel/dashboard/view/MetricController.java index 11039641..bd683c01 100755 --- a/sentinel-dashboard/src/main/java/com/taobao/csp/sentinel/dashboard/view/MetricController.java +++ b/sentinel-dashboard/src/main/java/com/taobao/csp/sentinel/dashboard/view/MetricController.java @@ -24,6 +24,7 @@ import java.util.Map; import java.util.TreeMap; import java.util.concurrent.ConcurrentHashMap; +import com.taobao.csp.sentinel.dashboard.repository.metric.MetricsRepository; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -35,7 +36,6 @@ import org.springframework.web.bind.annotation.ResponseBody; import com.alibaba.csp.sentinel.util.StringUtil; import com.taobao.csp.sentinel.dashboard.datasource.entity.MetricEntity; -import com.taobao.csp.sentinel.dashboard.inmem.InMemMetricStore; import com.taobao.csp.sentinel.dashboard.view.vo.MetricVo; /** @@ -50,7 +50,7 @@ public class MetricController { private static final long maxQueryIntervalMs = 1000 * 60 * 60; @Autowired - private InMemMetricStore metricStore; + private MetricsRepository metricStore; @ResponseBody @RequestMapping("/queryTopResourceMetric.json") @@ -83,7 +83,7 @@ public class MetricController { if (endTime - startTime > maxQueryIntervalMs) { return Result.ofFail(-1, "time intervalMs is too big, must <= 1h"); } - List resources = metricStore.findResourcesOfApp(app); + List resources = metricStore.listResourcesOfApp(app); logger.info("queryTopResourceMetric(), resources.size()={}", resources.size()); if (resources == null || resources.isEmpty()) { return Result.ofSuccess(null); @@ -110,7 +110,7 @@ public class MetricController { logger.info("topResource={}", topResource); long time = System.currentTimeMillis(); for (final String resource : topResource) { - List entities = metricStore.queryByAppAndResouce( + List entities = metricStore.queryByAppAndResourceBetween( app, resource, startTime, endTime); logger.info("resource={}, entities.size()={}", resource, entities == null ? "null" : entities.size()); List vos = MetricVo.fromMetricEntities(entities, resource); @@ -151,7 +151,7 @@ public class MetricController { if (endTime - startTime > maxQueryIntervalMs) { return Result.ofFail(-1, "time intervalMs is too big, must <= 1h"); } - List entities = metricStore.queryByAppAndResouce( + List entities = metricStore.queryByAppAndResourceBetween( app, identity, startTime, endTime); List vos = MetricVo.fromMetricEntities(entities, identity); return Result.ofSuccess(sortMetricVoAndDistinct(vos));