From a8d4f53dcadd4e5a151df42c1624a01b2593e1d8 Mon Sep 17 00:00:00 2001 From: phaneesh Date: Sat, 3 Aug 2019 17:12:52 +0530 Subject: [PATCH 1/3] Add degrade support for apis (force lower theradpool and timeout values to fail fast) --- pom.xml | 2 +- .../core/util/RevolverCommandHelper.java | 326 +++++++++--------- .../revolver/degrade/DegradeRegistry.java | 81 +++++ .../resource/RevolverApiManageResource.java | 104 +++++- 4 files changed, 348 insertions(+), 165 deletions(-) create mode 100644 src/main/java/io/dropwizard/revolver/degrade/DegradeRegistry.java diff --git a/pom.xml b/pom.xml index 76b3bef0..ca278e16 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ io.dropwizard.revolver dropwizard-revolver - 1.3.7-38-5 + 1.3.7-38-6 jar dropwizard-revolver diff --git a/src/main/java/io/dropwizard/revolver/core/util/RevolverCommandHelper.java b/src/main/java/io/dropwizard/revolver/core/util/RevolverCommandHelper.java index b6171e2d..95ec673f 100644 --- a/src/main/java/io/dropwizard/revolver/core/util/RevolverCommandHelper.java +++ b/src/main/java/io/dropwizard/revolver/core/util/RevolverCommandHelper.java @@ -36,177 +36,183 @@ import io.dropwizard.revolver.core.config.hystrix.ThreadPoolConfig; import io.dropwizard.revolver.core.model.RevolverRequest; import io.dropwizard.revolver.core.tracing.TraceInfo; +import io.dropwizard.revolver.degrade.DegradeRegistry; +import org.apache.commons.lang3.StringUtils; + import java.util.Map; import java.util.stream.Collectors; -import org.apache.commons.lang3.StringUtils; /** * @author phaneesh */ public class RevolverCommandHelper { - public static String getName(RevolverRequest request) { - return Joiner.on(".").join(request.getService(), request.getApi()); + public static String getName(RevolverRequest request) { + return Joiner.on(".").join(request.getService(), request.getApi()); + } + + public static T normalize(T request) { + if (null == request) { + throw new RevolverExecutionException(RevolverExecutionException.Type.BAD_REQUEST, + "Request cannot be null"); + } + TraceInfo traceInfo = request.getTrace(); + if (traceInfo == null) { + traceInfo = new TraceInfo(); + request.setTrace(traceInfo); + } + if (Strings.isNullOrEmpty(traceInfo.getRequestId())) { + throw new RevolverExecutionException(RevolverExecutionException.Type.BAD_REQUEST, + "Request ID must be passed in span"); + } + if (Strings.isNullOrEmpty(traceInfo.getTransactionId())) { + throw new RevolverExecutionException(RevolverExecutionException.Type.BAD_REQUEST, + "Transaction ID must be passed"); + } + if (0L == traceInfo.getTimestamp()) { + traceInfo.setTimestamp(System.currentTimeMillis()); + } + return request; + } + + /* + Group thread pools can be specified at service level. Different api of the service can subscribe to + different group thread pool + Timeout would be overridden if provided at individual api level + */ + public static HystrixCommand.Setter setter(RevolverCommand commandHandler, + String api) { + RuntimeConfig runtimeConfig = commandHandler.getRuntimeConfig(); + RevolverServiceConfig serviceConfiguration = commandHandler.getServiceConfiguration(); + CommandHandlerConfig config = commandHandler.getApiConfiguration(); + CircuitBreakerConfig circuitBreakerConfig; + if (null != runtimeConfig) { + circuitBreakerConfig = runtimeConfig.getCircuitBreaker(); + } else if (null != config.getRuntime() && null != config.getRuntime().getCircuitBreaker()) { + circuitBreakerConfig = config.getRuntime().getCircuitBreaker(); + } else if (null != serviceConfiguration.getRuntime() && null != serviceConfiguration + .getRuntime().getCircuitBreaker()) { + circuitBreakerConfig = serviceConfiguration.getRuntime().getCircuitBreaker(); + } else { + circuitBreakerConfig = new CircuitBreakerConfig(); + } + ThreadPoolConfig serviceThreadPoolConfig = null; + if (null != serviceConfiguration.getRuntime() && null != serviceConfiguration.getRuntime() + .getThreadPool()) { + serviceThreadPoolConfig = serviceConfiguration.getRuntime().getThreadPool(); + } + String keyName = StringUtils.EMPTY; + + MetricsConfig metricsConfig; + if (null != runtimeConfig) { + metricsConfig = runtimeConfig.getMetrics(); + } else { + metricsConfig = new MetricsConfig(); } - public static T normalize(T request) { - if (null == request) { - throw new RevolverExecutionException(RevolverExecutionException.Type.BAD_REQUEST, - "Request cannot be null"); - } - TraceInfo traceInfo = request.getTrace(); - if (traceInfo == null) { - traceInfo = new TraceInfo(); - request.setTrace(traceInfo); - } - if (Strings.isNullOrEmpty(traceInfo.getRequestId())) { - throw new RevolverExecutionException(RevolverExecutionException.Type.BAD_REQUEST, - "Request ID must be passed in span"); - } - if (Strings.isNullOrEmpty(traceInfo.getTransactionId())) { - throw new RevolverExecutionException(RevolverExecutionException.Type.BAD_REQUEST, - "Transaction ID must be passed"); - } - if (0L == traceInfo.getTimestamp()) { - traceInfo.setTimestamp(System.currentTimeMillis()); - } - return request; + Map threadPoolConfigMap; + if (null != serviceConfiguration.getThreadPoolGroupConfig() && null != serviceConfiguration + .getThreadPoolGroupConfig().getThreadPools()) { + threadPoolConfigMap = serviceConfiguration.getThreadPoolGroupConfig().getThreadPools() + .stream() + .collect(Collectors.toMap(ThreadPoolConfig::getThreadPoolName, t -> t)); + } else { + threadPoolConfigMap = Maps.newHashMap(); } - /* - Group thread pools can be specified at service level. Different api of the service can subscribe to - different group thread pool - Timeout would be overridden if provided at individual api level - */ - public static HystrixCommand.Setter setter(RevolverCommand commandHandler, - String api) { - RuntimeConfig runtimeConfig = commandHandler.getRuntimeConfig(); - RevolverServiceConfig serviceConfiguration = commandHandler.getServiceConfiguration(); - CommandHandlerConfig config = commandHandler.getApiConfiguration(); - CircuitBreakerConfig circuitBreakerConfig; - if (null != runtimeConfig) { - circuitBreakerConfig = runtimeConfig.getCircuitBreaker(); - } else if (null != config.getRuntime() && null != config.getRuntime().getCircuitBreaker()) { - circuitBreakerConfig = config.getRuntime().getCircuitBreaker(); - } else if (null != serviceConfiguration.getRuntime() && null != serviceConfiguration - .getRuntime().getCircuitBreaker()) { - circuitBreakerConfig = serviceConfiguration.getRuntime().getCircuitBreaker(); - } else { - circuitBreakerConfig = new CircuitBreakerConfig(); - } - ThreadPoolConfig serviceThreadPoolConfig = null; - if (null != serviceConfiguration.getRuntime() && null != serviceConfiguration.getRuntime() - .getThreadPool()) { - serviceThreadPoolConfig = serviceConfiguration.getRuntime().getThreadPool(); - } - String keyName = StringUtils.EMPTY; - - MetricsConfig metricsConfig; - if (null != runtimeConfig) { - metricsConfig = runtimeConfig.getMetrics(); - } else { - metricsConfig = new MetricsConfig(); - } - - Map threadPoolConfigMap; - if (null != serviceConfiguration.getThreadPoolGroupConfig() && null != serviceConfiguration - .getThreadPoolGroupConfig().getThreadPools()) { - threadPoolConfigMap = serviceConfiguration.getThreadPoolGroupConfig().getThreadPools() - .stream() - .collect(Collectors.toMap(ThreadPoolConfig::getThreadPoolName, t -> t)); - } else { - threadPoolConfigMap = Maps.newHashMap(); - } - - ThreadPoolConfig threadPoolConfig; - - if (null != config.getRuntime() && null != config.getRuntime().getThreadPool() - && StringUtils.isNotEmpty(config.getRuntime().getThreadPool().getThreadPoolName()) - && null != threadPoolConfigMap - .get(config.getRuntime().getThreadPool().getThreadPoolName())) { - - threadPoolConfig = threadPoolConfigMap - .get(config.getRuntime().getThreadPool().getThreadPoolName()); - keyName = threadPoolConfig.getThreadPoolName(); - - } else if (config.isSharedPool() && null != serviceThreadPoolConfig) { - - threadPoolConfig = serviceThreadPoolConfig; - if (StringUtils.isEmpty(keyName)) { - keyName = Joiner.on(".") - .join(commandHandler.getServiceConfiguration().getService(), "shared"); - } - - } else if (null != config.getRuntime() && null != config.getRuntime().getThreadPool()) { - - threadPoolConfig = config.getRuntime().getThreadPool(); - keyName = Joiner.on(".") - .join(commandHandler.getServiceConfiguration().getService(), api); - - } else if (null != serviceThreadPoolConfig) { - threadPoolConfig = serviceConfiguration.getRuntime().getThreadPool(); - if (StringUtils.isEmpty(keyName)) { - keyName = Joiner.on(".") - .join(commandHandler.getServiceConfiguration().getService(), api); - } - - } else if (null != runtimeConfig) { - threadPoolConfig = runtimeConfig.getThreadPool(); - keyName = Joiner.on(".") - .join(commandHandler.getServiceConfiguration().getService(), api); - - } else { - threadPoolConfig = new ThreadPoolConfig(); - keyName = Joiner.on(".") - .join(commandHandler.getServiceConfiguration().getService(), api); - } - - //Setting timeout from api thread pool config - if (null != config.getRuntime() && null != config.getRuntime().getThreadPool()) { - threadPoolConfig.setTimeout(config.getRuntime().getThreadPool().getTimeout()); - } - - int concurrency = threadPoolConfig.getConcurrency(); - int coreSize = (int) Math.ceil(concurrency * metricsConfig.getCorePoolSizeReductionParam()); - return HystrixCommand.Setter.withGroupKey( - HystrixCommandGroupKey.Factory.asKey(serviceConfiguration.getService())) - .andCommandPropertiesDefaults(HystrixCommandProperties.Setter() - .withExecutionIsolationStrategy(threadPoolConfig.isSemaphoreIsolated() - ? HystrixCommandProperties.ExecutionIsolationStrategy.SEMAPHORE - : HystrixCommandProperties.ExecutionIsolationStrategy.THREAD) - .withExecutionIsolationSemaphoreMaxConcurrentRequests( - threadPoolConfig.getConcurrency()) - .withFallbackIsolationSemaphoreMaxConcurrentRequests( - threadPoolConfig.getConcurrency()) - .withFallbackEnabled(commandHandler.isFallbackEnabled()) - .withCircuitBreakerErrorThresholdPercentage( - circuitBreakerConfig.getErrorThresholdPercentage()) - .withCircuitBreakerRequestVolumeThreshold( - circuitBreakerConfig.getNumAcceptableFailuresInTimeWindow()) - .withCircuitBreakerSleepWindowInMilliseconds( - circuitBreakerConfig.getWaitTimeBeforeRetry()) - .withExecutionTimeoutInMilliseconds(threadPoolConfig.getTimeout()) - .withMetricsHealthSnapshotIntervalInMilliseconds( - metricsConfig.getHealthCheckInterval()) - .withMetricsRollingPercentileBucketSize( - metricsConfig.getPercentileBucketSize()) - .withMetricsRollingPercentileWindowInMilliseconds( - metricsConfig.getPercentileTimeInMillis())).andCommandKey( - HystrixCommandKey.Factory.asKey(Joiner.on(".") - .join(commandHandler.getServiceConfiguration().getService(), api))) - .andThreadPoolKey(HystrixThreadPoolKey.Factory.asKey(keyName)) - .andThreadPoolPropertiesDefaults( - HystrixThreadPoolProperties.Setter().withCoreSize(coreSize) - .withMaxQueueSize(threadPoolConfig.getMaxRequestQueueSize()) - .withMaximumSize(concurrency) - .withKeepAliveTimeMinutes( - threadPoolConfig.getKeepAliveTimeInMinutes()) - .withQueueSizeRejectionThreshold( - threadPoolConfig.getDynamicRequestQueueSize()) - .withAllowMaximumSizeToDivergeFromCoreSize(true) - .withMetricsRollingStatisticalWindowBuckets( - metricsConfig.getStatsBucketSize()) - .withMetricsRollingStatisticalWindowInMilliseconds( - metricsConfig.getStatsTimeInMillis())); + ThreadPoolConfig threadPoolConfig; + + if (null != config.getRuntime() && null != config.getRuntime().getThreadPool() + && StringUtils.isNotEmpty(config.getRuntime().getThreadPool().getThreadPoolName()) + && null != threadPoolConfigMap + .get(config.getRuntime().getThreadPool().getThreadPoolName())) { + + threadPoolConfig = threadPoolConfigMap + .get(config.getRuntime().getThreadPool().getThreadPoolName()); + keyName = threadPoolConfig.getThreadPoolName(); + + } else if (config.isSharedPool() && null != serviceThreadPoolConfig) { + + threadPoolConfig = serviceThreadPoolConfig; + if (StringUtils.isEmpty(keyName)) { + keyName = Joiner.on(".") + .join(commandHandler.getServiceConfiguration().getService(), "shared"); + } + + } else if (null != config.getRuntime() && null != config.getRuntime().getThreadPool()) { + + threadPoolConfig = config.getRuntime().getThreadPool(); + keyName = Joiner.on(".") + .join(commandHandler.getServiceConfiguration().getService(), api); + + } else if (null != serviceThreadPoolConfig) { + threadPoolConfig = serviceConfiguration.getRuntime().getThreadPool(); + if (StringUtils.isEmpty(keyName)) { + keyName = Joiner.on(".") + .join(commandHandler.getServiceConfiguration().getService(), api); + } + + } else if (null != runtimeConfig) { + threadPoolConfig = runtimeConfig.getThreadPool(); + keyName = Joiner.on(".") + .join(commandHandler.getServiceConfiguration().getService(), api); + + } else { + threadPoolConfig = new ThreadPoolConfig(); + keyName = Joiner.on(".") + .join(commandHandler.getServiceConfiguration().getService(), api); } + + //Setting timeout from api thread pool config + if (null != config.getRuntime() && null != config.getRuntime().getThreadPool()) { + threadPoolConfig.setTimeout(config.getRuntime().getThreadPool().getTimeout()); + } + + final String hystrixCommandKey = Joiner.on(".") + .join(commandHandler.getServiceConfiguration().getService(), api); + int concurrency = DegradeRegistry.getInstance().getDegradedThreadPool(hystrixCommandKey, + threadPoolConfig.getConcurrency()); + int timeout = DegradeRegistry.getInstance().getDegradedTimeout(hystrixCommandKey, + threadPoolConfig.getTimeout()); + int coreSize = (int) Math.ceil(concurrency * metricsConfig.getCorePoolSizeReductionParam()); + return HystrixCommand.Setter.withGroupKey( + HystrixCommandGroupKey.Factory.asKey(serviceConfiguration.getService())) + .andCommandPropertiesDefaults(HystrixCommandProperties.Setter() + .withExecutionIsolationStrategy(threadPoolConfig.isSemaphoreIsolated() + ? HystrixCommandProperties.ExecutionIsolationStrategy.SEMAPHORE + : HystrixCommandProperties.ExecutionIsolationStrategy.THREAD) + .withExecutionIsolationSemaphoreMaxConcurrentRequests( + threadPoolConfig.getConcurrency()) + .withFallbackIsolationSemaphoreMaxConcurrentRequests( + threadPoolConfig.getConcurrency()) + .withFallbackEnabled(commandHandler.isFallbackEnabled()) + .withCircuitBreakerErrorThresholdPercentage( + circuitBreakerConfig.getErrorThresholdPercentage()) + .withCircuitBreakerRequestVolumeThreshold( + circuitBreakerConfig.getNumAcceptableFailuresInTimeWindow()) + .withCircuitBreakerSleepWindowInMilliseconds( + circuitBreakerConfig.getWaitTimeBeforeRetry()) + .withExecutionTimeoutInMilliseconds(timeout) + .withMetricsHealthSnapshotIntervalInMilliseconds( + metricsConfig.getHealthCheckInterval()) + .withMetricsRollingPercentileBucketSize( + metricsConfig.getPercentileBucketSize()) + .withMetricsRollingPercentileWindowInMilliseconds( + metricsConfig.getPercentileTimeInMillis())).andCommandKey( + HystrixCommandKey.Factory.asKey(hystrixCommandKey)) + .andThreadPoolKey(HystrixThreadPoolKey.Factory.asKey(keyName)) + .andThreadPoolPropertiesDefaults( + HystrixThreadPoolProperties.Setter().withCoreSize(coreSize) + .withMaxQueueSize(threadPoolConfig.getMaxRequestQueueSize()) + .withMaximumSize(concurrency) + .withKeepAliveTimeMinutes( + threadPoolConfig.getKeepAliveTimeInMinutes()) + .withQueueSizeRejectionThreshold( + threadPoolConfig.getDynamicRequestQueueSize()) + .withAllowMaximumSizeToDivergeFromCoreSize(true) + .withMetricsRollingStatisticalWindowBuckets( + metricsConfig.getStatsBucketSize()) + .withMetricsRollingStatisticalWindowInMilliseconds( + metricsConfig.getStatsTimeInMillis())); + } } diff --git a/src/main/java/io/dropwizard/revolver/degrade/DegradeRegistry.java b/src/main/java/io/dropwizard/revolver/degrade/DegradeRegistry.java new file mode 100644 index 00000000..4f0726ae --- /dev/null +++ b/src/main/java/io/dropwizard/revolver/degrade/DegradeRegistry.java @@ -0,0 +1,81 @@ +/* + * Copyright 2016 Phaneesh Nagaraja . + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package io.dropwizard.revolver.degrade; + +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +public class DegradeRegistry { + + private static DegradeRegistry instance; + + private ConcurrentHashMap threadPoolDegradeMap; + + private ConcurrentHashMap timeoutDegradeMap; + + private DegradeRegistry() { + threadPoolDegradeMap = new ConcurrentHashMap<>(); + timeoutDegradeMap = new ConcurrentHashMap<>(); + } + + public static synchronized DegradeRegistry getInstance() { + if(instance == null) { + instance = new DegradeRegistry(); + } + return instance; + } + + public void addThreadPoolDegrade(final String key, final double factor) { + threadPoolDegradeMap.put(key, factor); + } + + public void removeThreadPoolDegrade(final String key) { + threadPoolDegradeMap.remove(key); + } + + public void addTimeoutDegrade(final String key, final double factor) { + timeoutDegradeMap.put(key, factor); + } + + public void removeTimeoutDegrade(final String key) { + timeoutDegradeMap.remove(key); + } + + public int getDegradedThreadPool(final String key, int originalThreadpoolSize) { + if(!threadPoolDegradeMap.containsKey(key)) { + return originalThreadpoolSize; + } + return (int)Math.round(originalThreadpoolSize * threadPoolDegradeMap.get(key)); + } + + public int getDegradedTimeout(final String key, int originalTimeout) { + if(!timeoutDegradeMap.containsKey(key)) { + return originalTimeout; + } + return (int)Math.round(originalTimeout * threadPoolDegradeMap.get(key)); + } + + public ConcurrentMap getThreadPoolDegradeStatus() { + return threadPoolDegradeMap; + } + + public ConcurrentMap getTimeoutDegradeStatus() { + return timeoutDegradeMap; + } + +} diff --git a/src/main/java/io/dropwizard/revolver/resource/RevolverApiManageResource.java b/src/main/java/io/dropwizard/revolver/resource/RevolverApiManageResource.java index 8612690a..6d2259b2 100644 --- a/src/main/java/io/dropwizard/revolver/resource/RevolverApiManageResource.java +++ b/src/main/java/io/dropwizard/revolver/resource/RevolverApiManageResource.java @@ -20,8 +20,12 @@ import com.codahale.metrics.annotation.Metered; import com.google.common.collect.ImmutableMap; import io.dropwizard.revolver.RevolverBundle; +import io.dropwizard.revolver.degrade.DegradeRegistry; import io.swagger.annotations.ApiOperation; -import java.util.stream.Collectors; +import lombok.Builder; +import lombok.Data; +import lombok.extern.slf4j.Slf4j; + import javax.inject.Singleton; import javax.ws.rs.GET; import javax.ws.rs.POST; @@ -30,9 +34,7 @@ import javax.ws.rs.Produces; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; -import lombok.Builder; -import lombok.Data; -import lombok.extern.slf4j.Slf4j; +import java.util.stream.Collectors; @Path("/revolver") @Slf4j @@ -109,6 +111,100 @@ public Response disable(@PathParam("service") String service, } } + @Path("/v1/manage/api/degrade/threadpool/{service}/{api}/{factor}") + @POST + @Metered + @ApiOperation(value = "Enable threadpool degrade for api") + @Produces(MediaType.APPLICATION_JSON) + public Response degradeThreadPool(@PathParam("service") String service, + @PathParam("api") String api, @PathParam("factor") double factor) { + String key = service + "." + api; + if (RevolverBundle.apiStatus.containsKey(key)) { + DegradeRegistry.getInstance().addThreadPoolDegrade(key, factor); + return Response.ok(ImmutableMap.builder().put("service", service) + .put("api", api) + .put("factor", factor).build()) + .build(); + } else { + return Response.status(Response.Status.BAD_REQUEST) + .entity(ImmutableMap.builder().put("service", service) + .put("api", api).build()).build(); + } + } + + @Path("/v1/manage/api/degrade/threadpool/disable/{service}/{api}") + @POST + @Metered + @ApiOperation(value = "Disable threadpool degrade for api") + @Produces(MediaType.APPLICATION_JSON) + public Response disableDegradeThreadPool(@PathParam("service") String service, + @PathParam("api") String api) { + String key = service + "." + api; + if (RevolverBundle.apiStatus.containsKey(key)) { + DegradeRegistry.getInstance().removeThreadPoolDegrade(key); + return Response.ok(ImmutableMap.builder().put("service", service) + .put("api", api) + .build()).build(); + } else { + return Response.status(Response.Status.BAD_REQUEST) + .entity(ImmutableMap.builder().put("service", service) + .put("api", api).build()).build(); + } + } + + @Path("/v1/manage/api/degrade/timeout/{service}/{api}/{factor}") + @POST + @Metered + @ApiOperation(value = "Enable timeout degrade for api") + @Produces(MediaType.APPLICATION_JSON) + public Response degradeTimeout(@PathParam("service") String service, + @PathParam("api") String api, @PathParam("factor") double factor) { + String key = service + "." + api; + if (RevolverBundle.apiStatus.containsKey(key)) { + DegradeRegistry.getInstance().addTimeoutDegrade(key, factor); + return Response.ok(ImmutableMap.builder().put("service", service) + .put("api", api) + .put("factor", factor).build()) + .build(); + } else { + return Response.status(Response.Status.BAD_REQUEST) + .entity(ImmutableMap.builder().put("service", service) + .put("api", api).build()).build(); + } + } + + @Path("/v1/manage/api/degrade/timeout/disable/{service}/{api}") + @POST + @Metered + @ApiOperation(value = "Enable timeout degrade for api") + @Produces(MediaType.APPLICATION_JSON) + public Response degradeTimeout(@PathParam("service") String service, @PathParam("api") String api) { + String key = service + "." + api; + if (RevolverBundle.apiStatus.containsKey(key)) { + DegradeRegistry.getInstance().removeTimeoutDegrade(key); + return Response.ok(ImmutableMap.builder().put("service", service) + .put("api", api)) + .build(); + } else { + return Response.status(Response.Status.BAD_REQUEST) + .entity(ImmutableMap.builder().put("service", service) + .put("api", api).build()).build(); + } + } + + @Path("/v1/manage/api/degrade/status") + @GET + @Metered + @ApiOperation(value = "Degrade status for all apis") + @Produces(MediaType.APPLICATION_JSON) + public Response degradeStatus() { + return Response.status(Response.Status.BAD_REQUEST) + .entity(ImmutableMap.builder().put("threadpool", DegradeRegistry.getInstance().getThreadPoolDegradeStatus()) + .put("timeout", DegradeRegistry.getInstance().getTimeoutDegradeStatus()) + .build()).build(); + } + + @Path("/v1/manage/api/status") @GET @Metered From d2824ca96998bf67e9b801ef1160280a68f2a1cc Mon Sep 17 00:00:00 2001 From: phaneesh Date: Mon, 5 Aug 2019 12:16:14 +0530 Subject: [PATCH 2/3] Add support to degrade threadpool at service scope --- .../dropwizard/revolver/RevolverBundle.java | 3 + .../revolver/degrade/DegradeRegistry.java | 9 ++ .../optimizer/RevolverConfigUpdater.java | 14 ++- .../resource/RevolverApiManageResource.java | 98 +++++++++++++++++++ 4 files changed, 121 insertions(+), 3 deletions(-) diff --git a/src/main/java/io/dropwizard/revolver/RevolverBundle.java b/src/main/java/io/dropwizard/revolver/RevolverBundle.java index 2d3a5593..e2e7c13b 100644 --- a/src/main/java/io/dropwizard/revolver/RevolverBundle.java +++ b/src/main/java/io/dropwizard/revolver/RevolverBundle.java @@ -36,6 +36,7 @@ import io.dropwizard.revolver.core.config.RevolverConfig; import io.dropwizard.revolver.core.config.RevolverServiceConfig; import io.dropwizard.revolver.core.config.hystrix.ThreadPoolConfig; +import io.dropwizard.revolver.degrade.DegradeRegistry; import io.dropwizard.revolver.discovery.RevolverServiceResolver; import io.dropwizard.revolver.discovery.model.RangerEndpointSpec; import io.dropwizard.revolver.discovery.model.SimpleEndpointSpec; @@ -402,6 +403,8 @@ public void run(T configuration, Environment environment) { } + DegradeRegistry.getInstance(); + environment.jersey().register(new RevolverRequestFilter(revolverConfig)); environment.jersey().register( diff --git a/src/main/java/io/dropwizard/revolver/degrade/DegradeRegistry.java b/src/main/java/io/dropwizard/revolver/degrade/DegradeRegistry.java index 4f0726ae..358b0d9a 100644 --- a/src/main/java/io/dropwizard/revolver/degrade/DegradeRegistry.java +++ b/src/main/java/io/dropwizard/revolver/degrade/DegradeRegistry.java @@ -78,4 +78,13 @@ public ConcurrentMap getTimeoutDegradeStatus() { return timeoutDegradeMap; } + + public boolean isThreadPoolDegraded(final String key) { + return threadPoolDegradeMap.containsKey(key); + } + + public boolean isTimeoutDegraded(final String key) { + return timeoutDegradeMap.containsKey(key); + } + } diff --git a/src/main/java/io/dropwizard/revolver/optimizer/RevolverConfigUpdater.java b/src/main/java/io/dropwizard/revolver/optimizer/RevolverConfigUpdater.java index c6da4699..4f707df4 100644 --- a/src/main/java/io/dropwizard/revolver/optimizer/RevolverConfigUpdater.java +++ b/src/main/java/io/dropwizard/revolver/optimizer/RevolverConfigUpdater.java @@ -6,6 +6,7 @@ import io.dropwizard.revolver.core.config.RevolverConfig; import io.dropwizard.revolver.core.config.RevolverServiceConfig; import io.dropwizard.revolver.core.config.hystrix.ThreadPoolConfig; +import io.dropwizard.revolver.degrade.DegradeRegistry; import io.dropwizard.revolver.http.config.RevolverHttpApiConfig; import io.dropwizard.revolver.http.config.RevolverHttpServiceConfig; import io.dropwizard.revolver.optimizer.config.OptimizerConcurrencyConfig; @@ -220,16 +221,23 @@ private void updatedApiSettings(RevolverServiceConfig revolverServiceConfig, if (optimizerAggregatedMetrics == null) { return; } - updateConcurrencySetting(api.getRuntime().getThreadPool(), optimizerAggregatedMetrics, + if(!DegradeRegistry.getInstance().isThreadPoolDegraded(key)) { + updateConcurrencySetting(api.getRuntime().getThreadPool(), optimizerAggregatedMetrics, configUpdated, api.getApi()); - updateTimeoutSettings(api.getRuntime().getThreadPool(), optimizerAggregatedMetrics, + } + if(!DegradeRegistry.getInstance().isTimeoutDegraded(key)) { + updateTimeoutSettings(api.getRuntime().getThreadPool(), optimizerAggregatedMetrics, configUpdated, api); - updateLatencySettings(api, optimizerAggregatedMetrics); + updateLatencySettings(api, optimizerAggregatedMetrics); + } } private void updateConcurrencySetting(ThreadPoolConfig threadPoolConfig, OptimizerAggregatedMetrics optimizerAggregatedMetrics, AtomicBoolean configUpdated, String poolName) { + if(DegradeRegistry.getInstance().isThreadPoolDegraded(poolName)) { + return; + } OptimizerConcurrencyConfig optimizerConcurrencyConfig = optimizerConfig .getConcurrencyConfig(); if (optimizerConcurrencyConfig == null || !optimizerConfig.getConcurrencyConfig() diff --git a/src/main/java/io/dropwizard/revolver/resource/RevolverApiManageResource.java b/src/main/java/io/dropwizard/revolver/resource/RevolverApiManageResource.java index 6d2259b2..f3d7764f 100644 --- a/src/main/java/io/dropwizard/revolver/resource/RevolverApiManageResource.java +++ b/src/main/java/io/dropwizard/revolver/resource/RevolverApiManageResource.java @@ -20,6 +20,7 @@ import com.codahale.metrics.annotation.Metered; import com.google.common.collect.ImmutableMap; import io.dropwizard.revolver.RevolverBundle; +import io.dropwizard.revolver.core.config.hystrix.ThreadPoolConfig; import io.dropwizard.revolver.degrade.DegradeRegistry; import io.swagger.annotations.ApiOperation; import lombok.Builder; @@ -132,6 +133,36 @@ public Response degradeThreadPool(@PathParam("service") String service, } } + @Path("/v1/manage/api/degrade/threadpoolgroup/concurrency/{service}/{threadpool}/{factor}") + @POST + @Metered + @ApiOperation(value = "Enable threadpool degrade for service") + @Produces(MediaType.APPLICATION_JSON) + public Response degradeServiceThreadPool(@PathParam("service") String service, + @PathParam("threadpool") String threadpool, @PathParam("factor") double factor) { + if (!RevolverBundle.getServiceConfig().containsKey(service)) { + return Response.status(Response.Status.BAD_REQUEST) + .entity(ImmutableMap.builder().put("service", service) + .put("threadpool", threadpool).build()).build(); + } + if(null == RevolverBundle.getServiceConfig().get(service).getThreadPoolGroupConfig()) { + return Response.status(Response.Status.BAD_REQUEST) + .entity(ImmutableMap.builder().put("service", service) + .put("threadpool", threadpool).build()).build(); + } + if(RevolverBundle.getServiceConfig().get(service).getThreadPoolGroupConfig() + .getThreadPools().stream().map(ThreadPoolConfig::getThreadPoolName).noneMatch(n -> n.equals(threadpool))) { + return Response.status(Response.Status.BAD_REQUEST) + .entity(ImmutableMap.builder().put("service", service) + .put("threadpool", threadpool).build()).build(); + } + DegradeRegistry.getInstance().addThreadPoolDegrade(service +"." +threadpool, factor); + return Response.ok(ImmutableMap.builder().put("service", service) + .put("threadpool", threadpool) + .put("factor", factor).build()) + .build(); + } + @Path("/v1/manage/api/degrade/threadpool/disable/{service}/{api}") @POST @Metered @@ -152,6 +183,24 @@ public Response disableDegradeThreadPool(@PathParam("service") String service, } } + @Path("/v1/manage/api/degrade/disable/threadpoolgroup/concurrency/{service}/{threadpool}") + @POST + @Metered + @ApiOperation(value = "Enable threadpool degrade for service") + @Produces(MediaType.APPLICATION_JSON) + public Response disableDegradeServiceThreadPool(@PathParam("service") String service, + @PathParam("threadpool") String threadpool) { + if (!DegradeRegistry.getInstance().getThreadPoolDegradeStatus().containsKey(service +"." +threadpool)) { + return Response.status(Response.Status.BAD_REQUEST) + .entity(ImmutableMap.builder().put("service", service) + .put("threadpool", threadpool).build()).build(); + } + DegradeRegistry.getInstance().removeThreadPoolDegrade(service +"." +threadpool); + return Response.ok(ImmutableMap.builder().put("service", service) + .put("threadpool", threadpool)) + .build(); + } + @Path("/v1/manage/api/degrade/timeout/{service}/{api}/{factor}") @POST @Metered @@ -173,6 +222,37 @@ public Response degradeTimeout(@PathParam("service") String service, } } + @Path("/v1/manage/api/degrade/threadpoolgroup/timeout/{service}/{threadpool}/{factor}") + @POST + @Metered + @ApiOperation(value = "Enable timeout degrade for service") + @Produces(MediaType.APPLICATION_JSON) + public Response degradeServiceTimeout(@PathParam("service") String service, + @PathParam("threadpool") String threadpool, @PathParam("factor") double factor) { + if (!RevolverBundle.getServiceConfig().containsKey(service)) { + return Response.status(Response.Status.BAD_REQUEST) + .entity(ImmutableMap.builder().put("service", service) + .put("threadpool", threadpool).build()).build(); + } + if(null == RevolverBundle.getServiceConfig().get(service).getThreadPoolGroupConfig()) { + return Response.status(Response.Status.BAD_REQUEST) + .entity(ImmutableMap.builder().put("service", service) + .put("threadpool", threadpool).build()).build(); + } + if(RevolverBundle.getServiceConfig().get(service).getThreadPoolGroupConfig() + .getThreadPools().stream().map(ThreadPoolConfig::getThreadPoolName).noneMatch(n -> n.equals(threadpool))) { + return Response.status(Response.Status.BAD_REQUEST) + .entity(ImmutableMap.builder().put("service", service) + .put("threadpool", threadpool).build()).build(); + } + DegradeRegistry.getInstance().addTimeoutDegrade(service +"." +threadpool, factor); + return Response.ok(ImmutableMap.builder().put("service", service) + .put("threadpool", threadpool) + .put("factor", factor).build()) + .build(); + } + + @Path("/v1/manage/api/degrade/timeout/disable/{service}/{api}") @POST @Metered @@ -192,6 +272,24 @@ public Response degradeTimeout(@PathParam("service") String service, @PathParam( } } + @Path("/v1/manage/api/degrade/threadpoolgroup/timeout/disable/{service}/{threadpool}") + @POST + @Metered + @ApiOperation(value = "Enable timeout degrade for service") + @Produces(MediaType.APPLICATION_JSON) + public Response disableDegradeServiceTimeout(@PathParam("service") String service, + @PathParam("threadpool") String threadpool) { + if(!DegradeRegistry.getInstance().getTimeoutDegradeStatus().containsKey(service +"." +threadpool)) { + return Response.status(Response.Status.BAD_REQUEST) + .entity(ImmutableMap.builder().put("service", service) + .put("threadpool", threadpool).build()).build(); + } + DegradeRegistry.getInstance().removeTimeoutDegrade(service +"." +threadpool); + return Response.ok(ImmutableMap.builder().put("service", service) + .put("threadpool", threadpool)) + .build(); + } + @Path("/v1/manage/api/degrade/status") @GET @Metered From 082c9f1123153cf4fc981792cc78d664481b8d8a Mon Sep 17 00:00:00 2001 From: phaneesh Date: Mon, 5 Aug 2019 14:23:57 +0530 Subject: [PATCH 3/3] Cleanup api paths (more readable) --- .../revolver/resource/RevolverApiManageResource.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/io/dropwizard/revolver/resource/RevolverApiManageResource.java b/src/main/java/io/dropwizard/revolver/resource/RevolverApiManageResource.java index f3d7764f..29716cf6 100644 --- a/src/main/java/io/dropwizard/revolver/resource/RevolverApiManageResource.java +++ b/src/main/java/io/dropwizard/revolver/resource/RevolverApiManageResource.java @@ -112,7 +112,7 @@ public Response disable(@PathParam("service") String service, } } - @Path("/v1/manage/api/degrade/threadpool/{service}/{api}/{factor}") + @Path("/v1/manage/api/degrade/threadpool/concurrency/{service}/{api}/{factor}") @POST @Metered @ApiOperation(value = "Enable threadpool degrade for api") @@ -163,7 +163,7 @@ public Response degradeServiceThreadPool(@PathParam("service") String service, .build(); } - @Path("/v1/manage/api/degrade/threadpool/disable/{service}/{api}") + @Path("/v1/manage/api/degrade/threadpool/concurrency/disable/{service}/{api}") @POST @Metered @ApiOperation(value = "Disable threadpool degrade for api")