diff --git a/micrometer-core/src/main/java/io/micrometer/core/instrument/binder/hystrix/HystrixMetricsBinder.java b/micrometer-core/src/main/java/io/micrometer/core/instrument/binder/hystrix/HystrixMetricsBinder.java new file mode 100644 index 0000000000..c979f34c52 --- /dev/null +++ b/micrometer-core/src/main/java/io/micrometer/core/instrument/binder/hystrix/HystrixMetricsBinder.java @@ -0,0 +1,30 @@ +package io.micrometer.core.instrument.binder.hystrix; + +import com.netflix.hystrix.strategy.HystrixPlugins; +import com.netflix.hystrix.strategy.concurrency.HystrixConcurrencyStrategy; +import com.netflix.hystrix.strategy.eventnotifier.HystrixEventNotifier; +import com.netflix.hystrix.strategy.executionhook.HystrixCommandExecutionHook; +import com.netflix.hystrix.strategy.properties.HystrixPropertiesStrategy; +import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.binder.MeterBinder; + +public class HystrixMetricsBinder implements MeterBinder { + + @Override + public void bindTo(MeterRegistry registry) { + // Keeps references of existing Hystrix plugins. + HystrixEventNotifier eventNotifier = HystrixPlugins.getInstance().getEventNotifier(); + HystrixPropertiesStrategy propertiesStrategy = HystrixPlugins.getInstance().getPropertiesStrategy(); + HystrixCommandExecutionHook commandExecutionHook = HystrixPlugins.getInstance().getCommandExecutionHook(); + HystrixConcurrencyStrategy concurrencyStrategy = HystrixPlugins.getInstance().getConcurrencyStrategy(); + + HystrixPlugins.reset(); + + // Registers existing plugins except the new MicroMeter Strategy plugin. + HystrixPlugins.getInstance().registerMetricsPublisher(new MicrometerMetricsPublisher(registry)); + HystrixPlugins.getInstance().registerConcurrencyStrategy(concurrencyStrategy); + HystrixPlugins.getInstance().registerEventNotifier(eventNotifier); + HystrixPlugins.getInstance().registerPropertiesStrategy(propertiesStrategy); + HystrixPlugins.getInstance().registerCommandExecutionHook(commandExecutionHook); + } +} diff --git a/micrometer-core/src/main/java/io/micrometer/core/instrument/binder/hystrix/MicrometerMetricsPublisherCommand.java b/micrometer-core/src/main/java/io/micrometer/core/instrument/binder/hystrix/MicrometerMetricsPublisherCommand.java index ebbaa2a7ae..fa88de5d49 100644 --- a/micrometer-core/src/main/java/io/micrometer/core/instrument/binder/hystrix/MicrometerMetricsPublisherCommand.java +++ b/micrometer-core/src/main/java/io/micrometer/core/instrument/binder/hystrix/MicrometerMetricsPublisherCommand.java @@ -16,21 +16,43 @@ package io.micrometer.core.instrument.binder.hystrix; import com.netflix.hystrix.*; +import com.netflix.hystrix.metric.HystrixCommandCompletion; +import com.netflix.hystrix.metric.HystrixCommandCompletionStream; import com.netflix.hystrix.metric.consumer.CumulativeCommandEventCounterStream; import com.netflix.hystrix.strategy.metrics.HystrixMetricsPublisherCommand; import com.netflix.hystrix.util.HystrixRollingNumberEvent; import io.micrometer.core.instrument.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import rx.functions.Action1; +import rx.functions.Func0; +import java.util.Arrays; import java.util.List; +import java.util.concurrent.TimeUnit; import java.util.function.ToDoubleFunction; +import java.util.stream.StreamSupport; /** * @author Clint Checketts */ public class MicrometerMetricsPublisherCommand implements HystrixMetricsPublisherCommand { private static final Logger LOG = LoggerFactory.getLogger(MicrometerMetricsPublisherCommand.class); + private static final List executionEvents = Arrays.asList( + HystrixEventType.EMIT, + HystrixEventType.SUCCESS, + HystrixEventType.FAILURE, + HystrixEventType.TIMEOUT, + HystrixEventType.BAD_REQUEST, + HystrixEventType.SHORT_CIRCUITED, + HystrixEventType.THREAD_POOL_REJECTED, + HystrixEventType.SEMAPHORE_REJECTED); + private static final List fallbackEvents = Arrays.asList( + HystrixEventType.FALLBACK_EMIT, + HystrixEventType.FALLBACK_SUCCESS, + HystrixEventType.FALLBACK_FAILURE, + HystrixEventType.FALLBACK_REJECTION, + HystrixEventType.FALLBACK_MISSING); private final MeterRegistry meterRegistry; private final HystrixCommandMetrics metrics; @@ -46,6 +68,16 @@ public MicrometerMetricsPublisherCommand(MeterRegistry meterRegistry, HystrixCom this.commandKey = commandKey; this.properties = properties; tags = Tags.zip("group", commandGroupKey.name(), "key", commandKey.name()); + + //Initialize commands at zero + Counter.builder("hystrix.errors").tags(tags).register(meterRegistry); + Counter.builder("hystrix.requests").tags(tags).register(meterRegistry); + Timer.builder("hystrix.latency.execution").tags(tags).register(meterRegistry); + Timer.builder("hystrix.latency.total").tags(tags).register(meterRegistry); + executionEvents.forEach(this::getExecutionCounter); + fallbackEvents.forEach(this::getFallbackCounter); + Arrays.stream(HystrixEventType.values()).filter(e -> !executionEvents.contains(e) && !fallbackEvents.contains(e)) + .forEach(this::getOtherExecutionCounter); } @Override @@ -53,42 +85,107 @@ public void initialize() { Gauge.builder("hystrix.circuit.breaker.open", circuitBreaker, c -> c.isOpen() ? 1 : 0) .tags(tags).register(meterRegistry); - String executionName = "hystrix.execution"; - String executionDescription = "Execution results. See https://github.com/Netflix/Hystrix/wiki/Metrics-and-Monitoring#command-execution-event-types-comnetflixhystrixhystrixeventtype for type definitions"; - createCounter(executionName, executionDescription, HystrixRollingNumberEvent.EMIT); - createCounter(executionName, executionDescription, HystrixRollingNumberEvent.SUCCESS); - createCounter(executionName, executionDescription, HystrixRollingNumberEvent.FAILURE); - createCounter(executionName, executionDescription, HystrixRollingNumberEvent.TIMEOUT); - createCounter(executionName, executionDescription, HystrixRollingNumberEvent.BAD_REQUEST); - createCounter(executionName, executionDescription, HystrixRollingNumberEvent.SHORT_CIRCUITED); - createCounter(executionName, executionDescription, HystrixRollingNumberEvent.THREAD_POOL_REJECTED); - createCounter(executionName, executionDescription, HystrixRollingNumberEvent.SEMAPHORE_REJECTED); + HystrixCommandCompletionStream.getInstance(commandKey) + .observe() + .subscribe(hystrixCommandCompletion -> { + /* + our assumptions about latency as returned by hystrixCommandCompletion: + # a latency of >= 0 indicates that this the execution occurred. + # a latency of == -1 indicates that the execution didn't occur (default in execution result) + # a latency of < -1 indicates some clock problems. + We will only count executions, and ignore non-executions with a value of -1. + Latencies of < -1 are ignored as they will decrement the counts, and Prometheus will + take this as a reset of the counter, therefore this should be avoided by all means. + */ + long totalLatency = hystrixCommandCompletion.getTotalLatency(); + if (totalLatency >= 0) { + Timer.builder("hystrix.latency.execution") + .tags(tags) + .register(meterRegistry) + .record(totalLatency, TimeUnit.MILLISECONDS); + } else if (totalLatency < -1) { + LOG.warn("received negative totalLatency, event not counted. " + + "This indicates a clock skew? {}", + hystrixCommandCompletion); + } + long executionLatency = hystrixCommandCompletion.getExecutionLatency(); + if (executionLatency >= 0) { + Timer.builder("hystrix.latency.total") + .tags(tags) + .register(meterRegistry) + .record(executionLatency, TimeUnit.MILLISECONDS); + } else if (executionLatency < -1) { + LOG.warn("received negative executionLatency, event not counted. " + + "This indicates a clock skew? {}", + hystrixCommandCompletion); + } + for (HystrixEventType hystrixEventType : HystrixEventType.values()) { + int count = hystrixCommandCompletion.getEventCounts().getCount(hystrixEventType); + if (count > 0) { + switch (hystrixEventType) { + /* this list is derived from {@link HystrixCommandMetrics.HealthCounts.plus} */ + case FAILURE: + case TIMEOUT: + case THREAD_POOL_REJECTED: + case SEMAPHORE_REJECTED: + Counter.builder("hystrix.errors") + .tags(tags) + .register(meterRegistry) + .increment(count); + case SUCCESS: + Counter.builder("hystrix.requests") + .tags(tags) + .register(meterRegistry) + .increment(count); + break; + } - String fallbackEventName = "hystrix.fallback"; - String fallbackEventDescription = "Fallback execution results. See https://github.com/Netflix/Hystrix/wiki/Metrics-and-Monitoring#command-fallback-event-types-comnetflixhystrixhystrixeventtype for type definitions"; - createCounter(fallbackEventName, fallbackEventDescription, HystrixRollingNumberEvent.FALLBACK_EMIT); - createCounter(fallbackEventName, fallbackEventDescription, HystrixRollingNumberEvent.FALLBACK_SUCCESS); - createCounter(fallbackEventName, fallbackEventDescription, HystrixRollingNumberEvent.FALLBACK_FAILURE); - createCounter(fallbackEventName, fallbackEventDescription, HystrixRollingNumberEvent.FALLBACK_REJECTION); - createCounter(fallbackEventName, fallbackEventDescription, HystrixRollingNumberEvent.FALLBACK_MISSING); + if(executionEvents.contains(hystrixEventType)) { + getExecutionCounter(hystrixEventType).increment(count); + } else if(fallbackEvents.contains(hystrixEventType)){ + getFallbackCounter(hystrixEventType).increment(count); + } else { + getOtherExecutionCounter(hystrixEventType).increment(count); + } + } + } + }); + + String threadPool = metrics.getThreadPoolKey().name(); + Gauge.builder("hystrix.threadpool.concurrent.execution.current", metrics, HystrixCommandMetrics::getCurrentConcurrentExecutionCount) + .tags(Tags.concat(tags, "threadpool", threadPool)) + .register(meterRegistry); + Gauge.builder("hystrix.threadpool.concurrent.execution.rolling.max", metrics, HystrixCommandMetrics::getRollingMaxConcurrentExecutions) + .tags(Tags.concat(tags, "threadpool", threadPool)) + .register(meterRegistry); - CumulativeCommandEventCounterStream.getInstance(commandKey, properties).startCachingStreamValuesIfUnstarted(); } - private void createCounter(String name, String executionDescription, HystrixRollingNumberEvent event) { - ToDoubleFunction getCumulativeCount = m -> { - try { - return m.getCumulativeCount(event); - } catch (NoSuchFieldError error) { - LOG.error("While publishing metrics, error looking up eventType for : {}. Please check that all Hystrix versions are the same!", name); - return 0L; - } - }; + private Counter getOtherExecutionCounter(HystrixEventType hystrixEventType) { + String otherEventName = "hystrix.command.other"; + String otherEventDescription = "Other execution results. See https://github.com/Netflix/Hystrix/wiki/Metrics-and-Monitoring#other-command-event-types-comnetflixhystrixhystrixeventtype for type definitions"; + return Counter.builder(otherEventName) + .description(otherEventDescription) + .tags(Tags.concat(tags, "event", hystrixEventType.name().toLowerCase())) + .register(meterRegistry); + } - FunctionCounter.builder(name, metrics, getCumulativeCount).description(executionDescription) - .tags(Tags.concat(tags, "event", event.name().toLowerCase())) + private Counter getFallbackCounter(HystrixEventType hystrixEventType) { + String fallbackEventName = "hystrix.fallback"; + String fallbackEventDescription = "Fallback execution results. See https://github.com/Netflix/Hystrix/wiki/Metrics-and-Monitoring#command-fallback-event-types-comnetflixhystrixhystrixeventtype for type definitions"; + return Counter.builder(fallbackEventName) + .description(fallbackEventDescription) + .tags(Tags.concat(tags, "event", hystrixEventType.name().toLowerCase())) .register(meterRegistry); } + private Counter getExecutionCounter(HystrixEventType hystrixEventType) { + String executionName = "hystrix.execution"; + String executionDescription = "Execution results. See https://github.com/Netflix/Hystrix/wiki/Metrics-and-Monitoring#command-execution-event-types-comnetflixhystrixhystrixeventtype for type definitions"; + return Counter.builder(executionName) + .description(executionDescription) + .tags(Tags.concat(tags, "event", hystrixEventType.name().toLowerCase())) + .register(meterRegistry); + } } diff --git a/micrometer-spring-legacy/build.gradle b/micrometer-spring-legacy/build.gradle index 7837e5d84c..2e6f1ab743 100644 --- a/micrometer-spring-legacy/build.gradle +++ b/micrometer-spring-legacy/build.gradle @@ -75,4 +75,5 @@ dependencies { samplesCompile 'org.springframework.integration:spring-integration-java-dsl' samplesCompile 'org.springframework.integration:spring-integration-ws' samplesCompile 'org.springframework.integration:spring-integration-xml' + samplesCompile 'com.netflix.hystrix:hystrix-javanica:1.5.8' } diff --git a/micrometer-spring-legacy/src/main/java/io/micrometer/spring/autoconfigure/MetricsAutoConfiguration.java b/micrometer-spring-legacy/src/main/java/io/micrometer/spring/autoconfigure/MetricsAutoConfiguration.java index d5d244b52b..f1f264a827 100644 --- a/micrometer-spring-legacy/src/main/java/io/micrometer/spring/autoconfigure/MetricsAutoConfiguration.java +++ b/micrometer-spring-legacy/src/main/java/io/micrometer/spring/autoconfigure/MetricsAutoConfiguration.java @@ -19,6 +19,7 @@ import io.micrometer.core.instrument.MeterRegistry; import io.micrometer.core.instrument.Metrics; import io.micrometer.core.instrument.binder.MeterBinder; +import io.micrometer.core.instrument.binder.hystrix.HystrixMetricsBinder; import io.micrometer.core.instrument.composite.CompositeMeterRegistry; import io.micrometer.spring.SpringEnvironmentMeterFilter; import io.micrometer.spring.autoconfigure.export.MetricsExporter; @@ -138,4 +139,13 @@ static class MeterRegistryConfigurationSupport { } } } + + + @Bean + @ConditionalOnClass(name = "com.netflix.hystrix.strategy.HystrixPlugins") + @ConditionalOnProperty(value = "spring.metrics.hystrix.enabled", matchIfMissing = true) + public HystrixMetricsBinder hystrixMetricsBinder() { + return new HystrixMetricsBinder(); + } + } diff --git a/micrometer-spring-legacy/src/samples/java/io/micrometer/spring/samples/PrometheusSample.java b/micrometer-spring-legacy/src/samples/java/io/micrometer/spring/samples/PrometheusSample.java index 364592db85..392c206165 100644 --- a/micrometer-spring-legacy/src/samples/java/io/micrometer/spring/samples/PrometheusSample.java +++ b/micrometer-spring-legacy/src/samples/java/io/micrometer/spring/samples/PrometheusSample.java @@ -15,8 +15,10 @@ */ package io.micrometer.spring.samples; +import com.netflix.hystrix.contrib.javanica.aop.aspectj.HystrixCommandAspect; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.builder.SpringApplicationBuilder; +import org.springframework.context.annotation.Bean; import org.springframework.scheduling.annotation.EnableScheduling; @SpringBootApplication(scanBasePackages = "io.micrometer.spring.samples.components") @@ -25,4 +27,10 @@ public class PrometheusSample { public static void main(String[] args) { new SpringApplicationBuilder(PrometheusSample.class).profiles("prometheus").run(args); } + + @Bean + public HystrixCommandAspect hystrixAspect(){ + return new HystrixCommandAspect(); + } + } diff --git a/micrometer-spring-legacy/src/samples/java/io/micrometer/spring/samples/components/PersonController.java b/micrometer-spring-legacy/src/samples/java/io/micrometer/spring/samples/components/PersonController.java index d7c591f3ae..deb49e86e9 100644 --- a/micrometer-spring-legacy/src/samples/java/io/micrometer/spring/samples/components/PersonController.java +++ b/micrometer-spring-legacy/src/samples/java/io/micrometer/spring/samples/components/PersonController.java @@ -15,6 +15,7 @@ */ package io.micrometer.spring.samples.components; +import com.netflix.hystrix.contrib.javanica.annotation.HystrixCommand; import io.micrometer.core.annotation.Timed; import io.micrometer.core.instrument.MeterRegistry; import org.springframework.web.bind.annotation.GetMapping; @@ -37,6 +38,7 @@ public PersonController(MeterRegistry registry) { @GetMapping("/api/people") @Timed(percentiles = {0.5, 0.95, 0.999}, histogram = true) + @HystrixCommand(fallbackMethod = "fallbackPeople") public List allPeople() { try { Thread.sleep(200); @@ -46,6 +48,15 @@ public List allPeople() { return people; } + /** + * Fallback for {@link PersonController#allPeople()} + * @return people + */ + @SuppressWarnings("unused") + public List fallbackPeople() { + return Arrays.asList("old mike", "fallback frank"); + } + @GetMapping("/api/fail") public String fail() { throw new RuntimeException("boom");