Skip to content

Commit

Permalink
Use direct counter/timer instrumentationfor hystrix
Browse files Browse the repository at this point in the history
  • Loading branch information
@jkschneider
Clint Checketts authored and jkschneider committed Nov 27, 2017
1 parent 3af71a2 commit f1ef7cf
Show file tree
Hide file tree
Showing 6 changed files with 186 additions and 29 deletions.
30 changes: 30 additions & 0 deletions ...core/src/main/java/io/micrometer/core/instrument/binder/hystrix/HystrixMetricsBinder.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
package io.micrometer.core.instrument.binder.hystrix;

import com.netflix.hystrix.strategy.HystrixPlugins;
import com.netflix.hystrix.strategy.concurrency.HystrixConcurrencyStrategy;
import com.netflix.hystrix.strategy.eventnotifier.HystrixEventNotifier;
import com.netflix.hystrix.strategy.executionhook.HystrixCommandExecutionHook;
import com.netflix.hystrix.strategy.properties.HystrixPropertiesStrategy;
import io.micrometer.core.instrument.MeterRegistry;
import io.micrometer.core.instrument.binder.MeterBinder;

public class HystrixMetricsBinder implements MeterBinder {

@Override
public void bindTo(MeterRegistry registry) {
// Keeps references of existing Hystrix plugins.
HystrixEventNotifier eventNotifier = HystrixPlugins.getInstance().getEventNotifier();
HystrixPropertiesStrategy propertiesStrategy = HystrixPlugins.getInstance().getPropertiesStrategy();
HystrixCommandExecutionHook commandExecutionHook = HystrixPlugins.getInstance().getCommandExecutionHook();
HystrixConcurrencyStrategy concurrencyStrategy = HystrixPlugins.getInstance().getConcurrencyStrategy();

HystrixPlugins.reset();

// Registers existing plugins except the new MicroMeter Strategy plugin.
HystrixPlugins.getInstance().registerMetricsPublisher(new MicrometerMetricsPublisher(registry));
HystrixPlugins.getInstance().registerConcurrencyStrategy(concurrencyStrategy);
HystrixPlugins.getInstance().registerEventNotifier(eventNotifier);
HystrixPlugins.getInstance().registerPropertiesStrategy(propertiesStrategy);
HystrixPlugins.getInstance().registerCommandExecutionHook(commandExecutionHook);
}
}
155 changes: 126 additions & 29 deletions .../java/io/micrometer/core/instrument/binder/hystrix/MicrometerMetricsPublisherCommand.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,21 +16,43 @@
package io.micrometer.core.instrument.binder.hystrix;

import com.netflix.hystrix.*;
import com.netflix.hystrix.metric.HystrixCommandCompletion;
import com.netflix.hystrix.metric.HystrixCommandCompletionStream;
import com.netflix.hystrix.metric.consumer.CumulativeCommandEventCounterStream;
import com.netflix.hystrix.strategy.metrics.HystrixMetricsPublisherCommand;
import com.netflix.hystrix.util.HystrixRollingNumberEvent;
import io.micrometer.core.instrument.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import rx.functions.Action1;
import rx.functions.Func0;

import java.util.Arrays;
import java.util.List;
import java.util.concurrent.TimeUnit;
import java.util.function.ToDoubleFunction;
import java.util.stream.StreamSupport;

/**
* @author Clint Checketts
*/
public class MicrometerMetricsPublisherCommand implements HystrixMetricsPublisherCommand {
private static final Logger LOG = LoggerFactory.getLogger(MicrometerMetricsPublisherCommand.class);
private static final List<HystrixEventType> executionEvents = Arrays.asList(
HystrixEventType.EMIT,
HystrixEventType.SUCCESS,
HystrixEventType.FAILURE,
HystrixEventType.TIMEOUT,
HystrixEventType.BAD_REQUEST,
HystrixEventType.SHORT_CIRCUITED,
HystrixEventType.THREAD_POOL_REJECTED,
HystrixEventType.SEMAPHORE_REJECTED);
private static final List<HystrixEventType> fallbackEvents = Arrays.asList(
HystrixEventType.FALLBACK_EMIT,
HystrixEventType.FALLBACK_SUCCESS,
HystrixEventType.FALLBACK_FAILURE,
HystrixEventType.FALLBACK_REJECTION,
HystrixEventType.FALLBACK_MISSING);

private final MeterRegistry meterRegistry;
private final HystrixCommandMetrics metrics;
Expand All @@ -46,49 +68,124 @@ public MicrometerMetricsPublisherCommand(MeterRegistry meterRegistry, HystrixCom
this.commandKey = commandKey;
this.properties = properties;
tags = Tags.zip("group", commandGroupKey.name(), "key", commandKey.name());

//Initialize commands at zero
Counter.builder("hystrix.errors").tags(tags).register(meterRegistry);
Counter.builder("hystrix.requests").tags(tags).register(meterRegistry);
Timer.builder("hystrix.latency.execution").tags(tags).register(meterRegistry);
Timer.builder("hystrix.latency.total").tags(tags).register(meterRegistry);
executionEvents.forEach(this::getExecutionCounter);
fallbackEvents.forEach(this::getFallbackCounter);
Arrays.stream(HystrixEventType.values()).filter(e -> !executionEvents.contains(e) && !fallbackEvents.contains(e))
.forEach(this::getOtherExecutionCounter);
}

@Override
public void initialize() {
Gauge.builder("hystrix.circuit.breaker.open", circuitBreaker, c -> c.isOpen() ? 1 : 0)
.tags(tags).register(meterRegistry);

String executionName = "hystrix.execution";
String executionDescription = "Execution results. See https://github.com/Netflix/Hystrix/wiki/Metrics-and-Monitoring#command-execution-event-types-comnetflixhystrixhystrixeventtype for type definitions";
createCounter(executionName, executionDescription, HystrixRollingNumberEvent.EMIT);
createCounter(executionName, executionDescription, HystrixRollingNumberEvent.SUCCESS);
createCounter(executionName, executionDescription, HystrixRollingNumberEvent.FAILURE);
createCounter(executionName, executionDescription, HystrixRollingNumberEvent.TIMEOUT);
createCounter(executionName, executionDescription, HystrixRollingNumberEvent.BAD_REQUEST);
createCounter(executionName, executionDescription, HystrixRollingNumberEvent.SHORT_CIRCUITED);
createCounter(executionName, executionDescription, HystrixRollingNumberEvent.THREAD_POOL_REJECTED);
createCounter(executionName, executionDescription, HystrixRollingNumberEvent.SEMAPHORE_REJECTED);
HystrixCommandCompletionStream.getInstance(commandKey)
.observe()
.subscribe(hystrixCommandCompletion -> {
/*
our assumptions about latency as returned by hystrixCommandCompletion:
# a latency of >= 0 indicates that this the execution occurred.
# a latency of == -1 indicates that the execution didn't occur (default in execution result)
# a latency of < -1 indicates some clock problems.
We will only count executions, and ignore non-executions with a value of -1.
Latencies of < -1 are ignored as they will decrement the counts, and Prometheus will
take this as a reset of the counter, therefore this should be avoided by all means.
*/
long totalLatency = hystrixCommandCompletion.getTotalLatency();
if (totalLatency >= 0) {
Timer.builder("hystrix.latency.execution")
.tags(tags)
.register(meterRegistry)
.record(totalLatency, TimeUnit.MILLISECONDS);
} else if (totalLatency < -1) {
LOG.warn("received negative totalLatency, event not counted. " +
"This indicates a clock skew? {}",
hystrixCommandCompletion);
}
long executionLatency = hystrixCommandCompletion.getExecutionLatency();
if (executionLatency >= 0) {
Timer.builder("hystrix.latency.total")
.tags(tags)
.register(meterRegistry)
.record(executionLatency, TimeUnit.MILLISECONDS);
} else if (executionLatency < -1) {
LOG.warn("received negative executionLatency, event not counted. " +
"This indicates a clock skew? {}",
hystrixCommandCompletion);
}
for (HystrixEventType hystrixEventType : HystrixEventType.values()) {
int count = hystrixCommandCompletion.getEventCounts().getCount(hystrixEventType);
if (count > 0) {
switch (hystrixEventType) {
/* this list is derived from {@link HystrixCommandMetrics.HealthCounts.plus} */
case FAILURE:
case TIMEOUT:
case THREAD_POOL_REJECTED:
case SEMAPHORE_REJECTED:
Counter.builder("hystrix.errors")
.tags(tags)
.register(meterRegistry)
.increment(count);
case SUCCESS:
Counter.builder("hystrix.requests")
.tags(tags)
.register(meterRegistry)
.increment(count);

break;
}

String fallbackEventName = "hystrix.fallback";
String fallbackEventDescription = "Fallback execution results. See https://github.com/Netflix/Hystrix/wiki/Metrics-and-Monitoring#command-fallback-event-types-comnetflixhystrixhystrixeventtype for type definitions";
createCounter(fallbackEventName, fallbackEventDescription, HystrixRollingNumberEvent.FALLBACK_EMIT);
createCounter(fallbackEventName, fallbackEventDescription, HystrixRollingNumberEvent.FALLBACK_SUCCESS);
createCounter(fallbackEventName, fallbackEventDescription, HystrixRollingNumberEvent.FALLBACK_FAILURE);
createCounter(fallbackEventName, fallbackEventDescription, HystrixRollingNumberEvent.FALLBACK_REJECTION);
createCounter(fallbackEventName, fallbackEventDescription, HystrixRollingNumberEvent.FALLBACK_MISSING);
if(executionEvents.contains(hystrixEventType)) {
getExecutionCounter(hystrixEventType).increment(count);
} else if(fallbackEvents.contains(hystrixEventType)){
getFallbackCounter(hystrixEventType).increment(count);
} else {
getOtherExecutionCounter(hystrixEventType).increment(count);
}
}
}
});

String threadPool = metrics.getThreadPoolKey().name();
Gauge.builder("hystrix.threadpool.concurrent.execution.current", metrics, HystrixCommandMetrics::getCurrentConcurrentExecutionCount)
.tags(Tags.concat(tags, "threadpool", threadPool))
.register(meterRegistry);
Gauge.builder("hystrix.threadpool.concurrent.execution.rolling.max", metrics, HystrixCommandMetrics::getRollingMaxConcurrentExecutions)
.tags(Tags.concat(tags, "threadpool", threadPool))
.register(meterRegistry);

CumulativeCommandEventCounterStream.getInstance(commandKey, properties).startCachingStreamValuesIfUnstarted();
}

private void createCounter(String name, String executionDescription, HystrixRollingNumberEvent event) {
ToDoubleFunction<HystrixCommandMetrics> getCumulativeCount = m -> {
try {
return m.getCumulativeCount(event);
} catch (NoSuchFieldError error) {
LOG.error("While publishing metrics, error looking up eventType for : {}. Please check that all Hystrix versions are the same!", name);
return 0L;
}
};
private Counter getOtherExecutionCounter(HystrixEventType hystrixEventType) {
String otherEventName = "hystrix.command.other";
String otherEventDescription = "Other execution results. See https://github.com/Netflix/Hystrix/wiki/Metrics-and-Monitoring#other-command-event-types-comnetflixhystrixhystrixeventtype for type definitions";
return Counter.builder(otherEventName)
.description(otherEventDescription)
.tags(Tags.concat(tags, "event", hystrixEventType.name().toLowerCase()))
.register(meterRegistry);
}

FunctionCounter.builder(name, metrics, getCumulativeCount).description(executionDescription)
.tags(Tags.concat(tags, "event", event.name().toLowerCase()))
private Counter getFallbackCounter(HystrixEventType hystrixEventType) {
String fallbackEventName = "hystrix.fallback";
String fallbackEventDescription = "Fallback execution results. See https://github.com/Netflix/Hystrix/wiki/Metrics-and-Monitoring#command-fallback-event-types-comnetflixhystrixhystrixeventtype for type definitions";
return Counter.builder(fallbackEventName)
.description(fallbackEventDescription)
.tags(Tags.concat(tags, "event", hystrixEventType.name().toLowerCase()))
.register(meterRegistry);
}

private Counter getExecutionCounter(HystrixEventType hystrixEventType) {
String executionName = "hystrix.execution";
String executionDescription = "Execution results. See https://github.com/Netflix/Hystrix/wiki/Metrics-and-Monitoring#command-execution-event-types-comnetflixhystrixhystrixeventtype for type definitions";
return Counter.builder(executionName)
.description(executionDescription)
.tags(Tags.concat(tags, "event", hystrixEventType.name().toLowerCase()))
.register(meterRegistry);
}
}
1 change: 1 addition & 0 deletions micrometer-spring-legacy/build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,4 +75,5 @@ dependencies {
samplesCompile 'org.springframework.integration:spring-integration-java-dsl'
samplesCompile 'org.springframework.integration:spring-integration-ws'
samplesCompile 'org.springframework.integration:spring-integration-xml'
samplesCompile 'com.netflix.hystrix:hystrix-javanica:1.5.8'
}
10 changes: 10 additions & 0 deletions ...ing-legacy/src/main/java/io/micrometer/spring/autoconfigure/MetricsAutoConfiguration.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
import io.micrometer.core.instrument.MeterRegistry;
import io.micrometer.core.instrument.Metrics;
import io.micrometer.core.instrument.binder.MeterBinder;
import io.micrometer.core.instrument.binder.hystrix.HystrixMetricsBinder;
import io.micrometer.core.instrument.composite.CompositeMeterRegistry;
import io.micrometer.spring.SpringEnvironmentMeterFilter;
import io.micrometer.spring.autoconfigure.export.MetricsExporter;
Expand Down Expand Up @@ -138,4 +139,13 @@ static class MeterRegistryConfigurationSupport {
}
}
}


@Bean
@ConditionalOnClass(name = "com.netflix.hystrix.strategy.HystrixPlugins")
@ConditionalOnProperty(value = "spring.metrics.hystrix.enabled", matchIfMissing = true)
public HystrixMetricsBinder hystrixMetricsBinder() {
return new HystrixMetricsBinder();
}

}
8 changes: 8 additions & 0 deletions micrometer-spring-legacy/src/samples/java/io/micrometer/spring/samples/PrometheusSample.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,10 @@
*/
package io.micrometer.spring.samples;

import com.netflix.hystrix.contrib.javanica.aop.aspectj.HystrixCommandAspect;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.boot.builder.SpringApplicationBuilder;
import org.springframework.context.annotation.Bean;
import org.springframework.scheduling.annotation.EnableScheduling;

@SpringBootApplication(scanBasePackages = "io.micrometer.spring.samples.components")
Expand All @@ -25,4 +27,10 @@ public class PrometheusSample {
public static void main(String[] args) {
new SpringApplicationBuilder(PrometheusSample.class).profiles("prometheus").run(args);
}

@Bean
public HystrixCommandAspect hystrixAspect(){
return new HystrixCommandAspect();
}

}
11 changes: 11 additions & 0 deletions ...ing-legacy/src/samples/java/io/micrometer/spring/samples/components/PersonController.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
*/
package io.micrometer.spring.samples.components;

import com.netflix.hystrix.contrib.javanica.annotation.HystrixCommand;
import io.micrometer.core.annotation.Timed;
import io.micrometer.core.instrument.MeterRegistry;
import org.springframework.web.bind.annotation.GetMapping;
Expand All @@ -37,6 +38,7 @@ public PersonController(MeterRegistry registry) {

@GetMapping("/api/people")
@Timed(percentiles = {0.5, 0.95, 0.999}, histogram = true)
@HystrixCommand(fallbackMethod = "fallbackPeople")
public List<String> allPeople() {
try {
Thread.sleep(200);
Expand All @@ -46,6 +48,15 @@ public List<String> allPeople() {
return people;
}

/**
* Fallback for {@link PersonController#allPeople()}
* @return people
*/
@SuppressWarnings("unused")
public List<String> fallbackPeople() {
return Arrays.asList("old mike", "fallback frank");
}

@GetMapping("/api/fail")
public String fail() {
throw new RuntimeException("boom");
Expand Down

0 comments on commit f1ef7cf

Please sign in to comment.