Skip to content

Commit

Permalink
proof of concept micrometer instrumentation
Browse files Browse the repository at this point in the history 
adds metrics to the getOne catalog/timeseries endpoint
see documentation: https://micrometer.io/docs/concept
  • Loading branch information
@adamkorynta
adamkorynta committed Nov 9, 2023
1 parent d0a1d39 commit 43aead2
Show file tree
Hide file tree
Showing 4 changed files with 160 additions and 94 deletions.
3 changes: 3 additions & 0 deletions cwms-data-api/build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,9 @@ dependencies {
implementation "io.prometheus:simpleclient_dropwizard:0.15.0"
implementation "io.prometheus:simpleclient_servlet:0.15.0"

//Using OpenTelemetry implementations for Proof of Concept
implementation 'io.micrometer:micrometer-registry-otlp:latest.release'


implementation "com.fasterxml.jackson.core:jackson-databind:$JACKSON_VERSION"
implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-csv:$JACKSON_VERSION"
Expand Down
35 changes: 32 additions & 3 deletions cwms-data-api/src/main/java/cwms/cda/ApiServlet.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -85,13 +85,22 @@
import io.javalin.http.JavalinServlet;
import io.javalin.plugin.openapi.OpenApiOptions;
import io.javalin.plugin.openapi.OpenApiPlugin;
import io.micrometer.core.instrument.Clock;
import io.micrometer.core.instrument.MeterRegistry;
import io.micrometer.core.instrument.binder.jvm.ClassLoaderMetrics;
import io.micrometer.core.instrument.binder.jvm.JvmGcMetrics;
import io.micrometer.core.instrument.binder.jvm.JvmMemoryMetrics;
import io.micrometer.core.instrument.binder.jvm.JvmThreadMetrics;
import io.micrometer.core.instrument.binder.system.ProcessorMetrics;
import io.micrometer.core.instrument.binder.system.UptimeMetrics;
import io.micrometer.registry.otlp.OtlpConfig;
import io.micrometer.registry.otlp.OtlpMeterRegistry;
import io.swagger.v3.oas.models.Components;
import io.swagger.v3.oas.models.OpenAPI;
import io.swagger.v3.oas.models.Operation;
import io.swagger.v3.oas.models.PathItem;
import io.swagger.v3.oas.models.info.Info;
import io.swagger.v3.oas.models.security.SecurityRequirement;
import java.util.concurrent.TimeUnit;
import org.apache.http.entity.ContentType;
import org.jetbrains.annotations.NotNull;
import org.owasp.html.HtmlPolicyBuilder;
Expand All @@ -117,6 +126,7 @@
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import java.util.jar.Manifest;

import static io.javalin.apibuilder.ApiBuilder.*;
Expand Down Expand Up @@ -168,6 +178,7 @@ public class ApiServlet extends HttpServlet {
public static final String DEFAULT_PROVIDER = "MultipleAccessManager";

private MetricRegistry metrics;
private MeterRegistry meterRegistry;
private Meter totalRequests;

private static final long serialVersionUID = 1L;
Expand All @@ -186,9 +197,27 @@ public void destroy() {

@Override
public void init(ServletConfig config) throws ServletException {
logger.atInfo().log("Initializing CWMS Data API Version: " + obtainFullVersion(config));
String fullVersion = obtainFullVersion(config);
logger.atInfo().log("Initializing %s Version: %s", new Object[]{APPLICATION_TITLE, fullVersion});
metrics = (MetricRegistry)config.getServletContext()
.getAttribute(MetricsServlet.METRICS_REGISTRY);
OtlpConfig otlpConfig = key -> {
//From the docs: https://micrometer.io/docs/registry/otlp
//If you instead of returning null, bind it to a property source (e.g.: a simple Map can work),
//you can override the default configuration through properties.
return null;
};

meterRegistry = new OtlpMeterRegistry(otlpConfig, Clock.SYSTEM);
meterRegistry.config().commonTags("application", APPLICATION_TITLE);
meterRegistry.config().commonTags("application-version", fullVersion);
new ClassLoaderMetrics().bindTo(meterRegistry);
new JvmMemoryMetrics().bindTo(meterRegistry);
new JvmGcMetrics().bindTo(meterRegistry);
new JvmThreadMetrics().bindTo(meterRegistry);
new UptimeMetrics().bindTo(meterRegistry);
new ProcessorMetrics().bindTo(meterRegistry);

totalRequests = metrics.meter("cwms.dataapi.total_requests");
super.init(config);
}
Expand Down Expand Up @@ -384,7 +413,7 @@ protected void configureRoutes() {
cdaCrudCache("/ratings/{rating-id}",
new RatingController(metrics), requiredRoles,5, TimeUnit.MINUTES);
cdaCrudCache("/catalog/{dataset}",
new CatalogController(metrics), requiredRoles,5, TimeUnit.MINUTES);
new CatalogController(meterRegistry), requiredRoles,5, TimeUnit.MINUTES);
cdaCrudCache("/basins/{basin-id}",
new BasinController(metrics), requiredRoles,5, TimeUnit.MINUTES);
cdaCrudCache("/blobs/{blob-id}",
Expand Down
176 changes: 87 additions & 89 deletions cwms-data-api/src/main/java/cwms/cda/api/CatalogController.java
View file
Original file line number Diff line number Diff line change
@@ -1,36 +1,29 @@
/*
* MIT License
*
* Copyright (c) 2023 Hydrologic Engineering Center
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/

package cwms.cda.api;

import static com.codahale.metrics.MetricRegistry.name;
import static cwms.cda.api.Controllers.ACCEPT;
import static cwms.cda.api.Controllers.BOUNDING_OFFICE_LIKE;
import static cwms.cda.api.Controllers.CURSOR;
import static cwms.cda.api.Controllers.GET_ONE;
import static cwms.cda.api.Controllers.LIKE;
import static cwms.cda.api.Controllers.LOCATIONS;
import static cwms.cda.api.Controllers.LOCATION_CATEGORY_LIKE;
import static cwms.cda.api.Controllers.LOCATION_CATEGORY_LIKE2;
import static cwms.cda.api.Controllers.LOCATION_GROUP_LIKE;
import static cwms.cda.api.Controllers.LOCATION_GROUP_LIKE2;
import static cwms.cda.api.Controllers.OFFICE;
import static cwms.cda.api.Controllers.PAGE;
import static cwms.cda.api.Controllers.PAGESIZE2;
import static cwms.cda.api.Controllers.PAGESIZE3;
import static cwms.cda.api.Controllers.PAGE_SIZE;
import static cwms.cda.api.Controllers.RESULTS;
import static cwms.cda.api.Controllers.SIZE;
import static cwms.cda.api.Controllers.STATUS_200;
import static cwms.cda.api.Controllers.TIMESERIES;
import static cwms.cda.api.Controllers.TIMESERIESCATEGORYLIKE2;
import static cwms.cda.api.Controllers.TIMESERIES_CATEGORY_LIKE;
import static cwms.cda.api.Controllers.TIMESERIES_GROUP_LIKE;
import static cwms.cda.api.Controllers.TIMESERIES_GROUP_LIKE2;
import static cwms.cda.api.Controllers.UNITSYSTEM2;
import static cwms.cda.api.Controllers.UNIT_SYSTEM;
import static cwms.cda.api.Controllers.queryParamAsClass;

import com.codahale.metrics.Histogram;
import com.codahale.metrics.MetricRegistry;
import com.codahale.metrics.Timer;
import cwms.cda.api.enums.UnitSystem;
import cwms.cda.api.errors.CdaError;
import cwms.cda.data.dao.JooqDao;
Expand All @@ -49,31 +42,37 @@
import io.javalin.plugin.openapi.annotations.OpenApiContent;
import io.javalin.plugin.openapi.annotations.OpenApiParam;
import io.javalin.plugin.openapi.annotations.OpenApiResponse;
import java.util.logging.Logger;
import io.micrometer.core.instrument.DistributionSummary;
import io.micrometer.core.instrument.MeterRegistry;
import org.jetbrains.annotations.NotNull;
import org.jooq.DSLContext;
import org.owasp.html.PolicyFactory;

import java.util.logging.Logger;

import static com.codahale.metrics.MetricRegistry.name;
import static cwms.cda.api.Controllers.*;

public class CatalogController implements CrudHandler {

private static final Logger logger = Logger.getLogger(CatalogController.class.getName());
private static final String TAG = "Catalog-Beta";


private final MetricRegistry metrics;
private final MeterRegistry metrics;

private final Histogram requestResultSize;
private final DistributionSummary requestResultSize;

private final int defaultPageSize = 500;

public CatalogController(MetricRegistry metrics) {
public CatalogController(MeterRegistry metrics) {
this.metrics = metrics;
String className = this.getClass().getName();

requestResultSize = this.metrics.histogram((name(className, RESULTS, SIZE)));
requestResultSize = metrics.summary(className, RESULTS, SIZE);
}

private Timer.Context markAndTime(String subject) {
private io.micrometer.core.instrument.Timer markAndTime(String subject) {
return Controllers.markAndTime(metrics, getClass().getName(), subject);
}

Expand Down Expand Up @@ -185,71 +184,70 @@ public void getAll(Context ctx) {
@Override
public void getOne(Context ctx, @NotNull String dataSet) {

try (
final Timer.Context timeContext = markAndTime(GET_ONE);
DSLContext dsl = JooqDao.getDslContext(ctx)
) {
markAndTime(GET_ONE).record(() -> {
try (DSLContext dsl = JooqDao.getDslContext(ctx)) {

String valDataSet =
((PolicyFactory) ctx.appAttribute("PolicyFactory")).sanitize(dataSet);
String valDataSet =
((PolicyFactory) ctx.appAttribute("PolicyFactory")).sanitize(dataSet);

String cursor = queryParamAsClass(ctx, new String[]{PAGE, CURSOR},
String.class, "", metrics, name(CatalogController.class.getName(), GET_ONE));
String cursor = queryParamAsClass(ctx, new String[]{PAGE, CURSOR},
String.class, "", metrics, name(CatalogController.class.getName(), GET_ONE));

int pageSize = queryParamAsClass(ctx, new String[]{PAGE_SIZE, PAGESIZE3,
PAGESIZE2}, Integer.class, defaultPageSize, metrics,
name(CatalogController.class.getName(), GET_ONE));
int pageSize = queryParamAsClass(ctx, new String[]{PAGE_SIZE, PAGESIZE3,
PAGESIZE2}, Integer.class, defaultPageSize, metrics,
name(CatalogController.class.getName(), GET_ONE));

String unitSystem = queryParamAsClass(ctx,
new String[]{UNIT_SYSTEM, UNITSYSTEM2},
String.class, UnitSystem.SI.getValue(), metrics,
name(CatalogController.class.getName(), GET_ONE));
String unitSystem = queryParamAsClass(ctx,
new String[]{UNIT_SYSTEM, UNITSYSTEM2},
String.class, UnitSystem.SI.getValue(), metrics,
name(CatalogController.class.getName(), GET_ONE));

String office = ctx.queryParamAsClass(OFFICE, String.class).allowNullable()
.check(Office::validOfficeCanNull, "Invalid office provided")
.get();
String office = ctx.queryParamAsClass(OFFICE, String.class).allowNullable()
.check(Office::validOfficeCanNull, "Invalid office provided")
.get();

String like = ctx.queryParamAsClass(LIKE, String.class).getOrDefault(".*");
String like = ctx.queryParamAsClass(LIKE, String.class).getOrDefault(".*");

String tsCategoryLike = queryParamAsClass(ctx, new String[]{TIMESERIES_CATEGORY_LIKE, TIMESERIESCATEGORYLIKE2},
String.class, null, metrics, name(CatalogController.class.getName(), GET_ONE));
String tsCategoryLike = queryParamAsClass(ctx, new String[]{TIMESERIES_CATEGORY_LIKE, TIMESERIESCATEGORYLIKE2},
String.class, null, metrics, name(CatalogController.class.getName(), GET_ONE));

String tsGroupLike = queryParamAsClass(ctx, new String[]{TIMESERIES_GROUP_LIKE, TIMESERIES_GROUP_LIKE2},
String.class, null, metrics, name(CatalogController.class.getName(), GET_ONE));
String tsGroupLike = queryParamAsClass(ctx, new String[]{TIMESERIES_GROUP_LIKE, TIMESERIES_GROUP_LIKE2},
String.class, null, metrics, name(CatalogController.class.getName(), GET_ONE));

String locCategoryLike = queryParamAsClass(ctx, new String[]{LOCATION_CATEGORY_LIKE, LOCATION_CATEGORY_LIKE2},
String.class, null, metrics, name(CatalogController.class.getName(), GET_ONE));
String locCategoryLike = queryParamAsClass(ctx, new String[]{LOCATION_CATEGORY_LIKE, LOCATION_CATEGORY_LIKE2},
String.class, null, metrics, name(CatalogController.class.getName(), GET_ONE));

String locGroupLike = queryParamAsClass(ctx, new String[]{LOCATION_GROUP_LIKE, LOCATION_GROUP_LIKE2},
String.class, null, metrics, name(CatalogController.class.getName(), GET_ONE));
String locGroupLike = queryParamAsClass(ctx, new String[]{LOCATION_GROUP_LIKE, LOCATION_GROUP_LIKE2},
String.class, null, metrics, name(CatalogController.class.getName(), GET_ONE));

String boundingOfficeLike = queryParamAsClass(ctx, new String[]{BOUNDING_OFFICE_LIKE},
String.class, null, metrics, name(CatalogController.class.getName(), GET_ONE));
String boundingOfficeLike = queryParamAsClass(ctx, new String[]{BOUNDING_OFFICE_LIKE},
String.class, null, metrics, name(CatalogController.class.getName(), GET_ONE));

String acceptHeader = ctx.header(ACCEPT);
ContentType contentType = Formats.parseHeaderAndQueryParm(acceptHeader, null);
Catalog cat = null;
if (TIMESERIES.equalsIgnoreCase(valDataSet)) {
TimeSeriesDao tsDao = new TimeSeriesDaoImpl(dsl);
cat = tsDao.getTimeSeriesCatalog(cursor, pageSize, office, like, locCategoryLike,
locGroupLike, tsCategoryLike, tsGroupLike, boundingOfficeLike);
} else if (LOCATIONS.equalsIgnoreCase(valDataSet)) {
LocationsDao dao = new LocationsDaoImpl(dsl);
cat = dao.getLocationCatalog(cursor, pageSize, unitSystem, office, like,
locCategoryLike, locGroupLike, boundingOfficeLike);
}
if (cat != null) {
String data = Formats.format(contentType, cat);
ctx.result(data).contentType(contentType.toString());
requestResultSize.update(data.length());
} else {
final CdaError re = new CdaError("Cannot create catalog of requested "
+ "information");

logger.info(() -> re + "with url:" + ctx.fullUrl());
ctx.json(re).status(HttpCode.NOT_FOUND);
String acceptHeader = ctx.header(ACCEPT);
ContentType contentType = Formats.parseHeaderAndQueryParm(acceptHeader, null);
Catalog cat = null;
if (TIMESERIES.equalsIgnoreCase(valDataSet)) {
TimeSeriesDao tsDao = new TimeSeriesDaoImpl(dsl);
cat = tsDao.getTimeSeriesCatalog(cursor, pageSize, office, like, locCategoryLike,
locGroupLike, tsCategoryLike, tsGroupLike, boundingOfficeLike);
} else if (LOCATIONS.equalsIgnoreCase(valDataSet)) {
LocationsDao dao = new LocationsDaoImpl(dsl);
cat = dao.getLocationCatalog(cursor, pageSize, unitSystem, office, like,
locCategoryLike, locGroupLike, boundingOfficeLike);
}
if (cat != null) {
String data = Formats.format(contentType, cat);
ctx.result(data).contentType(contentType.toString());
requestResultSize.record(data.length());
} else {
final CdaError re = new CdaError("Cannot create catalog of requested "
+ "information");

logger.info(() -> re + "with url:" + ctx.fullUrl());
ctx.json(re).status(HttpCode.NOT_FOUND);
}
}
}
});
}

@OpenApi(tags = {"Catalog"}, ignore = true)
Expand Down
40 changes: 38 additions & 2 deletions cwms-data-api/src/main/java/cwms/cda/api/Controllers.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,14 +24,15 @@

package cwms.cda.api;

import static com.codahale.metrics.MetricRegistry.name;

import com.codahale.metrics.Meter;
import com.codahale.metrics.MetricRegistry;
import com.codahale.metrics.Timer;
import cwms.cda.data.dao.JooqDao;
import io.javalin.core.validation.JavalinValidation;
import io.javalin.core.validation.Validator;
import io.micrometer.core.instrument.MeterRegistry;

import static com.codahale.metrics.MetricRegistry.name;


public final class Controllers {
Expand Down Expand Up @@ -166,6 +167,12 @@ public static Timer.Context markAndTime(MetricRegistry registry, String classNam
return timer.time();
}

public static io.micrometer.core.instrument.Timer markAndTime(MeterRegistry registry, String className,
String subject) {
registry.counter(name(className, subject, COUNT)).increment();
return registry.timer(name(className, subject, TIME));
}

/**
* Returns the first matching query param or the provided default value if no match is found.
*
Expand Down Expand Up @@ -244,6 +251,35 @@ public static <T> T queryParamAsClass(io.javalin.http.Context ctx, String[] name
return retval;
}

public static <T> T queryParamAsClass(io.javalin.http.Context ctx, String[] names,
Class<T> clazz, T defaultValue, MeterRegistry metrics,
String className) {
T retval = null;

Validator<T> validator = ctx.queryParamAsClass(names[0], clazz);
if (validator.hasValue()) {
retval = validator.get();
metrics.counter(name(className, "correct")).increment();
} else {
for (int i = 1; i < names.length; i++) {
validator = ctx.queryParamAsClass(names[i], clazz);
if (validator.hasValue()) {
retval = validator.get();
metrics.counter(name(className, "deprecated")).increment();
break;
}
}

if (retval == null) {
retval = defaultValue;
metrics.counter(name(className, "default")).increment();
}

}

return retval;
}

public static JooqDao.DeleteMethod getDeleteMethod(String input) {
JooqDao.DeleteMethod retval = null;

Expand Down

0 comments on commit 43aead2

Please sign in to comment.