From 9b0617e3a5f76def61dcd843e3e7caa7f25f7336 Mon Sep 17 00:00:00 2001 From: Jason Thomas jjthom3 Date: Fri, 25 Oct 2024 13:58:00 +0000 Subject: [PATCH] Removed ThreadConfigurableLogger class for upcoming SLF4J change --- core/common-util/pom.xml | 4 + microservices/services/query-metric | 2 +- warehouse/query-core/pom.xml | 4 + .../query/index/lookup/RangeStream.java | 6 +- .../jexl/lookups/BoundedRangeIndexLookup.java | 6 +- .../query/jexl/lookups/RegexIndexLookup.java | 4 +- .../visitors/ExpandMultiNormalizedTerms.java | 2 +- .../UnfieldedIndexExpansionVisitor.java | 2 +- .../query/planner/DefaultQueryPlanner.java | 10 +- .../MetadataHelperQueryModelProvider.java | 2 +- .../planner/ThreadedRangeBundlerIterator.java | 2 +- .../query/tables/ShardQueryLogic.java | 5 +- web-services/metrics/pom.xml | 4 + .../query/map/SimpleQueryGeometryHandler.java | 2 +- .../metrics/ShardTableQueryMetricHandler.java | 226 +++++++++--------- 15 files changed, 142 insertions(+), 139 deletions(-) diff --git a/core/common-util/pom.xml b/core/common-util/pom.xml index d755fbf95a..e31de94740 100644 --- a/core/common-util/pom.xml +++ b/core/common-util/pom.xml @@ -37,6 +37,10 @@ + + org.slf4j + slf4j-api + diff --git a/microservices/services/query-metric b/microservices/services/query-metric index 8281bfc5d3..c16ecbc766 160000 --- a/microservices/services/query-metric +++ b/microservices/services/query-metric @@ -1 +1 @@ -Subproject commit 8281bfc5d3a608974e131ea304b325200fa7ff8e +Subproject commit c16ecbc766db2540826167d95645194b1d9cb114 diff --git a/warehouse/query-core/pom.xml b/warehouse/query-core/pom.xml index 04f5f2ff60..b4d9371d8f 100644 --- a/warehouse/query-core/pom.xml +++ b/warehouse/query-core/pom.xml @@ -205,6 +205,10 @@ org.powermock powermock-module-junit4 + + org.slf4j + slf4j-api + org.springframework spring-beans diff --git a/warehouse/query-core/src/main/java/datawave/query/index/lookup/RangeStream.java b/warehouse/query-core/src/main/java/datawave/query/index/lookup/RangeStream.java index b16a5bf583..c21d20eacb 100644 --- a/warehouse/query-core/src/main/java/datawave/query/index/lookup/RangeStream.java +++ b/warehouse/query-core/src/main/java/datawave/query/index/lookup/RangeStream.java @@ -544,7 +544,7 @@ public ScannerStream visit(ASTEQNode node, Object data) { return ScannerStream.unindexed(node); } } catch (TableNotFoundException e) { - log.error(String.valueOf(e)); + log.error("", e); throw new RuntimeException(e); } log.debug("{\"" + fieldName + "\": \"" + literal + "\"} is not an observed field."); @@ -617,7 +617,7 @@ public ScannerStream visit(ASTEQNode node, Object data) { return ScannerStream.initialized(scannerSession, entryParser, node); } catch (Exception e) { - log.error(String.valueOf(e)); + log.error("", e); throw new RuntimeException(e); } } @@ -673,7 +673,7 @@ public Object visit(ASTERNode node, Object data) { return ScannerStream.unknownField(node); } } catch (TableNotFoundException e) { - log.error(String.valueOf(e)); + log.error("", e); throw new RuntimeException(e); } diff --git a/warehouse/query-core/src/main/java/datawave/query/jexl/lookups/BoundedRangeIndexLookup.java b/warehouse/query-core/src/main/java/datawave/query/jexl/lookups/BoundedRangeIndexLookup.java index 9939581aba..11c1bdab18 100644 --- a/warehouse/query-core/src/main/java/datawave/query/jexl/lookups/BoundedRangeIndexLookup.java +++ b/warehouse/query-core/src/main/java/datawave/query/jexl/lookups/BoundedRangeIndexLookup.java @@ -122,7 +122,7 @@ public synchronized void submit() { range = new Range(startKey, true, endKey, literalRange.isUpperInclusive()); } catch (IllegalArgumentException e) { QueryException qe = new QueryException(DatawaveErrorCode.RANGE_CREATE_ERROR, e, MessageFormat.format("{0}", this.literalRange)); - log.debug(String.valueOf(qe)); + log.debug("", qe); throw new IllegalRangeArgumentException(qe); } @@ -183,12 +183,12 @@ public synchronized void submit() { } catch (TableNotFoundException e) { NotFoundQueryException qe = new NotFoundQueryException(DatawaveErrorCode.TABLE_NOT_FOUND, e, MessageFormat.format("Table: {0}", config.getIndexTableName())); - log.error(String.valueOf(qe)); + log.error("", qe); throw new DatawaveFatalQueryException(qe); } catch (IOException e) { QueryException qe = new QueryException(DatawaveErrorCode.RANGE_CREATE_ERROR, e, MessageFormat.format("{0}", this.literalRange)); - log.debug(String.valueOf(qe)); + log.debug("", qe); if (bs != null) { scannerFactory.close(bs); } diff --git a/warehouse/query-core/src/main/java/datawave/query/jexl/lookups/RegexIndexLookup.java b/warehouse/query-core/src/main/java/datawave/query/jexl/lookups/RegexIndexLookup.java index 93c3187454..577020fc59 100644 --- a/warehouse/query-core/src/main/java/datawave/query/jexl/lookups/RegexIndexLookup.java +++ b/warehouse/query-core/src/main/java/datawave/query/jexl/lookups/RegexIndexLookup.java @@ -130,7 +130,7 @@ public synchronized void submit() { if (config.getDisallowedRegexPatterns().contains(pattern)) { PreConditionFailedQueryException qe = new PreConditionFailedQueryException(DatawaveErrorCode.IGNORE_PATTERN_FOR_INDEX_LOOKUP, MessageFormat.format("Pattern: {0}", pattern)); - log.debug(String.valueOf(qe)); + log.debug("", qe); throw new DoNotPerformOptimizedQueryException(qe); } @@ -141,7 +141,7 @@ public synchronized void submit() { log.debug("Ignoring pattern that was not capable of being looked up in the index: " + pattern, e); continue; } catch (TableNotFoundException e) { - log.error(String.valueOf(e)); + log.error("", e); throw new DatawaveFatalQueryException(e); } catch (ExecutionException e) { throw new DatawaveFatalQueryException(e); diff --git a/warehouse/query-core/src/main/java/datawave/query/jexl/visitors/ExpandMultiNormalizedTerms.java b/warehouse/query-core/src/main/java/datawave/query/jexl/visitors/ExpandMultiNormalizedTerms.java index b5b0a4a533..8b9ce8bca7 100644 --- a/warehouse/query-core/src/main/java/datawave/query/jexl/visitors/ExpandMultiNormalizedTerms.java +++ b/warehouse/query-core/src/main/java/datawave/query/jexl/visitors/ExpandMultiNormalizedTerms.java @@ -448,7 +448,7 @@ else if (lenient) { } catch (Exception e) { QueryException qe = new QueryException(DatawaveErrorCode.NODE_EXPANSION_ERROR, e, MessageFormat.format("Node: {0}, Datatypes: {1}", PrintingVisitor.formattedQueryString(node), dataTypes)); - log.error(String.valueOf(qe)); + log.error("", qe); throw new DatawaveFatalQueryException(qe); } } diff --git a/warehouse/query-core/src/main/java/datawave/query/jexl/visitors/UnfieldedIndexExpansionVisitor.java b/warehouse/query-core/src/main/java/datawave/query/jexl/visitors/UnfieldedIndexExpansionVisitor.java index 1d0e9d17ce..227dd0263f 100644 --- a/warehouse/query-core/src/main/java/datawave/query/jexl/visitors/UnfieldedIndexExpansionVisitor.java +++ b/warehouse/query-core/src/main/java/datawave/query/jexl/visitors/UnfieldedIndexExpansionVisitor.java @@ -91,7 +91,7 @@ public static T expandUnfielded(ShardQueryConfiguration con private static T ensureTreeNotEmpty(T script) throws EmptyUnfieldedTermExpansionException { if (script.jjtGetNumChildren() == 0) { NotFoundQueryException qe = new NotFoundQueryException(DatawaveErrorCode.NO_UNFIELDED_TERM_EXPANSION_MATCH); - log.warn(String.valueOf(qe)); + log.warn("", qe); throw new EmptyUnfieldedTermExpansionException(qe); } return script; diff --git a/warehouse/query-core/src/main/java/datawave/query/planner/DefaultQueryPlanner.java b/warehouse/query-core/src/main/java/datawave/query/planner/DefaultQueryPlanner.java index a58f90a039..a698ce8c02 100644 --- a/warehouse/query-core/src/main/java/datawave/query/planner/DefaultQueryPlanner.java +++ b/warehouse/query-core/src/main/java/datawave/query/planner/DefaultQueryPlanner.java @@ -1240,7 +1240,7 @@ protected Set loadExpansionFields(ShardQueryConfiguration config) { return metadataHelper.getExpansionFields(config.getDatatypeFilter()); } catch (TableNotFoundException e) { QueryException qe = new QueryException(DatawaveErrorCode.METADATA_ACCESS_ERROR, e); - log.info(String.valueOf(qe)); + log.info("",qe); throw new DatawaveFatalQueryException(qe); } } @@ -1416,11 +1416,11 @@ protected ASTJexlScript timedExpandAnyFieldRegexNodes(QueryStopwatch timers, fin } catch (EmptyUnfieldedTermExpansionException e) { // The visitor will only throw this if we cannot expand anything resulting in empty query NotFoundQueryException qe = new NotFoundQueryException(DatawaveErrorCode.UNFIELDED_QUERY_ZERO_MATCHES, e, MessageFormat.format("Query: ", query)); - log.info(String.valueOf(qe)); + log.info("",qe); throw new NoResultsException(qe); } catch (TableNotFoundException e) { QueryException qe = new QueryException(DatawaveErrorCode.METADATA_ACCESS_ERROR, e); - log.info(String.valueOf(qe)); + log.info("",qe); throw new DatawaveFatalQueryException(qe); } } @@ -1488,7 +1488,7 @@ protected void timedTestForNonExistentFields(QueryStopwatch timers, final ASTJex BadRequestQueryException qe = new BadRequestQueryException(DatawaveErrorCode.FIELDS_NOT_IN_DATA_DICTIONARY, MessageFormat.format( "Datatype Filter: {0}, Missing Fields: {1}, Auths: {2}", datatypeFilterSet, nonexistentFields, settings.getQueryAuthorizations())); - log.error(String.valueOf(qe)); + log.error("", qe); throw new InvalidQueryException(qe); } @@ -1945,7 +1945,7 @@ protected ASTJexlScript applyQueryModel(MetadataHelper metadataHelper, ShardQuer } } catch (TableNotFoundException e) { QueryException qe = new QueryException(DatawaveErrorCode.FIELD_FETCH_ERROR, e); - log.error(String.valueOf(qe)); + log.error("", qe); throw new DatawaveFatalQueryException(qe); } diff --git a/warehouse/query-core/src/main/java/datawave/query/planner/MetadataHelperQueryModelProvider.java b/warehouse/query-core/src/main/java/datawave/query/planner/MetadataHelperQueryModelProvider.java index d43f07aa08..4d24377d70 100644 --- a/warehouse/query-core/src/main/java/datawave/query/planner/MetadataHelperQueryModelProvider.java +++ b/warehouse/query-core/src/main/java/datawave/query/planner/MetadataHelperQueryModelProvider.java @@ -36,7 +36,7 @@ public QueryModel getQueryModel() { config.setQueryModel(queryModel); } catch (TableNotFoundException e) { QueryException qe = new QueryException(DatawaveErrorCode.QUERY_MODEL_FETCH_ERROR, e); - log.error(String.valueOf(qe)); + log.error("", qe); throw new DatawaveFatalQueryException(qe); } diff --git a/warehouse/query-core/src/main/java/datawave/query/planner/ThreadedRangeBundlerIterator.java b/warehouse/query-core/src/main/java/datawave/query/planner/ThreadedRangeBundlerIterator.java index 6dc76d9da6..9d67c43bba 100644 --- a/warehouse/query-core/src/main/java/datawave/query/planner/ThreadedRangeBundlerIterator.java +++ b/warehouse/query-core/src/main/java/datawave/query/planner/ThreadedRangeBundlerIterator.java @@ -362,7 +362,7 @@ public synchronized void stop() { try { rangeIterable.close(); } catch (IOException e) { - log.error(String.valueOf(e)); + log.error("", e); } } diff --git a/warehouse/query-core/src/main/java/datawave/query/tables/ShardQueryLogic.java b/warehouse/query-core/src/main/java/datawave/query/tables/ShardQueryLogic.java index f8f765fc9f..fd7f90f70c 100644 --- a/warehouse/query-core/src/main/java/datawave/query/tables/ShardQueryLogic.java +++ b/warehouse/query-core/src/main/java/datawave/query/tables/ShardQueryLogic.java @@ -33,8 +33,7 @@ import org.apache.commons.jexl3.parser.JexlNode; import org.apache.commons.lang.builder.HashCodeBuilder; import org.apache.commons.lang3.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import org.apache.log4j.Logger; import com.google.common.base.Function; import com.google.common.cache.Cache; @@ -175,7 +174,7 @@ public class ShardQueryLogic extends BaseQueryLogic> implements public static final String NULL_BYTE = "\0"; public static final Class tableConfigurationType = ShardQueryConfiguration.class; - protected static final Logger log = LoggerFactory.getLogger(ShardQueryLogic.class); + protected static final Logger log = Logger.getLogger(ShardQueryLogic.class); static final ListeningExecutorService reloader = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(1)); private static Cache queryModelMap = CacheBuilder.newBuilder().maximumSize(100).concurrencyLevel(100) .expireAfterAccess(24, TimeUnit.HOURS).build(); diff --git a/web-services/metrics/pom.xml b/web-services/metrics/pom.xml index d15422cfa2..9fed678a1a 100644 --- a/web-services/metrics/pom.xml +++ b/web-services/metrics/pom.xml @@ -31,6 +31,10 @@ ${project.version} test + + org.slf4j + slf4j-api + ${project.artifactId} diff --git a/web-services/metrics/src/main/java/datawave/query/map/SimpleQueryGeometryHandler.java b/web-services/metrics/src/main/java/datawave/query/map/SimpleQueryGeometryHandler.java index 0578d2372a..3c7cd434dd 100644 --- a/web-services/metrics/src/main/java/datawave/query/map/SimpleQueryGeometryHandler.java +++ b/web-services/metrics/src/main/java/datawave/query/map/SimpleQueryGeometryHandler.java @@ -88,7 +88,7 @@ public static boolean isGeoQuery(BaseQueryMetric metric) { return !GeoFeatureVisitor.getGeoFeatures(JexlASTHelper.parseAndFlattenJexlQuery(jexlQuery)).isEmpty(); } catch (Exception e) { - log.trace(String.valueOf(new Exception("Unable to parse the geo features"))); + log.trace("", (new Exception("Unable to parse the geo features"))); } return false; } diff --git a/web-services/metrics/src/main/java/datawave/query/metrics/ShardTableQueryMetricHandler.java b/web-services/metrics/src/main/java/datawave/query/metrics/ShardTableQueryMetricHandler.java index ac9666fe25..a5c5a76f99 100644 --- a/web-services/metrics/src/main/java/datawave/query/metrics/ShardTableQueryMetricHandler.java +++ b/web-services/metrics/src/main/java/datawave/query/metrics/ShardTableQueryMetricHandler.java @@ -226,7 +226,7 @@ private void verifyTables() { private void writeMetrics(QueryMetric updatedQueryMetric, List storedQueryMetrics, Date lastUpdated, boolean delete) throws Exception { LiveContextWriter contextWriter = null; - MapContext context = null; + MapContext context = null; try { contextWriter = new LiveContextWriter(); @@ -239,7 +239,7 @@ private void writeMetrics(QueryMetric updatedQueryMetric, List stor AbstractColumnBasedHandler handler = new ContentQueryMetricsHandler<>(); handler.setup(context); - Multimap r = getEntries(handler, updatedQueryMetric, storedQueryMetric, lastUpdated, delete); + Multimap r = getEntries(handler, updatedQueryMetric, storedQueryMetric, lastUpdated, delete); try { if (r != null) { @@ -260,20 +260,20 @@ private void writeMetrics(QueryMetric updatedQueryMetric, List stor } } - public Map getEventFields(BaseQueryMetric queryMetric) { + public Map getEventFields(BaseQueryMetric queryMetric) { // ignore duplicates as none are expected - Map eventFields = new HashMap<>(); + Map eventFields = new HashMap<>(); ContentQueryMetricsIngestHelper ingestHelper = new ContentQueryMetricsIngestHelper(false); ingestHelper.setup(conf); - Multimap fieldsToWrite = ingestHelper.getEventFieldsToWrite(queryMetric); - for (Entry entry : fieldsToWrite.entries()) { + Multimap fieldsToWrite = ingestHelper.getEventFieldsToWrite(queryMetric); + for (Entry entry : fieldsToWrite.entries()) { eventFields.put(entry.getKey(), entry.getValue().getEventFieldValue()); } return eventFields; } - private Multimap getEntries(AbstractColumnBasedHandler handler, QueryMetric updatedQueryMetric, QueryMetric storedQueryMetric, - Date lastUpdated, boolean delete) { + private Multimap getEntries(AbstractColumnBasedHandler handler, QueryMetric updatedQueryMetric, QueryMetric storedQueryMetric, + Date lastUpdated, boolean delete) { Type type = TypeRegistry.getType("querymetrics"); ContentQueryMetricsIngestHelper ingestHelper = new ContentQueryMetricsIngestHelper(delete); @@ -284,7 +284,7 @@ private Multimap getEntries(AbstractColumnBasedHandler event.setDataType(type); event.setDate(storedQueryMetric.getCreateDate().getTime()); // get security markings from metric, otherwise default to PUBLIC - Map markings = updatedQueryMetric.getMarkings(); + Map markings = updatedQueryMetric.getMarkings(); if (markingFunctions == null || markings == null || markings.isEmpty()) { event.setVisibility(new ColumnVisibility(DEFAULT_SECURITY_MARKING)); } else { @@ -301,7 +301,7 @@ private Multimap getEntries(AbstractColumnBasedHandler event.setId(uidBuilder.newId(storedQueryMetric.getQueryId().getBytes(), (Date) null)); - final Multimap fields; + final Multimap fields; if (delete) { fields = ingestHelper.getEventFieldsToDelete(updatedQueryMetric, storedQueryMetric); @@ -319,12 +319,12 @@ private Multimap getEntries(AbstractColumnBasedHandler String indexTable = handler.getShardIndexTableName().toString(); String reverseIndexTable = handler.getShardReverseIndexTableName().toString(); int fieldSizeThreshold = ingestHelper.getFieldSizeThreshold(); - Multimap r = handler.processBulk(key, event, fields, reporter); + Multimap r = handler.processBulk(key, event, fields, reporter); List keysToRemove = new ArrayList<>(); - Map tfFields = new HashMap<>(); + Map tfFields = new HashMap<>(); // if an event has more than two entries for a given field, only keep the longest - for (Entry> entry : r.asMap().entrySet()) { + for (Entry> entry : r.asMap().entrySet()) { String table = entry.getKey().getTableName().toString(); BulkIngestKey bulkIngestKey = entry.getKey(); Key currentKey = bulkIngestKey.getKey(); @@ -343,11 +343,11 @@ private Multimap getEntries(AbstractColumnBasedHandler } // replace the longest of the keys from fields that get parsed as content - for (Entry l : tfFields.entrySet()) { + for (Entry l : tfFields.entrySet()) { r.put(l.getValue(), new Value(new byte[0])); } - for (Entry> entry : r.asMap().entrySet()) { + for (Entry> entry : r.asMap().entrySet()) { if (delete) { entry.getKey().getKey().setTimestamp(lastUpdated.getTime()); } else { @@ -365,81 +365,77 @@ private Multimap getEntries(AbstractColumnBasedHandler public void updateMetric(QueryMetric updatedQueryMetric, DatawavePrincipal datawavePrincipal) throws Exception { Date lastUpdated = updatedQueryMetric.getLastUpdated(); - try { - String sid = updatedQueryMetric.getUser(); - if (sid == null) { - sid = datawavePrincipal.getShortName(); - } + String sid = updatedQueryMetric.getUser(); + if (sid == null) { + sid = datawavePrincipal.getShortName(); + } - // find and remove previous entries - BaseQueryMetricListResponse response = new QueryMetricListResponse(); - Date end = new Date(); - Date begin = DateUtils.setYears(end, 2000); - - // user's DatawavePrincipal must have the Administrator role to use the Metrics query logic - QueryMetric cachedQueryMetric; - QueryMetric updatedQueryMetricCopy; - synchronized (ShardTableQueryMetricHandler.class) { - cachedQueryMetric = (QueryMetric) metricsCache.get(updatedQueryMetric.getQueryId()); - Map storedPageMetricMap = new TreeMap<>(); - if (cachedQueryMetric != null) { - List cachedPageMetrics = cachedQueryMetric.getPageTimes(); - if (cachedPageMetrics != null) { - for (PageMetric p : cachedPageMetrics) { - storedPageMetricMap.put(p.getPageNumber(), p); - } + // find and remove previous entries + BaseQueryMetricListResponse response = new QueryMetricListResponse(); + Date end = new Date(); + Date begin = DateUtils.setYears(end, 2000); + + // user's DatawavePrincipal must have the Administrator role to use the Metrics query logic + QueryMetric cachedQueryMetric; + QueryMetric updatedQueryMetricCopy; + synchronized (ShardTableQueryMetricHandler.class) { + cachedQueryMetric = (QueryMetric) metricsCache.get(updatedQueryMetric.getQueryId()); + Map storedPageMetricMap = new TreeMap<>(); + if (cachedQueryMetric != null) { + List cachedPageMetrics = cachedQueryMetric.getPageTimes(); + if (cachedPageMetrics != null) { + for (PageMetric p : cachedPageMetrics) { + storedPageMetricMap.put(p.getPageNumber(), p); } } - // combine all of the page metrics from the cached metric and the updated metric - for (PageMetric p : updatedQueryMetric.getPageTimes()) { - storedPageMetricMap.put(p.getPageNumber(), p); - } - ArrayList newPageMetrics = new ArrayList<>(); - newPageMetrics.addAll(storedPageMetricMap.values()); - updatedQueryMetric.setPageTimes(newPageMetrics); - metricsCache.put(updatedQueryMetric.getQueryId(), updatedQueryMetric); } - - List queryMetrics = new ArrayList<>(); - - if (cachedQueryMetric == null) { - // if numPages > 0 or Lifecycle > DEFINED, then we should have a metric cached already - // if we don't, then query for the current stored metric - if (updatedQueryMetric.getNumPages() > 0 || updatedQueryMetric.getLifecycle().compareTo(Lifecycle.DEFINED) > 0) { - QueryImpl query = new QueryImpl(); - query.setBeginDate(begin); - query.setEndDate(end); - query.setQueryLogicName(QUERY_METRICS_LOGIC_NAME); - query.setQuery("QUERY_ID == '" + updatedQueryMetric.getQueryId() + "'"); - query.setQueryName(QUERY_METRICS_LOGIC_NAME); - query.setColumnVisibility(visibilityString); - query.setQueryAuthorizations(connectorAuthorizations); - query.setUserDN(sid); - query.setExpirationDate(DateUtils.addDays(new Date(), 1)); - query.setPagesize(1000); - query.setId(UUID.randomUUID()); - query.setParameters(ImmutableMap.of(QueryOptions.INCLUDE_GROUPING_CONTEXT, "true")); - queryMetrics = getQueryMetrics(response, query, datawavePrincipal); - } - } else { - queryMetrics = Collections.singletonList(cachedQueryMetric); + // combine all of the page metrics from the cached metric and the updated metric + for (PageMetric p : updatedQueryMetric.getPageTimes()) { + storedPageMetricMap.put(p.getPageNumber(), p); } + ArrayList newPageMetrics = new ArrayList<>(); + newPageMetrics.addAll(storedPageMetricMap.values()); + updatedQueryMetric.setPageTimes(newPageMetrics); + metricsCache.put(updatedQueryMetric.getQueryId(), updatedQueryMetric); + } - if (!queryMetrics.isEmpty()) { - writeMetrics(updatedQueryMetric, queryMetrics, lastUpdated, true); - } + List queryMetrics = new ArrayList<>(); - populateMetricSelectors(updatedQueryMetric, this.luceneToJexlQueryParser); - incrementNumUpdates(updatedQueryMetric, queryMetrics); - synchronized (ShardTableQueryMetricHandler.class) { - metricsCache.put(updatedQueryMetric.getQueryId(), updatedQueryMetric); + if (cachedQueryMetric == null) { + // if numPages > 0 or Lifecycle > DEFINED, then we should have a metric cached already + // if we don't, then query for the current stored metric + if (updatedQueryMetric.getNumPages() > 0 || updatedQueryMetric.getLifecycle().compareTo(Lifecycle.DEFINED) > 0) { + QueryImpl query = new QueryImpl(); + query.setBeginDate(begin); + query.setEndDate(end); + query.setQueryLogicName(QUERY_METRICS_LOGIC_NAME); + query.setQuery("QUERY_ID == '" + updatedQueryMetric.getQueryId() + "'"); + query.setQueryName(QUERY_METRICS_LOGIC_NAME); + query.setColumnVisibility(visibilityString); + query.setQueryAuthorizations(connectorAuthorizations); + query.setUserDN(sid); + query.setExpirationDate(DateUtils.addDays(new Date(), 1)); + query.setPagesize(1000); + query.setId(UUID.randomUUID()); + query.setParameters(ImmutableMap.of(QueryOptions.INCLUDE_GROUPING_CONTEXT, "true")); + queryMetrics = getQueryMetrics(response, query, datawavePrincipal); } + } else { + queryMetrics = Collections.singletonList(cachedQueryMetric); + } - // write new entry - writeMetrics(updatedQueryMetric, Collections.singletonList(updatedQueryMetric), lastUpdated, false); - } catch (Exception e) { + if (!queryMetrics.isEmpty()) { + writeMetrics(updatedQueryMetric, queryMetrics, lastUpdated, true); + } + populateMetricSelectors(updatedQueryMetric, this.luceneToJexlQueryParser); + incrementNumUpdates(updatedQueryMetric, queryMetrics); + synchronized (ShardTableQueryMetricHandler.class) { + metricsCache.put(updatedQueryMetric.getQueryId(), updatedQueryMetric); } + + // write new entry + writeMetrics(updatedQueryMetric, Collections.singletonList(updatedQueryMetric), lastUpdated, false); } private List getQueryMetrics(BaseResponse response, Query query, DatawavePrincipal datawavePrincipal) { @@ -448,7 +444,7 @@ private List getQueryMetrics(BaseResponse response, Query query, Da AccumuloClient client = null; try { - Map trackingMap = this.connectionFactory.getTrackingMap(Thread.currentThread().getStackTrace()); + Map trackingMap = this.connectionFactory.getTrackingMap(Thread.currentThread().getStackTrace()); client = this.connectionFactory.getClient(null, null, Priority.ADMIN, trackingMap); QueryLogic queryLogic = queryLogicFactory.getQueryLogic(query.getQueryLogicName(), datawavePrincipal); runningQuery = new RunningQuery(null, client, Priority.ADMIN, queryLogic, query, query.getQueryAuthorizations(), datawavePrincipal, metricFactory); @@ -487,7 +483,7 @@ private List getQueryMetrics(BaseResponse response, Query query, Da List eventList = eventQueryResponse.getEvents(); if (eventList != null) { - for (EventBase event : eventList) { + for (EventBase event : eventList) { QueryMetric metric = (QueryMetric) toMetric(event); queryMetrics.add(metric); } @@ -521,35 +517,30 @@ private List getQueryMetrics(BaseResponse response, Query query, Da public QueryMetricListResponse query(String user, String queryId, DatawavePrincipal datawavePrincipal) { QueryMetricListResponse response = new QueryMetricListResponse(); - try { - - Collection> authorizations = datawavePrincipal.getAuthorizations(); - Date end = new Date(); - Date begin = DateUtils.setYears(end, 2000); - - QueryImpl query = new QueryImpl(); - query.setBeginDate(begin); - query.setEndDate(end); - query.setQueryLogicName(QUERY_METRICS_LOGIC_NAME); - // QueryMetricQueryLogic now enforces that you must be a QueryMetricsAdministrator to query metrics that do not belong to you - query.setQuery("QUERY_ID == '" + queryId + "'"); - query.setQueryName(QUERY_METRICS_LOGIC_NAME); - query.setColumnVisibility(visibilityString); - query.setQueryAuthorizations(WSAuthorizationsUtil.buildAuthorizationString(authorizations)); - query.setExpirationDate(DateUtils.addDays(new Date(), 1)); - query.setPagesize(1000); - query.setUserDN(datawavePrincipal.getShortName()); - query.setOwner(datawavePrincipal.getShortName()); - query.setId(UUID.randomUUID()); - query.setParameters(ImmutableMap.of(QueryOptions.INCLUDE_GROUPING_CONTEXT, "true")); - List queryMetrics = getQueryMetrics(response, query, datawavePrincipal); - - response.setResult(queryMetrics); - - response.setGeoQuery(queryMetrics.stream().anyMatch(SimpleQueryGeometryHandler::isGeoQuery)); - } catch (Exception e) { - - } + Collection> authorizations = datawavePrincipal.getAuthorizations(); + Date end = new Date(); + Date begin = DateUtils.setYears(end, 2000); + + QueryImpl query = new QueryImpl(); + query.setBeginDate(begin); + query.setEndDate(end); + query.setQueryLogicName(QUERY_METRICS_LOGIC_NAME); + // QueryMetricQueryLogic now enforces that you must be a QueryMetricsAdministrator to query metrics that do not belong to you + query.setQuery("QUERY_ID == '" + queryId + "'"); + query.setQueryName(QUERY_METRICS_LOGIC_NAME); + query.setColumnVisibility(visibilityString); + query.setQueryAuthorizations(WSAuthorizationsUtil.buildAuthorizationString(authorizations)); + query.setExpirationDate(DateUtils.addDays(new Date(), 1)); + query.setPagesize(1000); + query.setUserDN(datawavePrincipal.getShortName()); + query.setOwner(datawavePrincipal.getShortName()); + query.setId(UUID.randomUUID()); + query.setParameters(ImmutableMap.of(QueryOptions.INCLUDE_GROUPING_CONTEXT, "true")); + List queryMetrics = getQueryMetrics(response, query, datawavePrincipal); + + response.setResult(queryMetrics); + + response.setGeoQuery(queryMetrics.stream().anyMatch(SimpleQueryGeometryHandler::isGeoQuery)); return response; } @@ -565,7 +556,7 @@ public BaseQueryMetric toMetric(EventBase event) { BaseQueryMetric m = metricFactory.createMetric(false); List field = event.getFields(); m.setMarkings(event.getMarkings()); - TreeMap pageMetrics = Maps.newTreeMap(); + TreeMap pageMetrics = Maps.newTreeMap(); boolean createDateSet = false; for (FieldBase f : field) { @@ -721,13 +712,13 @@ public BaseQueryMetric toMetric(EventBase event) { } protected void createAndConfigureTablesIfNecessary(String[] tableNames, TableOperations tops, Configuration conf) - throws AccumuloSecurityException, AccumuloException, TableNotFoundException { + throws AccumuloSecurityException, AccumuloException, TableNotFoundException { for (String table : tableNames) { // If the tables don't exist, then create them. try { if (!tops.exists(table)) { tops.create(table); - Map tableConfigs = getTableConfigs(log, conf, tableNames); + Map tableConfigs = getTableConfigs(log, conf, tableNames); TableConfigHelper tableHelper = tableConfigs.get(table); @@ -745,8 +736,8 @@ protected void createAndConfigureTablesIfNecessary(String[] tableNames, TableOpe } @SuppressWarnings("unchecked") - private Map getTableConfigs(Logger log, Configuration conf, String[] tableNames) { - Map helperMap = new HashMap<>(tableNames.length); + private Map getTableConfigs(Logger log, Configuration conf, String[] tableNames) { + Map helperMap = new HashMap<>(tableNames.length); for (String table : tableNames) { String prop = table + TableConfigHelper.TABLE_CONFIG_CLASS_SUFFIX; @@ -760,7 +751,8 @@ private Map getTableConfigs(Logger log, Configuration if (tableHelper != null) tableHelper.setup(table, conf, log); - } catch (ClassNotFoundException | IllegalAccessException | InstantiationException | NoSuchMethodException | InvocationTargetException e) { + } catch (ClassNotFoundException | IllegalAccessException | InstantiationException | + NoSuchMethodException | InvocationTargetException e) { throw new IllegalArgumentException(e); } } @@ -806,7 +798,7 @@ public QueryMetricsSummaryResponse getUserQueriesSummary(Date begin, Date end, D } public QueryMetricsSummaryResponse getQueryMetricsSummary(Date begin, Date end, boolean onlyCurrentUser, DatawavePrincipal datawavePrincipal, - QueryMetricsSummaryResponse response) { + QueryMetricsSummaryResponse response) { try {