From 3924103f0020a5b0f627f1bdf6e1ea55d3084df9 Mon Sep 17 00:00:00 2001 From: zack-rma Date: Wed, 18 Sep 2024 12:03:15 -0700 Subject: [PATCH 1/2] Added global fetchsize variables to JooqDao and appropriate DAOs --- cwms-data-api/src/main/java/cwms/cda/data/dao/JooqDao.java | 3 ++- .../src/main/java/cwms/cda/data/dao/LocationGroupDao.java | 4 ++-- .../src/main/java/cwms/cda/data/dao/LocationsDaoImpl.java | 6 +++--- .../src/main/java/cwms/cda/data/dao/RatingSpecDao.java | 4 ++-- .../src/main/java/cwms/cda/data/dao/RatingTemplateDao.java | 4 ++-- 5 files changed, 11 insertions(+), 10 deletions(-) diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dao/JooqDao.java b/cwms-data-api/src/main/java/cwms/cda/data/dao/JooqDao.java index cdbd10089..3e1dfd0be 100644 --- a/cwms-data-api/src/main/java/cwms/cda/data/dao/JooqDao.java +++ b/cwms-data-api/src/main/java/cwms/cda/data/dao/JooqDao.java @@ -45,7 +45,6 @@ import java.util.Collections; import java.util.List; import java.util.Optional; -import java.util.logging.Level; import javax.servlet.http.HttpServletResponse; import javax.sql.DataSource; import org.jetbrains.annotations.NotNull; @@ -69,6 +68,8 @@ public abstract class JooqDao extends Dao { protected static final int ORACLE_CURSOR_TYPE = -10; private static final FluentLogger logger = FluentLogger.forEnclosingClass(); + public static final int FETCH_SIZE = 1000; + public static final int HALF_FETCH_SIZE = FETCH_SIZE / 2; static ExecuteListener listener = new ExceptionWrappingListener(); diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dao/LocationGroupDao.java b/cwms-data-api/src/main/java/cwms/cda/data/dao/LocationGroupDao.java index 64b47e467..520965d48 100644 --- a/cwms-data-api/src/main/java/cwms/cda/data/dao/LocationGroupDao.java +++ b/cwms-data-api/src/main/java/cwms/cda/data/dao/LocationGroupDao.java @@ -123,7 +123,7 @@ public Optional getLocationGroup(@NotNull String officeId, @NotNu .and(alcg.CAT_DB_OFFICE_ID.in(CWMS, officeId)) .and(assignmentOffice) ) - .orderBy(alga.ATTRIBUTE).fetchSize(1000).fetch(mapper); + .orderBy(alga.ATTRIBUTE).fetchSize(FETCH_SIZE).fetch(mapper); // Might want to verify that all the groups in the list are the same? LocationGroup locGroup = @@ -277,7 +277,7 @@ public List getLocationGroups(String officeId, String locCategory Map> map = new LinkedHashMap<>(); connectBy.orderBy(alcg.LOC_CATEGORY_ID, alcg.LOC_GROUP_ID, alga.ATTRIBUTE) - .fetchSize(1000) // This made the query go from 2 minutes to 10 seconds? + .fetchSize(FETCH_SIZE) // This made the query go from 2 minutes to 10 seconds? .stream().map(mapper::map).forEach(pair -> { LocationGroup locationGroup = pair.component1(); List list = map.computeIfAbsent(locationGroup, k -> new ArrayList<>()); diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dao/LocationsDaoImpl.java b/cwms-data-api/src/main/java/cwms/cda/data/dao/LocationsDaoImpl.java index 72a2256c8..90bce6d14 100644 --- a/cwms-data-api/src/main/java/cwms/cda/data/dao/LocationsDaoImpl.java +++ b/cwms-data-api/src/main/java/cwms/cda/data/dao/LocationsDaoImpl.java @@ -121,7 +121,7 @@ public List getLocations(String nameRegex, String unitSystem, String d return dsl.select(AV_LOC.asterisk()) .from(AV_LOC) .where(whereCondition) - .fetchSize(500) + .fetchSize(HALF_FETCH_SIZE) .fetch(this::buildLocation); } @@ -291,7 +291,7 @@ public FeatureCollection buildFeatureCollection(String names, String units, Stri selectQuery = selectQuery.and(AV_LOC.LOCATION_ID.in(identifiers)); } - List features = selectQuery.fetchSize(500).stream() + List features = selectQuery.fetchSize(HALF_FETCH_SIZE).stream() .map(LocationsDaoImpl::buildFeatureFromAvLocRecord) .collect(toList()); FeatureCollection collection = new FeatureCollection(); @@ -433,7 +433,7 @@ private Catalog getLocationCatalog(Catalog.CatalogPage catPage, int pageSize, Ca .orderBy(avLoc2.DB_OFFICE_ID.asc(),limitId.asc(),avLoc2.ALIASED_ITEM.asc()); logger.log(Level.FINER, () -> query.getSQL(ParamType.INLINED)); List entries = query - .fetchSize(1000) + .fetchSize(FETCH_SIZE) .fetchStream() .map(r -> r.into(AV_LOC2.AV_LOC2)) .collect(groupingBy(usace.cwms.db.jooq.codegen.tables.records.AV_LOC2::getLOCATION_CODE)) diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dao/RatingSpecDao.java b/cwms-data-api/src/main/java/cwms/cda/data/dao/RatingSpecDao.java index 9ea769ae6..a7deb4554 100644 --- a/cwms-data-api/src/main/java/cwms/cda/data/dao/RatingSpecDao.java +++ b/cwms-data-api/src/main/java/cwms/cda/data/dao/RatingSpecDao.java @@ -94,7 +94,7 @@ public Collection retrieveRatingSpecs(String office, String specIdMa .leftOuterJoin(ratView) .on(specView.RATING_SPEC_CODE.eq(ratView.RATING_SPEC_CODE)) .where(condition) - .fetchSize(1000); + .fetchSize(FETCH_SIZE); logger.fine(() -> query.getSQL(ParamType.INLINED)); @@ -248,7 +248,7 @@ public Optional retrieveRatingSpec(String office, String specId) { .on(specView.RATING_SPEC_CODE.eq(ratView.RATING_SPEC_CODE)) .where(condition) .orderBy(specView.OFFICE_ID, specView.RATING_ID, ratView.EFFECTIVE_DATE) - .fetchSize(1000); + .fetchSize(FETCH_SIZE); logger.fine(() -> query.getSQL(ParamType.INLINED)); diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dao/RatingTemplateDao.java b/cwms-data-api/src/main/java/cwms/cda/data/dao/RatingTemplateDao.java index 41d180a8e..3d49fa9ee 100644 --- a/cwms-data-api/src/main/java/cwms/cda/data/dao/RatingTemplateDao.java +++ b/cwms-data-api/src/main/java/cwms/cda/data/dao/RatingTemplateDao.java @@ -87,7 +87,7 @@ public Set retrieveRatingTemplates(String office, String templat .leftOuterJoin(specView) .on(specView.TEMPLATE_CODE.eq(tempView.TEMPLATE_CODE)) .where(condition) - .fetchSize(1000); + .fetchSize(FETCH_SIZE); logger.fine(() -> query.getSQL(ParamType.INLINED)); @@ -144,7 +144,7 @@ public Optional retrieveRatingTemplate(String office, String tem .leftOuterJoin(specView).on( specView.TEMPLATE_CODE.eq(tempView.TEMPLATE_CODE)) .where(condition) - .fetchSize(1000); + .fetchSize(FETCH_SIZE); logger.fine(() -> query.getSQL(ParamType.INLINED)); From 8c5052ace4d9074a7ed0c29afe41bc4a53bb8877 Mon Sep 17 00:00:00 2001 From: zack-rma Date: Thu, 19 Sep 2024 08:46:19 -0700 Subject: [PATCH 2/2] Updated fetch size global variable naming and small fetch value --- cwms-data-api/src/main/java/cwms/cda/data/dao/JooqDao.java | 4 ++-- .../src/main/java/cwms/cda/data/dao/LocationGroupDao.java | 4 ++-- .../src/main/java/cwms/cda/data/dao/LocationsDaoImpl.java | 6 +++--- .../src/main/java/cwms/cda/data/dao/RatingSpecDao.java | 4 ++-- .../src/main/java/cwms/cda/data/dao/RatingTemplateDao.java | 4 ++-- 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dao/JooqDao.java b/cwms-data-api/src/main/java/cwms/cda/data/dao/JooqDao.java index 3e1dfd0be..412132cde 100644 --- a/cwms-data-api/src/main/java/cwms/cda/data/dao/JooqDao.java +++ b/cwms-data-api/src/main/java/cwms/cda/data/dao/JooqDao.java @@ -68,8 +68,8 @@ public abstract class JooqDao extends Dao { protected static final int ORACLE_CURSOR_TYPE = -10; private static final FluentLogger logger = FluentLogger.forEnclosingClass(); - public static final int FETCH_SIZE = 1000; - public static final int HALF_FETCH_SIZE = FETCH_SIZE / 2; + public static final int DEFAULT_FETCH_SIZE = 1000; + public static final int DEFAULT_SMALL_FETCH_SIZE = 500; static ExecuteListener listener = new ExceptionWrappingListener(); diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dao/LocationGroupDao.java b/cwms-data-api/src/main/java/cwms/cda/data/dao/LocationGroupDao.java index 520965d48..68b570ef2 100644 --- a/cwms-data-api/src/main/java/cwms/cda/data/dao/LocationGroupDao.java +++ b/cwms-data-api/src/main/java/cwms/cda/data/dao/LocationGroupDao.java @@ -123,7 +123,7 @@ public Optional getLocationGroup(@NotNull String officeId, @NotNu .and(alcg.CAT_DB_OFFICE_ID.in(CWMS, officeId)) .and(assignmentOffice) ) - .orderBy(alga.ATTRIBUTE).fetchSize(FETCH_SIZE).fetch(mapper); + .orderBy(alga.ATTRIBUTE).fetchSize(DEFAULT_FETCH_SIZE).fetch(mapper); // Might want to verify that all the groups in the list are the same? LocationGroup locGroup = @@ -277,7 +277,7 @@ public List getLocationGroups(String officeId, String locCategory Map> map = new LinkedHashMap<>(); connectBy.orderBy(alcg.LOC_CATEGORY_ID, alcg.LOC_GROUP_ID, alga.ATTRIBUTE) - .fetchSize(FETCH_SIZE) // This made the query go from 2 minutes to 10 seconds? + .fetchSize(DEFAULT_FETCH_SIZE) // This made the query go from 2 minutes to 10 seconds? .stream().map(mapper::map).forEach(pair -> { LocationGroup locationGroup = pair.component1(); List list = map.computeIfAbsent(locationGroup, k -> new ArrayList<>()); diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dao/LocationsDaoImpl.java b/cwms-data-api/src/main/java/cwms/cda/data/dao/LocationsDaoImpl.java index 90bce6d14..1d34edbb6 100644 --- a/cwms-data-api/src/main/java/cwms/cda/data/dao/LocationsDaoImpl.java +++ b/cwms-data-api/src/main/java/cwms/cda/data/dao/LocationsDaoImpl.java @@ -121,7 +121,7 @@ public List getLocations(String nameRegex, String unitSystem, String d return dsl.select(AV_LOC.asterisk()) .from(AV_LOC) .where(whereCondition) - .fetchSize(HALF_FETCH_SIZE) + .fetchSize(DEFAULT_SMALL_FETCH_SIZE) .fetch(this::buildLocation); } @@ -291,7 +291,7 @@ public FeatureCollection buildFeatureCollection(String names, String units, Stri selectQuery = selectQuery.and(AV_LOC.LOCATION_ID.in(identifiers)); } - List features = selectQuery.fetchSize(HALF_FETCH_SIZE).stream() + List features = selectQuery.fetchSize(DEFAULT_SMALL_FETCH_SIZE).stream() .map(LocationsDaoImpl::buildFeatureFromAvLocRecord) .collect(toList()); FeatureCollection collection = new FeatureCollection(); @@ -433,7 +433,7 @@ private Catalog getLocationCatalog(Catalog.CatalogPage catPage, int pageSize, Ca .orderBy(avLoc2.DB_OFFICE_ID.asc(),limitId.asc(),avLoc2.ALIASED_ITEM.asc()); logger.log(Level.FINER, () -> query.getSQL(ParamType.INLINED)); List entries = query - .fetchSize(FETCH_SIZE) + .fetchSize(DEFAULT_FETCH_SIZE) .fetchStream() .map(r -> r.into(AV_LOC2.AV_LOC2)) .collect(groupingBy(usace.cwms.db.jooq.codegen.tables.records.AV_LOC2::getLOCATION_CODE)) diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dao/RatingSpecDao.java b/cwms-data-api/src/main/java/cwms/cda/data/dao/RatingSpecDao.java index a7deb4554..ef251e114 100644 --- a/cwms-data-api/src/main/java/cwms/cda/data/dao/RatingSpecDao.java +++ b/cwms-data-api/src/main/java/cwms/cda/data/dao/RatingSpecDao.java @@ -94,7 +94,7 @@ public Collection retrieveRatingSpecs(String office, String specIdMa .leftOuterJoin(ratView) .on(specView.RATING_SPEC_CODE.eq(ratView.RATING_SPEC_CODE)) .where(condition) - .fetchSize(FETCH_SIZE); + .fetchSize(DEFAULT_FETCH_SIZE); logger.fine(() -> query.getSQL(ParamType.INLINED)); @@ -248,7 +248,7 @@ public Optional retrieveRatingSpec(String office, String specId) { .on(specView.RATING_SPEC_CODE.eq(ratView.RATING_SPEC_CODE)) .where(condition) .orderBy(specView.OFFICE_ID, specView.RATING_ID, ratView.EFFECTIVE_DATE) - .fetchSize(FETCH_SIZE); + .fetchSize(DEFAULT_FETCH_SIZE); logger.fine(() -> query.getSQL(ParamType.INLINED)); diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dao/RatingTemplateDao.java b/cwms-data-api/src/main/java/cwms/cda/data/dao/RatingTemplateDao.java index 3d49fa9ee..bb60604a6 100644 --- a/cwms-data-api/src/main/java/cwms/cda/data/dao/RatingTemplateDao.java +++ b/cwms-data-api/src/main/java/cwms/cda/data/dao/RatingTemplateDao.java @@ -87,7 +87,7 @@ public Set retrieveRatingTemplates(String office, String templat .leftOuterJoin(specView) .on(specView.TEMPLATE_CODE.eq(tempView.TEMPLATE_CODE)) .where(condition) - .fetchSize(FETCH_SIZE); + .fetchSize(DEFAULT_FETCH_SIZE); logger.fine(() -> query.getSQL(ParamType.INLINED)); @@ -144,7 +144,7 @@ public Optional retrieveRatingTemplate(String office, String tem .leftOuterJoin(specView).on( specView.TEMPLATE_CODE.eq(tempView.TEMPLATE_CODE)) .where(condition) - .fetchSize(FETCH_SIZE); + .fetchSize(DEFAULT_FETCH_SIZE); logger.fine(() -> query.getSQL(ParamType.INLINED));