-
Notifications
You must be signed in to change notification settings - Fork 246
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Field configuration helper cache implementation
- Loading branch information
Showing
4 changed files
with
203 additions
and
1 deletion.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
75 changes: 75 additions & 0 deletions
75
warehouse/ingest-core/src/main/java/datawave/ingest/data/config/CachedFieldConfigHelper.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,75 @@ | ||
| package datawave.ingest.data.config; | ||
|
|
||
| import java.util.EnumMap; | ||
| import java.util.Map; | ||
| import java.util.function.Function; | ||
|
|
||
| import org.apache.commons.collections4.map.LRUMap; | ||
|
|
||
| import com.google.common.annotations.VisibleForTesting; | ||
|
|
||
| public class CachedFieldConfigHelper implements FieldConfigHelper { | ||
| private final FieldConfigHelper underlyingHelper; | ||
| private final Map<String, ResultEntry> resultCache; | ||
|
|
||
| enum AttributeType { | ||
| INDEXED_FIELD, REVERSE_INDEXED_FIELD, TOKENIZED_FIELD, REVERSE_TOKENIZED_FIELD, STORED_FIELD, INDEXED_ONLY | ||
| } | ||
|
|
||
| public CachedFieldConfigHelper(FieldConfigHelper helper, int limit) { | ||
| if (limit < 1) { | ||
| throw new IllegalArgumentException("Limit must be a positive integer"); | ||
| } | ||
| this.underlyingHelper = helper; | ||
| this.resultCache = new LRUMap<>(limit); | ||
| } | ||
|
|
||
| @Override | ||
| public boolean isStoredField(String fieldName) { | ||
| return getOrEvaluate(AttributeType.STORED_FIELD, fieldName, underlyingHelper::isStoredField); | ||
| } | ||
|
|
||
| @Override | ||
| public boolean isIndexedField(String fieldName) { | ||
| return getOrEvaluate(AttributeType.INDEXED_FIELD, fieldName, underlyingHelper::isIndexedField); | ||
| } | ||
|
|
||
| @Override | ||
| public boolean isIndexOnlyField(String fieldName) { | ||
| return getOrEvaluate(AttributeType.INDEXED_ONLY, fieldName, underlyingHelper::isIndexOnlyField); | ||
| } | ||
|
|
||
| @Override | ||
| public boolean isReverseIndexedField(String fieldName) { | ||
| return getOrEvaluate(AttributeType.REVERSE_INDEXED_FIELD, fieldName, underlyingHelper::isReverseIndexedField); | ||
| } | ||
|
|
||
| @Override | ||
| public boolean isTokenizedField(String fieldName) { | ||
| return getOrEvaluate(AttributeType.TOKENIZED_FIELD, fieldName, underlyingHelper::isTokenizedField); | ||
| } | ||
|
|
||
| @Override | ||
| public boolean isReverseTokenizedField(String fieldName) { | ||
| return getOrEvaluate(AttributeType.REVERSE_TOKENIZED_FIELD, fieldName, underlyingHelper::isReverseTokenizedField); | ||
| } | ||
|
|
||
| @VisibleForTesting | ||
| boolean getOrEvaluate(AttributeType attributeType, String fieldName, Function<String,Boolean> evaluateFn) { | ||
| return resultCache.computeIfAbsent(fieldName, ResultEntry::new).resolveResult(attributeType, evaluateFn); | ||
| } | ||
|
|
||
| private static class ResultEntry { | ||
| private final String fieldName; | ||
| private final EnumMap<AttributeType, Boolean> resultMap; | ||
|
|
||
| ResultEntry(String fieldName) { | ||
| this.fieldName = fieldName; | ||
| this.resultMap = new EnumMap<>(AttributeType.class); | ||
| } | ||
|
|
||
| boolean resolveResult(AttributeType attributeType, Function<String,Boolean> evaluateFn) { | ||
| return resultMap.computeIfAbsent(attributeType, (t) -> evaluateFn.apply(fieldName)); | ||
| } | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
99 changes: 99 additions & 0 deletions
99
...e/ingest-core/src/test/java/datawave/ingest/data/config/CachingFieldConfigHelperTest.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,99 @@ | ||
| package datawave.ingest.data.config; | ||
|
|
||
| import static org.junit.jupiter.api.Assertions.assertThrows; | ||
| import static org.mockito.ArgumentMatchers.eq; | ||
| import static org.mockito.Mockito.mock; | ||
| import static org.mockito.Mockito.verify; | ||
|
|
||
| import java.util.concurrent.atomic.AtomicLong; | ||
| import java.util.function.BiConsumer; | ||
| import java.util.function.Function; | ||
| import java.util.stream.Stream; | ||
|
|
||
| import org.junit.jupiter.api.Assertions; | ||
| import org.junit.jupiter.api.Test; | ||
| import org.junit.jupiter.params.ParameterizedTest; | ||
| import org.junit.jupiter.params.provider.ValueSource; | ||
|
|
||
| public class CachingFieldConfigHelperTest { | ||
| @SuppressWarnings("unchecked") | ||
| @Test | ||
| public void testCachingBehaviorWillCallBaseMethods() { | ||
| // @formatter:off | ||
| Stream.of(new Object[][] { | ||
| new Object[] { | ||
| (BiConsumer<FieldConfigHelper, String>) FieldConfigHelper::isIndexOnlyField, | ||
| (BiConsumer<FieldConfigHelper, String>) (h, f) -> verify(h).isIndexOnlyField(eq(f)), | ||
| (BiConsumer<FieldConfigHelper, String>) FieldConfigHelper::isIndexedField, | ||
| (BiConsumer<FieldConfigHelper, String>) (h, f) -> verify(h).isIndexedField(eq(f)), | ||
| (BiConsumer<FieldConfigHelper, String>) FieldConfigHelper::isTokenizedField, | ||
| (BiConsumer<FieldConfigHelper, String>) (h, f) -> verify(h).isTokenizedField(eq(f)), | ||
| (BiConsumer<FieldConfigHelper, String>) FieldConfigHelper::isStoredField, | ||
| (BiConsumer<FieldConfigHelper, String>) (h, f) -> verify(h).isStoredField(eq(f)), | ||
| (BiConsumer<FieldConfigHelper, String>) FieldConfigHelper::isReverseIndexedField, | ||
| (BiConsumer<FieldConfigHelper, String>) (h, f) -> verify(h).isReverseIndexedField(eq(f)), | ||
| (BiConsumer<FieldConfigHelper, String>) FieldConfigHelper::isReverseTokenizedField, | ||
| (BiConsumer<FieldConfigHelper, String>) (h, f) -> verify(h).isReverseTokenizedField(eq(f)), | ||
| } | ||
| }).forEach(arg -> { | ||
| // param[0] = helper method | ||
| // param[1] = validation method | ||
| String fieldName = "testField"; | ||
| BiConsumer<FieldConfigHelper, String> testAction = (BiConsumer<FieldConfigHelper, String>) arg[0]; | ||
| BiConsumer<FieldConfigHelper, String> verifyAction = (BiConsumer<FieldConfigHelper, String>) arg[1]; | ||
| FieldConfigHelper mockHelper = mock(FieldConfigHelper.class); | ||
| FieldConfigHelper cachedHelper = new CachedFieldConfigHelper(mockHelper, 1); | ||
| testAction.accept(cachedHelper, fieldName); | ||
| verifyAction.accept(mockHelper, fieldName); | ||
| }); | ||
| // @formatter:on | ||
| } | ||
|
|
||
| @ParameterizedTest | ||
| @ValueSource(ints = {-1,0}) | ||
| public void testConstructorWithNonPositiveLimitWillThrow(int limit) { | ||
| assertThrows(IllegalArgumentException.class, () -> new CachedFieldConfigHelper(mock(FieldConfigHelper.class), limit)); | ||
| } | ||
|
|
||
| @Test | ||
| public void testCachingLimitsBetweenFieldsAndAttributeTypes() { | ||
| AtomicLong counter = new AtomicLong(); | ||
| CachedFieldConfigHelper helper = new CachedFieldConfigHelper(mock(FieldConfigHelper.class), 2); | ||
| Function<String,Boolean> fn = (f) -> { | ||
| counter.incrementAndGet(); | ||
| return true; | ||
| }; | ||
|
|
||
| // following ensures that: | ||
| // 1. fields are computed, where appropriate per attribute-type | ||
| // 2. limit allows cache results to return | ||
| // 3. limit blocks results to return if exceeded | ||
| // 4. limit functions across attribute-types | ||
|
|
||
| helper.getOrEvaluate(CachedFieldConfigHelper.AttributeType.STORED_FIELD, "field1", fn); | ||
| Assertions.assertEquals(1, counter.get(), "field1 should compute result (new field)"); | ||
|
|
||
| helper.getOrEvaluate(CachedFieldConfigHelper.AttributeType.STORED_FIELD, "field1", fn); | ||
| Assertions.assertEquals(1, counter.get(), "field1 repeated (existing field)"); | ||
|
|
||
| helper.getOrEvaluate(CachedFieldConfigHelper.AttributeType.STORED_FIELD, "field2", fn); | ||
| Assertions.assertEquals(2, counter.get(), "field2 should compute result (new field)"); | ||
|
|
||
| helper.getOrEvaluate(CachedFieldConfigHelper.AttributeType.STORED_FIELD, "field2", fn); | ||
| Assertions.assertEquals(2, counter.get(), "field2 repeated (existing)"); | ||
|
|
||
| helper.getOrEvaluate(CachedFieldConfigHelper.AttributeType.INDEXED_FIELD, "field1", fn); | ||
| Assertions.assertEquals(3, counter.get(), "field1 should compute result (new attribute)"); | ||
|
|
||
| helper.getOrEvaluate(CachedFieldConfigHelper.AttributeType.STORED_FIELD, "field3", fn); | ||
| Assertions.assertEquals(4, counter.get(), "field3 exceeded limit (new field)"); | ||
|
|
||
| helper.getOrEvaluate(CachedFieldConfigHelper.AttributeType.STORED_FIELD, "field3", fn); | ||
| Assertions.assertEquals(4, counter.get(), "field3 exceeded limit (existing field)"); | ||
|
|
||
| // LRU map should evict field #2 | ||
| // we access field #1 above which has more accesses over field #2 | ||
| helper.getOrEvaluate(CachedFieldConfigHelper.AttributeType.STORED_FIELD, "field2", fn); | ||
| Assertions.assertEquals(5, counter.get(), "field1 exceeded limit (new field/eviction)"); | ||
| } | ||
| } |