diff --git a/boudicca.base/common-model/src/main/kotlin/base/boudicca/keyfilters/KeyFilters.kt b/boudicca.base/common-model/src/main/kotlin/base/boudicca/keyfilters/KeyFilters.kt index c8c07d40..68ea3cab 100644 --- a/boudicca.base/common-model/src/main/kotlin/base/boudicca/keyfilters/KeyFilters.kt +++ b/boudicca.base/common-model/src/main/kotlin/base/boudicca/keyfilters/KeyFilters.kt @@ -19,7 +19,7 @@ object KeyFilters { .sortedBy { it.first } } - private fun doesKeyMatchFilter( + fun doesKeyMatchFilter( key: Key, keyFilter: Key ): Boolean { @@ -36,7 +36,7 @@ object KeyFilters { return true } - private fun containsVariant(key: Key, variant: Variant): Boolean { + fun containsVariant(key: Key, variant: Variant): Boolean { if (variant.variantValue == "*") { return doesContainVariantName(key, variant.variantName) } @@ -54,8 +54,17 @@ object KeyFilters { } private fun doesContainVariantName(key: Key, variantName: String): Boolean { - for (selfVariant in key.variants) { - if (variantName == selfVariant.variantName) { + for (variant in key.variants) { + if (variantName == variant.variantName) { + return true + } + } + return false + } + + fun doesContainVariantValue(key: Key, variantName: String, variantValues: List): Boolean { + for (value in variantValues) { + if (containsVariant(key, Variant(variantName, value))) { return true } } diff --git a/boudicca.base/common-model/src/main/kotlin/base/boudicca/keyfilters/KeySelector.kt b/boudicca.base/common-model/src/main/kotlin/base/boudicca/keyfilters/KeySelector.kt index 9011e3e2..bb7b5623 100644 --- a/boudicca.base/common-model/src/main/kotlin/base/boudicca/keyfilters/KeySelector.kt +++ b/boudicca.base/common-model/src/main/kotlin/base/boudicca/keyfilters/KeySelector.kt @@ -69,6 +69,12 @@ class KeySelector private constructor( fun builder(propertyName: String): KeySelectorBuilder { return KeySelectorBuilder(propertyName) } + + fun builder(key: Key): KeySelectorBuilder { + val builder = KeySelectorBuilder(key.name) + key.variants.forEach { builder.thenVariant(it) } + return builder + } } class KeySelectorBuilder internal constructor(private val propertyName: String) { @@ -79,6 +85,11 @@ class KeySelector private constructor( return this } + fun thenVariant(variant: Variant): KeySelectorBuilder { + variants.add(Pair(variant.variantName, listOf(variant.variantValue))) + return this + } + fun build(): KeySelector { return KeySelector(propertyName, variants.toList()) } diff --git a/boudicca.base/publisher-event-html/src/main/kotlin/base/boudicca/publisher/event/html/service/EventService.kt b/boudicca.base/publisher-event-html/src/main/kotlin/base/boudicca/publisher/event/html/service/EventService.kt index 0df33e95..2093c031 100644 --- a/boudicca.base/publisher-event-html/src/main/kotlin/base/boudicca/publisher/event/html/service/EventService.kt +++ b/boudicca.base/publisher-event-html/src/main/kotlin/base/boudicca/publisher/event/html/service/EventService.kt @@ -117,7 +117,7 @@ class EventService @Autowired constructor( listOf( FilterQueryEntryDTO(SemanticKeys.LOCATION_NAME), FilterQueryEntryDTO(SemanticKeys.LOCATION_CITY), - FilterQueryEntryDTO(SemanticKeys.CONCERT_BANDLIST, true), + FilterQueryEntryDTO(SemanticKeys.CONCERT_BANDLIST), ) ) ) @@ -133,7 +133,7 @@ class EventService @Autowired constructor( fun getSources(): List { val allSources = - caller.getFiltersFor(FilterQueryDTO(listOf(FilterQueryEntryDTO(SemanticKeys.SOURCES, true)))) + caller.getFiltersFor(FilterQueryDTO(listOf(FilterQueryEntryDTO(SemanticKeys.SOURCES)))) return allSources[SemanticKeys.SOURCES]!! .map { normalize(it) } .distinct() diff --git a/boudicca.base/query-lib/src/main/kotlin/base/boudicca/query/Expression.kt b/boudicca.base/query-lib/src/main/kotlin/base/boudicca/query/Expression.kt index a0219d93..4aeb55d3 100644 --- a/boudicca.base/query-lib/src/main/kotlin/base/boudicca/query/Expression.kt +++ b/boudicca.base/query-lib/src/main/kotlin/base/boudicca/query/Expression.kt @@ -1,5 +1,6 @@ package base.boudicca.query +import base.boudicca.model.structured.Key import java.time.LocalDate import java.time.format.DateTimeFormatter import java.time.format.DateTimeParseException @@ -43,12 +44,14 @@ abstract class HasTwoChildren( abstract class FieldAndTextExpression( private val name: String, - private val fieldName: String, + keyFilter: String, private val text: String, ) : Expression { - fun getFieldName(): String { - return fieldName + private val keyFilter = parseKeyFilter(keyFilter) + + fun getKeyFilter(): Key { + return keyFilter } fun getText(): String { @@ -56,29 +59,31 @@ abstract class FieldAndTextExpression( } override fun toString(): String { - return "$name('$fieldName','$text')" + return "$name('${keyFilter.toKeyString()}','$text')" } } abstract class FieldExpression( private val name: String, - private val fieldName: String, + keyFilter: String, ) : Expression { - fun getFieldName(): String { - return fieldName + private val keyFilter = parseKeyFilter(keyFilter) + + fun getKeyFilter(): Key { + return keyFilter } override fun toString(): String { - return "$name('$fieldName')" + return "$name('${keyFilter.toKeyString()}')" } } abstract class DateExpression( private val name: String, - dateFieldName: String, + dateKeyFilter: String, dateText: String, -) : FieldAndTextExpression(name, dateFieldName, dateText) { +) : FieldAndTextExpression(name, dateKeyFilter, dateText) { private val date: LocalDate @@ -95,23 +100,26 @@ abstract class DateExpression( } override fun toString(): String { - return "$name('${getFieldName()}','${date.format(DateTimeFormatter.ISO_LOCAL_DATE)}')" + return "$name('${getKeyFilter().toKeyString()}','${date.format(DateTimeFormatter.ISO_LOCAL_DATE)}')" } } abstract class AbstractDurationExpression( private val name: String, - private val startDateField: String, - private val endDateField: String, + startDateKeyFilter: String, + endDateKeyFilter: String, private val duration: Number, ) : Expression { - fun getStartDateField(): String { - return startDateField + private val startDateKeyFilter = parseKeyFilter(startDateKeyFilter) + private val endDateKeyFilter = parseKeyFilter(endDateKeyFilter) + + fun getStartDateKeyFilter(): Key { + return startDateKeyFilter } - fun getEndDateField(): String { - return endDateField + fun getEndDateKeyFilter(): Key { + return endDateKeyFilter } fun getDuration(): Number { @@ -119,7 +127,15 @@ abstract class AbstractDurationExpression( } override fun toString(): String { - return "$name('${startDateField}','${endDateField}',${duration})" + return "$name('${startDateKeyFilter.toKeyString()}','${endDateKeyFilter.toKeyString()}',${duration})" + } +} + +private fun parseKeyFilter(keyFilter: String): Key { + try { + return Key.parse(keyFilter) + } catch (e: IllegalArgumentException) { + throw QueryException("invalid keyfilter", e) } } @@ -138,24 +154,24 @@ class NotExpression( ) : HasOneChild("NOT", child) class ContainsExpression( - fieldName: String, + keyFilter: String, text: String, -) : FieldAndTextExpression("CONTAINS", fieldName, text) +) : FieldAndTextExpression("CONTAINS", keyFilter, text) class EqualsExpression( - fieldName: String, + keyFilter: String, text: String, -) : FieldAndTextExpression("EQUALS", fieldName, text) +) : FieldAndTextExpression("EQUALS", keyFilter, text) class BeforeExpression( - dateFieldName: String, + dateKeyFilter: String, dateText: String, -) : DateExpression("BEFORE", dateFieldName, dateText) +) : DateExpression("BEFORE", dateKeyFilter, dateText) class AfterExpression( - dateFieldName: String, + dateKeyFilter: String, dateText: String, -) : DateExpression("AFTER", dateFieldName, dateText) +) : DateExpression("AFTER", dateKeyFilter, dateText) class DurationShorterExpression( startDateField: String, @@ -170,5 +186,5 @@ class DurationLongerExpression( ) : AbstractDurationExpression("DURATIONLONGER", startDateField, endDateField, duration) class HasFieldExpression( - fieldName: String, -) : FieldExpression("HASFIELD", fieldName) + keyFilter: String, +) : FieldExpression("HASFIELD", keyFilter) diff --git a/boudicca.base/query-lib/src/main/kotlin/base/boudicca/query/Utils.kt b/boudicca.base/query-lib/src/main/kotlin/base/boudicca/query/Utils.kt index daad3d0b..933451e6 100644 --- a/boudicca.base/query-lib/src/main/kotlin/base/boudicca/query/Utils.kt +++ b/boudicca.base/query-lib/src/main/kotlin/base/boudicca/query/Utils.kt @@ -1,7 +1,11 @@ package base.boudicca.query import base.boudicca.SemanticKeys +import base.boudicca.keyfilters.KeySelector import base.boudicca.model.Entry +import base.boudicca.model.structured.VariantConstants +import base.boudicca.model.structured.selectKey +import base.boudicca.model.toStructuredEntry import java.time.Instant import java.time.OffsetDateTime import java.time.ZoneId @@ -19,7 +23,7 @@ object Utils { fun order(entries: Collection, dateCache: ConcurrentHashMap): List { return entries.toList() - .map { Pair(it, getStartDate(it[SemanticKeys.STARTDATE], dateCache)) } + .map { Pair(it, getStartDate(it, dateCache)) } .sortedWith( Comparator .comparing, OffsetDateTime> { it.second } @@ -29,12 +33,22 @@ object Utils { } private fun getStartDate( - dateText: String?, + entry: Entry, startDateCache: ConcurrentHashMap ): OffsetDateTime { - if (dateText == null) { + val optionalDateText = entry.toStructuredEntry().selectKey( + KeySelector.builder(SemanticKeys.STARTDATE).thenVariant( + VariantConstants.FORMAT_VARIANT_NAME, + listOf( + VariantConstants.FormatVariantConstants.DATE_FORMAT_NAME, + VariantConstants.FormatVariantConstants.TEXT_FORMAT_NAME + ) + ).build() + ) + if (optionalDateText.isEmpty) { return Instant.ofEpochMilli(0).atOffset(ZoneOffset.MIN) } + val dateText = optionalDateText.get().second if (startDateCache.containsKey(dateText)) { return startDateCache[dateText]!! } diff --git a/boudicca.base/query-lib/src/main/kotlin/base/boudicca/query/evaluator/OptimizingEvaluator.kt b/boudicca.base/query-lib/src/main/kotlin/base/boudicca/query/evaluator/OptimizingEvaluator.kt index 154c0865..66674ff9 100644 --- a/boudicca.base/query-lib/src/main/kotlin/base/boudicca/query/evaluator/OptimizingEvaluator.kt +++ b/boudicca.base/query-lib/src/main/kotlin/base/boudicca/query/evaluator/OptimizingEvaluator.kt @@ -1,6 +1,11 @@ package base.boudicca.query.evaluator +import base.boudicca.format.ListFormat +import base.boudicca.keyfilters.KeyFilters import base.boudicca.model.Entry +import base.boudicca.model.structured.Key +import base.boudicca.model.structured.filterKeys +import base.boudicca.model.toStructuredEntry import base.boudicca.query.* import base.boudicca.query.evaluator.util.EvaluatorUtil import base.boudicca.query.evaluator.util.FullTextIndex @@ -34,7 +39,7 @@ class OptimizingEvaluator(rawEntries: Collection) : Evaluator { // it is faster to iterate over all entries then to sort the resultset entries.filterIndexed { i, _ -> resultSet.get(i) } } else { - Utils.order(resultSet.stream().mapToObj{ entries[it] }.toList(), dateCache) + Utils.order(resultSet.stream().mapToObj { entries[it] }.toList(), dateCache) } return QueryResult( orderedResult @@ -95,8 +100,9 @@ class OptimizingEvaluator(rawEntries: Collection) : Evaluator { private fun hasFieldExpression(expression: HasFieldExpression): BitSet { val resultSet = BitSet() - entries.forEachIndexed { i, event -> - if(event.containsKey(expression.getFieldName()) && event[expression.getFieldName()]!!.isNotEmpty()){ + entries.forEachIndexed { i, entry -> + //TODO this is not performant at all... + if (entry.toStructuredEntry().filterKeys(expression.getKeyFilter()).any { it.second.isNotEmpty() }) { resultSet.set(i) } } @@ -125,52 +131,72 @@ class OptimizingEvaluator(rawEntries: Collection) : Evaluator { private fun equalsExpression(expression: EqualsExpression): BitSet { val lowerCase = expression.getText().lowercase() - return starFieldSearch(expression.getFieldName()) { field -> + return keyFilterSearch(expression.getKeyFilter()) { field -> val index = getOrCreateSimpleIndex("equals", field) { - SimpleIndex(entries.map { it[field]?.lowercase() }, Comparator.naturalOrder()) + SimpleIndex( + entries.flatMapIndexed { entryIndex, entry -> + val value = entry[field] + if (EvaluatorUtil.isList(Key.parse(field))) { + if (value == null) { + emptyList() + } else { + ListFormat + .parseFromString(value) + .map { Pair(entryIndex, it.lowercase()) } + } + } else { + listOf(Pair(entryIndex, value?.lowercase())) + } + }, Comparator.naturalOrder() + ) } index.search { it?.compareTo(lowerCase) ?: -1 } } } private fun containsExpression(expression: ContainsExpression): BitSet { - return starFieldSearch(expression.getFieldName()) { field -> + return keyFilterSearch(expression.getKeyFilter()) { field -> val index = getOrCreateFullTextIndex(field) index.containsSearch(expression.getText()) } } private fun beforeExpression(expression: BeforeExpression): BitSet { - val index = getOrCreateLocalDateIndex(expression.getFieldName()) - return index.search { - if (it != null) { - if (it.isEqual(expression.getDate()) || it.isBefore(expression.getDate())) { - 0 + return keyFilterSearch(expression.getKeyFilter()) { field -> + val index = getOrCreateLocalDateIndex(field) + index.search { + if (it != null) { + if (it.isEqual(expression.getDate()) || it.isBefore(expression.getDate())) { + 0 + } else { + 1 + } } else { - 1 + -1 } - } else { - -1 } } } private fun afterExpression(expression: AfterExpression): BitSet { - val index = getOrCreateLocalDateIndex(expression.getFieldName()) - return index.search { - if (it != null) { - if (it.isEqual(expression.getDate()) || it.isAfter(expression.getDate())) { - 0 + return keyFilterSearch(expression.getKeyFilter()) { field -> + val index = getOrCreateLocalDateIndex(field) + index.search { + if (it != null) { + if (it.isEqual(expression.getDate()) || it.isAfter(expression.getDate())) { + 0 + } else { + -1 + } } else { -1 } - } else { - -1 } } } private fun durationLongerExpression(expression: DurationLongerExpression): BitSet { + //TODO this still does not work 100%, we need a completely different handling here val index = getDurationIndex(expression) val duration = expression.getDuration().toDouble() return index.search { @@ -187,6 +213,7 @@ class OptimizingEvaluator(rawEntries: Collection) : Evaluator { } private fun durationShorterExpression(expression: DurationShorterExpression): BitSet { + //TODO this still does not work 100%, we need a completely different handling here val index = getDurationIndex(expression) val duration = expression.getDuration().toDouble() return index.search { @@ -203,32 +230,32 @@ class OptimizingEvaluator(rawEntries: Collection) : Evaluator { } private fun getDurationIndex(expression: AbstractDurationExpression): SimpleIndex { + //TODO this index name generation can be wrong when & are used in keys val index = - getOrCreateSimpleIndex("duration", expression.getStartDateField() + "&" + expression.getEndDateField()) { - SimpleIndex(entries.map { + getOrCreateSimpleIndex( + "duration", + expression.getStartDateKeyFilter().toKeyString() + "&" + expression.getEndDateKeyFilter() + ) { + SimpleIndex.create(entries.map { EvaluatorUtil.getDuration( - expression.getStartDateField(), - expression.getEndDateField(), - it, dateCache + expression.getStartDateKeyFilter(), + expression.getEndDateKeyFilter(), + it.toStructuredEntry(), dateCache ) - }, Comparator.naturalOrder()) + }, Comparator.naturalOrder()) } return index } private fun getOrCreateLocalDateIndex(fieldName: String): SimpleIndex { val index = getOrCreateSimpleIndex("localDate", fieldName) { - SimpleIndex(entries.map { safeGetLocalDate(it[fieldName], dateCache) }, Comparator.naturalOrder()) + SimpleIndex.create(entries.map { safeGetLocalDate(it[fieldName], dateCache) }, Comparator.naturalOrder()) } return index } - private fun starFieldSearch(fieldName: String, search: (String) -> BitSet): BitSet { - val allFieldsToCheck = if (fieldName == "*") { - allFields - } else { - setOf(fieldName) - } + private fun keyFilterSearch(keyFilter: Key, search: (String) -> BitSet): BitSet { + val allFieldsToCheck = allFields.filter { KeyFilters.doesKeyMatchFilter(Key.parse(it), keyFilter) } val result = BitSet() for (field in allFieldsToCheck) { result.or(search(field)) @@ -252,7 +279,7 @@ class OptimizingEvaluator(rawEntries: Collection) : Evaluator { } } - synchronized(operationCache){ + synchronized(operationCache) { val index = if (!operationCache.containsKey(fieldName)) { val index = indexCreator() operationCache[fieldName] = index diff --git a/boudicca.base/query-lib/src/main/kotlin/base/boudicca/query/evaluator/SimpleEvaluator.kt b/boudicca.base/query-lib/src/main/kotlin/base/boudicca/query/evaluator/SimpleEvaluator.kt index 791fb88d..3ac336f0 100644 --- a/boudicca.base/query-lib/src/main/kotlin/base/boudicca/query/evaluator/SimpleEvaluator.kt +++ b/boudicca.base/query-lib/src/main/kotlin/base/boudicca/query/evaluator/SimpleEvaluator.kt @@ -1,19 +1,25 @@ package base.boudicca.query.evaluator +import base.boudicca.format.DateFormat +import base.boudicca.format.ListFormat import base.boudicca.model.Entry +import base.boudicca.model.structured.Key +import base.boudicca.model.structured.StructuredEntry +import base.boudicca.model.structured.filterKeys +import base.boudicca.model.structured.toFlatEntry +import base.boudicca.model.toStructuredEntry import base.boudicca.query.* import base.boudicca.query.evaluator.util.EvaluatorUtil import java.time.LocalDate import java.time.OffsetDateTime import java.time.ZoneId -import java.time.format.DateTimeFormatter import java.time.format.DateTimeParseException import java.util.concurrent.ConcurrentHashMap class SimpleEvaluator(rawEntries: Collection) : Evaluator { private val dateCache = ConcurrentHashMap() - private val events = Utils.order(rawEntries, dateCache) + private val events = Utils.order(rawEntries, dateCache).map { it.toStructuredEntry() } override fun evaluate(expression: Expression, page: Page): QueryResult { val results = events.filter { matchesExpression(expression, it) } @@ -21,51 +27,62 @@ class SimpleEvaluator(rawEntries: Collection) : Evaluator { results .drop(page.offset) .take(page.size) + .map { it.toFlatEntry() } .toList(), results.size ) } - private fun matchesExpression(expression: Expression, event: Map): Boolean { + private fun matchesExpression(expression: Expression, entry: StructuredEntry): Boolean { when (expression) { is EqualsExpression -> { - if (expression.getFieldName() == "*") { - return event.values.any { it.equals(expression.getText(), true) } - } - return event.containsKey(expression.getFieldName()) - && event[expression.getFieldName()].equals(expression.getText(), true) + return entry + .filterKeys(expression.getKeyFilter()) + .filter { EvaluatorUtil.isTextMarkdownOrList(it.first) } + .any { + if (EvaluatorUtil.isList(it.first)) { + parseList(it).any { listValue -> listValue.equals(expression.getText(), true) } + } else { + it.second.equals(expression.getText(), true) + } + } } is ContainsExpression -> { - if (expression.getFieldName() == "*") { - return event.values.any { it.contains(expression.getText(), true) } - } - return event.containsKey(expression.getFieldName()) - && event[expression.getFieldName()]!!.contains(expression.getText(), true) + return entry + .filterKeys(expression.getKeyFilter()) + .filter { EvaluatorUtil.isTextMarkdownOrList(it.first) } + .any { + if (EvaluatorUtil.isList(it.first)) { + parseList(it).any { listValue -> listValue.contains(expression.getText(), true) } + } else { + it.second.contains(expression.getText(), true) + } + } } is NotExpression -> { - return !matchesExpression(expression.getChild(), event) + return !matchesExpression(expression.getChild(), entry) } is AndExpression -> { - return matchesExpression(expression.getLeftChild(), event) - && matchesExpression(expression.getRightChild(), event) + return matchesExpression(expression.getLeftChild(), entry) + && matchesExpression(expression.getRightChild(), entry) } is OrExpression -> { - return matchesExpression(expression.getLeftChild(), event) - || matchesExpression(expression.getRightChild(), event) + return matchesExpression(expression.getLeftChild(), entry) + || matchesExpression(expression.getRightChild(), entry) } is BeforeExpression -> { try { - val dateFieldName = expression.getFieldName() - if (!event.containsKey(dateFieldName)) { - return false - } - val startDate = getLocalStartDate(event[dateFieldName]!!) - return startDate.isEqual(expression.getDate()) || startDate.isBefore(expression.getDate()) + val dateTexts = EvaluatorUtil.getDateValues(entry, expression.getKeyFilter()) + return dateTexts + .any { + val startDate = getLocalStartDate(it) + startDate.isEqual(expression.getDate()) || startDate.isBefore(expression.getDate()) + } } catch (e: DateTimeParseException) { return false } @@ -73,12 +90,12 @@ class SimpleEvaluator(rawEntries: Collection) : Evaluator { is AfterExpression -> { try { - val dateFieldName = expression.getFieldName() - if (!event.containsKey(dateFieldName)) { - return false - } - val startDate = getLocalStartDate(event[dateFieldName]!!) - return startDate.isEqual(expression.getDate()) || startDate.isAfter(expression.getDate()) + val dateTexts = EvaluatorUtil.getDateValues(entry, expression.getKeyFilter()) + return dateTexts + .any { + val startDate = getLocalStartDate(it) + startDate.isEqual(expression.getDate()) || startDate.isAfter(expression.getDate()) + } } catch (e: DateTimeParseException) { return false } @@ -87,9 +104,9 @@ class SimpleEvaluator(rawEntries: Collection) : Evaluator { is DurationLongerExpression -> { val duration = EvaluatorUtil.getDuration( - expression.getStartDateField(), - expression.getEndDateField(), - event, dateCache + expression.getStartDateKeyFilter(), + expression.getEndDateKeyFilter(), + entry, dateCache ) return duration >= expression.getDuration().toDouble() } @@ -97,15 +114,15 @@ class SimpleEvaluator(rawEntries: Collection) : Evaluator { is DurationShorterExpression -> { val duration = EvaluatorUtil.getDuration( - expression.getStartDateField(), - expression.getEndDateField(), - event, dateCache + expression.getStartDateKeyFilter(), + expression.getEndDateKeyFilter(), + entry, dateCache ) return duration <= expression.getDuration().toDouble() } is HasFieldExpression -> { - return event.containsKey(expression.getFieldName()) && event[expression.getFieldName()]!!.isNotEmpty() + return entry.filterKeys(expression.getKeyFilter()).any { it.second.isNotEmpty() } } else -> { @@ -114,8 +131,12 @@ class SimpleEvaluator(rawEntries: Collection) : Evaluator { } } + private fun parseList(keyValuePair: Pair): List { + return ListFormat.parseFromString(keyValuePair.second) + } + private fun getLocalStartDate(dateText: String): LocalDate = - OffsetDateTime.parse(dateText, DateTimeFormatter.ISO_DATE_TIME) + DateFormat.parseFromString(dateText) .atZoneSameInstant(ZoneId.of("Europe/Vienna")) .toLocalDate() diff --git a/boudicca.base/query-lib/src/main/kotlin/base/boudicca/query/evaluator/util/EvaluatorUtil.kt b/boudicca.base/query-lib/src/main/kotlin/base/boudicca/query/evaluator/util/EvaluatorUtil.kt index 79ad2cf4..b1cb34db 100644 --- a/boudicca.base/query-lib/src/main/kotlin/base/boudicca/query/evaluator/util/EvaluatorUtil.kt +++ b/boudicca.base/query-lib/src/main/kotlin/base/boudicca/query/evaluator/util/EvaluatorUtil.kt @@ -1,26 +1,34 @@ package base.boudicca.query.evaluator.util +import base.boudicca.format.DateFormat +import base.boudicca.keyfilters.KeyFilters +import base.boudicca.keyfilters.KeySelector +import base.boudicca.model.structured.Key +import base.boudicca.model.structured.StructuredEntry +import base.boudicca.model.structured.VariantConstants +import base.boudicca.model.structured.filterKeys import java.time.Duration import java.time.OffsetDateTime -import java.time.format.DateTimeFormatter import java.time.format.DateTimeParseException import java.time.temporal.ChronoUnit import java.util.concurrent.ConcurrentHashMap -import kotlin.time.measureTime +import kotlin.jvm.optionals.getOrNull object EvaluatorUtil { fun getDuration( - startDateField: String, - endDateField: String, - event: Map, + startDateKeyFilter: Key, + endDateKeyFilter: Key, + entry: StructuredEntry, dataCache: ConcurrentHashMap ): Double { - if (!event.containsKey(startDateField) || !event.containsKey(endDateField)) { + val startDateText = selectDateValue(entry, startDateKeyFilter) + val endDateText = selectDateValue(entry, endDateKeyFilter) + if (startDateText.isNullOrEmpty() || endDateText.isNullOrEmpty()) { return 0.0 } return try { - val startDate = parseDate(event[startDateField]!!, dataCache) - val endDate = parseDate(event[endDateField]!!, dataCache) + val startDate = parseDate(startDateText, dataCache) + val endDate = parseDate(endDateText, dataCache) Duration.of(endDate.toEpochSecond() - startDate.toEpochSecond(), ChronoUnit.SECONDS) .toMillis() .toDouble() / 1000.0 / 60.0 / 60.0 @@ -29,6 +37,27 @@ object EvaluatorUtil { } } + private fun selectDateValue(entry: StructuredEntry, keyFilter: Key): String? { + return KeySelector.builder(keyFilter) + .thenVariant( + VariantConstants.FORMAT_VARIANT_NAME, + listOf( + VariantConstants.FormatVariantConstants.DATE_FORMAT_NAME, + VariantConstants.FormatVariantConstants.TEXT_FORMAT_NAME + ) + ).build() + .selectSingle(entry) + .map { it.second } + .getOrNull() + } + + fun getDateValues(entry: StructuredEntry, dateKeyFilter: Key): List { + return entry + .filterKeys(dateKeyFilter) + .filter { isDate(it.first) } + .map { it.second } + } + fun parseDate( dateText: String, dataCache: ConcurrentHashMap @@ -37,7 +66,7 @@ object EvaluatorUtil { dataCache[dateText]!! } else { // try { - val parsedDate = OffsetDateTime.parse(dateText, DateTimeFormatter.ISO_DATE_TIME) + val parsedDate = DateFormat.parseFromString(dateText) dataCache[dateText] = parsedDate parsedDate // } catch (e: DateTimeParseException) { @@ -131,4 +160,34 @@ object EvaluatorUtil { i++ } } + + fun isTextMarkdownOrList(key: Key): Boolean { + return KeyFilters.doesContainVariantValue( + key, VariantConstants.FORMAT_VARIANT_NAME, + listOf( + VariantConstants.FormatVariantConstants.TEXT_FORMAT_NAME, + VariantConstants.FormatVariantConstants.MARKDOWN_FORMAT_NAME, + VariantConstants.FormatVariantConstants.LIST_FORMAT_NAME, + ) + ) + } + + fun isList(key: Key): Boolean { + return KeyFilters.doesContainVariantValue( + key, VariantConstants.FORMAT_VARIANT_NAME, + listOf( + VariantConstants.FormatVariantConstants.LIST_FORMAT_NAME, + ) + ) + } + + fun isDate(key: Key): Boolean { + return KeyFilters.doesContainVariantValue( + key, VariantConstants.FORMAT_VARIANT_NAME, + listOf( + VariantConstants.FormatVariantConstants.DATE_FORMAT_NAME, + VariantConstants.FormatVariantConstants.TEXT_FORMAT_NAME, //fallback for now... + ) + ) + } } \ No newline at end of file diff --git a/boudicca.base/query-lib/src/main/kotlin/base/boudicca/query/evaluator/util/FullTextIndex.kt b/boudicca.base/query-lib/src/main/kotlin/base/boudicca/query/evaluator/util/FullTextIndex.kt index c54ca928..283b6ebe 100644 --- a/boudicca.base/query-lib/src/main/kotlin/base/boudicca/query/evaluator/util/FullTextIndex.kt +++ b/boudicca.base/query-lib/src/main/kotlin/base/boudicca/query/evaluator/util/FullTextIndex.kt @@ -1,6 +1,8 @@ package base.boudicca.query.evaluator.util +import base.boudicca.format.ListFormat import base.boudicca.model.Entry +import base.boudicca.model.structured.Key import java.nio.ByteBuffer import java.nio.CharBuffer import java.text.BreakIterator @@ -93,11 +95,17 @@ class FullTextIndex(entries: List, field: String) { } private fun getWords(entries: List, field: String): List> { + val key = Key.parse(field) val words = mutableMapOf() entries.forEachIndexed { entryIndex, entry -> if (!entry[field].isNullOrEmpty()) { - val lowercase = entry[field]!!.lowercase() - val newWords = breakText(lowercase) + val entryValue = entry[field]!! + val values = if (EvaluatorUtil.isList(key)) { + ListFormat.parseFromString(entryValue) + } else { + listOf(entryValue) + } + val newWords = values.flatMap { breakText(it.lowercase()) } newWords.forEach { newWord -> if (words.containsKey(newWord)) { diff --git a/boudicca.base/query-lib/src/main/kotlin/base/boudicca/query/evaluator/util/SimpleIndex.kt b/boudicca.base/query-lib/src/main/kotlin/base/boudicca/query/evaluator/util/SimpleIndex.kt index c1fa9616..9a6dc72d 100644 --- a/boudicca.base/query-lib/src/main/kotlin/base/boudicca/query/evaluator/util/SimpleIndex.kt +++ b/boudicca.base/query-lib/src/main/kotlin/base/boudicca/query/evaluator/util/SimpleIndex.kt @@ -2,9 +2,9 @@ package base.boudicca.query.evaluator.util import java.util.* -class SimpleIndex(values: List, comparator: Comparator) { +class SimpleIndex(values: List>, comparator: Comparator) { + private val index = values - .mapIndexed { index, t -> Pair(index, t) } .filter { it.second != null } .sortedWith(Comparator.comparing({ pair -> pair.second }, comparator)) @@ -45,5 +45,10 @@ class SimpleIndex(values: List, comparator: Comparator) { return result } + companion object { + fun create(values: List, comparator: Comparator): SimpleIndex { + return SimpleIndex(values.mapIndexed { i, value -> Pair(i, value) }, comparator) + } + } } diff --git a/boudicca.base/query-lib/src/test/kotlin/base/boudicca/query/evaluator/AbstractEvaluatorTest.kt b/boudicca.base/query-lib/src/test/kotlin/base/boudicca/query/evaluator/AbstractEvaluatorTest.kt index 71d89388..2b20af50 100644 --- a/boudicca.base/query-lib/src/test/kotlin/base/boudicca/query/evaluator/AbstractEvaluatorTest.kt +++ b/boudicca.base/query-lib/src/test/kotlin/base/boudicca/query/evaluator/AbstractEvaluatorTest.kt @@ -22,6 +22,19 @@ abstract class AbstractEvaluatorTest { @Test fun simpleContains() { val events = callEvaluator(ContainsExpression("name", "event")) + assertEquals(4, events.size) + } + + @Test + fun simpleEqualsWithList() { + val events = callEvaluator(EqualsExpression("list", "val1")) + assertEquals(1, events.size) + assertEquals("listyEvent1", events.first()["name"]) + } + + @Test + fun simpleContainsWithList() { + val events = callEvaluator(ContainsExpression("list", "val")) assertEquals(2, events.size) } @@ -41,7 +54,7 @@ abstract class AbstractEvaluatorTest { @Test fun simpleNot() { val events = callEvaluator(NotExpression(EqualsExpression("name", "event1"))) - assertEquals(3, events.size) + assertEquals(5, events.size) } @Test @@ -144,6 +157,36 @@ abstract class AbstractEvaluatorTest { assertEquals("2023-05-29T00:00:00+02:00", events.first()["startDate"]) } + @Test + fun simpleBeforeWithNewFormat() { + val events = + callEvaluator( + BeforeExpression("startDate", "2023-05-27"), + listOf( + entryWithNewFormat("event1", "2023-05-25T00:00:00"), + entryWithNewFormat("event2", "2023-05-28T00:00:00"), + entryWithNewFormat("event3", "2023-05-29T00:00:00"), + ) + ) + assertEquals(1, events.size) + assertEquals("2023-05-25T00:00:00+02:00", events.first()["startDate:format=date"]) + } + + @Test + fun simpleAfterWithNewFormat() { + val events = + callEvaluator( + AfterExpression("startDate", "2023-05-27"), + listOf( + entryWithNewFormat("event1", "2023-05-25T00:00:00"), + entryWithNewFormat("event2", "2023-05-26T00:00:00"), + entryWithNewFormat("event3", "2023-05-29T00:00:00"), + ) + ) + assertEquals(1, events.size) + assertEquals("2023-05-29T00:00:00+02:00", events.first()["startDate:format=date"]) + } + @Test fun simpleAfterInclusiveToday() { val events = @@ -214,6 +257,50 @@ abstract class AbstractEvaluatorTest { assertEquals("event2", events.first()["name"]) } + @Test + fun durationLongerWithNewFormat() { + val events = + callEvaluator( + DurationLongerExpression("startDate", "endDate", 2.0), + listOf( + mapOf( + SemanticKeys.NAME to "event1", + SemanticKeys.STARTDATE + ":format=date" to "2024-05-31T00:00:00Z", + SemanticKeys.ENDDATE + ":format=date" to "2024-05-31T03:00:00Z", + ), + mapOf( + SemanticKeys.NAME to "event2", + SemanticKeys.STARTDATE + ":format=date" to "2024-05-31T00:00:00Z", + SemanticKeys.ENDDATE + ":format=date" to "2024-05-31T00:00:00Z", + ), + ) + ) + assertEquals(1, events.size) + assertEquals("event1", events.first()["name"]) + } + + @Test + fun durationShorterWithNewFormat() { + val events = + callEvaluator( + DurationShorterExpression("startDate", "endDate", 2.0), + listOf( + mapOf( + SemanticKeys.NAME to "event1", + SemanticKeys.STARTDATE + ":format=date" to "2024-05-31T00:00:00Z", + SemanticKeys.ENDDATE + ":format=date" to "2024-05-31T03:00:00Z", + ), + mapOf( + SemanticKeys.NAME to "event2", + SemanticKeys.STARTDATE + ":format=date" to "2024-05-31T00:00:00Z", + SemanticKeys.ENDDATE + ":format=date" to "2024-05-31T00:00:00Z", + ), + ) + ) + assertEquals(1, events.size) + assertEquals("event2", events.first()["name"]) + } + @Test fun durationZero() { val events = @@ -248,6 +335,30 @@ abstract class AbstractEvaluatorTest { assertEquals("event2", events.first()["name"]) } + @Test + fun hasFieldWithKeySelector() { + val events = + callEvaluator( + HasFieldExpression("*:format=date"), + listOf( + mapOf( + SemanticKeys.NAME to "event1", + ), + mapOf( + SemanticKeys.NAME to "event2", + SemanticKeys.STARTDATE+":format=date" to "2024-05-31T00:00:00Z", + ), + mapOf( + SemanticKeys.NAME to "event3", + "random:format=date" to "2024-05-31T00:00:00Z", + ), + ) + ) + assertEquals(2, events.size) + assertEquals("event2", events[0]["name"]) + assertEquals("event3", events[1]["name"]) + } + @Test fun resultsAreSorted() { val events = @@ -299,6 +410,15 @@ abstract class AbstractEvaluatorTest { entry("name" to "event2", "field" to "value2"), entry("name" to "somethingelse", "field" to "wuuut"), entry("name" to "somethingelse2", "field" to "wuuut"), + entry("name" to "listyEvent1", "list:format=list" to "val1,val2"), + entry("name" to "listyEvent2", "list:format=list" to "val3,val4"), + ) + } + + private fun entryWithNewFormat(name: String, startDate: String): Entry { + return entry( + "name" to name, + "startDate:format=date" to DateTimeFormatter.ISO_DATE_TIME.format(parseLocalDate(startDate)) ) } diff --git a/boudicca.base/query-lib/src/test/kotlin/base/boudicca/query/evaluator/util/EvaluatorUtilDurationTest.kt b/boudicca.base/query-lib/src/test/kotlin/base/boudicca/query/evaluator/util/EvaluatorUtilDurationTest.kt index 69b03caf..2961e995 100644 --- a/boudicca.base/query-lib/src/test/kotlin/base/boudicca/query/evaluator/util/EvaluatorUtilDurationTest.kt +++ b/boudicca.base/query-lib/src/test/kotlin/base/boudicca/query/evaluator/util/EvaluatorUtilDurationTest.kt @@ -1,6 +1,9 @@ package base.boudicca.query.evaluator.util import base.boudicca.SemanticKeys +import base.boudicca.model.Entry +import base.boudicca.model.structured.Key +import base.boudicca.model.toStructuredEntry import org.junit.jupiter.api.Assertions import org.junit.jupiter.api.Test import java.util.concurrent.ConcurrentHashMap @@ -87,8 +90,8 @@ class EvaluatorUtilDurationTest { ) } - fun getDuration(startDateField: String, endDateField: String, event: Map): Double { - return EvaluatorUtil.getDuration(startDateField, endDateField, event, ConcurrentHashMap()) + private fun getDuration(startDateField: String, endDateField: String, entry: Entry): Double { + return EvaluatorUtil.getDuration(Key.parse(startDateField), Key.parse(endDateField), entry.toStructuredEntry(), ConcurrentHashMap()) } } \ No newline at end of file diff --git a/boudicca.base/query-lib/src/test/kotlin/base/boudicca/query/evaluator/util/SimpleIndexTest.kt b/boudicca.base/query-lib/src/test/kotlin/base/boudicca/query/evaluator/util/SimpleIndexTest.kt index df602bc5..0b835ed8 100644 --- a/boudicca.base/query-lib/src/test/kotlin/base/boudicca/query/evaluator/util/SimpleIndexTest.kt +++ b/boudicca.base/query-lib/src/test/kotlin/base/boudicca/query/evaluator/util/SimpleIndexTest.kt @@ -33,12 +33,12 @@ class SimpleIndexTest { @Test fun searchNullableIndex() { val index = - SimpleIndex(listOf("c", null, "b", "a", null, null, null), Comparator.naturalOrder()) + SimpleIndex.create(listOf("c", null, "b", "a", null, null, null), Comparator.naturalOrder()) assertEquals(bitsetOf(3), index.search { it?.compareTo("a") ?: -1 }) } private fun createIndex(list: List): SimpleIndex { - return SimpleIndex(list, Comparator.naturalOrder()) + return SimpleIndex.create(list, Comparator.naturalOrder()) } } \ No newline at end of file diff --git a/boudicca.base/search-api/src/main/kotlin/base/boudicca/api/search/model/FilterQueryDTO.kt b/boudicca.base/search-api/src/main/kotlin/base/boudicca/api/search/model/FilterQueryDTO.kt index 7141e492..8dbea03f 100644 --- a/boudicca.base/search-api/src/main/kotlin/base/boudicca/api/search/model/FilterQueryDTO.kt +++ b/boudicca.base/search-api/src/main/kotlin/base/boudicca/api/search/model/FilterQueryDTO.kt @@ -6,6 +6,5 @@ data class FilterQueryDTO( data class FilterQueryEntryDTO( val name: String, - val multiline: Boolean = false ) diff --git a/boudicca.base/search-client/src/main/kotlin/base/boudicca/api/search/FilterQueryDTO.kt b/boudicca.base/search-client/src/main/kotlin/base/boudicca/api/search/FilterQueryDTO.kt index 154e9519..eb9953c4 100644 --- a/boudicca.base/search-client/src/main/kotlin/base/boudicca/api/search/FilterQueryDTO.kt +++ b/boudicca.base/search-client/src/main/kotlin/base/boudicca/api/search/FilterQueryDTO.kt @@ -6,6 +6,5 @@ data class FilterQueryDTO( data class FilterQueryEntryDTO( val name: String, - val multiline: Boolean = false ) diff --git a/boudicca.base/search-client/src/main/kotlin/base/boudicca/api/search/SearchClient.kt b/boudicca.base/search-client/src/main/kotlin/base/boudicca/api/search/SearchClient.kt index b104cf3c..c0023a3d 100644 --- a/boudicca.base/search-client/src/main/kotlin/base/boudicca/api/search/SearchClient.kt +++ b/boudicca.base/search-client/src/main/kotlin/base/boudicca/api/search/SearchClient.kt @@ -45,7 +45,7 @@ class SearchClient(private val searchUrl: String) { private fun mapFilterQueryDTOToApi(filterQueryDTO: FilterQueryDTO): SearchOpenapiFilterQueryDTO { return SearchOpenapiFilterQueryDTO() - .entries(filterQueryDTO.entries.map { FilterQueryEntryDTO().name(it.name).multiline(it.multiline) }) + .entries(filterQueryDTO.entries.map { FilterQueryEntryDTO().name(it.name)}) } private fun mapResultDto(resultDTO: base.boudicca.search.openapi.model.ResultDTO): ResultDTO { diff --git a/boudicca.base/search/src/main/kotlin/base/boudicca/search/service/FiltersService.kt b/boudicca.base/search/src/main/kotlin/base/boudicca/search/service/FiltersService.kt index 2b8ca261..03d63159 100644 --- a/boudicca.base/search/src/main/kotlin/base/boudicca/search/service/FiltersService.kt +++ b/boudicca.base/search/src/main/kotlin/base/boudicca/search/service/FiltersService.kt @@ -3,7 +3,10 @@ package base.boudicca.search.service import base.boudicca.api.search.model.FilterQueryDTO import base.boudicca.api.search.model.FilterQueryEntryDTO import base.boudicca.api.search.model.FilterResultDTO -import base.boudicca.model.Entry +import base.boudicca.format.ListFormat +import base.boudicca.keyfilters.KeyFilters +import base.boudicca.model.structured.* +import base.boudicca.model.toStructuredEntry import org.springframework.context.event.EventListener import org.springframework.stereotype.Service import java.util.concurrent.ConcurrentHashMap @@ -12,12 +15,12 @@ import java.util.concurrent.ConcurrentHashMap class FiltersService { @Volatile - private var entries = emptyList() + private var entries = emptyList() private val cache = ConcurrentHashMap>() @EventListener fun onEventsUpdate(event: EntriesUpdatedEvent) { - this.entries = event.entries.toList() + this.entries = event.entries.toList().map { it.toStructuredEntry() } this.cache.clear() } @@ -36,23 +39,37 @@ class FiltersService { return result } - private fun getFilterValuesFor(entry: FilterQueryEntryDTO): List { + private fun getFilterValuesFor(filterQuery: FilterQueryEntryDTO): List { val result = mutableSetOf() - for (e in entries) { - if (e.containsKey(entry.name)) { - val value = e[entry.name]!! - if (entry.multiline) { - for (line in value.split("\n")) { - result.add(line) + for (entry in entries) { + entry + .filterKeys(Key.parse(filterQuery.name)) + .flatMap { + //TODO can we find utils/a generic way to do this check? + if (isList(it.first)) { + ListFormat.parseFromString(it.second) + } else { + listOf(it.second) } - } else { - result.add(value) } - } + .forEach { + result.add(it) + } } return result.toList() } + //TODO find better place to place those utils? + private fun isList(key: Key): Boolean { + return KeyFilters.containsVariant( + key, + Variant( + VariantConstants.FORMAT_VARIANT_NAME, + VariantConstants.FormatVariantConstants.LIST_FORMAT_NAME + ) + ) + } + } \ No newline at end of file diff --git a/boudicca.base/search/src/main/kotlin/base/boudicca/search/service/QueryService.kt b/boudicca.base/search/src/main/kotlin/base/boudicca/search/service/QueryService.kt index b2858a78..f2f05c6c 100644 --- a/boudicca.base/search/src/main/kotlin/base/boudicca/search/service/QueryService.kt +++ b/boudicca.base/search/src/main/kotlin/base/boudicca/search/service/QueryService.kt @@ -60,8 +60,7 @@ class QueryService @Autowired constructor( private fun evaluateQuery(query: String, page: Page): ResultDTO { return try { - val expression = BoudiccaQueryRunner.parseQuery(query) - val queryResult = evaluator.evaluate(expression, page) + val queryResult = BoudiccaQueryRunner.evaluateQuery(query, page, evaluator) ResultDTO(queryResult.result, queryResult.totalResults, queryResult.error) } catch (e: QueryException) { //TODO this should return a 400 error or something, not a 200 message with an error message...