Merge pull request #165 from boudicca-events/abl/redo-timebased-queries

Abl/redo timebased queries

QUERY.md

@@ -11,18 +11,17 @@ Queries can look like `name contains Bandname or name contains Bandname2`
 A query is always UTF-8 encoded text and every query is exactly one `expression`, where expressions can take multiple (
 potentially nested) forms. Please note that expressions are case-insensitive:
 
-| Expression Name  | Meaning                                                                                      | Format                          |
-|------------------|----------------------------------------------------------------------------------------------|---------------------------------|
-| Equals           | If a field exactly (but case-insensitive) equals the text value                              | `<fieldname> equals <text>`     |
-| Contains         | If a field contains (case-insensitive) the text value                                        | `<fieldname> contains <text>`   |
-| And              | Both child-expressions have to be true so that the whole expression is true                  | `<expression> and <expression>` |
-| Or               | At least one child-expression has to be true so that the whole expression is true            | `<expression> or <expression>`  |
-| Not              | The child-expression has to be false so that the whole expression is true                    | `not <expression>`              |
-| After            | Filter events starting at or after the given date                                            | `after <date>`                  |
-| Before           | Filter events starting at or before the given date                                           | `before <date>`                 |
-| Grouping         | Marker to identify how expression should be grouped                                          | `( <expression> )`              |
-| Duration Longer  | Filter events on their duration in hours (inclusive), events without endDate have 0 duration | `durationLonger <number`        |
-| Duration Shorter | Filter events on their duration in hours (inclusive), events without endDate have 0 duration | `durationShorter <number>`      |
+| Expression Name | Meaning                                                                                                                                                  | Format                                                                              |
+|-----------------|----------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------|
+| Equals          | If a field exactly (but case-insensitive) equals the text value                                                                                          | `<fieldname> equals <text>`                                                         |
+| Contains        | If a field contains (case-insensitive) the text value                                                                                                    | `<fieldname> contains <text>`                                                       |
+| And             | Both child-expressions have to be true so that the whole expression is true                                                                              | `<expression> and <expression>`                                                     |
+| Or              | At least one child-expression has to be true so that the whole expression is true                                                                        | `<expression> or <expression>`                                                      |
+| Not             | The child-expression has to be false so that the whole expression is true                                                                                | `not <expression>`                                                                  |
+| After           | Filter events starting at or after the given date                                                                                                        | `<dateFieldname> after <date>`                                                      |
+| Before          | Filter events starting at or before the given date                                                                                                       | `<dateFieldname> before <date>`                                                     |
+| Grouping        | Marker to identify how expression should be grouped                                                                                                      | `( <expression> )`                                                                  |
+| Duration        | Filter events on their duration in hours (inclusive), events without startDate or endDate have 0 duration. You an filter for longer or shorter duration. | `duration <fieldname of startDate> <fieldname of endDate> longer\|shorter <number>` |
 
 where
 
@@ -53,7 +52,8 @@ In contrast to other queries or math there is no operator precedence here, they
 * Search for any event in Wien but not in
   Gasometer: `location.city equals Wien and (not location.name equals Gasometer)`
 * Search for any event while I am in Vienna on
-  holiday: `location.city equals Wien and after 2023-05-27 and before 2023-05-31`
+  holiday: `location.city equals Wien and startDate after 2023-05-27 and startDate before 2023-05-31`
+* Search for events with a duration of 2 hours or less: `duration "startDate" "endDate" shorter 2`
 
 See our [Semantic Conventions](SEMANTIC_CONVENTIONS.md) to find common field names.
 

boudicca.base/publisher-event-html/src/main/kotlin/base/boudicca/publisher/event/html/service/EventService.kt

@@ -66,16 +66,16 @@ class EventService @Autowired constructor(@Value("\${boudicca.search.url}") priv
             queryParts.add(equals(SemanticKeys.LOCATION_NAME, searchDTO.locationName!!))
         }
         if (!searchDTO.fromDate.isNullOrBlank()) {
-            queryParts.add(after(LocalDate.parse(searchDTO.fromDate!!, localDateFormatter)))
+            queryParts.add(after(SemanticKeys.STARTDATE, LocalDate.parse(searchDTO.fromDate!!, localDateFormatter)))
         }
         if (!searchDTO.toDate.isNullOrBlank()) {
-            queryParts.add(before(LocalDate.parse(searchDTO.toDate!!, localDateFormatter)))
+            queryParts.add(before(SemanticKeys.STARTDATE, LocalDate.parse(searchDTO.toDate!!, localDateFormatter)))
         }
         if (searchDTO.durationShorter != null) {
-            queryParts.add(durationShorter(searchDTO.durationShorter!!))
+            queryParts.add(durationShorter(SemanticKeys.STARTDATE, SemanticKeys.ENDDATE, searchDTO.durationShorter!!))
         }
         if (searchDTO.durationLonger != null) {
-            queryParts.add(durationLonger(searchDTO.durationLonger!!))
+            queryParts.add(durationLonger(SemanticKeys.STARTDATE, SemanticKeys.ENDDATE, searchDTO.durationLonger!!))
         }
         for (flag in (searchDTO.flags ?: emptyList()).filter { !it.isNullOrBlank() }) {
             queryParts.add(equals(flag!!, "true"))

boudicca.base/search-api/src/main/kotlin/base/boudicca/api/search/BoudiccaQueryBuilder.kt

@@ -28,12 +28,12 @@ object BoudiccaQueryBuilder {
         return "not ($query)"
     }
 
-    fun after(localDate: LocalDate): String {
-        return "after " + DateTimeFormatter.ISO_LOCAL_DATE.format(localDate)
+    fun after(dateFieldName: String, localDate: LocalDate): String {
+        return escapeText(dateFieldName) + " after " + DateTimeFormatter.ISO_LOCAL_DATE.format(localDate)
     }
 
-    fun before(localDate: LocalDate): String {
-        return "before " + DateTimeFormatter.ISO_LOCAL_DATE.format(localDate)
+    fun before(dateFieldName: String, localDate: LocalDate): String {
+        return escapeText(dateFieldName) + " before " + DateTimeFormatter.ISO_LOCAL_DATE.format(localDate)
     }
 
     fun equals(field: String, value: String): String {
@@ -50,12 +50,12 @@ object BoudiccaQueryBuilder {
         return escapeText(field) + " contains " + escapeText(value)
     }
 
-    fun durationLonger(hours: Double): String {
-        return "durationLonger $hours"
+    fun durationLonger(startDateField: String, endDateField: String, hours: Double): String {
+        return "duration ${escapeText(startDateField)} ${escapeText(endDateField)} longer $hours"
     }
 
-    fun durationShorter(hours: Double): String {
-        return "durationShorter $hours"
+    fun durationShorter(startDateField: String, endDateField: String, hours: Double): String {
+        return "duration ${escapeText(startDateField)} ${escapeText(endDateField)} shorter $hours"
     }
 
     fun escapeText(text: String): String {

boudicca.base/search-api/src/test/kotlin/base/boudicca/api/search/BoudiccaQueryBuilderTest.kt

@@ -114,30 +114,30 @@ class BoudiccaQueryBuilderTest {
 
     @Test
     fun simpleAfter() {
-        val query = after(LocalDate.of(2023, 10, 6))
+        val query = after("startDate",LocalDate.of(2023, 10, 6))
 
-        assertEquals("after 2023-10-06", query)
+        assertEquals("\"startDate\" after 2023-10-06", query)
     }
 
     @Test
     fun simpleBefore() {
-        val query = before(LocalDate.of(2023, 10, 6))
+        val query = before("startDate", LocalDate.of(2023, 10, 6))
 
-        assertEquals("before 2023-10-06", query)
+        assertEquals("\"startDate\" before 2023-10-06", query)
     }
 
     @Test
     fun simpleDurationLonger() {
-        val query = durationLonger(5.0)
+        val query = durationLonger("startDate", "endDate", 5.0)
 
-        assertEquals("durationLonger 5.0", query)
+        assertEquals("duration \"startDate\" \"endDate\" longer 5.0", query)
     }
 
     @Test
     fun simpleDurationShorter() {
-        val query = durationShorter(5.0)
+        val query = durationShorter("startDate", "endDate", 5.0)
 
-        assertEquals("durationShorter 5.0", query)
+        assertEquals("duration \"startDate\" \"endDate\" shorter 5.0", query)
     }
 
     @Test

boudicca.base/search/src/main/kotlin/base/boudicca/search/service/SearchService.kt

@@ -68,16 +68,22 @@ class SearchService @Autowired constructor(
             queryParts.add(SemanticKeys.LOCATION_NAME + " equals " + escape(searchDTO.locationName))
         }
         if (searchDTO.fromDate != null) {
-            queryParts.add("after " + formatDate(searchDTO.fromDate))
+            queryParts.add(SemanticKeys.STARTDATE + " after " + formatDate(searchDTO.fromDate))
         }
         if (searchDTO.toDate != null) {
-            queryParts.add("before " + formatDate(searchDTO.toDate))
+            queryParts.add(SemanticKeys.STARTDATE + " before " + formatDate(searchDTO.toDate))
         }
         if (searchDTO.durationShorter != null) {
-            queryParts.add("durationShorter " + formatNumber(searchDTO.durationShorter))
+            queryParts.add(
+                "duration ${escape(SemanticKeys.STARTDATE)} ${escape(SemanticKeys.ENDDATE)} shorter "
+                        + formatNumber(searchDTO.durationShorter)
+            )
         }
         if (searchDTO.durationLonger != null) {
-            queryParts.add("durationLonger " + formatNumber(searchDTO.durationLonger))
+            queryParts.add(
+                "duration ${escape(SemanticKeys.STARTDATE)} ${escape(SemanticKeys.ENDDATE)} longer "
+                        + formatNumber(searchDTO.durationLonger)
+            )
         }
         for (flag in (searchDTO.flags ?: emptyList()).filter { !it.isNullOrBlank() }) {
             queryParts.add(escape(flag!!) + " equals \"true\"")

boudicca.base/search/src/main/kotlin/base/boudicca/search/service/query/Expression.kt

@@ -60,38 +60,11 @@ abstract class FieldAndTextExpression(
     }
 }
 
-abstract class TextExpression(
-    private val name: String,
-    private val text: String,
-) : Expression {
-
-    fun getText(): String {
-        return text
-    }
-
-    override fun toString(): String {
-        return "$name('$text')"
-    }
-}
-
-abstract class NumberExpression(
-    private val name: String,
-    private val number: Number,
-) : Expression {
-
-    fun getNumber(): Number {
-        return number
-    }
-
-    override fun toString(): String {
-        return "$name($number)"
-    }
-}
-
 abstract class DateExpression(
     private val name: String,
+    dateFieldName: String,
     dateText: String,
-) : TextExpression(name, dateText) {
+) : FieldAndTextExpression(name, dateFieldName, dateText) {
 
     private val date: LocalDate
 
@@ -108,7 +81,31 @@ abstract class DateExpression(
     }
 
     override fun toString(): String {
-        return "$name('${date.format(DateTimeFormatter.ISO_LOCAL_DATE)}')"
+        return "$name('${getFieldName()}','${date.format(DateTimeFormatter.ISO_LOCAL_DATE)}')"
+    }
+}
+
+abstract class AbstractDurationExpression(
+    private val name: String,
+    private val startDateField: String,
+    private val endDateField: String,
+    private val duration: Number,
+) : Expression {
+
+    fun getStartDateField(): String {
+        return startDateField
+    }
+
+    fun getEndDateField(): String {
+        return endDateField
+    }
+
+    fun getDuration(): Number {
+        return duration
+    }
+
+    override fun toString(): String {
+        return "$name('${startDateField}','${endDateField}',${duration})"
     }
 }
 
@@ -137,17 +134,23 @@ class EqualsExpression(
 ) : FieldAndTextExpression("EQUALS", fieldName, text)
 
 class BeforeExpression(
-    text: String,
-) : DateExpression("BEFORE", text)
+    dateFieldName: String,
+    dateText: String,
+) : DateExpression("BEFORE", dateFieldName, dateText)
 
 class AfterExpression(
-    text: String,
-) : DateExpression("AFTER", text)
+    dateFieldName: String,
+    dateText: String,
+) : DateExpression("AFTER", dateFieldName, dateText)
 
 class DurationShorterExpression(
+    startDateField: String,
+    endDateField: String,
     duration: Number,
-) : NumberExpression("DURATIONSHORTER", duration)
+) : AbstractDurationExpression("DURATIONSHORTER", startDateField, endDateField, duration)
 
 class DurationLongerExpression(
+    startDateField: String,
+    endDateField: String,
     duration: Number,
-) : NumberExpression("DURATIONLONGER", duration)
+) : AbstractDurationExpression("DURATIONLONGER", startDateField, endDateField, duration)

boudicca.base/search/src/main/kotlin/base/boudicca/search/service/query/Lexer.kt

@@ -52,8 +52,9 @@ class Lexer(private val query: String) {
             "contains" -> tokens.add(Token(TokenType.CONTAINS, null))
             "before" -> tokens.add(Token(TokenType.BEFORE, null))
             "after" -> tokens.add(Token(TokenType.AFTER, null))
-            "durationlonger" -> tokens.add(Token(TokenType.DURATIONLONGER, null))
-            "durationshorter" -> tokens.add(Token(TokenType.DURATIONSHORTER, null))
+            "duration" -> tokens.add(Token(TokenType.DURATION, null))
+            "longer" -> tokens.add(Token(TokenType.LONGER, null))
+            "shorter" -> tokens.add(Token(TokenType.SHORTER, null))
             else -> tokens.add(Token(TokenType.TEXT, token))
         }
         i = tokenEnd

boudicca.base/search/src/main/kotlin/base/boudicca/search/service/query/Parser.kt

@@ -20,12 +20,10 @@ class Parser(private val tokens: List<Token>) {
         }
         val token = tokens[i]
         when (token.getType()) {
-            TokenType.TEXT -> parseFieldAndTextExpression()
+            TokenType.TEXT, TokenType.BEFORE, TokenType.AFTER -> parseFieldAndTextExpression()
             TokenType.NOT -> parseNotExpression()
             TokenType.GROUPING_OPEN -> parseGroupOpen()
-            TokenType.BEFORE, TokenType.AFTER, TokenType.DURATIONLONGER, TokenType.DURATIONSHORTER -> {
-                parseSingleFieldExpression(token)
-            }
+            TokenType.DURATION -> parseDurationExpression(token)
 
             else -> throw IllegalStateException("unexpected token ${token.getType()} at start of expression at index $i")
         }
@@ -39,23 +37,38 @@ class Parser(private val tokens: List<Token>) {
         }
     }
 
-    private fun parseSingleFieldExpression(token: Token) {
-        if (i + 1 >= tokens.size) {
-            throw IllegalStateException("expecting date, found end of query")
+    private fun parseDurationExpression(token: Token) {
+        if (i + 4 >= tokens.size) {
+            throw IllegalStateException("expecting duration query, found end of query")
         }
-        val textToken = getText(i + 1)
-        lastExpression = when (token.getType()) {
-            TokenType.BEFORE -> BeforeExpression(textToken.getToken()!!)
-            TokenType.AFTER -> AfterExpression(textToken.getToken()!!)
-            TokenType.DURATIONSHORTER -> DurationShorterExpression(parseNumber(textToken.getToken()!!))
-            TokenType.DURATIONLONGER -> DurationLongerExpression(parseNumber(textToken.getToken()!!))
+        val startDateField = getText(i + 1)
+        val endDateField = getText(i + 2)
+        val shorterOrLonger = tokens[i + 3]
+        val duration = parseNumber(getText(i + 4).getToken()!!)
+        lastExpression = when (shorterOrLonger.getType()) {
+            TokenType.LONGER -> DurationLongerExpression(
+                startDateField.getToken()!!,
+                endDateField.getToken()!!,
+                duration
+            )
+
+            TokenType.SHORTER -> DurationShorterExpression(
+                startDateField.getToken()!!,
+                endDateField.getToken()!!,
+                duration
+            )
+
             else -> throw IllegalStateException("unknown token type ${token.getType()}")
         }
-        i += 2
+        i += 5
     }
 
     private fun parseNumber(token: String): Number {
-        return BigDecimal(token)
+        try {
+            return BigDecimal(token)
+        } catch (e: NumberFormatException) {
+            throw IllegalStateException("error parsing expected number", e)
+        }
     }
 
     private fun parseGroupOpen() {
@@ -95,25 +108,19 @@ class Parser(private val tokens: List<Token>) {
             throw IllegalStateException("trying to parse text expression and needing 3 arguments, but there are not enough at index $i")
         }
         val fieldName = getText(i)
-        val expression = getFieldAndTextExpression(i + 1)
+        val expression = tokens[i + 1]
         val text = getText(i + 2)
 
         lastExpression = when (expression.getType()) {
             TokenType.CONTAINS -> ContainsExpression(fieldName.getToken()!!, text.getToken()!!)
             TokenType.EQUALS -> EqualsExpression(fieldName.getToken()!!, text.getToken()!!)
+            TokenType.AFTER -> AfterExpression(fieldName.getToken()!!, text.getToken()!!)
+            TokenType.BEFORE -> BeforeExpression(fieldName.getToken()!!, text.getToken()!!)
             else -> throw IllegalStateException("unknown token type ${expression.getType()}")
         }
         i += 3
     }
 
-    private fun getFieldAndTextExpression(i: Int): Token {
-        val token = tokens[i]
-        if (token.getType() != TokenType.CONTAINS && token.getType() != TokenType.EQUALS) {
-            throw IllegalStateException("expecting text expression token at index $i but was ${token.getType()}")
-        }
-        return token
-    }
-
     private fun getText(i: Int): Token {
         val token = tokens[i]
         if (token.getType() != TokenType.TEXT) {

boudicca.base/search/src/main/kotlin/base/boudicca/search/service/query/Token.kt

@@ -25,6 +25,7 @@ enum class TokenType {
     AFTER,
     GROUPING_OPEN,
     GROUPING_CLOSE,
-    DURATIONLONGER,
-    DURATIONSHORTER,
+    DURATION,
+    LONGER,
+    SHORTER,
 }

boudicca.base/search/src/main/kotlin/base/boudicca/search/service/query/evaluator/EvaluatorUtil.kt

@@ -27,13 +27,13 @@ object EvaluatorUtil {
     }
 
 
-    fun getDuration(event: Map<String, String>): Double {
-        if (!event.containsKey(SemanticKeys.STARTDATE) || !event.containsKey(SemanticKeys.ENDDATE)) {
+    fun getDuration(startDateField: String, endDateField: String, event: Map<String, String>): Double {
+        if (!event.containsKey(startDateField) || !event.containsKey(endDateField)) {
             return 0.0
         }
         return try {
-            val startDate = OffsetDateTime.parse(event[SemanticKeys.STARTDATE]!!, DateTimeFormatter.ISO_DATE_TIME)
-            val endDate = OffsetDateTime.parse(event[SemanticKeys.ENDDATE]!!, DateTimeFormatter.ISO_DATE_TIME)
+            val startDate = OffsetDateTime.parse(event[startDateField]!!, DateTimeFormatter.ISO_DATE_TIME)
+            val endDate = OffsetDateTime.parse(event[endDateField]!!, DateTimeFormatter.ISO_DATE_TIME)
             Duration.of(endDate.toEpochSecond() - startDate.toEpochSecond(), ChronoUnit.SECONDS)
                 .toMillis()
                 .toDouble() / 1000.0 / 60.0 / 60.0