MODELIX-914 Remove in-memory query cache and support querying multiple large branches in parallel

docs/global/modules/core/pages/howto/modelql.adoc

@@ -195,3 +195,31 @@ val html = query {
 
 `buildHtmlQuery` and the `requestFragment` operation are similar to the `buildLocalMapping` operation,
 but inside the `onSuccess` block you use the Kotlin HTML DSL.
+
+== Indices/Caching
+
+To search for a node efficiently, `.find` can be used. Internally, it creates a map of all the elements and reuses that
+in following queries.
+
+[source,kotlin]
+--
+val nodeId: String
+root.find({ it.descendants() }, { it.nodeReferenceAsString() }, nodeId.asMono())
+--
+
+It's also possible to search for multiple nodes:
+
+[source,kotlin]
+--
+val name: String
+root.findAll({ it.descendants().ofConcept(C_INamedConcept) }, { it.name }, name.asMono())
+--
+
+Internally, they both use the `memoize` operation. `memoize` stores the result of the query and reuses it without
+re-executing the query.
+
+The `find` example is equivalent to this:
+[source,kotlin]
+--
+root.memoize { it.descendants().associateBy { it.nodeReferenceAsString() } }.get(nodeId.asMono()).filterNotNull()
+--

gradle/libs.versions.toml

@@ -40,6 +40,7 @@ dokka = "1.9.20"
 detekt = "1.23.7"
 xmlunit = "2.10.0"
 kotest = "5.9.1"
+reaktive = "2.2.0"
 testcontainers = "1.20.2"
 
 [libraries]
@@ -135,3 +136,9 @@ jimfs = { group = "com.google.jimfs", name = "jimfs", version = "1.3.0" }
 
 testcontainers = { group = "org.testcontainers", name = "testcontainers", version.ref = "testcontainers" }
 testcontainers-postgresql = { group = "org.testcontainers", name = "postgresql", version.ref = "testcontainers" }
+
+# The official publication with the group ID com.badoo.reaktive is built for JVM 17
+reaktive = { group = "org.modelix.reaktive", name = "reaktive", version.ref = "reaktive" }
+reaktive-testing = { group = "org.modelix.reaktive", name = "reaktive-testing", version.ref = "reaktive" }
+reaktive-annotations = { group = "org.modelix.reaktive", name = "reaktive-annotations", version.ref = "reaktive" }
+reaktive-coroutines-interop = { group = "org.modelix.reaktive", name = "coroutines-interop", version.ref = "reaktive" }

kotlin-utils/src/commonMain/kotlin/org/modelix/kotlin/utils/Atomic.kt

@@ -0,0 +1,28 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.modelix.kotlin.utils
+
+expect class AtomicLong(initial: Long) {
+    fun incrementAndGet(): Long
+    fun get(): Long
+    fun set(newValue: Long)
+    fun addAndGet(delta: Long): Long
+}
+
+expect class AtomicBoolean(initial: Boolean) {
+    fun get(): Boolean
+    fun set(newValue: Boolean)
+    fun compareAndSet(expectedValue: Boolean, newValue: Boolean): Boolean
+    fun getAndSet(newValue: Boolean): Boolean
+}

kotlin-utils/src/commonMain/kotlin/org/modelix/kotlin/utils/ThreadLocal.kt

@@ -0,0 +1,23 @@
+/*
+ * Copyright (c) 2024.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.modelix.kotlin.utils
+
+expect class ThreadLocal<E>(initialValueSupplier: () -> E) {
+    fun get(): E
+    fun set(value: E)
+    fun remove()
+}

kotlin-utils/src/jsMain/kotlin/org/modelix/kotlin/utils/Atomic.js.kt

@@ -0,0 +1,63 @@
+/*
+ * Copyright (c) 2024.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.modelix.kotlin.utils
+
+actual class AtomicLong actual constructor(initial: Long) {
+    private var value: Long = initial
+    actual fun incrementAndGet(): Long {
+        return ++value
+    }
+
+    actual fun get(): Long {
+        return value
+    }
+
+    actual fun set(newValue: Long) {
+        value = newValue
+    }
+
+    actual fun addAndGet(delta: Long): Long {
+        value += delta
+        return value
+    }
+}
+
+actual class AtomicBoolean actual constructor(initial: Boolean) {
+    private var value: Boolean = initial
+    actual fun get(): Boolean {
+        return value
+    }
+
+    actual fun set(newValue: Boolean) {
+        value = newValue
+    }
+
+    actual fun compareAndSet(expectedValue: Boolean, newValue: Boolean): Boolean {
+        if (value == expectedValue) {
+            value = newValue
+            return true
+        } else {
+            return false
+        }
+    }
+
+    actual fun getAndSet(newValue: Boolean): Boolean {
+        val oldValue = value
+        value = newValue
+        return oldValue
+    }
+}

kotlin-utils/src/jsMain/kotlin/org/modelix/kotlin/utils/ThreadLocal.js.kt

@@ -0,0 +1,34 @@
+/*
+ * Copyright (c) 2024.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.modelix.kotlin.utils
+
+actual class ThreadLocal<E> actual constructor(val initialValueSupplier: () -> E) {
+
+    private var value: E = initialValueSupplier()
+
+    actual fun get(): E {
+        return value
+    }
+
+    actual fun set(value: E) {
+        this.value = value
+    }
+
+    actual fun remove() {
+        this.value = initialValueSupplier()
+    }
+}

kotlin-utils/src/jvmMain/kotlin/org/modelix/kotlin/utils/Atomic.jvm.kt

@@ -0,0 +1,20 @@
+/*
+ * Copyright (c) 2024.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.modelix.kotlin.utils
+
+public actual typealias AtomicLong = java.util.concurrent.atomic.AtomicLong
+public actual typealias AtomicBoolean = java.util.concurrent.atomic.AtomicBoolean

kotlin-utils/src/jvmMain/kotlin/org/modelix/kotlin/utils/ContextValue.jvm.kt

@@ -18,7 +18,7 @@ import kotlinx.coroutines.withContext
 
 actual class ContextValue<E>(private val initialStack: List<E>) {
 
-    private val valueStack = ThreadLocal.withInitial { initialStack }
+    private val valueStack = java.lang.ThreadLocal.withInitial { initialStack }
 
     actual constructor() : this(emptyList())
 

kotlin-utils/src/jvmMain/kotlin/org/modelix/kotlin/utils/ThreadLocal.jvm.kt

@@ -0,0 +1,36 @@
+/*
+ * Copyright (c) 2024.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.modelix.kotlin.utils
+
+import java.lang.ThreadLocal
+
+actual class ThreadLocal<E> actual constructor(initialValueSupplier: () -> E) {
+
+    private val threadLocal = ThreadLocal.withInitial(initialValueSupplier)
+
+    actual fun get(): E {
+        return threadLocal.get()
+    }
+
+    actual fun set(value: E) {
+        return threadLocal.set(value)
+    }
+
+    actual fun remove() {
+        return threadLocal.remove()
+    }
+}

model-api-gen-runtime/src/commonMain/kotlin/org/modelix/metamodel/GeneratedConcept.kt

@@ -2,11 +2,14 @@ package org.modelix.metamodel
 
 import org.modelix.model.api.ConceptReference
 import org.modelix.model.api.IChildLink
+import org.modelix.model.api.IChildLinkReference
 import org.modelix.model.api.IConcept
 import org.modelix.model.api.IConceptReference
 import org.modelix.model.api.INode
 import org.modelix.model.api.IProperty
+import org.modelix.model.api.IPropertyReference
 import org.modelix.model.api.IReferenceLink
+import org.modelix.model.api.IReferenceLinkReference
 import org.modelix.model.api.RoleAccessContext
 import org.modelix.model.api.getAllConcepts
 import kotlin.reflect.KClass
@@ -203,6 +206,10 @@ class GeneratedProperty<ValueT>(
     override fun deserializeValue(serialized: String?): ValueT = serializer.deserialize(serialized)
 
     override fun getSimpleName(): String = simpleName
+
+    override fun toReference(): IPropertyReference = IPropertyReference.fromIdAndName(uid, simpleName)
+
+    override fun toString(): String = "${getUID()}(${getConcept().getShortName()}.$simpleName)"
 }
 fun IProperty.typed() = this as? ITypedProperty<*>
 
@@ -233,6 +240,10 @@ abstract class GeneratedChildLink<ChildNodeT : ITypedNode, ChildConceptT : IConc
     }
 
     override fun getSimpleName(): String = simpleName
+
+    override fun toReference(): IChildLinkReference = IChildLinkReference.fromIdAndName(uid, simpleName)
+
+    override fun toString(): String = "${getUID()}(${getConcept().getShortName()}.$simpleName)"
 }
 fun IChildLink.typed(): ITypedChildLink<ITypedNode> {
     return this as? ITypedChildLink<ITypedNode>
@@ -286,5 +297,9 @@ class GeneratedReferenceLink<TargetNodeT : ITypedNode, TargetConceptT : IConcept
     }
 
     override fun getSimpleName(): String = simpleName
+
+    override fun toReference(): IReferenceLinkReference = IReferenceLinkReference.fromIdAndName(uid, simpleName)
+
+    override fun toString(): String = "${getUID()}(${getConcept().getShortName()}.$simpleName)"
 }
 fun IReferenceLink.typed() = this as? ITypedReferenceLink<ITypedNode> ?: UnknownTypedReferenceLink(this)

model-api-gen-runtime/src/commonMain/kotlin/org/modelix/metamodel/UnknownConcept.kt

@@ -5,63 +5,13 @@ import org.modelix.model.api.IConcept
 import org.modelix.model.api.IConceptReference
 import org.modelix.model.api.ILanguage
 import org.modelix.model.api.INode
-import org.modelix.model.api.IProperty
 import org.modelix.model.api.IReferenceLink
-import org.modelix.model.area.IArea
+import org.modelix.model.api.meta.EmptyConcept
+import org.modelix.model.api.meta.NullConcept
 import kotlin.reflect.KClass
 
-abstract class EmptyConcept : IConcept {
-    override fun isAbstract(): Boolean = true
-
-    override fun isSubConceptOf(superConcept: IConcept?): Boolean = superConcept == this
-
-    override fun getDirectSuperConcepts(): List<IConcept> = emptyList()
-
-    override fun isExactly(concept: IConcept?): Boolean = concept == this
-
-    override fun getOwnProperties(): List<IProperty> = emptyList()
-
-    override fun getOwnChildLinks(): List<IChildLink> = emptyList()
-
-    override fun getOwnReferenceLinks(): List<IReferenceLink> = emptyList()
-
-    override fun getAllProperties(): List<IProperty> = emptyList()
-
-    override fun getAllChildLinks(): List<IChildLink> = emptyList()
-
-    override fun getAllReferenceLinks(): List<IReferenceLink> = emptyList()
-
-    override fun getProperty(name: String): IProperty {
-        throw IllegalArgumentException("Cannot get property '$name'. No concept information available for '${getUID()}'.")
-    }
-
-    override fun getChildLink(name: String): IChildLink {
-        throw IllegalArgumentException("Cannot get link '$name'. No concept information available for '${getUID()}'.")
-    }
-
-    override fun getReferenceLink(name: String): IReferenceLink {
-        throw IllegalArgumentException("Cannot get link '$name'. No concept information available for '${getUID()}'.")
-    }
-}
-
-object NullConcept : EmptyConcept(), IConceptReference {
-    override fun getReference(): IConceptReference = this
-
-    override val language: ILanguage?
-        get() = null
-
-    override fun getUID(): String = "null"
-
-    override fun getShortName(): String = "null"
-
-    override fun getLongName(): String = getShortName()
-
-    override fun resolve(area: IArea?): IConcept? {
-        return this
-    }
-
-    override fun serialize(): String = "null"
-}
+@Deprecated("use org.modelix.model.api.meta.NullConcept", ReplaceWith("org.modelix.model.api.meta.NullConcept"))
+val NullConcept = org.modelix.model.api.meta.NullConcept
 
 data class UnknownConcept(private val ref: IConceptReference) : EmptyConcept() {
     override fun getReference(): IConceptReference {

model-api/build.gradle.kts

@@ -11,6 +11,7 @@ kotlin {
         val commonMain by getting {
             dependencies {
                 api(project(":kotlin-utils"))
+                api(project(":streams"))
                 implementation(kotlin("stdlib-common"))
                 implementation(libs.kotlin.logging)
                 implementation(libs.kotlin.serialization.json)