diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index bff43d0..e414399 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -7,15 +7,15 @@ jobs:
fail-fast: false
matrix:
os: [ubuntu-latest, macos-latest, windows-latest]
- jdk: [8]
+ jdk: [1.8.282,1.11.0,1.17.0-0]
runs-on: ${{ matrix.os }}
steps:
- name: Checkout
- uses: actions/checkout@v2
+ uses: actions/checkout@v4
- name: Setup Scala
- uses: olafurpg/setup-scala@v12
+ uses: olafurpg/setup-scala@v14
with:
- java-version: "adopt@1.${{ matrix.jdk }}"
+ java-version: "zulu@${{ matrix.jdk }}"
- name: Coursier cache
uses: coursier/cache-action@v6
- name: Build
@@ -31,4 +31,4 @@ jobs:
find $HOME/Library/Caches/Coursier/v1 -name "ivydata-*.properties" -delete || true
find $HOME/.ivy2/cache -name "ivydata-*.properties" -delete || true
find $HOME/.cache/coursier/v1 -name "ivydata-*.properties" -delete || true
- find $HOME/.sbt -name "*.lock" -delete || true
\ No newline at end of file
+ find $HOME/.sbt -name "*.lock" -delete || true
diff --git a/README.md b/README.md
index 1274c03..3c7a32c 100644
--- a/README.md
+++ b/README.md
@@ -11,6 +11,8 @@ eXist Indexer for Algolia is a configurable index plug-in for the [eXist-db](htt
## Installation
+The Algolia indexer is dependent on https://github.com/BCDH/cql-module follow the install instructions in the `README`.
+
It's probably a good idea to start with a clean database, which means a completely clean `$EXIST_HOME/webapp/WEB-INF/data` folder.
* The Index plugin requires at least eXist version 3.0.
@@ -19,7 +21,7 @@ It's probably a good idea to start with a clean database, which means a complete
- Make sure eXist is not running
-- Place the jar file named like `exist-algolia-index-assembly-2.13_1.0.0.jar` into eXist's `lib/user`.
+- Place the jar file named like `exist-algolia-index-assembly-2.13_1.0.0.jar` into eXist's `lib/user`, `lib` for eXist-db-6.x.x.
- Modify eXist's `conf.xml` file by adding the following line to the `indexer/modules` section:
@@ -30,6 +32,17 @@ It's probably a good idea to start with a clean database, which means a complete
admin-api-key="YOUR-ALGOLIA-ADMIN-API-KEY"/>
```
+### just for exist-6.x.x
+- add the dependency in `etc/startup.xml`
+```xml
+
+ Your group id
+ AlgoliaIndex
+ 1.1.0
+ exist-algolia-index-assembly-1.1.0-SNAPSHOT.jar
+
+```
+
- Startup eXist.
- For the Collection(s) that you want to index with Algolia, you need to add an Algolia index configuration to eXist's `collection.xconf` file. See [instructions](#collectionconf).
diff --git a/build.sbt b/build.sbt
index e19353e..d91ed2e 100644
--- a/build.sbt
+++ b/build.sbt
@@ -1,5 +1,9 @@
import ReleaseTransformations._
+val jaxbApiV = "3.0.1"
+
+val jaxbImplV = "3.0.2"
+
ThisBuild / versionScheme := Some("semver-spec")
lazy val root = Project("exist-algolia-index", file("."))
@@ -23,16 +27,26 @@ lazy val root = Project("exist-algolia-index", file("."))
name = "Adam Retter",
email = "adam@evolvedbinary.com",
url = url("https://www.evolvedbinary.com")
+ ),
+ Developer(
+ id = "mamroure",
+ name = "Younes Bahloul",
+ email = "younes@evolvedbinary.com",
+ url = url("https://www.evolvedbinary.com")
)
),
headerLicense := Some(HeaderLicense.GPLv3("2016", "Belgrade Center for Digital Humanities")),
+ xjcLibs := Seq(
+ "org.glassfish.jaxb" % "jaxb-xjc" % jaxbImplV,
+ "org.glassfish.jaxb" % "jaxb-runtime" % jaxbImplV
+ ),
libraryDependencies ++= {
val catsCoreV = "2.10.0"
- val existV = "4.4.0"
+ val existV = "6.3.0-SNAPSHOT"
val algoliaV = "2.19.0"
val akkaV = "2.6.20"
- val jacksonV = "2.9.7"
+ val jacksonV = "2.13.4"
Seq(
"org.scala-lang.modules" %% "scala-java8-compat" % "1.0.2",
@@ -40,14 +54,16 @@ lazy val root = Project("exist-algolia-index", file("."))
"org.parboiled" %% "parboiled" % "2.5.0",
+ "jakarta.xml.bind" % "jakarta.xml.bind-api" % jaxbApiV % Provided,
+ "org.glassfish.jaxb" % "jaxb-runtime" % jaxbImplV % Provided,
+
"org.clapper" %% "grizzled-slf4j" % "1.3.4"
exclude("org.slf4j", "slf4j-api"),
- "org.exist-db" % "exist-core" % existV % Provided
- exclude("org.exist-db.thirdparty.javax.xml.xquery", "xqjapi"),
- "net.sf.saxon" % "Saxon-HE" % "9.6.0-7" % Provided,
+ "org.exist-db" % "exist-core" % existV % Provided,
+ "net.sf.saxon" % "Saxon-HE" % "9.9.1-8" % Provided,
"com.fasterxml.jackson.core" % "jackson-core" % jacksonV % Provided,
- "commons-codec" % "commons-codec" % "1.11" % Provided,
+ "commons-codec" % "commons-codec" % "1.15" % Provided,
"com.fasterxml.jackson.core" % "jackson-databind" % jacksonV
exclude("com.fasterxml.jackson.core", "jackson-core"),
@@ -66,7 +82,7 @@ lazy val root = Project("exist-algolia-index", file("."))
"org.easymock" % "easymock" % "3.6" % Test,
"org.exist-db" % "exist-start" % existV % Test,
- "org.apache.httpcomponents" % "httpclient" % "4.5.6" % Test
+ "org.apache.httpcomponents" % "httpclient" % "4.5.14" % Test
)
},
publishMavenStyle := true,
@@ -84,7 +100,8 @@ lazy val root = Project("exist-algolia-index", file("."))
releasePublishArtifactsAction := PgpKeys.publishSigned.value,
resolvers += Resolver.mavenLocal,
- resolvers += "eXist Maven Repo" at "https://raw.github.com/eXist-db/mvn-repo/master/"
+ resolvers += "Evolved Binary eXist-db Release Maven Repo" at "https://repo.evolvedbinary.com/repository/exist-db/",
+ resolvers += "Evolved Binary eXist-db Snapshot Maven Repo" at "https://repo.evolvedbinary.com/repository/exist-db-snapshots/"
)
// Fancy up the Assembly JAR
@@ -122,8 +139,15 @@ Compile / assembly / artifact := {
art.withClassifier(Some("assembly"))
}
-addArtifact(Compile / assembly / artifact, assembly)
+assembly / assemblyMergeStrategy := {
+ case PathList("META-INF", "versions", "9", xs @ _*) => MergeStrategy.discard
+ case x if x.endsWith("module-info.class") => MergeStrategy.discard
+ case x =>
+ val oldStrategy = (assembly / assemblyMergeStrategy).value
+ oldStrategy(x)
+}
+addArtifact(Compile / assembly / artifact, assembly)
pomExtra := (
diff --git a/src/main/scala/org/humanistika/exist/index/algolia/AlgoliaIndexWorker.scala b/src/main/scala/org/humanistika/exist/index/algolia/AlgoliaIndexWorker.scala
index 64e303e..496cf42 100644
--- a/src/main/scala/org/humanistika/exist/index/algolia/AlgoliaIndexWorker.scala
+++ b/src/main/scala/org/humanistika/exist/index/algolia/AlgoliaIndexWorker.scala
@@ -17,7 +17,7 @@
package org.humanistika.exist.index.algolia
-import javax.xml.bind.JAXBContext
+import jakarta.xml.bind.JAXBContext
import org.exist.collections.Collection
import org.exist.dom.persistent._
diff --git a/src/main/scala/org/humanistika/exist/index/algolia/AlgoliaStreamListener.scala b/src/main/scala/org/humanistika/exist/index/algolia/AlgoliaStreamListener.scala
index 5f54316..6b39fd1 100644
--- a/src/main/scala/org/humanistika/exist/index/algolia/AlgoliaStreamListener.scala
+++ b/src/main/scala/org/humanistika/exist/index/algolia/AlgoliaStreamListener.scala
@@ -488,10 +488,11 @@ class AlgoliaStreamListener(indexWorker: AlgoliaIndexWorker, broker: DBBroker, i
private def isDocumentRootObject(rootObject: RootObject): Boolean = Option(rootObject.getPath).forall(path => path.isEmpty || path.equals("/"))
private def isElementRootObject(currentNode: ElementImpl, path: NodePath)(rootObject: RootObject): Boolean = {
- // nodePath(ns, rootObject.getPath) == path
-
val rootObjectPath = NodePathWithPredicates.parse(ns.asScala.toMap, rootObject.getPath)
- if(rootObjectPath.asNodePath == path) {
+ val rootNodePath: NodePath = rootObjectPath.asNodePath
+ val pathsEqual: Boolean = rootNodePath.equals(path)
+
+ if (pathsEqual) {
nodePathAndPredicatesMatch(currentNode)(rootObjectPath)
} else {
false
diff --git a/src/main/scala/org/humanistika/exist/index/algolia/JsonUtil.scala b/src/main/scala/org/humanistika/exist/index/algolia/JsonUtil.scala
index d348c80..ce1c464 100644
--- a/src/main/scala/org/humanistika/exist/index/algolia/JsonUtil.scala
+++ b/src/main/scala/org/humanistika/exist/index/algolia/JsonUtil.scala
@@ -1,6 +1,6 @@
package org.humanistika.exist.index.algolia
-import javax.xml.bind.DatatypeConverter
+import jakarta.xml.bind.DatatypeConverter
import com.fasterxml.jackson.core.JsonGenerator
import org.humanistika.exist.index.algolia.LiteralTypeConfig.LiteralTypeConfig
diff --git a/src/test/scala/org/humanistika/exist/index/algolia/AlgoliaStreamListenerIntegrationSpec.scala b/src/test/scala/org/humanistika/exist/index/algolia/AlgoliaStreamListenerIntegrationSpec.scala
index 7d4e6e3..3872e48 100644
--- a/src/test/scala/org/humanistika/exist/index/algolia/AlgoliaStreamListenerIntegrationSpec.scala
+++ b/src/test/scala/org/humanistika/exist/index/algolia/AlgoliaStreamListenerIntegrationSpec.scala
@@ -1,23 +1,22 @@
package org.humanistika.exist.index.algolia
import java.nio.file.{Path, Paths}
-
import akka.actor.{ActorRef, ActorSystem}
import org.exist.collections.CollectionConfiguration
import org.exist.indexing.IndexWorker
import org.exist.storage.{BrokerPool, DBBroker, ScalaBrokerPoolBridge}
-import org.exist.util.FileInputSource
+import org.exist.util.{FileInputSource, MimeType}
import org.exist.xmldb.XmldbURI
import org.humanistika.exist.index.algolia.AlgoliaIndex.Authentication
import org.humanistika.exist.index.algolia.backend.IncrementalIndexingManagerActor.{Add, FinishDocument, StartDocument}
import org.specs2.mutable.Specification
import org.w3c.dom.Element
-
import AlgoliaStreamListenerIntegrationSpec._
import ExistAPIHelper._
-
import cats.syntax.either._
+import scala.util.Using
+
object AlgoliaStreamListenerIntegrationSpec {
def getTestResource(filename: String): Path = Paths.get(classOf[AlgoliaStreamListenerIntegrationSpec].getClassLoader.getResource(filename).toURI)
}
@@ -44,7 +43,8 @@ class AlgoliaStreamListenerIntegrationSpec extends Specification with ExistServe
val algoliaIndex = createAndRegisterAlgoliaIndex(system, Some(testActor))
// set up an index configuration
- storeCollectionConfig(algoliaIndex, testCollectionPath, getTestResource("integration/basic/collection.xconf"))
+ val collectionConfigPath = storeCollectionConfig(algoliaIndex, testCollectionPath, getTestResource("integration/basic/collection.xconf"))
+ collectionConfigPath must beRight
// store some data (which will be indexed)
val (collectionId, docId) = storeTestDocument(algoliaIndex, testCollectionPath, getTestResource("integration/basic/VSK.TEST.xml"))
@@ -90,7 +90,8 @@ class AlgoliaStreamListenerIntegrationSpec extends Specification with ExistServe
val algoliaIndex = createAndRegisterAlgoliaIndex(system, Some(testActor))
// set up an index configuration
- storeCollectionConfig(algoliaIndex, testCollectionPath, getTestResource("integration/element-without-attributes/collection.xconf"))
+ val collectionConfigPath = storeCollectionConfig(algoliaIndex, testCollectionPath, getTestResource("integration/element-without-attributes/collection.xconf"))
+ collectionConfigPath must beRight
// store some data (which will be indexed)
val (collectionId, docId) = storeTestDocument(algoliaIndex, testCollectionPath, getTestResource("integration/element-without-attributes/algolia-test.xml"))
@@ -124,7 +125,8 @@ class AlgoliaStreamListenerIntegrationSpec extends Specification with ExistServe
val algoliaIndex = createAndRegisterAlgoliaIndex(system, Some(testActor))
// set up an index configuration
- storeCollectionConfig(algoliaIndex, testCollectionPath, getTestResource("integration/predicate/collection.xconf"))
+ val collectionConfigPath = storeCollectionConfig(algoliaIndex, testCollectionPath, getTestResource("integration/predicate/collection.xconf"))
+ collectionConfigPath must beRight
// store some data (which will be indexed)
val (collectionId, docId) = storeTestDocument(algoliaIndex, testCollectionPath, getTestResource("integration/predicate/VSK.TEST.xml"))
@@ -168,7 +170,8 @@ class AlgoliaStreamListenerIntegrationSpec extends Specification with ExistServe
val algoliaIndex = createAndRegisterAlgoliaIndex(system, Some(testActor))
// set up an index configuration
- storeCollectionConfig(algoliaIndex, testCollectionPath, getTestResource("integration/multi-predicates/collection.xconf"))
+ val collectionConfigPath = storeCollectionConfig(algoliaIndex, testCollectionPath, getTestResource("integration/multi-predicates/collection.xconf"))
+ collectionConfigPath must beRight
// store some data (which will be indexed)
val (collectionId, docId) = storeTestDocument(algoliaIndex, testCollectionPath, getTestResource("integration/multi-predicates/algolia-test.xml"))
@@ -198,7 +201,8 @@ class AlgoliaStreamListenerIntegrationSpec extends Specification with ExistServe
val algoliaIndex = createAndRegisterAlgoliaIndex(system, Some(testActor))
// set up an index configuration
- storeCollectionConfig(algoliaIndex, testCollectionPath, getTestResource("integration/user-specified-docId/collection.xconf"))
+ val collectionConfigPath = storeCollectionConfig(algoliaIndex, testCollectionPath, getTestResource("integration/user-specified-docId/collection.xconf"))
+ collectionConfigPath must beRight
// store some data (which will be indexed)
val (collectionId, docId) = storeTestDocument(algoliaIndex, testCollectionPath, getTestResource("integration/user-specified-docId/VSK.TEST.xml"))
@@ -245,7 +249,8 @@ class AlgoliaStreamListenerIntegrationSpec extends Specification with ExistServe
val algoliaIndex = createAndRegisterAlgoliaIndex(system, Some(testActor))
// set up an index configuration
- storeCollectionConfig(algoliaIndex, testCollectionPath, getTestResource("integration/user-specified-docId-and-nodeId/collection.xconf"))
+ val collectionConfigPath = storeCollectionConfig(algoliaIndex, testCollectionPath, getTestResource("integration/user-specified-docId-and-nodeId/collection.xconf"))
+ collectionConfigPath must beRight
// store some data (which will be indexed)
val (collectionId, docId) = storeTestDocument(algoliaIndex, testCollectionPath, getTestResource("integration/user-specified-docId-and-nodeId/VSK.TEST.xml"))
@@ -291,7 +296,8 @@ class AlgoliaStreamListenerIntegrationSpec extends Specification with ExistServe
val algoliaIndex = createAndRegisterAlgoliaIndex(system, Some(testActor))
// set up an index configuration
- storeCollectionConfig(algoliaIndex, testCollectionPath, getTestResource("integration/object-based-text-nodes/collection.xconf"))
+ val collectionConfigPath = storeCollectionConfig(algoliaIndex, testCollectionPath, getTestResource("integration/object-based-text-nodes/collection.xconf"))
+ collectionConfigPath must beRight
// store some data (which will be indexed)
val (collectionId, docId) = storeTestDocument(algoliaIndex, testCollectionPath, getTestResource("integration/object-based-text-nodes/MZ.RGJS.xml"))
@@ -319,7 +325,8 @@ class AlgoliaStreamListenerIntegrationSpec extends Specification with ExistServe
val algoliaIndex = createAndRegisterAlgoliaIndex(system, Some(testActor))
// set up an index configuration
- storeCollectionConfig(algoliaIndex, testCollectionPath, getTestResource("integration/object-based-attributes/collection.xconf"))
+ val collectionConfigPath = storeCollectionConfig(algoliaIndex, testCollectionPath, getTestResource("integration/object-based-attributes/collection.xconf"))
+ collectionConfigPath must beRight
// store some data (which will be indexed)
val (collectionId, docId) = storeTestDocument(algoliaIndex, testCollectionPath, getTestResource("integration/object-based-attributes/MZ.RGJS.xml"))
@@ -347,7 +354,8 @@ class AlgoliaStreamListenerIntegrationSpec extends Specification with ExistServe
val algoliaIndex = createAndRegisterAlgoliaIndex(system, Some(testActor))
// set up an index configuration
- storeCollectionConfig(algoliaIndex, testCollectionPath, getTestResource("integration/object-based-text-nodes-and-attributes/collection.xconf"))
+ val collectionConfigPath = storeCollectionConfig(algoliaIndex, testCollectionPath, getTestResource("integration/object-based-text-nodes-and-attributes/collection.xconf"))
+ collectionConfigPath must beRight
// store some data (which will be indexed)
val (collectionId, docId) = storeTestDocument(algoliaIndex, testCollectionPath, getTestResource("integration/object-based-text-nodes-and-attributes/MZ.RGJS.xml"))
@@ -375,7 +383,8 @@ class AlgoliaStreamListenerIntegrationSpec extends Specification with ExistServe
val algoliaIndex = createAndRegisterAlgoliaIndex(system, Some(testActor))
// set up an index configuration
- storeCollectionConfig(algoliaIndex, testCollectionPath, getTestResource("integration/object-based-mixed-content-nodes/collection.xconf"))
+ val collectionConfigPath = storeCollectionConfig(algoliaIndex, testCollectionPath, getTestResource("integration/object-based-mixed-content-nodes/collection.xconf"))
+ collectionConfigPath must beRight
// store some data (which will be indexed)
val (collectionId, docId) = storeTestDocument(algoliaIndex, testCollectionPath, getTestResource("integration/object-based-mixed-content-nodes/mixed-content-etyms.xml"))
@@ -403,7 +412,8 @@ class AlgoliaStreamListenerIntegrationSpec extends Specification with ExistServe
val algoliaIndex = createAndRegisterAlgoliaIndex(system, Some(testActor))
// set up an index configuration
- storeCollectionConfig(algoliaIndex, testCollectionPath, getTestResource("integration/attribute-based-text-nodes/collection.xconf"))
+ val collectionConfigPath = storeCollectionConfig(algoliaIndex, testCollectionPath, getTestResource("integration/attribute-based-text-nodes/collection.xconf"))
+ collectionConfigPath must beRight
// store some data (which will be indexed)
val (collectionId, docId) = storeTestDocument(algoliaIndex, testCollectionPath, getTestResource("integration/attribute-based-text-nodes/VSK.SR.xml"))
@@ -451,46 +461,48 @@ class AlgoliaStreamListenerIntegrationSpec extends Specification with ExistServe
algoliaIndex
}
- private def storeCollectionConfig(algoliaIndex: AlgoliaIndex, testCollectionPath: XmldbURI, collectionXconfFile: Path)(implicit brokerPool: BrokerPool) {
- val collectionConf = new FileInputSource(collectionXconfFile)
- withBroker { broker =>
- withTxn { txn =>
- injectAlgoliaIndexWorkerIfNotPresent(broker, algoliaIndex)
+ private def storeCollectionConfig(algoliaIndex: AlgoliaIndex, testCollectionPath: XmldbURI, collectionXconfFile: Path)(implicit brokerPool: BrokerPool) : Either[Exception, XmldbURI] = {
+ Using(new FileInputSource(collectionXconfFile)) { collectionConf =>
+ withBroker { broker =>
+ withTxn { txn =>
+ injectAlgoliaIndexWorkerIfNotPresent(broker, algoliaIndex)
- val collection = broker.getOrCreateCollection(txn, XmldbURI.CONFIG_COLLECTION_URI.append(testCollectionPath))
- broker.saveCollection(txn, collection)
+ Using(broker.getOrCreateCollection(txn, XmldbURI.CONFIG_COLLECTION_URI.append(testCollectionPath))) { collection =>
+ broker.saveCollection(txn, collection)
+ broker.storeDocument(txn, CollectionConfiguration.DEFAULT_COLLECTION_CONFIG_FILE_URI, collectionConf, MimeType.XML_TYPE, collection)
- val indexInfo = collection.validateXMLResource(txn, broker, CollectionConfiguration.DEFAULT_COLLECTION_CONFIG_FILE_URI, collectionConf)
- collection.store(txn, broker, indexInfo, collectionConf)
- //collection.close()
+ XmldbURI.CONFIG_COLLECTION_URI.append(testCollectionPath).append(CollectionConfiguration.DEFAULT_COLLECTION_CONFIG_FILE_URI)
+ }.get
+ }
}
- }
+ }.get.flatten
}
private def storeTestDocument(algoliaIndex: AlgoliaIndex, testCollectionPath: XmldbURI, documentFile: Path)(implicit brokerPool: BrokerPool): (Int, Int) = {
- val dataFile = new FileInputSource(documentFile)
- withBroker { broker =>
- withTxn { txn =>
-
- injectAlgoliaIndexWorkerIfNotPresent(broker, algoliaIndex)
-
- val collection = broker.getOrCreateCollection(txn, testCollectionPath)
- broker.saveCollection(txn, collection)
- val collectionId = collection.getId
-
- val indexInfo = collection.validateXMLResource(txn, broker, XmldbURI.create("VSK.TEST.xml"), dataFile)
- collection.store(txn, broker, indexInfo, dataFile)
- val docId = indexInfo.getDocument.getDocId
- //collection.close()
-
- (collectionId, docId)
+ Using(new FileInputSource(documentFile)) { dataFile =>
+ withBroker { broker =>
+ withTxn { txn =>
+
+ injectAlgoliaIndexWorkerIfNotPresent(broker, algoliaIndex)
+
+ Using(broker.getOrCreateCollection(txn, testCollectionPath)) { collection =>
+ broker.saveCollection(txn, collection)
+ val collectionId = collection.getId
+
+ broker.storeDocument(txn, XmldbURI.create("VSK.TEST.xml"), dataFile, MimeType.XML_TYPE, collection)
+ val doc = collection.getDocument(broker, XmldbURI.create("VSK.TEST.xml"))
+ val docId = doc.getDocId
+
+ (collectionId, docId)
+ }.get
+ }
+ }.flatMap(identity) match {
+ case Left(e) =>
+ throw e
+ case Right(result) =>
+ result
}
- }.flatMap(identity) match {
- case Left(e) =>
- throw e
- case Right(result) =>
- result
- }
+ }.get
}
private def mockIndexModuleConfig() : Element = {
diff --git a/src/test/scala/org/humanistika/exist/index/algolia/ExistSpecHelper.scala b/src/test/scala/org/humanistika/exist/index/algolia/ExistSpecHelper.scala
index 2e9bcc1..61f9eaf 100644
--- a/src/test/scala/org/humanistika/exist/index/algolia/ExistSpecHelper.scala
+++ b/src/test/scala/org/humanistika/exist/index/algolia/ExistSpecHelper.scala
@@ -62,7 +62,7 @@ trait ExistServerStartStopHelper {
var instanceName: Option[String] = None
val configFile: Option[Path] = None
var configProperties: Option[Properties] = None
- var useTemporaryStorage: Boolean = false
+ var useTemporaryStorage: Boolean = true
var disableAutoDeploy: Boolean = true
private var temporaryStorage : Option[Path] = None