diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bff43d0..e414399 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,15 +7,15 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest, macos-latest, windows-latest] - jdk: [8] + jdk: [1.8.282,1.11.0,1.17.0-0] runs-on: ${{ matrix.os }} steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Setup Scala - uses: olafurpg/setup-scala@v12 + uses: olafurpg/setup-scala@v14 with: - java-version: "adopt@1.${{ matrix.jdk }}" + java-version: "zulu@${{ matrix.jdk }}" - name: Coursier cache uses: coursier/cache-action@v6 - name: Build @@ -31,4 +31,4 @@ jobs: find $HOME/Library/Caches/Coursier/v1 -name "ivydata-*.properties" -delete || true find $HOME/.ivy2/cache -name "ivydata-*.properties" -delete || true find $HOME/.cache/coursier/v1 -name "ivydata-*.properties" -delete || true - find $HOME/.sbt -name "*.lock" -delete || true \ No newline at end of file + find $HOME/.sbt -name "*.lock" -delete || true diff --git a/README.md b/README.md index 1274c03..3c7a32c 100644 --- a/README.md +++ b/README.md @@ -11,6 +11,8 @@ eXist Indexer for Algolia is a configurable index plug-in for the [eXist-db](htt ## Installation +The Algolia indexer is dependent on https://github.com/BCDH/cql-module follow the install instructions in the `README`. + It's probably a good idea to start with a clean database, which means a completely clean `$EXIST_HOME/webapp/WEB-INF/data` folder. * The Index plugin requires at least eXist version 3.0. @@ -19,7 +21,7 @@ It's probably a good idea to start with a clean database, which means a complete - Make sure eXist is not running -- Place the jar file named like `exist-algolia-index-assembly-2.13_1.0.0.jar` into eXist's `lib/user`. +- Place the jar file named like `exist-algolia-index-assembly-2.13_1.0.0.jar` into eXist's `lib/user`, `lib` for eXist-db-6.x.x. - Modify eXist's `conf.xml` file by adding the following line to the `indexer/modules` section: @@ -30,6 +32,17 @@ It's probably a good idea to start with a clean database, which means a complete admin-api-key="YOUR-ALGOLIA-ADMIN-API-KEY"/> ``` +### just for exist-6.x.x +- add the dependency in `etc/startup.xml` +```xml + + Your group id + AlgoliaIndex + 1.1.0 + exist-algolia-index-assembly-1.1.0-SNAPSHOT.jar + +``` + - Startup eXist. - For the Collection(s) that you want to index with Algolia, you need to add an Algolia index configuration to eXist's `collection.xconf` file. See [instructions](#collectionconf). diff --git a/build.sbt b/build.sbt index e19353e..d91ed2e 100644 --- a/build.sbt +++ b/build.sbt @@ -1,5 +1,9 @@ import ReleaseTransformations._ +val jaxbApiV = "3.0.1" + +val jaxbImplV = "3.0.2" + ThisBuild / versionScheme := Some("semver-spec") lazy val root = Project("exist-algolia-index", file(".")) @@ -23,16 +27,26 @@ lazy val root = Project("exist-algolia-index", file(".")) name = "Adam Retter", email = "adam@evolvedbinary.com", url = url("https://www.evolvedbinary.com") + ), + Developer( + id = "mamroure", + name = "Younes Bahloul", + email = "younes@evolvedbinary.com", + url = url("https://www.evolvedbinary.com") ) ), headerLicense := Some(HeaderLicense.GPLv3("2016", "Belgrade Center for Digital Humanities")), + xjcLibs := Seq( + "org.glassfish.jaxb" % "jaxb-xjc" % jaxbImplV, + "org.glassfish.jaxb" % "jaxb-runtime" % jaxbImplV + ), libraryDependencies ++= { val catsCoreV = "2.10.0" - val existV = "4.4.0" + val existV = "6.3.0-SNAPSHOT" val algoliaV = "2.19.0" val akkaV = "2.6.20" - val jacksonV = "2.9.7" + val jacksonV = "2.13.4" Seq( "org.scala-lang.modules" %% "scala-java8-compat" % "1.0.2", @@ -40,14 +54,16 @@ lazy val root = Project("exist-algolia-index", file(".")) "org.parboiled" %% "parboiled" % "2.5.0", + "jakarta.xml.bind" % "jakarta.xml.bind-api" % jaxbApiV % Provided, + "org.glassfish.jaxb" % "jaxb-runtime" % jaxbImplV % Provided, + "org.clapper" %% "grizzled-slf4j" % "1.3.4" exclude("org.slf4j", "slf4j-api"), - "org.exist-db" % "exist-core" % existV % Provided - exclude("org.exist-db.thirdparty.javax.xml.xquery", "xqjapi"), - "net.sf.saxon" % "Saxon-HE" % "9.6.0-7" % Provided, + "org.exist-db" % "exist-core" % existV % Provided, + "net.sf.saxon" % "Saxon-HE" % "9.9.1-8" % Provided, "com.fasterxml.jackson.core" % "jackson-core" % jacksonV % Provided, - "commons-codec" % "commons-codec" % "1.11" % Provided, + "commons-codec" % "commons-codec" % "1.15" % Provided, "com.fasterxml.jackson.core" % "jackson-databind" % jacksonV exclude("com.fasterxml.jackson.core", "jackson-core"), @@ -66,7 +82,7 @@ lazy val root = Project("exist-algolia-index", file(".")) "org.easymock" % "easymock" % "3.6" % Test, "org.exist-db" % "exist-start" % existV % Test, - "org.apache.httpcomponents" % "httpclient" % "4.5.6" % Test + "org.apache.httpcomponents" % "httpclient" % "4.5.14" % Test ) }, publishMavenStyle := true, @@ -84,7 +100,8 @@ lazy val root = Project("exist-algolia-index", file(".")) releasePublishArtifactsAction := PgpKeys.publishSigned.value, resolvers += Resolver.mavenLocal, - resolvers += "eXist Maven Repo" at "https://raw.github.com/eXist-db/mvn-repo/master/" + resolvers += "Evolved Binary eXist-db Release Maven Repo" at "https://repo.evolvedbinary.com/repository/exist-db/", + resolvers += "Evolved Binary eXist-db Snapshot Maven Repo" at "https://repo.evolvedbinary.com/repository/exist-db-snapshots/" ) // Fancy up the Assembly JAR @@ -122,8 +139,15 @@ Compile / assembly / artifact := { art.withClassifier(Some("assembly")) } -addArtifact(Compile / assembly / artifact, assembly) +assembly / assemblyMergeStrategy := { + case PathList("META-INF", "versions", "9", xs @ _*) => MergeStrategy.discard + case x if x.endsWith("module-info.class") => MergeStrategy.discard + case x => + val oldStrategy = (assembly / assemblyMergeStrategy).value + oldStrategy(x) +} +addArtifact(Compile / assembly / artifact, assembly) pomExtra := ( diff --git a/src/main/scala/org/humanistika/exist/index/algolia/AlgoliaIndexWorker.scala b/src/main/scala/org/humanistika/exist/index/algolia/AlgoliaIndexWorker.scala index 64e303e..496cf42 100644 --- a/src/main/scala/org/humanistika/exist/index/algolia/AlgoliaIndexWorker.scala +++ b/src/main/scala/org/humanistika/exist/index/algolia/AlgoliaIndexWorker.scala @@ -17,7 +17,7 @@ package org.humanistika.exist.index.algolia -import javax.xml.bind.JAXBContext +import jakarta.xml.bind.JAXBContext import org.exist.collections.Collection import org.exist.dom.persistent._ diff --git a/src/main/scala/org/humanistika/exist/index/algolia/AlgoliaStreamListener.scala b/src/main/scala/org/humanistika/exist/index/algolia/AlgoliaStreamListener.scala index 5f54316..6b39fd1 100644 --- a/src/main/scala/org/humanistika/exist/index/algolia/AlgoliaStreamListener.scala +++ b/src/main/scala/org/humanistika/exist/index/algolia/AlgoliaStreamListener.scala @@ -488,10 +488,11 @@ class AlgoliaStreamListener(indexWorker: AlgoliaIndexWorker, broker: DBBroker, i private def isDocumentRootObject(rootObject: RootObject): Boolean = Option(rootObject.getPath).forall(path => path.isEmpty || path.equals("/")) private def isElementRootObject(currentNode: ElementImpl, path: NodePath)(rootObject: RootObject): Boolean = { - // nodePath(ns, rootObject.getPath) == path - val rootObjectPath = NodePathWithPredicates.parse(ns.asScala.toMap, rootObject.getPath) - if(rootObjectPath.asNodePath == path) { + val rootNodePath: NodePath = rootObjectPath.asNodePath + val pathsEqual: Boolean = rootNodePath.equals(path) + + if (pathsEqual) { nodePathAndPredicatesMatch(currentNode)(rootObjectPath) } else { false diff --git a/src/main/scala/org/humanistika/exist/index/algolia/JsonUtil.scala b/src/main/scala/org/humanistika/exist/index/algolia/JsonUtil.scala index d348c80..ce1c464 100644 --- a/src/main/scala/org/humanistika/exist/index/algolia/JsonUtil.scala +++ b/src/main/scala/org/humanistika/exist/index/algolia/JsonUtil.scala @@ -1,6 +1,6 @@ package org.humanistika.exist.index.algolia -import javax.xml.bind.DatatypeConverter +import jakarta.xml.bind.DatatypeConverter import com.fasterxml.jackson.core.JsonGenerator import org.humanistika.exist.index.algolia.LiteralTypeConfig.LiteralTypeConfig diff --git a/src/test/scala/org/humanistika/exist/index/algolia/AlgoliaStreamListenerIntegrationSpec.scala b/src/test/scala/org/humanistika/exist/index/algolia/AlgoliaStreamListenerIntegrationSpec.scala index 7d4e6e3..3872e48 100644 --- a/src/test/scala/org/humanistika/exist/index/algolia/AlgoliaStreamListenerIntegrationSpec.scala +++ b/src/test/scala/org/humanistika/exist/index/algolia/AlgoliaStreamListenerIntegrationSpec.scala @@ -1,23 +1,22 @@ package org.humanistika.exist.index.algolia import java.nio.file.{Path, Paths} - import akka.actor.{ActorRef, ActorSystem} import org.exist.collections.CollectionConfiguration import org.exist.indexing.IndexWorker import org.exist.storage.{BrokerPool, DBBroker, ScalaBrokerPoolBridge} -import org.exist.util.FileInputSource +import org.exist.util.{FileInputSource, MimeType} import org.exist.xmldb.XmldbURI import org.humanistika.exist.index.algolia.AlgoliaIndex.Authentication import org.humanistika.exist.index.algolia.backend.IncrementalIndexingManagerActor.{Add, FinishDocument, StartDocument} import org.specs2.mutable.Specification import org.w3c.dom.Element - import AlgoliaStreamListenerIntegrationSpec._ import ExistAPIHelper._ - import cats.syntax.either._ +import scala.util.Using + object AlgoliaStreamListenerIntegrationSpec { def getTestResource(filename: String): Path = Paths.get(classOf[AlgoliaStreamListenerIntegrationSpec].getClassLoader.getResource(filename).toURI) } @@ -44,7 +43,8 @@ class AlgoliaStreamListenerIntegrationSpec extends Specification with ExistServe val algoliaIndex = createAndRegisterAlgoliaIndex(system, Some(testActor)) // set up an index configuration - storeCollectionConfig(algoliaIndex, testCollectionPath, getTestResource("integration/basic/collection.xconf")) + val collectionConfigPath = storeCollectionConfig(algoliaIndex, testCollectionPath, getTestResource("integration/basic/collection.xconf")) + collectionConfigPath must beRight // store some data (which will be indexed) val (collectionId, docId) = storeTestDocument(algoliaIndex, testCollectionPath, getTestResource("integration/basic/VSK.TEST.xml")) @@ -90,7 +90,8 @@ class AlgoliaStreamListenerIntegrationSpec extends Specification with ExistServe val algoliaIndex = createAndRegisterAlgoliaIndex(system, Some(testActor)) // set up an index configuration - storeCollectionConfig(algoliaIndex, testCollectionPath, getTestResource("integration/element-without-attributes/collection.xconf")) + val collectionConfigPath = storeCollectionConfig(algoliaIndex, testCollectionPath, getTestResource("integration/element-without-attributes/collection.xconf")) + collectionConfigPath must beRight // store some data (which will be indexed) val (collectionId, docId) = storeTestDocument(algoliaIndex, testCollectionPath, getTestResource("integration/element-without-attributes/algolia-test.xml")) @@ -124,7 +125,8 @@ class AlgoliaStreamListenerIntegrationSpec extends Specification with ExistServe val algoliaIndex = createAndRegisterAlgoliaIndex(system, Some(testActor)) // set up an index configuration - storeCollectionConfig(algoliaIndex, testCollectionPath, getTestResource("integration/predicate/collection.xconf")) + val collectionConfigPath = storeCollectionConfig(algoliaIndex, testCollectionPath, getTestResource("integration/predicate/collection.xconf")) + collectionConfigPath must beRight // store some data (which will be indexed) val (collectionId, docId) = storeTestDocument(algoliaIndex, testCollectionPath, getTestResource("integration/predicate/VSK.TEST.xml")) @@ -168,7 +170,8 @@ class AlgoliaStreamListenerIntegrationSpec extends Specification with ExistServe val algoliaIndex = createAndRegisterAlgoliaIndex(system, Some(testActor)) // set up an index configuration - storeCollectionConfig(algoliaIndex, testCollectionPath, getTestResource("integration/multi-predicates/collection.xconf")) + val collectionConfigPath = storeCollectionConfig(algoliaIndex, testCollectionPath, getTestResource("integration/multi-predicates/collection.xconf")) + collectionConfigPath must beRight // store some data (which will be indexed) val (collectionId, docId) = storeTestDocument(algoliaIndex, testCollectionPath, getTestResource("integration/multi-predicates/algolia-test.xml")) @@ -198,7 +201,8 @@ class AlgoliaStreamListenerIntegrationSpec extends Specification with ExistServe val algoliaIndex = createAndRegisterAlgoliaIndex(system, Some(testActor)) // set up an index configuration - storeCollectionConfig(algoliaIndex, testCollectionPath, getTestResource("integration/user-specified-docId/collection.xconf")) + val collectionConfigPath = storeCollectionConfig(algoliaIndex, testCollectionPath, getTestResource("integration/user-specified-docId/collection.xconf")) + collectionConfigPath must beRight // store some data (which will be indexed) val (collectionId, docId) = storeTestDocument(algoliaIndex, testCollectionPath, getTestResource("integration/user-specified-docId/VSK.TEST.xml")) @@ -245,7 +249,8 @@ class AlgoliaStreamListenerIntegrationSpec extends Specification with ExistServe val algoliaIndex = createAndRegisterAlgoliaIndex(system, Some(testActor)) // set up an index configuration - storeCollectionConfig(algoliaIndex, testCollectionPath, getTestResource("integration/user-specified-docId-and-nodeId/collection.xconf")) + val collectionConfigPath = storeCollectionConfig(algoliaIndex, testCollectionPath, getTestResource("integration/user-specified-docId-and-nodeId/collection.xconf")) + collectionConfigPath must beRight // store some data (which will be indexed) val (collectionId, docId) = storeTestDocument(algoliaIndex, testCollectionPath, getTestResource("integration/user-specified-docId-and-nodeId/VSK.TEST.xml")) @@ -291,7 +296,8 @@ class AlgoliaStreamListenerIntegrationSpec extends Specification with ExistServe val algoliaIndex = createAndRegisterAlgoliaIndex(system, Some(testActor)) // set up an index configuration - storeCollectionConfig(algoliaIndex, testCollectionPath, getTestResource("integration/object-based-text-nodes/collection.xconf")) + val collectionConfigPath = storeCollectionConfig(algoliaIndex, testCollectionPath, getTestResource("integration/object-based-text-nodes/collection.xconf")) + collectionConfigPath must beRight // store some data (which will be indexed) val (collectionId, docId) = storeTestDocument(algoliaIndex, testCollectionPath, getTestResource("integration/object-based-text-nodes/MZ.RGJS.xml")) @@ -319,7 +325,8 @@ class AlgoliaStreamListenerIntegrationSpec extends Specification with ExistServe val algoliaIndex = createAndRegisterAlgoliaIndex(system, Some(testActor)) // set up an index configuration - storeCollectionConfig(algoliaIndex, testCollectionPath, getTestResource("integration/object-based-attributes/collection.xconf")) + val collectionConfigPath = storeCollectionConfig(algoliaIndex, testCollectionPath, getTestResource("integration/object-based-attributes/collection.xconf")) + collectionConfigPath must beRight // store some data (which will be indexed) val (collectionId, docId) = storeTestDocument(algoliaIndex, testCollectionPath, getTestResource("integration/object-based-attributes/MZ.RGJS.xml")) @@ -347,7 +354,8 @@ class AlgoliaStreamListenerIntegrationSpec extends Specification with ExistServe val algoliaIndex = createAndRegisterAlgoliaIndex(system, Some(testActor)) // set up an index configuration - storeCollectionConfig(algoliaIndex, testCollectionPath, getTestResource("integration/object-based-text-nodes-and-attributes/collection.xconf")) + val collectionConfigPath = storeCollectionConfig(algoliaIndex, testCollectionPath, getTestResource("integration/object-based-text-nodes-and-attributes/collection.xconf")) + collectionConfigPath must beRight // store some data (which will be indexed) val (collectionId, docId) = storeTestDocument(algoliaIndex, testCollectionPath, getTestResource("integration/object-based-text-nodes-and-attributes/MZ.RGJS.xml")) @@ -375,7 +383,8 @@ class AlgoliaStreamListenerIntegrationSpec extends Specification with ExistServe val algoliaIndex = createAndRegisterAlgoliaIndex(system, Some(testActor)) // set up an index configuration - storeCollectionConfig(algoliaIndex, testCollectionPath, getTestResource("integration/object-based-mixed-content-nodes/collection.xconf")) + val collectionConfigPath = storeCollectionConfig(algoliaIndex, testCollectionPath, getTestResource("integration/object-based-mixed-content-nodes/collection.xconf")) + collectionConfigPath must beRight // store some data (which will be indexed) val (collectionId, docId) = storeTestDocument(algoliaIndex, testCollectionPath, getTestResource("integration/object-based-mixed-content-nodes/mixed-content-etyms.xml")) @@ -403,7 +412,8 @@ class AlgoliaStreamListenerIntegrationSpec extends Specification with ExistServe val algoliaIndex = createAndRegisterAlgoliaIndex(system, Some(testActor)) // set up an index configuration - storeCollectionConfig(algoliaIndex, testCollectionPath, getTestResource("integration/attribute-based-text-nodes/collection.xconf")) + val collectionConfigPath = storeCollectionConfig(algoliaIndex, testCollectionPath, getTestResource("integration/attribute-based-text-nodes/collection.xconf")) + collectionConfigPath must beRight // store some data (which will be indexed) val (collectionId, docId) = storeTestDocument(algoliaIndex, testCollectionPath, getTestResource("integration/attribute-based-text-nodes/VSK.SR.xml")) @@ -451,46 +461,48 @@ class AlgoliaStreamListenerIntegrationSpec extends Specification with ExistServe algoliaIndex } - private def storeCollectionConfig(algoliaIndex: AlgoliaIndex, testCollectionPath: XmldbURI, collectionXconfFile: Path)(implicit brokerPool: BrokerPool) { - val collectionConf = new FileInputSource(collectionXconfFile) - withBroker { broker => - withTxn { txn => - injectAlgoliaIndexWorkerIfNotPresent(broker, algoliaIndex) + private def storeCollectionConfig(algoliaIndex: AlgoliaIndex, testCollectionPath: XmldbURI, collectionXconfFile: Path)(implicit brokerPool: BrokerPool) : Either[Exception, XmldbURI] = { + Using(new FileInputSource(collectionXconfFile)) { collectionConf => + withBroker { broker => + withTxn { txn => + injectAlgoliaIndexWorkerIfNotPresent(broker, algoliaIndex) - val collection = broker.getOrCreateCollection(txn, XmldbURI.CONFIG_COLLECTION_URI.append(testCollectionPath)) - broker.saveCollection(txn, collection) + Using(broker.getOrCreateCollection(txn, XmldbURI.CONFIG_COLLECTION_URI.append(testCollectionPath))) { collection => + broker.saveCollection(txn, collection) + broker.storeDocument(txn, CollectionConfiguration.DEFAULT_COLLECTION_CONFIG_FILE_URI, collectionConf, MimeType.XML_TYPE, collection) - val indexInfo = collection.validateXMLResource(txn, broker, CollectionConfiguration.DEFAULT_COLLECTION_CONFIG_FILE_URI, collectionConf) - collection.store(txn, broker, indexInfo, collectionConf) - //collection.close() + XmldbURI.CONFIG_COLLECTION_URI.append(testCollectionPath).append(CollectionConfiguration.DEFAULT_COLLECTION_CONFIG_FILE_URI) + }.get + } } - } + }.get.flatten } private def storeTestDocument(algoliaIndex: AlgoliaIndex, testCollectionPath: XmldbURI, documentFile: Path)(implicit brokerPool: BrokerPool): (Int, Int) = { - val dataFile = new FileInputSource(documentFile) - withBroker { broker => - withTxn { txn => - - injectAlgoliaIndexWorkerIfNotPresent(broker, algoliaIndex) - - val collection = broker.getOrCreateCollection(txn, testCollectionPath) - broker.saveCollection(txn, collection) - val collectionId = collection.getId - - val indexInfo = collection.validateXMLResource(txn, broker, XmldbURI.create("VSK.TEST.xml"), dataFile) - collection.store(txn, broker, indexInfo, dataFile) - val docId = indexInfo.getDocument.getDocId - //collection.close() - - (collectionId, docId) + Using(new FileInputSource(documentFile)) { dataFile => + withBroker { broker => + withTxn { txn => + + injectAlgoliaIndexWorkerIfNotPresent(broker, algoliaIndex) + + Using(broker.getOrCreateCollection(txn, testCollectionPath)) { collection => + broker.saveCollection(txn, collection) + val collectionId = collection.getId + + broker.storeDocument(txn, XmldbURI.create("VSK.TEST.xml"), dataFile, MimeType.XML_TYPE, collection) + val doc = collection.getDocument(broker, XmldbURI.create("VSK.TEST.xml")) + val docId = doc.getDocId + + (collectionId, docId) + }.get + } + }.flatMap(identity) match { + case Left(e) => + throw e + case Right(result) => + result } - }.flatMap(identity) match { - case Left(e) => - throw e - case Right(result) => - result - } + }.get } private def mockIndexModuleConfig() : Element = { diff --git a/src/test/scala/org/humanistika/exist/index/algolia/ExistSpecHelper.scala b/src/test/scala/org/humanistika/exist/index/algolia/ExistSpecHelper.scala index 2e9bcc1..61f9eaf 100644 --- a/src/test/scala/org/humanistika/exist/index/algolia/ExistSpecHelper.scala +++ b/src/test/scala/org/humanistika/exist/index/algolia/ExistSpecHelper.scala @@ -62,7 +62,7 @@ trait ExistServerStartStopHelper { var instanceName: Option[String] = None val configFile: Option[Path] = None var configProperties: Option[Properties] = None - var useTemporaryStorage: Boolean = false + var useTemporaryStorage: Boolean = true var disableAutoDeploy: Boolean = true private var temporaryStorage : Option[Path] = None