From 8928c1568253d25ed4fa28f41b8e97424b435539 Mon Sep 17 00:00:00 2001 From: Jonas Bulcke <127748878+jobulcke@users.noreply.github.com> Date: Wed, 28 Aug 2024 16:34:37 +0200 Subject: [PATCH] feat: testbed impl * chore: cleanup * feat: addition of some valueobjects and cleanup * feat: additional valueobjects * feat: use java instead of JSON template for pipeline creation * feat: extract event stream properties * feat: event stream properties fetched + managers cleaned up + busy waiting fixed * feat: little improvements * feat: refactor Request * fix: broken ldes client status * WIP * feat: shacl validation * WIP * feat: impl finished * chore: big cleanup * feat: pipelinename is distinct for each run * ci: add workflows * chore: add CODEOWNERS * chore: update README part 1 * ci: release process updated * feat: better naming for a param * fix: broken gh actions * fix: broken cicd pipeline --- .github/Dockerfile | 13 + .github/workflows/build-project.yml | 67 ++++ .github/workflows/pr-merged.yml | 110 ++++++ CODEOWNERS | 2 + README.md | 35 +- pom.xml | 41 ++- .../ldes/config/ShaclValidationConfig.java | 24 -- .../ldes/constants/RDFConstants.java | 10 +- .../ldes/gitb/MessagingServiceImpl.java | 163 --------- .../ldes/gitb/ProcessingServiceImpl.java | 101 ------ .../ldes/gitb/ProxyInfo.java | 66 ---- .../ldes/gitb/ServiceConfig.java | 39 --- .../ldes/gitb/StateManager.java | 117 ------- .../ldes/gitb/TestBedNotifier.java | 117 ------- .../informatievlaanderen/ldes/gitb/Utils.java | 330 ------------------ .../ldes/gitb/ValidationServiceImpl.java | 189 +++++----- .../ldes/handlers/ShaclValidationHandler.java | 23 -- .../ldes/http/HttpClientConfig.java | 15 + .../ldes/http/Request.java | 49 --- .../ldes/http/RequestExecutor.java | 26 +- .../ldes/http/requests/DeleteRequest.java | 32 ++ .../ldes/http/requests/GetRequest.java | 32 ++ .../ldes/http/requests/HttpRequest.java | 7 + .../ldes/http/requests/PostRequest.java | 29 ++ .../ldes/ldes/EventStreamFetcher.java | 48 +++ .../ldes/ldes/EventStreamProperties.java | 7 + .../ldes/ldio/LdesClientStatusManager.java | 85 +++++ .../ldes/ldio/LdioManager.java | 96 ----- .../ldes/ldio/LdioPipelineManager.java | 44 +++ .../ldio/config/LdioConfigProperties.java | 41 +++ .../LdesClientStatusUnavailableException.java | 8 + .../ldio/pipeline/LdioComponentBuilder.java | 25 ++ .../ldio/pipeline/LdioLdesClientBuilder.java | 18 + .../pipeline/LdioRepositorySinkBuilder.java | 22 ++ .../pipeline/ValidationPipelineSupplier.java | 50 +++ .../ldes/ldio/valuebojects/ClientStatus.java | 14 + .../ldes/ldio/valuebojects/LdioComponent.java | 10 + .../valuebojects/LdioComponentProperties.java | 8 + .../ldes/ldio/valuebojects/LdioPipeline.java | 18 + .../ldes/rdfrepo/Rdf4jRepositoryManager.java | 46 +-- .../ldes/rdfrepo/RepositoryValidator.java | 74 ++-- .../ldes/services/RDFConverter.java | 12 +- .../ldes/services/TarSupplier.java | 37 ++ .../services/ValidationReportToTarMapper.java | 44 +++ .../ldes/shacl/ShaclValidator.java | 37 ++ .../ldes/valueobjects/Parameters.java | 38 ++ .../valueobjects/ValidationParameters.java | 11 + .../ldes/valueobjects/ValidationReport.java | 43 +++ .../severitylevels/ErrorSeverityLevel.java | 29 ++ .../severitylevels/InfoSeverityLevel.java | 28 ++ .../severitylevels/SeverityLevel.java | 16 + .../severitylevels/SeverityLevels.java | 17 + .../severitylevels/WarningSeverityLevel.java | 30 ++ .../ldes/web/UserInputController.java | 78 ----- src/main/resources/application.properties | 6 +- src/main/resources/ldio-pipeline.json | 28 -- .../ldes/ApplicationTest.java | 6 +- .../ldes/PostRequestAssert.java | 47 +++ .../ldes/gitb/ValidationServiceImplTest.java | 109 ++++++ .../handlers/ShaclValidationHandlerTest.java | 26 -- .../valueobjects/EventStreamFetcherTest.java | 38 ++ .../ldio/LdesClientStatusManagerTest.java | 91 +++++ .../ldes/ldio/LdioPipelineManagerTest.java | 78 +++++ .../ValidationPipelineSupplierTest.java | 38 ++ .../ldes/rdfrepo/RepositoryValidatorTest.java | 79 +++++ .../ldes/shacl/ShaclValidatorTest.java | 49 +++ .../valueobjects/ValidationReportTest.java | 56 +++ src/test/resources/application-test.yaml | 3 + src/test/resources/event-stream.ttl | 53 +++ src/test/resources/ldio-pipeline.json | 25 ++ src/test/resources/test-shape.ttl | 19 + src/test/resources/validate-request.xml | 36 ++ .../resources/validation-report/invalid.ttl | 34 ++ .../resources/validation-report/valid.ttl | 12 + 74 files changed, 2014 insertions(+), 1490 deletions(-) create mode 100644 .github/Dockerfile create mode 100644 .github/workflows/build-project.yml create mode 100644 .github/workflows/pr-merged.yml create mode 100644 CODEOWNERS delete mode 100644 src/main/java/be/vlaanderen/informatievlaanderen/ldes/config/ShaclValidationConfig.java delete mode 100644 src/main/java/be/vlaanderen/informatievlaanderen/ldes/gitb/MessagingServiceImpl.java delete mode 100644 src/main/java/be/vlaanderen/informatievlaanderen/ldes/gitb/ProcessingServiceImpl.java delete mode 100644 src/main/java/be/vlaanderen/informatievlaanderen/ldes/gitb/ProxyInfo.java delete mode 100644 src/main/java/be/vlaanderen/informatievlaanderen/ldes/gitb/StateManager.java delete mode 100644 src/main/java/be/vlaanderen/informatievlaanderen/ldes/gitb/TestBedNotifier.java delete mode 100644 src/main/java/be/vlaanderen/informatievlaanderen/ldes/gitb/Utils.java delete mode 100644 src/main/java/be/vlaanderen/informatievlaanderen/ldes/handlers/ShaclValidationHandler.java create mode 100644 src/main/java/be/vlaanderen/informatievlaanderen/ldes/http/HttpClientConfig.java delete mode 100644 src/main/java/be/vlaanderen/informatievlaanderen/ldes/http/Request.java create mode 100644 src/main/java/be/vlaanderen/informatievlaanderen/ldes/http/requests/DeleteRequest.java create mode 100644 src/main/java/be/vlaanderen/informatievlaanderen/ldes/http/requests/GetRequest.java create mode 100644 src/main/java/be/vlaanderen/informatievlaanderen/ldes/http/requests/HttpRequest.java create mode 100644 src/main/java/be/vlaanderen/informatievlaanderen/ldes/http/requests/PostRequest.java create mode 100644 src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldes/EventStreamFetcher.java create mode 100644 src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldes/EventStreamProperties.java create mode 100644 src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdesClientStatusManager.java delete mode 100644 src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioManager.java create mode 100644 src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioPipelineManager.java create mode 100644 src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioConfigProperties.java create mode 100644 src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/excpeptions/LdesClientStatusUnavailableException.java create mode 100644 src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/pipeline/LdioComponentBuilder.java create mode 100644 src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/pipeline/LdioLdesClientBuilder.java create mode 100644 src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/pipeline/LdioRepositorySinkBuilder.java create mode 100644 src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/pipeline/ValidationPipelineSupplier.java create mode 100644 src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/valuebojects/ClientStatus.java create mode 100644 src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/valuebojects/LdioComponent.java create mode 100644 src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/valuebojects/LdioComponentProperties.java create mode 100644 src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/valuebojects/LdioPipeline.java create mode 100644 src/main/java/be/vlaanderen/informatievlaanderen/ldes/services/TarSupplier.java create mode 100644 src/main/java/be/vlaanderen/informatievlaanderen/ldes/services/ValidationReportToTarMapper.java create mode 100644 src/main/java/be/vlaanderen/informatievlaanderen/ldes/shacl/ShaclValidator.java create mode 100644 src/main/java/be/vlaanderen/informatievlaanderen/ldes/valueobjects/Parameters.java create mode 100644 src/main/java/be/vlaanderen/informatievlaanderen/ldes/valueobjects/ValidationParameters.java create mode 100644 src/main/java/be/vlaanderen/informatievlaanderen/ldes/valueobjects/ValidationReport.java create mode 100644 src/main/java/be/vlaanderen/informatievlaanderen/ldes/valueobjects/severitylevels/ErrorSeverityLevel.java create mode 100644 src/main/java/be/vlaanderen/informatievlaanderen/ldes/valueobjects/severitylevels/InfoSeverityLevel.java create mode 100644 src/main/java/be/vlaanderen/informatievlaanderen/ldes/valueobjects/severitylevels/SeverityLevel.java create mode 100644 src/main/java/be/vlaanderen/informatievlaanderen/ldes/valueobjects/severitylevels/SeverityLevels.java create mode 100644 src/main/java/be/vlaanderen/informatievlaanderen/ldes/valueobjects/severitylevels/WarningSeverityLevel.java delete mode 100644 src/main/java/be/vlaanderen/informatievlaanderen/ldes/web/UserInputController.java delete mode 100644 src/main/resources/ldio-pipeline.json create mode 100644 src/test/java/be/vlaanderen/informatievlaanderen/ldes/PostRequestAssert.java create mode 100644 src/test/java/be/vlaanderen/informatievlaanderen/ldes/gitb/ValidationServiceImplTest.java delete mode 100644 src/test/java/be/vlaanderen/informatievlaanderen/ldes/handlers/ShaclValidationHandlerTest.java create mode 100644 src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldes/valueobjects/EventStreamFetcherTest.java create mode 100644 src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdesClientStatusManagerTest.java create mode 100644 src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioPipelineManagerTest.java create mode 100644 src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/pipeline/ValidationPipelineSupplierTest.java create mode 100644 src/test/java/be/vlaanderen/informatievlaanderen/ldes/rdfrepo/RepositoryValidatorTest.java create mode 100644 src/test/java/be/vlaanderen/informatievlaanderen/ldes/shacl/ShaclValidatorTest.java create mode 100644 src/test/java/be/vlaanderen/informatievlaanderen/ldes/valueobjects/ValidationReportTest.java create mode 100644 src/test/resources/application-test.yaml create mode 100644 src/test/resources/event-stream.ttl create mode 100644 src/test/resources/ldio-pipeline.json create mode 100644 src/test/resources/test-shape.ttl create mode 100644 src/test/resources/validate-request.xml create mode 100644 src/test/resources/validation-report/invalid.ttl create mode 100644 src/test/resources/validation-report/valid.ttl diff --git a/.github/Dockerfile b/.github/Dockerfile new file mode 100644 index 0000000..ad2e832 --- /dev/null +++ b/.github/Dockerfile @@ -0,0 +1,13 @@ +# +# RUN THE APPLICATION +# +FROM amazoncorretto:21-alpine-jdk + +WORKDIR /validator + +COPY ./target/testbed-shacl-validator.jar testbed-shacl-validator.jar + +RUN adduser -D -u 2000 validator +USER validator + +CMD ["java", "-jar", "testbed-shacl-validator.jar"] \ No newline at end of file diff --git a/.github/workflows/build-project.yml b/.github/workflows/build-project.yml new file mode 100644 index 0000000..a7e6105 --- /dev/null +++ b/.github/workflows/build-project.yml @@ -0,0 +1,67 @@ +name: 1.a Build & Test Project +on: + pull_request: + types: [ opened, synchronize, reopened ] + workflow_dispatch: + +jobs: + build: + name: build + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 # Shallow clones should be disabled for a better relevancy of analysis + - name: Set up JDK 21 + uses: actions/setup-java@v4 + with: + java-version: '21' + distribution: 'zulu' + # TODO: setup sonar + # - name: Cache SonarCloud packages + # uses: actions/cache@v1 + # with: + # path: ~/.sonar/cache + # key: ${{ runner.os }}-sonar + # restore-keys: ${{ runner.os }}-sonar + - name: Cache Maven packages + uses: actions/cache@v1 + with: + path: ~/.m2 + key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} + restore-keys: ${{ runner.os }}-m2 + - name: Build and analyze + if: ${{ github.actor != 'dependabot[bot]' && !github.event.pull_request.head.repo.fork }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} +# SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} + run: mvn -B verify + - name: Build (Forks) # https://portal.productboard.com/sonarsource/1-sonarcloud/c/50-sonarcloud-analyzes-external-pull-request + if: ${{ github.actor == 'dependabot[bot]' || github.event.pull_request.head.repo.fork }} + run: mvn -B verify + - name: Upload JARs + uses: actions/upload-artifact@v2 + with: + name: artifacts + path: | + **/testbed-shacl-validator.jar + build-and-push-image: + runs-on: ubuntu-latest + needs: build + steps: + - name: Checkout repository + uses: actions/checkout@v3 + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + - name: Download JARs + uses: actions/download-artifact@v2 + with: + name: artifacts + path: .github + - name: Build and push Docker image + uses: docker/build-push-action@v3 + with: + context: .github + push: false \ No newline at end of file diff --git a/.github/workflows/pr-merged.yml b/.github/workflows/pr-merged.yml new file mode 100644 index 0000000..65af52e --- /dev/null +++ b/.github/workflows/pr-merged.yml @@ -0,0 +1,110 @@ +name: 2. Build & Deploy Project + +on: + release: + types: [ published ] + push: + branches: + - main + workflow_dispatch: + +env: + REGISTRY: ghcr.io + IMAGE_NAME: testbed-shacl-validator + +jobs: + build: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + steps: + - uses: actions/checkout@v4 + - name: Set up JDK 21 + uses: actions/setup-java@v4 + with: + distribution: zulu + java-version: 21 + - name: Cache Maven packages + uses: actions/cache@v1 + with: + path: ~/.m2 + key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} + restore-keys: ${{ runner.os }}-m2 + # TODO: setup Maven + # # Maven + # - name: Set up Maven Central Repository + # uses: actions/setup-java@v4 + # with: + # java-version: '21' + # distribution: 'zulu' + # server-id: ossrh + # server-username: MAVEN_USERNAME + # server-password: MAVEN_PASSWORD + # gpg-private-key: ${{ secrets.OSSRH_GPG_SECRET_KEY }} + # gpg-passphrase: MAVEN_GPG_PASSPHRASE + # TODO: setup Sonar + - name: Analyse & publish package + run: | + mvn -B verify deploy + export VERSION=$(mvn help:evaluate -Dexpression="project.version" -q -DforceStdout) + echo "version=$VERSION" >> $GITHUB_ENV + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Upload JARs + uses: actions/upload-artifact@v2 + with: + name: artifacts + path: | + **/testbed-shacl-validator.jar + create-image: + runs-on: ubuntu-latest + needs: build + steps: + - uses: actions/checkout@v4 + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + - name: Download JARs + uses: actions/download-artifact@v2 + with: + name: artifacts + path: .github + - name: Define docker variables + run: | + if [[ "${{ github.ref }}" == "refs/heads/main" ]]; then + echo "IMAGE_TAG=${{ env.version }}" >> $GITHUB_ENV + echo "IMAGES=ldes/${{ env.IMAGE_NAME }}" >> $GITHUB_ENV + if [[ "${{ env.version }}" != *"SNAPSHOT"* ]]; then + echo "LATEST=latest" >> $GITHUB_ENV + fi + else + echo "IMAGE_TAG=$(date +'%Y%m%d%H%M%S')" >> $GITHUB_ENV + echo "IMAGES=${{ env.REGISTRY }}/Informatievlaanderen/${{ env.IMAGE_NAME }}" >> $GITHUB_ENV + echo "LATEST=latest" >> $GITHUB_ENV + fi + # TODO: push to docker + - name: Log in to the GitHub Container registry + uses: docker/login-action@v2 + with: + registry: ${{ env.REGISTRY }} + username: Informatievlaanderen + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@v4 + with: + images: ${{ env.IMAGES }} + tags: | + type=raw,value=${{env.IMAGE_TAG}} + type=raw,value=${{env.LATEST}} + - name: Build and push Docker image + uses: docker/build-push-action@v3 + with: + context: .github + push: true + tags: ${{ steps.meta.outputs.tags }} + platforms: linux/amd64,linux/arm64 diff --git a/CODEOWNERS b/CODEOWNERS new file mode 100644 index 0000000..c2c3fdc --- /dev/null +++ b/CODEOWNERS @@ -0,0 +1,2 @@ +* @Yalz @rorlic @jobulcke + diff --git a/README.md b/README.md index fdeb3d8..ba73d2d 100644 --- a/README.md +++ b/README.md @@ -2,26 +2,7 @@ This application implements the [GITB test service APIs](https://www.itb.ec.europa.eu/docs/services/latest/) in a [Spring Boot](https://spring.io/projects/spring-boot) web application that is meant to support -[GITB TDL test cases](https://www.itb.ec.europa.eu/docs/tdl/latest/) running in the Interoperability Test Bed. - -## Messaging service implementation - -The sample messaging service is used by the Test Bed to send and receive a text message. When told to `send` a message -this service simply logs it. Regarding received messages, these are provided via HTTP GET call upon which time the -appropriate active test sessions get notified via callback. To manually complete a pending 'receive' call, make a GET -request to http://localhost:8080/input?message=MESSAGE&session=SESSION in which you set the 'MESSAGE' placeholder to the -text to send back, and the 'SESSION' placeholder to the test session ID to notify. Note that the 'session' parameter can -be altogether skipped to notify all pending test sessions. - -Once running, the messaging endpoint's WDSL is available at http://localhost:8080/services/messaging?WSDL. See -[here](https://www.itb.ec.europa.eu/docs/services/latest/messaging/) for further information on messaging service implementations. - -## Processing service implementation - -The sample processing service is used by the Test Bed to lowercase or uppercase a given text input. - -Once running, the processing endpoint's WDSL is available at http://localhost:8080/services/process?WSDL. See -[here](https://www.itb.ec.europa.eu/docs/services/latest/processing/) for further information on validation service implementations. +[GITB TDL test cases](https://www.itb.ec.europa.eu/docs/tdl/latest/) running in the Interoperability Test Bed. ## Validation service implementation @@ -32,6 +13,10 @@ is also returned in case values match but when ignoring casing. Once running, the validation endpoint's WDSL is available at http://localhost:8080/services/validation?WSDL. See [here](https://www.itb.ec.europa.eu/docs/services/latest/validation/) for further information on processing service implementations. +This validation services requires in the validation call two parameters: +1. **ldes-url**: the url of the LDES to validate +2. **shacl-shape**: the shacl shape that will be used to validate the server against to + # Prerequisites The following prerequisites are required: @@ -41,9 +26,7 @@ The following prerequisites are required: # Building and running 1. Build using `mvn clean package`. -2. Once built you can run the application in two ways: - a. With maven: `mvn spring-boot:run`. - b. Standalone: `java -jar ./target/validator-VERSION.jar`. +2. Once built you can run the application using `mvn spring-boot:run`. ## Live reload for development @@ -59,6 +42,8 @@ through Maven: 1. Build the JAR file with `mvn package`. 2. Build the Docker image with `mvn dockerfile:build`. -### Running the Docker container +[//]: # (TODO: how to run) +[//]: # (### Running the Docker container) -Assuming an image name of `local/validator`, it can be ran using `docker --name validator -p 8080:8080 -d local/validator`. \ No newline at end of file +[//]: # () +[//]: # (Assuming an image name of `local/validator`, it can be ran using `docker --name validator -p 8080:8080 -d local/validator`.) \ No newline at end of file diff --git a/pom.xml b/pom.xml index aa568bf..01994ae 100644 --- a/pom.xml +++ b/pom.xml @@ -9,8 +9,8 @@ 3.3.1 be.vlaanderen.informatievlaanderen.ldes - validator - 1.0-SNAPSHOT + testbed-shacl-validator + 1.0.0-SNAPSHOT 1.23.1 @@ -21,7 +21,8 @@ local 8.3.3 - 4.3.6 + 4.3.5 + 24.1.0 @@ -48,6 +49,11 @@ spring-boot-starter-test test + + org.jetbrains + annotations + ${jetbrains-annotations.version} + @@ -74,38 +80,35 @@ org.eclipse.rdf4j - rdf4j-client + rdf4j-runtime pom - 4.3.5 - + ${rdf4j.version} + + + org.eclipse.jetty + jetty-util + + + ${project.artifactId} - org.springframework.boot spring-boot-maven-plugin + + be.vlaanderen.informatievlaanderen.ldes.Application + + build-info repackage - - - com.spotify - dockerfile-maven-plugin - ${com.spotify.dockerfile-maven-plugin.version} - - ${docker.image.prefix}/${project.artifactId} - - target/${project.build.finalName}.jar - - - \ No newline at end of file diff --git a/src/main/java/be/vlaanderen/informatievlaanderen/ldes/config/ShaclValidationConfig.java b/src/main/java/be/vlaanderen/informatievlaanderen/ldes/config/ShaclValidationConfig.java deleted file mode 100644 index 4c9abef..0000000 --- a/src/main/java/be/vlaanderen/informatievlaanderen/ldes/config/ShaclValidationConfig.java +++ /dev/null @@ -1,24 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.config; - -import be.vlaanderen.informatievlaanderen.ldes.http.RequestExecutor; -import be.vlaanderen.informatievlaanderen.ldes.rdfrepo.RepositoryValidator; -import org.springframework.boot.context.properties.EnableConfigurationProperties; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.ComponentScan; -import org.springframework.context.annotation.Configuration; - -@Configuration -@EnableConfigurationProperties() -@ComponentScan("be.vlaanderen.informatievlaanderen.ldes") -public class ShaclValidationConfig { - - @Bean - public RequestExecutor requestExecutor() { - return new RequestExecutor(); - } - @Bean - public RepositoryValidator repositoryValidator() { - return new RepositoryValidator(); - } - -} diff --git a/src/main/java/be/vlaanderen/informatievlaanderen/ldes/constants/RDFConstants.java b/src/main/java/be/vlaanderen/informatievlaanderen/ldes/constants/RDFConstants.java index 3e38f16..18a80d8 100644 --- a/src/main/java/be/vlaanderen/informatievlaanderen/ldes/constants/RDFConstants.java +++ b/src/main/java/be/vlaanderen/informatievlaanderen/ldes/constants/RDFConstants.java @@ -4,11 +4,11 @@ import org.eclipse.rdf4j.model.impl.SimpleValueFactory; public class RDFConstants { - public static String SHACL = "http://www.w3.org/ns/shacl#"; - public static IRI SEVERITY = SimpleValueFactory.getInstance().createIRI(SHACL + "resultSeverity"); - public static IRI VIOLATION = SimpleValueFactory.getInstance().createIRI(SHACL + "Violation"); - public static IRI WARNING = SimpleValueFactory.getInstance().createIRI(SHACL + "Warning"); - public static IRI INFO = SimpleValueFactory.getInstance().createIRI(SHACL + "Info"); + public static final String SHACL = "http://www.w3.org/ns/shacl#"; + public static final IRI SEVERITY = SimpleValueFactory.getInstance().createIRI(SHACL + "resultSeverity"); + public static final IRI VIOLATION = SimpleValueFactory.getInstance().createIRI(SHACL + "Violation"); + public static final IRI WARNING = SimpleValueFactory.getInstance().createIRI(SHACL + "Warning"); + public static final IRI INFO = SimpleValueFactory.getInstance().createIRI(SHACL + "Info"); private RDFConstants() { } } diff --git a/src/main/java/be/vlaanderen/informatievlaanderen/ldes/gitb/MessagingServiceImpl.java b/src/main/java/be/vlaanderen/informatievlaanderen/ldes/gitb/MessagingServiceImpl.java deleted file mode 100644 index 30cd462..0000000 --- a/src/main/java/be/vlaanderen/informatievlaanderen/ldes/gitb/MessagingServiceImpl.java +++ /dev/null @@ -1,163 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.gitb; - -import com.gitb.ms.Void; -import com.gitb.ms.*; -import com.gitb.tr.TestResultType; -import jakarta.annotation.Resource; -import jakarta.xml.ws.WebServiceContext; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; - -/** - * Spring component that realises the messaging service. - */ -@Component -public class MessagingServiceImpl implements MessagingService { - - /** Logger. */ - private static final Logger LOG = LoggerFactory.getLogger(MessagingServiceImpl.class); - - @Autowired - private StateManager stateManager = null; - @Autowired - private Utils utils = null; - @Resource - private WebServiceContext wsContext = null; - - /** - * The purpose of the getModuleDefinition call is to inform its caller on how the service is supposed to be called. - *

- * Note that defining the implementation of this service is optional, and can be empty unless you plan to publish - * the service for use by third parties (in which case it serves as documentation on its expected inputs and outputs). - * - * @param parameters No parameters are expected. - * @return The response. - */ - @Override - public GetModuleDefinitionResponse getModuleDefinition(Void parameters) { - return new GetModuleDefinitionResponse(); - } - - /** - * The initiate operation is called by the test bed when a new test session is being prepared. - *

- * This call expects from the service to do the following: - *

- * - * @param parameters The actor configuration provided by the SUT. - * @return The session ID and any generated configuration to display for the SUT. - */ - @Override - public InitiateResponse initiate(InitiateRequest parameters) { - InitiateResponse response = new InitiateResponse(); - // Get the ReplyTo address for the test bed callbacks based on WS-Addressing. - String replyToAddress = utils.getReplyToAddressFromHeaders(wsContext).orElseThrow(); - // Get the test session ID to use for tracking session state. - String sessionId = utils.getTestSessionIdFromHeaders(wsContext).orElseThrow(); - stateManager.createSession(sessionId, replyToAddress); - LOG.info("Initiated a new session [{}] with callback address [{}]", sessionId, replyToAddress); - return response; - } - - /** - * The receive operation is called when the test bed is expecting to receive a message. - *

- * The goal here is to be informed by the test bed on the characteristics of the message we are expecting to receive. - * These characteristics would need to be recorded as part of this operation in the service's session state so - * that incoming messages can be matched against them. Once the expected message is received, the TestBedNotifier - * can then be used to ping the test bed. - *

- * Besides the expected message's characteristics, the service should also record: - *

- * - * @param parameters The input parameters to consider (if any). - * @return A void result. - */ - @Override - public Void receive(ReceiveRequest parameters) { - LOG.info("Received 'receive' command from test bed for session [{}]", parameters.getSessionId()); - return new Void(); - } - - /** - * The send operation is called when the test bed wants to send a message through this service. - *

- * This is the point where input is received for the call that this service needs to translate into an actual - * communication. This communication would be specific to a communication protocol or a separate system's API. - *

- * The result of the operation is typically an empty success or failure report depending on whether or not the - * communication was successful. This report could however include additional information that would be reported - * back to the test bed. - * - * @param parameters The input parameters and configuration to consider for the send operation. - * @return A status report for the call that will be returned to the test bed. - */ - @Override - public SendResponse send(SendRequest parameters) { - LOG.info("Received 'send' command from test bed for session [{}]", parameters.getSessionId()); - /* - At this point we would expect the actual communication or simulation to take place. In this sample implementation - we simply log the message received from the test bed. - */ - String messageToSend = utils.getRequiredString(parameters.getInput(), "messageToSend"); - LOG.info("The message to send is [{}]", messageToSend); - SendResponse response = new SendResponse(); - response.setReport(utils.createReport(TestResultType.SUCCESS)); - return response; - } - - /** - * The beginTransaction operation is called by the test bed with a transaction starts. - *

- * Often there is no need to take any action here but it could be interesting to do so if you need specific - * actions per transaction. - * - * @param parameters The transaction configuration. - * @return A void result. - */ - @Override - public Void beginTransaction(BeginTransactionRequest parameters) { - LOG.info("Transaction starting for session [{}]", parameters.getSessionId()); - return new Void(); - } - - /** - * The endTransaction operation is the counterpart of the beginTransaction and is called when the transaction terminates. - * - * @param parameters The session ID this transaction related to. - * @return A void result. - */ - @Override - public Void endTransaction(BasicRequest parameters) { - LOG.info("Transaction ending for session [{}]", parameters.getSessionId()); - return new Void(); - } - - /** - * The finalize operation is called by the test bed when a test session completes. - *

- * A typical action that needs to take place here is the cleanup of any resources that were specific to the session - * in question. This would typically involve the state recorded for the session. - * - * @param parameters The session ID that completed. - * @return A void result. - */ - @Override - public Void finalize(FinalizeRequest parameters) { - LOG.info("Finalising session [{}]", parameters.getSessionId()); - // Cleanup in-memory state for the completed session. - stateManager.destroySession(parameters.getSessionId()); - return new Void(); - } - -} diff --git a/src/main/java/be/vlaanderen/informatievlaanderen/ldes/gitb/ProcessingServiceImpl.java b/src/main/java/be/vlaanderen/informatievlaanderen/ldes/gitb/ProcessingServiceImpl.java deleted file mode 100644 index 9a101c4..0000000 --- a/src/main/java/be/vlaanderen/informatievlaanderen/ldes/gitb/ProcessingServiceImpl.java +++ /dev/null @@ -1,101 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.gitb; - -import com.gitb.core.ValueEmbeddingEnumeration; -import com.gitb.ps.Void; -import com.gitb.ps.*; -import com.gitb.tr.TestResultType; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; - -/** - * Spring component that realises the processing service. - */ -@Component -public class ProcessingServiceImpl implements ProcessingService { - - /** Logger. */ - private static final Logger LOG = LoggerFactory.getLogger(ProcessingServiceImpl.class); - - @Autowired - private Utils utils = null; - - /** - * The purpose of the getModuleDefinition call is to inform its caller on how the service is supposed to be called. - *

- * Note that defining the implementation of this service is optional, and can be empty unless you plan to publish - * the service for use by third parties (in which case it serves as documentation on its expected inputs and outputs). - * - * @param parameters No parameters are expected. - * @return The response. - */ - @Override - public GetModuleDefinitionResponse getModuleDefinition(Void parameters) { - return new GetModuleDefinitionResponse(); - } - - /** - * The purpose of the process operation is to execute one of the service's supported operations. - *

- * What would typically take place here is as follows: - *

    - *
  1. Check that the requested operation is indeed supported by the service.
    2. - *
  2. For the requested operation collect and check the provided input parameters.
    3. - *
  3. Perform the requested operation and return the result to the test bed.
    4. - *
- * - * @param processRequest The requested operation and input parameters. - * @return The result. - */ - @Override - public ProcessResponse process(ProcessRequest processRequest) { - LOG.info("Received 'process' command from test bed for session [{}]", processRequest.getSessionId()); - ProcessResponse response = new ProcessResponse(); - response.setReport(utils.createReport(TestResultType.SUCCESS)); - String operation = processRequest.getOperation(); - if (operation == null) { - throw new IllegalArgumentException("No processing operation provided"); - } - String input = utils.getRequiredString(processRequest.getInput(), "input"); - String result = switch (operation) { - case "uppercase" -> input.toUpperCase(); - case "lowercase" -> input.toLowerCase(); - default -> throw new IllegalArgumentException(String.format("Unexpected operation [%s].", operation)); - }; - response.getOutput().add(utils.createAnyContentSimple("output", result, ValueEmbeddingEnumeration.STRING)); - LOG.info("Completed operation [{}]. Input was [{}], output was [{}].", operation, input, result); - return response; - } - - /** - * The purpose of the beginTransaction operation is to begin a unique processing session. - *

- * Transactions are used when processing services need to maintain state across several calls. If this is needed - * then this implementation would generate a session identifier and record the session for subsequent 'process' calls. - *

- * In the typical case where no state needs to be maintained, you can provide an empty implementation for this method. - * - * @param beginTransactionRequest Optional configuration parameters to consider when starting a processing transaction. - * @return The response with the generated session ID for the processing transaction. - */ - @Override - public BeginTransactionResponse beginTransaction(BeginTransactionRequest beginTransactionRequest) { - return new BeginTransactionResponse(); - } - - /** - * The purpose of the endTransaction operation is to complete an ongoing processing session. - *

- * The main actions to be taken as part of this operation are to remove the provided session identifier (if this - * was being recorded to begin with), and to perform any custom cleanup tasks. - * - * @param parameters The identifier of the session to terminate. - * @return A void response. - */ - @Override - public Void endTransaction(BasicRequest parameters) { - return new Void(); - } - -} diff --git a/src/main/java/be/vlaanderen/informatievlaanderen/ldes/gitb/ProxyInfo.java b/src/main/java/be/vlaanderen/informatievlaanderen/ldes/gitb/ProxyInfo.java deleted file mode 100644 index d742494..0000000 --- a/src/main/java/be/vlaanderen/informatievlaanderen/ldes/gitb/ProxyInfo.java +++ /dev/null @@ -1,66 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.gitb; - -import org.apache.cxf.configuration.security.ProxyAuthorizationPolicy; -import org.apache.cxf.transport.http.HTTPConduit; -import org.apache.cxf.transports.http.configuration.ProxyServerType; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Component; - -/** - * Class used to hold and use the proxy configuration. - */ -@Component -public class ProxyInfo { - - @Value("${proxy.enabled:false}") - private boolean enabled; - - @Value("${proxy.server:''}") - private String server; - - @Value("${proxy.port:-1}") - private Integer port; - - @Value("${proxy.type:'HTTP'}") - private String type; - - @Value("${proxy.auth.enabled:false}") - private boolean authEnabled; - - @Value("${proxy.auth.username:''}") - private String username; - - @Value("${proxy.auth.password:''}") - private String password; - - @Value("${proxy.nonProxyHosts:''}") - private String nonProxyHosts; - - /** - * Check to see if a proxy should be used. - * - * @return The check result. - */ - public boolean isEnabled() { - return enabled; - } - - /** - * Apply the proxy configuration to the given CXF HTTPConduit. - * - * @param httpConduit The conduit to process. - */ - public void applyToCxfConduit(HTTPConduit httpConduit) { - httpConduit.getClient().setProxyServer(server); - httpConduit.getClient().setProxyServerPort(port); - httpConduit.getClient().setProxyServerType(ProxyServerType.fromValue(type)); - httpConduit.getClient().setNonProxyHosts(nonProxyHosts); - if (authEnabled) { - if (httpConduit.getProxyAuthorization() == null) { - httpConduit.setProxyAuthorization(new ProxyAuthorizationPolicy()); - } - httpConduit.getProxyAuthorization().setUserName(username); - httpConduit.getProxyAuthorization().setPassword(password); - } - } -} diff --git a/src/main/java/be/vlaanderen/informatievlaanderen/ldes/gitb/ServiceConfig.java b/src/main/java/be/vlaanderen/informatievlaanderen/ldes/gitb/ServiceConfig.java index 66f211f..55f8185 100644 --- a/src/main/java/be/vlaanderen/informatievlaanderen/ldes/gitb/ServiceConfig.java +++ b/src/main/java/be/vlaanderen/informatievlaanderen/ldes/gitb/ServiceConfig.java @@ -1,6 +1,5 @@ package be.vlaanderen.informatievlaanderen.ldes.gitb; -import com.gitb.tr.ObjectFactory; import org.apache.cxf.Bus; import org.apache.cxf.jaxws.EndpointImpl; import org.springframework.context.annotation.Bean; @@ -14,34 +13,6 @@ @Configuration public class ServiceConfig { - /** - * The CXF endpoint that will serve messaging service calls. - * - * @return The endpoint. - */ - @Bean - public EndpointImpl messagingService(Bus cxfBus, MessagingServiceImpl messagingServiceImplementation) { - EndpointImpl endpoint = new EndpointImpl(cxfBus, messagingServiceImplementation); - endpoint.setServiceName(new QName("http://www.gitb.com/ms/v1/", "MessagingServiceService")); - endpoint.setEndpointName(new QName("http://www.gitb.com/ms/v1/", "MessagingServicePort")); - endpoint.publish("/messaging"); - return endpoint; - } - - /** - * The CXF endpoint that will serve processing service calls. - * - * @return The endpoint. - */ - @Bean - public EndpointImpl processingService(Bus cxfBus, ProcessingServiceImpl processingServiceImplementation) { - EndpointImpl endpoint = new EndpointImpl(cxfBus, processingServiceImplementation); - endpoint.setServiceName(new QName("http://www.gitb.com/ps/v1/", "ProcessingServiceService")); - endpoint.setEndpointName(new QName("http://www.gitb.com/ps/v1/", "ProcessingServicePort")); - endpoint.publish("/process"); - return endpoint; - } - /** * The CXF endpoint that will serve validation service calls. * @@ -56,14 +27,4 @@ public EndpointImpl validationService(Bus cxfBus, ValidationServiceImpl validati return endpoint; } - /** - * The ObjectFactory used to construct GITB classes. - * - * @return The factory. - */ - @Bean - public ObjectFactory objectFactory() { - return new ObjectFactory(); - } - } diff --git a/src/main/java/be/vlaanderen/informatievlaanderen/ldes/gitb/StateManager.java b/src/main/java/be/vlaanderen/informatievlaanderen/ldes/gitb/StateManager.java deleted file mode 100644 index 00ba8a1..0000000 --- a/src/main/java/be/vlaanderen/informatievlaanderen/ldes/gitb/StateManager.java +++ /dev/null @@ -1,117 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.gitb; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.stereotype.Component; - -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; -import java.util.UUID; - -/** - * Component used to store sessions and their state. - *

- * This class is key in maintaining an overall context across a request and one or more - * responses. It allows mapping of received data to a given test session running in the - * test bed. - *

- * This implementation stores session information in memory. An alternative solution - * that would be fault-tolerant could store test session data in a DB. - */ -@Component -public class StateManager { - - /** Logger. */ - private static final Logger LOG = LoggerFactory.getLogger(StateManager.class); - - /** The map of in-memory active sessions. */ - private final Map> sessions = new HashMap<>(); - /** Lock object to use for synchronisation. */ - private final Object lock = new Object(); - - /** - * Create a new session. - * - * @param sessionId The session ID to use (if null a new one will be generated). - * @param callbackURL The callback URL to set for this session. - * @return The session ID that was generated (generated if not provided). - */ - public String createSession(String sessionId, String callbackURL) { - if (callbackURL == null) { - throw new IllegalArgumentException("A callback URL must be provided"); - } - if (sessionId == null) { - sessionId = UUID.randomUUID().toString(); - } - synchronized (lock) { - Map sessionInfo = new HashMap<>(); - sessionInfo.put(SessionData.CALLBACK_URL, callbackURL); - sessions.put(sessionId, sessionInfo); - } - return sessionId; - } - - /** - * Remove the provided session from the list of tracked sessions. - * - * @param sessionId The session ID to remove. - */ - public void destroySession(String sessionId) { - synchronized (lock) { - sessions.remove(sessionId); - } - } - - /** - * Get a given item of information linked to a specific session. - * - * @param sessionId The session ID we want to lookup. - * @param infoKey The key of the value that we want to retrieve. - * @return The retrieved value. - */ - public Object getSessionInfo(String sessionId, String infoKey) { - synchronized (lock) { - Object value = null; - if (sessions.containsKey(sessionId)) { - value = sessions.get(sessionId).get(infoKey); - } - return value; - } - } - - /** - * Set the given information item for a session. - * - * @param sessionId The session ID to set the information for. - * @param infoKey The information key. - * @param infoValue The information value. - */ - public void setSessionInfo(String sessionId, String infoKey, Object infoValue) { - synchronized (lock) { - sessions.get(sessionId).put(infoKey, infoValue); - } - } - - /** - * Get all the active sessions. - * - * @return An unmodifiable map of the sessions. - */ - public Map> getAllSessions() { - synchronized (lock) { - return Collections.unmodifiableMap(sessions); - } - } - - /** - * Constants used to identify data maintained as part of a session's state. - */ - public static class SessionData { - - /** The URL on which the test bed is to be called back. */ - public static final String CALLBACK_URL = "callbackURL"; - - } - -} diff --git a/src/main/java/be/vlaanderen/informatievlaanderen/ldes/gitb/TestBedNotifier.java b/src/main/java/be/vlaanderen/informatievlaanderen/ldes/gitb/TestBedNotifier.java deleted file mode 100644 index dee27b1..0000000 --- a/src/main/java/be/vlaanderen/informatievlaanderen/ldes/gitb/TestBedNotifier.java +++ /dev/null @@ -1,117 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.gitb; - -import com.gitb.core.LogLevel; -import com.gitb.ms.LogRequest; -import com.gitb.ms.MessagingClient; -import com.gitb.ms.NotifyForMessageRequest; -import com.gitb.tr.TAR; -import com.gitb.tr.TestResultType; -import org.apache.cxf.endpoint.Client; -import org.apache.cxf.frontend.ClientProxy; -import org.apache.cxf.jaxws.JaxWsProxyFactoryBean; -import org.apache.cxf.transport.http.HTTPConduit; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.scheduling.annotation.Async; -import org.springframework.stereotype.Component; - -import java.util.concurrent.ConcurrentHashMap; - -/** - * Component used to notify the Test Bed of received queries. - *

- * The main reason of defining this as a separate component is to facilitate making these notifications asynchronous - * (see the notifyTestBed method that is marked as async). - *

- * As an example, the configuration of a proxy to be used for this call is provided that can be optionally set on the - * call-back service proxy via configuration properties (set in application.properties). - */ -@Component -public class TestBedNotifier { - - private static final Logger LOG = LoggerFactory.getLogger(TestBedNotifier.class); - - private final ConcurrentHashMap messagingClientCache = new ConcurrentHashMap<>(); - - @Autowired - private ProxyInfo proxy = null; - @Autowired - private Utils utils = null; - /** - * Send a log message to the Test Bed at a given severity level. - * - * @param sessionId The session identifier. - * @param callbackAddress The Test Bed's callback address to use. - * @param message The log message. - * @param level The severity level. - */ - @Async - public void sendLogMessage(String sessionId, String callbackAddress, String message, LogLevel level) { - var logRequest = new LogRequest(); - logRequest.setSessionId(sessionId); - logRequest.setMessage(message); - logRequest.setLevel(level); - getMessagingClient(callbackAddress).log(logRequest); - } - - /** - * Notify the Test Bed for a given session. - * - * @param sessionId The session ID to notify the test bed for. - * @param callId The 'receive' call ID to notify the Test Bed for. - * @param report The report to notify the Test Bed with. - */ - @Async - public void notifyTestBed(String sessionId, String callId, String callback, TAR report){ - try { - LOG.info("Notifying Test Bed for session [{}]", sessionId); - callTestBed(sessionId, callId, report, callback); - } catch (Exception e) { - LOG.warn("Error while notifying test bed for session [{}]", sessionId, e); - callTestBed(sessionId, callId, utils.createReport(TestResultType.FAILURE), callback); - throw new IllegalStateException(e); - } - } - - /** - * Call the Test Bed to notify it of received communication. - * - * @param sessionId The session ID that this notification relates to. - * @param callId The 'receive' call ID to notify the test bed for. - * @param report The TAR report to send back. - * @param callbackAddress The address on which the call is to be made. - */ - private void callTestBed(String sessionId, String callId, TAR report, String callbackAddress) { - // Make the call. - NotifyForMessageRequest request = new NotifyForMessageRequest(); - request.setSessionId(sessionId); - request.setCallId(callId); - request.setReport(report); - getMessagingClient(callbackAddress).notifyForMessage(request); - } - - /** - * Get the messaging client to use for the given Test Bed instance. - * - * @param callbackAddress The Test Bed's messaging callback address. - * @return The client. - */ - private MessagingClient getMessagingClient(String callbackAddress) { - return messagingClientCache.computeIfAbsent(callbackAddress, (address) -> { - var proxyFactoryBean = new JaxWsProxyFactoryBean(); - proxyFactoryBean.setServiceClass(MessagingClient.class); - proxyFactoryBean.setAddress(callbackAddress); - MessagingClient serviceProxy = (MessagingClient)proxyFactoryBean.create(); - Client client = ClientProxy.getClient(serviceProxy); - HTTPConduit httpConduit = (HTTPConduit) client.getConduit(); - httpConduit.getClient().setAutoRedirect(true); - // Apply proxy settings (if applicable). - if (proxy.isEnabled()) { - proxy.applyToCxfConduit(httpConduit); - } - return serviceProxy; - }); - } - -} \ No newline at end of file diff --git a/src/main/java/be/vlaanderen/informatievlaanderen/ldes/gitb/Utils.java b/src/main/java/be/vlaanderen/informatievlaanderen/ldes/gitb/Utils.java deleted file mode 100644 index fc3b346..0000000 --- a/src/main/java/be/vlaanderen/informatievlaanderen/ldes/gitb/Utils.java +++ /dev/null @@ -1,330 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.gitb; - -import com.gitb.core.*; -import com.gitb.tr.*; -import com.gitb.tr.ObjectFactory; -import jakarta.xml.ws.WebServiceContext; -import org.w3c.dom.Element; -import org.apache.cxf.headers.Header; -import org.springframework.stereotype.Component; -import org.springframework.beans.factory.annotation.Autowired; - -import jakarta.xml.bind.JAXBElement; -import javax.xml.datatype.DatatypeConfigurationException; -import javax.xml.datatype.DatatypeFactory; -import javax.xml.namespace.QName; -import java.io.IOException; -import java.net.URI; -import java.net.URISyntaxException; -import java.net.http.*; -import java.util.*; -import java.util.function.Function; - -/** - * Class containing utility methods. - */ -@Component -public class Utils { - - /** SOAP header name for the ReplyTo address. */ - public static final QName REPLY_TO_QNAME = new QName("http://www.w3.org/2005/08/addressing", "ReplyTo"); - /** SOAP header name for the test session ID. */ - public static final QName TEST_SESSION_ID_QNAME = new QName("http://www.gitb.com", "TestSessionIdentifier", "gitb"); - - @Autowired - private ObjectFactory objectFactory; - - /** - * Create a report for the given result. - * - * This method creates the report, sets its time and constructs an empty context map to return values with. - * - * @param result The overall result of the report. - * @return The report. - */ - public TAR createReport(TestResultType result) { - TAR report = new TAR(); - report.setContext(new AnyContent()); - report.getContext().setType("map"); - report.setResult(result); - try { - report.setDate(DatatypeFactory.newInstance().newXMLGregorianCalendar(new GregorianCalendar())); - } catch (DatatypeConfigurationException e) { - throw new IllegalStateException(e); - } - return report; - } - - /** - * Create a parameter definition. - * - * @param name The name of the parameter. - * @param type The type of the parameter. This needs to match one of the GITB types. - * @param use The use (required or optional). - * @param kind The kind og parameter it is (whether it should be provided as the specific value, as BASE64 content or as a URL that needs to be looked up to obtain the value). - * @param description The description of the parameter. - * @return The created parameter. - */ - public TypedParameter createParameter(String name, String type, UsageEnumeration use, ConfigurationType kind, String description) { - TypedParameter parameter = new TypedParameter(); - parameter.setName(name); - parameter.setType(type); - parameter.setUse(use); - parameter.setKind(kind); - parameter.setDesc(description); - return parameter; - } - - /** - * Collect the inputs that match the provided name. - * - * @param parameterItems The items to look through. - * @param inputName The name of the input to look for. - * @return The collected inputs (not null). - */ - public List getInputsForName(List parameterItems, String inputName) { - List inputs = new ArrayList<>(); - if (parameterItems != null) { - for (AnyContent anInput: parameterItems) { - if (inputName.equals(anInput.getName())) { - inputs.add(anInput); - } - } - } - return inputs; - } - - /** - * Get a single required input for the provided name. - * - * @param parameterItems The items to look through. - * @param inputName The name of the input to look for. - * @return The input. - */ - public AnyContent getSingleRequiredInputForName(List parameterItems, String inputName) { - var inputs = getInputsForName(parameterItems, inputName); - if (inputs.isEmpty()) { - throw new IllegalArgumentException(String.format("No input named [%s] was found.", inputName)); - } else if (inputs.size() > 1) { - throw new IllegalArgumentException(String.format("Multiple inputs named [%s] were found when only one was expected.", inputName)); - } - return inputs.get(0); - } - - /** - * Get a single optional input for the provided name. - * - * @param parameterItems The items to look through. - * @param inputName The name of the input to look for. - * @return The input. - */ - public Optional getSingleOptionalInputForName(List parameterItems, String inputName) { - var inputs = getInputsForName(parameterItems, inputName); - if (inputs.isEmpty()) { - return Optional.empty(); - } else if (inputs.size() > 1) { - throw new IllegalArgumentException(String.format("Multiple inputs named [%s] were found when at most one was expected.", inputName)); - } else { - return Optional.of(inputs.get(0)); - } - } - - /** - * Convert the provided content to a string value. - * - * @param content The content to convert. - * @return The string value. - */ - public String asString(AnyContent content) { - if (content == null || content.getValue() == null) { - return null; - } else if (content.getEmbeddingMethod() == ValueEmbeddingEnumeration.BASE_64) { - // Value provided as BASE64 string. - return new String(Base64.getDecoder().decode(content.getValue())); - } else if (content.getEmbeddingMethod() == ValueEmbeddingEnumeration.URI) { - // Value provided as URI to look up. - try { - var request = HttpRequest.newBuilder() - .uri(new URI(content.getValue())) - .GET() - .build(); - return HttpClient.newHttpClient() - .send(request, HttpResponse.BodyHandlers.ofString()) - .body(); - } catch (URISyntaxException e) { - throw new IllegalArgumentException(String.format("The provided value [%s] was not a valid URI.", content.getValue()), e); - } catch (IOException | InterruptedException e) { - throw new IllegalArgumentException(String.format("Error while calling URI [%s]", content.getValue()), e); - } - } else { - // Value provided as String. - return content.getValue(); - } - } - - /** - * Get a single required input for the provided name as a string value. - * - * @param parameterItems The items to look through. - * @param inputName The name of the input to look for. - * @return The input's string value. - */ - public String getRequiredString(List parameterItems, String inputName) { - return asString(getSingleRequiredInputForName(parameterItems, inputName)); - } - - /** - * Get a single required input for the provided name as a binary value. - * - * @param parameterItems The items to look through. - * @param inputName The name of the input to look for. - * @return The input's byte[] value. - */ - public byte[] getRequiredBinary(List parameterItems, String inputName) { - var input = getSingleRequiredInputForName(parameterItems, inputName); - if (input.getEmbeddingMethod() == null || input.getEmbeddingMethod() == ValueEmbeddingEnumeration.BASE_64) { - // Base64 encoded string. - return Base64.getDecoder().decode(input.getValue()); - } else if (input.getEmbeddingMethod() == ValueEmbeddingEnumeration.URI) { - // Remote URI to read from. - try { - var request = HttpRequest.newBuilder() - .uri(new URI(input.getValue())) - .GET() - .build(); - return HttpClient.newHttpClient() - .send(request, HttpResponse.BodyHandlers.ofByteArray()) - .body(); - } catch (URISyntaxException e) { - throw new IllegalArgumentException(String.format("The provided value [%s] was not a valid URI.", input.getValue()), e); - } catch (IOException | InterruptedException e) { - throw new IllegalArgumentException(String.format("Error while calling URI [%s]", input.getValue()), e); - } - } else { - throw new IllegalArgumentException(String.format("Input [%s] was expected to be provided as a BASE64 string or a URI.", inputName)); - } - } - - /** - * Get a single optional input for the provided name as a string value. - * - * @param parameterItems The items to look through. - * @param inputName The name of the input to look for. - * @return The input's string value. - */ - public Optional getOptionalString(List parameterItems, String inputName) { - var input = getSingleOptionalInputForName(parameterItems, inputName); - return input.map(this::asString); - } - - /** - * Create a AnyContent object value based on the provided parameters. - * - * @param name The name of the value. - * @param value The value itself. - * @param embeddingMethod The way in which this value is to be considered. - * @return The value. - */ - public AnyContent createAnyContentSimple(String name, String value, ValueEmbeddingEnumeration embeddingMethod) { - AnyContent input = new AnyContent(); - input.setName(name); - input.setValue(value); - input.setType("string"); - input.setEmbeddingMethod(embeddingMethod); - return input; - } - - /** - * Parse the received SOAP headers to retrieve the "reply-to" address. - * - * @param context The call's context. - * @return The header's value. - */ - public Optional getReplyToAddressFromHeaders(WebServiceContext context) { - return getHeaderAsString(context, REPLY_TO_QNAME).map(h -> { - if (h.endsWith("?wsdl")) { - return h; - } else { - return h + "?wsdl"; - } - }); - } - - /** - * Parse the received SOAP headers to retrieve the test session identifier. - * - * @param context The call's context. - * @return The header's value. - */ - public Optional getTestSessionIdFromHeaders(WebServiceContext context) { - return getHeaderAsString(context, TEST_SESSION_ID_QNAME); - } - - /** - * Extract a value from the SOAP headers. - * - * @param name The name of the header to locate. - * @param valueExtractor The function used to extract the data. - * @return The extracted data. - * @param The type of data extracted. - */ - public T getHeaderValue(WebServiceContext context, QName name, Function valueExtractor) { - return ((List

) context.getMessageContext().get(Header.HEADER_LIST)) - .stream() - .filter(header -> name.equals(header.getName())).findFirst() - .map(valueExtractor).orElse(null); - } - - /** - * Get the specified header element as a string. - * - * @param name The name of the header element to lookup. - * @return The text value of the element. - */ - public Optional getHeaderAsString(WebServiceContext context, QName name) { - return Optional.ofNullable(getHeaderValue(context, name, (header) -> ((Element) header.getObject()).getTextContent().trim())); - } - - /** - * Add an information message to the report. - * - * @param message The message. - * @param reportItems The report's items. - */ - public void addReportItemInfo(String message, List> reportItems) { - reportItems.add(objectFactory.createTestAssertionGroupReportsTypeInfo(createReportItemContent(message))); - } - - /** - * Add a warning message to the report. - * - * @param message The message. - * @param reportItems The report's items. - */ - public void addReportItemWarning(String message, List> reportItems) { - reportItems.add(objectFactory.createTestAssertionGroupReportsTypeWarning(createReportItemContent(message))); - } - - /** - * Add an error message to the report. - * - * @param message The message. - * @param reportItems The report's items. - */ - public void addReportItemError(String message, List> reportItems) { - reportItems.add(objectFactory.createTestAssertionGroupReportsTypeError(createReportItemContent(message))); - } - - /** - * Create the internal content of a report's item. - * - * @param message The message. - * @return The content to wrap. - */ - private BAR createReportItemContent(String message) { - BAR itemContent = new BAR(); - itemContent.setDescription(message); - return itemContent; - } - -} diff --git a/src/main/java/be/vlaanderen/informatievlaanderen/ldes/gitb/ValidationServiceImpl.java b/src/main/java/be/vlaanderen/informatievlaanderen/ldes/gitb/ValidationServiceImpl.java index 451bf13..75cb611 100644 --- a/src/main/java/be/vlaanderen/informatievlaanderen/ldes/gitb/ValidationServiceImpl.java +++ b/src/main/java/be/vlaanderen/informatievlaanderen/ldes/gitb/ValidationServiceImpl.java @@ -1,29 +1,25 @@ package be.vlaanderen.informatievlaanderen.ldes.gitb; -import be.vlaanderen.informatievlaanderen.ldes.handlers.ShaclValidationHandler; import be.vlaanderen.informatievlaanderen.ldes.services.RDFConverter; +import be.vlaanderen.informatievlaanderen.ldes.services.ValidationReportToTarMapper; +import be.vlaanderen.informatievlaanderen.ldes.shacl.ShaclValidator; +import be.vlaanderen.informatievlaanderen.ldes.valueobjects.Parameters; +import be.vlaanderen.informatievlaanderen.ldes.valueobjects.ValidationParameters; +import be.vlaanderen.informatievlaanderen.ldes.valueobjects.ValidationReport; +import com.gitb.core.Metadata; +import com.gitb.core.TypedParameter; +import com.gitb.core.TypedParameters; import com.gitb.core.ValidationModule; -import com.gitb.tr.TAR; -import com.gitb.tr.TestAssertionGroupReportsType; -import com.gitb.tr.TestResultType; -import com.gitb.tr.ValidationCounters; import com.gitb.vs.Void; import com.gitb.vs.*; -import com.google.common.collect.Iterables; import org.eclipse.rdf4j.model.Model; -import org.eclipse.rdf4j.model.ValueFactory; -import org.eclipse.rdf4j.model.impl.SimpleValueFactory; import org.eclipse.rdf4j.rio.RDFFormat; -import org.eclipse.rdf4j.rio.Rio; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; -import java.io.ByteArrayInputStream; -import java.math.BigInteger; - -import static be.vlaanderen.informatievlaanderen.ldes.constants.RDFConstants.*; +import java.util.List; +import java.util.UUID; /** * Spring component that realises the validation service. @@ -31,98 +27,75 @@ @Component public class ValidationServiceImpl implements ValidationService { - private static final String SERVICE_NAME = "LdesmemberShaqlValidator"; - @Autowired - private ShaclValidationHandler shaclValidationHandler; - - /** Logger. **/ - private static final Logger LOG = LoggerFactory.getLogger(ValidationServiceImpl.class); - - @Autowired - private Utils utils = null; - - /** - * The purpose of the getModuleDefinition call is to inform its caller on how the service is supposed to be called. - *

- * Note that defining the implementation of this service is optional, and can be empty unless you plan to publish - * the service for use by third parties (in which case it serves as documentation on its expected inputs and outputs). - * - * @param parameters No parameters are expected. - * @return The response. - */ - @Override - public GetModuleDefinitionResponse getModuleDefinition(Void parameters) { - GetModuleDefinitionResponse response = new GetModuleDefinitionResponse(); - response.setModule(new ValidationModule()); - response.getModule().setId(SERVICE_NAME); -// response.getModule().setConfigs(); - return response; - } - - /** - * The validate operation is called to validate the input and produce a validation report. - * - * The expected input is described for the service's client through the getModuleDefinition call. - * - * @param parameters The input parameters and configuration for the validation. - * @return The response containing the validation report. - */ - @Override - public ValidationResponse validate(ValidateRequest parameters) { - LOG.info("Received 'validate' command from test bed for session [{}]", parameters.getSessionId()); - ValidationResponse result = new ValidationResponse(); - TAR report = utils.createReport(TestResultType.SUCCESS); - // First extract the parameters and check to see if they are as expected. - String shacl = utils.getRequiredString(parameters.getInput(), "shacl-shape"); - String url = utils.getRequiredString(parameters.getInput(), "server-url"); - - report.setReports(new TestAssertionGroupReportsType()); - int infos = 0; - int warnings = 0; - int errors = 0; - - try { - Model shaclModel = RDFConverter.readModel(shacl, RDFFormat.TURTLE); - Model validationReport = shaclValidationHandler.validate(url, shaclModel); - int count; - if ((count = getErrorCount(validationReport)) > 0) { - errors += count; - utils.addReportItemError(RDFConverter.writeModel(validationReport, RDFFormat.TURTLE), report.getReports().getInfoOrWarningOrError()); - } else if ((count = getWarnCount(validationReport)) > 0) { - warnings += count; - utils.addReportItemWarning(RDFConverter.writeModel(validationReport, RDFFormat.TURTLE), report.getReports().getInfoOrWarningOrError()); - } else if ((count = getInfoCount(validationReport)) > 0) { - infos += count; - utils.addReportItemInfo(RDFConverter.writeModel(validationReport, RDFFormat.TURTLE), report.getReports().getInfoOrWarningOrError()); - } - - } catch (Exception e) { - - } - - report.setCounters(new ValidationCounters()); - report.getCounters().setNrOfAssertions(BigInteger.valueOf(infos)); - report.getCounters().setNrOfWarnings(BigInteger.valueOf(warnings)); - report.getCounters().setNrOfErrors(BigInteger.valueOf(errors)); - if (errors > 0) { - report.setResult(TestResultType.FAILURE); - } else if (warnings > 0) { - report.setResult(TestResultType.WARNING); - } - // Return the report. - result.setReport(report); - return result; - } - - private int getErrorCount(Model report) { - return Iterables.size(report.getStatements(null, SEVERITY, VIOLATION)); - } - - private int getWarnCount(Model report) { - return Iterables.size(report.getStatements(null, SEVERITY, WARNING)); - } - - private int getInfoCount(Model report) { - return Iterables.size(report.getStatements(null, SEVERITY, INFO)); - } + private static final Logger LOG = LoggerFactory.getLogger(ValidationServiceImpl.class); + private static final String SERVICE_NAME = "LdesMemberShaclValidator"; + + private final ShaclValidator shaclValidator; + + public ValidationServiceImpl(ShaclValidator shaclValidator) { + this.shaclValidator = shaclValidator; + } + + /** + * The purpose of the getModuleDefinition call is to inform its caller on how the service is supposed to be called. + *

+ * Note that defining the implementation of this service is optional, and can be empty unless you plan to publish + * the service for use by third parties (in which case it serves as documentation on its expected inputs and outputs). + * + * @param parameters No parameters are expected. + * @return The response. + */ + @Override + public GetModuleDefinitionResponse getModuleDefinition(Void parameters) { + final var validationModule = new ValidationModule(); + validationModule.setId(SERVICE_NAME); + validationModule.setOperation("V"); + + final var metadata = new Metadata(); + metadata.setName(SERVICE_NAME); + validationModule.setMetadata(metadata); + + final var ldesServerParam = new TypedParameter(); + ldesServerParam.setName("ldes-url"); + ldesServerParam.setType("string"); + + final var shaclShapeParam = new TypedParameter(); + shaclShapeParam.setName("shacl-shape"); + shaclShapeParam.setType("string"); + + final var inputs = new TypedParameters(); + inputs.getParam().addAll(List.of(ldesServerParam, shaclShapeParam)); + validationModule.setInputs(inputs); + + GetModuleDefinitionResponse response = new GetModuleDefinitionResponse(); + response.setModule(validationModule); + return response; + } + + /** + * The validate operation is called to validate the input and produce a validation report. + *

+ * The expected input is described for the service's client through the getModuleDefinition call. + * + * @param validateRequest The input parameters and configuration for the validation. + * @return The response containing the validation report. + */ + @Override + public ValidationResponse validate(ValidateRequest validateRequest) { + final String sessionId = validateRequest.getSessionId() != null ? validateRequest.getSessionId() : UUID.randomUUID().toString(); + LOG.info("Received 'validate' command from test bed for session [{}]", sessionId); + ValidationResponse result = new ValidationResponse(); + // First extract the parameters and check to see if they are as expected. + final Parameters params = new Parameters(validateRequest.getInput()); + String shacl = params.getStringForName("shacl-shape"); + String url = params.getStringForName("ldes-url"); + + final Model shaclShape = RDFConverter.readModel(shacl, RDFFormat.TURTLE); + final ValidationParameters validationParams = new ValidationParameters(url, shaclShape, sessionId); + final ValidationReport validationReport = shaclValidator.validate(validationParams); + result.setReport(ValidationReportToTarMapper.mapToTar(validationReport)); + return result; + } + + } diff --git a/src/main/java/be/vlaanderen/informatievlaanderen/ldes/handlers/ShaclValidationHandler.java b/src/main/java/be/vlaanderen/informatievlaanderen/ldes/handlers/ShaclValidationHandler.java deleted file mode 100644 index cd8f8ee..0000000 --- a/src/main/java/be/vlaanderen/informatievlaanderen/ldes/handlers/ShaclValidationHandler.java +++ /dev/null @@ -1,23 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.handlers; - -import be.vlaanderen.informatievlaanderen.ldes.ldio.LdioManager; -import be.vlaanderen.informatievlaanderen.ldes.rdfrepo.RepositoryValidator; -import org.eclipse.rdf4j.model.Model; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; - -import java.io.IOException; - -@Component -public class ShaclValidationHandler { - @Autowired - private LdioManager ldioManager; - @Autowired - private RepositoryValidator validator; - - public Model validate(String url, Model shaclShape) throws IOException { - ldioManager.initPipeline(url); - return validator.validate(shaclShape); - } - -} diff --git a/src/main/java/be/vlaanderen/informatievlaanderen/ldes/http/HttpClientConfig.java b/src/main/java/be/vlaanderen/informatievlaanderen/ldes/http/HttpClientConfig.java new file mode 100644 index 0000000..4b9e376 --- /dev/null +++ b/src/main/java/be/vlaanderen/informatievlaanderen/ldes/http/HttpClientConfig.java @@ -0,0 +1,15 @@ +package be.vlaanderen.informatievlaanderen.ldes.http; + +import org.apache.http.client.HttpClient; +import org.apache.http.impl.client.HttpClientBuilder; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + + +@Configuration +public class HttpClientConfig { + @Bean + public HttpClient httpClient() { + return HttpClientBuilder.create().build(); + } +} diff --git a/src/main/java/be/vlaanderen/informatievlaanderen/ldes/http/Request.java b/src/main/java/be/vlaanderen/informatievlaanderen/ldes/http/Request.java deleted file mode 100644 index 541ca3b..0000000 --- a/src/main/java/be/vlaanderen/informatievlaanderen/ldes/http/Request.java +++ /dev/null @@ -1,49 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.http; - -import org.apache.http.client.methods.HttpDelete; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.client.methods.HttpPost; -import org.apache.http.client.methods.HttpRequestBase; -import org.apache.http.entity.ContentType; -import org.apache.http.entity.StringEntity; -import org.springframework.web.bind.annotation.RequestMethod; - -import java.io.UnsupportedEncodingException; - -public class Request { - private final String url; - private final String body; - private final RequestMethod method; - private final String contentType; - - public Request(String url, String body, RequestMethod method, ContentType contentType) { - this.url = url; - this.body = body; - this.method = method; - this.contentType = contentType.getMimeType(); - } - - public Request(String url, String body, RequestMethod method, String contentType) { - this.url = url; - this.body = body; - this.method = method; - this.contentType = contentType; - } - - public Request(String url, RequestMethod method) { - this(url, "", method, ContentType.DEFAULT_TEXT); - } - - public HttpRequestBase createRequest() throws UnsupportedEncodingException { - return switch (method) { - case GET -> new HttpGet(url); - case POST -> { - final HttpPost post = new HttpPost(url); - post.setEntity(new StringEntity(body, ContentType.parse(contentType))); - yield post; - } - case DELETE -> new HttpDelete(url); - default -> throw new IllegalStateException("Http method not supported: " + method); - }; - } -} diff --git a/src/main/java/be/vlaanderen/informatievlaanderen/ldes/http/RequestExecutor.java b/src/main/java/be/vlaanderen/informatievlaanderen/ldes/http/RequestExecutor.java index 6aba35a..1a93fb0 100644 --- a/src/main/java/be/vlaanderen/informatievlaanderen/ldes/http/RequestExecutor.java +++ b/src/main/java/be/vlaanderen/informatievlaanderen/ldes/http/RequestExecutor.java @@ -1,14 +1,15 @@ package be.vlaanderen.informatievlaanderen.ldes.http; +import be.vlaanderen.informatievlaanderen.ldes.http.requests.HttpRequest; import org.apache.http.HttpEntity; import org.apache.http.HttpResponse; import org.apache.http.client.HttpClient; -import org.apache.http.impl.client.HttpClientBuilder; +import org.apache.http.util.EntityUtils; import org.springframework.stereotype.Component; -import java.io.BufferedReader; import java.io.IOException; -import java.io.InputStreamReader; +import java.io.UncheckedIOException; +import java.util.Arrays; import java.util.List; @Component @@ -16,26 +17,29 @@ public class RequestExecutor { private static final List ACCEPTABLE_STATUS_CODES = List.of(200, 201); private final HttpClient httpClient; - public RequestExecutor() { - final HttpClientBuilder httpClientBuilder = HttpClientBuilder.create(); - this.httpClient = httpClientBuilder.build(); + public RequestExecutor(HttpClient httpClient) { + this.httpClient = httpClient; } - public HttpEntity execute(Request request) { + public HttpEntity execute(HttpRequest request) { return execute(request, ACCEPTABLE_STATUS_CODES); } - public HttpEntity execute(Request request, List expectedCodes) { + public HttpEntity execute(HttpRequest request, Integer... expectedStatusCodes) { + return execute(request, Arrays.asList(expectedStatusCodes)); + } + + public HttpEntity execute(HttpRequest request, List expectedCodes) { try { HttpResponse response = httpClient.execute(request.createRequest()); if (!expectedCodes.contains(response.getStatusLine().getStatusCode())) { - String msg = new BufferedReader(new InputStreamReader(response.getEntity().getContent())).readLine(); - throw new RuntimeException(msg); + final String message = EntityUtils.toString(response.getEntity()); + throw new IllegalStateException("Unexpected response status: " + response.getStatusLine().getStatusCode() + ":\n" + message); } return response.getEntity(); } catch (IOException e) { - throw new RuntimeException(e); + throw new UncheckedIOException(e); } } } diff --git a/src/main/java/be/vlaanderen/informatievlaanderen/ldes/http/requests/DeleteRequest.java b/src/main/java/be/vlaanderen/informatievlaanderen/ldes/http/requests/DeleteRequest.java new file mode 100644 index 0000000..281c54b --- /dev/null +++ b/src/main/java/be/vlaanderen/informatievlaanderen/ldes/http/requests/DeleteRequest.java @@ -0,0 +1,32 @@ +package be.vlaanderen.informatievlaanderen.ldes.http.requests; + +import org.apache.http.client.methods.HttpDelete; +import org.apache.http.client.methods.HttpRequestBase; + +import java.util.Objects; + +public class DeleteRequest implements HttpRequest { + private final String url; + + public DeleteRequest(String url) { + this.url = url; + } + + @Override + public HttpRequestBase createRequest() { + return new HttpDelete(url); + } + + @Override + public final boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof DeleteRequest that)) return false; + + return Objects.equals(url, that.url); + } + + @Override + public int hashCode() { + return Objects.hashCode(url); + } +} diff --git a/src/main/java/be/vlaanderen/informatievlaanderen/ldes/http/requests/GetRequest.java b/src/main/java/be/vlaanderen/informatievlaanderen/ldes/http/requests/GetRequest.java new file mode 100644 index 0000000..cfbba56 --- /dev/null +++ b/src/main/java/be/vlaanderen/informatievlaanderen/ldes/http/requests/GetRequest.java @@ -0,0 +1,32 @@ +package be.vlaanderen.informatievlaanderen.ldes.http.requests; + +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpRequestBase; + +import java.util.Objects; + +public class GetRequest implements HttpRequest { + private final String url; + + public GetRequest(String url) { + this.url = url; + } + + @Override + public HttpRequestBase createRequest() { + return new HttpGet(url); + } + + @Override + public final boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof GetRequest that)) return false; + + return Objects.equals(url, that.url); + } + + @Override + public int hashCode() { + return Objects.hashCode(url); + } +} diff --git a/src/main/java/be/vlaanderen/informatievlaanderen/ldes/http/requests/HttpRequest.java b/src/main/java/be/vlaanderen/informatievlaanderen/ldes/http/requests/HttpRequest.java new file mode 100644 index 0000000..eb1a844 --- /dev/null +++ b/src/main/java/be/vlaanderen/informatievlaanderen/ldes/http/requests/HttpRequest.java @@ -0,0 +1,7 @@ +package be.vlaanderen.informatievlaanderen.ldes.http.requests; + +import org.apache.http.client.methods.HttpRequestBase; + +public interface HttpRequest { + HttpRequestBase createRequest(); +} diff --git a/src/main/java/be/vlaanderen/informatievlaanderen/ldes/http/requests/PostRequest.java b/src/main/java/be/vlaanderen/informatievlaanderen/ldes/http/requests/PostRequest.java new file mode 100644 index 0000000..1278205 --- /dev/null +++ b/src/main/java/be/vlaanderen/informatievlaanderen/ldes/http/requests/PostRequest.java @@ -0,0 +1,29 @@ +package be.vlaanderen.informatievlaanderen.ldes.http.requests; + +import org.apache.http.client.methods.HttpPost; +import org.apache.http.client.methods.HttpRequestBase; +import org.apache.http.entity.ContentType; +import org.apache.http.entity.StringEntity; + +public class PostRequest implements HttpRequest { + private final String url; + private final String body; + private final ContentType contentType; + + public PostRequest(String url, String body, ContentType contentType) { + this.url = url; + this.body = body; + this.contentType = contentType; + } + + public PostRequest(String url, String body, String contentType) { + this(url, body, ContentType.parse(contentType)); + } + + @Override + public HttpRequestBase createRequest() { + final var request = new HttpPost(url); + request.setEntity(new StringEntity(body, contentType)); + return request; + } +} diff --git a/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldes/EventStreamFetcher.java b/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldes/EventStreamFetcher.java new file mode 100644 index 0000000..7872081 --- /dev/null +++ b/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldes/EventStreamFetcher.java @@ -0,0 +1,48 @@ +package be.vlaanderen.informatievlaanderen.ldes.ldes; + +import be.vlaanderen.informatievlaanderen.ldes.http.RequestExecutor; +import be.vlaanderen.informatievlaanderen.ldes.http.requests.GetRequest; +import org.apache.http.HttpEntity; +import org.eclipse.rdf4j.model.Model; +import org.eclipse.rdf4j.model.Statement; +import org.eclipse.rdf4j.model.impl.SimpleValueFactory; +import org.eclipse.rdf4j.rio.RDFFormat; +import org.eclipse.rdf4j.rio.Rio; +import org.springframework.stereotype.Service; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.Spliterator; +import java.util.stream.StreamSupport; + +@Service +public class EventStreamFetcher { + public static final String LDES_VERSION_OF = "https://w3id.org/ldes#versionOfPath"; + + final RequestExecutor requestExecutor; + + public EventStreamFetcher(RequestExecutor requestExecutor) { + this.requestExecutor = requestExecutor; + } + + public EventStreamProperties fetchProperties(String url) { + final RDFFormat rdfFormat = RDFFormat.TURTLE; + final HttpEntity response = requestExecutor.execute(new GetRequest(url)); + final Model model = extractModel(response, rdfFormat); + + final Spliterator statements = model.getStatements(null, SimpleValueFactory.getInstance().createIRI(LDES_VERSION_OF), null).spliterator(); + return StreamSupport.stream(statements, false) + .findFirst() + .map(statement -> new EventStreamProperties(url, statement.getObject().stringValue())) + .orElseThrow(() -> new IllegalStateException("Required properties of event stream for %s could not be found".formatted(url))); + } + + private Model extractModel(HttpEntity response, RDFFormat rdfFormat) { + try { + return Rio.parse(response.getContent(), rdfFormat); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + +} diff --git a/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldes/EventStreamProperties.java b/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldes/EventStreamProperties.java new file mode 100644 index 0000000..b4e6f4b --- /dev/null +++ b/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldes/EventStreamProperties.java @@ -0,0 +1,7 @@ +package be.vlaanderen.informatievlaanderen.ldes.ldes; + +public record EventStreamProperties( + String ldesServerUrl, + String versionOfPath +) { +} diff --git a/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdesClientStatusManager.java b/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdesClientStatusManager.java new file mode 100644 index 0000000..557cf03 --- /dev/null +++ b/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdesClientStatusManager.java @@ -0,0 +1,85 @@ +package be.vlaanderen.informatievlaanderen.ldes.ldio; + +import be.vlaanderen.informatievlaanderen.ldes.http.RequestExecutor; +import be.vlaanderen.informatievlaanderen.ldes.http.requests.GetRequest; +import be.vlaanderen.informatievlaanderen.ldes.ldio.config.LdioConfigProperties; +import be.vlaanderen.informatievlaanderen.ldes.ldio.excpeptions.LdesClientStatusUnavailableException; +import be.vlaanderen.informatievlaanderen.ldes.ldio.valuebojects.ClientStatus; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.http.HttpEntity; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; + +import java.io.IOException; +import java.util.concurrent.*; +import java.util.concurrent.atomic.AtomicInteger; + +@Service +public class LdesClientStatusManager { + private static final Logger log = LoggerFactory.getLogger(LdesClientStatusManager.class); + private static final int POLLING_PERIOD_IN_SECONDS = 5; + private static final int CLIENT_STATUS_FETCHING_RETRIES = 5; + private final RequestExecutor requestExecutor; + private final LdioConfigProperties ldioConfigProperties; + + public LdesClientStatusManager(RequestExecutor requestExecutor, LdioConfigProperties ldioConfigProperties) { + this.requestExecutor = requestExecutor; + this.ldioConfigProperties = ldioConfigProperties; + } + + public void waitUntilReplicated(String pipelineName) { + final CompletableFuture hasReplicated = new CompletableFuture<>(); + final AtomicInteger retryCount = new AtomicInteger(); + final TimeUnit timeUnit = TimeUnit.SECONDS; + final ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor(); + + log.atInfo().log("Waiting for the LDES client to complete REPLICATING"); + scheduler.scheduleAtFixedRate(() -> { + try { + final ClientStatus clientStatus = getClientStatus(pipelineName); + log.atDebug().log("Checking for LDES client status"); + if (ClientStatus.isSuccessfullyReplicated(clientStatus)) { + log.atInfo().log("LDES client status is now {}", clientStatus.toString()); + hasReplicated.complete(true); + } + } catch (LdesClientStatusUnavailableException e) { + if(retryCount.incrementAndGet() == CLIENT_STATUS_FETCHING_RETRIES) { + hasReplicated.complete(false); + } + log.atWarn().log("LDES client status is not available yet, trying again in {} {} ...", POLLING_PERIOD_IN_SECONDS, timeUnit.toString().toLowerCase()); + } catch (Exception e) { + hasReplicated.complete(false); + } + }, 0, POLLING_PERIOD_IN_SECONDS, timeUnit); + + try { + if(Boolean.FALSE.equals(hasReplicated.get())) { + throw new IllegalStateException("Unable to fetch the LDES client status"); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + log.atError().log("Thread interrupted", e); + } catch (ExecutionException e) { + log.atError().log("Something went wrong while waiting for LDES client to be fully replicated", e); + } finally { + scheduler.shutdown(); + } + } + + public ClientStatus getClientStatus(String pipelineName) { + final String clientStatusUrl = ldioConfigProperties.getLdioLdesClientStatusUrlTemplate().formatted(pipelineName); + final HttpEntity response = requestExecutor.execute(new GetRequest(clientStatusUrl), 200, 404); + + if (response.getContentLength() == 0) { + throw new LdesClientStatusUnavailableException(); + } + + final ObjectMapper objectMapper = new ObjectMapper(); + try { + return objectMapper.readValue(response.getContent(), ClientStatus.class); + } catch (IOException e) { + throw new IllegalStateException("Invalid client status received from %s".formatted(clientStatusUrl)); + } + } +} diff --git a/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioManager.java b/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioManager.java deleted file mode 100644 index 75d3e95..0000000 --- a/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioManager.java +++ /dev/null @@ -1,96 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.ldio; - - -import be.vlaanderen.informatievlaanderen.ldes.http.Request; -import be.vlaanderen.informatievlaanderen.ldes.http.RequestExecutor; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.ObjectNode; -import com.fasterxml.jackson.databind.node.TextNode; -import org.apache.http.entity.ContentType; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; -import org.springframework.web.bind.annotation.RequestMethod; - -import java.io.*; - -@Service -public class LdioManager { - private static final String PIPELINE_FILE_PATH = "src/main/resources/ldio-pipeline.json"; - private static final String PIPELINE_NAME = "validation-pipeline"; - @Autowired - private RequestExecutor requestExecutor; - - - public void initPipeline(String serverUrl) throws IOException { - String ldioUrl = "http://localhost:8082" + "/admin/api/v1/pipeline"; - - ObjectMapper objectMapper = new ObjectMapper(); - JsonNode jsonNode = objectMapper.readTree(new File(PIPELINE_FILE_PATH)); - setUrl(jsonNode, serverUrl); - setRepository(jsonNode, serverUrl); - setVersionOf(jsonNode, serverUrl); - - requestExecutor.execute(new Request(ldioUrl, jsonNode.toString(), RequestMethod.POST, ContentType.APPLICATION_JSON)); - waitForReplication(ldioUrl); - - } - - private void setUrl(JsonNode jsonNode, String serverUrl) { - ((ObjectNode) jsonNode.path("input").path("config")).replace("urls", new TextNode(serverUrl)); - } - private void setVersionOf(JsonNode jsonNode, String serverUrl) { - ObjectNode config = ((ObjectNode) jsonNode.path("transformers").get(0).get("config")); - config.replace("versionOf-property", new TextNode(serverUrl)); - } - - private void setRepository(JsonNode jsonNode, String serverUrl) { - ObjectNode config = ((ObjectNode) jsonNode.path("outputs").get(0).get("config")); - config.replace("sparql-host", new TextNode(serverUrl)); - config.replace("repository-id", new TextNode(serverUrl)); - } - - private void waitForReplication(String ldioUrl) throws IOException { - int seconds = 5; - boolean replicating = true; - while (replicating) { - try { - Thread.sleep(seconds * 1000); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - if (getIsPipelineFinished(ldioUrl)) { - requestExecutor.execute( - new Request(ldioUrl + "/" + PIPELINE_NAME, RequestMethod.DELETE) - ); - replicating = false; - } - } - } - - -// Checks if the pipeline is finished replicating - private boolean getIsPipelineFinished(String ldioUrl) throws IOException { - InputStream in = requestExecutor.execute( - new Request(ldioUrl + "/ldes-client/" + PIPELINE_NAME + "/status", "", - RequestMethod.GET, ContentType.DEFAULT_TEXT)).getContent(); - BufferedReader streamReader = new BufferedReader(new InputStreamReader(in)); - StringBuilder responseStrBuilder = new StringBuilder(); - - String inputStr; - while ((inputStr = streamReader.readLine()) != null) - responseStrBuilder.append(inputStr); - String responseStr = responseStrBuilder.toString().toUpperCase(); - if (!responseStr.contains("REPLICATING")) { - if (responseStr.contains("SYNCHRONISING") || responseStr.contains("COMPLETED")) { - return true; - } - else { - throw new RuntimeException(); - } - } - else { - return false; - } - } -} diff --git a/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioPipelineManager.java b/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioPipelineManager.java new file mode 100644 index 0000000..269243e --- /dev/null +++ b/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioPipelineManager.java @@ -0,0 +1,44 @@ +package be.vlaanderen.informatievlaanderen.ldes.ldio; + + +import be.vlaanderen.informatievlaanderen.ldes.http.RequestExecutor; +import be.vlaanderen.informatievlaanderen.ldes.http.requests.DeleteRequest; +import be.vlaanderen.informatievlaanderen.ldes.http.requests.PostRequest; +import be.vlaanderen.informatievlaanderen.ldes.ldes.EventStreamFetcher; +import be.vlaanderen.informatievlaanderen.ldes.ldes.EventStreamProperties; +import be.vlaanderen.informatievlaanderen.ldes.ldio.config.LdioConfigProperties; +import be.vlaanderen.informatievlaanderen.ldes.ldio.pipeline.ValidationPipelineSupplier; +import org.apache.http.entity.ContentType; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; + +@Service +public class LdioPipelineManager { + private static final Logger log = LoggerFactory.getLogger(LdioPipelineManager.class); + private final EventStreamFetcher eventStreamFetcher; + private final RequestExecutor requestExecutor; + private final LdioConfigProperties ldioConfigProperties; + + public LdioPipelineManager(EventStreamFetcher eventStreamFetcher, RequestExecutor requestExecutor, LdioConfigProperties ldioConfigProperties) { + this.eventStreamFetcher = eventStreamFetcher; + this.requestExecutor = requestExecutor; + this.ldioConfigProperties = ldioConfigProperties; + } + + public void initPipeline(String serverUrl, String pipelineName) { + final String ldioAdminPipelineUrl = ldioConfigProperties.getLdioAdminPipelineUrl(); + final EventStreamProperties eventStreamProperties = eventStreamFetcher.fetchProperties(serverUrl); + final String json = new ValidationPipelineSupplier(eventStreamProperties, ldioConfigProperties.getSparqlHost(), pipelineName).getValidationPipelineAsJson(); + requestExecutor.execute(new PostRequest(ldioAdminPipelineUrl, json, ContentType.APPLICATION_JSON), 201); + log.atInfo().log("LDIO pipeline created: {}", pipelineName); + } + + public void deletePipeline(String pipelineName) { + requestExecutor.execute( + new DeleteRequest(ldioConfigProperties.getLdioAdminPipelineUrl() + "/" + pipelineName), 202, 204 + ); + log.atInfo().log("LDIO pipeline deleted: {}", pipelineName); + } + +} diff --git a/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioConfigProperties.java b/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioConfigProperties.java new file mode 100644 index 0000000..c4ed311 --- /dev/null +++ b/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioConfigProperties.java @@ -0,0 +1,41 @@ +package be.vlaanderen.informatievlaanderen.ldes.ldio.config; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.context.annotation.Configuration; + +@Configuration +@ConfigurationProperties(prefix = "ldio") +public class LdioConfigProperties { + public static final String REPOSITORY_ID = "validation"; + + private String host; + private String sparqlHost; + + public String getHost() { + return host; + } + + public void setHost(String host) { + this.host = host; + } + + public String getSparqlHost() { + return sparqlHost; + } + + public void setSparqlHost(String sparqlHost) { + this.sparqlHost = sparqlHost; + } + + public String getLdioAdminPipelineUrl() { + return "%s/admin/api/v1/pipeline".formatted(host); + } + + public String getLdioLdesClientStatusUrlTemplate() { + return getLdioAdminPipelineUrl() + "/ldes-client/%s"; + } + + public String getRepositoryValidationUrl() { + return "%s/rest/repositories/%s/validate/text".formatted(sparqlHost, REPOSITORY_ID); + } +} diff --git a/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/excpeptions/LdesClientStatusUnavailableException.java b/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/excpeptions/LdesClientStatusUnavailableException.java new file mode 100644 index 0000000..2cc138c --- /dev/null +++ b/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/excpeptions/LdesClientStatusUnavailableException.java @@ -0,0 +1,8 @@ +package be.vlaanderen.informatievlaanderen.ldes.ldio.excpeptions; + +public class LdesClientStatusUnavailableException extends RuntimeException { + @Override + public String getMessage() { + return "Ldes client status not available."; + } +} diff --git a/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/pipeline/LdioComponentBuilder.java b/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/pipeline/LdioComponentBuilder.java new file mode 100644 index 0000000..05f2ed8 --- /dev/null +++ b/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/pipeline/LdioComponentBuilder.java @@ -0,0 +1,25 @@ +package be.vlaanderen.informatievlaanderen.ldes.ldio.pipeline; + +import be.vlaanderen.informatievlaanderen.ldes.ldio.valuebojects.LdioComponentProperties; +import be.vlaanderen.informatievlaanderen.ldes.ldio.valuebojects.LdioComponent; + +import java.util.Map; + +public abstract class LdioComponentBuilder> { + private final String name; + private final Map properties; + + protected LdioComponentBuilder(String name, Map properties) { + this.name = name; + this.properties = properties; + } + + protected T withProperty(String key, Object value) { + properties.put(key, value); + return (T) this; + } + + public LdioComponent build() { + return new LdioComponent(name, new LdioComponentProperties(properties)); + } +} diff --git a/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/pipeline/LdioLdesClientBuilder.java b/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/pipeline/LdioLdesClientBuilder.java new file mode 100644 index 0000000..f2790e2 --- /dev/null +++ b/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/pipeline/LdioLdesClientBuilder.java @@ -0,0 +1,18 @@ +package be.vlaanderen.informatievlaanderen.ldes.ldio.pipeline; + +import java.util.HashMap; +import java.util.Map; + +public class LdioLdesClientBuilder extends LdioComponentBuilder { + public LdioLdesClientBuilder() { + super("Ldio:LdesClient", new HashMap<>(Map.of("source-format", "application/n-quads"))); + } + + public LdioLdesClientBuilder withUrl(String url) { + return withProperty("urls", url); + } + + public LdioLdesClientBuilder withVersionOfProperty(String versionOfProperty) { + return withProperty("materialisation", Map.of("enabled", true, "version-of-property", versionOfProperty)); + } +} diff --git a/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/pipeline/LdioRepositorySinkBuilder.java b/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/pipeline/LdioRepositorySinkBuilder.java new file mode 100644 index 0000000..86ddea2 --- /dev/null +++ b/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/pipeline/LdioRepositorySinkBuilder.java @@ -0,0 +1,22 @@ +package be.vlaanderen.informatievlaanderen.ldes.ldio.pipeline; + +import java.util.HashMap; + +public class LdioRepositorySinkBuilder extends LdioComponentBuilder{ + + protected LdioRepositorySinkBuilder() { + super("Ldio:RepositorySink", new HashMap<>()); + } + + public LdioRepositorySinkBuilder withSparqlHost(String sparqlHost) { + return withProperty("sparql-host", sparqlHost); + } + + public LdioRepositorySinkBuilder withRepositoryId(String repositoryId) { + return withProperty("repository-id", repositoryId); + } + + public LdioRepositorySinkBuilder withBatchSize(int batchSize) { + return withProperty("batch-size", batchSize); + } +} diff --git a/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/pipeline/ValidationPipelineSupplier.java b/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/pipeline/ValidationPipelineSupplier.java new file mode 100644 index 0000000..ed077a0 --- /dev/null +++ b/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/pipeline/ValidationPipelineSupplier.java @@ -0,0 +1,50 @@ +package be.vlaanderen.informatievlaanderen.ldes.ldio.pipeline; + +import be.vlaanderen.informatievlaanderen.ldes.ldes.EventStreamProperties; +import be.vlaanderen.informatievlaanderen.ldes.ldio.valuebojects.LdioPipeline; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; + +import java.util.List; + +import static be.vlaanderen.informatievlaanderen.ldes.ldio.config.LdioConfigProperties.REPOSITORY_ID; + +public class ValidationPipelineSupplier { + public static final String PIPELINE_NAME_TEMPLATE = "validation-pipeline-%s"; + private static final String PIPELINE_DESCRIPTION = "Pipeline that will only replicate an LDES for validation purposes"; + private final EventStreamProperties eventStreamProperties; + private final String sparqlHost; + private final String pipelineName; + + public ValidationPipelineSupplier(EventStreamProperties eventStreamProperties, String sparqlHost, String pipelineName) { + this.eventStreamProperties = eventStreamProperties; + this.sparqlHost = sparqlHost; + this.pipelineName = pipelineName; + } + + public LdioPipeline getValidationPipeline() { + return new LdioPipeline( + pipelineName, + PIPELINE_DESCRIPTION, + new LdioLdesClientBuilder() + .withUrl(eventStreamProperties.ldesServerUrl()) + .withVersionOfProperty(eventStreamProperties.versionOfPath()) + .build(), + List.of(new LdioRepositorySinkBuilder() + .withSparqlHost(sparqlHost) + .withRepositoryId(REPOSITORY_ID) + .withBatchSize(1) + .build()) + ); + } + + public String getValidationPipelineAsJson() { + final LdioPipeline pipeline = getValidationPipeline(); + final ObjectMapper objectMapper = new ObjectMapper(); + try { + return objectMapper.writeValueAsString(pipeline); + } catch (JsonProcessingException e) { + throw new IllegalStateException("Could not serialize pipeline to JSON", e); + } + } +} diff --git a/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/valuebojects/ClientStatus.java b/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/valuebojects/ClientStatus.java new file mode 100644 index 0000000..4d2e85f --- /dev/null +++ b/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/valuebojects/ClientStatus.java @@ -0,0 +1,14 @@ +package be.vlaanderen.informatievlaanderen.ldes.ldio.valuebojects; + +public enum ClientStatus { + + REPLICATING, + SYNCHRONISING, + COMPLETED, + ERROR; + + public static boolean isSuccessfullyReplicated(ClientStatus status) { + return SYNCHRONISING.equals(status) || ClientStatus.COMPLETED.equals(status); + } + +} diff --git a/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/valuebojects/LdioComponent.java b/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/valuebojects/LdioComponent.java new file mode 100644 index 0000000..3279c3b --- /dev/null +++ b/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/valuebojects/LdioComponent.java @@ -0,0 +1,10 @@ +package be.vlaanderen.informatievlaanderen.ldes.ldio.valuebojects; + +import com.fasterxml.jackson.annotation.JsonUnwrapped; + +public record LdioComponent( + String name, + @JsonUnwrapped + LdioComponentProperties properties +) { +} diff --git a/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/valuebojects/LdioComponentProperties.java b/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/valuebojects/LdioComponentProperties.java new file mode 100644 index 0000000..5581536 --- /dev/null +++ b/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/valuebojects/LdioComponentProperties.java @@ -0,0 +1,8 @@ +package be.vlaanderen.informatievlaanderen.ldes.ldio.valuebojects; + +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.Map; + +public record LdioComponentProperties(@JsonProperty("config") Map properties) { +} diff --git a/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/valuebojects/LdioPipeline.java b/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/valuebojects/LdioPipeline.java new file mode 100644 index 0000000..c292ca1 --- /dev/null +++ b/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/valuebojects/LdioPipeline.java @@ -0,0 +1,18 @@ +package be.vlaanderen.informatievlaanderen.ldes.ldio.valuebojects; + +import com.fasterxml.jackson.annotation.JsonInclude; + +import java.util.List; + +public record LdioPipeline( + String name, + String description, + LdioComponent input, + @JsonInclude(value = JsonInclude.Include.NON_EMPTY) + List transformers, + List outputs +) { + public LdioPipeline(String name, String description, LdioComponent input, List outputs) { + this(name, description, input, List.of(), outputs); + } +} diff --git a/src/main/java/be/vlaanderen/informatievlaanderen/ldes/rdfrepo/Rdf4jRepositoryManager.java b/src/main/java/be/vlaanderen/informatievlaanderen/ldes/rdfrepo/Rdf4jRepositoryManager.java index 622b033..d530458 100644 --- a/src/main/java/be/vlaanderen/informatievlaanderen/ldes/rdfrepo/Rdf4jRepositoryManager.java +++ b/src/main/java/be/vlaanderen/informatievlaanderen/ldes/rdfrepo/Rdf4jRepositoryManager.java @@ -1,32 +1,38 @@ package be.vlaanderen.informatievlaanderen.ldes.rdfrepo; -import org.eclipse.rdf4j.repository.Repository; +import be.vlaanderen.informatievlaanderen.ldes.ldio.config.LdioConfigProperties; import org.eclipse.rdf4j.repository.config.RepositoryConfig; -import org.eclipse.rdf4j.repository.http.config.HTTPRepositoryConfig; +import org.eclipse.rdf4j.repository.config.RepositoryImplConfig; +import org.eclipse.rdf4j.repository.manager.RemoteRepositoryManager; import org.eclipse.rdf4j.repository.manager.RepositoryManager; -import org.eclipse.rdf4j.repository.manager.RepositoryProvider; +import org.eclipse.rdf4j.repository.sail.config.SailRepositoryConfig; +import org.eclipse.rdf4j.sail.memory.config.MemoryStoreConfig; +import org.eclipse.rdf4j.sail.shacl.config.ShaclSailConfig; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; +import static be.vlaanderen.informatievlaanderen.ldes.ldio.config.LdioConfigProperties.REPOSITORY_ID; + @Component public class Rdf4jRepositoryManager { + private static final Logger log = LoggerFactory.getLogger(Rdf4jRepositoryManager.class); + private final RepositoryManager repositoryManager; - private static final String repoIdBase = "validation"; - private final String serverUrl = "http://localhost:8080/rdf4j-server"; - private final RepositoryManager repositoryManager; - - public Rdf4jRepositoryManager() { - repositoryManager = RepositoryProvider.getRepositoryManager(serverUrl); - repositoryManager.init(); - } + public Rdf4jRepositoryManager(LdioConfigProperties ldioProperties) { + repositoryManager = new RemoteRepositoryManager(ldioProperties.getSparqlHost()); + repositoryManager.init(); + } - public String initRepo() { - String repoId = repositoryManager.getNewRepositoryID(repoIdBase); - new RepositoryConfig(repoId, new HTTPRepositoryConfig(serverUrl)); - repositoryManager.addRepositoryConfig(new RepositoryConfig()); - return repoId; - } + public void createRepository() { + final RepositoryImplConfig repositoryTypeSpec = new SailRepositoryConfig(new ShaclSailConfig(new MemoryStoreConfig(true))); + final RepositoryConfig config = new RepositoryConfig(REPOSITORY_ID, repositoryTypeSpec); + repositoryManager.addRepositoryConfig(config); + log.atInfo().log("Repository created with repository id: {}", REPOSITORY_ID); + } - public Repository getRepo(String id) { - return repositoryManager.getRepository(id); - } + public void deleteRepository() { + repositoryManager.removeRepository(REPOSITORY_ID); + log.atInfo().log("Repository deleted with repository id: {}", REPOSITORY_ID); + } } diff --git a/src/main/java/be/vlaanderen/informatievlaanderen/ldes/rdfrepo/RepositoryValidator.java b/src/main/java/be/vlaanderen/informatievlaanderen/ldes/rdfrepo/RepositoryValidator.java index bd39e74..c50c773 100644 --- a/src/main/java/be/vlaanderen/informatievlaanderen/ldes/rdfrepo/RepositoryValidator.java +++ b/src/main/java/be/vlaanderen/informatievlaanderen/ldes/rdfrepo/RepositoryValidator.java @@ -1,48 +1,42 @@ package be.vlaanderen.informatievlaanderen.ldes.rdfrepo; -import be.vlaanderen.informatievlaanderen.ldes.http.Request; import be.vlaanderen.informatievlaanderen.ldes.http.RequestExecutor; -import be.vlaanderen.informatievlaanderen.ldes.services.RDFConverter; +import be.vlaanderen.informatievlaanderen.ldes.http.requests.PostRequest; +import be.vlaanderen.informatievlaanderen.ldes.ldio.config.LdioConfigProperties; +import org.apache.http.HttpEntity; import org.eclipse.rdf4j.model.Model; -import org.eclipse.rdf4j.model.impl.DynamicModelFactory; -import org.eclipse.rdf4j.repository.Repository; -import org.eclipse.rdf4j.repository.RepositoryConnection; import org.eclipse.rdf4j.rio.RDFFormat; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.web.bind.annotation.RequestMethod; +import org.eclipse.rdf4j.rio.Rio; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; -public class RepositoryValidator { - - private static final String REPO_VALIDATION_URL_TEMPLATE = "%s/rest/repositories/%s/validate/text"; - private static final RDFFormat CONTENT_TYPE = RDFFormat.TURTLE; - private String repoUrl; - @Autowired - private Rdf4jRepositoryManager repositoryManager; - @Autowired - private RequestExecutor requestExecutor; - private RepositoryConnection connection; - - public RepositoryValidator() { - } - - public Model validate(Model shaclShape) { - String repositoryId = repositoryManager.initRepo(); - Repository repository = repositoryManager.getRepo(repositoryId); - try { - Model validationReport = new DynamicModelFactory().createEmptyModel(); - requestExecutor.execute(new Request( - String.format(REPO_VALIDATION_URL_TEMPLATE, repoUrl, repositoryId), - RDFConverter.writeModel(shaclShape, RDFFormat.TURTLE), - RequestMethod.POST, CONTENT_TYPE.getName())); +import java.io.IOException; +import java.io.StringWriter; +import java.io.UncheckedIOException; -// RepositoryConnection connection = repository.getConnection(); -// connection.begin(); -// connection.getStatements(null, null, null, ); - - return validationReport; - } finally { - repository.shutDown(); - } - - } +@Component +public class RepositoryValidator { + private static final RDFFormat CONTENT_TYPE = RDFFormat.TURTLE; + private static final Logger log = LoggerFactory.getLogger(RepositoryValidator.class); + private final RequestExecutor requestExecutor; + private final String repositoryValidationUrl; + + public RepositoryValidator(RequestExecutor requestExecutor, LdioConfigProperties ldioProperties) { + this.requestExecutor = requestExecutor; + this.repositoryValidationUrl = ldioProperties.getRepositoryValidationUrl(); + } + + public Model validate(Model shaclShape) { + log.atInfo().log("Validating repository ..."); + final StringWriter shaclShapeWriter = new StringWriter(); + Rio.write(shaclShape, shaclShapeWriter, CONTENT_TYPE); + final PostRequest postRequest = new PostRequest(repositoryValidationUrl, shaclShapeWriter.toString(), CONTENT_TYPE.getDefaultMIMEType()); + final HttpEntity response = requestExecutor.execute(postRequest); + try { + return Rio.parse(response.getContent(), CONTENT_TYPE); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } } diff --git a/src/main/java/be/vlaanderen/informatievlaanderen/ldes/services/RDFConverter.java b/src/main/java/be/vlaanderen/informatievlaanderen/ldes/services/RDFConverter.java index aa921d9..4219e10 100644 --- a/src/main/java/be/vlaanderen/informatievlaanderen/ldes/services/RDFConverter.java +++ b/src/main/java/be/vlaanderen/informatievlaanderen/ldes/services/RDFConverter.java @@ -7,15 +7,23 @@ import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.io.UncheckedIOException; public class RDFConverter { + private RDFConverter() { + + } public static String writeModel(Model model, RDFFormat format) { ByteArrayOutputStream out = new ByteArrayOutputStream(); Rio.write(model, out, format); return out.toString(); } - public static Model readModel(String content, RDFFormat format) throws IOException { - return Rio.parse(new ByteArrayInputStream(content.getBytes()), format); + public static Model readModel(String content, RDFFormat format) { + try { + return Rio.parse(new ByteArrayInputStream(content.getBytes()), format); + } catch (IOException e) { + throw new UncheckedIOException(e); + } } } diff --git a/src/main/java/be/vlaanderen/informatievlaanderen/ldes/services/TarSupplier.java b/src/main/java/be/vlaanderen/informatievlaanderen/ldes/services/TarSupplier.java new file mode 100644 index 0000000..39d835d --- /dev/null +++ b/src/main/java/be/vlaanderen/informatievlaanderen/ldes/services/TarSupplier.java @@ -0,0 +1,37 @@ +package be.vlaanderen.informatievlaanderen.ldes.services; + +import com.gitb.core.AnyContent; +import com.gitb.tr.TAR; +import com.gitb.tr.TestResultType; + +import javax.xml.datatype.DatatypeConfigurationException; +import javax.xml.datatype.DatatypeFactory; +import java.util.GregorianCalendar; +import java.util.function.Supplier; + +public class TarSupplier implements Supplier { + private final TestResultType testResultType; + + public TarSupplier(TestResultType testResultType) { + this.testResultType = testResultType; + } + + @Override + public TAR get() { + final TAR tar = new TAR(); + final AnyContent context = new AnyContent(); + context.setType("map"); + tar.setContext(context); + tar.setResult(testResultType); + try { + tar.setDate(DatatypeFactory.newInstance().newXMLGregorianCalendar(new GregorianCalendar())); + } catch (DatatypeConfigurationException e) { + throw new IllegalStateException(e); + } + return tar; + } + + public static TAR success() { + return new TarSupplier(TestResultType.SUCCESS).get(); + } +} diff --git a/src/main/java/be/vlaanderen/informatievlaanderen/ldes/services/ValidationReportToTarMapper.java b/src/main/java/be/vlaanderen/informatievlaanderen/ldes/services/ValidationReportToTarMapper.java new file mode 100644 index 0000000..d125628 --- /dev/null +++ b/src/main/java/be/vlaanderen/informatievlaanderen/ldes/services/ValidationReportToTarMapper.java @@ -0,0 +1,44 @@ +package be.vlaanderen.informatievlaanderen.ldes.services; + +import be.vlaanderen.informatievlaanderen.ldes.valueobjects.ValidationReport; +import be.vlaanderen.informatievlaanderen.ldes.valueobjects.severitylevels.SeverityLevel; +import com.gitb.tr.BAR; +import com.gitb.tr.TAR; +import com.gitb.tr.TestAssertionGroupReportsType; +import com.gitb.tr.ValidationCounters; +import org.eclipse.rdf4j.model.Model; +import org.eclipse.rdf4j.rio.RDFFormat; + +import java.math.BigInteger; + +public class ValidationReportToTarMapper { + private ValidationReportToTarMapper() { + } + + public static TAR mapToTar(ValidationReport validationReport) { + final SeverityLevel highestSeverityLevel = validationReport.getHighestSeverityLevel(); + final TAR tarReport = highestSeverityLevel.createTarReport(); + + final TestAssertionGroupReportsType reportsType = new TestAssertionGroupReportsType(); + reportsType.getInfoOrWarningOrError().add(highestSeverityLevel + .mapToJaxbElement(createReportItemContent(validationReport.shaclReport()))); + tarReport.setReports(reportsType); + tarReport.setCounters(extractValidationCounters(validationReport)); + return tarReport; + } + + private static BAR createReportItemContent(Model shaclReport) { + BAR itemContent = new BAR(); + itemContent.setDescription(RDFConverter.writeModel(shaclReport, RDFFormat.TURTLE)); + return itemContent; + } + + private static ValidationCounters extractValidationCounters(ValidationReport validationReport) { + final ValidationCounters validationCounters = new ValidationCounters(); + validationCounters.setNrOfAssertions(BigInteger.valueOf(validationReport.infoCount())); + validationCounters.setNrOfWarnings(BigInteger.valueOf(validationReport.warningCount())); + validationCounters.setNrOfErrors(BigInteger.valueOf(validationReport.errorCount())); + return new ValidationCounters(); + + } +} diff --git a/src/main/java/be/vlaanderen/informatievlaanderen/ldes/shacl/ShaclValidator.java b/src/main/java/be/vlaanderen/informatievlaanderen/ldes/shacl/ShaclValidator.java new file mode 100644 index 0000000..3b4a038 --- /dev/null +++ b/src/main/java/be/vlaanderen/informatievlaanderen/ldes/shacl/ShaclValidator.java @@ -0,0 +1,37 @@ +package be.vlaanderen.informatievlaanderen.ldes.shacl; + +import be.vlaanderen.informatievlaanderen.ldes.ldio.LdesClientStatusManager; +import be.vlaanderen.informatievlaanderen.ldes.ldio.LdioPipelineManager; +import be.vlaanderen.informatievlaanderen.ldes.rdfrepo.Rdf4jRepositoryManager; +import be.vlaanderen.informatievlaanderen.ldes.rdfrepo.RepositoryValidator; +import be.vlaanderen.informatievlaanderen.ldes.valueobjects.ValidationParameters; +import be.vlaanderen.informatievlaanderen.ldes.valueobjects.ValidationReport; +import org.eclipse.rdf4j.model.Model; +import org.springframework.stereotype.Component; + +import static be.vlaanderen.informatievlaanderen.ldes.ldio.pipeline.ValidationPipelineSupplier.PIPELINE_NAME_TEMPLATE; + +@Component +public class ShaclValidator { + private final LdioPipelineManager ldioPipelineManager; + private final LdesClientStatusManager clientStatusManager; + private final Rdf4jRepositoryManager repositoryManager; + private final RepositoryValidator validator; + + public ShaclValidator(LdioPipelineManager ldioPipelineManager, LdesClientStatusManager clientStatusManager, Rdf4jRepositoryManager repositoryManager, RepositoryValidator validator) { + this.ldioPipelineManager = ldioPipelineManager; + this.clientStatusManager = clientStatusManager; + this.repositoryManager = repositoryManager; + this.validator = validator; + } + + public ValidationReport validate(ValidationParameters params) { + repositoryManager.createRepository(); + ldioPipelineManager.initPipeline(params.ldesUrl(), params.pipelineName()); + clientStatusManager.waitUntilReplicated(PIPELINE_NAME_TEMPLATE.formatted(params.sessionId())); + ldioPipelineManager.deletePipeline(params.pipelineName()); + final Model shaclValidationReport = validator.validate(params.shaclShape()); + repositoryManager.deleteRepository(); + return new ValidationReport(shaclValidationReport); + } +} diff --git a/src/main/java/be/vlaanderen/informatievlaanderen/ldes/valueobjects/Parameters.java b/src/main/java/be/vlaanderen/informatievlaanderen/ldes/valueobjects/Parameters.java new file mode 100644 index 0000000..157debf --- /dev/null +++ b/src/main/java/be/vlaanderen/informatievlaanderen/ldes/valueobjects/Parameters.java @@ -0,0 +1,38 @@ +package be.vlaanderen.informatievlaanderen.ldes.valueobjects; + +import com.gitb.core.AnyContent; +import com.gitb.core.ValueEmbeddingEnumeration; +import org.jetbrains.annotations.NotNull; + +import java.util.Base64; +import java.util.List; + +public class Parameters { + private final List items; + + public Parameters(@NotNull List items) { + this.items = items; + } + + public String getStringForName(String inputName) { + final AnyContent item = getSingleContentForName(inputName); + if(item.getEmbeddingMethod().equals(ValueEmbeddingEnumeration.BASE_64)) { + return new String(Base64.getDecoder().decode(item.getValue())); + } + return item.getValue(); + } + + private AnyContent getSingleContentForName(String name) { + var inputs = getInputsForName(name); + if (inputs.isEmpty()) { + throw new IllegalArgumentException(String.format("No input named [%s] was found.", name)); + } else if (inputs.size() > 1) { + throw new IllegalArgumentException(String.format("Multiple inputs named [%s] were found when only one was expected.", name)); + } + return inputs.get(0); + } + + private List getInputsForName(String name) { + return items.stream().filter(content -> content.getName().equals(name)).toList(); + } +} diff --git a/src/main/java/be/vlaanderen/informatievlaanderen/ldes/valueobjects/ValidationParameters.java b/src/main/java/be/vlaanderen/informatievlaanderen/ldes/valueobjects/ValidationParameters.java new file mode 100644 index 0000000..3485f15 --- /dev/null +++ b/src/main/java/be/vlaanderen/informatievlaanderen/ldes/valueobjects/ValidationParameters.java @@ -0,0 +1,11 @@ +package be.vlaanderen.informatievlaanderen.ldes.valueobjects; + +import org.eclipse.rdf4j.model.Model; + +import static be.vlaanderen.informatievlaanderen.ldes.ldio.pipeline.ValidationPipelineSupplier.PIPELINE_NAME_TEMPLATE; + +public record ValidationParameters(String ldesUrl, Model shaclShape, String sessionId) { + public String pipelineName() { + return PIPELINE_NAME_TEMPLATE.formatted(sessionId); + } +} diff --git a/src/main/java/be/vlaanderen/informatievlaanderen/ldes/valueobjects/ValidationReport.java b/src/main/java/be/vlaanderen/informatievlaanderen/ldes/valueobjects/ValidationReport.java new file mode 100644 index 0000000..6c4e820 --- /dev/null +++ b/src/main/java/be/vlaanderen/informatievlaanderen/ldes/valueobjects/ValidationReport.java @@ -0,0 +1,43 @@ +package be.vlaanderen.informatievlaanderen.ldes.valueobjects; + +import be.vlaanderen.informatievlaanderen.ldes.valueobjects.severitylevels.SeverityLevel; +import be.vlaanderen.informatievlaanderen.ldes.valueobjects.severitylevels.SeverityLevels; +import com.google.common.collect.Iterables; +import org.eclipse.rdf4j.model.IRI; +import org.eclipse.rdf4j.model.Model; + +import java.util.Arrays; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; + +import static be.vlaanderen.informatievlaanderen.ldes.constants.RDFConstants.*; + +public record ValidationReport(Model shaclReport) { + + public int errorCount() { + return getCountFor(VIOLATION); + } + + public int warningCount() { + return getCountFor(WARNING); + } + + public int infoCount() { + return getCountFor(INFO); + } + + public SeverityLevel getHighestSeverityLevel() { + return Arrays.stream(SeverityLevels.all()) + .collect(Collectors.toMap(Function.identity(), severityLevel -> getCountFor(severityLevel.getIri()))) + .entrySet().stream() + .filter(entry -> entry.getValue() > 0) + .findFirst() + .map(Map.Entry::getKey) + .orElse(SeverityLevels.INFO); + } + + private int getCountFor(IRI severity) { + return Iterables.size(shaclReport.getStatements(null, SEVERITY, severity)); + } +} diff --git a/src/main/java/be/vlaanderen/informatievlaanderen/ldes/valueobjects/severitylevels/ErrorSeverityLevel.java b/src/main/java/be/vlaanderen/informatievlaanderen/ldes/valueobjects/severitylevels/ErrorSeverityLevel.java new file mode 100644 index 0000000..c339a05 --- /dev/null +++ b/src/main/java/be/vlaanderen/informatievlaanderen/ldes/valueobjects/severitylevels/ErrorSeverityLevel.java @@ -0,0 +1,29 @@ +package be.vlaanderen.informatievlaanderen.ldes.valueobjects.severitylevels; + +import be.vlaanderen.informatievlaanderen.ldes.constants.RDFConstants; +import be.vlaanderen.informatievlaanderen.ldes.services.TarSupplier; +import com.gitb.tr.TAR; +import com.gitb.tr.TestAssertionReportType; +import com.gitb.tr.TestResultType; +import jakarta.xml.bind.JAXBElement; +import org.eclipse.rdf4j.model.IRI; + +public class ErrorSeverityLevel implements SeverityLevel { + ErrorSeverityLevel() { + } + + @Override + public IRI getIri() { + return RDFConstants.VIOLATION; + } + + @Override + public JAXBElement mapToJaxbElement(TestAssertionReportType testAssertionReportType) { + return SeverityLevel.objectMapper.createTestAssertionGroupReportsTypeError(testAssertionReportType); + } + + @Override + public TAR createTarReport() { + return new TarSupplier(TestResultType.FAILURE).get(); + } +} diff --git a/src/main/java/be/vlaanderen/informatievlaanderen/ldes/valueobjects/severitylevels/InfoSeverityLevel.java b/src/main/java/be/vlaanderen/informatievlaanderen/ldes/valueobjects/severitylevels/InfoSeverityLevel.java new file mode 100644 index 0000000..fcf1945 --- /dev/null +++ b/src/main/java/be/vlaanderen/informatievlaanderen/ldes/valueobjects/severitylevels/InfoSeverityLevel.java @@ -0,0 +1,28 @@ +package be.vlaanderen.informatievlaanderen.ldes.valueobjects.severitylevels; + +import be.vlaanderen.informatievlaanderen.ldes.constants.RDFConstants; +import be.vlaanderen.informatievlaanderen.ldes.services.TarSupplier; +import com.gitb.tr.TAR; +import com.gitb.tr.TestAssertionReportType; +import jakarta.xml.bind.JAXBElement; +import org.eclipse.rdf4j.model.IRI; + +public class InfoSeverityLevel implements SeverityLevel { + InfoSeverityLevel() { + } + + @Override + public IRI getIri() { + return RDFConstants.INFO; + } + + @Override + public JAXBElement mapToJaxbElement(TestAssertionReportType testAssertionReportType) { + return SeverityLevel.objectMapper.createTestAssertionGroupReportsTypeInfo(testAssertionReportType); + } + + @Override + public TAR createTarReport() { + return TarSupplier.success(); + } +} diff --git a/src/main/java/be/vlaanderen/informatievlaanderen/ldes/valueobjects/severitylevels/SeverityLevel.java b/src/main/java/be/vlaanderen/informatievlaanderen/ldes/valueobjects/severitylevels/SeverityLevel.java new file mode 100644 index 0000000..dd0fb3d --- /dev/null +++ b/src/main/java/be/vlaanderen/informatievlaanderen/ldes/valueobjects/severitylevels/SeverityLevel.java @@ -0,0 +1,16 @@ +package be.vlaanderen.informatievlaanderen.ldes.valueobjects.severitylevels; + +import com.gitb.tr.ObjectFactory; +import com.gitb.tr.TAR; +import com.gitb.tr.TestAssertionReportType; +import jakarta.xml.bind.JAXBElement; +import org.eclipse.rdf4j.model.IRI; + +public interface SeverityLevel { + ObjectFactory objectMapper = new ObjectFactory(); + IRI getIri(); + + JAXBElement mapToJaxbElement(TestAssertionReportType testAssertionReportType); + + TAR createTarReport(); +} diff --git a/src/main/java/be/vlaanderen/informatievlaanderen/ldes/valueobjects/severitylevels/SeverityLevels.java b/src/main/java/be/vlaanderen/informatievlaanderen/ldes/valueobjects/severitylevels/SeverityLevels.java new file mode 100644 index 0000000..0e75ac9 --- /dev/null +++ b/src/main/java/be/vlaanderen/informatievlaanderen/ldes/valueobjects/severitylevels/SeverityLevels.java @@ -0,0 +1,17 @@ +package be.vlaanderen.informatievlaanderen.ldes.valueobjects.severitylevels; + +public class SeverityLevels { + private SeverityLevels() {} + + public static final SeverityLevel ERROR = new ErrorSeverityLevel(); + public static final SeverityLevel WARNING = new WarningSeverityLevel(); + public static final SeverityLevel INFO = new InfoSeverityLevel(); + + public static SeverityLevel[] all() { + return new SeverityLevel[] { + ERROR, + WARNING, + INFO + }; + } +} diff --git a/src/main/java/be/vlaanderen/informatievlaanderen/ldes/valueobjects/severitylevels/WarningSeverityLevel.java b/src/main/java/be/vlaanderen/informatievlaanderen/ldes/valueobjects/severitylevels/WarningSeverityLevel.java new file mode 100644 index 0000000..fe55e5b --- /dev/null +++ b/src/main/java/be/vlaanderen/informatievlaanderen/ldes/valueobjects/severitylevels/WarningSeverityLevel.java @@ -0,0 +1,30 @@ +package be.vlaanderen.informatievlaanderen.ldes.valueobjects.severitylevels; + +import be.vlaanderen.informatievlaanderen.ldes.constants.RDFConstants; +import be.vlaanderen.informatievlaanderen.ldes.services.TarSupplier; +import com.gitb.tr.TAR; +import com.gitb.tr.TestAssertionReportType; +import com.gitb.tr.TestResultType; +import jakarta.xml.bind.JAXBElement; +import org.eclipse.rdf4j.model.IRI; + +public class WarningSeverityLevel implements SeverityLevel { + WarningSeverityLevel() { + + } + + @Override + public IRI getIri() { + return RDFConstants.WARNING; + } + + @Override + public JAXBElement mapToJaxbElement(TestAssertionReportType testAssertionReportType) { + return SeverityLevel.objectMapper.createTestAssertionGroupReportsTypeWarning(testAssertionReportType); + } + + @Override + public TAR createTarReport() { + return new TarSupplier(TestResultType.WARNING).get(); + } +} diff --git a/src/main/java/be/vlaanderen/informatievlaanderen/ldes/web/UserInputController.java b/src/main/java/be/vlaanderen/informatievlaanderen/ldes/web/UserInputController.java deleted file mode 100644 index a9c5c23..0000000 --- a/src/main/java/be/vlaanderen/informatievlaanderen/ldes/web/UserInputController.java +++ /dev/null @@ -1,78 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.web; - -import com.gitb.core.ValueEmbeddingEnumeration; -import com.gitb.tr.TAR; -import com.gitb.tr.TestResultType; -import be.vlaanderen.informatievlaanderen.ldes.gitb.StateManager; -import be.vlaanderen.informatievlaanderen.ldes.gitb.TestBedNotifier; -import be.vlaanderen.informatievlaanderen.ldes.gitb.Utils; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RequestMethod; -import org.springframework.web.bind.annotation.RequestParam; -import org.springframework.web.bind.annotation.RestController; - -import java.util.ArrayList; -import java.util.List; - -/** - * Simple REST controller to allow an easy way of providing a message for the test bed. - *

- * This implementation acts a sample of how messages could be sent to the test bed. In this case this is done - * via a simple HTTP GET service that accepts two parameters: - *

    - *
  • session: The test session ID. Not providing this will send notifications for all active sessions.
    • - *
  • message: The message to send. Not providing this will consider an empty string.
    • - *
- * One of the key points to define when using a messaging service is the approach to match received messages to - * waiting test bed sessions. In this example a very simple approach is foreseen, expecting the session ID to be - * passed as a parameter (or be omitted to signal all sessions). A more realistic approach would be as follows: - *
    - *
  1. The messaging service records as part of the state for each session a property that will serve to - * uniquely identify it. This could be a transaction identifier, an endpoint address, or some other metadata.
    2. - *
  2. The input provided to the messaging service includes the identifier to use for session matching.
    3. - *
  3. Given such an identifier, the current session state is scanned to find the corresponding session.
    4. - *
- * In addition, keep in mind that the communication protocol involved in sending and receiving messages could be anything. - * In this example we use a HTTP GET request but this could be an email, a SOAP web service call, a polled endpoint or - * filesystem location; anything that corresponds to the actual messaging needs. - */ -@RestController -public class UserInputController { - - @Autowired - private StateManager stateManager = null; - @Autowired - private TestBedNotifier testBedNotifier = null; - @Autowired - private Utils utils = null; - - /** - * HTTP GET service to receive input for the test bed. - *

- * Input received here will be provided back to the test bed as a response to its 'receive' step. - * - * @param session The test session ID this relates to. Omitting this will consider all active sessions. - * @param message The message to send. No message will result in an empty string. - * @return A text configuration message. - */ - @RequestMapping(value = "/input", method = RequestMethod.GET) - public String provideMessage(@RequestParam(value="session", required = false) String session, @RequestParam(value="message", defaultValue = "") String message) { - List sessionIds = new ArrayList<>(); - if (session == null) { - // Send message to all current sessions. - sessionIds.addAll(stateManager.getAllSessions().keySet()); - } else { - sessionIds.add(session); - } - // Input for the test bed is provided by means of a report. - TAR notificationReport = utils.createReport(TestResultType.SUCCESS); - // The report can include any properties and with any nesting (by nesting list of map types). In this case we add a simple string. - notificationReport.getContext().getItem().add(utils.createAnyContentSimple("messageReceived", message, ValueEmbeddingEnumeration.STRING)); - for (String sessionId: sessionIds) { - testBedNotifier.notifyTestBed(sessionId, null, (String)stateManager.getSessionInfo(sessionId, StateManager.SessionData.CALLBACK_URL), notificationReport); - } - return String.format("Sent message [%s] to %s session(s)", message, sessionIds.size()); - } - -} diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index 577062a..5a8cfee 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -24,4 +24,8 @@ # - The username used for proxy authentication. # proxy.auth.username = # - The password used for proxy authentication. -# proxy.auth.password = \ No newline at end of file +# proxy.auth.password = + +server.port=8888 +ldio.host=http://localhost:8383 +ldio.sparql-host=http://CI00321761:7200 \ No newline at end of file diff --git a/src/main/resources/ldio-pipeline.json b/src/main/resources/ldio-pipeline.json deleted file mode 100644 index f672c74..0000000 --- a/src/main/resources/ldio-pipeline.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "name": "validation-pipeline", - "description": "", - "input": { - "name": "Ldio:LdesClient", - "config": { - "urls": "%s", - "source-format": "application/n-quads" - } - }, - "transformers": [ - { - "name": "Ldio:VersionMaterialiser", - "config": { - "versionOf-property": "%s" - } - } - ], - "outputs": [ - { - "name": "Ldio:RepositorySink", - "config": { - "sparql-host": "%s", - "repository-id": "%s" - } - } - ] -} \ No newline at end of file diff --git a/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ApplicationTest.java b/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ApplicationTest.java index 7c9c8c0..a5844a7 100644 --- a/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ApplicationTest.java +++ b/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ApplicationTest.java @@ -2,18 +2,20 @@ import org.junit.jupiter.api.Test; import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.ActiveProfiles; /** * Unit test to ensure that the Spring context loads. */ @SpringBootTest -public class ApplicationTest { +@ActiveProfiles("test") +class ApplicationTest { /** * Test that the context loads. */ @Test - public void contextLoads() { + void contextLoads() { } } diff --git a/src/test/java/be/vlaanderen/informatievlaanderen/ldes/PostRequestAssert.java b/src/test/java/be/vlaanderen/informatievlaanderen/ldes/PostRequestAssert.java new file mode 100644 index 0000000..12d0f1d --- /dev/null +++ b/src/test/java/be/vlaanderen/informatievlaanderen/ldes/PostRequestAssert.java @@ -0,0 +1,47 @@ +package be.vlaanderen.informatievlaanderen.ldes; + +import be.vlaanderen.informatievlaanderen.ldes.http.requests.PostRequest; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.http.entity.ContentType; +import org.assertj.core.api.AbstractAssert; +import org.assertj.core.api.InstanceOfAssertFactories; + +import java.io.UncheckedIOException; + +import static org.assertj.core.api.Assertions.assertThat; + +public class PostRequestAssert extends AbstractAssert { + + public PostRequestAssert(PostRequest postRequest) { + super(postRequest, PostRequestAssert.class); + } + + public PostRequestAssert hasUrl(String expected) { + assertThat(actual) + .extracting("url") + .isEqualTo(expected); + return this; + } + + public PostRequestAssert hasBody(JsonNode expected) { + assertThat(actual) + .extracting("body", InstanceOfAssertFactories.STRING) + .matches(actualBody -> { + try { + return new ObjectMapper().readTree(actualBody).equals(expected); + } catch (JsonProcessingException e) { + throw new UncheckedIOException(e); + } + }); + return this; + } + + public PostRequestAssert hasContentType(ContentType expected) { + assertThat(actual) + .extracting("contentType") + .isEqualTo(expected); + return this; + } +} diff --git a/src/test/java/be/vlaanderen/informatievlaanderen/ldes/gitb/ValidationServiceImplTest.java b/src/test/java/be/vlaanderen/informatievlaanderen/ldes/gitb/ValidationServiceImplTest.java new file mode 100644 index 0000000..09910b4 --- /dev/null +++ b/src/test/java/be/vlaanderen/informatievlaanderen/ldes/gitb/ValidationServiceImplTest.java @@ -0,0 +1,109 @@ +package be.vlaanderen.informatievlaanderen.ldes.gitb; + +import be.vlaanderen.informatievlaanderen.ldes.http.RequestExecutor; +import be.vlaanderen.informatievlaanderen.ldes.http.requests.DeleteRequest; +import be.vlaanderen.informatievlaanderen.ldes.http.requests.GetRequest; +import be.vlaanderen.informatievlaanderen.ldes.http.requests.PostRequest; +import be.vlaanderen.informatievlaanderen.ldes.rdfrepo.Rdf4jRepositoryManager; +import org.apache.http.entity.BasicHttpEntity; +import org.apache.http.entity.InputStreamEntity; +import org.assertj.core.api.InstanceOfAssertFactories; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.boot.test.web.client.TestRestTemplate; +import org.springframework.context.annotation.ComponentScan; +import org.springframework.http.HttpEntity; +import org.springframework.http.HttpHeaders; +import org.springframework.http.MediaType; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.util.ResourceUtils; + +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.nio.file.Files; +import java.util.stream.Stream; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.*; + +@EnableAutoConfiguration +@SpringBootTest(properties = {"ldio.host=http://ldio-workbench:8080", "ldio.sparql-host=http://graph-db:7200"}, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) +@ContextConfiguration(classes = ServiceConfig.class) +@ComponentScan(value = {"be.vlaanderen.informatievlaanderen.ldes"}) +class ValidationServiceImplTest { + private static final String LDIO_HOST = "http://ldio-workbench:8080"; + private static final String PIPELINE_UUID = "test-pipeline-uuid"; + private static final String LDIO_LDES_CLIENT_STATUS_URL = LDIO_HOST + "/admin/api/v1/pipeline/ldes-client/validation-pipeline-" + PIPELINE_UUID; + private static final String LDES_SERVER_URL = "http://ldes-server:8080/verkeersmetingen"; + @MockBean + private RequestExecutor requestExecutor; + @MockBean + private Rdf4jRepositoryManager rdf4jRepositoryManager; + @Autowired + private TestRestTemplate restTemplate; + + static Stream provideShaclShapes() { + return Stream.of( + Arguments.of("validation-report/invalid.ttl", "sh:conforms false;"), + Arguments.of("validation-report/valid.ttl", "sh:conforms true;") + ); + } + + @ParameterizedTest + @MethodSource("provideShaclShapes") + void test_ValidationServiceImpl(String fileName, String expectedShaclConformity) throws IOException { + when(requestExecutor.execute(new GetRequest(LDES_SERVER_URL))) + .thenReturn(createResponse(ResourceUtils.getFile("classpath:event-stream.ttl"))); + when(requestExecutor.execute(any(PostRequest.class), eq(201))) + .thenReturn(new BasicHttpEntity()); + when(requestExecutor.execute(new GetRequest(LDIO_LDES_CLIENT_STATUS_URL), 200, 404)) + .thenReturn(createEmptyResponse()) + .thenReturn(createResponse("\"REPLICATING\"")) + .thenReturn(createResponse("\"SYNCHRONISING\"")); + when(requestExecutor.execute(any(DeleteRequest.class), eq(202), eq(204))) + .thenReturn(new BasicHttpEntity()); + when(requestExecutor.execute(any(PostRequest.class))) + .thenReturn(createResponse(ResourceUtils.getFile("classpath:" + fileName))); + + final var result = restTemplate.postForEntity("/services/validation?wsdl", createRequest(), String.class); + + assertThat(result) + .extracting(HttpEntity::getBody, InstanceOfAssertFactories.STRING) + .contains(expectedShaclConformity); + verify(rdf4jRepositoryManager).createRepository(); + verify(rdf4jRepositoryManager).deleteRepository(); + verify(requestExecutor).execute(any(PostRequest.class), eq(201)); + verify(requestExecutor).execute(any(DeleteRequest.class), eq(202), eq(204)); + } + + private static BasicHttpEntity createEmptyResponse() { + final BasicHttpEntity response = new BasicHttpEntity(); + response.setContentLength(0); + return response; + } + + private static BasicHttpEntity createResponse(String content) { + final BasicHttpEntity response = new BasicHttpEntity(); + response.setContent(new ByteArrayInputStream(content.getBytes())); + response.setContentLength(content.length()); + return response; + } + + private static InputStreamEntity createResponse(File file) throws IOException { + return new InputStreamEntity(new FileInputStream(file)); + } + + private static HttpEntity createRequest() throws IOException { + final HttpHeaders headers = new HttpHeaders(); + headers.setContentType(MediaType.TEXT_XML); + final String requestPayload = Files.readString(ResourceUtils.getFile("classpath:validate-request.xml").toPath()); + return new HttpEntity<>(requestPayload, headers); + } +} \ No newline at end of file diff --git a/src/test/java/be/vlaanderen/informatievlaanderen/ldes/handlers/ShaclValidationHandlerTest.java b/src/test/java/be/vlaanderen/informatievlaanderen/ldes/handlers/ShaclValidationHandlerTest.java deleted file mode 100644 index b787da9..0000000 --- a/src/test/java/be/vlaanderen/informatievlaanderen/ldes/handlers/ShaclValidationHandlerTest.java +++ /dev/null @@ -1,26 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.handlers; - -import org.eclipse.rdf4j.model.impl.DynamicModelFactory; -import org.junit.jupiter.api.Test; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.autoconfigure.EnableAutoConfiguration; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.context.annotation.ComponentScan; - -import java.io.IOException; - -import static org.junit.jupiter.api.Assertions.*; - -@EnableAutoConfiguration -@SpringBootTest -@ComponentScan(value = { "be.vlaanderen.informatievlaanderen.ldes.server" }) -class ShaclValidationHandlerTest { - @Autowired - ShaclValidationHandler validationHandler; - - @Test - void test() throws IOException { - validationHandler.validate("http://localhost:8082", new DynamicModelFactory().createEmptyModel()); - } - -} \ No newline at end of file diff --git a/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldes/valueobjects/EventStreamFetcherTest.java b/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldes/valueobjects/EventStreamFetcherTest.java new file mode 100644 index 0000000..d03b42f --- /dev/null +++ b/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldes/valueobjects/EventStreamFetcherTest.java @@ -0,0 +1,38 @@ +package be.vlaanderen.informatievlaanderen.ldes.ldes.valueobjects; + +import be.vlaanderen.informatievlaanderen.ldes.http.RequestExecutor; +import be.vlaanderen.informatievlaanderen.ldes.ldes.EventStreamFetcher; +import be.vlaanderen.informatievlaanderen.ldes.ldes.EventStreamProperties; +import org.apache.http.entity.BasicHttpEntity; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.io.FileInputStream; +import java.io.IOException; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +class EventStreamFetcherTest { + @Mock + private RequestExecutor requestExecutor; + @InjectMocks + private EventStreamFetcher eventStreamFetcher; + + @Test + void test_FetchEventStream() throws IOException { + final EventStreamProperties expected = new EventStreamProperties("http://test.com", "http://purl.org/dc/terms/isVersionOf"); + final BasicHttpEntity httpEntity = new BasicHttpEntity(); + httpEntity.setContent(new FileInputStream("src/test/resources/event-stream.ttl")); + when(requestExecutor.execute(any())).thenReturn(httpEntity); + + final EventStreamProperties actual = eventStreamFetcher.fetchProperties("http://test.com"); + + assertThat(actual).isEqualTo(expected); + } +} \ No newline at end of file diff --git a/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdesClientStatusManagerTest.java b/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdesClientStatusManagerTest.java new file mode 100644 index 0000000..5a4e4d0 --- /dev/null +++ b/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdesClientStatusManagerTest.java @@ -0,0 +1,91 @@ +package be.vlaanderen.informatievlaanderen.ldes.ldio; + +import be.vlaanderen.informatievlaanderen.ldes.http.RequestExecutor; +import be.vlaanderen.informatievlaanderen.ldes.http.requests.GetRequest; +import be.vlaanderen.informatievlaanderen.ldes.ldio.config.LdioConfigProperties; +import be.vlaanderen.informatievlaanderen.ldes.ldio.excpeptions.LdesClientStatusUnavailableException; +import be.vlaanderen.informatievlaanderen.ldes.ldio.valuebojects.ClientStatus; +import org.apache.http.HttpEntity; +import org.apache.http.entity.BasicHttpEntity; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.io.ByteArrayInputStream; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.*; + +@ExtendWith(MockitoExtension.class) +class LdesClientStatusManagerTest { + private static final String PIPELINE_NAME = "test-pipeline"; + private final Integer[] expectedStatusCodes = {200, 404}; + @Mock + private RequestExecutor requestExecutor; + private LdesClientStatusManager ldesClientStatusManager; + + @BeforeEach + void setUp() { + final LdioConfigProperties ldioConfigProperties = new LdioConfigProperties(); + ldioConfigProperties.setHost("http://ldio-workben-host.vlaanderen.be"); + ldesClientStatusManager = new LdesClientStatusManager(requestExecutor, ldioConfigProperties); + } + + @Test + void test_WaitUntilReplicated() { + when(requestExecutor.execute(any(), eq(expectedStatusCodes))) + .thenReturn(createResponse(ClientStatus.REPLICATING)) + .thenReturn(createResponse(ClientStatus.REPLICATING)) + .thenReturn(createResponse(ClientStatus.SYNCHRONISING)); + + ldesClientStatusManager.waitUntilReplicated(PIPELINE_NAME); + + verify(requestExecutor, timeout(10000).times(3)).execute(any(), eq(expectedStatusCodes)); + } + + @Test + void test_WaitUntilReplicated_when_StatusUnavailable() { + final BasicHttpEntity response = new BasicHttpEntity(); + response.setContentLength(0); + when(requestExecutor.execute(any(GetRequest.class), eq(200), eq(404))).thenReturn(response); + + assertThatThrownBy(() -> ldesClientStatusManager.waitUntilReplicated(PIPELINE_NAME)) + .isInstanceOf(IllegalStateException.class) + .hasMessage("Unable to fetch the LDES client status"); + + verify(requestExecutor, times(5)).execute(any(GetRequest.class), eq(200), eq(404)); + } + + @Test + void when_ClientStatusCannotBeFound_then_ThrowException() { + final BasicHttpEntity response = new BasicHttpEntity(); + response.setContentLength(0); + when(requestExecutor.execute(any(), eq(expectedStatusCodes))).thenReturn(response); + + assertThatThrownBy(() -> ldesClientStatusManager.getClientStatus(PIPELINE_NAME)) + .isInstanceOf(LdesClientStatusUnavailableException.class) + .hasMessage("Ldes client status not available."); + } + + @ParameterizedTest + @EnumSource(ClientStatus.class) + void test_GetClientStatus(ClientStatus status) { + when(requestExecutor.execute(any(), eq(expectedStatusCodes))).thenReturn(createResponse(status)); + + final ClientStatus actualStatus = ldesClientStatusManager.getClientStatus(PIPELINE_NAME); + + assertThat(actualStatus).isEqualTo(status); + } + + private HttpEntity createResponse(ClientStatus status) { + final BasicHttpEntity response = new BasicHttpEntity(); + response.setContent(new ByteArrayInputStream(('"' + status.toString() + '"').getBytes())); + return response; + } +} \ No newline at end of file diff --git a/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioPipelineManagerTest.java b/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioPipelineManagerTest.java new file mode 100644 index 0000000..6ec0ef2 --- /dev/null +++ b/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioPipelineManagerTest.java @@ -0,0 +1,78 @@ +package be.vlaanderen.informatievlaanderen.ldes.ldio; + +import be.vlaanderen.informatievlaanderen.ldes.PostRequestAssert; +import be.vlaanderen.informatievlaanderen.ldes.http.RequestExecutor; +import be.vlaanderen.informatievlaanderen.ldes.http.requests.DeleteRequest; +import be.vlaanderen.informatievlaanderen.ldes.http.requests.PostRequest; +import be.vlaanderen.informatievlaanderen.ldes.ldes.EventStreamFetcher; +import be.vlaanderen.informatievlaanderen.ldes.ldes.EventStreamProperties; +import be.vlaanderen.informatievlaanderen.ldes.ldio.config.LdioConfigProperties; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.http.entity.ContentType; +import org.assertj.core.api.InstanceOfAssertFactory; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.util.ResourceUtils; + +import java.io.IOException; + +import static be.vlaanderen.informatievlaanderen.ldes.ldio.pipeline.ValidationPipelineSupplier.PIPELINE_NAME_TEMPLATE; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.assertArg; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +class LdioPipelineManagerTest { + private static final String LDIO_HOST = "http://localhost:8080"; + private static final String SPARQL_HOST = "http://my-sparql-host.net"; + private static final String LDES_SERVER_URL = "http://test-server/test-collection"; + private static final String PIPELINE_UUID = "test-pipeline-uuid"; + private static final String PIPELINE_NAME = PIPELINE_NAME_TEMPLATE.formatted(PIPELINE_UUID); + @Mock + private EventStreamFetcher eventStreamFetcher; + @Mock + private RequestExecutor requestExecutor; + private LdioPipelineManager ldioPipelineManager; + + @BeforeEach + void setUp() { + final LdioConfigProperties ldioConfigProperties = new LdioConfigProperties(); + ldioConfigProperties.setHost(LDIO_HOST); + ldioConfigProperties.setSparqlHost(SPARQL_HOST); + ldioPipelineManager = new LdioPipelineManager(eventStreamFetcher, requestExecutor, ldioConfigProperties); + } + + @Test + void test_InitPipeline() throws IOException { + final JsonNode expectedJson = new ObjectMapper().readTree(ResourceUtils.getFile("classpath:ldio-pipeline.json")); + when(eventStreamFetcher.fetchProperties(LDES_SERVER_URL)).thenReturn(new EventStreamProperties(LDES_SERVER_URL, "http://purl.org/dc/terms/isVersionOf")); + + ldioPipelineManager.initPipeline(LDES_SERVER_URL, PIPELINE_NAME); + + verify(eventStreamFetcher).fetchProperties(LDES_SERVER_URL); + verify(requestExecutor).execute( + assertArg(actual -> assertThat(actual) + .asInstanceOf(new InstanceOfAssertFactory<>(PostRequest.class, PostRequestAssert::new)) + .hasUrl(LDIO_HOST + "/admin/api/v1/pipeline") + .hasBody(expectedJson) + .hasContentType(ContentType.APPLICATION_JSON)), + eq(201)); + } + + @Test + void test_DeletePipeline() { + final DeleteRequest expectedDeleteRequest = new DeleteRequest(LDIO_HOST + "/admin/api/v1/pipeline/" + PIPELINE_NAME); + + ldioPipelineManager.deletePipeline(PIPELINE_NAME); + + verify(requestExecutor).execute( + assertArg(actual -> assertThat(actual).usingRecursiveComparison().isEqualTo(expectedDeleteRequest)), + eq(202), eq(204)); + } +} \ No newline at end of file diff --git a/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/pipeline/ValidationPipelineSupplierTest.java b/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/pipeline/ValidationPipelineSupplierTest.java new file mode 100644 index 0000000..339059c --- /dev/null +++ b/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/pipeline/ValidationPipelineSupplierTest.java @@ -0,0 +1,38 @@ +package be.vlaanderen.informatievlaanderen.ldes.ldio.pipeline; + +import be.vlaanderen.informatievlaanderen.ldes.ldes.EventStreamProperties; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.Test; +import org.springframework.util.ResourceUtils; + +import java.io.IOException; + +import static be.vlaanderen.informatievlaanderen.ldes.ldio.pipeline.ValidationPipelineSupplier.PIPELINE_NAME_TEMPLATE; +import static org.assertj.core.api.Assertions.assertThat; + +class ValidationPipelineSupplierTest { + private static final String LDES_SERVER_URL = "http://test-server/test-collection"; + private static final String SPARQL_HOST = "http://my-sparql-host.net"; + private static final String PIPELINE_UUID = "test-pipeline-uuid"; + private static final String PIPELINE_NAME = PIPELINE_NAME_TEMPLATE.formatted(PIPELINE_UUID); + private final ObjectMapper objectMapper = new ObjectMapper(); + + @Test + void test_createJson() throws IOException { + final ValidationPipelineSupplier factory = new ValidationPipelineSupplier(new EventStreamProperties(LDES_SERVER_URL, "http://purl.org/dc/terms/isVersionOf"), SPARQL_HOST, PIPELINE_NAME); + final JsonNode expectedJson = readJsonNode(); + + final String result = factory.getValidationPipelineAsJson(); + final JsonNode actualJson = objectMapper.readTree(result); + + assertThat(actualJson) + .isEqualTo(expectedJson); + + } + + private JsonNode readJsonNode() throws IOException { + final var jsonFile = ResourceUtils.getFile("classpath:ldio-pipeline.json"); + return objectMapper.readTree(jsonFile); + } +} \ No newline at end of file diff --git a/src/test/java/be/vlaanderen/informatievlaanderen/ldes/rdfrepo/RepositoryValidatorTest.java b/src/test/java/be/vlaanderen/informatievlaanderen/ldes/rdfrepo/RepositoryValidatorTest.java new file mode 100644 index 0000000..5f5bc5f --- /dev/null +++ b/src/test/java/be/vlaanderen/informatievlaanderen/ldes/rdfrepo/RepositoryValidatorTest.java @@ -0,0 +1,79 @@ +package be.vlaanderen.informatievlaanderen.ldes.rdfrepo; + +import be.vlaanderen.informatievlaanderen.ldes.http.RequestExecutor; +import be.vlaanderen.informatievlaanderen.ldes.ldio.config.LdioConfigProperties; +import org.apache.http.entity.InputStreamEntity; +import org.assertj.core.api.InstanceOfAssertFactories; +import org.eclipse.rdf4j.model.Literal; +import org.eclipse.rdf4j.model.Model; +import org.eclipse.rdf4j.rio.RDFFormat; +import org.eclipse.rdf4j.rio.Rio; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.io.*; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.Objects; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +class RepositoryValidatorTest { + private static final RDFFormat RDF_FORMAT = RDFFormat.TURTLE; + private static final String SHACL_CONFORMS_URI = "http://www.w3.org/ns/shacl#conforms"; + private static Model shaclShape; + private RequestExecutor requestExecutor; + private RepositoryValidator repoValidator; + + @BeforeAll + static void beforeAll() { + try { + shaclShape = Rio.parse(new FileInputStream("src/test/resources/test-shape.ttl"), RDF_FORMAT); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + @BeforeEach + void setUp() { + requestExecutor = mock(); + final LdioConfigProperties ldioConfigProperties = new LdioConfigProperties(); + ldioConfigProperties.setHost("http://localhost:8080"); + ldioConfigProperties.setSparqlHost("http://localhost:7200"); + repoValidator = new RepositoryValidator(requestExecutor, ldioConfigProperties); + } + + @Test + void given_ValidRepo_when_Validate_then_ReturnEmptyModel() throws FileNotFoundException, URISyntaxException { + final URI resource = Objects.requireNonNull(this.getClass().getClassLoader().getResource("validation-report/valid.ttl")).toURI(); + when(requestExecutor.execute(any())).thenReturn(new InputStreamEntity(new FileInputStream(new File(resource)))); + + final Model result = repoValidator.validate(shaclShape); + + assertThat(result) + .filteredOn(statement -> statement.getPredicate().toString().equals(SHACL_CONFORMS_URI)) + .hasSize(1) + .map(statement -> ((Literal) statement.getObject()).booleanValue()) + .first(InstanceOfAssertFactories.BOOLEAN) + .isTrue(); + } + + @Test + void given_InvalidRepo_when_Validate_then_ReturnNonEmptyModel() throws FileNotFoundException, URISyntaxException { + final URI resource = Objects.requireNonNull(this.getClass().getClassLoader().getResource("validation-report/invalid.ttl")).toURI(); + when(requestExecutor.execute(any())).thenReturn(new InputStreamEntity(new FileInputStream(new File(resource)))); + + final Model result = repoValidator.validate(shaclShape); + + assertThat(result) + .filteredOn(statement -> statement.getPredicate().toString().equals(SHACL_CONFORMS_URI)) + .hasSize(1) + .map(statement -> ((Literal) statement.getObject()).booleanValue()) + .first(InstanceOfAssertFactories.BOOLEAN) + .isFalse(); + } +} \ No newline at end of file diff --git a/src/test/java/be/vlaanderen/informatievlaanderen/ldes/shacl/ShaclValidatorTest.java b/src/test/java/be/vlaanderen/informatievlaanderen/ldes/shacl/ShaclValidatorTest.java new file mode 100644 index 0000000..e73fa44 --- /dev/null +++ b/src/test/java/be/vlaanderen/informatievlaanderen/ldes/shacl/ShaclValidatorTest.java @@ -0,0 +1,49 @@ +package be.vlaanderen.informatievlaanderen.ldes.shacl; + +import be.vlaanderen.informatievlaanderen.ldes.ldio.LdesClientStatusManager; +import be.vlaanderen.informatievlaanderen.ldes.ldio.LdioPipelineManager; +import be.vlaanderen.informatievlaanderen.ldes.rdfrepo.Rdf4jRepositoryManager; +import be.vlaanderen.informatievlaanderen.ldes.rdfrepo.RepositoryValidator; +import be.vlaanderen.informatievlaanderen.ldes.valueobjects.ValidationParameters; +import org.eclipse.rdf4j.model.impl.LinkedHashModel; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InOrder; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import static be.vlaanderen.informatievlaanderen.ldes.ldio.pipeline.ValidationPipelineSupplier.PIPELINE_NAME_TEMPLATE; +import static org.mockito.Mockito.inOrder; + +@ExtendWith(MockitoExtension.class) +class ShaclValidatorTest { + private static final String LDES_SERVER_URL = "http://ldes-server:8080/collection"; + private static final String PIPELINE_UUID = "test-pipeline-uuid"; + private static final String PIPELINE_NAME = PIPELINE_NAME_TEMPLATE.formatted(PIPELINE_UUID); + @Mock + private Rdf4jRepositoryManager repositoryManager; + @Mock + private LdioPipelineManager ldioPipelineManager; + @Mock + private LdesClientStatusManager ldesClientStatusManager; + @Mock + private RepositoryValidator repositoryValidator; + + @InjectMocks + private ShaclValidator shaclValidator; + + @Test + void test() { + shaclValidator.validate(new ValidationParameters(LDES_SERVER_URL, new LinkedHashModel(), PIPELINE_UUID)); + + final InOrder inOrder = inOrder(ldioPipelineManager, ldesClientStatusManager, repositoryManager, repositoryValidator); + inOrder.verify(repositoryManager).createRepository(); + inOrder.verify(ldioPipelineManager).initPipeline(LDES_SERVER_URL, PIPELINE_NAME); + inOrder.verify(ldesClientStatusManager).waitUntilReplicated(PIPELINE_NAME); + inOrder.verify(ldioPipelineManager).deletePipeline(PIPELINE_NAME); + inOrder.verify(repositoryValidator).validate(new LinkedHashModel()); + inOrder.verify(repositoryManager).deleteRepository(); + inOrder.verifyNoMoreInteractions(); + } +} \ No newline at end of file diff --git a/src/test/java/be/vlaanderen/informatievlaanderen/ldes/valueobjects/ValidationReportTest.java b/src/test/java/be/vlaanderen/informatievlaanderen/ldes/valueobjects/ValidationReportTest.java new file mode 100644 index 0000000..a03667b --- /dev/null +++ b/src/test/java/be/vlaanderen/informatievlaanderen/ldes/valueobjects/ValidationReportTest.java @@ -0,0 +1,56 @@ +package be.vlaanderen.informatievlaanderen.ldes.valueobjects; + +import be.vlaanderen.informatievlaanderen.ldes.valueobjects.severitylevels.SeverityLevels; +import org.eclipse.rdf4j.model.Model; +import org.eclipse.rdf4j.rio.RDFFormat; +import org.eclipse.rdf4j.rio.Rio; +import org.junit.jupiter.api.extension.ExtensionContext; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.ArgumentsProvider; +import org.junit.jupiter.params.provider.ArgumentsSource; + +import java.io.IOException; +import java.util.function.Consumer; +import java.util.stream.Stream; + +import static org.assertj.core.api.Assertions.assertThat; + +class ValidationReportTest { + + @ParameterizedTest(name = "ShaclValidationReportFile={0}") + @ArgumentsSource(ShaclValidationReportProvider.class) + void test_ValidationReport(String resourceName, Consumer throwingConsumer) throws IOException { + final Model shaclReport = Rio.parse(this.getClass().getClassLoader().getResourceAsStream(resourceName), RDFFormat.TURTLE); + final ValidationReport validationReport = new ValidationReport(shaclReport); + + assertThat(validationReport).satisfies(throwingConsumer); + } + + + static class ShaclValidationReportProvider implements ArgumentsProvider { + @Override + public Stream provideArguments(ExtensionContext extensionContext) { + return Stream.of( + Arguments.of( + "validation-report/invalid.ttl", + (Consumer) actualValidationReport -> { + assertThat(actualValidationReport.getHighestSeverityLevel()).isSameAs(SeverityLevels.ERROR); + assertThat(actualValidationReport.errorCount()).isEqualTo(2); + assertThat(actualValidationReport.warningCount()).isZero(); + assertThat(actualValidationReport.infoCount()).isZero(); + } + ), + Arguments.of( + "validation-report/valid.ttl", + (Consumer) acutalValidationReport -> { + assertThat(acutalValidationReport.getHighestSeverityLevel()).isSameAs(SeverityLevels.INFO); + assertThat(acutalValidationReport.errorCount()).isZero(); + assertThat(acutalValidationReport.warningCount()).isZero(); + assertThat(acutalValidationReport.infoCount()).isZero(); + } + ) + ); + } + } +} \ No newline at end of file diff --git a/src/test/resources/application-test.yaml b/src/test/resources/application-test.yaml new file mode 100644 index 0000000..505eae1 --- /dev/null +++ b/src/test/resources/application-test.yaml @@ -0,0 +1,3 @@ +ldio: + host: http://localhost:8888 + sparql-host: http://localhost:7200 \ No newline at end of file diff --git a/src/test/resources/event-stream.ttl b/src/test/resources/event-stream.ttl new file mode 100644 index 0000000..2eff50f --- /dev/null +++ b/src/test/resources/event-stream.ttl @@ -0,0 +1,53 @@ +@prefix by-location: . +@prefix by-page: . +@prefix by-time: . +@prefix dcat: . +@prefix ldes: . +@prefix prov: . +@prefix rdf: . +@prefix shacl: . +@prefix terms: . +@prefix tree: . +@prefix verkeersmetingen: . + + + rdf:type dcat:Dataset , ldes:EventStream; + terms:conformsTo , ; + terms:identifier "http://localhost:8080/verkeersmetingen"^^; + ldes:createVersions false; + ldes:eventSource [ rdf:type ldes:EventSource ]; + ldes:timestampPath prov:generatedAtTime; + ldes:versionOfPath terms:isVersionOf; + tree:shape [ rdf:type shacl:NodeShape ]; + tree:view verkeersmetingen:by-page , verkeersmetingen:by-time . + + + rdf:type terms:Standard . + +by-page:description rdf:type tree:ViewDescription; + ldes:retentionPolicy [ ]; + tree:fragmentationStrategy (); + tree:pageSize "250"^^ . + + +by-time:description rdf:type tree:ViewDescription; + ldes:retentionPolicy [ rdf:type ldes:DurationAgoPolicy; + tree:value "P5Y"^^ + ]; + tree:fragmentationStrategy ( [ rdf:type tree:HierarchicalTimeBasedFragmentation; + tree:fragmentationPath "http://www.w3.org/ns/prov#generatedAtTime"; + tree:maxGranularity "hour" + ] + ); + tree:pageSize "250"^^ . + +verkeersmetingen:by-time + rdf:type tree:Node; + tree:viewDescription by-time:description . + +verkeersmetingen:by-page + rdf:type tree:Node; + tree:viewDescription by-page:description . + + + rdf:type terms:Standard . diff --git a/src/test/resources/ldio-pipeline.json b/src/test/resources/ldio-pipeline.json new file mode 100644 index 0000000..4a8ba97 --- /dev/null +++ b/src/test/resources/ldio-pipeline.json @@ -0,0 +1,25 @@ +{ + "name": "validation-pipeline-test-pipeline-uuid", + "description": "Pipeline that will only replicate an LDES for validation purposes", + "input": { + "name": "Ldio:LdesClient", + "config": { + "urls": "http://test-server/test-collection", + "source-format": "application/n-quads", + "materialisation": { + "enabled": true, + "version-of-property": "http://purl.org/dc/terms/isVersionOf" + } + } + }, + "outputs": [ + { + "name": "Ldio:RepositorySink", + "config": { + "sparql-host": "http://my-sparql-host.net", + "repository-id": "validation", + "batch-size": 1 + } + } + ] +} \ No newline at end of file diff --git a/src/test/resources/test-shape.ttl b/src/test/resources/test-shape.ttl new file mode 100644 index 0000000..a3d4594 --- /dev/null +++ b/src/test/resources/test-shape.ttl @@ -0,0 +1,19 @@ +@prefix sh: . +@prefix xsd: . +@prefix rdf: . +@prefix verkeers: . +@prefix vsds: . +@prefix geo: . +@prefix time: . + +[] a sh:NodeShape ; + sh:targetClass verkeers:Verkeersmeting ; + sh:property [ + sh:path ; + sh:class ; + ] ; + sh:property [ + sh:path geo:asWKT ; + sh:minCount 10 ; + ] ; +. diff --git a/src/test/resources/validate-request.xml b/src/test/resources/validate-request.xml new file mode 100644 index 0000000..bf67b2f --- /dev/null +++ b/src/test/resources/validate-request.xml @@ -0,0 +1,36 @@ + + + + + + test-pipeline-uuid + + + http://ldes-server:8080/verkeersmetingen + + + . +@prefix xsd: . +@prefix rdf: . +@prefix verkeers: . +@prefix vsds: . +@prefix geo: . +@prefix time: . + +[] a sh:NodeShape ; + sh:targetClass verkeers:Verkeersmeting ; + sh:property [ + sh:path ; + sh:class ; + ] ; + sh:property [ + sh:path geo:asWKT ; + sh:minCount 10 ; + ] ; +. + ]]> + + + + \ No newline at end of file diff --git a/src/test/resources/validation-report/invalid.ttl b/src/test/resources/validation-report/invalid.ttl new file mode 100644 index 0000000..e8d5b4e --- /dev/null +++ b/src/test/resources/validation-report/invalid.ttl @@ -0,0 +1,34 @@ +@prefix sh: . +@prefix rsx: . +@prefix dash: . +@prefix rdf: . +@prefix rdfs: . +@prefix owl: . +@prefix xsd: . +@prefix rdf4j: . + +[] a sh:ValidationReport; + sh:conforms false; + rdf4j:truncated false; + sh:result [ a sh:ValidationResult; + sh:focusNode ; + rsx:shapesGraph rdf4j:nil; + sh:value []; + sh:resultPath ; + sh:sourceConstraintComponent sh:ClassConstraintComponent; + sh:resultSeverity sh:Violation; + sh:sourceShape [ a sh:PropertyShape; + sh:path ; + sh:class + ] + ], [ a sh:ValidationResult; + sh:focusNode ; + rsx:shapesGraph rdf4j:nil; + sh:resultPath ; + sh:sourceConstraintComponent sh:MinCountConstraintComponent; + sh:resultSeverity sh:Violation; + sh:sourceShape [ a sh:PropertyShape; + sh:path ; + sh:minCount 10 + ] + ] . diff --git a/src/test/resources/validation-report/valid.ttl b/src/test/resources/validation-report/valid.ttl new file mode 100644 index 0000000..5a446c6 --- /dev/null +++ b/src/test/resources/validation-report/valid.ttl @@ -0,0 +1,12 @@ +@prefix sh: . +@prefix rsx: . +@prefix dash: . +@prefix rdf: . +@prefix rdfs: . +@prefix owl: . +@prefix xsd: . +@prefix rdf4j: . + +[] a sh:ValidationReport; + sh:conforms true; + rdf4j:truncated false .