From 7760c165feb16517546811709120ed33f1a6c426 Mon Sep 17 00:00:00 2001 From: Christophe Loiseau Date: Wed, 8 May 2024 10:25:20 +0200 Subject: [PATCH] Merge the docker build --- .env.dev | 1 + .../actions/build-connector-image/action.yml | 1 + .github/workflows/ci.yml | 44 +++++++++++++ connector/Dockerfile | 65 +++++++------------ .../src/main/resources/logging.properties | 8 --- connector/.env => launchers/.env.broker | 2 +- launchers/.env.extensions | 2 +- launchers/Dockerfile | 8 ++- .../broker-server-ce}/build.gradle.kts | 11 +--- .../broker-server-dev/build.gradle.kts | 38 +++++++++++ launchers/logging.properties | 1 + settings.gradle.kts | 2 + 12 files changed, 120 insertions(+), 63 deletions(-) delete mode 100644 connector/src/main/resources/logging.properties rename connector/.env => launchers/.env.broker (98%) rename {connector => launchers/connectors/broker-server-ce}/build.gradle.kts (75%) create mode 100644 launchers/connectors/broker-server-dev/build.gradle.kts diff --git a/.env.dev b/.env.dev index 498abf37b..40bce4255 100644 --- a/.env.dev +++ b/.env.dev @@ -3,3 +3,4 @@ EDC_IMAGE=ghcr.io/sovity/edc-dev:latest TEST_BACKEND_IMAGE=ghcr.io/sovity/test-backend:latest EDC_UI_IMAGE=ghcr.io/sovity/edc-ui:latest EDC_UI_ACTIVE_PROFILE=sovity-open-source +BROKER_IMAGE=broker-server-dev:test diff --git a/.github/actions/build-connector-image/action.yml b/.github/actions/build-connector-image/action.yml index 1dfbefa10..eb4ca964c 100644 --- a/.github/actions/build-connector-image/action.yml +++ b/.github/actions/build-connector-image/action.yml @@ -71,5 +71,6 @@ runs: labels: ${{ steps.meta.outputs.labels }} build-args: | CONNECTOR_NAME=${{ inputs.connector-name }} + CONNECTOR_TYPE=${{ inputs.connector-type }} "EDC_LAST_COMMIT_INFO_ARG=${{ env.LAST_COMMIT_INFO }}" EDC_BUILD_DATE_ARG=${{ env.BUILD_DATE }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 68abaac9d..c412c7786 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,6 +22,22 @@ jobs: permissions: contents: read packages: write + + services: + postgres: + image: postgres:15 + env: + POSTGRES_USER: edc + POSTGRES_PASSWORD: edc + POSTGRES_DB: edc + ports: + - 5432:5432 + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + steps: - uses: FranzDiebold/github-env-vars-action@v2 - uses: actions/checkout@v3 @@ -65,6 +81,7 @@ jobs: image-base-name: ${{ env.IMAGE_BASE_NAME }} image-name: "edc-dev" connector-name: "sovity-dev" + connector-type: "extensions" title: "sovity Dev EDC Connector" description: "Extended EDC Connector built by sovity. This dev version contains no dataspace auth and can be used to quickly start a locally running EDC + EDC UI." - name: "Docker Image: edc-ce" @@ -76,6 +93,7 @@ jobs: image-base-name: ${{ env.IMAGE_BASE_NAME }} image-name: "edc-ce" connector-name: "sovity-ce" + connector-type: "extensions" title: "sovity Community Edition EDC Connector" description: "EDC Connector built by sovity. Contains sovity's Community Edition EDC extensions and requires dataspace credentials to join an existing dataspace." - name: "Docker Image: edc-ce-mds" @@ -87,6 +105,7 @@ jobs: image-base-name: ${{ env.IMAGE_BASE_NAME }} image-name: "edc-ce-mds" connector-name: "mds-ce" + connector-type: "extensions" title: "MDS Community Edition EDC Connector" description: "EDC Connector built by sovity and configured for compatibility with the Mobility Data Space (MDS). This EDC requires dataspace credentials, and additional MDS Services such as a Clearing House." - name: "Docker Image: test-backend" @@ -98,8 +117,33 @@ jobs: image-base-name: ${{ env.IMAGE_BASE_NAME }} image-name: "test-backend" connector-name: "test-backend" + connector-type: "extensions" title: "Test Data Source / Data Sink" description: "Provides a minimal data source / data sink for E2E tests." + - name: "Docker Image: broker-server-dev" + uses: ./.github/actions/build-connector-image + with: + registry-url: ${{ env.REGISTRY_URL }} + registry-user: ${{ env.REGISTRY_USER }} + registry-password: ${{ secrets.GITHUB_TOKEN }} + image-base-name: ${{ env.IMAGE_BASE_NAME }} + image-name: "broker-server-dev" + connector-name: "broker-server-dev" + connector-type: "broker" + title: "Broker Server (Dev)" + description: "EDC IDS Broker Server. This dev version contains no persistence or auth and can be used to quickly start a locally running Broker Server + Broker UI." + - name: "Docker Image: broker-server-ce" + uses: ./.github/actions/build-connector-image + with: + registry-url: ${{ env.REGISTRY_URL }} + registry-user: ${{ env.REGISTRY_USER }} + registry-password: ${{ secrets.GITHUB_TOKEN }} + image-base-name: ${{ env.IMAGE_BASE_NAME }} + image-name: "broker-server-ce" + connector-name: "broker-server-ce" + connector-type: "broker" + title: "Broker Server (Community Edition)" + description: "EDC IDS Broker Server. Contains DB extensions and requires dataspace credentials to join an existing dataspace." ts-api-client-library: name: TS API Client Library runs-on: ubuntu-latest diff --git a/connector/Dockerfile b/connector/Dockerfile index 71205d0d6..047da1950 100644 --- a/connector/Dockerfile +++ b/connector/Dockerfile @@ -1,54 +1,33 @@ -# TODO https://github.com/sovity/edc-broker-server-extension/issues/425 -USER fixme_and_stop_using_root - -FROM gradle:7-jdk17-alpine AS build - -ARG USERNAME -ARG TOKEN -ARG BUILD_ARGS -ARG TEST_POSTGRES_JDBC_URL -ARG TEST_POSTGRES_JDBC_USER -ARG TEST_POSTGRES_JDBC_PASSWORD - -ENV USERNAME=$USERNAME -ENV TOKEN=$TOKEN - -ENV SKIP_TESTCONTAINERS=true -ENV TEST_POSTGRES_JDBC_URL=$TEST_POSTGRES_JDBC_URL -ENV TEST_POSTGRES_JDBC_USER=$TEST_POSTGRES_JDBC_USER -ENV TEST_POSTGRES_JDBC_PASSWORD=$TEST_POSTGRES_JDBC_PASSWORD - -COPY --chown=gradle:gradle . /home/gradle/project/ -WORKDIR /home/gradle/project/ -RUN --mount=type=cache,target=/home/gradle/.gradle/caches gradle build --no-daemon $BUILD_ARGS - FROM eclipse-temurin:17-jre-alpine -# TODO https://github.com/sovity/edc-broker-server-extension/issues/425 -USER fixme_and_stop_using_root - -# Optional JVM arguments, such as memory settings -ARG JVM_ARGS="" - -# Install curl for healthcheck and create an empty properties file as migitation for a core EDC issue -RUN apk add --no-cache curl bash && touch /emtpy-properties-file.properties +# Install curl for healthcheck, bash for entrypoint +RUN apk add --no-cache curl bash SHELL ["/bin/bash", "-c"] -WORKDIR /app +# Use a non-root user +RUN adduser -D -H -s /sbin/nologin edc +USER edc:edc -COPY --from=build /home/gradle/project/connector/build/libs/app.jar /app -COPY ./connector/src/main/resources/logging.properties /app +# Which app.jar to include +ARG CONNECTOR_NAME="sovity-ce" +ARG CONNECTOR_TYPE="extensions" -# health status is determined by the availability of the /health endpoint -HEALTHCHECK --interval=5s --timeout=5s --retries=10 CMD curl -H "x-api-key: $EDC_API_AUTH_KEY" --fail http://localhost:11001/backend/api/check/health +# For last-commit-info extension +ARG EDC_LAST_COMMIT_INFO_ARG="The docker container was built outside of github actions and you didn't provide the build arg EDC_LAST_COMMIT_INFO_ARG, so there's no last commit info." +ARG EDC_BUILD_DATE_ARG="The docker container was built outside of github actions and you didn't provide the build arg EDC_BUILD_DATE_ARG, so there's no build date." -# Use "exec" for graceful termination (SIGINT) to reach JVM. -# ARG can not be used in ENTRYPOINT so storing values in ENV variables -ENV JVM_ARGS=$JVM_ARGS +WORKDIR /app +COPY ./launchers/connectors/$CONNECTOR_NAME/build/libs/app.jar /app +COPY ./launchers/logging.properties /app +COPY ./launchers/logging.dev.properties /app +COPY ./launchers/.env.broker /app/.env -# Read ENV Vars from .env with substitution -COPY ./connector/.env /app/.env +RUN touch /app/empty-properties-file.properties # Replaces ENV Var statements so they don't overwrite existing ENV Vars RUN sed -ri 's/^\s*(\S+)=(.*)$/\1=${\1:-"\2"}/' .env -ENTRYPOINT set -a && source /app/.env && set +a && exec java -Djava.util.logging.config.file=/app/logging.properties $JVM_ARGS -jar app.jar +ENTRYPOINT ["/app/entrypoint.sh"] +CMD ["start"] + +# health status is determined by the availability of the /health endpoint +HEALTHCHECK --interval=5s --timeout=5s --retries=10 CMD curl -H "x-api-key: $EDC_API_AUTH_KEY" --fail http://localhost:11001/backend/api/check/health diff --git a/connector/src/main/resources/logging.properties b/connector/src/main/resources/logging.properties deleted file mode 100644 index 17dfd8a75..000000000 --- a/connector/src/main/resources/logging.properties +++ /dev/null @@ -1,8 +0,0 @@ -handlers = java.util.logging.ConsoleHandler -.level = INFO -java.util.logging.ConsoleHandler.level = ALL -java.util.logging.ConsoleHandler.formatter = java.util.logging.SimpleFormatter -java.util.logging.SimpleFormatter.format = %1$tY-%1$tm-%1$td %1$tH:%1$tM:%1$tS %5$s %6$s%n -org.eclipse.dataspaceconnector.level = FINE -org.eclipse.dataspaceconnector.handler = java.util.logging.ConsoleHandler -org.eclipse.edc.api.observability.ObservabilityApiController.level = ERROR diff --git a/connector/.env b/launchers/.env.broker similarity index 98% rename from connector/.env rename to launchers/.env.broker index ca37ee84c..cc006d7cd 100644 --- a/connector/.env +++ b/launchers/.env.broker @@ -103,4 +103,4 @@ EDC_AGENT_IDENTITY_KEY=referringConnector # This file could contain an entry replacing the EDC_KEYSTORE ENV var, # but for some reason it is required, and EDC won't start up if it isn't configured. # It will be created in the Dockerfile -EDC_VAULT=/emtpy-properties-file.properties +EDC_VAULT=/app/empty-properties-file.properties diff --git a/launchers/.env.extensions b/launchers/.env.extensions index 19a9d73b5..d58c1267a 100644 --- a/launchers/.env.extensions +++ b/launchers/.env.extensions @@ -94,4 +94,4 @@ EDC_AGENT_IDENTITY_KEY=referringConnector # This file could contain an entry replacing the EDC_KEYSTORE ENV var # but for some reason it is required, and EDC won't start up if it isn't configured # it is created in the Dockerfile -EDC_VAULT=/app/emtpy-properties-file.properties +EDC_VAULT=/app/empty-properties-file.properties diff --git a/launchers/Dockerfile b/launchers/Dockerfile index 060f4deb6..274e33b34 100644 --- a/launchers/Dockerfile +++ b/launchers/Dockerfile @@ -10,6 +10,7 @@ USER edc:edc # Which app.jar to include ARG CONNECTOR_NAME="sovity-ce" +ARG CONNECTOR_TYPE="extensions" # For last-commit-info extension ARG EDC_LAST_COMMIT_INFO_ARG="The docker container was built outside of github actions and you didn't provide the build arg EDC_LAST_COMMIT_INFO_ARG, so there's no last commit info." @@ -19,8 +20,9 @@ WORKDIR /app COPY ./launchers/connectors/$CONNECTOR_NAME/build/libs/app.jar /app COPY ./launchers/logging.properties /app COPY ./launchers/logging.dev.properties /app -COPY ./launchers/.env.extensions /app/.env -RUN touch /app/emtpy-properties-file.properties +COPY ./launchers/.env.$CONNECTOR_TYPE /app/.env + +RUN touch /app/empty-properties-file.properties # Replaces var statements so when they are sourced as bash they don't overwrite existing env vars RUN sed -ri 's/^\s*(\S+)=(.*)$/\1=${\1:-"\2"}/' .env @@ -35,3 +37,5 @@ CMD ["start"] # health status is determined by the availability of the /health endpoint HEALTHCHECK --interval=5s --timeout=5s --retries=10 CMD curl --fail http://localhost:11001/api/check/health +# TODO: do I need the API key? +#HEALTHCHECK --interval=5s --timeout=5s --retries=10 CMD curl -H "x-api-key: $EDC_API_AUTH_KEY" --fail http://localhost:11001/backend/api/check/health diff --git a/connector/build.gradle.kts b/launchers/connectors/broker-server-ce/build.gradle.kts similarity index 75% rename from connector/build.gradle.kts rename to launchers/connectors/broker-server-ce/build.gradle.kts index 81e0dd58e..f1555fd09 100644 --- a/connector/build.gradle.kts +++ b/launchers/connectors/broker-server-ce/build.gradle.kts @@ -1,7 +1,7 @@ plugins { `java-library` id("application") - id("com.github.johnrengelman.shadow") version "7.1.2" + alias(libs.plugins.shadow) } val edcVersion: String by project @@ -24,13 +24,8 @@ dependencies { // Broker Server + PostgreSQL + Flyway implementation(project(":extensions:broker-server")) - // Optional: Connector-To-Connector IAM - if (project.hasProperty("oauth2")) { - implementation("${edcGroup}:vault-filesystem:${edcVersion}") - implementation("${edcGroup}:oauth2-core:${edcVersion}") - } else { - implementation("${edcGroup}:iam-mock:${edcVersion}") - } + implementation("${edcGroup}:vault-filesystem:${edcVersion}") + implementation("${edcGroup}:oauth2-core:${edcVersion}") } application { diff --git a/launchers/connectors/broker-server-dev/build.gradle.kts b/launchers/connectors/broker-server-dev/build.gradle.kts new file mode 100644 index 000000000..cca40e607 --- /dev/null +++ b/launchers/connectors/broker-server-dev/build.gradle.kts @@ -0,0 +1,38 @@ +plugins { + `java-library` + id("application") + alias(libs.plugins.shadow) +} + +val edcVersion: String by project +val edcGroup: String by project + +dependencies { + // Control-Plane + implementation("${edcGroup}:control-plane-core:${edcVersion}") + implementation("${edcGroup}:data-plane-selector-core:${edcVersion}") + implementation("${edcGroup}:api-observability:${edcVersion}") + implementation("${edcGroup}:configuration-filesystem:${edcVersion}") + implementation("${edcGroup}:control-plane-aggregate-services:${edcVersion}") + implementation("${edcGroup}:http:${edcVersion}") + implementation("${edcGroup}:dsp:${edcVersion}") + implementation("${edcGroup}:json-ld:${edcVersion}") + + // JDK Logger + implementation("${edcGroup}:monitor-jdk-logger:${edcVersion}") + + // Broker Server + PostgreSQL + Flyway + implementation(project(":extensions:broker-server")) + + // Connector-To-Connector IAM + implementation("${edcGroup}:iam-mock:${edcVersion}") +} + +application { + mainClass.set("org.eclipse.edc.boot.system.runtime.BaseRuntime") +} + +tasks.withType { + mergeServiceFiles() + archiveFileName.set("app.jar") +} diff --git a/launchers/logging.properties b/launchers/logging.properties index b4d12f28f..17dfd8a75 100644 --- a/launchers/logging.properties +++ b/launchers/logging.properties @@ -5,3 +5,4 @@ java.util.logging.ConsoleHandler.formatter = java.util.logging.SimpleFormatter java.util.logging.SimpleFormatter.format = %1$tY-%1$tm-%1$td %1$tH:%1$tM:%1$tS %5$s %6$s%n org.eclipse.dataspaceconnector.level = FINE org.eclipse.dataspaceconnector.handler = java.util.logging.ConsoleHandler +org.eclipse.edc.api.observability.ObservabilityApiController.level = ERROR diff --git a/settings.gradle.kts b/settings.gradle.kts index ee6d678c8..04e381a05 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -26,6 +26,8 @@ include(":launchers:common:auth-mock") include(":launchers:common:base") include(":launchers:common:base-mds") include(":launchers:common:observability") +include(":launchers:connectors:broker-server-ce") +include(":launchers:connectors:broker-server-dev") include(":launchers:connectors:mds-ce") include(":launchers:connectors:sovity-ce") include(":launchers:connectors:sovity-dev")