From 0ed83a621197de572374fe2de1739facc32bc55e Mon Sep 17 00:00:00 2001 From: Thomas Johansen Date: Wed, 14 Aug 2024 14:03:17 +0200 Subject: [PATCH] Tweaks etter flytting av kafka-key-generator --- .github/workflows/kafka-key-generator.yaml | 62 +++++--- apps/kafka-key-generator/build.gradle.kts | 140 +++++++++--------- .../nav/paw/kafkakeygenerator/AppStarter.kt | 10 +- .../kafkakeygenerator/ktor/helse_endepunkt.kt | 10 +- .../paw/kafkakeygenerator/ktor/init_ktor.kt | 6 +- .../kafkakeygenerator/ktor/ktor_oppsett.kt | 31 ++-- settings.gradle.kts | 33 ++++- 7 files changed, 173 insertions(+), 119 deletions(-) diff --git a/.github/workflows/kafka-key-generator.yaml b/.github/workflows/kafka-key-generator.yaml index de4d4e3a..68f5dad9 100644 --- a/.github/workflows/kafka-key-generator.yaml +++ b/.github/workflows/kafka-key-generator.yaml @@ -1,9 +1,20 @@ -name: Build, push, and deploy +name: PAW Kafka Key Generator -on: [push] +on: + push: + paths: + - 'apps/kafka-key-generator/**' + - 'lib/**' + - 'domain/**' + - '.github/workflows/kafka-key-generator.yaml' + - 'gradle/**' + - 'settings.gradle.kts' + - 'gradle.properties' + - 'gradlew' + - 'gradlew.bat' env: - IMAGE: europe-north1-docker.pkg.dev/${{ vars.NAIS_MANAGEMENT_PROJECT_ID }}/paw/${{ github.event.repository.name }} + IMAGE: europe-north1-docker.pkg.dev/${{ vars.NAIS_MANAGEMENT_PROJECT_ID }}/paw/paw-kafka-key-generator jobs: build: name: Build and push Docker container @@ -14,13 +25,17 @@ jobs: outputs: image: ${{ steps.docker-build-push.outputs.image }} steps: - - uses: actions/checkout@v3 - - uses: actions/setup-java@v3 + - name: Checkout + uses: actions/checkout@v3 + - name: Setup Java + uses: actions/setup-java@v3 with: java-version: 21 distribution: temurin cache: gradle - - name: Set version + - name: Specify module + run: echo "MODULE=:apps:kafka-key-generator" >> $GITHUB_ENV + - name: Specify version run: echo "VERSION=$(date +'%y.%m.%d').${{ github.run_number }}-${{ github.run_attempt }}" >> $GITHUB_ENV - name: Login GAR uses: nais/login@v0 @@ -28,30 +43,35 @@ jobs: project_id: ${{ vars.NAIS_MANAGEMENT_PROJECT_ID }} identity_provider: ${{ secrets.NAIS_WORKLOAD_IDENTITY_PROVIDER }} team: paw - - name: Build and push image with Gradle + - name: Build and push image id: docker-build-push working-directory: ./ run: | echo "image=${{ env.IMAGE }}:${{ env.VERSION }}" >> $GITHUB_OUTPUT - ./gradlew -Pversion=${{ env.VERSION }} -Pimage=${{ env.IMAGE }} build jib + echo -Pversion=${{ env.VERSION }} -Pimage=${{ env.IMAGE }} ${{ env.MODULE }}:build ${{ env.MODULE }}:test ${{ env.MODULE }}:jib + ./gradlew -Pversion=${{ env.VERSION }} -Pimage=${{ env.IMAGE }} ${{ env.MODULE }}:build ${{ env.MODULE }}:test ${{ env.MODULE }}:jib echo "DIGEST=$(cat app/build/jib-image.digest)" >> $GITHUB_ENV env: ORG_GRADLE_PROJECT_githubPassword: ${{ secrets.GITHUB_TOKEN }} - - name: Attest and sign + - name: Attest and sign image uses: nais/attest-sign@v1.3.4 with: image_ref: ${{ env.IMAGE }}@${{ env.DIGEST }} deploy-dev: - name: Deploy to NAIS dev + if: github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/heads/dev') + name: Deploy to dev-gcp + needs: + - build + runs-on: ubuntu-latest permissions: contents: read id-token: write - needs: build - runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - uses: nais/deploy/actions/deploy@v2 + - name: Checkout + uses: actions/checkout@v3 + - name: Deploy to GCP + uses: nais/deploy/actions/deploy@v2 env: CLUSTER: dev-gcp RESOURCE: nais/nais-dev.yaml @@ -59,15 +79,19 @@ jobs: deploy-prod: if: github.ref == 'refs/heads/main' - name: Deploy to NAIS prod + name: Deploy to prod-gcp + needs: + - build + - deploy-dev + runs-on: ubuntu-latest permissions: contents: read id-token: write - needs: [deploy-dev,build] - runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - uses: nais/deploy/actions/deploy@v2 + - name: Checkout + uses: actions/checkout@v3 + - name: Deploy to GCP + uses: nais/deploy/actions/deploy@v2 env: CLUSTER: prod-gcp RESOURCE: nais/nais-prod.yaml diff --git a/apps/kafka-key-generator/build.gradle.kts b/apps/kafka-key-generator/build.gradle.kts index 9665fa5a..139a29d6 100644 --- a/apps/kafka-key-generator/build.gradle.kts +++ b/apps/kafka-key-generator/build.gradle.kts @@ -1,87 +1,79 @@ import org.jetbrains.kotlin.gradle.tasks.KotlinCompile plugins { - kotlin("jvm") version "2.0.0" + kotlin("jvm") application - id("com.google.cloud.tools.jib") version "3.4.3" + id("com.google.cloud.tools.jib") } -val jvmVersion = 21 -val image: String? by project - -val exposedVersion = "0.52.0" -val logbackVersion = "1.5.2" -val logstashVersion = "7.4" -val navCommonModulesVersion = "3.2024.05.23_05.46-2b29fa343e8e" -val tokenSupportVersion = "5.0.1" -val ktorVersion = "2.3.12" +val image: String? by project +val baseImage: String by project +val jvmMajorVersion: String by project dependencies { - implementation("io.opentelemetry.instrumentation:opentelemetry-instrumentation-annotations:2.1.0") - // Konfigurasjon - implementation("com.sksamuel.hoplite:hoplite-core:2.8.0.RC3") - implementation("com.sksamuel.hoplite:hoplite-toml:2.8.0.RC3") + // PAW + implementation(project(":lib:hoplite-config")) + implementation(pawClients.pawPdlClient) // NAV - implementation("no.nav.security:token-validation-ktor-v2:$tokenSupportVersion") - implementation("no.nav.security:token-client-core:$tokenSupportVersion") - implementation("no.nav.common:token-client:$navCommonModulesVersion") - implementation("no.nav.common:log:$navCommonModulesVersion") - implementation("no.nav.security:token-validation-ktor-v2:$tokenSupportVersion") - implementation("no.nav.security:token-client-core:$tokenSupportVersion") - implementation("no.nav.paw:pdl-client:24.07.04.39-1") - - // Database - implementation("org.postgresql:postgresql:42.7.3") - implementation("org.flywaydb:flyway-core:9.21.2") - implementation("org.jetbrains.exposed:exposed-core:$exposedVersion") - implementation("org.jetbrains.exposed:exposed-crypt:$exposedVersion") - implementation("org.jetbrains.exposed:exposed-dao:$exposedVersion") - implementation("org.jetbrains.exposed:exposed-jdbc:$exposedVersion") - implementation("com.zaxxer:HikariCP:5.1.0") + implementation(navCommon.log) + implementation(navCommon.tokenClient) + implementation(navSecurity.tokenClient) + implementation(navSecurity.tokenValidationKtorV2) // Ktor - implementation("io.ktor:ktor-server-core:$ktorVersion") - implementation("io.ktor:ktor-server-core-jvm:$ktorVersion") - implementation("io.ktor:ktor-server-netty:$ktorVersion") - implementation("io.ktor:ktor-server-metrics-micrometer:$ktorVersion") - implementation("io.ktor:ktor-server-content-negotiation-jvm:$ktorVersion") - implementation("io.ktor:ktor-server-swagger:$ktorVersion") - implementation("io.ktor:ktor-server-status-pages:$ktorVersion") - implementation("io.ktor:ktor-serialization-jackson:$ktorVersion") - implementation("io.ktor:ktor-client-okhttp-jvm:$ktorVersion") - implementation("io.ktor:ktor-client-logging-jvm:$ktorVersion") - implementation("io.ktor:ktor-client-content-negotiation:$ktorVersion") - - //Otel - implementation("io.opentelemetry:opentelemetry-api:1.39.0") - implementation("io.opentelemetry.instrumentation:opentelemetry-ktor-2.0:2.4.0-alpha") - - // Micrometer - implementation("io.micrometer:micrometer-registry-prometheus:1.12.3") + implementation(ktor.serializationJackson) + + // Ktor Server + implementation(ktorServer.bundles.withNettyAndMicrometer) + implementation(ktorServer.cors) + implementation(ktorServer.swagger) + implementation(ktorServer.callId) + implementation(ktorServer.statusPages) + implementation(ktorServer.contentNegotiation) + + // Ktor Client + implementation(ktorClient.contentNegotiation) + implementation(ktorClient.core) + implementation(ktorClient.cio) + implementation(ktorClient.okhttp) + implementation(ktorClient.logging) + + // Micrometer & OTEL + implementation(micrometer.registryPrometheus) + implementation(otel.api) + implementation(otel.annotations) + implementation(otel.ktor) + + // Database + implementation(exposed.core) + implementation(exposed.crypt) + implementation(exposed.dao) + implementation(exposed.jdbc) + implementation(exposed.javaTime) + implementation(postgres.driver) + implementation(flyway.core) + implementation(flyway.postgres) + implementation(hikari.connectionPool) + + // Config + implementation(hoplite.hopliteToml) // Logging - implementation("ch.qos.logback:logback-classic:$logbackVersion") - implementation("net.logstash.logback:logstash-logback-encoder:$logstashVersion") + implementation(loggingLibs.logbackClassic) + implementation(loggingLibs.logstashLogbackEncoder) // Tester - testImplementation("org.jetbrains.kotlin:kotlin-test-junit5") - testImplementation("org.junit.jupiter:junit-jupiter-engine:5.9.2") - testRuntimeOnly("org.junit.platform:junit-platform-launcher") - testImplementation("io.kotest:kotest-runner-junit5-jvm:4.6.3") - testImplementation("io.ktor:ktor-client-mock:$ktorVersion") - testImplementation("org.testcontainers:postgresql:1.19.8") -} - -tasks.withType().configureEach { - compilerOptions { - freeCompilerArgs.add("-Xcontext-receivers") - } + testImplementation(testLibs.bundles.withUnitTesting) + testImplementation(testLibs.testContainers) + testImplementation(testLibs.postgresql) + testImplementation(ktorServer.testJvm) + testImplementation(ktorClient.mock) } java { toolchain { - languageVersion.set(JavaLanguageVersion.of(jvmVersion)) + languageVersion.set(JavaLanguageVersion.of(jvmMajorVersion)) } } @@ -89,12 +81,22 @@ application { mainClass.set("no.nav.paw.kafkakeygenerator.AppStarterKt") } -tasks.named("test") { - useJUnitPlatform() +tasks.withType().configureEach { + compilerOptions { + freeCompilerArgs.add("-Xcontext-receivers") + } +} + +tasks.withType(Jar::class) { + manifest { + attributes["Implementation-Version"] = project.version + attributes["Main-Class"] = application.mainClass.get() + attributes["Implementation-Title"] = rootProject.name + } } jib { - from.image = "ghcr.io/navikt/baseimages/temurin:$jvmVersion" + from.image = "$baseImage:$jvmMajorVersion" to.image = "${image ?: project.name}:${project.version}" container { environment = mapOf( @@ -107,6 +109,10 @@ jib { } } +tasks.named("test") { + useJUnitPlatform() +} + tasks.create("runTestApp", JavaExec::class) { classpath = sourceSets["test"].runtimeClasspath + sourceSets["main"].runtimeClasspath diff --git a/apps/kafka-key-generator/src/main/kotlin/no/nav/paw/kafkakeygenerator/AppStarter.kt b/apps/kafka-key-generator/src/main/kotlin/no/nav/paw/kafkakeygenerator/AppStarter.kt index 6820f9a5..5862ede5 100644 --- a/apps/kafka-key-generator/src/main/kotlin/no/nav/paw/kafkakeygenerator/AppStarter.kt +++ b/apps/kafka-key-generator/src/main/kotlin/no/nav/paw/kafkakeygenerator/AppStarter.kt @@ -1,15 +1,15 @@ package no.nav.paw.kafkakeygenerator -import io.micrometer.prometheus.PrometheusConfig -import io.micrometer.prometheus.PrometheusMeterRegistry +import io.micrometer.prometheusmetrics.PrometheusConfig +import io.micrometer.prometheusmetrics.PrometheusMeterRegistry import no.nav.paw.kafkakeygenerator.config.Autentiseringskonfigurasjon import no.nav.paw.kafkakeygenerator.config.DatabaseKonfigurasjon import no.nav.paw.kafkakeygenerator.config.dataSource import no.nav.paw.kafkakeygenerator.config.lastKonfigurasjon import no.nav.paw.kafkakeygenerator.database.flywayMigrate +import no.nav.paw.kafkakeygenerator.ktor.initKtorServer import no.nav.paw.kafkakeygenerator.pdl.PdlIdentitesTjeneste import no.nav.paw.kafkakeygenerator.pdl.opprettPdlKlient -import no.nav.paw.kafkakeygenerator.ktor.initKtorServer import no.nav.paw.pdl.PdlClient import org.jetbrains.exposed.sql.Database import javax.sql.DataSource @@ -33,6 +33,7 @@ fun main() { pdlKlient ) } + fun startApplikasjon( autentiseringKonfig: Autentiseringskonfigurasjon, dataSource: DataSource, @@ -47,5 +48,6 @@ fun startApplikasjon( Applikasjon( KafkaKeys(database), PdlIdentitesTjeneste(pdlKlient) - )).start(wait = true) + ) + ).start(wait = true) } diff --git a/apps/kafka-key-generator/src/main/kotlin/no/nav/paw/kafkakeygenerator/ktor/helse_endepunkt.kt b/apps/kafka-key-generator/src/main/kotlin/no/nav/paw/kafkakeygenerator/ktor/helse_endepunkt.kt index a31de368..9eb46885 100644 --- a/apps/kafka-key-generator/src/main/kotlin/no/nav/paw/kafkakeygenerator/ktor/helse_endepunkt.kt +++ b/apps/kafka-key-generator/src/main/kotlin/no/nav/paw/kafkakeygenerator/ktor/helse_endepunkt.kt @@ -1,9 +1,11 @@ package no.nav.paw.kafkakeygenerator.ktor -import io.ktor.server.application.* -import io.ktor.server.response.* -import io.ktor.server.routing.* -import io.micrometer.prometheus.PrometheusMeterRegistry +import io.ktor.server.application.call +import io.ktor.server.response.respond +import io.ktor.server.response.respondText +import io.ktor.server.routing.Routing +import io.ktor.server.routing.get +import io.micrometer.prometheusmetrics.PrometheusMeterRegistry fun Routing.konfigurereHelse(prometheusMeterRegistry: PrometheusMeterRegistry) { get("/internal/isAlive") { diff --git a/apps/kafka-key-generator/src/main/kotlin/no/nav/paw/kafkakeygenerator/ktor/init_ktor.kt b/apps/kafka-key-generator/src/main/kotlin/no/nav/paw/kafkakeygenerator/ktor/init_ktor.kt index 93889e39..87da2246 100644 --- a/apps/kafka-key-generator/src/main/kotlin/no/nav/paw/kafkakeygenerator/ktor/init_ktor.kt +++ b/apps/kafka-key-generator/src/main/kotlin/no/nav/paw/kafkakeygenerator/ktor/init_ktor.kt @@ -1,8 +1,8 @@ package no.nav.paw.kafkakeygenerator.ktor -import io.ktor.server.engine.* -import io.ktor.server.netty.* -import io.micrometer.prometheus.PrometheusMeterRegistry +import io.ktor.server.engine.embeddedServer +import io.ktor.server.netty.Netty +import io.micrometer.prometheusmetrics.PrometheusMeterRegistry import no.nav.paw.kafkakeygenerator.Applikasjon import no.nav.paw.kafkakeygenerator.config.Autentiseringskonfigurasjon diff --git a/apps/kafka-key-generator/src/main/kotlin/no/nav/paw/kafkakeygenerator/ktor/ktor_oppsett.kt b/apps/kafka-key-generator/src/main/kotlin/no/nav/paw/kafkakeygenerator/ktor/ktor_oppsett.kt index 62a29408..0a136dc0 100644 --- a/apps/kafka-key-generator/src/main/kotlin/no/nav/paw/kafkakeygenerator/ktor/ktor_oppsett.kt +++ b/apps/kafka-key-generator/src/main/kotlin/no/nav/paw/kafkakeygenerator/ktor/ktor_oppsett.kt @@ -1,23 +1,25 @@ package no.nav.paw.kafkakeygenerator.ktor import com.fasterxml.jackson.databind.DatabindException -import io.ktor.http.* -import io.ktor.serialization.jackson.* -import io.ktor.server.application.* -import io.ktor.server.auth.* -import io.ktor.server.metrics.micrometer.* -import io.ktor.server.plugins.callloging.* -import io.ktor.server.plugins.contentnegotiation.* -import io.ktor.server.plugins.statuspages.* -import io.ktor.server.plugins.swagger.* -import io.ktor.server.request.* -import io.ktor.server.response.* -import io.ktor.server.routing.* +import io.ktor.http.ContentType +import io.ktor.http.HttpStatusCode +import io.ktor.serialization.jackson.jackson +import io.ktor.server.application.Application +import io.ktor.server.application.install +import io.ktor.server.auth.authentication +import io.ktor.server.metrics.micrometer.MicrometerMetrics +import io.ktor.server.plugins.callloging.CallLogging +import io.ktor.server.plugins.contentnegotiation.ContentNegotiation +import io.ktor.server.plugins.statuspages.StatusPages +import io.ktor.server.plugins.swagger.swaggerUI +import io.ktor.server.request.path +import io.ktor.server.response.respondText +import io.ktor.server.routing.routing import io.micrometer.core.instrument.binder.jvm.JvmGcMetrics import io.micrometer.core.instrument.binder.jvm.JvmMemoryMetrics import io.micrometer.core.instrument.binder.system.ProcessorMetrics import io.micrometer.core.instrument.distribution.DistributionStatisticConfig -import io.micrometer.prometheus.PrometheusMeterRegistry +import io.micrometer.prometheusmetrics.PrometheusMeterRegistry import no.nav.paw.kafkakeygenerator.Applikasjon import no.nav.paw.kafkakeygenerator.api.v2.konfigurerApiV2 import no.nav.paw.kafkakeygenerator.config.Autentiseringskonfigurasjon @@ -111,6 +113,7 @@ fun Application.statusPages() { HttpStatusCode.BadRequest ) } + else -> { feilLogger.error( "Kall {}, feilet, grunnet: {}", @@ -131,6 +134,6 @@ fun Application.statusPages() { fun Application.configureLogging() { install(CallLogging) { disableDefaultColors() - filter { !it.request.path().startsWith("/internal") && it.response.status() != HttpStatusCode.OK} + filter { !it.request.path().startsWith("/internal") && it.response.status() != HttpStatusCode.OK } } } \ No newline at end of file diff --git a/settings.gradle.kts b/settings.gradle.kts index 19019b63..79946932 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -13,18 +13,19 @@ include( "lib:kafka", "lib:kafka-streams", "lib:kafka-key-generator-client", - "apps:api-start-stopp-perioder", - "apps:hendelseprosessor", - "apps:utgang-formidlingsgruppe", - "apps:hendelselogg-backup", - "apps:utgang-pdl", "domain:rapportering-interne-hendelser", "domain:rapporteringsansvar-schema", "domain:rapporteringsmelding-schema", "domain:main-avro-schema", "domain:interne-hendelser", "domain:arbeidssoekerregisteret-kotlin", - "domain:arbeidssoeker-regler" + "domain:arbeidssoeker-regler", + "apps:api-start-stopp-perioder", + "apps:hendelseprosessor", + "apps:utgang-formidlingsgruppe", + "apps:hendelselogg-backup", + "apps:utgang-pdl", + "apps:kafka-key-generator" ) dependencyResolutionManagement { @@ -49,6 +50,13 @@ dependencyResolutionManagement { password = githubPassword } } + maven { + setUrl("https://maven.pkg.github.com/navikt/paw-kotlin-clients") + credentials { + username = "x-access-token" + password = githubPassword + } + } } versionCatalogs { // PAW greier @@ -85,10 +93,12 @@ dependencyResolutionManagement { val micrometerVersion = "1.13.1" val otelTargetSdkVersion = "1.39.0" val otelInstrumentationVersion = "2.4.0" + val otelInstrumentationKtorVersion = "2.4.0-alpha" val coroutinesVersion = "1.8.1" val rapporteringsSchemaVersion = "24.05.15.2-1" val postgresDriverVersion = "42.7.3" val flywayVersion = "10.15.0" + val hikariVersion = "5.1.0" fun VersionCatalogBuilder.ktorLib(alias: String, artifactId: String) = library(alias, "io.ktor", artifactId).version("2.3.12") @@ -97,6 +107,7 @@ dependencyResolutionManagement { aliases.forEach { (artifactId, alias) -> ktorLib(alias, artifactId) } infix fun String.alias(alias: String) = this to alias + create("kotlinx") { library("coroutinesCore", "org.jetbrains.kotlinx", "kotlinx-coroutines-core").version(coroutinesVersion) } @@ -113,7 +124,10 @@ dependencyResolutionManagement { ktorLibs( "ktor-client-content-negotiation" alias "contentNegotiation", "ktor-client-core" alias "core", - "ktor-client-cio" alias "cio" + "ktor-client-cio" alias "cio", + "ktor-client-mock" alias "mock", + "ktor-client-logging" alias "logging", + "ktor-client-okhttp" alias "okhttp" ) } create("ktorServer") { @@ -151,7 +165,7 @@ dependencyResolutionManagement { create("otel") { library("api", "io.opentelemetry", "opentelemetry-api").version(otelTargetSdkVersion) library("ktor", "io.opentelemetry.instrumentation", "opentelemetry-ktor-2.0").version( - otelInstrumentationVersion + otelInstrumentationKtorVersion ) library( "annotations", @@ -241,6 +255,9 @@ dependencyResolutionManagement { library("core", "org.flywaydb", "flyway-core").version(flywayVersion) library("postgres", "org.flywaydb", "flyway-database-postgresql").version(flywayVersion) } + create("hikari") { + library("connectionPool", "com.zaxxer", "HikariCP").version(hikariVersion) + } create("poao") { library("tilgangClient", "no.nav.poao-tilgang", "client").version("2024.04.29_13.59-a0ddddd36ac9") }