-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Lagt til store og punctuator i opplysninger aggregering
- Loading branch information
1 parent
d37cdf6
commit 3895254
Showing
14 changed files
with
291 additions
and
27 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
30 changes: 29 additions & 1 deletion
30
...ysninger-aggregering/src/main/kotlin/no/nav/paw/arbeidssoekerregisteret/config/Metrics.kt
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,11 +1,39 @@ | ||
| package no.nav.paw.arbeidssoekerregisteret.config | ||
|
|
||
| import io.micrometer.core.instrument.MeterRegistry | ||
| import io.micrometer.core.instrument.Tag | ||
| import io.micrometer.core.instrument.Tags | ||
| import java.time.Instant | ||
| import java.time.ZoneId | ||
| import java.util.concurrent.atomic.AtomicLong | ||
|
|
||
| private const val METRIC_PREFIX = "paw_arbeidssoeker_opplysninger_aggregering" | ||
|
|
||
| fun MeterRegistry.tellAntallMottatteOpplysninger() { | ||
| fun MeterRegistry.tellMottatteOpplysninger() { | ||
| counter( | ||
| "${METRIC_PREFIX}_antall_mottatte_opplysninger_total", | ||
| ).increment() | ||
| } | ||
|
|
||
| fun MeterRegistry.antallLagredeOpplysningerTotal(antallReference: AtomicLong) { | ||
| gauge( | ||
| "${METRIC_PREFIX}_antall_lagrede_opplysninger_total", | ||
| Tags.empty(), | ||
| antallReference | ||
| ) { | ||
| antallReference.get().toDouble() | ||
| } | ||
| } | ||
|
|
||
| fun MeterRegistry.antallLagredeOpplysningerSumPerPeriode(timestamp: Instant, antallReference: AtomicLong) { | ||
| val zonedDateTime = timestamp.atZone(ZoneId.systemDefault()) | ||
| gauge( | ||
| "${METRIC_PREFIX}_antall_lagrede_opplysninger_sum_per_periode", | ||
| Tags.of( | ||
| Tag.of("minute", "${zonedDateTime.minute}") | ||
| ), | ||
| antallReference | ||
| ) { | ||
| antallReference.get().toDouble() | ||
| } | ||
| } |
4 changes: 2 additions & 2 deletions
4
...regering/src/main/kotlin/no/nav/paw/arbeidssoekerregisteret/context/ApplicationContext.kt
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,6 +1,6 @@ | ||
| package no.nav.paw.arbeidssoekerregisteret.context | ||
|
|
||
| import no.nav.paw.arbeidssoekerregisteret.config.AppConfig | ||
| import no.nav.paw.arbeidssoekerregisteret.properties.ApplicationProperties | ||
| import org.slf4j.Logger | ||
|
|
||
| data class ApplicationContext(val logger: Logger, val properties: AppConfig) | ||
| data class ApplicationContext(val logger: Logger, val properties: ApplicationProperties) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
4 changes: 2 additions & 2 deletions
4
...dssoekerregisteret/config/ServerConfig.kt → ...registeret/properties/ServerProperties.kt
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
104 changes: 100 additions & 4 deletions
104
...ering/src/main/kotlin/no/nav/paw/arbeidssoekerregisteret/topology/OpplysningerTopology.kt
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,22 +1,118 @@ | ||
| package no.nav.paw.arbeidssoekerregisteret.topology | ||
|
|
||
| import io.micrometer.core.instrument.MeterRegistry | ||
| import no.nav.paw.arbeidssoekerregisteret.config.tellAntallMottatteOpplysninger | ||
| import no.nav.paw.arbeidssoekerregisteret.config.antallLagredeOpplysningerSumPerPeriode | ||
| import no.nav.paw.arbeidssoekerregisteret.config.antallLagredeOpplysningerTotal | ||
| import no.nav.paw.arbeidssoekerregisteret.config.buildOpplysningerOmArbeidssoekerAvroSerde | ||
| import no.nav.paw.arbeidssoekerregisteret.config.tellMottatteOpplysninger | ||
| import no.nav.paw.arbeidssoekerregisteret.context.ApplicationContext | ||
| import no.nav.paw.arbeidssokerregisteret.api.v4.OpplysningerOmArbeidssoeker | ||
| import no.nav.paw.config.kafka.streams.Punctuation | ||
| import no.nav.paw.config.kafka.streams.genericProcess | ||
| import org.apache.kafka.common.serialization.Serdes | ||
| import org.apache.kafka.streams.StreamsBuilder | ||
| import org.apache.kafka.streams.Topology | ||
| import org.apache.kafka.streams.kstream.Consumed | ||
| import org.apache.kafka.streams.processor.PunctuationType | ||
| import org.apache.kafka.streams.state.Stores | ||
| import org.apache.kafka.streams.state.TimestampedKeyValueStore | ||
| import org.apache.kafka.streams.state.ValueAndTimestamp | ||
| import java.time.Instant | ||
| import java.util.* | ||
| import java.util.concurrent.atomic.AtomicLong | ||
|
|
||
| context(ApplicationContext) | ||
| fun buildOpplysningerTopology( | ||
| meterRegistry: MeterRegistry | ||
| ): Topology = StreamsBuilder().apply { | ||
| addOpplysningerStateStore() | ||
| addOpplysningerKStream(meterRegistry) | ||
| }.build() | ||
|
|
||
| context(ApplicationContext) | ||
| private fun StreamsBuilder.addOpplysningerStateStore() { | ||
| logger.info("Oppretter state store for opplysninger om arbeidssøker") | ||
| val kafkaStreamsProperties = properties.kafkaStreams | ||
|
|
||
| this.addStateStore( | ||
| Stores.timestampedKeyValueStoreBuilder( | ||
| Stores.persistentKeyValueStore(kafkaStreamsProperties.opplysningerStore), | ||
| Serdes.Long(), | ||
| buildOpplysningerOmArbeidssoekerAvroSerde() | ||
| ) | ||
| ) | ||
| } | ||
|
|
||
| context(ApplicationContext) | ||
| private fun StreamsBuilder.addOpplysningerKStream(meterRegistry: MeterRegistry) { | ||
| logger.info("Oppretter KStream for opplysninger om arbeidssøker") | ||
| val kafkaStreamsProperties = properties.kafkaStreams | ||
|
|
||
| this.stream<Long, OpplysningerOmArbeidssoeker>(kafkaStreamsProperties.opplysningerTopic) | ||
| this | ||
| .stream( | ||
| kafkaStreamsProperties.opplysningerTopic, | ||
| Consumed.with(Serdes.Long(), buildOpplysningerOmArbeidssoekerAvroSerde()) | ||
| ) | ||
| .peek { key, _ -> | ||
| logger.debug("Mottok event på {} med key {}", kafkaStreamsProperties.opplysningerTopic, key) | ||
| meterRegistry.tellAntallMottatteOpplysninger() | ||
| meterRegistry.tellMottatteOpplysninger() | ||
| }.genericProcess<Long, OpplysningerOmArbeidssoeker, Long, OpplysningerOmArbeidssoeker>( | ||
| name = "processOpplysningerOmArbeidssoeker", | ||
| stateStoreNames = arrayOf(kafkaStreamsProperties.opplysningerStore), | ||
| punctuation = buildPunctuation(meterRegistry) | ||
| ) { record -> | ||
| val stateStore: TimestampedKeyValueStore<Long, OpplysningerOmArbeidssoeker> = | ||
| getStateStore(kafkaStreamsProperties.opplysningerStore) | ||
| logger.debug("Lagrer opplysninger for periode {}", record.value().periodeId) | ||
| stateStore.put(record.key(), ValueAndTimestamp.make(record.value(), Instant.now().toEpochMilli())) | ||
| } | ||
| }.build() | ||
| } | ||
|
|
||
| context(ApplicationContext) | ||
| private fun buildPunctuation(meterRegistry: MeterRegistry): Punctuation<Long, OpplysningerOmArbeidssoeker> { | ||
| logger.info("Oppretter Punctuation for opplysninger om arbeidssøker") | ||
| val kafkaStreamsProperties = properties.kafkaStreams | ||
|
|
||
| return Punctuation( | ||
| kafkaStreamsProperties.opplysningerPunctuatorSchedule, PunctuationType.WALL_CLOCK_TIME | ||
| ) { timestamp, context -> | ||
| logger.info("Punctuation kjører for tidspunkt {}", timestamp) | ||
|
|
||
| with(context) { | ||
| val antallTotalt = AtomicLong(0) | ||
| val histogram = mutableMapOf<UUID, AtomicLong>() | ||
|
|
||
| val stateStore: TimestampedKeyValueStore<Long, OpplysningerOmArbeidssoeker> = | ||
| getStateStore(kafkaStreamsProperties.opplysningerStore) | ||
| for (keyValue in stateStore.all()) { | ||
| antallTotalt.incrementAndGet() | ||
| val lagretTidspunkt = Instant.ofEpochMilli(keyValue.value.timestamp()) | ||
| val utloepTidspunkt = Instant.now().minus(kafkaStreamsProperties.opplysningerLagretTidsperiode) | ||
| if (utloepTidspunkt.isAfter(lagretTidspunkt) | ||
| ) { | ||
| logger.debug( | ||
| "Sletter opplysninger for periode {} fordi de har vært lagret mer enn {}m (utløp {} > lagret {})", | ||
| keyValue.value.value().periodeId, | ||
| kafkaStreamsProperties.opplysningerLagretTidsperiode.toMinutes(), | ||
| utloepTidspunkt, | ||
| lagretTidspunkt | ||
| ) | ||
| stateStore.delete(keyValue.key) | ||
| continue | ||
| } | ||
|
|
||
| val opplysninger = keyValue.value.value() | ||
| val antall = histogram[opplysninger.periodeId] | ||
| if (antall != null) { | ||
| antall.incrementAndGet() | ||
| histogram[opplysninger.periodeId] = antall | ||
| } else { | ||
| histogram[opplysninger.periodeId] = AtomicLong(1) | ||
| } | ||
| } | ||
|
|
||
| histogram.forEach { (_, antall) -> meterRegistry.antallLagredeOpplysningerSumPerPeriode(timestamp, antall) } | ||
| meterRegistry.antallLagredeOpplysningerTotal(antallTotalt) | ||
| } | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.