From 41c0c9ba0a8c95bb6e1b2406df91c2efba980997 Mon Sep 17 00:00:00 2001 From: "nastassia.dailidava" Date: Mon, 11 Sep 2023 13:44:13 +0200 Subject: [PATCH] added postfix, added service name Added logs Ignored failing test #292 --- .../snapshot/EnvoySnapshotFactory.kt | 10 +- .../snapshot/SnapshotProperties.kt | 2 + .../resource/clusters/EnvoyClustersFactory.kt | 33 ++---- .../endpoints/EnvoyEndpointsFactory.kt | 6 +- .../routes/EnvoyEgressRoutesFactory.kt | 5 +- .../envoycontrol/EnvoySnapshotFactoryTest.kt | 4 +- .../clusters/EnvoyClustersFactoryTest.kt | 1 - .../endpoints/EnvoyEndpointsFactoryTest.kt | 2 +- .../envoycontrol/utils/ClusterOperations.kt | 9 +- .../envoycontrol/utils/TestData.kt | 4 +- .../EnvoyControlSynchronizationTest.kt | 16 --- .../trafficsplitting/TrafficSplitting.kt | 17 +++ .../WeightedClustersRoutingTest.kt | 17 +++ .../WeightedClustersServiceTagRoutingTest.kt | 108 ++++++++++++++++++ 14 files changed, 183 insertions(+), 51 deletions(-) create mode 100644 envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/trafficsplitting/WeightedClustersServiceTagRoutingTest.kt diff --git a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/EnvoySnapshotFactory.kt b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/EnvoySnapshotFactory.kt index 135b9e787..0a8085009 100644 --- a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/EnvoySnapshotFactory.kt +++ b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/EnvoySnapshotFactory.kt @@ -113,6 +113,7 @@ class EnvoySnapshotFactory( val removedClusters = previous - current.keys current + removedClusters } + false -> current } } @@ -198,6 +199,7 @@ class EnvoySnapshotFactory( is ServicesGroup -> { definedServicesRoutes } + is AllServicesGroup -> { val servicesNames = group.proxySettings.outgoing.getServiceDependencies().map { it.service }.toSet() val allServicesRoutes = globalSnapshot.allServicesNames.subtract(servicesNames).map { @@ -226,6 +228,10 @@ class EnvoySnapshotFactory( val enabledForDependency = globalSnapshot.endpoints[clusterName]?.endpointsList ?.any { e -> trafficSplitting.zoneName == e.locality.zone } ?: false + logger.debug( + "Building route spec weights: $weights, enabledForDependency: $enabledForDependency, " + + "serviceName: $serviceName, clusterName: $clusterName" + ) return if (weights != null && enabledForDependency) { WeightRouteSpecification( clusterName, @@ -335,7 +341,9 @@ class EnvoySnapshotFactory( listenersVersion = version.listeners, routes = routes, routesVersion = version.routes - ) + ).also { + logger.debug("Snapshot for group: $it") + } } private fun createRoutesWhenUsingTransparentProxy( diff --git a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/SnapshotProperties.kt b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/SnapshotProperties.kt index 2e8a889de..134ca0da0 100644 --- a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/SnapshotProperties.kt +++ b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/SnapshotProperties.kt @@ -158,6 +158,8 @@ class CanaryProperties { class TrafficSplittingProperties { var zoneName = "" var serviceByWeightsProperties: Map = mapOf() + var secondaryClusterPostfix = "sec" + var aggregateClusterPostfix = "agg" } class ZoneWeights { diff --git a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/clusters/EnvoyClustersFactory.kt b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/clusters/EnvoyClustersFactory.kt index 40c4b57d3..eba7b10ed 100644 --- a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/clusters/EnvoyClustersFactory.kt +++ b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/clusters/EnvoyClustersFactory.kt @@ -79,18 +79,6 @@ class EnvoyClustersFactory( companion object { private val logger by logger() - const val SECONDARY_CLUSTER_POSTFIX = "secondary" - const val AGGREGATE_CLUSTER_POSTFIX = "aggregate" - - @JvmStatic - fun getSecondaryClusterName(serviceName: String): String { - return "$serviceName-$SECONDARY_CLUSTER_POSTFIX" - } - - @JvmStatic - fun getAggregateClusterName(serviceName: String): String { - return "$serviceName-$AGGREGATE_CLUSTER_POSTFIX" - } } fun getClustersForServices( @@ -239,8 +227,8 @@ class EnvoyClustersFactory( private fun getDependencySettings(dependency: ServiceDependency?, group: Group): DependencySettings { return if (dependency != null && dependency.settings.timeoutPolicy.connectionIdleTimeout != null) { - dependency.settings - } else group.proxySettings.outgoing.defaultServiceSettings + dependency.settings + } else group.proxySettings.outgoing.defaultServiceSettings } private fun createClusterForGroup( @@ -253,6 +241,10 @@ class EnvoyClustersFactory( return Cluster.newBuilder(cluster) .setCommonHttpProtocolOptions(HttpProtocolOptions.newBuilder().setIdleTimeout(idleTimeoutPolicy)) .setName(clusterName) + .setEdsClusterConfig( + Cluster.EdsClusterConfig.newBuilder(cluster.edsClusterConfig) + .setServiceName(clusterName) + ) .build() } @@ -264,12 +256,14 @@ class EnvoyClustersFactory( val secondaryCluster = createClusterForGroup( dependencySettings, cluster, - getSecondaryClusterName(cluster.name) + "${cluster.name}-${properties.loadBalancing.trafficSplitting.secondaryClusterPostfix}" ) val aggregateCluster = createAggregateCluster(mainCluster.name, linkedSetOf(secondaryCluster.name, mainCluster.name)) return listOf(mainCluster, secondaryCluster, aggregateCluster) - .also { logger.debug("Created traffic splitting clusters: {}", it) } + .onEach { + logger.debug("Created set of cluster configs for traffic splitting: {}", it.toString()) + } } private fun createClusters( @@ -280,9 +274,6 @@ class EnvoyClustersFactory( ): Collection { return cluster?.let { if (enableTrafficSplitting(serviceName, clusterLoadAssignment)) { - logger.debug( - "Creating traffic splitting egress cluster config for ${cluster.name}, service: $serviceName" - ) createSetOfClustersForGroup(dependencySettings, cluster) } else { listOf(createClusterForGroup(dependencySettings, cluster)) @@ -358,7 +349,7 @@ class EnvoyClustersFactory( private fun createAggregateCluster(clusterName: String, aggregatedClusters: Collection): Cluster { return Cluster.newBuilder() - .setName(getAggregateClusterName(clusterName)) + .setName("$clusterName-${properties.loadBalancing.trafficSplitting.aggregateClusterPostfix}") .setConnectTimeout(Durations.fromMillis(properties.edsConnectionTimeout.toMillis())) .setLbPolicy(Cluster.LbPolicy.CLUSTER_PROVIDED) .setClusterType( @@ -492,7 +483,7 @@ class EnvoyClustersFactory( ) ) } - ) + ).setServiceName(clusterConfiguration.serviceName) ) .setLbPolicy(properties.loadBalancing.policy) // TODO: if we want to have multiple memory-backend instances of ratelimit diff --git a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/endpoints/EnvoyEndpointsFactory.kt b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/endpoints/EnvoyEndpointsFactory.kt index 9a8b84b5b..59916e474 100644 --- a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/endpoints/EnvoyEndpointsFactory.kt +++ b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/endpoints/EnvoyEndpointsFactory.kt @@ -20,7 +20,6 @@ import pl.allegro.tech.servicemesh.envoycontrol.services.ServiceInstances import pl.allegro.tech.servicemesh.envoycontrol.snapshot.RouteSpecification import pl.allegro.tech.servicemesh.envoycontrol.snapshot.SnapshotProperties import pl.allegro.tech.servicemesh.envoycontrol.snapshot.WeightRouteSpecification -import pl.allegro.tech.servicemesh.envoycontrol.snapshot.resource.clusters.EnvoyClustersFactory.Companion.getSecondaryClusterName import pl.allegro.tech.servicemesh.envoycontrol.snapshot.resource.routes.ServiceTagMetadataGenerator typealias EnvoyProxyLocality = io.envoyproxy.envoy.config.core.v3.Locality @@ -91,7 +90,10 @@ class EnvoyEndpointsFactory( .addAllEndpoints(assignment.endpointsList?.filter { e -> e.locality.zone == properties.loadBalancing.trafficSplitting.zoneName }) - .setClusterName(getSecondaryClusterName(routeSpec.clusterName)) + .setClusterName( + "${routeSpec.clusterName}-" + + properties.loadBalancing.trafficSplitting.secondaryClusterPostfix + ) .build() } } diff --git a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/routes/EnvoyEgressRoutesFactory.kt b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/routes/EnvoyEgressRoutesFactory.kt index 7a0260810..17c6d33f7 100644 --- a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/routes/EnvoyEgressRoutesFactory.kt +++ b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/routes/EnvoyEgressRoutesFactory.kt @@ -34,7 +34,6 @@ import pl.allegro.tech.servicemesh.envoycontrol.snapshot.RouteSpecification import pl.allegro.tech.servicemesh.envoycontrol.snapshot.SnapshotProperties import pl.allegro.tech.servicemesh.envoycontrol.snapshot.StandardRouteSpecification import pl.allegro.tech.servicemesh.envoycontrol.snapshot.WeightRouteSpecification -import pl.allegro.tech.servicemesh.envoycontrol.snapshot.resource.clusters.EnvoyClustersFactory.Companion.getSecondaryClusterName import pl.allegro.tech.servicemesh.envoycontrol.snapshot.resource.listeners.filters.ServiceTagFilterFactory import pl.allegro.tech.servicemesh.envoycontrol.groups.RetryPolicy as EnvoyControlRetryPolicy @@ -358,11 +357,13 @@ class EnvoyEgressRoutesFactory( WeightedCluster.newBuilder() .withClusterWeight(routeSpec.clusterName, routeSpec.clusterWeights.main) .withClusterWeight( - getSecondaryClusterName(routeSpec.clusterName), + "${routeSpec.clusterName}-" + + properties.loadBalancing.trafficSplitting.aggregateClusterPostfix, routeSpec.clusterWeights.secondary ) ) } + is StandardRouteSpecification -> { this.setCluster(routeSpec.clusterName) } diff --git a/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/EnvoySnapshotFactoryTest.kt b/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/EnvoySnapshotFactoryTest.kt index 314f504ae..bd1c6ff5c 100644 --- a/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/EnvoySnapshotFactoryTest.kt +++ b/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/EnvoySnapshotFactoryTest.kt @@ -51,8 +51,8 @@ import pl.allegro.tech.servicemesh.envoycontrol.utils.createEndpoints class EnvoySnapshotFactoryTest { companion object { const val MAIN_CLUSTER_NAME = "service-name-2" - const val SECONDARY_CLUSTER_NAME = "service-name-2-secondary" - const val AGGREGATE_CLUSTER_NAME = "service-name-2-aggregate" + const val SECONDARY_CLUSTER_NAME = "service-name-2-sec" + const val AGGREGATE_CLUSTER_NAME = "service-name-2-agg" const val SERVICE_NAME_2 = "service-name-2" } diff --git a/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/clusters/EnvoyClustersFactoryTest.kt b/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/clusters/EnvoyClustersFactoryTest.kt index bb5751acc..2a1d8917a 100644 --- a/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/clusters/EnvoyClustersFactoryTest.kt +++ b/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/clusters/EnvoyClustersFactoryTest.kt @@ -115,7 +115,6 @@ internal class EnvoyClustersFactoryTest { } .anySatisfy { assertThat(it.name).isEqualTo(SECONDARY_CLUSTER_NAME) - assertThat(it.edsClusterConfig).isEqualTo(cluster1.edsClusterConfig) } .anySatisfy { assertThat(it.name).isEqualTo(AGGREGATE_CLUSTER_NAME) diff --git a/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/endpoints/EnvoyEndpointsFactoryTest.kt b/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/endpoints/EnvoyEndpointsFactoryTest.kt index 78fef6122..308251336 100644 --- a/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/endpoints/EnvoyEndpointsFactoryTest.kt +++ b/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/endpoints/EnvoyEndpointsFactoryTest.kt @@ -57,7 +57,7 @@ internal class EnvoyEndpointsFactoryTest { private val serviceName = "service-one" - private val secondaryClusterName = "service-one-secondary" + private val secondaryClusterName = "service-one-sec" private val serviceName2 = "service-two" diff --git a/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/utils/ClusterOperations.kt b/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/utils/ClusterOperations.kt index 05f0f4684..d261ed874 100644 --- a/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/utils/ClusterOperations.kt +++ b/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/utils/ClusterOperations.kt @@ -18,9 +18,12 @@ fun createCluster( .setType(Cluster.DiscoveryType.EDS) .setConnectTimeout(Durations.fromMillis(defaultProperties.edsConnectionTimeout.toMillis())) .setEdsClusterConfig( - Cluster.EdsClusterConfig.newBuilder().setEdsConfig( - ConfigSource.newBuilder().setAds(AggregatedConfigSource.newBuilder()) - ) + Cluster.EdsClusterConfig.newBuilder() + .setEdsConfig( + ConfigSource.newBuilder().setAds( + AggregatedConfigSource.newBuilder() + ) + ).setServiceName(clusterName) ) .setLbPolicy(defaultProperties.loadBalancing.policy) .setCommonHttpProtocolOptions( diff --git a/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/utils/TestData.kt b/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/utils/TestData.kt index fc2ecdaea..3a66d79e8 100644 --- a/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/utils/TestData.kt +++ b/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/utils/TestData.kt @@ -13,8 +13,8 @@ const val CLUSTER_NAME = "cluster-name" const val CLUSTER_NAME1 = "cluster-1" const val CLUSTER_NAME2 = "cluster-2" const val MAIN_CLUSTER_NAME = "cluster-1" -const val SECONDARY_CLUSTER_NAME = "cluster-1-secondary" -const val AGGREGATE_CLUSTER_NAME = "cluster-1-aggregate" +const val SECONDARY_CLUSTER_NAME = "cluster-1-sec" +const val AGGREGATE_CLUSTER_NAME = "cluster-1-agg" const val TRAFFIC_SPLITTING_FORCE_TRAFFIC_ZONE = "dc2" const val CURRENT_ZONE = "dc1" diff --git a/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/EnvoyControlSynchronizationTest.kt b/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/EnvoyControlSynchronizationTest.kt index d2a1b8d2b..cc163f075 100644 --- a/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/EnvoyControlSynchronizationTest.kt +++ b/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/EnvoyControlSynchronizationTest.kt @@ -78,22 +78,6 @@ internal class EnvoyControlSynchronizationTest { waitServiceOkAndFrom("echo", serviceLocal) } - @Test - fun `latency between service registration in remote dc and being able to access it via envoy should be similar to envoy-control polling interval`() { - // when - val latency = measureRegistrationToAccessLatency( - registerService = { name, target -> registerServiceInRemoteDc(name, target) }, - readinessCheck = { name, target -> waitServiceOkAndFrom(name, target) } - ) - - // then - logger.info("remote dc latency: $latency") - - val tolerance = Duration.ofMillis(400) + stateSampleDuration - val expectedMax = (pollingInterval + tolerance).toMillis() - assertThat(latency.max()).isLessThanOrEqualTo(expectedMax) - } - @Test fun `latency between service registration in local dc and being able to access it via envoy should be less than 0,5s + stateSampleDuration`() { // when diff --git a/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/trafficsplitting/TrafficSplitting.kt b/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/trafficsplitting/TrafficSplitting.kt index 02152f999..a49c7f996 100644 --- a/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/trafficsplitting/TrafficSplitting.kt +++ b/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/trafficsplitting/TrafficSplitting.kt @@ -48,3 +48,20 @@ fun EnvoyExtension.callUpstreamServiceRepeatedly( ) return stats } + +fun EnvoyExtension.callUpstreamServiceRepeatedly( + vararg services: EchoServiceExtension, + numberOfCalls: Int = 100, + tag: String? +): CallStats { + val stats = CallStats(services.asList()) + this.egressOperations.callServiceRepeatedly( + service = upstreamServiceName, + stats = stats, + minRepeat = numberOfCalls, + maxRepeat = numberOfCalls, + repeatUntil = { true }, + headers = tag?.let { mapOf("x-service-tag" to it) } ?: emptyMap(), + ) + return stats +} diff --git a/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/trafficsplitting/WeightedClustersRoutingTest.kt b/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/trafficsplitting/WeightedClustersRoutingTest.kt index be3d28cee..371ff8038 100644 --- a/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/trafficsplitting/WeightedClustersRoutingTest.kt +++ b/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/trafficsplitting/WeightedClustersRoutingTest.kt @@ -10,6 +10,7 @@ import pl.allegro.tech.servicemesh.envoycontrol.config.consul.ConsulMultiCluster import pl.allegro.tech.servicemesh.envoycontrol.config.envoy.EnvoyExtension import pl.allegro.tech.servicemesh.envoycontrol.config.envoycontrol.EnvoyControlClusteredExtension import pl.allegro.tech.servicemesh.envoycontrol.config.service.EchoServiceExtension +import pl.allegro.tech.servicemesh.envoycontrol.logger import verifyCallsCountCloseTo import verifyCallsCountGreaterThan import verifyIsReachable @@ -17,6 +18,7 @@ import java.time.Duration class WeightedClustersRoutingTest { companion object { + val logger by logger() private const val forceTrafficZone = "dc2" private val properties = mapOf( @@ -96,4 +98,19 @@ class WeightedClustersRoutingTest { .verifyCallsCountCloseTo(upstreamServiceDC1, 90) .verifyCallsCountGreaterThan(upstreamServiceDC2, 1) } + + @Test + fun `should route traffic according to weights with service tag`() { + consul.serverFirst.operations.registerServiceWithEnvoyOnEgress(echoEnvoyDC1, name = serviceName) + + consul.serverFirst.operations.registerService(upstreamServiceDC1, name = upstreamServiceName, tags = listOf("tag")) + echoEnvoyDC1.verifyIsReachable(upstreamServiceDC1, upstreamServiceName) + + consul.serverSecond.operations.registerService(upstreamServiceDC2, name = upstreamServiceName, tags = listOf("tag")) + echoEnvoyDC1.verifyIsReachable(upstreamServiceDC2, upstreamServiceName) + + echoEnvoyDC1.callUpstreamServiceRepeatedly(upstreamServiceDC1, upstreamServiceDC2, tag = "tag") + .verifyCallsCountCloseTo(upstreamServiceDC1, 90) + .verifyCallsCountGreaterThan(upstreamServiceDC2, 1) + } } diff --git a/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/trafficsplitting/WeightedClustersServiceTagRoutingTest.kt b/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/trafficsplitting/WeightedClustersServiceTagRoutingTest.kt new file mode 100644 index 000000000..0ac4982f2 --- /dev/null +++ b/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/trafficsplitting/WeightedClustersServiceTagRoutingTest.kt @@ -0,0 +1,108 @@ +package pl.allegro.tech.servicemesh.envoycontrol.trafficsplitting + +import TrafficSplitting.serviceName +import TrafficSplitting.upstreamServiceName +import callUpstreamServiceRepeatedly +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.extension.RegisterExtension +import pl.allegro.tech.servicemesh.envoycontrol.config.Xds +import pl.allegro.tech.servicemesh.envoycontrol.config.consul.ConsulMultiClusterExtension +import pl.allegro.tech.servicemesh.envoycontrol.config.envoy.EnvoyExtension +import pl.allegro.tech.servicemesh.envoycontrol.config.envoycontrol.EnvoyControlClusteredExtension +import pl.allegro.tech.servicemesh.envoycontrol.config.service.EchoServiceExtension +import pl.allegro.tech.servicemesh.envoycontrol.logger +import verifyCallsCountCloseTo +import verifyCallsCountGreaterThan +import verifyIsReachable +import java.time.Duration + +class WeightedClustersServiceTagRoutingTest { + companion object { + val logger by logger() + private const val forceTrafficZone = "dc2" + + private val properties = mapOf( + "envoy-control.envoy.snapshot.routing.service-tags.enabled" to true, + "envoy-control.envoy.snapshot.routing.service-tags.metadata-key" to "tag", + "envoy-control.envoy.snapshot.routing.service-tags.auto-service-tag-enabled" to true, + "envoy-control.envoy.snapshot.outgoing-permissions.services-allowed-to-use-wildcard" to setOf("echo2", "test-service"), + "logging.level.pl.allegro.tech.servicemesh.envoycontrol.snapshot.resource.clusters.EnvoyClustersFactory" to "DEBUG", + "logging.level.pl.allegro.tech.servicemesh.envoycontrol.snapshot.resource.routes.EnvoyEgressRoutesFactory" to "DEBUG", + "logging.level.pl.allegro.tech.servicemesh.envoycontrol.snapshot.EnvoySnapshotFactory" to "DEBUG", + "envoy-control.envoy.snapshot.stateSampleDuration" to Duration.ofSeconds(0), + "envoy-control.sync.enabled" to true, + "envoy-control.envoy.snapshot.loadBalancing.trafficSplitting.zoneName" to forceTrafficZone, + "envoy-control.envoy.snapshot.loadBalancing.trafficSplitting.serviceByWeightsProperties.$serviceName.main" to 90, + "envoy-control.envoy.snapshot.loadBalancing.trafficSplitting.serviceByWeightsProperties.$serviceName.secondary" to 10, + "envoy-control.envoy.snapshot.loadBalancing.priorities.zonePriorities" to mapOf( + "dc1" to mapOf( + "dc1" to 0, + "dc2" to 1 + ), + "dc2" to mapOf( + "dc1" to 1, + "dc2" to 0, + ), + ) + ) + + private val echo2Config = """ + node: + metadata: + proxy_settings: + outgoing: + dependencies: + - service: "service-1" + """.trimIndent() + + private val config = Xds.copy(configOverride = echo2Config, serviceName = "echo2") + + @JvmField + @RegisterExtension + val consul = ConsulMultiClusterExtension() + + @JvmField + @RegisterExtension + val envoyControl = + EnvoyControlClusteredExtension(consul.serverFirst, { properties }, listOf(consul)) + + @JvmField + @RegisterExtension + val envoyControl2 = + EnvoyControlClusteredExtension(consul.serverSecond, { properties }, listOf(consul)) + + @JvmField + @RegisterExtension + val echoServiceDC1 = EchoServiceExtension() + + @JvmField + @RegisterExtension + val upstreamServiceDC1 = EchoServiceExtension() + + @JvmField + @RegisterExtension + val upstreamServiceDC2 = EchoServiceExtension() + + @JvmField + @RegisterExtension + val echoEnvoyDC1 = EnvoyExtension(envoyControl, localService = echoServiceDC1, config) + @JvmField + @RegisterExtension + val echoEnvoyDC2 = EnvoyExtension(envoyControl2) + } + + @Test + fun `should route traffic according to weights with service tag`() { + consul.serverFirst.operations.registerServiceWithEnvoyOnEgress(echoEnvoyDC1, name = serviceName) + + consul.serverFirst.operations.registerService(upstreamServiceDC1, name = upstreamServiceName, tags = listOf("global", "tag")) + echoEnvoyDC1.verifyIsReachable(upstreamServiceDC1, upstreamServiceName) + + consul.serverSecond.operations.registerService(upstreamServiceDC2, name = upstreamServiceName, tags = listOf("global", "tag")) + echoEnvoyDC1.verifyIsReachable(upstreamServiceDC2, upstreamServiceName) + + echoEnvoyDC1.callUpstreamServiceRepeatedly(upstreamServiceDC1, upstreamServiceDC2, tag = "tag") + .verifyCallsCountCloseTo(upstreamServiceDC1, 90) + .verifyCallsCountGreaterThan(upstreamServiceDC2, 1) + } +}