diff --git a/src/common/Constants.cs b/src/common/Constants.cs index bac25ccf..efb862ef 100644 --- a/src/common/Constants.cs +++ b/src/common/Constants.cs @@ -132,6 +132,7 @@ internal static class Routing public const string FeatureFlagsAnnotationName = RoutingLabelPrefix + "feature-flags"; public const string RoutingComponentLabel = RoutingLabelPrefix + "component"; public const string RouteFromLabelName = RoutingLabelPrefix + "route-from"; + public const string RouteUniqueName = RoutingLabelPrefix + "route-unique-name"; public const string KubernetesRouteAsHeaderName = "kubernetes-route-as"; public const string RoutingManagerNameLower = "routingmanager"; public const string RoutingManagerServiceName = RoutingManagerNameLower + "-service"; diff --git a/src/devhostagent.restorationjob/RestorationJobApp.cs b/src/devhostagent.restorationjob/RestorationJobApp.cs index 5955755e..1bdda858 100644 --- a/src/devhostagent.restorationjob/RestorationJobApp.cs +++ b/src/devhostagent.restorationjob/RestorationJobApp.cs @@ -368,7 +368,7 @@ private async Task _GetAgentEndpointAsync(PodDeployment podDeployment, Canc return null; } - return new Uri(string.Format(AgentPingEndpointFormat, pod.Status.PodIP)); + return new Uri(string.Format(AgentPingEndpointFormat, $"{pod.Metadata.Name}.{pod.Metadata.NamespaceProperty}")); } catch (Exception e) when (!cancellationToken.IsCancellationRequested) { diff --git a/src/library/Connect/KubernetesRemoteEnvironmentManager.cs b/src/library/Connect/KubernetesRemoteEnvironmentManager.cs index a4b80400..a26e5e42 100644 --- a/src/library/Connect/KubernetesRemoteEnvironmentManager.cs +++ b/src/library/Connect/KubernetesRemoteEnvironmentManager.cs @@ -585,6 +585,7 @@ private async Task _WaitForPodToRunAsync(V1Pod pod, TimeSpan maxWaitTime, } clonedPod.Metadata.Labels[Routing.RouteFromLabelName] = sourceServiceName; + clonedPod.Metadata.Labels[Routing.RouteUniqueName] = clonedPod.Metadata.Name; // Set routing annotation value to the routing header value provided by user. if (clonedPod.Metadata.Annotations == null) diff --git a/src/routingmanager/Envoy/EnvoyConfig.cs b/src/routingmanager/Envoy/EnvoyConfig.cs index 70ba0877..7da1b988 100644 --- a/src/routingmanager/Envoy/EnvoyConfig.cs +++ b/src/routingmanager/Envoy/EnvoyConfig.cs @@ -251,6 +251,9 @@ internal partial class Route // https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/route/v3/route_components.proto#envoy-v3-api-field-config-route-v3-routeaction-idle-timeout [YamlMember(Alias = "idle_timeout", DefaultValuesHandling = DefaultValuesHandling.OmitNull)] public string IdleTimeout { get; set; } = "0s"; + + [YamlMember(Alias = "auto_host_rewrite", DefaultValuesHandling = DefaultValuesHandling.OmitNull)] + public bool AutoHostRewrite { get; set; } } internal partial class RequestHeadersToAdd diff --git a/src/routingmanager/Envoy/EnvoyConfigBuilder.cs b/src/routingmanager/Envoy/EnvoyConfigBuilder.cs index a34c6285..08a1b628 100644 --- a/src/routingmanager/Envoy/EnvoyConfigBuilder.cs +++ b/src/routingmanager/Envoy/EnvoyConfigBuilder.cs @@ -38,7 +38,8 @@ public EnvoyConfigBuilder(ILog log) /// /// Returns the envoy configuration based on trigger information /// - public EnvoyConfig GetEnvoyConfig(V1Service triggerService, RoutingStateEstablisherInput routingStateEstablisherInput, IEnumerable allPodTriggersInNamespace) + public EnvoyConfig GetEnvoyConfig(V1Service triggerService, RoutingStateEstablisherInput routingStateEstablisherInput, + IEnumerable allPodTriggersInNamespace, List destinationServices) { var envoyConfig = CreateEmptyEnvoyConfig(); @@ -62,20 +63,21 @@ public EnvoyConfig GetEnvoyConfig(V1Service triggerService, RoutingStateEstablis ConfigureVirtualHostForMatchAllHost(triggerService, routingStateEstablisherInput.PodTriggers, servicePort, httpFilterVirtualHosts); // Now we will start adding clusters to this envoy configuration - ConfigureClusters(triggerService, routingStateEstablisherInput.PodTriggers, envoyConfig, servicePort); + ConfigureClusters(triggerService, routingStateEstablisherInput.PodTriggers, envoyConfig, servicePort, destinationServices); } _log.Info("Envoy Config is: {0}", JsonSerializer.Serialize(envoyConfig)); return envoyConfig; } - private void ConfigureClusters(V1Service triggerService, IEnumerable podTriggers, EnvoyConfig envoyConfig, V1ServicePort servicePort) + private void ConfigureClusters(V1Service triggerService, IEnumerable podTriggers, EnvoyConfig envoyConfig, V1ServicePort servicePort, + List destinationServices) { var portName = (servicePort.Name ?? string.Empty).ToLowerInvariant(); var cloneCluster = new Cluster { Name = string.Format(_serviceCloneWithPortsFormatString, servicePort.Port, servicePort.TargetPort.Value), ConnectTimeout = "1.00s", - Type = "strict_dns", + Type = "logical_dns", LoadAssignment = new LoadAssignment { ClusterName = string.Format(_serviceCloneWithPortsFormatString, servicePort.Port, servicePort.TargetPort.Value), @@ -120,11 +122,14 @@ private void ConfigureClusters(V1Service triggerService, IEnumerable ds.Metadata.Name == podTrigger.RouteUniqueName); + var routeCluster = new Cluster { Name = string.Format(_serviceStableWithHeaderWithPortsFormatString, podTrigger.RouteOnHeader.Key, podTrigger.RouteOnHeader.Value, servicePort.Port, servicePort.TargetPort.Value), ConnectTimeout = "1.00s", - Type = "static", + Type = "logical_dns", LoadAssignment = new LoadAssignment { ClusterName = string.Format(_serviceStableWithHeaderWithPortsFormatString, podTrigger.RouteOnHeader.Key, podTrigger.RouteOnHeader.Value, servicePort.Port, servicePort.TargetPort.Value), @@ -143,7 +148,7 @@ private void ConfigureClusters(V1Service triggerService, IEnumerable input.Value.PodTriggers.Count()).Sum(), @@ -667,15 +668,18 @@ await WebUtilities.RetryUntilTimeAsync(async _ => _log.Warning(ex.Message); } - var podTriggerToAdd = - new PodTriggerConfig( - namespaceName: triggerService.Metadata.NamespaceProperty, - triggerService: triggerService, - lpkPodName: pod.Metadata.Name, - routeOnHeaderKey: routeOnHeader.headerName, - routeOnHeaderValue: routeOnHeader.headerValue, - triggerPodIP: pod_latest == null ? pod.Status.PodIP : pod_latest.Status.PodIP, - correlationId: correlationId); + var routeUniqueName = pod.Metadata.Labels[Common.Constants.Routing.RouteUniqueName]; + + var podTriggerToAdd = new PodTriggerConfig( + namespaceName: triggerService.Metadata.NamespaceProperty, + triggerService: triggerService, + lpkPodName: pod.Metadata.Name, + routeOnHeaderKey: routeOnHeader.headerName, + routeOnHeaderValue: routeOnHeader.headerValue, + triggerPodIP: pod_latest == null ? pod.Status.PodIP : pod_latest.Status.PodIP, + correlationId: correlationId, + routeUniqueName: routeUniqueName + ); routingStateEstablisherInputMap.AddOrUpdateWithTrigger(triggerService, podTriggerToAdd); } } diff --git a/src/routingmanager/RoutingStateEstablisher.cs b/src/routingmanager/RoutingStateEstablisher.cs index 9864387d..9f1819a9 100644 --- a/src/routingmanager/RoutingStateEstablisher.cs +++ b/src/routingmanager/RoutingStateEstablisher.cs @@ -23,6 +23,7 @@ using System.Text.Json; using System.Threading; using System.Threading.Tasks; +using Microsoft.BridgeToKubernetes.Common.DevHostAgent; using static Microsoft.BridgeToKubernetes.Common.Constants; namespace Microsoft.BridgeToKubernetes.RoutingManager @@ -109,9 +110,14 @@ public async Task> RunAsync(IDictionary> RunAsync(IDictionary i.Value.PodTriggers.Count)) != expectedDeployments.Count + || (expectedServices.Count - inputs.Sum(i => i.Value.PodTriggers.Count)) != expectedConfigMaps.Count) { _log.Error("Number of generated envoy resources do not match. "); throw new RoutingException(Resources.FailedToValidateEnvoyResources); @@ -485,13 +491,13 @@ private V1Deployment GenerateEnvoyDeployment(KeyValuePair { "/bin/bash" }, Args = new List { "-c", - "touch envoy-logs.txt && /usr/local/bin/envoy --log-path envoy-logs.txt --log-level trace --config-path /etc/envoy/envoy.yaml" + "touch envoy-logs.txt && /usr/local/bin/envoy --log-path envoy-logs.txt --log-level trace --base-id 3 --config-path /etc/envoy/envoy.yaml" }, Ports = containerPorts, VolumeMounts = new List @@ -683,6 +689,64 @@ private V1Service GenerateClonedService(KeyValuePair + /// Generates services that exposes the target pods with a cluster IP + /// + /// + private List GenerateDestinationServices(KeyValuePair input) + { + var services = new List(); + + foreach (var pod in input.Value.PodTriggers) + { + var service = new V1Service + { + Metadata = new V1ObjectMeta + { + Name = pod.RouteUniqueName, + NamespaceProperty = pod.NamespaceName, + Labels = new Dictionary + { + [Routing.GeneratedLabel] = "true", + }, + }, + Spec = new V1ServiceSpec + { + Selector = new Dictionary + { + [Routing.RouteUniqueName] = pod.RouteUniqueName, + }, + Type = "ClusterIP", + Ports = new List(), + }, + }; + + foreach (var port in input.Key.Spec.Ports) + { + service.Spec.Ports.Add(new V1ServicePort + { + AppProtocol = port.AppProtocol, + Name = port.Name, + Port = port.Port, + Protocol = port.Protocol, + TargetPort = port.TargetPort, + }); + } + + service.Spec.Ports.Add(new V1ServicePort + { + Name = "devhostagent", + Port = DevHostConstants.DevHostAgent.Port, + Protocol = "TCP", + TargetPort = DevHostConstants.DevHostAgent.Port, + }); + + services.Add(service); + } + + return services; + } /// /// Update the cluster with the expected objects diff --git a/src/routingmanager/TriggerConfig/PodTriggerConfig.cs b/src/routingmanager/TriggerConfig/PodTriggerConfig.cs index 51f8d6ac..d4d70db8 100644 --- a/src/routingmanager/TriggerConfig/PodTriggerConfig.cs +++ b/src/routingmanager/TriggerConfig/PodTriggerConfig.cs @@ -21,7 +21,9 @@ public PodTriggerConfig( string routeOnHeaderKey, string routeOnHeaderValue, string triggerPodIP, - string correlationId) + string correlationId, + string routeUniqueName + ) { NamespaceName = string.IsNullOrWhiteSpace(namespaceName) ? throw new ArgumentNullException(nameof(namespaceName)) : namespaceName; TriggerService = triggerService ?? throw new ArgumentNullException(nameof(triggerService)); @@ -32,6 +34,7 @@ public PodTriggerConfig( TriggerPodIp = triggerPodIP ?? throw new ArgumentNullException(nameof(triggerPodIP)); TriggerEntityName = string.IsNullOrWhiteSpace(lpkPodName) ? throw new ArgumentNullException(nameof(lpkPodName)) : lpkPodName; CorrelationId = correlationId; + RouteUniqueName = routeUniqueName; } /// @@ -53,6 +56,11 @@ public PodTriggerConfig( /// LPK agent pod name /// public string TriggerEntityName { get; } + + /// + /// the unique name of the pod used to expose it via a service + /// + public string RouteUniqueName { get; } /// /// LPK agent Pod IP