feat: support service meshes (only tested with OSM)

src/common/Constants.cs

@@ -132,6 +132,7 @@ internal static class Routing
             public const string FeatureFlagsAnnotationName = RoutingLabelPrefix + "feature-flags";
             public const string RoutingComponentLabel = RoutingLabelPrefix + "component";
             public const string RouteFromLabelName = RoutingLabelPrefix + "route-from";
+            public const string RouteUniqueName = RoutingLabelPrefix + "route-unique-name";
             public const string KubernetesRouteAsHeaderName = "kubernetes-route-as";
             public const string RoutingManagerNameLower = "routingmanager";
             public const string RoutingManagerServiceName = RoutingManagerNameLower + "-service";

src/devhostagent.restorationjob/RestorationJobApp.cs

@@ -368,7 +368,7 @@ private async Task<Uri> _GetAgentEndpointAsync(PodDeployment podDeployment, Canc
                     return null;
                 }
 
-                return new Uri(string.Format(AgentPingEndpointFormat, pod.Status.PodIP));
+                return new Uri(string.Format(AgentPingEndpointFormat, $"{pod.Metadata.Name}.{pod.Metadata.NamespaceProperty}"));
             }
             catch (Exception e) when (!cancellationToken.IsCancellationRequested)
             {

src/library/Connect/KubernetesRemoteEnvironmentManager.cs

@@ -585,6 +585,7 @@ private async Task<V1Pod> _WaitForPodToRunAsync(V1Pod pod, TimeSpan maxWaitTime,
                 }
 
                 clonedPod.Metadata.Labels[Routing.RouteFromLabelName] = sourceServiceName;
+                clonedPod.Metadata.Labels[Routing.RouteUniqueName] = clonedPod.Metadata.Name;
 
                 // Set routing annotation value to the routing header value provided by user.
                 if (clonedPod.Metadata.Annotations == null)

src/routingmanager/Envoy/EnvoyConfig.cs

@@ -251,6 +251,9 @@ internal partial class Route
         // https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/route/v3/route_components.proto#envoy-v3-api-field-config-route-v3-routeaction-idle-timeout
         [YamlMember(Alias = "idle_timeout", DefaultValuesHandling = DefaultValuesHandling.OmitNull)]
         public string IdleTimeout { get; set; } = "0s";
+
+        [YamlMember(Alias = "auto_host_rewrite", DefaultValuesHandling = DefaultValuesHandling.OmitNull)]
+        public bool AutoHostRewrite { get; set; }
     }
 
     internal partial class RequestHeadersToAdd

src/routingmanager/Envoy/EnvoyConfigBuilder.cs

@@ -38,7 +38,8 @@ public EnvoyConfigBuilder(ILog log)
         /// <summary>
         /// Returns the envoy configuration based on trigger information
         /// </summary>
-        public EnvoyConfig GetEnvoyConfig(V1Service triggerService, RoutingStateEstablisherInput routingStateEstablisherInput, IEnumerable<PodTriggerConfig> allPodTriggersInNamespace)
+        public EnvoyConfig GetEnvoyConfig(V1Service triggerService, RoutingStateEstablisherInput routingStateEstablisherInput,
+            IEnumerable<PodTriggerConfig> allPodTriggersInNamespace, List<V1Service> destinationServices)
         {
             var envoyConfig = CreateEmptyEnvoyConfig();
 
@@ -62,20 +63,21 @@ public EnvoyConfig GetEnvoyConfig(V1Service triggerService, RoutingStateEstablis
                 ConfigureVirtualHostForMatchAllHost(triggerService, routingStateEstablisherInput.PodTriggers, servicePort, httpFilterVirtualHosts);
 
                 // Now we will start adding clusters to this envoy configuration
-                ConfigureClusters(triggerService, routingStateEstablisherInput.PodTriggers, envoyConfig, servicePort);
+                ConfigureClusters(triggerService, routingStateEstablisherInput.PodTriggers, envoyConfig, servicePort, destinationServices);
             }
             _log.Info("Envoy Config is: {0}", JsonSerializer.Serialize(envoyConfig));
             return envoyConfig;
         }
 
-        private void ConfigureClusters(V1Service triggerService, IEnumerable<PodTriggerConfig> podTriggers, EnvoyConfig envoyConfig, V1ServicePort servicePort)
+        private void ConfigureClusters(V1Service triggerService, IEnumerable<PodTriggerConfig> podTriggers, EnvoyConfig envoyConfig, V1ServicePort servicePort,
+            List<V1Service> destinationServices)
         {
             var portName = (servicePort.Name ?? string.Empty).ToLowerInvariant();
             var cloneCluster = new Cluster
             {
                 Name = string.Format(_serviceCloneWithPortsFormatString, servicePort.Port, servicePort.TargetPort.Value),
                 ConnectTimeout = "1.00s",
-                Type = "strict_dns",
+                Type = "logical_dns",
                 LoadAssignment = new LoadAssignment
                 {
                     ClusterName = string.Format(_serviceCloneWithPortsFormatString, servicePort.Port, servicePort.TargetPort.Value),
@@ -120,11 +122,14 @@ private void ConfigureClusters(V1Service triggerService, IEnumerable<PodTriggerC
 
             foreach (var podTrigger in podTriggers)
             {
+                var destinationService = destinationServices
+                    .Single(ds => ds.Metadata.Name == podTrigger.RouteUniqueName);
+
                 var routeCluster = new Cluster
                 {
                     Name = string.Format(_serviceStableWithHeaderWithPortsFormatString, podTrigger.RouteOnHeader.Key, podTrigger.RouteOnHeader.Value, servicePort.Port, servicePort.TargetPort.Value),
                     ConnectTimeout = "1.00s",
-                    Type = "static",
+                    Type = "logical_dns",
                     LoadAssignment = new LoadAssignment
                     {
                         ClusterName = string.Format(_serviceStableWithHeaderWithPortsFormatString, podTrigger.RouteOnHeader.Key, podTrigger.RouteOnHeader.Value, servicePort.Port, servicePort.TargetPort.Value),
@@ -143,7 +148,7 @@ private void ConfigureClusters(V1Service triggerService, IEnumerable<PodTriggerC
                                                 {
                                                     SocketAddress = new SocketAddress
                                                     {
-                                                        Address = podTrigger.TriggerPodIp,
+                                                        Address = $"{destinationService.Metadata.Name}.{destinationService.Metadata.NamespaceProperty}",
                                                         // envoy listens on the target port and sends the request on the target port itself
                                                         // because we are directly sending the request to the pod
                                                         PortValue = long.Parse(servicePort.TargetPort.Value)
@@ -204,7 +209,8 @@ private void ConfigureVirtualHostForMatchAllHost(V1Service triggerService, IEnum
                         },
                         Route = new Route
                         {
-                            Cluster = string.Format(_serviceStableWithHeaderWithPortsFormatString, inputPodTrigger.RouteOnHeader.Key, inputPodTrigger.RouteOnHeader.Value, servicePort.Port, servicePort.TargetPort.Value)
+                            Cluster = string.Format(_serviceStableWithHeaderWithPortsFormatString, inputPodTrigger.RouteOnHeader.Key, inputPodTrigger.RouteOnHeader.Value, servicePort.Port, servicePort.TargetPort.Value),
+                            AutoHostRewrite = true,
                         }
                     });
             }
@@ -219,7 +225,8 @@ private void ConfigureVirtualHostForMatchAllHost(V1Service triggerService, IEnum
                     },
                     Route = new Route
                     {
-                        Cluster = string.Format(_serviceCloneWithPortsFormatString, servicePort.Port, servicePort.TargetPort.Value)
+                        Cluster = string.Format(_serviceCloneWithPortsFormatString, servicePort.Port, servicePort.TargetPort.Value),
+                        AutoHostRewrite = true,
                     }
                 });
         }
@@ -289,14 +296,16 @@ private void ConfigureVirtualHostForPrefixedMatchAllHost(V1Service triggerServic
                         // This means input.TriggerService has an ingress and pod trigger both, so we need to route to service_stable for the given domains
                         ingressVirtualHost.Routes.First().Route = new Route
                         {
-                            Cluster = string.Format(_serviceStableWithHeaderWithPortsFormatString, podTrigger.RouteOnHeader.Key, podTrigger.RouteOnHeader.Value, servicePort.Port, servicePort.TargetPort.Value)
+                            Cluster = string.Format(_serviceStableWithHeaderWithPortsFormatString, podTrigger.RouteOnHeader.Key, podTrigger.RouteOnHeader.Value, servicePort.Port, servicePort.TargetPort.Value),
+                            AutoHostRewrite = true,
                         };
                     }
                     else
                     {
                         ingressVirtualHost.Routes.First().Route = new Route
                         {
-                            Cluster = string.Format(_serviceCloneWithPortsFormatString, servicePort.Port, servicePort.TargetPort.Value)
+                            Cluster = string.Format(_serviceCloneWithPortsFormatString, servicePort.Port, servicePort.TargetPort.Value),
+                            AutoHostRewrite = true,
                         };
                     }
 
@@ -385,14 +394,16 @@ private void ConfigureVirtualHostForPrefixedHost(V1Service triggerService, Routi
                         // This means input.TriggerService has an ingress and pod trigger both, so we need to route to service_stable for the given domains
                         ingressVirtualHost.Routes.First().Route = new Route
                         {
-                            Cluster = string.Format(_serviceStableWithHeaderWithPortsFormatString, podTrigger.RouteOnHeader.Key, podTrigger.RouteOnHeader.Value, servicePort.Port, servicePort.TargetPort.Value)
+                            Cluster = string.Format(_serviceStableWithHeaderWithPortsFormatString, podTrigger.RouteOnHeader.Key, podTrigger.RouteOnHeader.Value, servicePort.Port, servicePort.TargetPort.Value),
+                            AutoHostRewrite = true,
                         };
                     }
                     else
                     {
                         ingressVirtualHost.Routes.First().Route = new Route
                         {
-                            Cluster = string.Format(_serviceCloneWithPortsFormatString, servicePort.Port, servicePort.TargetPort.Value)
+                            Cluster = string.Format(_serviceCloneWithPortsFormatString, servicePort.Port, servicePort.TargetPort.Value),
+                            AutoHostRewrite = true,
                         };
                     }
 

src/routingmanager/RoutingManagerApp.cs

@@ -230,7 +230,8 @@ await Task.WhenAll(
                         AddPodTriggersAsync(routingStateEstablisherInputMap, pods, userServices, cancellationToken),
                         AddIngressTriggersAsync(routingStateEstablisherInputMap, userIngresses, userServices, pods, cancellationToken),
                         AddLoadBalancerTriggersAsync(routingStateEstablisherInputMap, userServices, pods, cancellationToken),
-                        AddIngressRouteTriggersAsync(routingStateEstablisherInputMap, userIngressRoutes, userServices, pods, cancellationToken));
+                        AddIngressRouteTriggersAsync(routingStateEstablisherInputMap, userIngressRoutes, userServices, pods, cancellationToken)
+                    );
 
                     _log.Info("Created '{0}' pod triggers, '{1}' ingress triggers, '{2}' ingressRoute triggers and '{3}' load balancer triggers",
                         routingStateEstablisherInputMap.Select(input => input.Value.PodTriggers.Count()).Sum(),
@@ -667,15 +668,18 @@ await WebUtilities.RetryUntilTimeAsync(async _ =>
                         _log.Warning(ex.Message);
                     }
 
-                    var podTriggerToAdd =
-                        new PodTriggerConfig(
-                            namespaceName: triggerService.Metadata.NamespaceProperty,
-                            triggerService: triggerService,
-                            lpkPodName: pod.Metadata.Name,
-                            routeOnHeaderKey: routeOnHeader.headerName,
-                            routeOnHeaderValue: routeOnHeader.headerValue,
-                            triggerPodIP: pod_latest == null ? pod.Status.PodIP : pod_latest.Status.PodIP,
-                            correlationId: correlationId);
+                    var routeUniqueName = pod.Metadata.Labels[Common.Constants.Routing.RouteUniqueName];
+
+                    var podTriggerToAdd = new PodTriggerConfig(
+                        namespaceName: triggerService.Metadata.NamespaceProperty,
+                        triggerService: triggerService,
+                        lpkPodName: pod.Metadata.Name,
+                        routeOnHeaderKey: routeOnHeader.headerName,
+                        routeOnHeaderValue: routeOnHeader.headerValue,
+                        triggerPodIP: pod_latest == null ? pod.Status.PodIP : pod_latest.Status.PodIP,
+                        correlationId: correlationId,
+                        routeUniqueName: routeUniqueName
+                    );
                     routingStateEstablisherInputMap.AddOrUpdateWithTrigger(triggerService, podTriggerToAdd);
                 }
             }

src/routingmanager/RoutingStateEstablisher.cs

@@ -23,6 +23,7 @@
 using System.Text.Json;
 using System.Threading;
 using System.Threading.Tasks;
+using Microsoft.BridgeToKubernetes.Common.DevHostAgent;
 using static Microsoft.BridgeToKubernetes.Common.Constants;
 
 namespace Microsoft.BridgeToKubernetes.RoutingManager
@@ -109,9 +110,14 @@ public async Task<IDictionary<string, string>> RunAsync(IDictionary<V1Service, R
                             {
                                 cancellationToken.ThrowIfCancellationRequested();
                                 _log.Verbose("Namespace '{0}' : Trigger service name : '{1}'", new PII(input.Key.Metadata.NamespaceProperty), new PII(input.Key.Metadata.Name));
+
+                                var destinationServices = GenerateDestinationServices(input);
+                                var envoyConfig =  _envoyConfigBuilder.GetEnvoyConfig(input.Key, input.Value, allPodTriggersInNamespace, destinationServices);
+
+                                expectedServices.AddRange(destinationServices);
                                 expectedServices.Add(GenerateClonedService(input));
                                 expectedDeployments.Add(GenerateEnvoyDeployment(input));
-                                expectedConfigMaps.Add(GenerateEnvoyConfigMap(input, _envoyConfigBuilder.GetEnvoyConfig(input.Key, input.Value, allPodTriggersInNamespace)));
+                                expectedConfigMaps.Add(GenerateEnvoyConfigMap(input, envoyConfig));
                             }
                         },
                         cancellationToken: cancellationToken)
@@ -195,8 +201,8 @@ public async Task<IDictionary<string, string>> RunAsync(IDictionary<V1Service, R
                 }
             }
 
-            if (expectedServices.Count() != expectedDeployments.Count()
-                    || expectedServices.Count() != expectedConfigMaps.Count())
+            if ((expectedServices.Count - inputs.Sum(i => i.Value.PodTriggers.Count)) != expectedDeployments.Count
+                || (expectedServices.Count - inputs.Sum(i => i.Value.PodTriggers.Count)) != expectedConfigMaps.Count)
             {
                 _log.Error("Number of generated envoy resources do not match. ");
                 throw new RoutingException(Resources.FailedToValidateEnvoyResources);
@@ -485,13 +491,13 @@ private V1Deployment GenerateEnvoyDeployment(KeyValuePair<V1Service, RoutingStat
                             {
                                 new V1Container
                                 {
-                                    Name = "envoy",
+                                    Name = "btk-envoy",
                                     Image = Constants.EnvoyImageName,
                                     Command = new List<string> { "/bin/bash" },
                                     Args = new List<string>
                                     {
                                         "-c",
-                                        "touch envoy-logs.txt && /usr/local/bin/envoy --log-path envoy-logs.txt --log-level trace --config-path /etc/envoy/envoy.yaml"
+                                        "touch envoy-logs.txt && /usr/local/bin/envoy --log-path envoy-logs.txt --log-level trace --base-id 3 --config-path /etc/envoy/envoy.yaml"
                                     },
                                     Ports = containerPorts,
                                     VolumeMounts = new List<V1VolumeMount>
@@ -683,6 +689,64 @@ private V1Service GenerateClonedService(KeyValuePair<V1Service, RoutingStateEsta
 
             return clonedService;
         }
+
+        /// <summary>
+        /// Generates services that exposes the target pods with a cluster IP
+        /// </summary>
+        /// <param name="input"></param>
+        private List<V1Service> GenerateDestinationServices(KeyValuePair<V1Service, RoutingStateEstablisherInput> input)
+        {
+            var services = new List<V1Service>();
+
+            foreach (var pod in input.Value.PodTriggers)
+            {
+                var service = new V1Service
+                {
+                    Metadata = new V1ObjectMeta
+                    {
+                        Name = pod.RouteUniqueName,
+                        NamespaceProperty = pod.NamespaceName,
+                        Labels = new Dictionary<string, string>
+                        {
+                            [Routing.GeneratedLabel] = "true",
+                        },
+                    },
+                    Spec = new V1ServiceSpec
+                    {
+                        Selector = new Dictionary<string, string>
+                        {
+                            [Routing.RouteUniqueName] = pod.RouteUniqueName,
+                        },
+                        Type = "ClusterIP",
+                        Ports = new List<V1ServicePort>(),
+                    },
+                };
+
+                foreach (var port in input.Key.Spec.Ports)
+                {
+                    service.Spec.Ports.Add(new V1ServicePort
+                    {
+                        AppProtocol = port.AppProtocol,
+                        Name = port.Name,
+                        Port = port.Port,
+                        Protocol = port.Protocol,
+                        TargetPort = port.TargetPort,
+                    });
+                }
+
+                service.Spec.Ports.Add(new V1ServicePort
+                {
+                    Name = "devhostagent",
+                    Port = DevHostConstants.DevHostAgent.Port,
+                    Protocol = "TCP",
+                    TargetPort = DevHostConstants.DevHostAgent.Port,
+                });
+
+                services.Add(service);
+            }
+
+            return services;
+        }
 
         /// <summary>
         /// Update the cluster with the expected objects