diff --git a/charts/nginx-gateway-fabric/templates/deployment.yaml b/charts/nginx-gateway-fabric/templates/deployment.yaml index 107c258990..6ce6240c29 100644 --- a/charts/nginx-gateway-fabric/templates/deployment.yaml +++ b/charts/nginx-gateway-fabric/templates/deployment.yaml @@ -129,6 +129,8 @@ spec: volumeMounts: - name: nginx-conf mountPath: /etc/nginx/conf.d + - name: nginx-stream-conf + mountPath: /etc/nginx/stream-conf.d - name: module-includes mountPath: /etc/nginx/module-includes - name: nginx-secrets @@ -166,6 +168,8 @@ spec: volumeMounts: - name: nginx-conf mountPath: /etc/nginx/conf.d + - name: nginx-stream-conf + mountPath: /etc/nginx/stream-conf.d - name: module-includes mountPath: /etc/nginx/module-includes - name: nginx-secrets @@ -200,6 +204,8 @@ spec: volumes: - name: nginx-conf emptyDir: {} + - name: nginx-stream-conf + emptyDir: {} - name: module-includes emptyDir: {} - name: nginx-secrets diff --git a/config/tests/static-deployment.yaml b/config/tests/static-deployment.yaml index 73ad539084..bb2fb62765 100644 --- a/config/tests/static-deployment.yaml +++ b/config/tests/static-deployment.yaml @@ -72,6 +72,8 @@ spec: volumeMounts: - name: nginx-conf mountPath: /etc/nginx/conf.d + - name: nginx-stream-conf + mountPath: /etc/nginx/stream-conf.d - name: module-includes mountPath: /etc/nginx/module-includes - name: nginx-secrets @@ -102,6 +104,8 @@ spec: volumeMounts: - name: nginx-conf mountPath: /etc/nginx/conf.d + - name: nginx-stream-conf + mountPath: /etc/nginx/stream-conf.d - name: module-includes mountPath: /etc/nginx/module-includes - name: nginx-secrets @@ -121,6 +125,8 @@ spec: volumes: - name: nginx-conf emptyDir: {} + - name: nginx-stream-conf + emptyDir: {} - name: module-includes emptyDir: {} - name: nginx-secrets diff --git a/deploy/aws-nlb/deploy.yaml b/deploy/aws-nlb/deploy.yaml index 00839c0396..49b29bf988 100644 --- a/deploy/aws-nlb/deploy.yaml +++ b/deploy/aws-nlb/deploy.yaml @@ -246,6 +246,8 @@ spec: volumeMounts: - mountPath: /etc/nginx/conf.d name: nginx-conf + - mountPath: /etc/nginx/stream-conf.d + name: nginx-stream-conf - mountPath: /etc/nginx/module-includes name: module-includes - mountPath: /etc/nginx/secrets @@ -276,6 +278,8 @@ spec: volumeMounts: - mountPath: /etc/nginx/conf.d name: nginx-conf + - mountPath: /etc/nginx/stream-conf.d + name: nginx-stream-conf - mountPath: /etc/nginx/module-includes name: module-includes - mountPath: /etc/nginx/secrets @@ -295,6 +299,8 @@ spec: volumes: - emptyDir: {} name: nginx-conf + - emptyDir: {} + name: nginx-stream-conf - emptyDir: {} name: module-includes - emptyDir: {} diff --git a/deploy/azure/deploy.yaml b/deploy/azure/deploy.yaml index 5cfbec8b65..968c1a2926 100644 --- a/deploy/azure/deploy.yaml +++ b/deploy/azure/deploy.yaml @@ -243,6 +243,8 @@ spec: volumeMounts: - mountPath: /etc/nginx/conf.d name: nginx-conf + - mountPath: /etc/nginx/stream-conf.d + name: nginx-stream-conf - mountPath: /etc/nginx/module-includes name: module-includes - mountPath: /etc/nginx/secrets @@ -273,6 +275,8 @@ spec: volumeMounts: - mountPath: /etc/nginx/conf.d name: nginx-conf + - mountPath: /etc/nginx/stream-conf.d + name: nginx-stream-conf - mountPath: /etc/nginx/module-includes name: module-includes - mountPath: /etc/nginx/secrets @@ -294,6 +298,8 @@ spec: volumes: - emptyDir: {} name: nginx-conf + - emptyDir: {} + name: nginx-stream-conf - emptyDir: {} name: module-includes - emptyDir: {} diff --git a/deploy/default/deploy.yaml b/deploy/default/deploy.yaml index 7347443192..6245a2bbc7 100644 --- a/deploy/default/deploy.yaml +++ b/deploy/default/deploy.yaml @@ -243,6 +243,8 @@ spec: volumeMounts: - mountPath: /etc/nginx/conf.d name: nginx-conf + - mountPath: /etc/nginx/stream-conf.d + name: nginx-stream-conf - mountPath: /etc/nginx/module-includes name: module-includes - mountPath: /etc/nginx/secrets @@ -273,6 +275,8 @@ spec: volumeMounts: - mountPath: /etc/nginx/conf.d name: nginx-conf + - mountPath: /etc/nginx/stream-conf.d + name: nginx-stream-conf - mountPath: /etc/nginx/module-includes name: module-includes - mountPath: /etc/nginx/secrets @@ -292,6 +296,8 @@ spec: volumes: - emptyDir: {} name: nginx-conf + - emptyDir: {} + name: nginx-stream-conf - emptyDir: {} name: module-includes - emptyDir: {} diff --git a/deploy/experimental-nginx-plus/deploy.yaml b/deploy/experimental-nginx-plus/deploy.yaml index 2a850aa19a..e6cb4a795e 100644 --- a/deploy/experimental-nginx-plus/deploy.yaml +++ b/deploy/experimental-nginx-plus/deploy.yaml @@ -256,6 +256,8 @@ spec: volumeMounts: - mountPath: /etc/nginx/conf.d name: nginx-conf + - mountPath: /etc/nginx/stream-conf.d + name: nginx-stream-conf - mountPath: /etc/nginx/module-includes name: module-includes - mountPath: /etc/nginx/secrets @@ -286,6 +288,8 @@ spec: volumeMounts: - mountPath: /etc/nginx/conf.d name: nginx-conf + - mountPath: /etc/nginx/stream-conf.d + name: nginx-stream-conf - mountPath: /etc/nginx/module-includes name: module-includes - mountPath: /etc/nginx/secrets @@ -305,6 +309,8 @@ spec: volumes: - emptyDir: {} name: nginx-conf + - emptyDir: {} + name: nginx-stream-conf - emptyDir: {} name: module-includes - emptyDir: {} diff --git a/deploy/experimental/deploy.yaml b/deploy/experimental/deploy.yaml index 5cd1c2b0bb..40c7ad96f6 100644 --- a/deploy/experimental/deploy.yaml +++ b/deploy/experimental/deploy.yaml @@ -247,6 +247,8 @@ spec: volumeMounts: - mountPath: /etc/nginx/conf.d name: nginx-conf + - mountPath: /etc/nginx/stream-conf.d + name: nginx-stream-conf - mountPath: /etc/nginx/module-includes name: module-includes - mountPath: /etc/nginx/secrets @@ -277,6 +279,8 @@ spec: volumeMounts: - mountPath: /etc/nginx/conf.d name: nginx-conf + - mountPath: /etc/nginx/stream-conf.d + name: nginx-stream-conf - mountPath: /etc/nginx/module-includes name: module-includes - mountPath: /etc/nginx/secrets @@ -296,6 +300,8 @@ spec: volumes: - emptyDir: {} name: nginx-conf + - emptyDir: {} + name: nginx-stream-conf - emptyDir: {} name: module-includes - emptyDir: {} diff --git a/deploy/nginx-plus/deploy.yaml b/deploy/nginx-plus/deploy.yaml index 9c6a4bd132..76249e80c2 100644 --- a/deploy/nginx-plus/deploy.yaml +++ b/deploy/nginx-plus/deploy.yaml @@ -254,6 +254,8 @@ spec: volumeMounts: - mountPath: /etc/nginx/conf.d name: nginx-conf + - mountPath: /etc/nginx/stream-conf.d + name: nginx-stream-conf - mountPath: /etc/nginx/module-includes name: module-includes - mountPath: /etc/nginx/secrets @@ -284,6 +286,8 @@ spec: volumeMounts: - mountPath: /etc/nginx/conf.d name: nginx-conf + - mountPath: /etc/nginx/stream-conf.d + name: nginx-stream-conf - mountPath: /etc/nginx/module-includes name: module-includes - mountPath: /etc/nginx/secrets @@ -303,6 +307,8 @@ spec: volumes: - emptyDir: {} name: nginx-conf + - emptyDir: {} + name: nginx-stream-conf - emptyDir: {} name: module-includes - emptyDir: {} diff --git a/deploy/nodeport/deploy.yaml b/deploy/nodeport/deploy.yaml index 4f9b78acde..db81fdf259 100644 --- a/deploy/nodeport/deploy.yaml +++ b/deploy/nodeport/deploy.yaml @@ -243,6 +243,8 @@ spec: volumeMounts: - mountPath: /etc/nginx/conf.d name: nginx-conf + - mountPath: /etc/nginx/stream-conf.d + name: nginx-stream-conf - mountPath: /etc/nginx/module-includes name: module-includes - mountPath: /etc/nginx/secrets @@ -273,6 +275,8 @@ spec: volumeMounts: - mountPath: /etc/nginx/conf.d name: nginx-conf + - mountPath: /etc/nginx/stream-conf.d + name: nginx-stream-conf - mountPath: /etc/nginx/module-includes name: module-includes - mountPath: /etc/nginx/secrets @@ -292,6 +296,8 @@ spec: volumes: - emptyDir: {} name: nginx-conf + - emptyDir: {} + name: nginx-stream-conf - emptyDir: {} name: module-includes - emptyDir: {} diff --git a/deploy/openshift/deploy.yaml b/deploy/openshift/deploy.yaml index 213cedcb55..cb78ce0f39 100644 --- a/deploy/openshift/deploy.yaml +++ b/deploy/openshift/deploy.yaml @@ -251,6 +251,8 @@ spec: volumeMounts: - mountPath: /etc/nginx/conf.d name: nginx-conf + - mountPath: /etc/nginx/stream-conf.d + name: nginx-stream-conf - mountPath: /etc/nginx/module-includes name: module-includes - mountPath: /etc/nginx/secrets @@ -281,6 +283,8 @@ spec: volumeMounts: - mountPath: /etc/nginx/conf.d name: nginx-conf + - mountPath: /etc/nginx/stream-conf.d + name: nginx-stream-conf - mountPath: /etc/nginx/module-includes name: module-includes - mountPath: /etc/nginx/secrets @@ -300,6 +304,8 @@ spec: volumes: - emptyDir: {} name: nginx-conf + - emptyDir: {} + name: nginx-stream-conf - emptyDir: {} name: module-includes - emptyDir: {} diff --git a/internal/mode/static/nginx/conf/nginx-plus.conf b/internal/mode/static/nginx/conf/nginx-plus.conf index d4499652d1..6006b5c484 100644 --- a/internal/mode/static/nginx/conf/nginx-plus.conf +++ b/internal/mode/static/nginx/conf/nginx-plus.conf @@ -54,6 +54,21 @@ http { } } +stream { + variables_hash_bucket_size 512; + variables_hash_max_size 1024; + + map_hash_max_size 2048; + map_hash_bucket_size 256; + + log_format stream-main '$remote_addr [$time_local] ' + '$protocol $status $bytes_sent $bytes_received ' + '$session_time "$ssl_preread_server_name"'; + access_log /dev/stdout stream-main; + + include /etc/nginx/stream-conf.d/*.conf; +} + mgmt { usage_report interval=0s; } diff --git a/internal/mode/static/nginx/conf/nginx.conf b/internal/mode/static/nginx/conf/nginx.conf index c253b641eb..681962f17c 100644 --- a/internal/mode/static/nginx/conf/nginx.conf +++ b/internal/mode/static/nginx/conf/nginx.conf @@ -38,3 +38,18 @@ http { } } } + +stream { + variables_hash_bucket_size 512; + variables_hash_max_size 1024; + + map_hash_max_size 2048; + map_hash_bucket_size 256; + + log_format stream-main '$remote_addr [$time_local] ' + '$protocol $status $bytes_sent $bytes_received ' + '$session_time "$ssl_preread_server_name"'; + access_log /dev/stdout stream-main; + + include /etc/nginx/stream-conf.d/*.conf; +} diff --git a/internal/mode/static/nginx/config/base_http_config_template.go b/internal/mode/static/nginx/config/base_http_config_template.go index a909001ab6..e5f20e48e2 100644 --- a/internal/mode/static/nginx/config/base_http_config_template.go +++ b/internal/mode/static/nginx/config/base_http_config_template.go @@ -2,4 +2,20 @@ package config const baseHTTPTemplateText = ` {{- if .HTTP2 }}http2 on;{{ end }} + +# Set $gw_api_compliant_host variable to the value of $http_host unless $http_host is empty, then set it to the value +# of $host. We prefer $http_host because it contains the original value of the host header, which is required by the +# Gateway API. However, in an HTTP/1.0 request, it's possible that $http_host can be empty. In this case, we will use +# the value of $host. See http://nginx.org/en/docs/http/ngx_http_core_module.html#var_host. +map $http_host $gw_api_compliant_host { + '' $host; + default $http_host; +} + +# Set $connection_header variable to upgrade when the $http_upgrade header is set, otherwise, set it to close. This +# allows support for websocket connections. See https://nginx.org/en/docs/http/websocket.html. +map $http_upgrade $connection_upgrade { + default upgrade; + '' close; +} ` diff --git a/internal/mode/static/nginx/config/base_http_config_test.go b/internal/mode/static/nginx/config/base_http_config_test.go index 4eb202ba1b..151d3283b3 100644 --- a/internal/mode/static/nginx/config/base_http_config_test.go +++ b/internal/mode/static/nginx/config/base_http_config_test.go @@ -47,5 +47,7 @@ func TestExecuteBaseHttp(t *testing.T) { res := executeBaseHTTPConfig(test.conf) g.Expect(res).To(HaveLen(1)) g.Expect(test.expCount).To(Equal(strings.Count(string(res[0].data), expSubStr))) + g.Expect(strings.Count(string(res[0].data), "map $http_host $gw_api_compliant_host {")).To(Equal(1)) + g.Expect(strings.Count(string(res[0].data), "map $http_upgrade $connection_upgrade {")).To(Equal(1)) } } diff --git a/internal/mode/static/nginx/config/generator.go b/internal/mode/static/nginx/config/generator.go index a0509194e0..688ebe8353 100644 --- a/internal/mode/static/nginx/config/generator.go +++ b/internal/mode/static/nginx/config/generator.go @@ -17,6 +17,9 @@ const ( // httpFolder is the folder where NGINX HTTP configuration files are stored. httpFolder = configFolder + "/conf.d" + // streamFolder is the folder where NGINX Stream configuration files are stored. + streamFolder = configFolder + "/stream-conf.d" + // modulesIncludesFolder is the folder where the included "load_module" file is stored. modulesIncludesFolder = configFolder + "/module-includes" @@ -29,6 +32,9 @@ const ( // httpConfigFile is the path to the configuration file with HTTP configuration. httpConfigFile = httpFolder + "/http.conf" + // streamConfigFile is the path to the configuration file with Stream configuration. + streamConfigFile = streamFolder + "/stream.conf" + // configVersionFile is the path to the config version configuration file. configVersionFile = httpFolder + "/config-version.conf" @@ -40,7 +46,7 @@ const ( ) // ConfigFolders is a list of folders where NGINX configuration files are stored. -var ConfigFolders = []string{httpFolder, secretsFolder, includesFolder, modulesIncludesFolder} +var ConfigFolders = []string{httpFolder, secretsFolder, includesFolder, modulesIncludesFolder, streamFolder} // Generator generates NGINX configuration files. // This interface is used for testing purposes only. @@ -157,6 +163,9 @@ func (g GeneratorImpl) getExecuteFuncs() []executeFunc { executeSplitClients, executeMaps, executeTelemetry, + executeStreamServers, + g.executeStreamUpstreams, + executeStreamMaps, } } diff --git a/internal/mode/static/nginx/config/generator_test.go b/internal/mode/static/nginx/config/generator_test.go index d4e6a5f841..67311f4223 100644 --- a/internal/mode/static/nginx/config/generator_test.go +++ b/internal/mode/static/nginx/config/generator_test.go @@ -47,12 +47,25 @@ func TestGenerate(t *testing.T) { Port: 443, }, }, + TLSPassthroughServers: []dataplane.Layer4VirtualServer{ + { + Hostname: "app.example.com", + Port: 443, + UpstreamName: "stream_up", + }, + }, Upstreams: []dataplane.Upstream{ { Name: "up", Endpoints: nil, }, }, + StreamUpstreams: []dataplane.Upstream{ + { + Name: "stream_up", + Endpoints: nil, + }, + }, BackendGroups: []dataplane.BackendGroup{bg}, SSLKeyPairs: map[dataplane.SSLKeyPairID]dataplane.SSLKeyPair{ "test-keypair": { @@ -81,7 +94,7 @@ func TestGenerate(t *testing.T) { files := generator.Generate(conf) - g.Expect(files).To(HaveLen(6)) + g.Expect(files).To(HaveLen(7)) arrange := func(i, j int) bool { return files[i].Path < files[j].Path } @@ -98,7 +111,7 @@ func TestGenerate(t *testing.T) { // Note: this only verifies that Generate() returns a byte array with upstream, server, and split_client blocks. // It does not test the correctness of those blocks. That functionality is covered by other tests in this package. g.Expect(httpCfg).To(ContainSubstring("listen 80")) - g.Expect(httpCfg).To(ContainSubstring("listen 443")) + g.Expect(httpCfg).To(ContainSubstring("listen unix:/var/run/nginx/https443.sock")) g.Expect(httpCfg).To(ContainSubstring("upstream")) g.Expect(httpCfg).To(ContainSubstring("split_clients")) @@ -127,4 +140,12 @@ func TestGenerate(t *testing.T) { Path: "/etc/nginx/secrets/test-keypair.pem", Content: []byte("test-cert\ntest-key"), })) + + g.Expect(files[6].Path).To(Equal("/etc/nginx/stream-conf.d/stream.conf")) + g.Expect(files[6].Type).To(Equal(file.TypeRegular)) + streamCfg := string(files[6].Content) + g.Expect(streamCfg).To(ContainSubstring("listen unix:/var/run/nginx/app.example.com-443.sock")) + g.Expect(streamCfg).To(ContainSubstring("listen 443")) + g.Expect(streamCfg).To(ContainSubstring("app.example.com unix:/var/run/nginx/app.example.com-443.sock")) + g.Expect(streamCfg).To(ContainSubstring("example.com unix:/var/run/nginx/https443.sock")) } diff --git a/internal/mode/static/nginx/config/http/config.go b/internal/mode/static/nginx/config/http/config.go index 9326ebb439..4e9e4f2f85 100644 --- a/internal/mode/static/nginx/config/http/config.go +++ b/internal/mode/static/nginx/config/http/config.go @@ -4,9 +4,9 @@ package http type Server struct { SSL *SSL ServerName string + Listen string Locations []Location Includes []string - Port int32 IsDefaultHTTP bool IsDefaultSSL bool GRPC bool @@ -94,19 +94,6 @@ type SplitClientDistribution struct { Value string } -// Map defines an NGINX map. -type Map struct { - Source string - Variable string - Parameters []MapParameter -} - -// Parameter defines a Value and Result pair in a Map. -type MapParameter struct { - Value string - Result string -} - // ProxySSLVerify holds the proxied HTTPS server verification configuration. type ProxySSLVerify struct { TrustedCertificate string diff --git a/internal/mode/static/nginx/config/maps.go b/internal/mode/static/nginx/config/maps.go index a784390170..703b2b7b04 100644 --- a/internal/mode/static/nginx/config/maps.go +++ b/internal/mode/static/nginx/config/maps.go @@ -5,12 +5,19 @@ import ( gotemplate "text/template" "github.com/nginxinc/nginx-gateway-fabric/internal/framework/helpers" - "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/nginx/config/http" + "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/nginx/config/shared" "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/state/dataplane" ) var mapsTemplate = gotemplate.Must(gotemplate.New("maps").Parse(mapsTemplateText)) +// emptyStringSocket is used when the stream server has an invalid upstream. In this case, we pass the connection +// to the empty socket so that NGINX will close the connection with an error in the error log -- +// no host in pass "" -- and set $status variable to 500 (logged by stream access log), +// which will indicate the problem to the user. +// https://nginx.org/en/docs/stream/ngx_stream_core_module.html#var_status +const emptyStringSocket = `""` + func executeMaps(conf dataplane.Configuration) []executeResult { maps := buildAddHeaderMaps(append(conf.HTTPServers, conf.SSLServers...)) result := executeResult{ @@ -21,7 +28,81 @@ func executeMaps(conf dataplane.Configuration) []executeResult { return []executeResult{result} } -func buildAddHeaderMaps(servers []dataplane.VirtualServer) []http.Map { +func executeStreamMaps(conf dataplane.Configuration) []executeResult { + maps := createStreamMaps(conf) + + result := executeResult{ + dest: streamConfigFile, + data: helpers.MustExecuteTemplate(mapsTemplate, maps), + } + + return []executeResult{result} +} + +func createStreamMaps(conf dataplane.Configuration) []shared.Map { + if len(conf.TLSPassthroughServers) == 0 { + return nil + } + portsToMap := make(map[int32]shared.Map) + + for _, server := range conf.TLSPassthroughServers { + streamMap, portInUse := portsToMap[server.Port] + + socket := emptyStringSocket + + if server.UpstreamName != "" { + socket = getSocketNameTLS(server.Port, server.Hostname) + } + + mapParam := shared.MapParameter{ + Value: server.Hostname, + Result: socket, + } + + if !portInUse { + m := shared.Map{ + Source: "$ssl_preread_server_name", + Variable: getTLSPassthroughVarName(server.Port), + Parameters: []shared.MapParameter{ + mapParam, + }, + UseHostnames: true, + } + portsToMap[server.Port] = m + } else { + streamMap.Parameters = append(streamMap.Parameters, mapParam) + portsToMap[server.Port] = streamMap + } + } + + for _, server := range conf.SSLServers { + streamMap, portInUse := portsToMap[server.Port] + + hostname := server.Hostname + + if server.IsDefault { + hostname = "default" + } + + if portInUse { + streamMap.Parameters = append(streamMap.Parameters, shared.MapParameter{ + Value: hostname, + Result: getSocketNameHTTPS(server.Port), + }) + portsToMap[server.Port] = streamMap + } + } + + maps := make([]shared.Map, 0, len(portsToMap)) + + for _, m := range portsToMap { + maps = append(maps, m) + } + + return maps +} + +func buildAddHeaderMaps(servers []dataplane.VirtualServer) []shared.Map { addHeaderNames := make(map[string]struct{}) for _, s := range servers { @@ -39,7 +120,7 @@ func buildAddHeaderMaps(servers []dataplane.VirtualServer) []http.Map { } } - maps := make([]http.Map, 0, len(addHeaderNames)) + maps := make([]shared.Map, 0, len(addHeaderNames)) for m := range addHeaderNames { maps = append(maps, createAddHeadersMap(m)) } @@ -52,11 +133,11 @@ const ( anyStringFmt = `~.*` ) -func createAddHeadersMap(name string) http.Map { +func createAddHeadersMap(name string) shared.Map { underscoreName := convertStringToSafeVariableName(name) httpVarSource := "${http_" + underscoreName + "}" mapVarName := generateAddHeaderMapVariableName(name) - params := []http.MapParameter{ + params := []shared.MapParameter{ { Value: "default", Result: "''", @@ -66,7 +147,7 @@ func createAddHeadersMap(name string) http.Map { Result: httpVarSource + ",", }, } - return http.Map{ + return shared.Map{ Source: httpVarSource, Variable: "$" + mapVarName, Parameters: params, diff --git a/internal/mode/static/nginx/config/maps_template.go b/internal/mode/static/nginx/config/maps_template.go index 4b4407a1a4..00604410c1 100644 --- a/internal/mode/static/nginx/config/maps_template.go +++ b/internal/mode/static/nginx/config/maps_template.go @@ -3,25 +3,13 @@ package config const mapsTemplateText = ` {{ range $m := . }} map {{ $m.Source }} {{ $m.Variable }} { - {{ range $p := $m.Parameters }} - {{ $p.Value }} {{ $p.Result }}; - {{ end }} -} -{{- end }} + {{- if $m.UseHostnames -}} + hostnames; + {{ end }} -# Set $gw_api_compliant_host variable to the value of $http_host unless $http_host is empty, then set it to the value -# of $host. We prefer $http_host because it contains the original value of the host header, which is required by the -# Gateway API. However, in an HTTP/1.0 request, it's possible that $http_host can be empty. In this case, we will use -# the value of $host. See http://nginx.org/en/docs/http/ngx_http_core_module.html#var_host. -map $http_host $gw_api_compliant_host { - '' $host; - default $http_host; -} - -# Set $connection_header variable to upgrade when the $http_upgrade header is set, otherwise, set it to close. This -# allows support for websocket connections. See https://nginx.org/en/docs/http/websocket.html. -map $http_upgrade $connection_upgrade { - default upgrade; - '' close; + {{ range $p := $m.Parameters }} + {{ $p.Value }} {{ $p.Result }}; + {{ end }} } +{{- end }} ` diff --git a/internal/mode/static/nginx/config/maps_test.go b/internal/mode/static/nginx/config/maps_test.go index e903c17151..96a3c8381a 100644 --- a/internal/mode/static/nginx/config/maps_test.go +++ b/internal/mode/static/nginx/config/maps_test.go @@ -6,7 +6,7 @@ import ( . "github.com/onsi/gomega" - "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/nginx/config/http" + "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/nginx/config/shared" "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/state/dataplane" ) @@ -84,8 +84,6 @@ func TestExecuteMaps(t *testing.T) { "map ${http_my_second_add_header} $my_second_add_header_header_var {": 1, "~.* ${http_my_second_add_header},;": 1, "map ${http_my_set_header} $my_set_header_header_var {": 0, - "map $http_host $gw_api_compliant_host {": 1, - "map $http_upgrade $connection_upgrade {": 1, } mapResult := executeMaps(conf) @@ -161,11 +159,11 @@ func TestBuildAddHeaderMaps(t *testing.T) { IsDefault: true, }, } - expectedMap := []http.Map{ + expectedMap := []shared.Map{ { Source: "${http_my_add_header}", Variable: "$my_add_header_header_var", - Parameters: []http.MapParameter{ + Parameters: []shared.MapParameter{ {Value: "default", Result: "''"}, { Value: "~.*", @@ -176,7 +174,7 @@ func TestBuildAddHeaderMaps(t *testing.T) { { Source: "${http_my_second_add_header}", Variable: "$my_second_add_header_header_var", - Parameters: []http.MapParameter{ + Parameters: []shared.MapParameter{ {Value: "default", Result: "''"}, { Value: "~.*", @@ -189,3 +187,125 @@ func TestBuildAddHeaderMaps(t *testing.T) { g.Expect(maps).To(ConsistOf(expectedMap)) } + +func TestExecuteStreamMaps(t *testing.T) { + g := NewWithT(t) + conf := dataplane.Configuration{ + TLSPassthroughServers: []dataplane.Layer4VirtualServer{ + { + Hostname: "example.com", + Port: 8081, + UpstreamName: "backend1", + }, + { + Hostname: "example.com", + Port: 8080, + UpstreamName: "backend1", + }, + { + Hostname: "cafe.example.com", + Port: 8080, + UpstreamName: "backend2", + }, + }, + SSLServers: []dataplane.VirtualServer{ + { + Hostname: "app.example.com", + Port: 8080, + }, + }, + } + + expSubStrings := map[string]int{ + "example.com unix:/var/run/nginx/example.com-8081.sock;": 1, + "example.com unix:/var/run/nginx/example.com-8080.sock;": 1, + "cafe.example.com unix:/var/run/nginx/cafe.example.com-8080.sock;": 1, + "app.example.com unix:/var/run/nginx/https8080.sock;": 1, + "hostnames": 2, + } + + results := executeStreamMaps(conf) + g.Expect(results).To(HaveLen(1)) + result := results[0] + + g.Expect(result.dest).To(Equal(streamConfigFile)) + for expSubStr, expCount := range expSubStrings { + g.Expect(strings.Count(string(result.data), expSubStr)).To(Equal(expCount)) + } +} + +func TestCreateStreamMaps(t *testing.T) { + g := NewWithT(t) + conf := dataplane.Configuration{ + TLSPassthroughServers: []dataplane.Layer4VirtualServer{ + { + Hostname: "example.com", + Port: 8081, + UpstreamName: "backend1", + }, + { + Hostname: "example.com", + Port: 8080, + UpstreamName: "backend1", + }, + { + Hostname: "cafe.example.com", + Port: 8080, + UpstreamName: "backend2", + }, + { + Hostname: "wrong.example.com", + Port: 8080, + UpstreamName: "", + }, + }, + SSLServers: []dataplane.VirtualServer{ + { + Hostname: "app.example.com", + Port: 8080, + }, + { + Port: 8080, + IsDefault: true, + }, + }, + } + + maps := createStreamMaps(conf) + + expectedMaps := []shared.Map{ + { + Source: "$ssl_preread_server_name", + Variable: getTLSPassthroughVarName(8081), + Parameters: []shared.MapParameter{ + {Value: "example.com", Result: getSocketNameTLS(8081, "example.com")}, + }, + UseHostnames: true, + }, + { + Source: "$ssl_preread_server_name", + Variable: getTLSPassthroughVarName(8080), + Parameters: []shared.MapParameter{ + {Value: "example.com", Result: getSocketNameTLS(8080, "example.com")}, + {Value: "cafe.example.com", Result: getSocketNameTLS(8080, "cafe.example.com")}, + {Value: "wrong.example.com", Result: `""`}, + {Value: "app.example.com", Result: getSocketNameHTTPS(8080)}, + {Value: "default", Result: getSocketNameHTTPS(8080)}, + }, + UseHostnames: true, + }, + } + + g.Expect(maps).To(ConsistOf(expectedMaps)) +} + +func TestCreateStreamMapsWithEmpty(t *testing.T) { + g := NewWithT(t) + conf := dataplane.Configuration{ + TLSPassthroughServers: nil, + } + + maps := createStreamMaps(conf) + + g.Expect(maps).To(BeNil()) +} diff --git a/internal/mode/static/nginx/config/servers.go b/internal/mode/static/nginx/config/servers.go index 3aeefa47c7..b239e378ed 100644 --- a/internal/mode/static/nginx/config/servers.go +++ b/internal/mode/static/nginx/config/servers.go @@ -58,7 +58,7 @@ var grpcBaseHeaders = []http.Header{ } func executeServers(conf dataplane.Configuration) []executeResult { - servers, httpMatchPairs := createServers(conf.HTTPServers, conf.SSLServers) + servers, httpMatchPairs := createServers(conf.HTTPServers, conf.SSLServers, conf.TLSPassthroughServers) serverConfig := http.ServerConfig{ Servers: servers, @@ -158,10 +158,20 @@ func createIncludes(additions []dataplane.Addition) []string { return includes } -func createServers(httpServers, sslServers []dataplane.VirtualServer) ([]http.Server, httpMatchPairs) { +func createServers( + httpServers, + sslServers []dataplane.VirtualServer, + tlsPassthroughServers []dataplane.Layer4VirtualServer, +) ([]http.Server, httpMatchPairs) { servers := make([]http.Server, 0, len(httpServers)+len(sslServers)) finalMatchPairs := make(httpMatchPairs) + sharedTLSPorts := make(map[int32]struct{}) + + for _, passthroughServer := range tlsPassthroughServers { + sharedTLSPorts[passthroughServer.Port] = struct{}{} + } + for serverID, s := range httpServers { httpServer, matchPairs := createServer(s, serverID) servers = append(servers, httpServer) @@ -169,7 +179,12 @@ func createServers(httpServers, sslServers []dataplane.VirtualServer) ([]http.Se } for serverID, s := range sslServers { - sslServer, matchPair := createSSLServer(s, serverID) + listen := fmt.Sprint(s.Port) + + if _, portInUse := sharedTLSPorts[s.Port]; portInUse { + listen = getSocketNameHTTPS(s.Port) + } + sslServer, matchPair := createSSLServer(s, serverID, listen) servers = append(servers, sslServer) maps.Copy(finalMatchPairs, matchPair) } @@ -177,11 +192,15 @@ func createServers(httpServers, sslServers []dataplane.VirtualServer) ([]http.Se return servers, finalMatchPairs } -func createSSLServer(virtualServer dataplane.VirtualServer, serverID int) (http.Server, httpMatchPairs) { +func createSSLServer( + virtualServer dataplane.VirtualServer, + serverID int, + listen string, +) (http.Server, httpMatchPairs) { if virtualServer.IsDefault { return http.Server{ IsDefaultSSL: true, - Port: virtualServer.Port, + Listen: listen, }, nil } @@ -194,17 +213,19 @@ func createSSLServer(virtualServer dataplane.VirtualServer, serverID int) (http. CertificateKey: generatePEMFileName(virtualServer.SSL.KeyPairID), }, Locations: locs, - Port: virtualServer.Port, GRPC: grpc, Includes: createIncludes(virtualServer.Additions), + Listen: listen, }, matchPairs } func createServer(virtualServer dataplane.VirtualServer, serverID int) (http.Server, httpMatchPairs) { + listen := fmt.Sprint(virtualServer.Port) + if virtualServer.IsDefault { return http.Server{ IsDefaultHTTP: true, - Port: virtualServer.Port, + Listen: listen, }, nil } @@ -213,7 +234,7 @@ func createServer(virtualServer dataplane.VirtualServer, serverID int) (http.Ser return http.Server{ ServerName: virtualServer.Hostname, Locations: locs, - Port: virtualServer.Port, + Listen: listen, GRPC: grpc, Includes: createIncludes(virtualServer.Additions), }, matchPairs diff --git a/internal/mode/static/nginx/config/servers_template.go b/internal/mode/static/nginx/config/servers_template.go index 4d8196b180..3f6afde32f 100644 --- a/internal/mode/static/nginx/config/servers_template.go +++ b/internal/mode/static/nginx/config/servers_template.go @@ -6,10 +6,10 @@ js_preload_object matches from /etc/nginx/conf.d/matches.json; {{ if $s.IsDefaultSSL -}} server { {{- if $.IPFamily.IPv4 }} - listen {{ $s.Port }} ssl default_server; + listen {{ $s.Listen }} ssl default_server; {{- end }} {{- if $.IPFamily.IPv6 }} - listen [::]:{{ $s.Port }} ssl default_server; + listen [::]:{{ $s.Listen }} ssl default_server; {{- end }} ssl_reject_handshake on; @@ -17,10 +17,10 @@ server { {{- else if $s.IsDefaultHTTP }} server { {{- if $.IPFamily.IPv4 }} - listen {{ $s.Port }} default_server; + listen {{ $s.Listen }} default_server; {{- end }} {{- if $.IPFamily.IPv6 }} - listen [::]:{{ $s.Port }} default_server; + listen [::]:{{ $s.Listen }} default_server; {{- end }} default_type text/html; @@ -30,10 +30,10 @@ server { server { {{- if $s.SSL }} {{- if $.IPFamily.IPv4 }} - listen {{ $s.Port }} ssl; + listen {{ $s.Listen }} ssl; {{- end }} {{- if $.IPFamily.IPv6 }} - listen [::]:{{ $s.Port }} ssl; + listen [::]:{{ $s.Listen }} ssl; {{- end }} ssl_certificate {{ $s.SSL.Certificate }}; ssl_certificate_key {{ $s.SSL.CertificateKey }}; @@ -43,10 +43,10 @@ server { } {{- else }} {{- if $.IPFamily.IPv4 }} - listen {{ $s.Port }}; + listen {{ $s.Listen }}; {{- end }} {{- if $.IPFamily.IPv6 }} - listen [::]:{{ $s.Port }}; + listen [::]:{{ $s.Listen }}; {{- end }} {{- end }} diff --git a/internal/mode/static/nginx/config/servers_test.go b/internal/mode/static/nginx/config/servers_test.go index effb0099ab..ec203235a9 100644 --- a/internal/mode/static/nginx/config/servers_test.go +++ b/internal/mode/static/nginx/config/servers_test.go @@ -775,6 +775,14 @@ func TestCreateServers(t *testing.T) { }, } + tlsPassthroughServers := []dataplane.Layer4VirtualServer{ + { + Hostname: "app.example.com", + Port: 8443, + UpstreamName: "sup", + }, + } + expMatchPairs := httpMatchPairs{ "1_0": { {Method: "POST", RedirectPath: "@rule0-route0"}, @@ -1135,12 +1143,12 @@ func TestCreateServers(t *testing.T) { expectedServers := []http.Server{ { IsDefaultHTTP: true, - Port: 8080, + Listen: "8080", }, { ServerName: "cafe.example.com", Locations: getExpectedLocations(false), - Port: 8080, + Listen: "8080", GRPC: true, Includes: []string{ includesFolder + "/server-addition-1.conf", @@ -1149,7 +1157,7 @@ func TestCreateServers(t *testing.T) { }, { IsDefaultSSL: true, - Port: 8443, + Listen: getSocketNameHTTPS(8443), }, { ServerName: "cafe.example.com", @@ -1158,7 +1166,7 @@ func TestCreateServers(t *testing.T) { CertificateKey: expectedPEMPath, }, Locations: getExpectedLocations(true), - Port: 8443, + Listen: getSocketNameHTTPS(8443), GRPC: true, Includes: []string{ includesFolder + "/server-addition-1.conf", @@ -1169,7 +1177,7 @@ func TestCreateServers(t *testing.T) { g := NewWithT(t) - result, httpMatchPair := createServers(httpServers, sslServers) + result, httpMatchPair := createServers(httpServers, sslServers, tlsPassthroughServers) g.Expect(httpMatchPair).To(Equal(allExpMatchPair)) g.Expect(helpers.Diff(expectedServers, result)).To(BeEmpty()) @@ -1367,18 +1375,18 @@ func TestCreateServersConflicts(t *testing.T) { expectedServers := []http.Server{ { IsDefaultHTTP: true, - Port: 8080, + Listen: "8080", }, { ServerName: "cafe.example.com", Locations: test.expLocs, - Port: 8080, + Listen: "8080", }, } g := NewWithT(t) - result, _ := createServers(httpServers, []dataplane.VirtualServer{}) + result, _ := createServers(httpServers, []dataplane.VirtualServer{}, []dataplane.Layer4VirtualServer{}) g.Expect(helpers.Diff(expectedServers, result)).To(BeEmpty()) }) } diff --git a/internal/mode/static/nginx/config/shared/config.go b/internal/mode/static/nginx/config/shared/config.go new file mode 100644 index 0000000000..baab86c73a --- /dev/null +++ b/internal/mode/static/nginx/config/shared/config.go @@ -0,0 +1,15 @@ +package shared + +// Map defines an NGINX map. +type Map struct { + Source string + Variable string + Parameters []MapParameter + UseHostnames bool +} + +// MapParameter defines a Value and Result pair in a Map. +type MapParameter struct { + Value string + Result string +} diff --git a/internal/mode/static/nginx/config/sockets.go b/internal/mode/static/nginx/config/sockets.go new file mode 100644 index 0000000000..9707ef01a8 --- /dev/null +++ b/internal/mode/static/nginx/config/sockets.go @@ -0,0 +1,17 @@ +package config + +import ( + "fmt" +) + +func getSocketNameTLS(port int32, hostname string) string { + return fmt.Sprintf("unix:/var/run/nginx/%s-%d.sock", hostname, port) +} + +func getSocketNameHTTPS(port int32) string { + return fmt.Sprintf("unix:/var/run/nginx/https%d.sock", port) +} + +func getTLSPassthroughVarName(port int32) string { + return fmt.Sprintf("$dest%d", port) +} diff --git a/internal/mode/static/nginx/config/sockets_test.go b/internal/mode/static/nginx/config/sockets_test.go new file mode 100644 index 0000000000..cbab84aea3 --- /dev/null +++ b/internal/mode/static/nginx/config/sockets_test.go @@ -0,0 +1,28 @@ +package config + +import ( + "testing" + + . "github.com/onsi/gomega" +) + +func TestGetSocketNameTLS(t *testing.T) { + res := getSocketNameTLS(800, "*.cafe.example.com") + + g := NewGomegaWithT(t) + g.Expect(res).To(Equal("unix:/var/run/nginx/*.cafe.example.com-800.sock")) +} + +func TestGetSocketNameHTTPS(t *testing.T) { + res := getSocketNameHTTPS(800) + + g := NewGomegaWithT(t) + g.Expect(res).To(Equal("unix:/var/run/nginx/https800.sock")) +} + +func TestGetTLSPassthroughVarName(t *testing.T) { + res := getTLSPassthroughVarName(800) + + g := NewGomegaWithT(t) + g.Expect(res).To(Equal("$dest800")) +} diff --git a/internal/mode/static/nginx/config/stream/config.go b/internal/mode/static/nginx/config/stream/config.go new file mode 100644 index 0000000000..93f16b22cc --- /dev/null +++ b/internal/mode/static/nginx/config/stream/config.go @@ -0,0 +1,21 @@ +package stream + +// Server holds all configuration for a stream server. +type Server struct { + Listen string + ProxyPass string + Pass string + SSLPreread bool +} + +// Upstream holds all configuration for a stream upstream. +type Upstream struct { + Name string + ZoneSize string // format: 512k, 1m + Servers []UpstreamServer +} + +// UpstreamServer holds all configuration for a stream upstream server. +type UpstreamServer struct { + Address string +} diff --git a/internal/mode/static/nginx/config/stream_servers.go b/internal/mode/static/nginx/config/stream_servers.go new file mode 100644 index 0000000000..29f0991cf0 --- /dev/null +++ b/internal/mode/static/nginx/config/stream_servers.go @@ -0,0 +1,56 @@ +package config + +import ( + "fmt" + gotemplate "text/template" + + "github.com/nginxinc/nginx-gateway-fabric/internal/framework/helpers" + "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/nginx/config/stream" + "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/state/dataplane" +) + +var streamServersTemplate = gotemplate.Must(gotemplate.New("streamServers").Parse(streamServersTemplateText)) + +func executeStreamServers(conf dataplane.Configuration) []executeResult { + streamServers := createStreamServers(conf) + + streamServerResult := executeResult{ + dest: streamConfigFile, + data: helpers.MustExecuteTemplate(streamServersTemplate, streamServers), + } + + return []executeResult{ + streamServerResult, + } +} + +func createStreamServers(conf dataplane.Configuration) []stream.Server { + if len(conf.TLSPassthroughServers) == 0 { + return nil + } + + streamServers := make([]stream.Server, 0, len(conf.TLSPassthroughServers)*2) + portSet := make(map[int32]struct{}) + + for _, server := range conf.TLSPassthroughServers { + if server.UpstreamName != "" { + streamServers = append(streamServers, stream.Server{ + Listen: getSocketNameTLS(server.Port, server.Hostname), + ProxyPass: server.UpstreamName, + }) + } + + if _, inPortSet := portSet[server.Port]; inPortSet { + continue + } + + portSet[server.Port] = struct{}{} + streamServers = append(streamServers, stream.Server{ + Listen: fmt.Sprint(server.Port), + Pass: getTLSPassthroughVarName(server.Port), + SSLPreread: true, + }) + } + + return streamServers +} diff --git a/internal/mode/static/nginx/config/stream_servers_template.go b/internal/mode/static/nginx/config/stream_servers_template.go new file mode 100644 index 0000000000..e0e1c00ba8 --- /dev/null +++ b/internal/mode/static/nginx/config/stream_servers_template.go @@ -0,0 +1,21 @@ +package config + +const streamServersTemplateText = ` +{{- range $s := . }} +server { + listen {{ $s.Listen }}; + + {{- if $s.ProxyPass }} + proxy_pass {{ $s.ProxyPass }}; + {{- end }} + + {{- if $s.Pass }} + pass {{ $s.Pass }}; + {{- end }} + + {{- if $s.SSLPreread }} + ssl_preread on; + {{- end }} +} +{{- end }} +` diff --git a/internal/mode/static/nginx/config/stream_servers_test.go b/internal/mode/static/nginx/config/stream_servers_test.go new file mode 100644 index 0000000000..1f6a94d9b7 --- /dev/null +++ b/internal/mode/static/nginx/config/stream_servers_test.go @@ -0,0 +1,123 @@ +package config + +import ( + "fmt" + "strings" + "testing" + + . "github.com/onsi/gomega" + + "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/nginx/config/stream" + "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/state/dataplane" +) + +func TestExecuteStreamServers(t *testing.T) { + conf := dataplane.Configuration{ + TLSPassthroughServers: []dataplane.Layer4VirtualServer{ + { + Hostname: "example.com", + Port: 8081, + UpstreamName: "backend1", + }, + { + Hostname: "example.com", + Port: 8080, + UpstreamName: "backend1", + }, + { + Hostname: "cafe.example.com", + Port: 8080, + UpstreamName: "backend2", + }, + }, + } + + expSubStrings := map[string]int{ + "pass $dest8081;": 1, + "pass $dest8080;": 1, + "ssl_preread on;": 2, + "proxy_pass": 3, + } + g := NewWithT(t) + + results := executeStreamServers(conf) + g.Expect(results).To(HaveLen(1)) + result := results[0] + + g.Expect(result.dest).To(Equal(streamConfigFile)) + for expSubStr, expCount := range expSubStrings { + g.Expect(strings.Count(string(result.data), expSubStr)).To(Equal(expCount)) + } +} + +func TestCreateStreamServers(t *testing.T) { + conf := dataplane.Configuration{ + TLSPassthroughServers: []dataplane.Layer4VirtualServer{ + { + Hostname: "example.com", + Port: 8081, + UpstreamName: "backend1", + }, + { + Hostname: "example.com", + Port: 8080, + UpstreamName: "backend1", + }, + { + Hostname: "cafe.example.com", + Port: 8080, + UpstreamName: "backend2", + }, + { + Hostname: "wrong.example.com", + Port: 8081, + UpstreamName: "", + }, + }, + } + + streamServers := createStreamServers(conf) + + g := NewWithT(t) + + expectedStreamServers := []stream.Server{ + { + Listen: getSocketNameTLS(conf.TLSPassthroughServers[0].Port, conf.TLSPassthroughServers[0].Hostname), + ProxyPass: conf.TLSPassthroughServers[0].UpstreamName, + SSLPreread: false, + }, + { + Listen: getSocketNameTLS(conf.TLSPassthroughServers[1].Port, conf.TLSPassthroughServers[1].Hostname), + ProxyPass: conf.TLSPassthroughServers[1].UpstreamName, + SSLPreread: false, + }, + { + Listen: getSocketNameTLS(conf.TLSPassthroughServers[2].Port, conf.TLSPassthroughServers[2].Hostname), + ProxyPass: conf.TLSPassthroughServers[2].UpstreamName, + SSLPreread: false, + }, + { + Listen: fmt.Sprint(8081), + Pass: getTLSPassthroughVarName(8081), + SSLPreread: true, + }, + { + Listen: fmt.Sprint(8080), + Pass: getTLSPassthroughVarName(8080), + SSLPreread: true, + }, + } + g.Expect(streamServers).To(ConsistOf(expectedStreamServers)) +} + +func TestCreateStreamServersWithNone(t *testing.T) { + conf := dataplane.Configuration{ + TLSPassthroughServers: nil, + } + + streamServers := createStreamServers(conf) + + g := NewWithT(t) + + g.Expect(streamServers).To(BeNil()) +} diff --git a/internal/mode/static/nginx/config/upstreams.go b/internal/mode/static/nginx/config/upstreams.go index a76ee23a6a..f15a89d5d8 100644 --- a/internal/mode/static/nginx/config/upstreams.go +++ b/internal/mode/static/nginx/config/upstreams.go @@ -6,6 +6,7 @@ import ( "github.com/nginxinc/nginx-gateway-fabric/internal/framework/helpers" "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/nginx/config/http" + "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/nginx/config/stream" "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/state/dataplane" ) @@ -22,6 +23,10 @@ const ( ossZoneSize = "512k" // plusZoneSize is the upstream zone size for nginx plus. plusZoneSize = "1m" + // ossZoneSize is the upstream zone size for nginx open source. + ossZoneSizeStream = "512k" + // plusZoneSize is the upstream zone size for nginx plus. + plusZoneSizeStream = "1m" ) func (g GeneratorImpl) executeUpstreams(conf dataplane.Configuration) []executeResult { @@ -35,6 +40,49 @@ func (g GeneratorImpl) executeUpstreams(conf dataplane.Configuration) []executeR return []executeResult{result} } +func (g GeneratorImpl) executeStreamUpstreams(conf dataplane.Configuration) []executeResult { + upstreams := g.createStreamUpstreams(conf.StreamUpstreams) + + result := executeResult{ + dest: streamConfigFile, + data: helpers.MustExecuteTemplate(upstreamsTemplate, upstreams), + } + + return []executeResult{result} +} + +func (g GeneratorImpl) createStreamUpstreams(upstreams []dataplane.Upstream) []stream.Upstream { + ups := make([]stream.Upstream, 0, len(upstreams)) + + for _, u := range upstreams { + if len(u.Endpoints) != 0 { + ups = append(ups, g.createStreamUpstream(u)) + } + } + + return ups +} + +func (g GeneratorImpl) createStreamUpstream(up dataplane.Upstream) stream.Upstream { + zoneSize := ossZoneSizeStream + if g.plus { + zoneSize = plusZoneSizeStream + } + + upstreamServers := make([]stream.UpstreamServer, len(up.Endpoints)) + for idx, ep := range up.Endpoints { + upstreamServers[idx] = stream.UpstreamServer{ + Address: fmt.Sprintf("%s:%d", ep.Address, ep.Port), + } + } + + return stream.Upstream{ + Name: up.Name, + ZoneSize: zoneSize, + Servers: upstreamServers, + } +} + func (g GeneratorImpl) createUpstreams(upstreams []dataplane.Upstream) []http.Upstream { // capacity is the number of upstreams + 1 for the invalid backend ref upstream ups := make([]http.Upstream, 0, len(upstreams)+1) diff --git a/internal/mode/static/nginx/config/upstreams_template.go b/internal/mode/static/nginx/config/upstreams_template.go index fd5130dc2b..a04915bec8 100644 --- a/internal/mode/static/nginx/config/upstreams_template.go +++ b/internal/mode/static/nginx/config/upstreams_template.go @@ -1,8 +1,9 @@ package config // FIXME(kate-osborn): Dynamically calculate upstream zone size based on the number of upstreams. -// 512k will support up to 648 upstream servers for OSS. -// NGINX Plus needs 1m to support roughly the same amount of servers (556 upstream servers). +// 512k will support up to 648 http upstream servers for OSS. +// NGINX Plus needs 1m to support roughly the same amount of http servers (556 upstream servers). +// For stream upstream servers, 512k will support 576 in OSS and 1m will support 991 in NGINX Plus // https://github.com/nginxinc/nginx-gateway-fabric/issues/483 const upstreamsTemplateText = ` {{ range $u := . }} diff --git a/internal/mode/static/nginx/config/upstreams_test.go b/internal/mode/static/nginx/config/upstreams_test.go index 9b6dbbf7ec..eb7b123542 100644 --- a/internal/mode/static/nginx/config/upstreams_test.go +++ b/internal/mode/static/nginx/config/upstreams_test.go @@ -6,6 +6,7 @@ import ( . "github.com/onsi/gomega" "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/nginx/config/http" + "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/nginx/config/stream" "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/state/dataplane" "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/state/resolver" ) @@ -307,3 +308,184 @@ func TestCreateUpstreamPlus(t *testing.T) { g := NewWithT(t) g.Expect(result).To(Equal(expectedUpstream)) } + +func TestExecuteStreamUpstreams(t *testing.T) { + gen := GeneratorImpl{} + stateUpstreams := []dataplane.Upstream{ + { + Name: "up1", + Endpoints: []resolver.Endpoint{ + { + Address: "10.0.0.0", + Port: 80, + }, + }, + }, + { + Name: "up2", + Endpoints: []resolver.Endpoint{ + { + Address: "11.0.0.0", + Port: 80, + }, + }, + }, + { + Name: "up3", + Endpoints: []resolver.Endpoint{}, + }, + } + + expectedSubStrings := []string{ + "upstream up1", + "upstream up2", + "server 10.0.0.0:80;", + "server 11.0.0.0:80;", + } + + upstreamResults := gen.executeStreamUpstreams(dataplane.Configuration{StreamUpstreams: stateUpstreams}) + g := NewWithT(t) + g.Expect(upstreamResults).To(HaveLen(1)) + upstreams := string(upstreamResults[0].data) + + g.Expect(upstreamResults[0].dest).To(Equal(streamConfigFile)) + for _, expSubString := range expectedSubStrings { + g.Expect(upstreams).To(ContainSubstring(expSubString)) + } +} + +func TestCreateStreamUpstreams(t *testing.T) { + gen := GeneratorImpl{} + stateUpstreams := []dataplane.Upstream{ + { + Name: "up1", + Endpoints: []resolver.Endpoint{ + { + Address: "10.0.0.0", + Port: 80, + }, + { + Address: "10.0.0.1", + Port: 80, + }, + { + Address: "10.0.0.2", + Port: 80, + }, + }, + }, + { + Name: "up2", + Endpoints: []resolver.Endpoint{ + { + Address: "11.0.0.0", + Port: 80, + }, + }, + }, + { + Name: "up3", + Endpoints: []resolver.Endpoint{}, + }, + } + + expUpstreams := []stream.Upstream{ + { + Name: "up1", + ZoneSize: ossZoneSize, + Servers: []stream.UpstreamServer{ + { + Address: "10.0.0.0:80", + }, + { + Address: "10.0.0.1:80", + }, + { + Address: "10.0.0.2:80", + }, + }, + }, + { + Name: "up2", + ZoneSize: ossZoneSize, + Servers: []stream.UpstreamServer{ + { + Address: "11.0.0.0:80", + }, + }, + }, + } + + g := NewWithT(t) + result := gen.createStreamUpstreams(stateUpstreams) + g.Expect(result).To(Equal(expUpstreams)) +} + +func TestCreateStreamUpstream(t *testing.T) { + gen := GeneratorImpl{} + up := dataplane.Upstream{ + Name: "multiple-endpoints", + Endpoints: []resolver.Endpoint{ + { + Address: "10.0.0.1", + Port: 80, + }, + { + Address: "10.0.0.2", + Port: 80, + }, + { + Address: "10.0.0.3", + Port: 80, + }, + }, + } + + expectedUpstream := stream.Upstream{ + Name: "multiple-endpoints", + ZoneSize: ossZoneSize, + Servers: []stream.UpstreamServer{ + { + Address: "10.0.0.1:80", + }, + { + Address: "10.0.0.2:80", + }, + { + Address: "10.0.0.3:80", + }, + }, + } + + g := NewWithT(t) + result := gen.createStreamUpstream(up) + g.Expect(result).To(Equal(expectedUpstream)) +} + +func TestCreateStreamUpstreamPlus(t *testing.T) { + gen := GeneratorImpl{plus: true} + + stateUpstream := dataplane.Upstream{ + Name: "multiple-endpoints", + Endpoints: []resolver.Endpoint{ + { + Address: "10.0.0.1", + Port: 80, + }, + }, + } + expectedUpstream := stream.Upstream{ + Name: "multiple-endpoints", + ZoneSize: plusZoneSize, + Servers: []stream.UpstreamServer{ + { + Address: "10.0.0.1:80", + }, + }, + } + + result := gen.createStreamUpstream(stateUpstream) + + g := NewWithT(t) + g.Expect(result).To(Equal(expectedUpstream)) +} diff --git a/internal/mode/static/state/dataplane/types.go b/internal/mode/static/state/dataplane/types.go index d342ff3b0c..5eef247340 100644 --- a/internal/mode/static/state/dataplane/types.go +++ b/internal/mode/static/state/dataplane/types.go @@ -29,8 +29,12 @@ type Configuration struct { HTTPServers []VirtualServer // SSLServers holds all SSLServers. SSLServers []VirtualServer - // Upstreams holds all unique Upstreams. + // TLSPassthroughServers hold all TLSPassthroughServers + TLSPassthroughServers []Layer4VirtualServer + // Upstreams holds all unique http Upstreams. Upstreams []Upstream + // StreamUpstreams holds all unique stream Upstreams + StreamUpstreams []Upstream // BackendGroups holds all unique BackendGroups. BackendGroups []BackendGroup // BaseHTTPConfig holds the configuration options at the http context. @@ -76,6 +80,16 @@ type VirtualServer struct { IsDefault bool } +// Layer4VirtualServer is a virtual server for Layer 4 traffic. +type Layer4VirtualServer struct { + // Hostname is the hostname of the server. + Hostname string + // UpstreamName refers to the name of the upstream that is used. + UpstreamName string + // Port is the port of the server. + Port int32 +} + // Addition holds additional configuration. type Addition struct { // Identifier is a unique identifier for the addition. diff --git a/internal/mode/static/state/graph/route_common.go b/internal/mode/static/state/graph/route_common.go index f8fedd50d0..36d14a11f3 100644 --- a/internal/mode/static/state/graph/route_common.go +++ b/internal/mode/static/state/graph/route_common.go @@ -288,7 +288,7 @@ func bindRouteToListeners( continue } - // Case 2: Attachment is not possible due to unsupported configuration + // Case 2: Attachment is not possible due to unsupported configuration. if ref.Port != nil { valErr := field.Forbidden(path.Child("port"), "cannot be set")