Add support for TLS Passthrough using TLSRoutes

.github/workflows/conformance.yml

@@ -75,6 +75,7 @@ jobs:
         run: |
           ngf_prefix=ghcr.io/nginxinc/nginx-gateway-fabric
           ngf_tag=${{ steps.ngf-meta.outputs.version }}
+          if [ ${{ inputs.enable-experimental }} == "true" ]; then export ENABLE_EXPERIMENTAL=true; fi
           make generate-static-deployment PLUS_ENABLED=${{ inputs.image == 'plus' && 'true' || 'false' }} PREFIX=${ngf_prefix} TAG=${ngf_tag}
         working-directory: ./tests
 
@@ -146,6 +147,7 @@ jobs:
 
       - name: Run conformance tests
         run: |
+          if [ ${{ inputs.enable-experimental }} == "true" ]; then export ENABLE_EXPERIMENTAL=true; fi
           make run-conformance-tests CONFORMANCE_TAG=${{ github.sha }} NGF_VERSION=${{ github.ref_name }} CLUSTER_NAME=${{ github.run_id }}
           core_result=$(cat conformance-profile.yaml | yq '.profiles[0].core.result')
           extended_result=$(cat conformance-profile.yaml | yq '.profiles[0].extended.result')

charts/nginx-gateway-fabric/templates/clusterrole.yaml

@@ -72,6 +72,7 @@ rules:
   - grpcroutes
 {{- if .Values.nginxGateway.gwAPIExperimentalFeatures.enable }}
   - backendtlspolicies
+  - tlsroutes
 {{- end }}
   verbs:
   - list
@@ -85,6 +86,7 @@ rules:
   - grpcroutes/status
 {{- if .Values.nginxGateway.gwAPIExperimentalFeatures.enable }}
   - backendtlspolicies/status
+  - tlsroutes/status
 {{- end }}
   verbs:
   - update

charts/nginx-gateway-fabric/templates/deployment.yaml

@@ -129,6 +129,10 @@ spec:
         volumeMounts:
         - name: nginx-conf
           mountPath: /etc/nginx/conf.d
+          {{- if .Values.nginxGateway.gwAPIExperimentalFeatures.enable }}
+        - name: nginx-stream-conf
+          mountPath: /etc/nginx/stream-conf.d
+          {{ end }}
         - name: module-includes
           mountPath: /etc/nginx/module-includes
         - name: nginx-secrets
@@ -166,6 +170,10 @@ spec:
         volumeMounts:
         - name: nginx-conf
           mountPath: /etc/nginx/conf.d
+          {{- if .Values.nginxGateway.gwAPIExperimentalFeatures.enable }}
+        - name: nginx-stream-conf
+          mountPath: /etc/nginx/stream-conf.d
+          {{ end }}
         - name: module-includes
           mountPath: /etc/nginx/module-includes
         - name: nginx-secrets
@@ -200,6 +208,10 @@ spec:
       volumes:
       - name: nginx-conf
         emptyDir: {}
+        {{- if .Values.nginxGateway.gwAPIExperimentalFeatures.enable }}
+      - name: nginx-stream-conf
+        emptyDir: {}
+        {{ end }}
       - name: module-includes
         emptyDir: {}
       - name: nginx-secrets

deploy/experimental-nginx-plus/deploy.yaml

@@ -82,6 +82,7 @@ rules:
   - referencegrants
   - grpcroutes
   - backendtlspolicies
+  - tlsroutes
   verbs:
   - list
   - watch
@@ -93,6 +94,7 @@ rules:
   - gatewayclasses/status
   - grpcroutes/status
   - backendtlspolicies/status
+  - tlsroutes/status
   verbs:
   - update
 - apiGroups:
@@ -256,6 +258,8 @@ spec:
         volumeMounts:
         - mountPath: /etc/nginx/conf.d
           name: nginx-conf
+        - mountPath: /etc/nginx/stream-conf.d
+          name: nginx-stream-conf
         - mountPath: /etc/nginx/module-includes
           name: module-includes
         - mountPath: /etc/nginx/secrets
@@ -286,6 +290,8 @@ spec:
         volumeMounts:
         - mountPath: /etc/nginx/conf.d
           name: nginx-conf
+        - mountPath: /etc/nginx/stream-conf.d
+          name: nginx-stream-conf
         - mountPath: /etc/nginx/module-includes
           name: module-includes
         - mountPath: /etc/nginx/secrets
@@ -305,6 +311,8 @@ spec:
       volumes:
       - emptyDir: {}
         name: nginx-conf
+      - emptyDir: {}
+        name: nginx-stream-conf
       - emptyDir: {}
         name: module-includes
       - emptyDir: {}

deploy/experimental/deploy.yaml

@@ -74,6 +74,7 @@ rules:
   - referencegrants
   - grpcroutes
   - backendtlspolicies
+  - tlsroutes
   verbs:
   - list
   - watch
@@ -85,6 +86,7 @@ rules:
   - gatewayclasses/status
   - grpcroutes/status
   - backendtlspolicies/status
+  - tlsroutes/status
   verbs:
   - update
 - apiGroups:
@@ -247,6 +249,8 @@ spec:
         volumeMounts:
         - mountPath: /etc/nginx/conf.d
           name: nginx-conf
+        - mountPath: /etc/nginx/stream-conf.d
+          name: nginx-stream-conf
         - mountPath: /etc/nginx/module-includes
           name: module-includes
         - mountPath: /etc/nginx/secrets
@@ -277,6 +281,8 @@ spec:
         volumeMounts:
         - mountPath: /etc/nginx/conf.d
           name: nginx-conf
+        - mountPath: /etc/nginx/stream-conf.d
+          name: nginx-stream-conf
         - mountPath: /etc/nginx/module-includes
           name: module-includes
         - mountPath: /etc/nginx/secrets
@@ -296,6 +302,8 @@ spec:
       volumes:
       - emptyDir: {}
         name: nginx-conf
+      - emptyDir: {}
+        name: nginx-stream-conf
       - emptyDir: {}
         name: module-includes
       - emptyDir: {}

internal/framework/gatewayclass/validate.go

@@ -23,6 +23,7 @@ var gatewayCRDs = map[string]apiVersion{
 	"referencegrants.gateway.networking.k8s.io":    {},
 	"backendtlspolicies.gateway.networking.k8s.io": {},
 	"grpcroutes.gateway.networking.k8s.io":         {},
+	"tlsroutes.gateway.networking.k8s.io":          {},
 }
 
 type apiVersion struct {

internal/framework/kinds/kinds.go

@@ -19,6 +19,8 @@ const (
 	HTTPRoute = "HTTPRoute"
 	// GRPCRoute is the GRPCRoute kind.
 	GRPCRoute = "GRPCRoute"
+	// TLSRoute is the TLSRoute kind.
+	TLSRoute = "TLSRoute"
 )
 
 // NGINX Gateway Fabric kinds.

internal/mode/static/handler.go

@@ -246,6 +246,7 @@ func (h *eventHandlerImpl) updateStatuses(ctx context.Context, logger logr.Logge
 		gcReqs = status.PrepareGatewayClassRequests(graph.GatewayClass, graph.IgnoredGatewayClasses, transitionTime)
 	}
 	routeReqs := status.PrepareRouteRequests(
+		graph.L4Routes,
 		graph.Routes,
 		transitionTime,
 		h.latestReloadResult,

internal/mode/static/manager.go

@@ -31,6 +31,7 @@
 	metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
 	k8spredicate "sigs.k8s.io/controller-runtime/pkg/predicate"
 	gatewayv1 "sigs.k8s.io/gateway-api/apis/v1"
+	gatewayv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2"
 	gatewayv1alpha3 "sigs.k8s.io/gateway-api/apis/v1alpha3"
 	gatewayv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1"
 
@@ -73,6 +74,7 @@
 	utilruntime.Must(gatewayv1beta1.Install(scheme))
 	utilruntime.Must(gatewayv1.Install(scheme))
 	utilruntime.Must(gatewayv1alpha3.Install(scheme))
+	utilruntime.Must(gatewayv1alpha2.Install(scheme))
 	utilruntime.Must(apiv1.AddToScheme(scheme))
 	utilruntime.Must(discoveryV1.AddToScheme(scheme))
 	utilruntime.Must(ngfAPI.AddToScheme(scheme))
@@ -489,6 +491,12 @@
 				// https://github.com/nginxinc/nginx-gateway-fabric/issues/1545
 				objectType: &apiv1.ConfigMap{},
 			},
+			{
+				objectType: &gatewayv1alpha2.TLSRoute{},
+				options: []controller.Option{
+					controller.WithK8sPredicate(k8spredicate.GenerationChangedPredicate{}),
+				},
+			},
 		}
 		controllerRegCfgs = append(controllerRegCfgs, gwExpFeatures...)
 	}
@@ -663,6 +671,7 @@
 			objectLists,
 			&gatewayv1alpha3.BackendTLSPolicyList{},
 			&apiv1.ConfigMapList{},
+			&gatewayv1alpha2.TLSRouteList{},
 		)
 	}
 

internal/mode/static/manager_test.go

@@ -13,6 +13,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
 	gatewayv1 "sigs.k8s.io/gateway-api/apis/v1"
+	gatewayv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2"
 	gatewayv1alpha3 "sigs.k8s.io/gateway-api/apis/v1alpha3"
 	gatewayv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1"
 
@@ -105,6 +106,7 @@ func TestPrepareFirstEventBatchPreparerArgs(t *testing.T) {
 				&ngfAPI.NginxProxyList{},
 				partialObjectMetadataList,
 				&gatewayv1alpha3.BackendTLSPolicyList{},
+				&gatewayv1alpha2.TLSRouteList{},
 				&gatewayv1.GRPCRouteList{},
 				&ngfAPI.ClientSettingsPolicyList{},
 				&ngfAPI.ObservabilityPolicyList{},

internal/mode/static/nginx/conf/nginx-plus.conf

@@ -54,6 +54,20 @@ http {
   }
 }
 
+stream {
+  variables_hash_bucket_size 512;
+  variables_hash_max_size 1024;
+
+  map_hash_max_size 2048;
+  map_hash_bucket_size 256;
+
+  log_format stream-main '$remote_addr [$time_local] '
+                         '$protocol $status $bytes_sent $bytes_received '
+                         '$session_time "$ssl_preread_server_name"';
+  access_log /dev/stdout stream-main;
+  include /etc/nginx/stream-conf.d/*.conf;
+}
+
 mgmt {
   usage_report interval=0s;
 }

internal/mode/static/nginx/conf/nginx.conf

@@ -38,3 +38,17 @@ http {
     }
   }
 }
+
+stream {
+  variables_hash_bucket_size 512;
+  variables_hash_max_size 1024;
+
+  map_hash_max_size 2048;
+  map_hash_bucket_size 256;
+
+  log_format stream-main '$remote_addr [$time_local] '
+                         '$protocol $status $bytes_sent $bytes_received '
+                         '$session_time "$ssl_preread_server_name"';
+  access_log /dev/stdout stream-main;
+  include /etc/nginx/stream-conf.d/*.conf;
+}

internal/mode/static/nginx/config/base_http_config_template.go

@@ -2,4 +2,25 @@ package config
 
 const baseHTTPTemplateText = `
 {{- if .HTTP2 }}http2 on;{{ end }}
+
+# Set $gw_api_compliant_host variable to the value of $http_host unless $http_host is empty, then set it to the value
+# of $host. We prefer $http_host because it contains the original value of the host header, which is required by the
+# Gateway API. However, in an HTTP/1.0 request, it's possible that $http_host can be empty. In this case, we will use
+# the value of $host. See http://nginx.org/en/docs/http/ngx_http_core_module.html#var_host.
+map $http_host $gw_api_compliant_host {
+    '' $host;
+    default $http_host;
+}
+
+# Set $connection_header variable to upgrade when the $http_upgrade header is set, otherwise, set it to close. This
+# allows support for websocket connections. See https://nginx.org/en/docs/http/websocket.html.
+map $http_upgrade $connection_upgrade {
+    default upgrade;
+    '' close;
+}
+
+## Returns just the path from the original request URI.
+map $request_uri $request_uri_path {
+  "~^(?P<path>[^?]*)(\?.*)?$"  $path;
+}
 `

internal/mode/static/nginx/config/base_http_config_test.go

@@ -47,5 +47,8 @@ func TestExecuteBaseHttp(t *testing.T) {
 		res := executeBaseHTTPConfig(test.conf)
 		g.Expect(res).To(HaveLen(1))
 		g.Expect(test.expCount).To(Equal(strings.Count(string(res[0].data), expSubStr)))
+		g.Expect(strings.Count(string(res[0].data), "map $http_host $gw_api_compliant_host {")).To(Equal(1))
+		g.Expect(strings.Count(string(res[0].data), "map $http_upgrade $connection_upgrade {")).To(Equal(1))
+		g.Expect(strings.Count(string(res[0].data), "map $request_uri $request_uri_path {")).To(Equal(1))
 	}
 }

internal/mode/static/nginx/config/generator.go

@@ -20,6 +20,9 @@ const (
 	// httpFolder is the folder where NGINX HTTP configuration files are stored.
 	httpFolder = configFolder + "/conf.d"
 
+	// streamFolder is the folder where NGINX Stream configuration files are stored.
+	streamFolder = configFolder + "/stream-conf.d"
+
 	// modulesIncludesFolder is the folder where the included "load_module" file is stored.
 	modulesIncludesFolder = configFolder + "/module-includes"
 
@@ -32,6 +35,9 @@ const (
 	// httpConfigFile is the path to the configuration file with HTTP configuration.
 	httpConfigFile = httpFolder + "/http.conf"
 
+	// streamConfigFile is the path to the configuration file with Stream configuration.
+	streamConfigFile = streamFolder + "/stream.conf"
+
 	// configVersionFile is the path to the config version configuration file.
 	configVersionFile = httpFolder + "/config-version.conf"
 
@@ -43,7 +49,7 @@ const (
 )
 
 // ConfigFolders is a list of folders where NGINX configuration files are stored.
-var ConfigFolders = []string{httpFolder, secretsFolder, includesFolder, modulesIncludesFolder}
+var ConfigFolders = []string{httpFolder, secretsFolder, includesFolder, modulesIncludesFolder, streamFolder}
 
 // Generator generates NGINX configuration files.
 // This interface is used for testing purposes only.
@@ -168,6 +174,9 @@ func (g GeneratorImpl) getExecuteFuncs(generator policies.Generator) []executeFu
 		executeSplitClients,
 		executeMaps,
 		executeTelemetry,
+		executeStreamServers,
+		g.executeStreamUpstreams,
+		executeStreamMaps,
 	}
 }