add support for TLS route (#2211)

Problem: TLSRoute was not supported by NGF.

Solution: Watched for changes to TLSRoutes, added validation and tests, added TLSRoute to graph, and converted Kubernetes TLSRoute spec to nginx config.

Author: sarthyparty

.github/workflows/conformance.yml

@@ -75,6 +75,7 @@ jobs:
         run: |
           ngf_prefix=ghcr.io/nginxinc/nginx-gateway-fabric
           ngf_tag=${{ steps.ngf-meta.outputs.version }}
+          if [ ${{ inputs.enable-experimental }} == "true" ]; then export ENABLE_EXPERIMENTAL=true; fi
           make generate-static-deployment PLUS_ENABLED=${{ inputs.image == 'plus' && 'true' || 'false' }} PREFIX=${ngf_prefix} TAG=${ngf_tag}
         working-directory: ./tests
 
@@ -146,6 +147,7 @@ jobs:
 
       - name: Run conformance tests
         run: |
+          if [ ${{ inputs.enable-experimental }} == "true" ]; then export ENABLE_EXPERIMENTAL=true; fi
           make run-conformance-tests CONFORMANCE_TAG=${{ github.sha }} NGF_VERSION=${{ github.ref_name }} CLUSTER_NAME=${{ github.run_id }}
           core_result=$(cat conformance-profile.yaml | yq '.profiles[0].core.result')
           extended_result=$(cat conformance-profile.yaml | yq '.profiles[0].extended.result')

charts/nginx-gateway-fabric/templates/clusterrole.yaml

@@ -72,6 +72,7 @@ rules:
   - grpcroutes
 {{- if .Values.nginxGateway.gwAPIExperimentalFeatures.enable }}
   - backendtlspolicies
+  - tlsroutes
 {{- end }}
   verbs:
   - list
@@ -85,6 +86,7 @@ rules:
   - grpcroutes/status
 {{- if .Values.nginxGateway.gwAPIExperimentalFeatures.enable }}
   - backendtlspolicies/status
+  - tlsroutes/status
 {{- end }}
   verbs:
   - update

deploy/experimental-nginx-plus/deploy.yaml

@@ -82,6 +82,7 @@ rules:
   - referencegrants
   - grpcroutes
   - backendtlspolicies
+  - tlsroutes
   verbs:
   - list
   - watch
@@ -93,6 +94,7 @@ rules:
   - gatewayclasses/status
   - grpcroutes/status
   - backendtlspolicies/status
+  - tlsroutes/status
   verbs:
   - update
 - apiGroups:

deploy/experimental/deploy.yaml

@@ -74,6 +74,7 @@ rules:
   - referencegrants
   - grpcroutes
   - backendtlspolicies
+  - tlsroutes
   verbs:
   - list
   - watch
@@ -85,6 +86,7 @@ rules:
   - gatewayclasses/status
   - grpcroutes/status
   - backendtlspolicies/status
+  - tlsroutes/status
   verbs:
   - update
 - apiGroups:

internal/framework/gatewayclass/validate.go

@@ -23,6 +23,7 @@ var gatewayCRDs = map[string]apiVersion{
 	"referencegrants.gateway.networking.k8s.io":    {},
 	"backendtlspolicies.gateway.networking.k8s.io": {},
 	"grpcroutes.gateway.networking.k8s.io":         {},
+	"tlsroutes.gateway.networking.k8s.io":          {},
 }
 
 type apiVersion struct {

internal/framework/kinds/kinds.go

@@ -19,6 +19,8 @@ const (
 	HTTPRoute = "HTTPRoute"
 	// GRPCRoute is the GRPCRoute kind.
 	GRPCRoute = "GRPCRoute"
+	// TLSRoute is the TLSRoute kind.
+	TLSRoute = "TLSRoute"
 )
 
 // NGINX Gateway Fabric kinds.

internal/mode/static/handler.go

@@ -246,6 +246,7 @@ func (h *eventHandlerImpl) updateStatuses(ctx context.Context, logger logr.Logge
 		gcReqs = status.PrepareGatewayClassRequests(graph.GatewayClass, graph.IgnoredGatewayClasses, transitionTime)
 	}
 	routeReqs := status.PrepareRouteRequests(
+		graph.L4Routes,
 		graph.Routes,
 		transitionTime,
 		h.latestReloadResult,

internal/mode/static/manager.go

@@ -31,6 +31,7 @@ import (
 	metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
 	k8spredicate "sigs.k8s.io/controller-runtime/pkg/predicate"
 	gatewayv1 "sigs.k8s.io/gateway-api/apis/v1"
+	gatewayv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2"
 	gatewayv1alpha3 "sigs.k8s.io/gateway-api/apis/v1alpha3"
 	gatewayv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1"
 
@@ -73,6 +74,7 @@ func init() {
 	utilruntime.Must(gatewayv1beta1.Install(scheme))
 	utilruntime.Must(gatewayv1.Install(scheme))
 	utilruntime.Must(gatewayv1alpha3.Install(scheme))
+	utilruntime.Must(gatewayv1alpha2.Install(scheme))
 	utilruntime.Must(apiv1.AddToScheme(scheme))
 	utilruntime.Must(discoveryV1.AddToScheme(scheme))
 	utilruntime.Must(ngfAPI.AddToScheme(scheme))
@@ -489,6 +491,12 @@ func registerControllers(
 				// https://github.com/nginxinc/nginx-gateway-fabric/issues/1545
 				objectType: &apiv1.ConfigMap{},
 			},
+			{
+				objectType: &gatewayv1alpha2.TLSRoute{},
+				options: []controller.Option{
+					controller.WithK8sPredicate(k8spredicate.GenerationChangedPredicate{}),
+				},
+			},
 		}
 		controllerRegCfgs = append(controllerRegCfgs, gwExpFeatures...)
 	}
@@ -663,6 +671,7 @@ func prepareFirstEventBatchPreparerArgs(
 			objectLists,
 			&gatewayv1alpha3.BackendTLSPolicyList{},
 			&apiv1.ConfigMapList{},
+			&gatewayv1alpha2.TLSRouteList{},
 		)
 	}
 

internal/mode/static/manager_test.go

@@ -13,6 +13,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
 	gatewayv1 "sigs.k8s.io/gateway-api/apis/v1"
+	gatewayv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2"
 	gatewayv1alpha3 "sigs.k8s.io/gateway-api/apis/v1alpha3"
 	gatewayv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1"
 
@@ -105,6 +106,7 @@ func TestPrepareFirstEventBatchPreparerArgs(t *testing.T) {
 				&ngfAPI.NginxProxyList{},
 				partialObjectMetadataList,
 				&gatewayv1alpha3.BackendTLSPolicyList{},
+				&gatewayv1alpha2.TLSRouteList{},
 				&gatewayv1.GRPCRouteList{},
 				&ngfAPI.ClientSettingsPolicyList{},
 				&ngfAPI.ObservabilityPolicyList{},

internal/mode/static/nginx/config/base_http_config_template.go

@@ -18,4 +18,9 @@ map $http_upgrade $connection_upgrade {
     default upgrade;
     '' close;
 }
+
+## Returns just the path from the original request URI.
+map $request_uri $request_uri_path {
+  "~^(?P<path>[^?]*)(\?.*)?$"  $path;
+}
 `

internal/mode/static/nginx/config/base_http_config_test.go

@@ -49,5 +49,6 @@ func TestExecuteBaseHttp(t *testing.T) {
 		g.Expect(test.expCount).To(Equal(strings.Count(string(res[0].data), expSubStr)))
 		g.Expect(strings.Count(string(res[0].data), "map $http_host $gw_api_compliant_host {")).To(Equal(1))
 		g.Expect(strings.Count(string(res[0].data), "map $http_upgrade $connection_upgrade {")).To(Equal(1))
+		g.Expect(strings.Count(string(res[0].data), "map $request_uri $request_uri_path {")).To(Equal(1))
 	}
 }

internal/mode/static/nginx/config/generator_test.go

@@ -11,6 +11,7 @@ import (
 	"github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/nginx/config"
 	"github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/nginx/file"
 	"github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/state/dataplane"
+	"github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/state/resolver"
 )
 
 func TestGenerate(t *testing.T) {
@@ -62,8 +63,13 @@ func TestGenerate(t *testing.T) {
 		},
 		StreamUpstreams: []dataplane.Upstream{
 			{
-				Name:      "stream_up",
-				Endpoints: nil,
+				Name: "stream_up",
+				Endpoints: []resolver.Endpoint{
+					{
+						Address: "1.1.1.1",
+						Port:    80,
+					},
+				},
 			},
 		},
 		BackendGroups: []dataplane.BackendGroup{bg},

internal/mode/static/nginx/config/http/config.go

@@ -1,5 +1,7 @@
 package http
 
+import "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/nginx/config/shared"
+
 const InternalRoutePathPrefix = "/_ngf-internal"
 
 // Server holds all configuration for an HTTP server.
@@ -12,12 +14,7 @@ type Server struct {
 	IsDefaultHTTP bool
 	IsDefaultSSL  bool
 	GRPC          bool
-}
-
-// IPFamily holds the IP family configuration to be used by NGINX.
-type IPFamily struct {
-	IPv4 bool
-	IPv6 bool
+	IsSocket      bool
 }
 
 type LocationType string
@@ -113,7 +110,7 @@ type ProxySSLVerify struct {
 // ServerConfig holds configuration for an HTTP server and IP family to be used by NGINX.
 type ServerConfig struct {
 	Servers  []Server
-	IPFamily IPFamily
+	IPFamily shared.IPFamily
 }
 
 // Include defines a file that's included via the include directive.

internal/mode/static/nginx/config/maps.go

@@ -11,12 +11,18 @@ import (
 
 var mapsTemplate = gotemplate.Must(gotemplate.New("maps").Parse(mapsTemplateText))
 
-// emptyStringSocket is used when the stream server has an invalid upstream. In this case, we pass the connection
-// to the empty socket so that NGINX will close the connection with an error in the error log --
-// no host in pass "" -- and set $status variable to 500 (logged by stream access log),
-// which will indicate the problem to the user.
-// https://nginx.org/en/docs/stream/ngx_stream_core_module.html#var_status
-const emptyStringSocket = `""`
+const (
+	// emptyStringSocket is used when the stream server has an invalid upstream. In this case, we pass the connection
+	// to the empty socket so that NGINX will close the connection with an error in the error log --
+	// no host in pass "" -- and set $status variable to 500 (logged by stream access log),
+	// which will indicate the problem to the user.
+	// https://nginx.org/en/docs/stream/ngx_stream_core_module.html#var_status
+	emptyStringSocket = `""`
+
+	// connectionClosedStreamServerSocket is used when we want to listen on a port but have no service configured,
+	// so we pass to this server that just returns an empty string to tell users that we are listening.
+	connectionClosedStreamServerSocket = "unix:/var/run/nginx/connection-closed-server.sock"
+)
 
 func executeMaps(conf dataplane.Configuration) []executeResult {
 	maps := buildAddHeaderMaps(append(conf.HTTPServers, conf.SSLServers...))
@@ -44,32 +50,43 @@ func createStreamMaps(conf dataplane.Configuration) []shared.Map {
 		return nil
 	}
 	portsToMap := make(map[int32]shared.Map)
+	portHasDefault := make(map[int32]struct{})
+	upstreams := make(map[string]dataplane.Upstream)
+
+	for _, u := range conf.StreamUpstreams {
+		upstreams[u.Name] = u
+	}
 
 	for _, server := range conf.TLSPassthroughServers {
 		streamMap, portInUse := portsToMap[server.Port]
 
 		socket := emptyStringSocket
 
-		if server.UpstreamName != "" {
+		if u, ok := upstreams[server.UpstreamName]; ok && server.UpstreamName != "" && len(u.Endpoints) > 0 {
 			socket = getSocketNameTLS(server.Port, server.Hostname)
 		}
 
-		mapParam := shared.MapParameter{
-			Value:  server.Hostname,
-			Result: socket,
+		if server.IsDefault {
+			socket = connectionClosedStreamServerSocket
 		}
 
 		if !portInUse {
-			m := shared.Map{
-				Source:   "$ssl_preread_server_name",
-				Variable: getTLSPassthroughVarName(server.Port),
-				Parameters: []shared.MapParameter{
-					mapParam,
-				},
+			streamMap = shared.Map{
+				Source:       "$ssl_preread_server_name",
+				Variable:     getTLSPassthroughVarName(server.Port),
+				Parameters:   make([]shared.MapParameter, 0),
 				UseHostnames: true,
 			}
-			portsToMap[server.Port] = m
-		} else {
+			portsToMap[server.Port] = streamMap
+		}
+
+		// If the hostname is empty, we don't want to add an entry to the map. This case occurs when
+		// the gateway listener hostname is not specified
+		if server.Hostname != "" {
+			mapParam := shared.MapParameter{
+				Value:  server.Hostname,
+				Result: socket,
+			}
 			streamMap.Parameters = append(streamMap.Parameters, mapParam)
 			portsToMap[server.Port] = streamMap
 		}
@@ -82,6 +99,7 @@ func createStreamMaps(conf dataplane.Configuration) []shared.Map {
 
 		if server.IsDefault {
 			hostname = "default"
+			portHasDefault[server.Port] = struct{}{}
 		}
 
 		if portInUse {
@@ -95,7 +113,13 @@ func createStreamMaps(conf dataplane.Configuration) []shared.Map {
 
 	maps := make([]shared.Map, 0, len(portsToMap))
 
-	for _, m := range portsToMap {
+	for p, m := range portsToMap {
+		if _, ok := portHasDefault[p]; !ok {
+			m.Parameters = append(m.Parameters, shared.MapParameter{
+				Value:  "default",
+				Result: connectionClosedStreamServerSocket,
+			})
+		}
 		maps = append(maps, m)
 	}
 

internal/mode/static/nginx/config/maps_template.go

@@ -3,34 +3,12 @@ package config
 const mapsTemplateText = `
 {{ range $m := . }}
 map {{ $m.Source }} {{ $m.Variable }} {
-	{{- if $m.UseHostnames -}}
+	{{- if $m.UseHostnames }}
 	hostnames;
 	{{ end }}
-
 	{{ range $p := $m.Parameters }}
 	{{ $p.Value }} {{ $p.Result }};
 	{{ end }}
 }
 {{- end }}
-
-# Set $gw_api_compliant_host variable to the value of $http_host unless $http_host is empty, then set it to the value
-# of $host. We prefer $http_host because it contains the original value of the host header, which is required by the
-# Gateway API. However, in an HTTP/1.0 request, it's possible that $http_host can be empty. In this case, we will use
-# the value of $host. See http://nginx.org/en/docs/http/ngx_http_core_module.html#var_host.
-map $http_host $gw_api_compliant_host {
-    '' $host;
-    default $http_host;
-}
-
-# Set $connection_header variable to upgrade when the $http_upgrade header is set, otherwise, set it to close. This
-# allows support for websocket connections. See https://nginx.org/en/docs/http/websocket.html.
-map $http_upgrade $connection_upgrade {
-    default upgrade;
-    '' close;
-}
-
-## Returns just the path from the original request URI.
-map $request_uri $request_uri_path {
-  "~^(?P<path>[^?]*)(\?.*)?$"  $path;
-}
 `