Support NginxProxy CRD and global tracing settings

Problem: As a user of NGF
I want to set the collection point for my traces for my installation of NGF
So that I can ensure all my traces are sent to the same collection platform.

Solution: Implement the NginxProxy CRD which contains the fields required to configure the collection point for tracing. This resource is attached to the GatewayClass. If the resource is not found, a condition will be set on the GatewayClass to indicate this. The GatewayClass will continue to be Accepted even if the parametersRef is invalid.

This configuration sets the `http` context-level otel directives. The otel module is loaded conditionally based on the existence of this configuration.

Note: tracing is not enabled by this configuration, this only sets high level options. #1828 is required to actually enable tracing on a per-route basis.

Author: sjberman

build/Dockerfile.nginx

@@ -1,11 +1,18 @@
 # syntax=docker/dockerfile:1.6
+FROM scratch as nginx-files
+
+# the following links can be replaced with local files if needed, i.e. ADD --chown=101:1001 <local_file> <container_file>
+ADD --link --chown=101:1001 https://cs.nginx.com/static/keys/nginx_signing.rsa.pub nginx_signing.rsa.pub
+
 FROM nginx:1.25.5-alpine
 
 ARG NJS_DIR
 ARG NGINX_CONF_DIR
 ARG BUILD_AGENT
 
-RUN apk add --no-cache libcap \
+RUN --mount=type=bind,from=nginx-files,src=nginx_signing.rsa.pub,target=/etc/apk/keys/nginx_signing.rsa.pub \
+    printf "%s\n" "http://nginx.org/packages/mainline/alpine/v$(egrep -o '^[0-9]+\.[0-9]+' /etc/alpine-release)/main" >> /etc/apk/repositories \
+    && apk add --no-cache libcap nginx-module-otel \
     && mkdir -p /var/lib/nginx /usr/lib/nginx/modules \
     && setcap 'cap_net_bind_service=+ep' /usr/sbin/nginx \
     && setcap -v 'cap_net_bind_service=+ep' /usr/sbin/nginx \

build/Dockerfile.nginxplus

@@ -18,7 +18,7 @@ RUN --mount=type=secret,id=nginx-repo.crt,dst=/etc/apk/cert.pem,mode=0644 \
     addgroup -g 1001 -S nginx \
     && adduser -S -D -H -u 101 -h /var/cache/nginx -s /sbin/nologin -G nginx -g nginx nginx \
     && printf "%s\n" "https://pkgs.nginx.com/plus/${NGINX_PLUS_VERSION}/alpine/v$(grep -E -o '^[0-9]+\.[0-9]+' /etc/alpine-release)/main" >> /etc/apk/repositories \
-    && apk add --no-cache nginx-plus nginx-plus-module-njs libcap \
+    && apk add --no-cache nginx-plus nginx-plus-module-njs nginx-plus-module-otel libcap \
     && mkdir -p /var/lib/nginx /usr/lib/nginx/modules \
     && setcap 'cap_net_bind_service=+ep' /usr/sbin/nginx \
     && setcap -v 'cap_net_bind_service=+ep' /usr/sbin/nginx \

conformance/provisioner/static-deployment.yaml

@@ -70,6 +70,8 @@ spec:
         volumeMounts:
         - name: nginx-conf
           mountPath: /etc/nginx/conf.d
+        - name: nginx-includes
+          mountPath: /etc/nginx/includes
         - name: nginx-secrets
           mountPath: /etc/nginx/secrets
         - name: nginx-run
@@ -94,6 +96,8 @@ spec:
         volumeMounts:
         - name: nginx-conf
           mountPath: /etc/nginx/conf.d
+        - name: nginx-includes
+          mountPath: /etc/nginx/includes
         - name: nginx-secrets
           mountPath: /etc/nginx/secrets
         - name: nginx-run
@@ -111,6 +115,8 @@ spec:
       volumes:
       - name: nginx-conf
         emptyDir: {}
+      - name: nginx-includes
+        emptyDir: {}
       - name: nginx-secrets
         emptyDir: {}
       - name: nginx-run

deploy/helm-chart/README.md

@@ -249,7 +249,7 @@ To uninstall/delete the release `ngf`:
 ```shell
 helm uninstall ngf -n nginx-gateway
 kubectl delete ns nginx-gateway
-kubectl delete crd nginxgateways.gateway.nginx.org
+for crd in `kubectl get crds -oname | grep gateway.nginx.org | awk -F / '{ print $2 }'`; do kubectl delete crd $crd; done
 ```
 
 These commands remove all the Kubernetes components associated with the release and deletes the release.

deploy/helm-chart/templates/deployment.yaml

@@ -118,6 +118,8 @@ spec:
         volumeMounts:
         - name: nginx-conf
           mountPath: /etc/nginx/conf.d
+        - name: nginx-includes
+          mountPath: /etc/nginx/includes
         - name: nginx-secrets
           mountPath: /etc/nginx/secrets
         - name: nginx-run
@@ -149,6 +151,8 @@ spec:
         volumeMounts:
         - name: nginx-conf
           mountPath: /etc/nginx/conf.d
+        - name: nginx-includes
+          mountPath: /etc/nginx/includes
         - name: nginx-secrets
           mountPath: /etc/nginx/secrets
         - name: nginx-run
@@ -181,6 +185,8 @@ spec:
       volumes:
       - name: nginx-conf
         emptyDir: {}
+      - name: nginx-includes
+        emptyDir: {}
       - name: nginx-secrets
         emptyDir: {}
       - name: nginx-run

deploy/helm-chart/templates/rbac.yaml

@@ -10,7 +10,7 @@ metadata:
   {{- if or .Values.serviceAccount.imagePullSecret .Values.serviceAccount.imagePullSecrets }}
   imagePullSecrets:
     {{- if .Values.serviceAccount.imagePullSecret }}
-    - name: {{ .Values.serviceAccount.imagePullSecret}}
+    - name: {{ .Values.serviceAccount.imagePullSecret }}
     {{- end }}
     {{- if .Values.serviceAccount.imagePullSecrets }}
     {{- range .Values.serviceAccount.imagePullSecrets }}
@@ -111,6 +111,7 @@ rules:
   - gateway.nginx.org
   resources:
   - nginxgateways
+  - nginxproxies
   verbs:
   - get
   - list

deploy/manifests/nginx-gateway-experimental.yaml

@@ -93,6 +93,7 @@ rules:
   - gateway.nginx.org
   resources:
   - nginxgateways
+  - nginxproxies
   verbs:
   - get
   - list
@@ -213,6 +214,8 @@ spec:
         volumeMounts:
         - name: nginx-conf
           mountPath: /etc/nginx/conf.d
+        - name: nginx-includes
+          mountPath: /etc/nginx/includes
         - name: nginx-secrets
           mountPath: /etc/nginx/secrets
         - name: nginx-run
@@ -237,6 +240,8 @@ spec:
         volumeMounts:
         - name: nginx-conf
           mountPath: /etc/nginx/conf.d
+        - name: nginx-includes
+          mountPath: /etc/nginx/includes
         - name: nginx-secrets
           mountPath: /etc/nginx/secrets
         - name: nginx-run
@@ -254,6 +259,8 @@ spec:
       volumes:
       - name: nginx-conf
         emptyDir: {}
+      - name: nginx-includes
+        emptyDir: {}
       - name: nginx-secrets
         emptyDir: {}
       - name: nginx-run

deploy/manifests/nginx-gateway.yaml

@@ -90,6 +90,7 @@ rules:
   - gateway.nginx.org
   resources:
   - nginxgateways
+  - nginxproxies
   verbs:
   - get
   - list
@@ -209,6 +210,8 @@ spec:
         volumeMounts:
         - name: nginx-conf
           mountPath: /etc/nginx/conf.d
+        - name: nginx-includes
+          mountPath: /etc/nginx/includes
         - name: nginx-secrets
           mountPath: /etc/nginx/secrets
         - name: nginx-run
@@ -233,6 +236,8 @@ spec:
         volumeMounts:
         - name: nginx-conf
           mountPath: /etc/nginx/conf.d
+        - name: nginx-includes
+          mountPath: /etc/nginx/includes
         - name: nginx-secrets
           mountPath: /etc/nginx/secrets
         - name: nginx-run
@@ -250,6 +255,8 @@ spec:
       volumes:
       - name: nginx-conf
         emptyDir: {}
+      - name: nginx-includes
+        emptyDir: {}
       - name: nginx-secrets
         emptyDir: {}
       - name: nginx-run

deploy/manifests/nginx-plus-gateway-experimental.yaml

@@ -99,6 +99,7 @@ rules:
   - gateway.nginx.org
   resources:
   - nginxgateways
+  - nginxproxies
   verbs:
   - get
   - list
@@ -220,6 +221,8 @@ spec:
         volumeMounts:
         - name: nginx-conf
           mountPath: /etc/nginx/conf.d
+        - name: nginx-includes
+          mountPath: /etc/nginx/includes
         - name: nginx-secrets
           mountPath: /etc/nginx/secrets
         - name: nginx-run
@@ -244,6 +247,8 @@ spec:
         volumeMounts:
         - name: nginx-conf
           mountPath: /etc/nginx/conf.d
+        - name: nginx-includes
+          mountPath: /etc/nginx/includes
         - name: nginx-secrets
           mountPath: /etc/nginx/secrets
         - name: nginx-run
@@ -261,6 +266,8 @@ spec:
       volumes:
       - name: nginx-conf
         emptyDir: {}
+      - name: nginx-includes
+        emptyDir: {}
       - name: nginx-secrets
         emptyDir: {}
       - name: nginx-run

deploy/manifests/nginx-plus-gateway.yaml

@@ -96,6 +96,7 @@ rules:
   - gateway.nginx.org
   resources:
   - nginxgateways
+  - nginxproxies
   verbs:
   - get
   - list
@@ -216,6 +217,8 @@ spec:
         volumeMounts:
         - name: nginx-conf
           mountPath: /etc/nginx/conf.d
+        - name: nginx-includes
+          mountPath: /etc/nginx/includes
         - name: nginx-secrets
           mountPath: /etc/nginx/secrets
         - name: nginx-run
@@ -240,6 +243,8 @@ spec:
         volumeMounts:
         - name: nginx-conf
           mountPath: /etc/nginx/conf.d
+        - name: nginx-includes
+          mountPath: /etc/nginx/includes
         - name: nginx-secrets
           mountPath: /etc/nginx/secrets
         - name: nginx-run
@@ -257,6 +262,8 @@ spec:
       volumes:
       - name: nginx-conf
         emptyDir: {}
+      - name: nginx-includes
+        emptyDir: {}
       - name: nginx-secrets
         emptyDir: {}
       - name: nginx-run

docs/proposals/gateway-settings.md

@@ -1,7 +1,7 @@
 # Enhancement Proposal-1775: Gateway Settings
 
 - Issue: https://github.com/nginxinc/nginx-gateway-fabric/issues/1775
-- Status: Implementable
+- Status: Completed
 
 ## Summary
 
@@ -93,7 +93,7 @@ type Telemetry struct {
     // SpanAttributes are custom key/value attributes that are added to each span.
     //
     // +optional
-    SpanAttributes map[string]string `json:"spanAttributes,omitempty"`
+    SpanAttributes []SpanAttribute `json:"spanAttributes,omitempty"`
 }
 
 // TelemetryExporter specifies OpenTelemetry export parameters.
@@ -122,6 +122,15 @@ type TelemetryExporter struct {
 // The format is a subset of the syntax parsed by Golang time.ParseDuration.
 // Examples: 1h, 12m, 30s, 150ms.
 type Duration string
+
+// SpanAttribute is a key value pair to be added to a tracing span.
+type SpanAttribute struct {
+	// Key is the key for a span attribute.
+	Key string `json:"key"`
+
+	// Value is the value for a span attribute.
+	Value string `json:"value"`
+}
 ```
 
 ### Status

internal/mode/static/manager.go

@@ -414,6 +414,12 @@ func registerControllers(
 				),
 			},
 		},
+		{
+			objectType: &ngfAPI.NginxProxy{},
+			options: []controller.Option{
+				controller.WithK8sPredicate(k8spredicate.GenerationChangedPredicate{}),
+			},
+		},
 	}
 
 	if cfg.ExperimentalFeatures {
@@ -592,6 +598,7 @@ func prepareFirstEventBatchPreparerArgs(
 		&discoveryV1.EndpointSliceList{},
 		&gatewayv1.HTTPRouteList{},
 		&gatewayv1beta1.ReferenceGrantList{},
+		&ngfAPI.NginxProxyList{},
 		partialObjectMetadataList,
 	}
 

internal/mode/static/manager_test.go

@@ -16,6 +16,7 @@ import (
 	gatewayv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2"
 	gatewayv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1"
 
+	ngfAPI "github.com/nginxinc/nginx-gateway-fabric/apis/v1alpha1"
 	"github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/config"
 )
 
@@ -52,6 +53,7 @@ func TestPrepareFirstEventBatchPreparerArgs(t *testing.T) {
 				&gatewayv1.HTTPRouteList{},
 				&gatewayv1.GatewayList{},
 				&gatewayv1beta1.ReferenceGrantList{},
+				&ngfAPI.NginxProxyList{},
 				partialObjectMetadataList,
 			},
 		},
@@ -72,6 +74,7 @@ func TestPrepareFirstEventBatchPreparerArgs(t *testing.T) {
 				&discoveryV1.EndpointSliceList{},
 				&gatewayv1.HTTPRouteList{},
 				&gatewayv1beta1.ReferenceGrantList{},
+				&ngfAPI.NginxProxyList{},
 				partialObjectMetadataList,
 			},
 		},
@@ -93,6 +96,7 @@ func TestPrepareFirstEventBatchPreparerArgs(t *testing.T) {
 				&discoveryV1.EndpointSliceList{},
 				&gatewayv1.HTTPRouteList{},
 				&gatewayv1beta1.ReferenceGrantList{},
+				&ngfAPI.NginxProxyList{},
 				partialObjectMetadataList,
 				&gatewayv1alpha2.BackendTLSPolicyList{},
 			},

internal/mode/static/nginx/conf/nginx-plus.conf

@@ -1,4 +1,5 @@
 load_module /usr/lib/nginx/modules/ngx_http_js_module.so;
+include /etc/nginx/includes/*.conf;
 
 worker_processes auto;
 

internal/mode/static/nginx/conf/nginx.conf

@@ -1,4 +1,5 @@
 load_module /usr/lib/nginx/modules/ngx_http_js_module.so;
+include /etc/nginx/includes/*.conf;
 
 worker_processes auto;
 

internal/mode/static/nginx/config/generator.go

@@ -15,6 +15,7 @@ const (
 
 	// httpFolder is the folder where NGINX HTTP configuration files are stored.
 	httpFolder = configFolder + "/conf.d"
+
 	// secretsFolder is the folder where secrets (like TLS certs/keys) are stored.
 	secretsFolder = configFolder + "/secrets"
 
@@ -23,6 +24,9 @@ const (
 
 	// configVersionFile is the path to the config version configuration file.
 	configVersionFile = httpFolder + "/config-version.conf"
+
+	// loadModulesFile is the path to the file containing any load_module directives.
+	loadModulesFile = configFolder + "/includes/load_modules.conf"
 )
 
 // ConfigFolders is a list of folders where NGINX configuration files are stored.
@@ -74,6 +78,8 @@ func (g GeneratorImpl) Generate(conf dataplane.Configuration) []file.File {
 		files = append(files, generateCertBundle(id, bundle))
 	}
 
+	files = append(files, generateLoadModulesConf(conf))
+
 	return files
 }
 
@@ -125,6 +131,7 @@ func (g GeneratorImpl) getExecuteFuncs() []executeFunc {
 		executeSplitClients,
 		executeServers,
 		executeMaps,
+		executeTelemetry,
 	}
 }
 
@@ -138,3 +145,16 @@ func generateConfigVersion(configVersion int) file.File {
 		Type:    file.TypeRegular,
 	}
 }
+
+func generateLoadModulesConf(conf dataplane.Configuration) file.File {
+	var c []byte
+	if conf.Telemetry.Endpoint != "" {
+		c = []byte("load_module modules/ngx_otel_module.so;")
+	}
+
+	return file.File{
+		Content: c,
+		Path:    loadModulesFile,
+		Type:    file.TypeRegular,
+	}
+}