Skip to content

Commit 6ac55e0

Browse files
sjbermansjberman
committed
NGINX Plus R33 support
Adding support for NGINX Plus R33. The major change with this release is that NGINX Plus now requires a JWT in order to run. A user must create a Secret with this JWT and supply the secret name to NGF when installing. A user can also create client SSL and CA Secrets for NIM connections. All of these Secrets are mounted to the nginx container. Because of the new usage reporting method, the old usage reporting method has been removed and CLI arguments have been altered. Since this release is a breaking change for N+ users, the choice was made to remove the unused usage reporting flags instead of deprecating them. Updated documentation to describe this process, while also cleaning up the JWT docker registry process for N+.
1 parent 63bcb26 commit 6ac55e0

File tree

75 files changed

+1361
-2383
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

75 files changed

+1361
-2383
lines changed

‎.github/workflows/conformance.yml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -135,6 +135,12 @@ jobs:
135135
kind create cluster --name ${{ github.run_id }} --image=kindest/node:${{ inputs.k8s-version }}
136136
kind load docker-image ${{ join(fromJSON(steps.ngf-meta.outputs.json).tags, ' ') }} ${{ join(fromJSON(steps.nginx-meta.outputs.json).tags, ' ') }} --name ${{ github.run_id }}
137137
138+
- name: Setup license file for plus
139+
if: ${{ inputs.image == 'plus' }}
140+
run: echo "$PLUS_LICENSE" > license.jwt
141+
env:
142+
PLUS_LICENSE: ${{ secrets.JWT_PLUS_REGISTRY }}
143+
138144
- name: Setup conformance tests
139145
run: |
140146
ngf_prefix=ghcr.io/nginxinc/nginx-gateway-fabric

‎.github/workflows/functional.yml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -100,6 +100,12 @@ jobs:
100100
NGINX_CONF_DIR=internal/mode/static/nginx/conf
101101
BUILD_AGENT=gha
102102
103+
- name: Setup license file for plus
104+
if: ${{ inputs.image == 'plus' }}
105+
run: echo "$PLUS_LICENSE" > license.jwt
106+
env:
107+
PLUS_LICENSE: ${{ secrets.JWT_PLUS_REGISTRY }}
108+
103109
- name: Install cloud-provider-kind
104110
run: |
105111
CLOUD_PROVIDER_KIND_VERSION=v0.4.0 # renovate: datasource=github-tags depName=kubernetes-sigs/cloud-provider-kind

‎.github/workflows/helm.yml

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -143,10 +143,14 @@ jobs:
143143
kubectl kustomize config/crd/gateway-api/standard | kubectl apply -f -
144144
kubectl create namespace nginx-gateway
145145
146-
- name: Create k8s secret
146+
- name: Create plus secrets
147147
if: ${{ inputs.image == 'plus' }}
148+
env:
149+
PLUS_LICENSE: ${{ secrets.JWT_PLUS_REGISTRY }}
148150
run: |
151+
echo "$PLUS_LICENSE" > license.jwt
149152
kubectl create secret docker-registry nginx-plus-registry-secret --docker-server=private-registry.nginx.com --docker-username=${{ secrets.JWT_PLUS_REGISTRY }} --docker-password=none -n nginx-gateway
153+
kubectl create secret generic nplus-license --from-file license.jwt -n nginx-gateway
150154
151155
- name: Set up Python
152156
uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0

‎.github/workflows/nfr.yml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -111,6 +111,12 @@ jobs:
111111
echo "GKE_NUM_NODES=12" >> vars.env
112112
echo "GKE_MACHINE_TYPE=n2d-standard-16" >> vars.env
113113
114+
- name: Setup license file for plus
115+
if: matrix.type == 'plus'
116+
run: echo "$PLUS_LICENSE" > license.jwt
117+
env:
118+
PLUS_LICENSE: ${{ secrets.JWT_PLUS_REGISTRY }}
119+
114120
- name: Create GKE cluster
115121
working-directory: ./tests
116122
run: make create-gke-cluster CI=true

‎.gitignore

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,9 @@ internal/mode/static/nginx/modules/coverage
4646
*.crt
4747
*.key
4848

49+
# JWT files
50+
*.jwt
51+
4952
# Dotenv files
5053
**/*.env
5154

‎.hugo_build.lock

Whitespace-only changes.

‎.yamllint.yaml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ ignore:
33
- charts/nginx-gateway-fabric/templates
44
- config/crd/bases/
55
- deploy/crds.yaml
6+
- deploy/*nginx-plus
67
- site/static
78

89
rules:

‎Makefile

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,12 @@
11
# variables that should not be overridden by the user
22
VERSION = edge
3-
SELF_DIR := $(dir $(lastword $(MAKEFILE_LIST)))
3+
SELF_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST))))
44
CHART_DIR = $(SELF_DIR)charts/nginx-gateway-fabric
55
NGINX_CONF_DIR = internal/mode/static/nginx/conf
66
NJS_DIR = internal/mode/static/nginx/modules/src
77
KIND_CONFIG_FILE = $(SELF_DIR)config/cluster/kind-cluster.yaml
88
NGINX_DOCKER_BUILD_PLUS_ARGS = --secret id=nginx-repo.crt,src=nginx-repo.crt --secret id=nginx-repo.key,src=nginx-repo.key
9-
BUILD_AGENT=local
10-
PLUS_ENABLED ?= false
9+
BUILD_AGENT = local
1110

1211
PROD_TELEMETRY_ENDPOINT = oss.edge.df.f5.com:443
1312
# the telemetry related variables below are also configured in goreleaser.yml
@@ -49,6 +48,8 @@ TARGET ?= local## The target of the build. Possible values: local and container
4948
OUT_DIR ?= build/out## The folder where the binary will be stored
5049
GOARCH ?= amd64## The architecture of the image and/or binary. For example: amd64 or arm64
5150
GOOS ?= linux## The OS of the image and/or binary. For example: linux or darwin
51+
PLUS_ENABLED ?= false
52+
PLUS_LICENSE_FILE ?= $(SELF_DIR)license.jwt
5253
override NGINX_DOCKER_BUILD_OPTIONS += --build-arg NJS_DIR=$(NJS_DIR) --build-arg NGINX_CONF_DIR=$(NGINX_CONF_DIR) --build-arg BUILD_AGENT=$(BUILD_AGENT)
5354

5455
.DEFAULT_GOAL := help
@@ -227,7 +228,9 @@ helm-install-local: install-gateway-crds ## Helm install NGF on configured kind
227228

228229
.PHONY: helm-install-local-with-plus
229230
helm-install-local-with-plus: install-gateway-crds ## Helm install NGF with NGINX Plus on configured kind cluster with local images. To build, load, and install with helm run make install-ngf-local-build-with-plus.
230-
helm install nginx-gateway $(CHART_DIR) --set nginx.image.repository=$(NGINX_PLUS_PREFIX) --create-namespace --wait --set nginxGateway.image.pullPolicy=Never --set service.type=NodePort --set nginxGateway.image.repository=$(PREFIX) --set nginxGateway.image.tag=$(TAG) --set nginx.image.tag=$(TAG) --set nginx.image.pullPolicy=Never --set nginxGateway.gwAPIExperimentalFeatures.enable=$(ENABLE_EXPERIMENTAL) -n nginx-gateway --set nginx.plus=true $(HELM_PARAMETERS)
231+
kubectl create namespace nginx-gateway || true
232+
kubectl -n nginx-gateway create secret generic nplus-license --from-file $(PLUS_LICENSE_FILE) || true
233+
helm install nginx-gateway $(CHART_DIR) --set nginx.image.repository=$(NGINX_PLUS_PREFIX) --wait --set nginxGateway.image.pullPolicy=Never --set service.type=NodePort --set nginxGateway.image.repository=$(PREFIX) --set nginxGateway.image.tag=$(TAG) --set nginx.image.tag=$(TAG) --set nginx.image.pullPolicy=Never --set nginxGateway.gwAPIExperimentalFeatures.enable=$(ENABLE_EXPERIMENTAL) -n nginx-gateway --set nginx.plus=true $(HELM_PARAMETERS)
231234

232235
# Debug Targets
233236
.PHONY: debug-build

‎build/Dockerfile.nginxplus

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ ADD --link --chown=101:1001 https://cs.nginx.com/static/keys/nginx_signing.rsa.p
77

88
FROM alpine:3.20
99

10-
ARG NGINX_PLUS_VERSION=R32
10+
ARG NGINX_PLUS_VERSION=R33
1111
ARG NJS_DIR
1212
ARG NGINX_CONF_DIR
1313
ARG BUILD_AGENT

‎charts/nginx-gateway-fabric/README.md

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -268,10 +268,12 @@ The following table lists the configurable parameters of the NGINX Gateway Fabri
268268
| `nginx.image.tag` | | string | `"edge"` |
269269
| `nginx.lifecycle` | The lifecycle of the nginx container. | object | `{}` |
270270
| `nginx.plus` | Is NGINX Plus image being used | bool | `false` |
271-
| `nginx.usage.clusterName` | The display name of the Kubernetes cluster in the NGINX Plus usage reporting server. | string | `""` |
272-
| `nginx.usage.insecureSkipVerify` | Disable client verification of the NGINX Plus usage reporting server certificate. | bool | `false` |
273-
| `nginx.usage.secretName` | The namespace/name of the Secret containing the credentials for NGINX Plus usage reporting. | string | `""` |
274-
| `nginx.usage.serverURL` | The base server URL of the NGINX Plus usage reporting server. | string | `""` |
271+
| `nginx.usage.caSecretName` | The name of the Secret containing the CA cert for verifying the NGINX Plus usage reporting server. Must exist in the same namespace that the NGINX Gateway Fabric control plane is running in (default namespace: nginx-gateway). | string | `""` |
272+
| `nginx.usage.clientSSLSecretName` | The name of the Secret containing the client cert/key for communicating with the NGINX Plus usage reporting server. Must exist in the same namespace that the NGINX Gateway Fabric control plane is running in (default namespace: nginx-gateway). | string | `""` |
273+
| `nginx.usage.endpoint` | The endpoint of the NGINX Plus usage reporting server. Default: product.connect.nginx.com | string | `""` |
274+
| `nginx.usage.resolver` | The resolver domain name or IP address with optional port for resolving the endpoint. | string | `""` |
275+
| `nginx.usage.secretName` | The name of the Secret containing the JWT for NGINX Plus usage reporting. Must exist in the same namespace that the NGINX Gateway Fabric control plane is running in (default namespace: nginx-gateway). | string | `"nplus-license"` |
276+
| `nginx.usage.skipVerify` | Disable client verification of the NGINX Plus usage reporting server certificate. | bool | `false` |
275277
| `nginxGateway.config.logging.level` | Log level. | string | `"info"` |
276278
| `nginxGateway.configAnnotations` | Set of custom annotations for NginxGateway objects. | object | `{}` |
277279
| `nginxGateway.extraVolumeMounts` | extraVolumeMounts are the additional volume mounts for the nginx-gateway container. | list | `[]` |

‎charts/nginx-gateway-fabric/templates/clusterrole.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ rules:
1818
- get
1919
- list
2020
- watch
21-
{{- if .Values.nginxGateway.productTelemetry.enable }}
21+
{{- if or .Values.nginxGateway.productTelemetry.enable .Values.nginx.plus }}
2222
- apiGroups:
2323
- ""
2424
resources:

‎charts/nginx-gateway-fabric/templates/configmap.yaml

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
apiVersion: v1
2+
kind: ConfigMap
3+
metadata:
4+
name: nginx-includes-bootstrap
5+
namespace: {{ .Release.Namespace }}
6+
labels:
7+
{{- include "nginx-gateway.labels" . | nindent 4 }}
8+
data:
9+
main.conf: |
10+
{{- if and .Values.nginx.config .Values.nginx.config.logging .Values.nginx.config.logging.errorLevel }}
11+
error_log stderr {{ .Values.nginx.config.logging.errorLevel }};
12+
{{ else }}
13+
error_log stderr info;
14+
{{- end }}
15+
{{- if .Values.nginx.plus }}
16+
mgmt.conf: |
17+
mgmt {
18+
license_token /etc/nginx/license/license.jwt;
19+
{{- if .Values.nginx.usage.endpoint }}
20+
usage_report endpoint={{ .Values.nginx.usage.endpoint }};
21+
{{- end }}
22+
{{- if .Values.nginx.usage.skipVerify }}
23+
ssl_verify off;
24+
{{- end }}
25+
{{- if .Values.nginx.usage.caSecretName }}
26+
ssl_trusted_certificate /etc/nginx/usage-certs/ca/ca.crt;
27+
{{- end }}
28+
{{- if .Values.nginx.usage.clientSSLSecretName }}
29+
ssl_certificate /etc/nginx/usage-certs/client/tls.crt;
30+
ssl_certificate_key /etc/nginx/usage-certs/client/tls.key;
31+
{{- end }}
32+
enforce_initial_report off;
33+
}
34+
{{- end }}

‎charts/nginx-gateway-fabric/templates/deployment.yaml

Lines changed: 61 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -42,8 +42,12 @@ spec:
4242
- copy
4343
- --source
4444
- /includes/main.conf
45+
{{- if .Values.nginx.plus }}
46+
- --source
47+
- /includes/mgmt.conf
48+
{{- end }}
4549
- --destination
46-
- /etc/nginx/main-includes/main.conf
50+
- /etc/nginx/main-includes
4751
securityContext:
4852
seccompProfile:
4953
type: RuntimeDefault
@@ -56,7 +60,7 @@ spec:
5660
runAsUser: 102
5761
runAsGroup: 1001
5862
volumeMounts:
59-
- name: nginx-includes-configmap
63+
- name: nginx-includes-bootstrap
6064
mountPath: /includes
6165
- name: nginx-main-includes
6266
mountPath: /etc/nginx/main-includes
@@ -69,6 +73,24 @@ spec:
6973
- --service={{ include "nginx-gateway.fullname" . }}
7074
{{- if .Values.nginx.plus }}
7175
- --nginx-plus
76+
{{- if .Values.nginx.usage.secretName }}
77+
- --usage-report-secret={{ .Values.nginx.usage.secretName }}
78+
{{- end }}
79+
{{- if .Values.nginx.usage.endpoint }}
80+
- --usage-report-endpoint={{ .Values.nginx.usage.endpoint }}
81+
{{- end }}
82+
{{- if .Values.nginx.usage.resolver }}
83+
- --usage-report-resolver={{ .Values.nginx.usage.resolver }}
84+
{{- end }}
85+
{{- if .Values.nginx.usage.skipVerify }}
86+
- --usage-report-skip-verify
87+
{{- end }}
88+
{{- if .Values.nginx.usage.caSecretName }}
89+
- --usage-report-ca-secret={{ .Values.nginx.usage.caSecretName }}
90+
{{- end }}
91+
{{- if .Values.nginx.usage.clientSSLSecretName }}
92+
- --usage-report-client-ssl-secret={{ .Values.nginx.usage.clientSSLSecretName }}
93+
{{- end }}
7294
{{- end }}
7395
{{- if .Values.metrics.enable }}
7496
- --metrics-port={{ .Values.metrics.port }}
@@ -94,18 +116,6 @@ spec:
94116
{{- if .Values.nginxGateway.gwAPIExperimentalFeatures.enable }}
95117
- --gateway-api-experimental-features
96118
{{- end }}
97-
{{- if .Values.nginx.usage.secretName }}
98-
- --usage-report-secret={{ .Values.nginx.usage.secretName }}
99-
{{- end }}
100-
{{- if .Values.nginx.usage.serverURL }}
101-
- --usage-report-server-url={{ .Values.nginx.usage.serverURL }}
102-
{{- end }}
103-
{{- if .Values.nginx.usage.clusterName }}
104-
- --usage-report-cluster-name={{ .Values.nginx.usage.clusterName }}
105-
{{- end }}
106-
{{- if .Values.nginx.usage.insecureSkipVerify }}
107-
- --usage-report-skip-verify
108-
{{- end }}
109119
{{- if .Values.nginxGateway.snippetsFilters.enable }}
110120
- --snippets-filters
111121
{{- end }}
@@ -212,8 +222,24 @@ spec:
212222
mountPath: /var/run/nginx
213223
- name: nginx-cache
214224
mountPath: /var/cache/nginx
225+
- name: nginx-lib
226+
mountPath: /var/lib/nginx
215227
- name: nginx-includes
216228
mountPath: /etc/nginx/includes
229+
{{- if .Values.nginx.plus }}
230+
{{- if .Values.nginx.usage.secretName }}
231+
- name: nginx-plus-license
232+
mountPath: /etc/nginx/license
233+
{{- end }}
234+
{{- if .Values.nginx.usage.clientSSLSecretName }}
235+
- name: usage-client-ssl-secret
236+
mountPath: /etc/nginx/usage-certs/client
237+
{{- end }}
238+
{{- if .Values.nginx.usage.caSecretName }}
239+
- name: usage-ca-secret
240+
mountPath: /etc/nginx/usage-certs/ca
241+
{{- end }}
242+
{{- end }}
217243
{{- with .Values.nginx.extraVolumeMounts -}}
218244
{{ toYaml . | nindent 8 }}
219245
{{- end }}
@@ -255,11 +281,30 @@ spec:
255281
emptyDir: {}
256282
- name: nginx-cache
257283
emptyDir: {}
284+
- name: nginx-lib
285+
emptyDir: {}
258286
- name: nginx-includes
259287
emptyDir: {}
260-
- name: nginx-includes-configmap
288+
- name: nginx-includes-bootstrap
261289
configMap:
262-
name: nginx-includes
290+
name: nginx-includes-bootstrap
291+
{{- if .Values.nginx.plus }}
292+
{{- if .Values.nginx.usage.secretName }}
293+
- name: nginx-plus-license
294+
secret:
295+
secretName: {{ .Values.nginx.usage.secretName }}
296+
{{- end }}
297+
{{- if .Values.nginx.usage.clientSSLSecretName }}
298+
- name: usage-client-ssl-secret
299+
secret:
300+
secretName: {{ .Values.nginx.usage.clientSSLSecretName }}
301+
{{- end }}
302+
{{- if .Values.nginx.usage.caSecretName }}
303+
- name: usage-ca-secret
304+
secret:
305+
secretName: {{ .Values.nginx.usage.caSecretName }}
306+
{{- end }}
307+
{{- end }}
263308
{{- with .Values.extraVolumes -}}
264309
{{ toYaml . | nindent 6 }}
265310
{{- end }}

‎charts/nginx-gateway-fabric/templates/include-configmap.yaml

Lines changed: 0 additions & 14 deletions
This file was deleted.

‎charts/nginx-gateway-fabric/values.schema.json

Lines changed: 28 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -262,33 +262,47 @@
262262
"usage": {
263263
"description": "Configuration for NGINX Plus usage reporting.",
264264
"properties": {
265-
"clusterName": {
265+
"caSecretName": {
266266
"default": "",
267-
"description": "The display name of the Kubernetes cluster in the NGINX Plus usage reporting server.",
267+
"description": "The name of the Secret containing the CA cert for verifying the NGINX Plus usage reporting\nserver. Must exist in the same namespace that the NGINX Gateway Fabric control plane is running in (default namespace: nginx-gateway).",
268268
"required": [],
269-
"title": "clusterName",
269+
"title": "caSecretName",
270270
"type": "string"
271271
},
272-
"insecureSkipVerify": {
273-
"default": false,
274-
"description": "Disable client verification of the NGINX Plus usage reporting server certificate.",
272+
"clientSSLSecretName": {
273+
"default": "",
274+
"description": "The name of the Secret containing the client cert/key for communicating with the NGINX Plus usage reporting\nserver. Must exist in the same namespace that the NGINX Gateway Fabric control plane is running in (default namespace: nginx-gateway).",
275275
"required": [],
276-
"title": "insecureSkipVerify",
277-
"type": "boolean"
276+
"title": "clientSSLSecretName",
277+
"type": "string"
278278
},
279-
"secretName": {
279+
"endpoint": {
280280
"default": "",
281-
"description": "The namespace/name of the Secret containing the credentials for NGINX Plus usage reporting.",
281+
"description": "The endpoint of the NGINX Plus usage reporting server. Default: product.connect.nginx.com",
282282
"required": [],
283-
"title": "secretName",
283+
"title": "endpoint",
284284
"type": "string"
285285
},
286-
"serverURL": {
286+
"resolver": {
287287
"default": "",
288-
"description": "The base server URL of the NGINX Plus usage reporting server.",
288+
"description": "The resolver domain name or IP address with optional port for resolving the endpoint.",
289289
"required": [],
290-
"title": "serverURL",
290+
"title": "resolver",
291291
"type": "string"
292+
},
293+
"secretName": {
294+
"default": "nplus-license",
295+
"description": "The name of the Secret containing the JWT for NGINX Plus usage reporting. Must exist in the same namespace\nthat the NGINX Gateway Fabric control plane is running in (default namespace: nginx-gateway).",
296+
"required": [],
297+
"title": "secretName",
298+
"type": "string"
299+
},
300+
"skipVerify": {
301+
"default": false,
302+
"description": "Disable client verification of the NGINX Plus usage reporting server certificate.",
303+
"required": [],
304+
"title": "skipVerify",
305+
"type": "boolean"
292306
}
293307
},
294308
"required": [],

0 commit comments

Comments
 (0)