Skip to content

Commit 2ce993f

Browse files
kate-osbornkate-osborn
authored
Use staging endpoint for N+ reporting in pipeline (#2885) (#2907)
Problem: Pipeline uses production endpoint for N+ usage reporting, which should be reserved for production deployments. Solution: Use staging endpoint in the pipeline. Devs will need to export the staging endpoint in their environments in order to run the NFR, functional, and conformance tests locally.
1 parent a5e94c2 commit 2ce993f

File tree

10 files changed

+48
-21
lines changed

10 files changed

+48
-21
lines changed

‎.github/workflows/conformance.yml

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,7 @@ jobs:
7777
ngf_prefix=ghcr.io/nginxinc/nginx-gateway-fabric
7878
ngf_tag=${{ steps.ngf-meta.outputs.version }}
7979
if [ ${{ inputs.enable-experimental }} == "true" ]; then export ENABLE_EXPERIMENTAL=true; fi
80+
export PLUS_USAGE_ENDPOINT=${{ secrets.JWT_PLUS_REPORTING_ENDPOINT }}
8081
make generate-static-deployment PLUS_ENABLED=${{ inputs.image == 'plus' && 'true' || 'false' }} PREFIX=${ngf_prefix} TAG=${ngf_tag}
8182
working-directory: ./tests
8283

@@ -139,7 +140,7 @@ jobs:
139140
- name: Setup license file for plus
140141
if: ${{ inputs.image == 'plus' }}
141142
env:
142-
PLUS_LICENSE: ${{ secrets.JWT_PLUS_REGISTRY }}
143+
PLUS_LICENSE: ${{ secrets.JWT_PLUS_REPORTING }}
143144
run: echo "${PLUS_LICENSE}" > license.jwt
144145

145146
- name: Setup conformance tests
@@ -148,6 +149,7 @@ jobs:
148149
ngf_tag=${{ steps.ngf-meta.outputs.version }}
149150
if [ ${{ github.event_name }} == "schedule" ]; then export GW_API_VERSION=main; fi
150151
if [ ${{ inputs.enable-experimental }} == "true" ]; then export ENABLE_EXPERIMENTAL=true; fi
152+
export PLUS_USAGE_ENDPOINT=${{ secrets.JWT_PLUS_REPORTING_ENDPOINT }}
151153
make helm-install-local${{ inputs.image == 'plus' && '-with-plus' || ''}} PREFIX=${ngf_prefix} TAG=${ngf_tag}
152154
make deploy-updated-provisioner PREFIX=${ngf_prefix} TAG=${ngf_tag}
153155
working-directory: ./tests

‎.github/workflows/functional.yml

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -104,7 +104,7 @@ jobs:
104104
- name: Setup license file for plus
105105
if: ${{ inputs.image == 'plus' }}
106106
env:
107-
PLUS_LICENSE: ${{ secrets.JWT_PLUS_REGISTRY }}
107+
PLUS_LICENSE: ${{ secrets.JWT_PLUS_REPORTING }}
108108
run: echo "${PLUS_LICENSE}" > license.jwt
109109

110110
- name: Install cloud-provider-kind
@@ -125,19 +125,22 @@ jobs:
125125
run: |
126126
ngf_prefix=ghcr.io/nginxinc/nginx-gateway-fabric
127127
ngf_tag=${{ steps.ngf-meta.outputs.version }}
128+
export PLUS_USAGE_ENDPOINT=${{ secrets.JWT_PLUS_REPORTING_ENDPOINT }}
128129
make test${{ inputs.image == 'plus' && '-with-plus' || ''}} PREFIX=${ngf_prefix} TAG=${ngf_tag} GINKGO_LABEL=telemetry GW_SERVICE_TYPE=LoadBalancer CLUSTER_NAME=${{ github.run_id }} CI=true
129130
working-directory: ./tests
130131

131132
- name: Run functional graceful-recovery tests
132133
run: |
133134
ngf_prefix=ghcr.io/nginxinc/nginx-gateway-fabric
134135
ngf_tag=${{ steps.ngf-meta.outputs.version }}
136+
export PLUS_USAGE_ENDPOINT=${{ secrets.JWT_PLUS_REPORTING_ENDPOINT }}
135137
make test${{ inputs.image == 'plus' && '-with-plus' || ''}} PREFIX=${ngf_prefix} TAG=${ngf_tag} GINKGO_LABEL=graceful-recovery GW_SERVICE_TYPE=LoadBalancer CLUSTER_NAME=${{ github.run_id }} CI=true
136138
working-directory: ./tests
137139

138140
- name: Run functional tests
139141
run: |
140142
ngf_prefix=ghcr.io/nginxinc/nginx-gateway-fabric
141143
ngf_tag=${{ steps.ngf-meta.outputs.version }}
144+
export PLUS_USAGE_ENDPOINT=${{ secrets.JWT_PLUS_REPORTING_ENDPOINT }}
142145
make test${{ inputs.image == 'plus' && '-with-plus' || ''}} PREFIX=${ngf_prefix} TAG=${ngf_tag} GW_SERVICE_TYPE=LoadBalancer CLUSTER_NAME=${{ github.run_id }} CI=true
143146
working-directory: ./tests

‎.github/workflows/helm.yml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -103,7 +103,7 @@ jobs:
103103
- name: Create plus secret
104104
if: ${{ inputs.image == 'plus' }}
105105
env:
106-
PLUS_LICENSE: ${{ secrets.JWT_PLUS_REGISTRY }}
106+
PLUS_LICENSE: ${{ secrets.JWT_PLUS_REPORTING }}
107107
run: |
108108
echo "${PLUS_LICENSE}" > license.jwt
109109
kubectl create secret generic nplus-license --from-file license.jwt -n nginx-gateway
@@ -155,7 +155,7 @@ jobs:
155155
- name: Create plus secrets
156156
if: ${{ inputs.image == 'plus' }}
157157
env:
158-
PLUS_LICENSE: ${{ secrets.JWT_PLUS_REGISTRY }}
158+
PLUS_LICENSE: ${{ secrets.JWT_PLUS_REPORTING }}
159159
run: |
160160
echo "${PLUS_LICENSE}" > license.jwt
161161
kubectl create secret docker-registry nginx-plus-registry-secret --docker-server=private-registry.nginx.com --docker-username=${{ secrets.JWT_PLUS_REGISTRY }} --docker-password=none -n nginx-gateway

‎.github/workflows/nfr.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -114,7 +114,7 @@ jobs:
114114
- name: Setup license file for plus
115115
if: matrix.type == 'plus'
116116
env:
117-
PLUS_LICENSE: ${{ secrets.JWT_PLUS_REGISTRY }}
117+
PLUS_LICENSE: ${{ secrets.JWT_PLUS_REPORTING }}
118118
run: echo "${PLUS_LICENSE}" > license.jwt
119119

120120
- name: Create GKE cluster

‎Makefile

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,8 @@ GOARCH ?= amd64## The architecture of the image and/or binary. For example: amd6
5050
GOOS ?= linux## The OS of the image and/or binary. For example: linux or darwin
5151
PLUS_ENABLED ?= false
5252
PLUS_LICENSE_FILE ?= $(SELF_DIR)license.jwt
53+
PLUS_USAGE_ENDPOINT ?=## The N+ usage endpoint. For development, please set to the N1 staging endpoint.
54+
5355
override NGINX_DOCKER_BUILD_OPTIONS += --build-arg NJS_DIR=$(NJS_DIR) --build-arg NGINX_CONF_DIR=$(NGINX_CONF_DIR) --build-arg BUILD_AGENT=$(BUILD_AGENT)
5456

5557
.DEFAULT_GOAL := help
@@ -220,17 +222,23 @@ load-images-with-plus: ## Load NGF and NGINX Plus images on configured kind clus
220222
install-ngf-local-build: build-images load-images helm-install-local ## Install NGF from local build on configured kind cluster.
221223

222224
.PHONY: install-ngf-local-build-with-plus
223-
install-ngf-local-build-with-plus: build-images-with-plus load-images-with-plus helm-install-local-with-plus ## Install NGF with NGINX Plus from local build on configured kind cluster.
225+
install-ngf-local-build-with-plus: check-for-plus-usage-endpoint build-images-with-plus load-images-with-plus helm-install-local-with-plus ## Install NGF with NGINX Plus from local build on configured kind cluster.
224226

225227
.PHONY: helm-install-local
226228
helm-install-local: install-gateway-crds ## Helm install NGF on configured kind cluster with local images. To build, load, and install with helm run make install-ngf-local-build.
227229
helm install nginx-gateway $(CHART_DIR) --set nginx.image.repository=$(NGINX_PREFIX) --create-namespace --wait --set nginxGateway.image.pullPolicy=Never --set service.type=NodePort --set nginxGateway.image.repository=$(PREFIX) --set nginxGateway.image.tag=$(TAG) --set nginx.image.tag=$(TAG) --set nginx.image.pullPolicy=Never --set nginxGateway.gwAPIExperimentalFeatures.enable=$(ENABLE_EXPERIMENTAL) -n nginx-gateway $(HELM_PARAMETERS)
228230

229231
.PHONY: helm-install-local-with-plus
230-
helm-install-local-with-plus: install-gateway-crds ## Helm install NGF with NGINX Plus on configured kind cluster with local images. To build, load, and install with helm run make install-ngf-local-build-with-plus.
232+
helm-install-local-with-plus: check-for-plus-usage-endpoint install-gateway-crds ## Helm install NGF with NGINX Plus on configured kind cluster with local images. To build, load, and install with helm run make install-ngf-local-build-with-plus.
231233
kubectl create namespace nginx-gateway || true
232234
kubectl -n nginx-gateway create secret generic nplus-license --from-file $(PLUS_LICENSE_FILE) || true
233-
helm install nginx-gateway $(CHART_DIR) --set nginx.image.repository=$(NGINX_PLUS_PREFIX) --wait --set nginxGateway.image.pullPolicy=Never --set service.type=NodePort --set nginxGateway.image.repository=$(PREFIX) --set nginxGateway.image.tag=$(TAG) --set nginx.image.tag=$(TAG) --set nginx.image.pullPolicy=Never --set nginxGateway.gwAPIExperimentalFeatures.enable=$(ENABLE_EXPERIMENTAL) -n nginx-gateway --set nginx.plus=true $(HELM_PARAMETERS)
235+
helm install nginx-gateway $(CHART_DIR) --set nginx.image.repository=$(NGINX_PLUS_PREFIX) --wait --set nginxGateway.image.pullPolicy=Never --set service.type=NodePort --set nginxGateway.image.repository=$(PREFIX) --set nginxGateway.image.tag=$(TAG) --set nginx.image.tag=$(TAG) --set nginx.image.pullPolicy=Never --set nginxGateway.gwAPIExperimentalFeatures.enable=$(ENABLE_EXPERIMENTAL) -n nginx-gateway --set nginx.plus=true --set nginx.usage.endpoint=$(PLUS_USAGE_ENDPOINT) $(HELM_PARAMETERS)
236+
237+
.PHONY: check-for-plus-usage-endpoint
238+
check-for-plus-usage-endpoint: ## Checks that the PLUS_USAGE_ENDPOINT is set in the environment. This env var is required when deploying or testing with N+.
239+
ifndef PLUS_USAGE_ENDPOINT
240+
$(error PLUS_USAGE_ENDPOINT must be defined in your environment)
241+
endif
234242

235243
# Debug Targets
236244
.PHONY: debug-build

‎docs/developer/quickstart.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -122,6 +122,7 @@ This will build the docker images `nginx-gateway-fabric:<your-user>` and `nginx-
122122
> Note: You will need a valid NGINX Plus license certificate and key named `nginx-repo.crt` and `nginx-repo.key` in the
123123
> root of this repo to build the NGINX Plus image.
124124
> You will also need a valid NGINX Plus JSON Web Token (JWT) to deploy NGF with NGINX Plus. That JWT should be stored in the `license.jwt` file in the root of the `nginx-gateway-fabric/` directory. See the [documentation](https://docs.nginx.com/nginx-gateway-fabric/installation/nginx-plus-jwt/) for instructions on how to download and set up the JWT.
125+
> Additionally, you need to set the NGINX Plus usage endpoint in your environment. For development and testing, export `PLUS_USAGE_ENDPOINT=<N1 staging endpoint>`.
125126
126127
To build the NGINX Gateway Fabric and NGINX Plus container images from source run the following make command:
127128

‎tests/Makefile

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -109,7 +109,7 @@ sync-files-to-vm: ## Syncs your local NGF files with the NGF repo on the VM
109109
./scripts/sync-files-to-vm.sh
110110

111111
.PHONY: nfr-test
112-
nfr-test: ## Run the NFR tests on a GCP VM
112+
nfr-test: check-for-plus-usage-endpoint ## Run the NFR tests on a GCP VM
113113
CI=$(CI) ./scripts/run-tests-gcp-vm.sh
114114

115115
.PHONY: start-longevity-test
@@ -130,10 +130,10 @@ stop-longevity-test: nfr-test ## Stop the longevity test and collects results
130130
--ngf-image-repo=$(PREFIX) --nginx-image-repo=$(NGINX_PREFIX) --nginx-plus-image-repo=$(NGINX_PLUS_PREFIX) \
131131
--pull-policy=$(PULL_POLICY) --service-type=$(GW_SERVICE_TYPE) \
132132
--is-gke-internal-lb=$(GW_SVC_GKE_INTERNAL) --plus-enabled=$(PLUS_ENABLED) \
133-
--plus-license-file-name=$(PLUS_LICENSE_FILE)
133+
--plus-license-file-name=$(PLUS_LICENSE_FILE) --plus-usage-endpoint=$(PLUS_USAGE_ENDPOINT)
134134

135135
.PHONY: test
136-
test: build-crossplane-image ## Runs the functional tests on your kind k8s cluster
136+
test: check-for-plus-usage-endpoint build-crossplane-image ## Runs the functional tests on your kind k8s cluster
137137
kind load docker-image nginx-crossplane:latest --name $(CLUSTER_NAME)
138138
go run github.com/onsi/ginkgo/v2/ginkgo --randomize-all --randomize-suites --keep-going --fail-on-pending \
139139
--trace -r -v --buildvcs --force-newlines $(GITHUB_OUTPUT) \
@@ -143,7 +143,7 @@ test: build-crossplane-image ## Runs the functional tests on your kind k8s clust
143143
--ngf-image-repo=$(PREFIX) --nginx-image-repo=$(NGINX_PREFIX) --nginx-plus-image-repo=$(NGINX_PLUS_PREFIX) \
144144
--pull-policy=$(PULL_POLICY) --service-type=$(GW_SERVICE_TYPE) \
145145
--is-gke-internal-lb=$(GW_SVC_GKE_INTERNAL) --cluster-name=$(CLUSTER_NAME) --plus-enabled=$(PLUS_ENABLED) \
146-
--plus-license-file-name=$(PLUS_LICENSE_FILE)
146+
--plus-license-file-name=$(PLUS_LICENSE_FILE) --plus-usage-endpoint=$(PLUS_USAGE_ENDPOINT)
147147

148148
.PHONY: test-with-plus
149149
test-with-plus: PLUS_ENABLED=true
@@ -177,7 +177,7 @@ deploy-updated-provisioner: ## Update provisioner manifest and deploy to the con
177177

178178
.PHONY: generate-static-deployment
179179
generate-static-deployment:
180-
helm template nginx-gateway $(CHART_DIR) --set nameOverride=nginx-gateway --set metrics.enable=false --set nginxGateway.productTelemetry.enable=false -n nginx-gateway -s templates/deployment.yaml --set nginxGateway.image.repository=$(PREFIX) --set nginxGateway.image.tag=$(TAG) --set nginxGateway.image.pullPolicy=Never --set nginx.image.repository=$(NGINX_PREFIX) --set nginx.image.tag=$(TAG) --set nginx.image.pullPolicy=Never --set nginxGateway.gwAPIExperimentalFeatures.enable=$(ENABLE_EXPERIMENTAL) --set nginx.plus=$(PLUS_ENABLED) > $(SELF_DIR)config/tests/static-deployment.yaml
180+
helm template nginx-gateway $(CHART_DIR) --set nameOverride=nginx-gateway --set metrics.enable=false --set nginxGateway.productTelemetry.enable=false -n nginx-gateway -s templates/deployment.yaml --set nginxGateway.image.repository=$(PREFIX) --set nginxGateway.image.tag=$(TAG) --set nginxGateway.image.pullPolicy=Never --set nginx.image.repository=$(NGINX_PREFIX) --set nginx.image.tag=$(TAG) --set nginx.image.pullPolicy=Never --set nginxGateway.gwAPIExperimentalFeatures.enable=$(ENABLE_EXPERIMENTAL) --set nginx.plus=$(PLUS_ENABLED) --set nginx.usage.endpoint=$(PLUS_USAGE_ENDPOINT) > $(SELF_DIR)config/tests/static-deployment.yaml
181181

182182
# this target is used to install the gateway-api CRDs from the main branch (only used in the nightly CI job)
183183
# it overrides the target in the main Makefile when the GW_API_VERSION is set to main

‎tests/framework/ngf.go

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,8 +33,9 @@ type InstallationConfig struct {
3333
ImageTag string
3434
ImagePullPolicy string
3535
ServiceType string
36-
IsGKEInternalLB bool
36+
PlusUsageEndpoint string
3737
Plus bool
38+
IsGKEInternalLB bool
3839
}
3940

4041
// InstallGatewayAPI installs the specified version of the Gateway API resources.
@@ -78,6 +79,7 @@ func InstallNGF(cfg InstallationConfig, extraArgs ...string) ([]byte, error) {
7879
}
7980

8081
args = append(args, setImageArgs(cfg)...)
82+
args = append(args, setPlusUsageEndpointArg(cfg)...)
8183
fullArgs := append(args, extraArgs...) //nolint:gocritic
8284

8385
GinkgoWriter.Printf("Installing NGF with command: helm %v\n", strings.Join(fullArgs, " "))
@@ -226,6 +228,15 @@ func setImageArgs(cfg InstallationConfig) []string {
226228
return args
227229
}
228230

231+
func setPlusUsageEndpointArg(cfg InstallationConfig) []string {
232+
var args []string
233+
if cfg.Plus && cfg.PlusUsageEndpoint != "" {
234+
args = append(args, formatValueSet("nginx.usage.endpoint", cfg.PlusUsageEndpoint)...)
235+
}
236+
237+
return args
238+
}
239+
229240
func formatValueSet(key, value string) []string {
230241
return []string{"--set", fmt.Sprintf("%s=%s", key, value)}
231242
}

‎tests/suite/graceful_recovery_test.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -367,7 +367,7 @@ func checkContainerLogsForErrors(ngfPodName string, checkNginxLogsOnly bool) {
367367
Expect(line).ToNot(ContainSubstring("[emerg]"), line)
368368
if strings.Contains(line, "[error]") {
369369
expectedError1 := "connect() failed (111: Connection refused)"
370-
expectedError2 := "product.connect.nginx.com could not be resolved"
370+
expectedError2 := "could not be resolved (host not found) during usage report"
371371
expectedError3 := "server returned 429"
372372
// FIXME(salonichf5) remove this error message check
373373
// when https://github.com/nginxinc/nginx-gateway-fabric/issues/2090 is completed.

‎tests/suite/system_suite_test.go

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,7 @@ var (
6262
isGKEInternalLB = flag.Bool("is-gke-internal-lb", false, "Is the LB service GKE internal only")
6363
plusEnabled = flag.Bool("plus-enabled", false, "Is NGINX Plus enabled")
6464
plusLicenseFileName = flag.String("plus-license-file-name", "", "File name containing the NGINX Plus JWT")
65+
plusUsageEndpoint = flag.String("plus-usage-endpoint", "", "Endpoint for reporting NGINX Plus usage")
6566
clusterName = flag.String("cluster-name", "kind", "Cluster name")
6667
)
6768

@@ -190,12 +191,13 @@ func setup(cfg setupConfig, extraInstallArgs ...string) {
190191

191192
func createNGFInstallConfig(cfg setupConfig, extraInstallArgs ...string) framework.InstallationConfig {
192193
installCfg := framework.InstallationConfig{
193-
ReleaseName: cfg.releaseName,
194-
Namespace: ngfNamespace,
195-
ChartPath: cfg.chartPath,
196-
ServiceType: *serviceType,
197-
IsGKEInternalLB: *isGKEInternalLB,
198-
Plus: *plusEnabled,
194+
ReleaseName: cfg.releaseName,
195+
Namespace: ngfNamespace,
196+
ChartPath: cfg.chartPath,
197+
ServiceType: *serviceType,
198+
IsGKEInternalLB: *isGKEInternalLB,
199+
Plus: *plusEnabled,
200+
PlusUsageEndpoint: *plusUsageEndpoint,
199201
}
200202

201203
// if we aren't installing from the public charts, then set the custom images

0 commit comments

Comments
 (0)