diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 663ef8b766..35d8d4111f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -55,9 +55,6 @@ jobs: - name: Check if generated go files are up to date run: make generate && git diff --exit-code - - name: Check if njs-modules yaml is up to date - run: make generate-njs-yaml && git diff --exit-code - - name: Check if generated manifests are up to date run: make generate-manifests && git diff --exit-code @@ -173,8 +170,8 @@ jobs: - name: Docker Buildx uses: docker/setup-buildx-action@4c0219f9ac95b02789c1075625400b2acbff50b1 # v2.9.1 - - name: Docker meta - id: meta + - name: NKG Docker meta + id: nkg-meta uses: docker/metadata-action@818d4b7b91585d195f67373fd9cb0332e31a7175 # v4.6.0 with: images: | @@ -185,17 +182,43 @@ jobs: type=ref,event=pr type=ref,event=branch,suffix=-rc,enable=${{ startsWith(github.ref, 'refs/heads/release') }} - - name: Build Docker Image + - name: NGINX Docker meta + id: nginx-meta + uses: docker/metadata-action@818d4b7b91585d195f67373fd9cb0332e31a7175 # v4.6.0 + with: + images: | + name=ghcr.io/nginxinc/nginx-kubernetes-gateway/nginx + tags: | + type=semver,pattern={{version}} + type=edge + type=ref,event=pr + type=ref,event=branch,suffix=-rc,enable=${{ startsWith(github.ref, 'refs/heads/release') }} + + - name: Build NKG Docker Image uses: docker/build-push-action@2eb1c1961a95fc15694676618e422e8ba1d63825 # v4.1.1 with: file: build/Dockerfile - tags: ${{ steps.meta.outputs.tags }} + tags: ${{ steps.nkg-meta.outputs.tags }} context: "." target: goreleaser load: true - cache-from: type=gha - cache-to: type=gha,mode=max + cache-from: type=gha,scope=nkg + cache-to: type=gha,scope=nkg,mode=max + pull: true + + - name: Build NGINX Docker Image + uses: docker/build-push-action@2eb1c1961a95fc15694676618e422e8ba1d63825 # v4.1.1 + with: + file: build/Dockerfile.nginx + tags: ${{ steps.nginx-meta.outputs.tags }} + context: "." + load: true + cache-from: type=gha,scope=nginx + cache-to: type=gha,scope=nginx,mode=max pull: true + build-args: | + NJS_DIR=internal/mode/static/nginx/modules/src + NGINX_CONF_DIR=internal/mode/static/nginx/conf - name: Deploy Kubernetes id: k8s @@ -203,19 +226,23 @@ jobs: kube_config=${{ github.workspace }}/deploy/helm-chart/kube-${{ github.run_id }}-helm make create-kind-cluster KIND_KUBE_CONFIG=${kube_config} echo "KUBECONFIG=${kube_config}" >> "$GITHUB_ENV" - kind load docker-image ${{ steps.meta.outputs.tags }} + kind load docker-image ${{ steps.nkg-meta.outputs.tags }} ${{ steps.nginx-meta.outputs.tags }} kubectl apply -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v0.7.1/standard-install.yaml kubectl wait --for=condition=complete job/gateway-api-admission-patch job/gateway-api-admission -n gateway-system - name: Install Chart run: > helm install - helm-$(echo ${{ steps.meta.outputs.tags }} | cut -d ":" -f 2) + helm-$(echo ${{ steps.nkg-meta.outputs.tags }} | cut -d ":" -f 2) . --wait --create-namespace - --set controller.image.repository=$(echo ${{ steps.meta.outputs.tags }} | cut -d ":" -f 1) - --set controller.image.tag=$(echo ${{ steps.meta.outputs.tags }} | cut -d ":" -f 2) + --set nginxGateway.image.repository=$(echo ${{ steps.nkg-meta.outputs.tags }} | cut -d ":" -f 1) + --set nginxGateway.image.tag=$(echo ${{ steps.nkg-meta.outputs.tags }} | cut -d ":" -f 2) + --set nginxGateway.image.pullPolicy=Never + --set nginx.image.repository=$(echo ${{ steps.nginx-meta.outputs.tags }} | cut -d ":" -f 1) + --set nginx.image.tag=$(echo ${{ steps.nginx-meta.outputs.tags }} | cut -d ":" -f 2) + --set nginx.image.pullPolicy=Never --set service.type=NodePort -n nginx-gateway working-directory: ${{ github.workspace }}/deploy/helm-chart @@ -224,6 +251,10 @@ jobs: name: Build Image runs-on: ubuntu-22.04 needs: [vars, binary] + strategy: + fail-fast: false + matrix: + container: [nkg, nginx] permissions: contents: read # for docker/build-push-action to read repo content security-events: write # for github/codeql-action/upload-sarif to upload SARIF results @@ -259,7 +290,7 @@ jobs: uses: docker/metadata-action@818d4b7b91585d195f67373fd9cb0332e31a7175 # v4.6.0 with: images: | - name=ghcr.io/nginxinc/nginx-kubernetes-gateway + name=ghcr.io/nginxinc/nginx-kubernetes-gateway${{ matrix.container == 'nginx' && '/nginx' || '' }} tags: | type=semver,pattern={{version}} type=edge @@ -269,42 +300,46 @@ jobs: - name: Build Docker Image uses: docker/build-push-action@2eb1c1961a95fc15694676618e422e8ba1d63825 # v4.1.1 with: - file: build/Dockerfile + file: ${{ matrix.container == 'nginx' && 'build/Dockerfile.nginx' || 'build/Dockerfile' }} context: "." - target: goreleaser + target: ${{ matrix.container == 'nkg' && 'goreleaser' || '' }} tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} load: ${{ github.event_name == 'pull_request' }} push: ${{ github.event_name != 'pull_request' }} platforms: ${{ github.event_name != 'pull_request' && env.platforms || '' }} - cache-from: type=gha - cache-to: type=gha,mode=max + cache-from: type=gha,scope=${{ matrix.container }} + cache-to: type=gha,scope=${{ matrix.container }},mode=max pull: true no-cache: ${{ github.event_name != 'pull_request' }} sbom: ${{ github.event_name != 'pull_request' }} provenance: false + build-args: | + NJS_DIR=internal/mode/static/nginx/modules/src + NGINX_CONF_DIR=internal/mode/static/nginx/conf + - name: Run Trivy vulnerability scanner uses: aquasecurity/trivy-action@41f05d9ecffa2ed3f1580af306000f734b733e54 # 0.11.2 continue-on-error: true with: - image-ref: ghcr.io/nginxinc/nginx-kubernetes-gateway:${{ steps.meta.outputs.version }} + image-ref: ghcr.io/nginxinc/nginx-kubernetes-gateway${{ matrix.container == 'nginx' && '/nginx' || '' }}:${{ steps.meta.outputs.version }} format: "sarif" - output: "trivy-results-nginx-kubernetes-gateway.sarif" + output: trivy-results-nginx-kubernetes-gateway${{ matrix.container == 'nginx' && '-nginx' || '' }}.sarif ignore-unfixed: "true" - name: Upload Trivy scan results to GitHub Security tab uses: github/codeql-action/upload-sarif@5b6282e01c62d02e720b81eb8a51204f527c3624 # v2.21.3 continue-on-error: true with: - sarif_file: "trivy-results-nginx-kubernetes-gateway.sarif" + sarif_file: trivy-results-nginx-kubernetes-gateway${{ matrix.container == 'nginx' && '-nginx' || '' }}.sarif - name: Upload Scan Results uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2 continue-on-error: true with: - name: "trivy-results-nginx-kubernetes-gateway.sarif" - path: "trivy-results-nginx-kubernetes-gateway.sarif" + name: trivy-results-nginx-kubernetes-gateway${{ matrix.container == 'nginx' && '-nginx' || '' }}.sarif + path: trivy-results-nginx-kubernetes-gateway${{ matrix.container == 'nginx' && '-nginx' || '' }}.sarif if: always() publish-helm: diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml index 54cddbb02b..be73862a7b 100644 --- a/.github/workflows/conformance.yml +++ b/.github/workflows/conformance.yml @@ -42,8 +42,8 @@ jobs: - name: Docker Buildx uses: docker/setup-buildx-action@4c0219f9ac95b02789c1075625400b2acbff50b1 # v2.9.1 - - name: Docker meta - id: meta + - name: NKG Docker meta + id: nkg-meta uses: docker/metadata-action@818d4b7b91585d195f67373fd9cb0332e31a7175 # v4.6.0 with: images: | @@ -54,10 +54,22 @@ jobs: type=ref,event=pr type=ref,event=branch,suffix=-rc,enable=${{ startsWith(github.ref, 'refs/heads/release') }} + - name: NGINX Docker meta + id: nginx-meta + uses: docker/metadata-action@818d4b7b91585d195f67373fd9cb0332e31a7175 # v4.6.0 + with: + images: | + name=ghcr.io/nginxinc/nginx-kubernetes-gateway/nginx + tags: | + type=semver,pattern={{version}} + type=edge + type=ref,event=pr + type=ref,event=branch,suffix=-rc,enable=${{ startsWith(github.ref, 'refs/heads/release') }} + - name: Prepare NKG files run: | - nkg_prefix=$(echo ${{ steps.meta.outputs.tags }} | cut -d ":" -f 1) - nkg_tag=$(echo ${{ steps.meta.outputs.tags }} | cut -d ":" -f 2) + nkg_prefix=$(echo ${{ steps.nkg-meta.outputs.tags }} | cut -d ":" -f 1) + nkg_tag=$(echo ${{ steps.nkg-meta.outputs.tags }} | cut -d ":" -f 2) make update-nkg-manifest NKG_PREFIX=${nkg_prefix} NKG_TAG=${nkg_tag} working-directory: ./conformance @@ -67,17 +79,31 @@ jobs: version: latest args: build --snapshot --clean - - name: Build Docker Image + - name: Build NKG Docker Image uses: docker/build-push-action@2eb1c1961a95fc15694676618e422e8ba1d63825 # v4.1.1 with: file: build/Dockerfile - tags: ${{ steps.meta.outputs.tags }} + tags: ${{ steps.nkg-meta.outputs.tags }} context: "." target: goreleaser load: true - cache-from: type=gha - cache-to: type=gha,mode=max + cache-from: type=gha,scope=nkg + cache-to: type=gha,scope=nkg,mode=max + pull: true + + - name: Build NGINX Docker Image + uses: docker/build-push-action@2eb1c1961a95fc15694676618e422e8ba1d63825 # v4.1.1 + with: + file: build/Dockerfile.nginx + tags: ${{ steps.nginx-meta.outputs.tags }} + context: "." + load: true + cache-from: type=gha,scope=nginx + cache-to: type=gha,scope=nginx,mode=max pull: true + build-args: | + NJS_DIR=internal/mode/static/nginx/modules/src + NGINX_CONF_DIR=internal/mode/static/nginx/conf - name: Update Go Modules if: ${{ github.event_name == 'schedule' }} @@ -104,8 +130,8 @@ jobs: - name: Setup conformance tests run: | - nkg_prefix=$(echo ${{ steps.meta.outputs.tags }} | cut -d ":" -f 1) - nkg_tag=$(echo ${{ steps.meta.outputs.tags }} | cut -d ":" -f 2) + nkg_prefix=$(echo ${{ steps.nkg-meta.outputs.tags }} | cut -d ":" -f 1) + nkg_tag=$(echo ${{ steps.nkg-meta.outputs.tags }} | cut -d ":" -f 2) if [ ${{ github.event_name }} == "schedule" ]; then export GW_API_VERSION=main fi diff --git a/Makefile b/Makefile index c5b3807f9a..e3ec3f8316 100644 --- a/Makefile +++ b/Makefile @@ -3,8 +3,9 @@ VERSION = edge GIT_COMMIT = $(shell git rev-parse HEAD || echo "unknown") DATE = $(shell date -u +"%Y-%m-%dT%H:%M:%SZ") MANIFEST_DIR = $(shell pwd)/deploy/manifests -NJS_DIR = $(shell pwd)/internal/mode/static/nginx/modules/src CHART_DIR = $(shell pwd)/deploy/helm-chart +NGINX_CONF_DIR = internal/mode/static/nginx/conf +NJS_DIR = internal/mode/static/nginx/modules/src # go build flags - should not be overridden by the user GO_LINKER_FlAGS_VARS = -X main.version=${VERSION} -X main.commit=${GIT_COMMIT} -X main.date=${DATE} @@ -12,7 +13,8 @@ GO_LINKER_FLAGS_OPTIMIZATIONS = -s -w GO_LINKER_FLAGS = $(GO_LINKER_FLAGS_OPTIMIZATIONS) $(GO_LINKER_FlAGS_VARS) # variables that can be overridden by the user -PREFIX ?= nginx-kubernetes-gateway## The name of the image. For example, nginx-kubernetes-gateway +PREFIX ?= nginx-kubernetes-gateway## The name of the NKG image. For example, nginx-kubernetes-gateway +NGINX_PREFIX ?= $(PREFIX)/nginx## The name of the nginx image. For example: nginx-kubernetes-gateway/nginx TAG ?= $(VERSION:v%=%)## The tag of the image. For example, 0.3.0 TARGET ?= local## The target of the build. Possible values: local and container KIND_KUBE_CONFIG=$${HOME}/.kube/kind/config## The location of the kind kubeconfig @@ -21,7 +23,7 @@ ARCH ?= amd64## The architecture of the image and/or binary. For example: amd64 override HELM_TEMPLATE_COMMON_ARGS += --set creator=template --set nameOverride=nginx-gateway## The common options for the Helm template command. override HELM_TEMPLATE_EXTRA_ARGS_FOR_ALL_MANIFESTS_FILE += --set service.create=false## The options to be passed to the full Helm templating command only. override DOCKER_BUILD_OPTIONS += --build-arg VERSION=$(VERSION) --build-arg GIT_COMMIT=$(GIT_COMMIT) --build-arg DATE=$(DATE)## The options for the docker build command. For example, --pull - +override NGINX_DOCKER_BUILD_OPTIONS += --build-arg NJS_DIR=$(NJS_DIR) --build-arg NGINX_CONF_DIR=$(NGINX_CONF_DIR) .DEFAULT_GOAL := help .PHONY: help @@ -29,11 +31,21 @@ help: Makefile ## Display this help @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "; printf "Usage:\n\n make \033[36m\033[0m [VARIABLE=value...]\n\nTargets:\n\n"}; {printf " \033[36m%-30s\033[0m %s\n", $$1, $$2}' @grep -E '^(override )?[a-zA-Z_-]+ \??\+?= .*?## .*$$' $< | sort | awk 'BEGIN {FS = " \\??\\+?= .*?## "; printf "\nVariables:\n\n"}; {gsub(/override /, "", $$1); printf " \033[36m%-30s\033[0m %s\n", $$1, $$2}' -.PHONY: container -container: build ## Build the container - @docker -v || (code=$$?; printf "\033[0;31mError\033[0m: there was a problem with Docker\n"; exit $$code) +.PHONY: build-images +build-images: build-nkg-image build-nginx-image ## Build the NKG and nginx docker images + +.PHONY: build-nkg-image +build-nkg-image: check-for-docker build ## Build the NKG docker image docker build --platform linux/$(ARCH) $(strip $(DOCKER_BUILD_OPTIONS)) --target $(strip $(TARGET)) -f build/Dockerfile -t $(strip $(PREFIX)):$(strip $(TAG)) . +.PHONY: build-nginx-image +build-nginx-image: check-for-docker ## Build the custom nginx image + docker build --platform linux/$(ARCH) $(strip $(NGINX_DOCKER_BUILD_OPTIONS)) -f build/Dockerfile.nginx -t $(strip $(NGINX_PREFIX)):$(strip $(TAG)) . + +.PHONY: check-for-docker +check-for-docker: ## Check if Docker is installed + @docker -v || (code=$$?; printf "\033[0;31mError\033[0m: there was a problem with Docker\n"; exit $$code) + .PHONY: build build: ## Build the binary ifeq (${TARGET},local) @@ -103,10 +115,6 @@ njs-unit-test: ## Run unit tests for the njs httpmatches module node:18 \ /bin/bash -c "npm install && npm test && npm run clean" -.PHONY: generate-njs-yaml -generate-njs-yaml: ## Generate the njs-modules ConfigMap - kubectl create configmap njs-modules --from-file=$(NJS_DIR)/httpmatches.js --dry-run=client --output=yaml > $(strip $(MANIFEST_DIR))/njs-modules.yaml - .PHONY: lint-helm lint-helm: ## Run the helm chart linter helm lint $(CHART_DIR) @@ -116,8 +124,8 @@ debug-build: GO_LINKER_FLAGS=$(GO_LINKER_FlAGS_VARS) debug-build: ADDITIONAL_GO_BUILD_FLAGS=-gcflags "all=-N -l" debug-build: build ## Build binary with debug info, symbols, and no optimizations -.PHONY: debug-container -debug-container: debug-build container ## Build container with debug binary +.PHONY: build-nkg-debug-image +build-nkg-debug-image: debug-build build-nkg-image ## Build NKG image with debug binary .PHONY: generate-manifests generate-manifests: ## Generate manifests using Helm. diff --git a/README.md b/README.md index b33c7847ca..6cf2a3faea 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ Learn about our [design principles](/docs/developer/design-principles.md) and [a 1. [Quick Start on a kind cluster](docs/running-on-kind.md). 2. [Install](docs/installation.md) NGINX Kubernetes Gateway. -3. [Build](docs/building-the-image.md) an NGINX Kubernetes Gateway container image from source or use a pre-built image +3. [Build](docs/building-the-images.md) an NGINX Kubernetes Gateway container image from source or use a pre-built image available on [GitHub Container Registry](https://github.com/nginxinc/nginx-kubernetes-gateway/pkgs/container/nginx-kubernetes-gateway). 4. Deploy various [examples](examples). @@ -59,7 +59,7 @@ The following table lists the software versions NGINX Kubernetes Gateway support | NGINX Kubernetes Gateway | Gateway API | Kubernetes | NGINX OSS | |--------------------------|-------------|------------|-----------| -| Edge | 0.7.1 | 1.21+ | 1.25.x * | +| Edge | 0.7.1 | 1.21+ | 1.25.1 | | 0.5.0 | 0.7.1 | 1.21+ | 1.25.x * | | 0.4.0 | 0.7.1 | 1.21+ | 1.25.x * | | 0.3.0 | 0.6.2 | 1.21+ | 1.23.x * | diff --git a/build/Dockerfile b/build/Dockerfile index 7ecbf97681..3cebe4f25e 100644 --- a/build/Dockerfile +++ b/build/Dockerfile @@ -31,7 +31,7 @@ COPY dist/gateway_linux_$TARGETARCH*/gateway /usr/bin/ RUN setcap 'cap_kill=+ep' /usr/bin/gateway FROM scratch as common -USER 1001:1001 +USER 102:1001 ENTRYPOINT [ "/usr/bin/gateway" ] FROM common as container diff --git a/build/Dockerfile.nginx b/build/Dockerfile.nginx new file mode 100644 index 0000000000..6053430cbd --- /dev/null +++ b/build/Dockerfile.nginx @@ -0,0 +1,18 @@ +# syntax=docker/dockerfile:1.4 +FROM nginx:1.25.1-alpine + +ARG NJS_DIR +ARG NGINX_CONF_DIR + +RUN apk update && apk add --no-cache libcap \ + && mkdir -p /var/lib/nginx /usr/lib/nginx/modules \ + && setcap 'cap_net_bind_service=+ep' /usr/sbin/nginx \ + && setcap -v 'cap_net_bind_service=+ep' /usr/sbin/nginx \ + && apk del libcap + +COPY ${NJS_DIR}/httpmatches.js /usr/lib/nginx/modules/njs/httpmatches.js +COPY ${NGINX_CONF_DIR}/nginx.conf /etc/nginx/nginx.conf + +RUN chown -R 101:1001 /etc/nginx /var/cache/nginx /var/lib/nginx + +USER 101:1001 diff --git a/conformance/Makefile b/conformance/Makefile index 4520a19c48..841d575e12 100644 --- a/conformance/Makefile +++ b/conformance/Makefile @@ -1,5 +1,6 @@ NKG_TAG = edge NKG_PREFIX = nginx-kubernetes-gateway +NGINX_IMAGE_NAME = $(NKG_PREFIX)/nginx GW_API_VERSION ?= 0.7.1 GATEWAY_CLASS = nginx SUPPORTED_FEATURES = HTTPRoute,HTTPRouteQueryParamMatching,HTTPRouteMethodMatching,HTTPRoutePortRedirect,HTTPRouteSchemeRedirect,GatewayClassObservedGenerationBump @@ -32,22 +33,17 @@ create-kind-cluster: ## Create a kind cluster kind create cluster --image $(KIND_IMAGE) kind export kubeconfig --kubeconfig $(KIND_KUBE_CONFIG) -.PHONY: preload-nginx-container -preload-nginx-container: ## Preload NGINX container on configured kind cluster - docker pull $(NGINX_IMAGE) - kind load docker-image $(NGINX_IMAGE) - .PHONY: update-nkg-manifest -update-nkg-manifest: ## Update the NKG deployment manifest image name and imagePullPolicy - cd .. && make generate-manifests HELM_TEMPLATE_EXTRA_ARGS_FOR_ALL_MANIFESTS_FILE="--set nginxGateway.kind=skip" HELM_TEMPLATE_COMMON_ARGS="--set nginxGateway.image.repository=$(NKG_PREFIX) --set nginxGateway.image.tag=$(NKG_TAG) --set nginxGateway.image.pullPolicy=Never" && cd - +update-nkg-manifest: ## Update the NKG deployment manifest image names and imagePullPolicies + cd .. && make generate-manifests HELM_TEMPLATE_EXTRA_ARGS_FOR_ALL_MANIFESTS_FILE="--set nginxGateway.kind=skip" HELM_TEMPLATE_COMMON_ARGS="--set nginxGateway.image.repository=$(NKG_PREFIX) --set nginxGateway.image.tag=$(NKG_TAG) --set nginxGateway.image.pullPolicy=Never --set nginx.image.repository=$(NGINX_IMAGE_NAME) --set nginx.image.tag=$(NKG_TAG) --set nginx.image.pullPolicy=Never" && cd - -.PHONY: build-nkg-image -build-nkg-image: ## Build NKG container and load it and NGINX container on configured kind cluster - cd .. && make PREFIX=$(NKG_PREFIX) TAG=$(NKG_TAG) container +.PHONY: build-images +build-images: ## Build NKG and nginx images + cd .. && make PREFIX=$(NKG_PREFIX) TAG=$(NKG_TAG) build-images .PHONY: load-images -load-images: preload-nginx-container ## Load NKG and NGINX containers on configured kind cluster - kind load docker-image $(NKG_PREFIX):$(NKG_TAG) +load-images: ## Load NKG and NGINX images on configured kind cluster + kind load docker-image $(NKG_PREFIX):$(NKG_TAG) $(NGINX_IMAGE_NAME):$(NKG_TAG) .PHONY: prepare-nkg-dependencies prepare-nkg-dependencies: update-nkg-manifest ## Install NKG dependencies on configured kind cluster @@ -62,13 +58,13 @@ deploy-updated-provisioner: ## Update provisioner manifest and deploy to the con yq '(select(.spec.template.spec.containers[].image) | .spec.template.spec.containers[].image="$(NKG_PREFIX):$(NKG_TAG)" | .spec.template.spec.containers[].imagePullPolicy = "Never")' $(PROVISIONER_MANIFEST) | kubectl apply -f - .PHONY: install-nkg-local-build -install-nkg-local-build: prepare-nkg-dependencies build-nkg-image load-images deploy-updated-provisioner ## Install NKG from local build with provisioner on configured kind cluster +install-nkg-local-build: prepare-nkg-dependencies build-images load-images deploy-updated-provisioner ## Install NKG from local build with provisioner on configured kind cluster .PHONY: install-nkg-local-no-build install-nkg-local-no-build: prepare-nkg-dependencies load-images deploy-updated-provisioner ## Install NKG from local build with provisioner on configured kind cluster but do not build the NKG image .PHONY: install-nkg-edge -install-nkg-edge: preload-nginx-container prepare-nkg-dependencies ## Install NKG with provisioner from edge on configured kind cluster +install-nkg-edge: prepare-nkg-dependencies ## Install NKG with provisioner from edge on configured kind cluster kubectl apply -f $(PROVISIONER_MANIFEST) .PHONY: run-conformance-tests diff --git a/conformance/README.md b/conformance/README.md index a549e055f6..51c4e107e6 100644 --- a/conformance/README.md +++ b/conformance/README.md @@ -16,7 +16,7 @@ make ``` ```text -build-nkg-image Build NKG container and load it and NGINX container on configured kind cluster +build-images Build NKG and nginx images build-test-runner-image Build conformance test runner image cleanup-conformance-tests Clean up conformance tests fixtures create-kind-cluster Create a kind cluster @@ -26,15 +26,14 @@ help Display this help install-nkg-edge Install NKG with provisioner from edge on configured kind cluster install-nkg-local-build Install NKG from local build with provisioner on configured kind cluster install-nkg-local-no-build Install NKG from local build with provisioner on configured kind cluster but do not build the NKG image -load-images Load NKG and NGINX containers on configured kind cluster -preload-nginx-container Preload NGINX container on configured kind cluster +load-images Load NKG and NGINX images on configured kind cluster prepare-nkg-dependencies Install NKG dependencies on configured kind cluster reset-go-modules Reset the go modules changes run-conformance-tests Run conformance tests undo-manifests-update Undo the changes in the manifest files uninstall-nkg Uninstall NKG on configured kind cluster and undo manifest changes update-go-modules Update the gateway-api go modules to latest main version -update-nkg-manifest Update the NKG deployment manifest image name and imagePullPolicy +update-nkg-manifest Update the NKG deployment manifest image names and imagePullPolicies ``` **Note:** The following variables are configurable when running the below `make` commands: diff --git a/conformance/provisioner/static-deployment.yaml b/conformance/provisioner/static-deployment.yaml index f2a3415392..b311285182 100644 --- a/conformance/provisioner/static-deployment.yaml +++ b/conformance/provisioner/static-deployment.yaml @@ -41,11 +41,16 @@ spec: - KILL drop: - ALL - runAsUser: 1001 + runAsUser: 102 + runAsGroup: 1001 volumeMounts: - - mountPath: /etc/nginx - name: nginx - - image: nginx:1.25 + - name: nginx-conf + mountPath: /etc/nginx/conf.d + - name: nginx-secrets + mountPath: /etc/nginx/secrets + - name: nginx-run + mountPath: /var/run/nginx + - image: ghcr.io/nginxinc/nginx-kubernetes-gateway/nginx:edge imagePullPolicy: Always name: nginx ports: @@ -56,44 +61,26 @@ spec: securityContext: capabilities: add: - - CHOWN - NET_BIND_SERVICE - - SETGID - - SETUID - - DAC_OVERRIDE drop: - ALL + runAsUser: 101 + runAsGroup: 1001 volumeMounts: - - mountPath: /etc/nginx - name: nginx - - mountPath: /etc/nginx/nginx.conf - name: nginx-conf - subPath: nginx.conf - - mountPath: /var/lib/nginx - name: var-lib-nginx - - mountPath: /usr/lib/nginx/modules/njs - name: njs-modules - initContainers: - - command: - - sh - - -c - - rm -r /etc/nginx/conf.d /etc/nginx/secrets; mkdir /etc/nginx/conf.d /etc/nginx/secrets - && chown 1001:0 /etc/nginx/conf.d /etc/nginx/secrets - image: busybox:1.36 - name: set-permissions - volumeMounts: - - mountPath: /etc/nginx - name: nginx + - name: nginx-conf + mountPath: /etc/nginx/conf.d + - name: nginx-secrets + mountPath: /etc/nginx/secrets + - name: nginx-run + mountPath: /var/run/nginx serviceAccountName: nginx-gateway shareProcessNamespace: true + securityContext: + fsGroup: 1001 volumes: - - emptyDir: {} - name: nginx - - configMap: - name: nginx-gateway-conf - name: nginx-conf - - emptyDir: {} - name: var-lib-nginx - - configMap: - name: nginx-gateway-njs-modules - name: njs-modules + - name: nginx-conf + emptyDir: {} + - name: nginx-secrets + emptyDir: {} + - name: nginx-run + emptyDir: {} diff --git a/deploy/helm-chart/README.md b/deploy/helm-chart/README.md index 07ed4d41f4..8bae0eab54 100644 --- a/deploy/helm-chart/README.md +++ b/deploy/helm-chart/README.md @@ -105,23 +105,21 @@ kubectl delete -f https://github.com/kubernetes-sigs/gateway-api/releases/downlo The following tables lists the configurable parameters of the NGINX Kubernetes Gateway chart and their default values. -|Parameter | Description | Default Value | -| --- | --- | --- | -|`nginxGateway.image.repository` | The repository for the NGINX Kubernetes Gateway image. | ghcr.io/nginxinc/nginx-kubernetes-gateway | -|`nginxGateway.image.tag` | The tag for the NGINX Kubernetes Gateway image. | edge | -|`nginxGateway.image.pullPolicy` | The `imagePullPolicy` for the NGINX Kubernetes Gateway image. | Always | -|`nginxGateway.gatewayClassName` | The name of the GatewayClass for the NGINX Kubernetes Gateway deployment. | nginx | -|`nginxGateway.gatewayControllerName` | The name of the Gateway controller. The controller name must be of the form: DOMAIN/PATH. The controller's domain is k8s-gateway.nginx.org. | k8s-gateway.nginx.org/nginx-gateway-controller | -|`nginxGateway.kind` | The kind of the NGINX Kubernetes Gateway installation - currently, only Deployment is supported. | deployment | -|`nginx.image.repository` | The repository for the NGINX image. | nginx | -|`nginx.image.tag` | The tag for the NGINX image. | 1.25 | -|`nginx.image.pullPolicy` | The `imagePullPolicy` for the NGINX image. | Always | -|`initContainer.image.repository` | The repository for the `initContainer` image. | busybox | -|`initContainer.image.tag` | The tag for the `initContainer` image. | 1.36 | -|`serviceAccount.annotations` | The `annotations` for the ServiceAccount used by the NGINX Kubernetes Gateway deployment. | {} | -|`serviceAccount.name` | Name of the ServiceAccount used by the NGINX Kubernetes Gateway deployment. | Autogenerated | -|`service.create` | Creates a service to expose the NGINX Kubernetes Gateway pods. | true | -|`service.type` | The type of service to create for the NGINX Kubernetes Gateway. | Loadbalancer | -|`service.externalTrafficPolicy` | The `externalTrafficPolicy` of the service. The value `Local` preserves the client source IP. | Local | -|`service.annotations` | The `annotations` of the NGINX Kubernetes Gateway service. | {} | -|`service.ports` | A list of ports to expose through the NGINX Kubernetes Gateway service. Update it to match the listener ports from your Gateway resource. Follows the conventional Kubernetes yaml syntax for service ports. | [ port: 80, targetPort: 80, protocol: TCP, name: http; port: 443, targetPort: 443, protocol: TCP, name: https ] | +| Parameter | Description | Default Value | +|--------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------| +| `nginxGateway.image.repository` | The repository for the NGINX Kubernetes Gateway image. | ghcr.io/nginxinc/nginx-kubernetes-gateway | +| `nginxGateway.image.tag` | The tag for the NGINX Kubernetes Gateway image. | edge | +| `nginxGateway.image.pullPolicy` | The `imagePullPolicy` for the NGINX Kubernetes Gateway image. | Always | +| `nginxGateway.gatewayClassName` | The name of the GatewayClass for the NGINX Kubernetes Gateway deployment. | nginx | +| `nginxGateway.gatewayControllerName` | The name of the Gateway controller. The controller name must be of the form: DOMAIN/PATH. The controller's domain is k8s-gateway.nginx.org. | k8s-gateway.nginx.org/nginx-gateway-controller | +| `nginxGateway.kind` | The kind of the NGINX Kubernetes Gateway installation - currently, only Deployment is supported. | deployment | +| `nginx.image.repository` | The repository for the NGINX image. | ghcr.io/nginxinc/nginx-kubernetes-gateway/nginx | +| `nginx.image.tag` | The tag for the NGINX image. | edge | +| `nginx.image.pullPolicy` | The `imagePullPolicy` for the NGINX image. | Always | +| `serviceAccount.annotations` | The `annotations` for the ServiceAccount used by the NGINX Kubernetes Gateway deployment. | {} | +| `serviceAccount.name` | Name of the ServiceAccount used by the NGINX Kubernetes Gateway deployment. | Autogenerated | +| `service.create` | Creates a service to expose the NGINX Kubernetes Gateway pods. | true | +| `service.type` | The type of service to create for the NGINX Kubernetes Gateway. | Loadbalancer | +| `service.externalTrafficPolicy` | The `externalTrafficPolicy` of the service. The value `Local` preserves the client source IP. | Local | +| `service.annotations` | The `annotations` of the NGINX Kubernetes Gateway service. | {} | +| `service.ports` | A list of ports to expose through the NGINX Kubernetes Gateway service. Update it to match the listener ports from your Gateway resource. Follows the conventional Kubernetes yaml syntax for service ports. | [ port: 80, targetPort: 80, protocol: TCP, name: http; port: 443, targetPort: 443, protocol: TCP, name: https ] | diff --git a/deploy/helm-chart/templates/_helpers.tpl b/deploy/helm-chart/templates/_helpers.tpl index 5488944e85..c067c5aa33 100644 --- a/deploy/helm-chart/templates/_helpers.tpl +++ b/deploy/helm-chart/templates/_helpers.tpl @@ -58,17 +58,3 @@ Create the name of the ServiceAccount to use {{- define "nginx-gateway.serviceAccountName" -}} {{- default (include "nginx-gateway.fullname" .) .Values.serviceAccount.name }} {{- end }} - -{{/* -Expand default NGINX conf ConfigMap name. -*/}} -{{- define "nginx-gateway.nginx-conf" -}} -{{- printf "%s-%s" (include "nginx-gateway.fullname" .) "conf" -}} -{{- end -}} - -{{/* -Expand default njs-modules ConfigMap name. -*/}} -{{- define "nginx-gateway.njs-modules" -}} -{{- printf "%s-%s" (include "nginx-gateway.fullname" .) "njs-modules" -}} -{{- end -}} diff --git a/deploy/helm-chart/templates/deployment.yaml b/deploy/helm-chart/templates/deployment.yaml index e1cae67599..abd3874dab 100644 --- a/deploy/helm-chart/templates/deployment.yaml +++ b/deploy/helm-chart/templates/deployment.yaml @@ -36,11 +36,16 @@ spec: - KILL drop: - ALL - runAsUser: 1001 + runAsUser: 102 + runAsGroup: 1001 volumeMounts: - - mountPath: /etc/nginx - name: nginx - - image: {{ .Values.nginx.image.repository }}:{{ .Values.nginx.image.tag }} + - name: nginx-conf + mountPath: /etc/nginx/conf.d + - name: nginx-secrets + mountPath: /etc/nginx/secrets + - name: nginx-run + mountPath: /var/run/nginx + - image: {{ .Values.nginx.image.repository }}:{{ .Values.nginx.image.tag | default .Chart.AppVersion }} imagePullPolicy: {{ .Values.nginx.image.pullPolicy }} name: nginx ports: @@ -51,45 +56,27 @@ spec: securityContext: capabilities: add: - - CHOWN - NET_BIND_SERVICE - - SETGID - - SETUID - - DAC_OVERRIDE drop: - ALL + runAsUser: 101 + runAsGroup: 1001 volumeMounts: - - mountPath: /etc/nginx - name: nginx - - mountPath: /etc/nginx/nginx.conf - name: nginx-conf - subPath: nginx.conf - - mountPath: /var/lib/nginx - name: var-lib-nginx - - mountPath: /usr/lib/nginx/modules/njs - name: njs-modules - initContainers: - - command: - - sh - - -c - - rm -r /etc/nginx/conf.d /etc/nginx/secrets; mkdir /etc/nginx/conf.d /etc/nginx/secrets - && chown 1001:0 /etc/nginx/conf.d /etc/nginx/secrets - image: {{ .Values.initContainer.image.repository }}:{{ .Values.initContainer.image.tag }} - name: set-permissions - volumeMounts: - - mountPath: /etc/nginx - name: nginx + - name: nginx-conf + mountPath: /etc/nginx/conf.d + - name: nginx-secrets + mountPath: /etc/nginx/secrets + - name: nginx-run + mountPath: /var/run/nginx serviceAccountName: {{ include "nginx-gateway.serviceAccountName" . }} shareProcessNamespace: true + securityContext: + fsGroup: 1001 volumes: - - emptyDir: {} - name: nginx - - configMap: - name: {{ include "nginx-gateway.nginx-conf" . }} - name: nginx-conf - - emptyDir: {} - name: var-lib-nginx - - configMap: - name: {{ include "nginx-gateway.njs-modules" . }} - name: njs-modules + - name: nginx-conf + emptyDir: {} + - name: nginx-secrets + emptyDir: {} + - name: nginx-run + emptyDir: {} {{- end }} diff --git a/deploy/helm-chart/templates/nginx-conf.yaml b/deploy/helm-chart/templates/nginx-conf.yaml deleted file mode 100644 index dd64888508..0000000000 --- a/deploy/helm-chart/templates/nginx-conf.yaml +++ /dev/null @@ -1,23 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ include "nginx-gateway.nginx-conf" . }} - namespace: {{ .Release.Namespace }} - labels: - {{- include "nginx-gateway.labels" . | nindent 4 }} -data: - nginx.conf: |- - load_module /usr/lib/nginx/modules/ngx_http_js_module.so; - events {} - pid /etc/nginx/nginx.pid; - error_log stderr debug; - http { - include /etc/nginx/conf.d/*.conf; - js_import /usr/lib/nginx/modules/njs/httpmatches.js; - proxy_headers_hash_bucket_size 512; - proxy_headers_hash_max_size 1024; - server_names_hash_bucket_size 256; - server_names_hash_max_size 1024; - variables_hash_bucket_size 512; - variables_hash_max_size 1024; - } diff --git a/deploy/helm-chart/templates/njs-modules.yaml b/deploy/helm-chart/templates/njs-modules.yaml deleted file mode 100644 index d67650d584..0000000000 --- a/deploy/helm-chart/templates/njs-modules.yaml +++ /dev/null @@ -1,216 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ include "nginx-gateway.njs-modules" . }} - namespace: {{ .Release.Namespace }} - labels: - {{- include "nginx-gateway.labels" . | nindent 4 }} -data: - httpmatches.js: | - const MATCHES_VARIABLE = 'http_matches'; - const HTTP_CODES = { - notFound: 404, - internalServerError: 500, - }; - - function redirect(r) { - let matches; - - try { - matches = extractMatchesFromRequest(r); - } catch (e) { - r.error(e.message); - r.return(HTTP_CODES.internalServerError); - return; - } - - // Matches is a list of http matches in order of precedence. - // We will accept the first match that the request satisfies. - // If there's a match, redirect request to internal location block. - // If an exception occurs, return 500. - // If no matches are found, return 404. - let match; - try { - match = findWinningMatch(r, matches); - } catch (e) { - r.error(e.message); - r.return(HTTP_CODES.internalServerError); - return; - } - - if (!match) { - r.return(HTTP_CODES.notFound); - return; - } - - if (!match.redirectPath) { - r.error( - `cannot redirect the request; the match ${JSON.stringify( - match, - )} does not have a redirectPath set`, - ); - r.return(HTTP_CODES.internalServerError); - return; - } - - r.internalRedirect(match.redirectPath); - } - - function extractMatchesFromRequest(r) { - if (!r.variables[MATCHES_VARIABLE]) { - throw Error( - `cannot redirect the request; the variable ${MATCHES_VARIABLE} is not defined on the request object`, - ); - } - - let matches; - - try { - matches = JSON.parse(r.variables[MATCHES_VARIABLE]); - } catch (e) { - throw Error( - `cannot redirect the request; error parsing ${r.variables[MATCHES_VARIABLE]} into a JSON object: ${e}`, - ); - } - - if (!Array.isArray(matches)) { - throw Error(`cannot redirect the request; expected a list of matches, got ${matches}`); - } - - if (matches.length === 0) { - throw Error(`cannot redirect the request; matches is an empty list`); - } - - return matches; - } - - function findWinningMatch(r, matches) { - for (let i = 0; i < matches.length; i++) { - try { - let found = testMatch(r, matches[i]); - if (found) { - return matches[i]; - } - } catch (e) { - throw e; - } - } - - return null; - } - - function testMatch(r, match) { - // check for any - if (match.any) { - return true; - } - - // check method - if (match.method && r.method !== match.method) { - return false; - } - - // check headers - if (match.headers) { - try { - let found = headersMatch(r.headersIn, match.headers); - if (!found) { - return false; - } - } catch (e) { - throw e; - } - } - - // check params - if (match.params) { - try { - let found = paramsMatch(r.args, match.params); - if (!found) { - return false; - } - } catch (e) { - throw e; - } - } - - // all match conditions are satisfied so return true - return true; - } - - function headersMatch(requestHeaders, headers) { - for (let i = 0; i < headers.length; i++) { - const h = headers[i]; - const kv = h.split(':'); - - if (kv.length !== 2) { - throw Error(`invalid header match: ${h}`); - } - // Header names are compared in a case-insensitive manner, meaning header name "FOO" is equivalent to "foo". - // The NGINX request's headersIn object lookup is case-insensitive as well. - // This means that requestHeaders['FOO'] is equivalent to requestHeaders['foo']. - let val = requestHeaders[kv[0]]; - - if (!val) { - return false; - } - - // split on comma because nginx uses commas to delimit multiple header values - const values = val.split(','); - if (!values.includes(kv[1])) { - return false; - } - } - - return true; - } - - function paramsMatch(requestParams, params) { - for (let i = 0; i < params.length; i++) { - let p = params[i]; - // We store query parameter matches as strings with the format "key=value"; however, there may be more than one - // instance of "=" in the string. - // To recover the key and value, we need to find the first occurrence of "=" in the string. - const idx = params[i].indexOf('='); - // Check for an improperly constructed query parameter match. There are three possible error cases: - // (1) if the index is -1, then there are no "=" in the string (e.g. "keyvalue") - // (2) if the index is 0, then there is no value in the string (e.g. "key="). - // (3) if the index is equal to length -1, then there is no key in the string (e.g. "=value"). - if (idx === -1 || (idx === 0) | (idx === p.length - 1)) { - throw Error(`invalid query parameter: ${p}`); - } - - // Divide string into key value using the index. - let kv = [p.slice(0, idx), p.slice(idx + 1)]; - - // val can either be a string or an array of strings. - // Also, the NGINX request's args object lookup is case-sensitive. - // For example, 'a=1&b=2&A=3&b=4' will be parsed into {a: "1", b: ["2", "4"], A: "3"} - let val = requestParams[kv[0]]; - if (!val) { - return false; - } - - // If val is an array, we will match against the first element in the array according to the Gateway API spec. - if (Array.isArray(val)) { - val = val[0]; - } - - if (val !== kv[1]) { - return false; - } - } - - return true; - } - - export default { - redirect, - testMatch, - findWinningMatch, - headersMatch, - paramsMatch, - extractMatchesFromRequest, - HTTP_CODES, - MATCHES_VARIABLE, - }; diff --git a/deploy/helm-chart/values.yaml b/deploy/helm-chart/values.yaml index 28158bcd8a..e80a726cf3 100644 --- a/deploy/helm-chart/values.yaml +++ b/deploy/helm-chart/values.yaml @@ -17,16 +17,10 @@ nginxGateway: nginx: ## The NGINX image to use image: - repository: nginx - tag: "1.25" + repository: ghcr.io/nginxinc/nginx-kubernetes-gateway/nginx + tag: edge pullPolicy: Always -initContainer: - image: - ## The image the init container should use. - repository: busybox - tag: "1.36" - serviceAccount: annotations: {} ## The name of the service account of the NGINX Kubernetes Gateway pods. Used for RBAC. diff --git a/deploy/manifests/nginx-gateway.yaml b/deploy/manifests/nginx-gateway.yaml index 79837d0253..a419e9df58 100644 --- a/deploy/manifests/nginx-gateway.yaml +++ b/deploy/manifests/nginx-gateway.yaml @@ -16,253 +16,6 @@ metadata: annotations: {} --- -# Source: nginx-kubernetes-gateway/templates/nginx-conf.yaml -apiVersion: v1 -kind: ConfigMap -metadata: - name: nginx-gateway-conf - namespace: nginx-gateway - labels: - app.kubernetes.io/name: nginx-gateway - app.kubernetes.io/instance: nginx-gateway - app.kubernetes.io/version: "edge" -data: - nginx.conf: |- - load_module /usr/lib/nginx/modules/ngx_http_js_module.so; - events {} - pid /etc/nginx/nginx.pid; - error_log stderr debug; - http { - include /etc/nginx/conf.d/*.conf; - js_import /usr/lib/nginx/modules/njs/httpmatches.js; - proxy_headers_hash_bucket_size 512; - proxy_headers_hash_max_size 1024; - server_names_hash_bucket_size 256; - server_names_hash_max_size 1024; - variables_hash_bucket_size 512; - variables_hash_max_size 1024; - } ---- -# Source: nginx-kubernetes-gateway/templates/njs-modules.yaml -apiVersion: v1 -kind: ConfigMap -metadata: - name: nginx-gateway-njs-modules - namespace: nginx-gateway - labels: - app.kubernetes.io/name: nginx-gateway - app.kubernetes.io/instance: nginx-gateway - app.kubernetes.io/version: "edge" -data: - httpmatches.js: | - const MATCHES_VARIABLE = 'http_matches'; - const HTTP_CODES = { - notFound: 404, - internalServerError: 500, - }; - - function redirect(r) { - let matches; - - try { - matches = extractMatchesFromRequest(r); - } catch (e) { - r.error(e.message); - r.return(HTTP_CODES.internalServerError); - return; - } - - // Matches is a list of http matches in order of precedence. - // We will accept the first match that the request satisfies. - // If there's a match, redirect request to internal location block. - // If an exception occurs, return 500. - // If no matches are found, return 404. - let match; - try { - match = findWinningMatch(r, matches); - } catch (e) { - r.error(e.message); - r.return(HTTP_CODES.internalServerError); - return; - } - - if (!match) { - r.return(HTTP_CODES.notFound); - return; - } - - if (!match.redirectPath) { - r.error( - `cannot redirect the request; the match ${JSON.stringify( - match, - )} does not have a redirectPath set`, - ); - r.return(HTTP_CODES.internalServerError); - return; - } - - r.internalRedirect(match.redirectPath); - } - - function extractMatchesFromRequest(r) { - if (!r.variables[MATCHES_VARIABLE]) { - throw Error( - `cannot redirect the request; the variable ${MATCHES_VARIABLE} is not defined on the request object`, - ); - } - - let matches; - - try { - matches = JSON.parse(r.variables[MATCHES_VARIABLE]); - } catch (e) { - throw Error( - `cannot redirect the request; error parsing ${r.variables[MATCHES_VARIABLE]} into a JSON object: ${e}`, - ); - } - - if (!Array.isArray(matches)) { - throw Error(`cannot redirect the request; expected a list of matches, got ${matches}`); - } - - if (matches.length === 0) { - throw Error(`cannot redirect the request; matches is an empty list`); - } - - return matches; - } - - function findWinningMatch(r, matches) { - for (let i = 0; i < matches.length; i++) { - try { - let found = testMatch(r, matches[i]); - if (found) { - return matches[i]; - } - } catch (e) { - throw e; - } - } - - return null; - } - - function testMatch(r, match) { - // check for any - if (match.any) { - return true; - } - - // check method - if (match.method && r.method !== match.method) { - return false; - } - - // check headers - if (match.headers) { - try { - let found = headersMatch(r.headersIn, match.headers); - if (!found) { - return false; - } - } catch (e) { - throw e; - } - } - - // check params - if (match.params) { - try { - let found = paramsMatch(r.args, match.params); - if (!found) { - return false; - } - } catch (e) { - throw e; - } - } - - // all match conditions are satisfied so return true - return true; - } - - function headersMatch(requestHeaders, headers) { - for (let i = 0; i < headers.length; i++) { - const h = headers[i]; - const kv = h.split(':'); - - if (kv.length !== 2) { - throw Error(`invalid header match: ${h}`); - } - // Header names are compared in a case-insensitive manner, meaning header name "FOO" is equivalent to "foo". - // The NGINX request's headersIn object lookup is case-insensitive as well. - // This means that requestHeaders['FOO'] is equivalent to requestHeaders['foo']. - let val = requestHeaders[kv[0]]; - - if (!val) { - return false; - } - - // split on comma because nginx uses commas to delimit multiple header values - const values = val.split(','); - if (!values.includes(kv[1])) { - return false; - } - } - - return true; - } - - function paramsMatch(requestParams, params) { - for (let i = 0; i < params.length; i++) { - let p = params[i]; - // We store query parameter matches as strings with the format "key=value"; however, there may be more than one - // instance of "=" in the string. - // To recover the key and value, we need to find the first occurrence of "=" in the string. - const idx = params[i].indexOf('='); - // Check for an improperly constructed query parameter match. There are three possible error cases: - // (1) if the index is -1, then there are no "=" in the string (e.g. "keyvalue") - // (2) if the index is 0, then there is no value in the string (e.g. "key="). - // (3) if the index is equal to length -1, then there is no key in the string (e.g. "=value"). - if (idx === -1 || (idx === 0) | (idx === p.length - 1)) { - throw Error(`invalid query parameter: ${p}`); - } - - // Divide string into key value using the index. - let kv = [p.slice(0, idx), p.slice(idx + 1)]; - - // val can either be a string or an array of strings. - // Also, the NGINX request's args object lookup is case-sensitive. - // For example, 'a=1&b=2&A=3&b=4' will be parsed into {a: "1", b: ["2", "4"], A: "3"} - let val = requestParams[kv[0]]; - if (!val) { - return false; - } - - // If val is an array, we will match against the first element in the array according to the Gateway API spec. - if (Array.isArray(val)) { - val = val[0]; - } - - if (val !== kv[1]) { - return false; - } - } - - return true; - } - - export default { - redirect, - testMatch, - findWinningMatch, - headersMatch, - paramsMatch, - extractMatchesFromRequest, - HTTP_CODES, - MATCHES_VARIABLE, - }; ---- # Source: nginx-kubernetes-gateway/templates/rbac.yaml apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole @@ -382,11 +135,16 @@ spec: - KILL drop: - ALL - runAsUser: 1001 + runAsUser: 102 + runAsGroup: 1001 volumeMounts: - - mountPath: /etc/nginx - name: nginx - - image: nginx:1.25 + - name: nginx-conf + mountPath: /etc/nginx/conf.d + - name: nginx-secrets + mountPath: /etc/nginx/secrets + - name: nginx-run + mountPath: /var/run/nginx + - image: ghcr.io/nginxinc/nginx-kubernetes-gateway/nginx:edge imagePullPolicy: Always name: nginx ports: @@ -397,47 +155,29 @@ spec: securityContext: capabilities: add: - - CHOWN - NET_BIND_SERVICE - - SETGID - - SETUID - - DAC_OVERRIDE drop: - ALL + runAsUser: 101 + runAsGroup: 1001 volumeMounts: - - mountPath: /etc/nginx - name: nginx - - mountPath: /etc/nginx/nginx.conf - name: nginx-conf - subPath: nginx.conf - - mountPath: /var/lib/nginx - name: var-lib-nginx - - mountPath: /usr/lib/nginx/modules/njs - name: njs-modules - initContainers: - - command: - - sh - - -c - - rm -r /etc/nginx/conf.d /etc/nginx/secrets; mkdir /etc/nginx/conf.d /etc/nginx/secrets - && chown 1001:0 /etc/nginx/conf.d /etc/nginx/secrets - image: busybox:1.36 - name: set-permissions - volumeMounts: - - mountPath: /etc/nginx - name: nginx + - name: nginx-conf + mountPath: /etc/nginx/conf.d + - name: nginx-secrets + mountPath: /etc/nginx/secrets + - name: nginx-run + mountPath: /var/run/nginx serviceAccountName: nginx-gateway shareProcessNamespace: true + securityContext: + fsGroup: 1001 volumes: - - emptyDir: {} - name: nginx - - configMap: - name: nginx-gateway-conf - name: nginx-conf - - emptyDir: {} - name: var-lib-nginx - - configMap: - name: nginx-gateway-njs-modules - name: njs-modules + - name: nginx-conf + emptyDir: {} + - name: nginx-secrets + emptyDir: {} + - name: nginx-run + emptyDir: {} --- # Source: nginx-kubernetes-gateway/templates/gatewayclass.yaml apiVersion: gateway.networking.k8s.io/v1beta1 diff --git a/docs/architecture.md b/docs/architecture.md index cd081b39b6..c73a91189f 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -98,27 +98,33 @@ parentheses. To enhance readability, the suffix "process" has been omitted from 1. (HTTPS) *NKG* reads the *Kubernetes API* to get the latest versions of the resources in the cluster and writes to the API to update the handled resources' statuses and emit events. -2. (File I/O) *NKG* generates NGINX *configuration* based on the cluster resources and writes them as `.conf` files to -the mounted `nginx` volume, located at `/etc/nginx`. It also writes *TLS certificates* and *keys* -from [TLS Secrets][secrets] referenced in the accepted Gateway resource to the volume at the -path `/etc/nginx/secrets`. +2. (File I/O) + - Write: *NKG* generates NGINX *configuration* based on the cluster resources and writes them as `.conf` files to the + mounted `nginx-conf` volume, located at `/etc/nginx/conf.d`. It also writes *TLS certificates* and *keys* + from [TLS Secrets][secrets] referenced in the accepted Gateway resource to the `nginx-secrets` volume at the + path `/etc/nginx/secrets`. + - Read: *NKG* reads the PID file `nginx.pid` from the `nginx-run` volume, located at `/var/run/nginx`. *NKG* + extracts the PID of the nginx process from this file in order to send reload signals to *NGINX master*. 3. (File I/O) *NKG* writes logs to its *stdout* and *stderr*, which are collected by the container runtime. 4. (Signal) To reload NGINX, *NKG* sends the [reload signal][reload] to the **NGINX master**. -5. (File I/O) The *NGINX master* reads *configuration files* and the *TLS cert and keys* referenced in the -configuration when it starts or during a reload. These files, certificates, and keys are stored in the `nginx` volume -that is mounted to both the `nginx-gateway` and `nginx` containers. -6. (File I/O): The *NGINX master* writes to the auxiliary Unix sockets folder, which is mounted to the `nginx` -container as the `var-lib-nginx` volume. The mounted path for this volume is `/var/lib/nginx`. +5. (File I/O) + - Write: The *NGINX master* writes its PID to the `nginx.pid` file stored in the `nginx-run` volume. + - Read: The *NGINX master* reads *configuration files* and the *TLS cert and keys* referenced in the configuration when + it starts or during a reload. These files, certificates, and keys are stored in the `nginx-conf` and `nginx-secrets` + volumes that are mounted to both the `nginx-gateway` and `nginx` containers. +6. (File I/O) + - Write: The *NGINX master* writes to the auxiliary Unix sockets folder, which is located in the `/var/lib/nginx` + directory. + - Read: The *NGINX master* reads the `nginx.conf` file from the `/etc/nginx` directory. This [file][conf-file] contains + the global and http configuration settings for NGINX. In addition, *NGINX master* + reads the NJS modules referenced in the configuration when it starts or during a reload. NJS modules are stored in + the `/usr/lib/nginx/modules` directory. 7. (File I/O) The *NGINX master* sends logs to its *stdout* and *stderr*, which are collected by the container runtime. -8. (File I/O): The *NGINX master* reads the NJS modules referenced in the configuration when it starts or during a -reload. NJS modules are stored in the `njs-modules` volume that is mounted to the `nginx` container. -9. (File I/O) An *NGINX worker* writes logs to its *stdout* and *stderr*, which are collected by the container runtime. -10. (File I/O): The *NGINX master* reads the `nginx.conf` file from the mounted `nginx-conf` volume. -This [file][conf-file] contains the global and http configuration settings for NGINX. -11. (Signal) The *NGINX master* controls the [lifecycle of *NGINX workers*][lifecycle] it creates workers with the new +8. (File I/O) An *NGINX worker* writes logs to its *stdout* and *stderr*, which are collected by the container runtime. +9. (Signal) The *NGINX master* controls the [lifecycle of *NGINX workers*][lifecycle] it creates workers with the new configuration and shutdowns workers with the old configuration. -12. (HTTP,HTTPS) A *client* sends traffic to and receives traffic from any of the *NGINX workers* on ports 80 and 443. -13. (HTTP,HTTPS) An *NGINX worker* sends traffic to and receives traffic from the *backends*. +10. (HTTP,HTTPS) A *client* sends traffic to and receives traffic from any of the *NGINX workers* on ports 80 and 443. +11. (HTTP,HTTPS) An *NGINX worker* sends traffic to and receives traffic from the *backends*. [controller]: https://kubernetes.io/docs/concepts/architecture/controller/ @@ -130,6 +136,6 @@ configuration and shutdowns workers with the old configuration. [lifecycle]: https://nginx.org/en/docs/control.html#reconfiguration -[conf-file]: https://github.com/nginxinc/nginx-kubernetes-gateway/blob/main/deploy/manifests/nginx-conf.yaml +[conf-file]: https://github.com/nginxinc/nginx-kubernetes-gateway/blob/main/internal/mode/static/nginx/conf/nginx.conf [share]: https://kubernetes.io/docs/tasks/configure-pod-container/share-process-namespace/ diff --git a/docs/building-the-image.md b/docs/building-the-image.md deleted file mode 100644 index 8f63e895e6..0000000000 --- a/docs/building-the-image.md +++ /dev/null @@ -1,35 +0,0 @@ -# Building the Image - -## Prerequisites - -Before you can build the NGINX Kubernetes Gateway, make sure you have the following software installed on your machine: - -- [git](https://git-scm.com/) -- [GNU Make](https://www.gnu.org/software/software.html) -- [Docker](https://www.docker.com/) v18.09+ - -## Steps - -1. Clone the repo and change into the `nginx-kubernetes-gateway` directory: - - ```shell - git clone https://github.com/nginxinc/nginx-kubernetes-gateway.git - cd nginx-kubernetes-gateway - ``` - -1. Build the image: - - ```makefile - make PREFIX=myregistry.example.com/nginx-kubernetes-gateway container - ``` - - Set the `PREFIX` variable to the name of the registry you'd like to push the image to. By default, the image will be - named `nginx-kubernetes-gateway:edge`. - -1. Push the image to your container registry: - - ```shell - docker push myregistry.example.com/nginx-kubernetes-gateway:edge - ``` - - Make sure to substitute `myregistry.example.com/nginx-kubernetes-gateway` with your registry. diff --git a/docs/building-the-images.md b/docs/building-the-images.md new file mode 100644 index 0000000000..14ab698c3a --- /dev/null +++ b/docs/building-the-images.md @@ -0,0 +1,51 @@ +# Building the Images + +## Prerequisites + +Before you can build the NGINX Kubernetes Gateway and NGINX images, make sure you have the following software +installed on your machine: + +- [git](https://git-scm.com/) +- [GNU Make](https://www.gnu.org/software/software.html) +- [Docker](https://www.docker.com/) v18.09+ +- [Go](https://go.dev/doc/install) v1.20 + +## Steps + +1. Clone the repo and change into the `nginx-kubernetes-gateway` directory: + + ```shell + git clone https://github.com/nginxinc/nginx-kubernetes-gateway.git + cd nginx-kubernetes-gateway + ``` + +1. Build the images: + - To build both the NGINX Kubernetes Gateway and NGINX images: + + ```makefile + make PREFIX=myregistry.example.com/nginx-kubernetes-gateway build-images + ``` + + - To build just the NGINX Kubernetes Gateway image: + + ```makefile + make PREFIX=myregistry.example.com/nginx-kubernetes-gateway build-nkg-image + ``` + + - To build just the NGINX image: + + ```makefile + make PREFIX=myregistry.example.com/nginx-kubernetes-gateway build-nginx-image + ``` + + Set the `PREFIX` variable to the name of the registry you'd like to push the image to. By default, the images will be + named `nginx-kubernetes-gateway:edge` and `nginx-kubernetes-gateway/nginx:edge`. + +1. Push the images to your container registry: + + ```shell + docker push myregistry.example.com/nginx-kubernetes-gateway:edge + docker push myregistry.example.com/nginx-kubernetes-gateway/nginx:edge + ``` + + Make sure to substitute `myregistry.example.com/nginx-kubernetes-gateway` with your registry. diff --git a/docs/developer/quickstart.md b/docs/developer/quickstart.md index 12f132df05..71f3b38368 100644 --- a/docs/developer/quickstart.md +++ b/docs/developer/quickstart.md @@ -60,15 +60,15 @@ make build This command will build the binary and output it to the `/build/.out` directory. -### Build the Image +### Build the Images -To build an NGINX Kubernetes Gateway container image from source run the following make command: +To build the NGINX Kubernetes Gateway and NGINX container images from source run the following make command: ```makefile -make TAG=$(whoami) container +make TAG=$(whoami) build-images ``` -This will build the docker image `nginx-kubernetes-gateway:`. +This will build the docker images `nginx-kubernetes-gateway:` and `nginx-kubernetes-gateway/nginx:`. ## Deploy on Kind @@ -78,10 +78,10 @@ This will build the docker image `nginx-kubernetes-gateway:`. make create-kind-cluster ``` -2. Load the previously built image onto your `kind` cluster: +2. Load the previously built images onto your `kind` cluster: ```shell - kind load docker-image nginx-kubernetes-gateway:$(whoami) + kind load docker-image nginx-kubernetes-gateway:$(whoami) nginx-kubernetes-gateway/nginx:$(whoami) ``` 3. Install Gateway API Resources @@ -95,7 +95,7 @@ This will build the docker image `nginx-kubernetes-gateway:`. - To install with Helm (where your release name is `my-release`): ```shell - helm install my-release ./deploy/helm-chart --create-namespace --wait --set service.type=NodePort --set nginxGateway.image.repository=nginx-kubernetes-gateway --set nginxGateway.image.tag=$(whoami) --set nginxGateway.image.pullPolicy=Never -n nginx-gateway + helm install my-release ./deploy/helm-chart --create-namespace --wait --set service.type=NodePort --set nginxGateway.image.repository=nginx-kubernetes-gateway --set nginxGateway.image.tag=$(whoami) --set nginxGateway.image.pullPolicy=Never --set nginx.image.repository=nginx-kubernetes-gateway/nginx --set nginx.image.tag=$(whoami) --set nginx.image.pullPolicy=Never -n nginx-gateway ``` > For more information on helm configuration options see the Helm [README](/deploy/helm-chart/README.md). @@ -103,9 +103,9 @@ This will build the docker image `nginx-kubernetes-gateway:`. - To install with manifests: ```shell - make generate-manifests HELM_TEMPLATE_COMMON_ARGS="--set nginxGateway.image.repository=nginx-kubernetes-gateway --set nginxGateway.image.tag=$(whoami) --set nginxGateway.image.pullPolicy=Never" + make generate-manifests HELM_TEMPLATE_COMMON_ARGS="--set nginxGateway.image.repository=nginx-kubernetes-gateway --set nginxGateway.image.tag=$(whoami) --set nginxGateway.image.pullPolicy=Never --set nginx.image.repository=nginx-kubernetes-gateway/nginx --set nginx.image.tag=$(whoami) --set nginx.image.pullPolicy=Never" kubectl apply -f deploy/manifests/nginx-gateway.yaml - kubectl apply -f deploy/manifests/nodeport.yaml + kubectl apply -f deploy/manifests/service/nodeport.yaml ``` ### Run Examples @@ -155,17 +155,7 @@ make generate ## Update Generated Manifests -To update the NJS ConfigMap yaml, run the following make command from the project's root directory: - -```shell -make generate-njs-yaml -``` - -Additionally, the [NJS ConfigMap Helm template](/deploy/helm-chart/templates/njs-modules.yaml) will need to be updated. -This is currently a manual process - ensure the content in the `data` field matches that in the -[NJS ConfigMap manifest](/deploy/manifests/njs-modules.yaml) `data` field. - -Finally, to update all other generated manifests, run the following make command from the project's root directory: +To update the generated manifests, run the following make command from the project's root directory: ```shell make generate-manifests diff --git a/docs/images/nkg-pod.png b/docs/images/nkg-pod.png index 7ea499cbe6..5f58045480 100644 Binary files a/docs/images/nkg-pod.png and b/docs/images/nkg-pod.png differ diff --git a/internal/mode/static/nginx/conf/nginx.conf b/internal/mode/static/nginx/conf/nginx.conf new file mode 100644 index 0000000000..02f6c9f91f --- /dev/null +++ b/internal/mode/static/nginx/conf/nginx.conf @@ -0,0 +1,17 @@ +load_module /usr/lib/nginx/modules/ngx_http_js_module.so; + +events {} + +pid /var/run/nginx/nginx.pid; +error_log stderr debug; + +http { + include /etc/nginx/conf.d/*.conf; + js_import /usr/lib/nginx/modules/njs/httpmatches.js; + proxy_headers_hash_bucket_size 512; + proxy_headers_hash_max_size 1024; + server_names_hash_bucket_size 256; + server_names_hash_max_size 1024; + variables_hash_bucket_size 512; + variables_hash_max_size 1024; +} diff --git a/internal/mode/static/nginx/file/manager.go b/internal/mode/static/nginx/file/manager.go index e991a455d9..560ab878af 100644 --- a/internal/mode/static/nginx/file/manager.go +++ b/internal/mode/static/nginx/file/manager.go @@ -11,7 +11,7 @@ import ( const ( // secretFileMode defines the default file mode for files with secrets. - secretFileMode = 0o600 + secretFileMode = 0o640 ) // Type is the type of File. diff --git a/internal/mode/static/nginx/file/manager_test.go b/internal/mode/static/nginx/file/manager_test.go index 4ef6d1aae9..783a3314fc 100644 --- a/internal/mode/static/nginx/file/manager_test.go +++ b/internal/mode/static/nginx/file/manager_test.go @@ -43,7 +43,7 @@ var _ = Describe("EventHandler", func() { if f.Type == file.TypeRegular { Expect(info.Mode()).To(Equal(os.FileMode(0o644))) } else { - Expect(info.Mode()).To(Equal(os.FileMode(0o600))) + Expect(info.Mode()).To(Equal(os.FileMode(0o640))) } bytes, err := os.ReadFile(f.Path) diff --git a/internal/mode/static/nginx/runtime/manager.go b/internal/mode/static/nginx/runtime/manager.go index 88be206d5e..88fa44afbb 100644 --- a/internal/mode/static/nginx/runtime/manager.go +++ b/internal/mode/static/nginx/runtime/manager.go @@ -15,7 +15,7 @@ import ( ) const ( - pidFile = "/etc/nginx/nginx.pid" + pidFile = "/var/run/nginx/nginx.pid" pidFileTimeout = 10 * time.Second )