Build and use custom nginx container

.github/workflows/ci.yml

@@ -55,9 +55,6 @@ jobs:
       - name: Check if generated go files are up to date
         run: make generate && git diff --exit-code
 
-      - name: Check if njs-modules yaml is up to date
-        run: make generate-njs-yaml && git diff --exit-code
-
       - name: Check if generated manifests are up to date
         run: make generate-manifests && git diff --exit-code
 
@@ -173,8 +170,8 @@ jobs:
       - name: Docker Buildx
         uses: docker/setup-buildx-action@4c0219f9ac95b02789c1075625400b2acbff50b1 # v2.9.1
 
-      - name: Docker meta
-        id: meta
+      - name: NKG Docker meta
+        id: nkg-meta
         uses: docker/metadata-action@818d4b7b91585d195f67373fd9cb0332e31a7175 # v4.6.0
         with:
           images: |
@@ -185,37 +182,67 @@ jobs:
             type=ref,event=pr
             type=ref,event=branch,suffix=-rc,enable=${{ startsWith(github.ref, 'refs/heads/release') }}
 
-      - name: Build Docker Image
+      - name: NGINX Docker meta
+        id: nginx-meta
+        uses: docker/metadata-action@818d4b7b91585d195f67373fd9cb0332e31a7175 # v4.6.0
+        with:
+          images: |
+            name=ghcr.io/nginxinc/nginx-kubernetes-gateway/nginx
+          tags: |
+            type=semver,pattern={{version}}
+            type=edge
+            type=ref,event=pr
+            type=ref,event=branch,suffix=-rc,enable=${{ startsWith(github.ref, 'refs/heads/release') }}
+
+      - name: Build NKG Docker Image
         uses: docker/build-push-action@2eb1c1961a95fc15694676618e422e8ba1d63825 # v4.1.1
         with:
           file: build/Dockerfile
-          tags: ${{ steps.meta.outputs.tags }}
+          tags: ${{ steps.nkg-meta.outputs.tags }}
           context: "."
           target: goreleaser
           load: true
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
+          cache-from: type=gha,scope=nkg
+          cache-to: type=gha,scope=nkg,mode=max
+          pull: true
+
+      - name: Build NGINX Docker Image
+        uses: docker/build-push-action@2eb1c1961a95fc15694676618e422e8ba1d63825 # v4.1.1
+        with:
+          file: build/Dockerfile.nginx
+          tags: ${{ steps.nginx-meta.outputs.tags }}
+          context: "."
+          load: true
+          cache-from: type=gha,scope=nginx
+          cache-to: type=gha,scope=nginx,mode=max
           pull: true
+          build-args: |
+            NJS_DIR=internal/mode/static/nginx/modules/src
+            NGINX_CONF_DIR=internal/mode/static/nginx/conf
 
       - name: Deploy Kubernetes
         id: k8s
         run: |
           kube_config=${{ github.workspace }}/deploy/helm-chart/kube-${{ github.run_id }}-helm
           make create-kind-cluster KIND_KUBE_CONFIG=${kube_config}
           echo "KUBECONFIG=${kube_config}" >> "$GITHUB_ENV"
-          kind load docker-image ${{ steps.meta.outputs.tags }}
+          kind load docker-image ${{ steps.nkg-meta.outputs.tags }} ${{ steps.nginx-meta.outputs.tags }}
           kubectl apply -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v0.7.1/standard-install.yaml
           kubectl wait --for=condition=complete job/gateway-api-admission-patch job/gateway-api-admission -n gateway-system
 
       - name: Install Chart
         run: >
           helm install
-          helm-$(echo ${{ steps.meta.outputs.tags }} | cut -d ":" -f 2)
+          helm-$(echo ${{ steps.nkg-meta.outputs.tags }} | cut -d ":" -f 2)
           .
           --wait
           --create-namespace
-          --set controller.image.repository=$(echo ${{ steps.meta.outputs.tags }} | cut -d ":" -f 1)
-          --set controller.image.tag=$(echo ${{ steps.meta.outputs.tags }} | cut -d ":" -f 2)
+          --set nginxGateway.image.repository=$(echo ${{ steps.nkg-meta.outputs.tags }} | cut -d ":" -f 1)
+          --set nginxGateway.image.tag=$(echo ${{ steps.nkg-meta.outputs.tags }} | cut -d ":" -f 2)
+          --set nginxGateway.image.pullPolicy=Never
+          --set nginx.image.repository=$(echo ${{ steps.nginx-meta.outputs.tags }} | cut -d ":" -f 1)
+          --set nginx.image.tag=$(echo ${{ steps.nginx-meta.outputs.tags }} | cut -d ":" -f 2)
+          --set nginx.image.pullPolicy=Never
           --set service.type=NodePort
           -n nginx-gateway
         working-directory: ${{ github.workspace }}/deploy/helm-chart
@@ -224,6 +251,10 @@ jobs:
     name: Build Image
     runs-on: ubuntu-22.04
     needs: [vars, binary]
+    strategy:
+      fail-fast: false
+      matrix:
+        container: [nkg, nginx]
     permissions:
       contents: read # for docker/build-push-action to read repo content
       security-events: write # for github/codeql-action/upload-sarif to upload SARIF results
@@ -259,7 +290,7 @@ jobs:
         uses: docker/metadata-action@818d4b7b91585d195f67373fd9cb0332e31a7175 # v4.6.0
         with:
           images: |
-            name=ghcr.io/nginxinc/nginx-kubernetes-gateway
+            name=ghcr.io/nginxinc/nginx-kubernetes-gateway${{ matrix.container == 'nginx' && '/nginx' || '' }}
           tags: |
             type=semver,pattern={{version}}
             type=edge
@@ -269,42 +300,46 @@ jobs:
       - name: Build Docker Image
         uses: docker/build-push-action@2eb1c1961a95fc15694676618e422e8ba1d63825 # v4.1.1
         with:
-          file: build/Dockerfile
+          file: ${{ matrix.container == 'nginx' && 'build/Dockerfile.nginx' || 'build/Dockerfile' }}
           context: "."
-          target: goreleaser
+          target: ${{ matrix.container == 'nkg' && 'goreleaser' || '' }}
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
           load: ${{ github.event_name == 'pull_request' }}
           push: ${{ github.event_name != 'pull_request' }}
           platforms: ${{ github.event_name != 'pull_request' && env.platforms || '' }}
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
+          cache-from: type=gha,scope=${{ matrix.container }}
+          cache-to: type=gha,scope=${{ matrix.container }},mode=max
           pull: true
           no-cache: ${{ github.event_name != 'pull_request' }}
           sbom: ${{ github.event_name != 'pull_request' }}
           provenance: false
+          build-args: |
+            NJS_DIR=internal/mode/static/nginx/modules/src
+            NGINX_CONF_DIR=internal/mode/static/nginx/conf
+
 
       - name: Run Trivy vulnerability scanner
         uses: aquasecurity/trivy-action@41f05d9ecffa2ed3f1580af306000f734b733e54 # 0.11.2
         continue-on-error: true
         with:
-          image-ref: ghcr.io/nginxinc/nginx-kubernetes-gateway:${{ steps.meta.outputs.version }}
+          image-ref: ghcr.io/nginxinc/nginx-kubernetes-gateway${{ matrix.container == 'nginx' && '/nginx' || '' }}:${{ steps.meta.outputs.version }}
           format: "sarif"
-          output: "trivy-results-nginx-kubernetes-gateway.sarif"
+          output: trivy-results-nginx-kubernetes-gateway${{ matrix.container == 'nginx' && '-nginx' || '' }}.sarif
           ignore-unfixed: "true"
 
       - name: Upload Trivy scan results to GitHub Security tab
         uses: github/codeql-action/upload-sarif@5b6282e01c62d02e720b81eb8a51204f527c3624 # v2.21.3
         continue-on-error: true
         with:
-          sarif_file: "trivy-results-nginx-kubernetes-gateway.sarif"
+          sarif_file: trivy-results-nginx-kubernetes-gateway${{ matrix.container == 'nginx' && '-nginx' || '' }}.sarif
 
       - name: Upload Scan Results
         uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
         continue-on-error: true
         with:
-          name: "trivy-results-nginx-kubernetes-gateway.sarif"
-          path: "trivy-results-nginx-kubernetes-gateway.sarif"
+          name: trivy-results-nginx-kubernetes-gateway${{ matrix.container == 'nginx' && '-nginx' || '' }}.sarif
+          path: trivy-results-nginx-kubernetes-gateway${{ matrix.container == 'nginx' && '-nginx' || '' }}.sarif
         if: always()
 
   publish-helm:

.github/workflows/conformance.yml

@@ -42,8 +42,8 @@ jobs:
       - name: Docker Buildx
         uses: docker/setup-buildx-action@4c0219f9ac95b02789c1075625400b2acbff50b1 # v2.9.1
 
-      - name: Docker meta
-        id: meta
+      - name: NKG Docker meta
+        id: nkg-meta
         uses: docker/metadata-action@818d4b7b91585d195f67373fd9cb0332e31a7175 # v4.6.0
         with:
           images: |
@@ -54,10 +54,22 @@ jobs:
             type=ref,event=pr
             type=ref,event=branch,suffix=-rc,enable=${{ startsWith(github.ref, 'refs/heads/release') }}
 
+      - name: NGINX Docker meta
+        id: nginx-meta
+        uses: docker/metadata-action@818d4b7b91585d195f67373fd9cb0332e31a7175 # v4.6.0
+        with:
+          images: |
+            name=ghcr.io/nginxinc/nginx-kubernetes-gateway/nginx
+          tags: |
+            type=semver,pattern={{version}}
+            type=edge
+            type=ref,event=pr
+            type=ref,event=branch,suffix=-rc,enable=${{ startsWith(github.ref, 'refs/heads/release') }}
+
       - name: Prepare NKG files
         run: |
-          nkg_prefix=$(echo ${{ steps.meta.outputs.tags }} | cut -d ":" -f 1)
-          nkg_tag=$(echo ${{ steps.meta.outputs.tags }} | cut -d ":" -f 2)
+          nkg_prefix=$(echo ${{ steps.nkg-meta.outputs.tags }} | cut -d ":" -f 1)
+          nkg_tag=$(echo ${{ steps.nkg-meta.outputs.tags }} | cut -d ":" -f 2)
           make update-nkg-manifest NKG_PREFIX=${nkg_prefix} NKG_TAG=${nkg_tag}
         working-directory: ./conformance
 
@@ -67,17 +79,31 @@ jobs:
           version: latest
           args: build --snapshot --clean
 
-      - name: Build Docker Image
+      - name: Build NKG Docker Image
         uses: docker/build-push-action@2eb1c1961a95fc15694676618e422e8ba1d63825 # v4.1.1
         with:
           file: build/Dockerfile
-          tags: ${{ steps.meta.outputs.tags }}
+          tags: ${{ steps.nkg-meta.outputs.tags }}
           context: "."
           target: goreleaser
           load: true
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
+          cache-from: type=gha,scope=nkg
+          cache-to: type=gha,scope=nkg,mode=max
+          pull: true
+
+      - name: Build NGINX Docker Image
+        uses: docker/build-push-action@2eb1c1961a95fc15694676618e422e8ba1d63825 # v4.1.1
+        with:
+          file: build/Dockerfile.nginx
+          tags: ${{ steps.nginx-meta.outputs.tags }}
+          context: "."
+          load: true
+          cache-from: type=gha,scope=nginx
+          cache-to: type=gha,scope=nginx,mode=max
           pull: true
+          build-args: |
+            NJS_DIR=internal/mode/static/nginx/modules/src
+            NGINX_CONF_DIR=internal/mode/static/nginx/conf
 
       - name: Update Go Modules
         if: ${{ github.event_name == 'schedule' }}
@@ -104,8 +130,8 @@ jobs:
 
       - name: Setup conformance tests
         run: |
-          nkg_prefix=$(echo ${{ steps.meta.outputs.tags }} | cut -d ":" -f 1)
-          nkg_tag=$(echo ${{ steps.meta.outputs.tags }} | cut -d ":" -f 2)
+          nkg_prefix=$(echo ${{ steps.nkg-meta.outputs.tags }} | cut -d ":" -f 1)
+          nkg_tag=$(echo ${{ steps.nkg-meta.outputs.tags }} | cut -d ":" -f 2)
           if [ ${{ github.event_name }} == "schedule" ]; then
           export GW_API_VERSION=main
           fi

Makefile

@@ -3,16 +3,18 @@ VERSION = edge
 GIT_COMMIT = $(shell git rev-parse HEAD || echo "unknown")
 DATE = $(shell date -u +"%Y-%m-%dT%H:%M:%SZ")
 MANIFEST_DIR = $(shell pwd)/deploy/manifests
-NJS_DIR = $(shell pwd)/internal/mode/static/nginx/modules/src
 CHART_DIR = $(shell pwd)/deploy/helm-chart
+NGINX_CONF_DIR = internal/mode/static/nginx/conf
+NJS_DIR = internal/mode/static/nginx/modules/src
 
 # go build flags - should not be overridden by the user
 GO_LINKER_FlAGS_VARS = -X main.version=${VERSION} -X main.commit=${GIT_COMMIT} -X main.date=${DATE}
 GO_LINKER_FLAGS_OPTIMIZATIONS = -s -w
 GO_LINKER_FLAGS = $(GO_LINKER_FLAGS_OPTIMIZATIONS) $(GO_LINKER_FlAGS_VARS)
 
 # variables that can be overridden by the user
-PREFIX ?= nginx-kubernetes-gateway## The name of the image. For example, nginx-kubernetes-gateway
+PREFIX ?= nginx-kubernetes-gateway## The name of the NKG image. For example, nginx-kubernetes-gateway
+NGINX_PREFIX ?= $(PREFIX)/nginx## The name of the nginx image. For example: nginx-kubernetes-gateway/nginx
 TAG ?= $(VERSION:v%=%)## The tag of the image. For example, 0.3.0
 TARGET ?= local## The target of the build. Possible values: local and container
 KIND_KUBE_CONFIG=$${HOME}/.kube/kind/config## The location of the kind kubeconfig
@@ -21,19 +23,29 @@ ARCH ?= amd64## The architecture of the image and/or binary. For example: amd64
 override HELM_TEMPLATE_COMMON_ARGS += --set creator=template --set nameOverride=nginx-gateway## The common options for the Helm template command.
 override HELM_TEMPLATE_EXTRA_ARGS_FOR_ALL_MANIFESTS_FILE += --set service.create=false## The options to be passed to the full Helm templating command only.
 override DOCKER_BUILD_OPTIONS += --build-arg VERSION=$(VERSION) --build-arg GIT_COMMIT=$(GIT_COMMIT) --build-arg DATE=$(DATE)## The options for the docker build command. For example, --pull
-
+override NGINX_DOCKER_BUILD_OPTIONS += --build-arg NJS_DIR=$(NJS_DIR) --build-arg NGINX_CONF_DIR=$(NGINX_CONF_DIR)
 .DEFAULT_GOAL := help
 
 .PHONY: help
 help: Makefile ## Display this help
 	@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "; printf "Usage:\n\n    make \033[36m<target>\033[0m [VARIABLE=value...]\n\nTargets:\n\n"}; {printf "    \033[36m%-30s\033[0m %s\n", $$1, $$2}'
 	@grep -E '^(override )?[a-zA-Z_-]+ \??\+?= .*?## .*$$' $< | sort | awk 'BEGIN {FS = " \\??\\+?= .*?## "; printf "\nVariables:\n\n"}; {gsub(/override /, "", $$1); printf "    \033[36m%-30s\033[0m %s\n", $$1, $$2}'
 
-.PHONY: container
-container: build ## Build the container
-	@docker -v || (code=$$?; printf "\033[0;31mError\033[0m: there was a problem with Docker\n"; exit $$code)
+.PHONY: build-images
+build-images: build-nkg-image build-nginx-image ## Build the NKG and nginx docker images
+
+.PHONY: build-nkg-image
+build-nkg-image: check-for-docker build ## Build the NKG docker image
 	docker build --platform linux/$(ARCH) $(strip $(DOCKER_BUILD_OPTIONS)) --target $(strip $(TARGET)) -f build/Dockerfile -t $(strip $(PREFIX)):$(strip $(TAG)) .
 
+.PHONY: build-nginx-image
+build-nginx-image: check-for-docker ## Build the custom nginx image
+	docker build --platform linux/$(ARCH) $(strip $(NGINX_DOCKER_BUILD_OPTIONS)) -f build/Dockerfile.nginx -t $(strip $(NGINX_PREFIX)):$(strip $(TAG)) .
+
+.PHONY: check-for-docker
+check-for-docker: ## Check if Docker is installed
+	@docker -v || (code=$$?; printf "\033[0;31mError\033[0m: there was a problem with Docker\n"; exit $$code)
+
 .PHONY: build
 build: ## Build the binary
 ifeq (${TARGET},local)
@@ -103,10 +115,6 @@ njs-unit-test: ## Run unit tests for the njs httpmatches module
 		node:18 \
 		/bin/bash -c "npm install && npm test && npm run clean"
 
-.PHONY: generate-njs-yaml
-generate-njs-yaml: ## Generate the njs-modules ConfigMap
-	kubectl create configmap njs-modules --from-file=$(NJS_DIR)/httpmatches.js --dry-run=client --output=yaml > $(strip $(MANIFEST_DIR))/njs-modules.yaml
-
 .PHONY: lint-helm
 lint-helm: ## Run the helm chart linter
 	helm lint $(CHART_DIR)
@@ -116,8 +124,8 @@ debug-build: GO_LINKER_FLAGS=$(GO_LINKER_FlAGS_VARS)
 debug-build: ADDITIONAL_GO_BUILD_FLAGS=-gcflags "all=-N -l"
 debug-build: build ## Build binary with debug info, symbols, and no optimizations
 
-.PHONY: debug-container
-debug-container: debug-build container ## Build container with debug binary
+.PHONY: build-nkg-debug-image
+build-nkg-debug-image: debug-build build-nkg-image ## Build NKG image with debug binary
 
 .PHONY: generate-manifests
 generate-manifests: ## Generate manifests using Helm.

README.md

@@ -21,7 +21,7 @@ Learn about our [design principles](/docs/developer/design-principles.md) and [a
 
 1. [Quick Start on a kind cluster](docs/running-on-kind.md).
 2. [Install](docs/installation.md) NGINX Kubernetes Gateway.
-3. [Build](docs/building-the-image.md) an NGINX Kubernetes Gateway container image from source or use a pre-built image
+3. [Build](docs/building-the-images.md) an NGINX Kubernetes Gateway container image from source or use a pre-built image
    available
    on [GitHub Container Registry](https://github.com/nginxinc/nginx-kubernetes-gateway/pkgs/container/nginx-kubernetes-gateway).
 4. Deploy various [examples](examples).
@@ -59,7 +59,7 @@ The following table lists the software versions NGINX Kubernetes Gateway support
 
 | NGINX Kubernetes Gateway | Gateway API | Kubernetes | NGINX OSS |
 |--------------------------|-------------|------------|-----------|
-| Edge                     | 0.7.1       | 1.21+      | 1.25.x *  |
+| Edge                     | 0.7.1       | 1.21+      | 1.25.1    |
 | 0.5.0                    | 0.7.1       | 1.21+      | 1.25.x *  |
 | 0.4.0                    | 0.7.1       | 1.21+      | 1.25.x *  |
 | 0.3.0                    | 0.6.2       | 1.21+      | 1.23.x *  |

build/Dockerfile

@@ -31,7 +31,7 @@ COPY dist/gateway_linux_$TARGETARCH*/gateway /usr/bin/
 RUN setcap 'cap_kill=+ep' /usr/bin/gateway
 
 FROM scratch as common
-USER 1001:1001
+USER 102:1001
 ENTRYPOINT [ "/usr/bin/gateway" ]
 
 FROM common as container

build/Dockerfile.nginx

@@ -0,0 +1,18 @@
+# syntax=docker/dockerfile:1.4
+FROM nginx:1.25.1-alpine
+
+ARG NJS_DIR
+ARG NGINX_CONF_DIR
+
+RUN apk update && apk add --no-cache libcap \
+    && mkdir -p /var/lib/nginx /usr/lib/nginx/modules \
+    && setcap 'cap_net_bind_service=+ep' /usr/sbin/nginx \
+    && setcap -v 'cap_net_bind_service=+ep' /usr/sbin/nginx \
+    && apk del libcap
+
+COPY ${NJS_DIR}/httpmatches.js /usr/lib/nginx/modules/njs/httpmatches.js
+COPY ${NGINX_CONF_DIR}/nginx.conf /etc/nginx/nginx.conf
+
+RUN chown -R 101:1001 /etc/nginx /var/cache/nginx /var/lib/nginx
+
+USER 101:1001