From 558f34c4406cee8920d635e942e67004527e7ab8 Mon Sep 17 00:00:00 2001 From: Ciara Stacke Date: Wed, 6 Mar 2024 10:33:51 +0000 Subject: [PATCH 1/6] Add convenience scripts and update workflow token --- .github/workflows/nfr.yml | 1 + tests/Makefile | 4 ++++ tests/README.md | 7 +++++++ tests/scripts/add-local-ip-auth-networks.sh | 8 ++++++++ tests/scripts/remote-scripts/run-tests.sh | 4 +++- tests/scripts/scp-files-to-vm.sh | 12 ++++++++++++ 6 files changed, 35 insertions(+), 1 deletion(-) create mode 100644 tests/scripts/add-local-ip-auth-networks.sh create mode 100644 tests/scripts/scp-files-to-vm.sh diff --git a/.github/workflows/nfr.yml b/.github/workflows/nfr.yml index 922b3621a5..b796a70233 100644 --- a/.github/workflows/nfr.yml +++ b/.github/workflows/nfr.yml @@ -41,6 +41,7 @@ jobs: permissions: contents: write # needed for opening PR with the results files pull-requests: write # needed for opening PR with the results files + id-token: write # needed for authenticating to GCP steps: - name: Checkout Repository diff --git a/tests/Makefile b/tests/Makefile index 3770196a00..b88e90377d 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -91,3 +91,7 @@ create-gke-cluster: ## Create a GKE cluster .PHONY: delete-gke-cluster delete-gke-cluster: ## Delete the GKE cluster bash scripts/delete-gke-cluster.sh + +.PHONY: add-local-ip-to-cluster +add-local-ip-to-cluster: ## Add local IP to the GKE cluster master-authorized-networks + bash scripts/add-local-ip-to-cluster.sh diff --git a/tests/README.md b/tests/README.md index 50dda8b2f7..444fec0d7e 100644 --- a/tests/README.md +++ b/tests/README.md @@ -101,6 +101,13 @@ and `GKE_NODES_SERVICE_ACCOUNT` needs to be the name of a service account that h make create-gke-cluster ``` +> Note: The GKE cluster is created with no local kubectl access by default. To access the cluster locally, you can add +> your local IP to the `master-authorized-networks` of the cluster by running the following: + +```makefile +make add-local-ip-to-cluster +``` + ## Step 2 - Build and Load Images Loading the images only applies to a `kind` cluster. If using a cloud provider, you will need to tag and push diff --git a/tests/scripts/add-local-ip-auth-networks.sh b/tests/scripts/add-local-ip-auth-networks.sh new file mode 100644 index 0000000000..525260fe93 --- /dev/null +++ b/tests/scripts/add-local-ip-auth-networks.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +source scripts/vars.env + +CURRENT_AUTH_NETWORK=$(gcloud container clusters describe ${GKE_CLUSTER_NAME} --zone=${GKE_CLUSTER_ZONE} \ + --format="value(masterAuthorizedNetworksConfig.cidrBlocks[0])" | sed 's/cidrBlock=//') + +gcloud container clusters update ${GKE_CLUSTER_NAME} --zone=${GKE_CLUSTER_ZONE} --enable-master-authorized-networks --master-authorized-networks=${SOURCE_IP_RANGE},${CURRENT_AUTH_NETWORK} diff --git a/tests/scripts/remote-scripts/run-tests.sh b/tests/scripts/remote-scripts/run-tests.sh index 67024ef56d..25956636ba 100644 --- a/tests/scripts/remote-scripts/run-tests.sh +++ b/tests/scripts/remote-scripts/run-tests.sh @@ -6,4 +6,6 @@ source ~/vars.env echo "export PATH=$PATH:/usr/local/go/bin" >> $HOME/.profile && . $HOME/.profile -cd nginx-gateway-fabric/tests && make test TAG=${TAG} PREFIX=${PREFIX} NGINX_PREFIX=${NGINX_PREFIX} NGINX_PLUS_PREFIX=${NGINX_PLUS_PREFIX} PLUS_ENABLED=${PLUS_ENABLED} GINKGO_LABEL=${GINKGO_LABEL} GINKGO_FLAGS=${GINKGO_FLAGS} PULL_POLICY=Always GW_SERVICE_TYPE=LoadBalancer GW_SVC_GKE_INTERNAL=true NGF_VERSION=${NGF_VERSION} +cd nginx-gateway-fabric && git pull + +cd tests && make test TAG=${TAG} PREFIX=${PREFIX} NGINX_PREFIX=${NGINX_PREFIX} NGINX_PLUS_PREFIX=${NGINX_PLUS_PREFIX} PLUS_ENABLED=${PLUS_ENABLED} GINKGO_LABEL=${GINKGO_LABEL} GINKGO_FLAGS=${GINKGO_FLAGS} PULL_POLICY=Always GW_SERVICE_TYPE=LoadBalancer GW_SVC_GKE_INTERNAL=true NGF_VERSION=${NGF_VERSION} diff --git a/tests/scripts/scp-files-to-vm.sh b/tests/scripts/scp-files-to-vm.sh new file mode 100644 index 0000000000..9e0f2a6c44 --- /dev/null +++ b/tests/scripts/scp-files-to-vm.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +# Usage: bash /scp-files-to-vm.sh +# e.g. bash scripts/scp-files-to-vm.sh framework/results.go /nginx-gateway-fabric/tests/framework/results.go +PATH_TO_LOCAL=$1 + +# The remote path will be appended to '~'. Requires leading /. +PATH_REMOTE=$2 + +source scripts/vars.env + +gcloud compute scp --zone ${GKE_CLUSTER_ZONE} --project=${GKE_PROJECT} ${PATH_TO_LOCAL} username@${RESOURCE_NAME}:~${PATH_REMOTE} From eae156f367154f75426a40f5daaf1e212e8a8045 Mon Sep 17 00:00:00 2001 From: Ciara Stacke Date: Wed, 6 Mar 2024 15:07:25 +0000 Subject: [PATCH 2/6] Remove scp script --- tests/scripts/scp-files-to-vm.sh | 12 ------------ 1 file changed, 12 deletions(-) delete mode 100644 tests/scripts/scp-files-to-vm.sh diff --git a/tests/scripts/scp-files-to-vm.sh b/tests/scripts/scp-files-to-vm.sh deleted file mode 100644 index 9e0f2a6c44..0000000000 --- a/tests/scripts/scp-files-to-vm.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -# Usage: bash /scp-files-to-vm.sh -# e.g. bash scripts/scp-files-to-vm.sh framework/results.go /nginx-gateway-fabric/tests/framework/results.go -PATH_TO_LOCAL=$1 - -# The remote path will be appended to '~'. Requires leading /. -PATH_REMOTE=$2 - -source scripts/vars.env - -gcloud compute scp --zone ${GKE_CLUSTER_ZONE} --project=${GKE_PROJECT} ${PATH_TO_LOCAL} username@${RESOURCE_NAME}:~${PATH_REMOTE} From 3ef209a8d4e3f2e57e63e363a66a5974b4ec0c48 Mon Sep 17 00:00:00 2001 From: Ciara Stacke Date: Wed, 6 Mar 2024 15:16:43 +0000 Subject: [PATCH 3/6] Remove git pull --- tests/scripts/remote-scripts/run-tests.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/scripts/remote-scripts/run-tests.sh b/tests/scripts/remote-scripts/run-tests.sh index 25956636ba..67024ef56d 100644 --- a/tests/scripts/remote-scripts/run-tests.sh +++ b/tests/scripts/remote-scripts/run-tests.sh @@ -6,6 +6,4 @@ source ~/vars.env echo "export PATH=$PATH:/usr/local/go/bin" >> $HOME/.profile && . $HOME/.profile -cd nginx-gateway-fabric && git pull - -cd tests && make test TAG=${TAG} PREFIX=${PREFIX} NGINX_PREFIX=${NGINX_PREFIX} NGINX_PLUS_PREFIX=${NGINX_PLUS_PREFIX} PLUS_ENABLED=${PLUS_ENABLED} GINKGO_LABEL=${GINKGO_LABEL} GINKGO_FLAGS=${GINKGO_FLAGS} PULL_POLICY=Always GW_SERVICE_TYPE=LoadBalancer GW_SVC_GKE_INTERNAL=true NGF_VERSION=${NGF_VERSION} +cd nginx-gateway-fabric/tests && make test TAG=${TAG} PREFIX=${PREFIX} NGINX_PREFIX=${NGINX_PREFIX} NGINX_PLUS_PREFIX=${NGINX_PLUS_PREFIX} PLUS_ENABLED=${PLUS_ENABLED} GINKGO_LABEL=${GINKGO_LABEL} GINKGO_FLAGS=${GINKGO_FLAGS} PULL_POLICY=Always GW_SERVICE_TYPE=LoadBalancer GW_SVC_GKE_INTERNAL=true NGF_VERSION=${NGF_VERSION} From 22a5c6d4bbb44ab6fcbf0137d9126d5c6b8cf627 Mon Sep 17 00:00:00 2001 From: Ciara Stacke Date: Wed, 6 Mar 2024 15:55:34 +0000 Subject: [PATCH 4/6] Add local IP to cluster access list --- .github/workflows/nfr.yml | 2 +- tests/Makefile | 3 ++- tests/README.md | 6 ++++-- tests/scripts/create-gke-cluster.sh | 8 ++++++++ 4 files changed, 15 insertions(+), 4 deletions(-) diff --git a/.github/workflows/nfr.yml b/.github/workflows/nfr.yml index b796a70233..0a33888cb9 100644 --- a/.github/workflows/nfr.yml +++ b/.github/workflows/nfr.yml @@ -133,7 +133,7 @@ jobs: - name: Create GKE cluster working-directory: ./tests run: - make create-gke-cluster + make create-gke-cluster CI=true - name: Create and setup VM working-directory: ./tests diff --git a/tests/Makefile b/tests/Makefile index b88e90377d..6b32e47d07 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -12,6 +12,7 @@ GW_SVC_GKE_INTERNAL=false GINKGO_LABEL= GINKGO_FLAGS= NGF_VERSION= +CI=false ifneq ($(GINKGO_LABEL),) override GINKGO_FLAGS += -ginkgo.label-filter "$(GINKGO_LABEL)" @@ -86,7 +87,7 @@ cleanup-gcp: cleanup-router cleanup-vm delete-gke-cluster ## Cleanup all GCP res .PHONY: create-gke-cluster create-gke-cluster: ## Create a GKE cluster - bash scripts/create-gke-cluster.sh + bash scripts/create-gke-cluster.sh $(CI) .PHONY: delete-gke-cluster delete-gke-cluster: ## Delete the GKE cluster diff --git a/tests/README.md b/tests/README.md index 444fec0d7e..688ef77873 100644 --- a/tests/README.md +++ b/tests/README.md @@ -101,8 +101,10 @@ and `GKE_NODES_SERVICE_ACCOUNT` needs to be the name of a service account that h make create-gke-cluster ``` -> Note: The GKE cluster is created with no local kubectl access by default. To access the cluster locally, you can add -> your local IP to the `master-authorized-networks` of the cluster by running the following: +> Note: The GKE cluster is created with `master-authorized-networks`, meaning only IPs from explitly allowed CIDR ranges +> will be able to access the cluster. The script will automatically add your current IP to the authorized list, but if +> your IP changes, you can add your new local IP to the `master-authorized-networks` of the cluster by running the +> following: ```makefile make add-local-ip-to-cluster diff --git a/tests/scripts/create-gke-cluster.sh b/tests/scripts/create-gke-cluster.sh index d048eda368..20e7c08bcf 100644 --- a/tests/scripts/create-gke-cluster.sh +++ b/tests/scripts/create-gke-cluster.sh @@ -4,6 +4,8 @@ source scripts/vars.env ip_random_digit=$((1 + $RANDOM % 250)) +IS_CI=${1:-false} + gcloud container clusters create ${GKE_CLUSTER_NAME} \ --project ${GKE_PROJECT} \ --zone ${GKE_CLUSTER_ZONE} \ @@ -13,3 +15,9 @@ gcloud container clusters create ${GKE_CLUSTER_NAME} \ --enable-private-nodes \ --master-ipv4-cidr 172.16.${ip_random_digit}.32/28 \ --metadata=block-project-ssh-keys=TRUE + +# Add current IP to GKE master control node access, if this script is not invoked during a CI run. +if [ "${IS_CI}" = "false" ]; then + SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + bash ${SCRIPT_DIR}/add-local-ip-auth-networks.sh +fi From dca7fe6c6eda18414082fb893ca29f2c0f19cd35 Mon Sep 17 00:00:00 2001 From: Ciara Stacke <18287516+ciarams87@users.noreply.github.com> Date: Wed, 6 Mar 2024 16:15:25 +0000 Subject: [PATCH 5/6] Update tests/README.md Co-authored-by: Saylor Berman --- tests/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/README.md b/tests/README.md index 688ef77873..8f839c86f7 100644 --- a/tests/README.md +++ b/tests/README.md @@ -101,7 +101,7 @@ and `GKE_NODES_SERVICE_ACCOUNT` needs to be the name of a service account that h make create-gke-cluster ``` -> Note: The GKE cluster is created with `master-authorized-networks`, meaning only IPs from explitly allowed CIDR ranges +> Note: The GKE cluster is created with `master-authorized-networks`, meaning only IPs from explicitly allowed CIDR ranges > will be able to access the cluster. The script will automatically add your current IP to the authorized list, but if > your IP changes, you can add your new local IP to the `master-authorized-networks` of the cluster by running the > following: From b164ef3c92833874278c4b2697be166d8205c73f Mon Sep 17 00:00:00 2001 From: Ciara Stacke Date: Wed, 6 Mar 2024 16:18:02 +0000 Subject: [PATCH 6/6] Update README --- tests/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/README.md b/tests/README.md index 8f839c86f7..07a8ea141e 100644 --- a/tests/README.md +++ b/tests/README.md @@ -37,6 +37,7 @@ make ``` ```text +add-local-ip-to-cluster Add local IP to the GKE cluster master-authorized-networks build-images-with-plus Build NGF and NGINX Plus images build-images Build NGF and NGINX images cleanup-gcp Cleanup all GCP resources