Skip to content

Commit 43c1100

Browse files
kate-osbornkate-osborn
authored
Support Horizontal Scaling (#1048)
Problem: NKG cannot be scaled horizontally because all replicas will write statuses to the Gateway API resources. Solution: Add leader election to the status updater so that only one replica of NKG will write statuses to the Gateway API resources. Leader election is enabled by default but can be disabled via a cli arg --leader-election-disable. The lock name used for leader election can be configured via the cli arg --leader-election-lock-name.
1 parent 1d44e2b commit 43c1100

File tree

26 files changed

+852
-226
lines changed

26 files changed

+852
-226
lines changed

‎Makefile

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,8 @@ TAG ?= $(VERSION:v%=%)## The tag of the image. For example, 0.3.0
1919
TARGET ?= local## The target of the build. Possible values: local and container
2020
KIND_KUBE_CONFIG=$${HOME}/.kube/kind/config## The location of the kind kubeconfig
2121
OUT_DIR ?= $(shell pwd)/build/out## The folder where the binary will be stored
22-
ARCH ?= amd64## The architecture of the image and/or binary. For example: amd64 or arm64
22+
GOARCH ?= amd64## The architecture of the image and/or binary. For example: amd64 or arm64
23+
GOOS ?= linux## The OS of the image and/or binary. For example: linux or darwin
2324
override HELM_TEMPLATE_COMMON_ARGS += --set creator=template --set nameOverride=nginx-gateway## The common options for the Helm template command.
2425
override HELM_TEMPLATE_EXTRA_ARGS_FOR_ALL_MANIFESTS_FILE += --set service.create=false## The options to be passed to the full Helm templating command only.
2526
override NGINX_DOCKER_BUILD_OPTIONS += --build-arg NJS_DIR=$(NJS_DIR) --build-arg NGINX_CONF_DIR=$(NGINX_CONF_DIR)
@@ -35,11 +36,11 @@ build-images: build-nkg-image build-nginx-image ## Build the NKG and nginx docke
3536

3637
.PHONY: build-nkg-image
3738
build-nkg-image: check-for-docker build ## Build the NKG docker image
38-
docker build --platform linux/$(ARCH) --target $(strip $(TARGET)) -f build/Dockerfile -t $(strip $(PREFIX)):$(strip $(TAG)) .
39+
docker build --platform linux/$(GOARCH) --target $(strip $(TARGET)) -f build/Dockerfile -t $(strip $(PREFIX)):$(strip $(TAG)) .
3940

4041
.PHONY: build-nginx-image
4142
build-nginx-image: check-for-docker ## Build the custom nginx image
42-
docker build --platform linux/$(ARCH) $(strip $(NGINX_DOCKER_BUILD_OPTIONS)) -f build/Dockerfile.nginx -t $(strip $(NGINX_PREFIX)):$(strip $(TAG)) .
43+
docker build --platform linux/$(GOARCH) $(strip $(NGINX_DOCKER_BUILD_OPTIONS)) -f build/Dockerfile.nginx -t $(strip $(NGINX_PREFIX)):$(strip $(TAG)) .
4344

4445
.PHONY: check-for-docker
4546
check-for-docker: ## Check if Docker is installed
@@ -49,13 +50,13 @@ check-for-docker: ## Check if Docker is installed
4950
build: ## Build the binary
5051
ifeq (${TARGET},local)
5152
@go version || (code=$$?; printf "\033[0;31mError\033[0m: unable to build locally\n"; exit $$code)
52-
CGO_ENABLED=0 GOOS=linux GOARCH=$(ARCH) go build -trimpath -a -ldflags "$(GO_LINKER_FLAGS)" $(ADDITIONAL_GO_BUILD_FLAGS) -o $(OUT_DIR)/gateway github.com/nginxinc/nginx-kubernetes-gateway/cmd/gateway
53+
CGO_ENABLED=0 GOOS=$(GOOS) GOARCH=$(GOARCH) go build -trimpath -a -ldflags "$(GO_LINKER_FLAGS)" $(ADDITIONAL_GO_BUILD_FLAGS) -o $(OUT_DIR)/gateway github.com/nginxinc/nginx-kubernetes-gateway/cmd/gateway
5354
endif
5455

5556
.PHONY: build-goreleaser
5657
build-goreleaser: ## Build the binary using GoReleaser
5758
@goreleaser -v || (code=$$?; printf "\033[0;31mError\033[0m: there was a problem with GoReleaser. Follow the docs to install it https://goreleaser.com/install\n"; exit $$code)
58-
GOOS=linux GOPATH=$(shell go env GOPATH) GOARCH=$(ARCH) goreleaser build --clean --snapshot --single-target
59+
GOOS=linux GOPATH=$(shell go env GOPATH) GOARCH=$(GOARCH) goreleaser build --clean --snapshot --single-target
5960

6061
.PHONY: generate
6162
generate: ## Run go generate

‎cmd/gateway/commands.go

Lines changed: 80 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,6 @@ const (
2525
gatewayCtrlNameFlag = "gateway-ctlr-name"
2626
gatewayCtrlNameUsageFmt = `The name of the Gateway controller. ` +
2727
`The controller name must be of the form: DOMAIN/PATH. The controller's domain is '%s'`
28-
gatewayFlag = "gateway"
2928
)
3029

3130
var (
@@ -38,25 +37,6 @@ var (
3837
gatewayClassName = stringValidatingValue{
3938
validator: validateResourceName,
4039
}
41-
42-
// Backing values for static subcommand cli flags.
43-
updateGCStatus bool
44-
disableMetrics bool
45-
metricsSecure bool
46-
disableHealth bool
47-
48-
metricsListenPort = intValidatingValue{
49-
validator: validatePort,
50-
value: 9113,
51-
}
52-
healthListenPort = intValidatingValue{
53-
validator: validatePort,
54-
value: 8081,
55-
}
56-
gateway = namespacedNameValue{}
57-
configName = stringValidatingValue{
58-
validator: validateResourceName,
59-
}
6040
)
6141

6242
func createRootCommand() *cobra.Command {
@@ -85,6 +65,46 @@ func createRootCommand() *cobra.Command {
8565
}
8666

8767
func createStaticModeCommand() *cobra.Command {
68+
// flag names
69+
const (
70+
gatewayFlag = "gateway"
71+
configFlag = "config"
72+
updateGCStatusFlag = "update-gatewayclass-status"
73+
metricsDisableFlag = "metrics-disable"
74+
metricsSecureFlag = "metrics-secure-serving"
75+
metricsPortFlag = "metrics-port"
76+
healthDisableFlag = "health-disable"
77+
healthPortFlag = "health-port"
78+
leaderElectionDisableFlag = "leader-election-disable"
79+
leaderElectionLockNameFlag = "leader-election-lock-name"
80+
)
81+
82+
// flag values
83+
var (
84+
updateGCStatus bool
85+
gateway = namespacedNameValue{}
86+
configName = stringValidatingValue{
87+
validator: validateResourceName,
88+
}
89+
disableMetrics bool
90+
metricsSecure bool
91+
metricsListenPort = intValidatingValue{
92+
validator: validatePort,
93+
value: 9113,
94+
}
95+
disableHealth bool
96+
healthListenPort = intValidatingValue{
97+
validator: validatePort,
98+
value: 8081,
99+
}
100+
101+
disableLeaderElection bool
102+
leaderElectionLockName = stringValidatingValue{
103+
validator: validateResourceName,
104+
value: "nginx-gateway-leader-election-lock",
105+
}
106+
)
107+
88108
cmd := &cobra.Command{
89109
Use: "static-mode",
90110
Short: "Configure NGINX in the scope of a single Gateway resource",
@@ -109,23 +129,21 @@ func createStaticModeCommand() *cobra.Command {
109129
return fmt.Errorf("error validating POD_IP environment variable: %w", err)
110130
}
111131

112-
namespace := os.Getenv("MY_NAMESPACE")
132+
namespace := os.Getenv("POD_NAMESPACE")
113133
if namespace == "" {
114-
return errors.New("MY_NAMESPACE environment variable must be set")
134+
return errors.New("POD_NAMESPACE environment variable must be set")
135+
}
136+
137+
podName := os.Getenv("POD_NAME")
138+
if podName == "" {
139+
return errors.New("POD_NAME environment variable must be set")
115140
}
116141

117142
var gwNsName *types.NamespacedName
118143
if cmd.Flags().Changed(gatewayFlag) {
119144
gwNsName = &gateway.value
120145
}
121146

122-
metricsConfig := config.MetricsConfig{}
123-
if !disableMetrics {
124-
metricsConfig.Enabled = true
125-
metricsConfig.Port = metricsListenPort.value
126-
metricsConfig.Secure = metricsSecure
127-
}
128-
129147
conf := config.Config{
130148
GatewayCtlrName: gatewayCtlrName.value,
131149
ConfigName: configName.String(),
@@ -136,11 +154,20 @@ func createStaticModeCommand() *cobra.Command {
136154
Namespace: namespace,
137155
GatewayNsName: gwNsName,
138156
UpdateGatewayClassStatus: updateGCStatus,
139-
MetricsConfig: metricsConfig,
140157
HealthConfig: config.HealthConfig{
141158
Enabled: !disableHealth,
142159
Port: healthListenPort.value,
143160
},
161+
MetricsConfig: config.MetricsConfig{
162+
Enabled: !disableMetrics,
163+
Port: metricsListenPort.value,
164+
Secure: metricsSecure,
165+
},
166+
LeaderElection: config.LeaderElection{
167+
Enabled: !disableLeaderElection,
168+
LockName: leaderElectionLockName.String(),
169+
Identity: podName,
170+
},
144171
}
145172

146173
if err := static.StartManager(conf); err != nil {
@@ -163,53 +190,69 @@ func createStaticModeCommand() *cobra.Command {
163190

164191
cmd.Flags().VarP(
165192
&configName,
166-
"config",
193+
configFlag,
167194
"c",
168195
`The name of the NginxGateway resource to be used for this controller's dynamic configuration.`+
169196
` Lives in the same Namespace as the controller.`,
170197
)
171198

172199
cmd.Flags().BoolVar(
173200
&updateGCStatus,
174-
"update-gatewayclass-status",
201+
updateGCStatusFlag,
175202
true,
176203
"Update the status of the GatewayClass resource.",
177204
)
178205

179206
cmd.Flags().BoolVar(
180207
&disableMetrics,
181-
"metrics-disable",
208+
metricsDisableFlag,
182209
false,
183210
"Disable exposing metrics in the Prometheus format.",
184211
)
185212

186213
cmd.Flags().Var(
187214
&metricsListenPort,
188-
"metrics-port",
215+
metricsPortFlag,
189216
"Set the port where the metrics are exposed. Format: [1024 - 65535]",
190217
)
191218

192219
cmd.Flags().BoolVar(
193220
&metricsSecure,
194-
"metrics-secure-serving",
221+
metricsSecureFlag,
195222
false,
196223
"Enable serving metrics via https. By default metrics are served via http."+
197224
" Please note that this endpoint will be secured with a self-signed certificate.",
198225
)
199226

200227
cmd.Flags().BoolVar(
201228
&disableHealth,
202-
"health-disable",
229+
healthDisableFlag,
203230
false,
204231
"Disable running the health probe server.",
205232
)
206233

207234
cmd.Flags().Var(
208235
&healthListenPort,
209-
"health-port",
236+
healthPortFlag,
210237
"Set the port where the health probe server is exposed. Format: [1024 - 65535]",
211238
)
212239

240+
cmd.Flags().BoolVar(
241+
&disableLeaderElection,
242+
leaderElectionDisableFlag,
243+
false,
244+
"Disable leader election. Leader election is used to avoid multiple replicas of the NGINX Kubernetes Gateway"+
245+
" reporting the status of the Gateway API resources. If disabled, "+
246+
"all replicas of NGINX Kubernetes Gateway will update the statuses of the Gateway API resources.",
247+
)
248+
249+
cmd.Flags().Var(
250+
&leaderElectionLockName,
251+
leaderElectionLockNameFlag,
252+
"The name of the leader election lock. "+
253+
"A Lease object with this name will be created in the same Namespace as the controller.",
254+
)
255+
213256
return cmd
214257
}
215258

‎cmd/gateway/commands_test.go

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -123,6 +123,8 @@ func TestStaticModeCmdFlagValidation(t *testing.T) {
123123
"--metrics-secure-serving",
124124
"--health-port=8081",
125125
"--health-disable",
126+
"--leader-election-lock-name=my-lock",
127+
"--leader-election-disable=false",
126128
},
127129
wantErr: false,
128130
},
@@ -243,6 +245,22 @@ func TestStaticModeCmdFlagValidation(t *testing.T) {
243245
expectedErrPrefix: `invalid argument "999" for "--health-disable" flag: strconv.ParseBool:` +
244246
` parsing "999": invalid syntax`,
245247
},
248+
{
249+
name: "leader-election-lock-name is set to invalid string",
250+
args: []string{
251+
"--leader-election-lock-name=!@#$",
252+
},
253+
wantErr: true,
254+
expectedErrPrefix: `invalid argument "!@#$" for "--leader-election-lock-name" flag: invalid format`,
255+
},
256+
{
257+
name: "leader-election-disable is set to empty string",
258+
args: []string{
259+
"--leader-election-disable=",
260+
},
261+
wantErr: true,
262+
expectedErrPrefix: `invalid argument "" for "--leader-election-disable" flag: strconv.ParseBool`,
263+
},
246264
}
247265

248266
for _, test := range tests {

‎conformance/provisioner/static-deployment.yaml

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,6 @@ metadata:
1010
app.kubernetes.io/instance: nginx-gateway
1111
app.kubernetes.io/version: "edge"
1212
spec:
13-
# We only support a single replica for now
1413
replicas: 1
1514
selector:
1615
matchLabels:
@@ -30,15 +29,20 @@ spec:
3029
- --config=nginx-gateway-config
3130
- --metrics-disable
3231
- --health-port=8081
32+
- --leader-election-lock-name=nginx-gateway-leader-election
3333
env:
3434
- name: POD_IP
3535
valueFrom:
3636
fieldRef:
3737
fieldPath: status.podIP
38-
- name: MY_NAMESPACE
38+
- name: POD_NAMESPACE
3939
valueFrom:
4040
fieldRef:
4141
fieldPath: metadata.namespace
42+
- name: POD_NAME
43+
valueFrom:
44+
fieldRef:
45+
fieldPath: metadata.name
4246
image: ghcr.io/nginxinc/nginx-kubernetes-gateway:edge
4347
imagePullPolicy: Always
4448
name: nginx-gateway

0 commit comments

Comments
 (0)