Skip to content

Commit 37490ef

Browse files
ciarams87ciarams87
authored
Integrate NGINX prometheus exporter and enable metrics server (#999)
* Integrate prometheus exporter and enable metrics server
1 parent 44cefeb commit 37490ef

30 files changed

+636
-106
lines changed

‎Makefile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -135,7 +135,7 @@ build-nkg-debug-image: debug-build build-nkg-image ## Build NKG image with debug
135135
generate-manifests: ## Generate manifests using Helm.
136136
cp $(CHART_DIR)/crds/* $(MANIFEST_DIR)/crds/
137137
helm template nginx-gateway $(CHART_DIR) $(HELM_TEMPLATE_COMMON_ARGS) $(HELM_TEMPLATE_EXTRA_ARGS_FOR_ALL_MANIFESTS_FILE) -n nginx-gateway | cat $(strip $(MANIFEST_DIR))/namespace.yaml - > $(strip $(MANIFEST_DIR))/nginx-gateway.yaml
138-
helm template nginx-gateway $(CHART_DIR) $(HELM_TEMPLATE_COMMON_ARGS) -n nginx-gateway -s templates/deployment.yaml > conformance/provisioner/static-deployment.yaml
138+
helm template nginx-gateway $(CHART_DIR) $(HELM_TEMPLATE_COMMON_ARGS) --set metrics.disable=true -n nginx-gateway -s templates/deployment.yaml > conformance/provisioner/static-deployment.yaml
139139
helm template nginx-gateway $(CHART_DIR) $(HELM_TEMPLATE_COMMON_ARGS) -n nginx-gateway -s templates/service.yaml > $(strip $(MANIFEST_DIR))/service/loadbalancer.yaml
140140
helm template nginx-gateway $(CHART_DIR) $(HELM_TEMPLATE_COMMON_ARGS) --set service.annotations.'service\.beta\.kubernetes\.io\/aws-load-balancer-type'="nlb" -n nginx-gateway -s templates/service.yaml > $(strip $(MANIFEST_DIR))/service/loadbalancer-aws-nlb.yaml
141141
helm template nginx-gateway $(CHART_DIR) $(HELM_TEMPLATE_COMMON_ARGS) --set service.type=NodePort --set service.externalTrafficPolicy="" -n nginx-gateway -s templates/service.yaml > $(strip $(MANIFEST_DIR))/service/nodeport.yaml

‎cmd/gateway/commands.go

Lines changed: 41 additions & 57 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ const (
2424
gatewayCtrlNameFlag = "gateway-ctlr-name"
2525
gatewayCtrlNameUsageFmt = `The name of the Gateway controller. ` +
2626
`The controller name must be of the form: DOMAIN/PATH. The controller's domain is '%s'`
27+
gatewayFlag = "gateway"
2728
)
2829

2930
var (
@@ -36,58 +37,21 @@ var (
3637
gatewayClassName = stringValidatingValue{
3738
validator: validateResourceName,
3839
}
39-
)
4040

41-
// stringValidatingValue is a string flag value with custom validation logic.
42-
// it implements the pflag.Value interface.
43-
type stringValidatingValue struct {
44-
validator func(v string) error
45-
value string
46-
}
41+
// Backing values for static subcommand cli flags.
42+
updateGCStatus bool
43+
disableMetrics bool
44+
metricsSecure bool
4745

48-
func (v *stringValidatingValue) String() string {
49-
return v.value
50-
}
51-
52-
func (v *stringValidatingValue) Set(param string) error {
53-
if err := v.validator(param); err != nil {
54-
return err
46+
metricsListenPort = intValidatingValue{
47+
validator: validatePort,
48+
value: 9113,
5549
}
56-
v.value = param
57-
return nil
58-
}
59-
60-
func (v *stringValidatingValue) Type() string {
61-
return "string"
62-
}
63-
64-
// namespacedNameValue is a string flag value that represents a namespaced name.
65-
// it implements the pflag.Value interface.
66-
type namespacedNameValue struct {
67-
value types.NamespacedName
68-
}
69-
70-
func (v *namespacedNameValue) String() string {
71-
if (v.value == types.NamespacedName{}) {
72-
// if we don't do that, the default value in the help message will be printed as "/"
73-
return ""
74-
}
75-
return v.value.String()
76-
}
77-
78-
func (v *namespacedNameValue) Set(param string) error {
79-
nsname, err := parseNamespacedResourceName(param)
80-
if err != nil {
81-
return err
50+
gateway = namespacedNameValue{}
51+
configName = stringValidatingValue{
52+
validator: validateResourceName,
8253
}
83-
84-
v.value = nsname
85-
return nil
86-
}
87-
88-
func (v *namespacedNameValue) Type() string {
89-
return "string"
90-
}
54+
)
9155

9256
func createRootCommand() *cobra.Command {
9357
rootCmd := &cobra.Command{
@@ -115,15 +79,6 @@ func createRootCommand() *cobra.Command {
11579
}
11680

11781
func createStaticModeCommand() *cobra.Command {
118-
const gatewayFlag = "gateway"
119-
120-
// flag values
121-
gateway := namespacedNameValue{}
122-
var updateGCStatus bool
123-
configName := stringValidatingValue{
124-
validator: validateResourceName,
125-
}
126-
12782
cmd := &cobra.Command{
12883
Use: "static-mode",
12984
Short: "Configure NGINX in the scope of a single Gateway resource",
@@ -153,6 +108,13 @@ func createStaticModeCommand() *cobra.Command {
153108
gwNsName = &gateway.value
154109
}
155110

111+
metricsConfig := config.MetricsConfig{}
112+
if !disableMetrics {
113+
metricsConfig.Enabled = true
114+
metricsConfig.Port = metricsListenPort.value
115+
metricsConfig.Secure = metricsSecure
116+
}
117+
156118
conf := config.Config{
157119
GatewayCtlrName: gatewayCtlrName.value,
158120
ConfigName: configName.String(),
@@ -163,6 +125,7 @@ func createStaticModeCommand() *cobra.Command {
163125
Namespace: namespace,
164126
GatewayNsName: gwNsName,
165127
UpdateGatewayClassStatus: updateGCStatus,
128+
MetricsConfig: metricsConfig,
166129
}
167130

168131
if err := static.StartManager(conf); err != nil {
@@ -198,6 +161,27 @@ func createStaticModeCommand() *cobra.Command {
198161
"Update the status of the GatewayClass resource.",
199162
)
200163

164+
cmd.Flags().BoolVar(
165+
&disableMetrics,
166+
"metrics-disable",
167+
false,
168+
"Disable exposing metrics in the Prometheus format.",
169+
)
170+
171+
cmd.Flags().Var(
172+
&metricsListenPort,
173+
"metrics-port",
174+
"Set the port where the metrics are exposed. Format: [1023 - 65535]",
175+
)
176+
177+
cmd.Flags().BoolVar(
178+
&metricsSecure,
179+
"metrics-secure-serving",
180+
false,
181+
"Enable serving metrics via https. By default metrics are served via http."+
182+
" Please note that this endpoint will be secured with a self-signed certificate.",
183+
)
184+
201185
return cmd
202186
}
203187

‎cmd/gateway/commands_test.go

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -118,6 +118,9 @@ func TestStaticModeCmdFlagValidation(t *testing.T) {
118118
"--gateway=nginx-gateway/nginx",
119119
"--config=nginx-gateway-config",
120120
"--update-gatewayclass-status=true",
121+
"--metrics-port=9114",
122+
"--metrics-disable",
123+
"--metrics-secure-serving",
121124
},
122125
wantErr: false,
123126
},
@@ -175,6 +178,42 @@ func TestStaticModeCmdFlagValidation(t *testing.T) {
175178
wantErr: true,
176179
expectedErrPrefix: `invalid argument "invalid" for "--update-gatewayclass-status" flag: strconv.ParseBool`,
177180
},
181+
{
182+
name: "metrics-port is invalid type",
183+
args: []string{
184+
"--metrics-port=invalid", // not an int
185+
},
186+
wantErr: true,
187+
expectedErrPrefix: `invalid argument "invalid" for "--metrics-port" flag: failed to parse int value:` +
188+
` strconv.ParseInt: parsing "invalid": invalid syntax`,
189+
},
190+
{
191+
name: "metrics-port is outside of range",
192+
args: []string{
193+
"--metrics-port=999", // outside of range
194+
},
195+
wantErr: true,
196+
expectedErrPrefix: `invalid argument "999" for "--metrics-port" flag:` +
197+
` port outside of valid port range [1024 - 65535]: 999`,
198+
},
199+
{
200+
name: "metrics-disable is not a bool",
201+
args: []string{
202+
"--metrics-disable=999", // not a bool
203+
},
204+
wantErr: true,
205+
expectedErrPrefix: `invalid argument "999" for "--metrics-disable" flag: strconv.ParseBool:` +
206+
` parsing "999": invalid syntax`,
207+
},
208+
{
209+
name: "metrics-secure-serving is not a bool",
210+
args: []string{
211+
"--metrics-secure-serving=999", // not a bool
212+
},
213+
wantErr: true,
214+
expectedErrPrefix: `invalid argument "999" for "--metrics-secure-serving" flag: strconv.ParseBool:` +
215+
` parsing "999": invalid syntax`,
216+
},
178217
}
179218

180219
for _, test := range tests {

‎cmd/gateway/validating_types.go

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
package main
2+
3+
import (
4+
"fmt"
5+
"strconv"
6+
7+
"k8s.io/apimachinery/pkg/types"
8+
)
9+
10+
// stringValidatingValue is a string flag value with custom validation logic.
11+
// it implements the pflag.Value interface.
12+
type stringValidatingValue struct {
13+
validator func(v string) error
14+
value string
15+
}
16+
17+
func (v *stringValidatingValue) String() string {
18+
return v.value
19+
}
20+
21+
func (v *stringValidatingValue) Set(param string) error {
22+
if err := v.validator(param); err != nil {
23+
return err
24+
}
25+
v.value = param
26+
return nil
27+
}
28+
29+
func (v *stringValidatingValue) Type() string {
30+
return "string"
31+
}
32+
33+
type intValidatingValue struct {
34+
validator func(v int) error
35+
value int
36+
}
37+
38+
func (v *intValidatingValue) String() string {
39+
return strconv.Itoa(v.value)
40+
}
41+
42+
func (v *intValidatingValue) Set(param string) error {
43+
intVal, err := strconv.ParseInt(param, 10, 32)
44+
if err != nil {
45+
return fmt.Errorf("failed to parse int value: %w", err)
46+
}
47+
48+
if err := v.validator(int(intVal)); err != nil {
49+
return err
50+
}
51+
52+
v.value = int(intVal)
53+
return nil
54+
}
55+
56+
func (v *intValidatingValue) Type() string {
57+
return "int"
58+
}
59+
60+
// namespacedNameValue is a string flag value that represents a namespaced name.
61+
// it implements the pflag.Value interface.
62+
type namespacedNameValue struct {
63+
value types.NamespacedName
64+
}
65+
66+
func (v *namespacedNameValue) String() string {
67+
if (v.value == types.NamespacedName{}) {
68+
// if we don't do that, the default value in the help message will be printed as "/"
69+
return ""
70+
}
71+
return v.value.String()
72+
}
73+
74+
func (v *namespacedNameValue) Set(param string) error {
75+
nsname, err := parseNamespacedResourceName(param)
76+
if err != nil {
77+
return err
78+
}
79+
80+
v.value = nsname
81+
return nil
82+
}
83+
84+
func (v *namespacedNameValue) Type() string {
85+
return "string"
86+
}

‎cmd/gateway/validation.go

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -99,3 +99,11 @@ func validateIP(ip string) error {
9999

100100
return nil
101101
}
102+
103+
// validatePort makes sure a given port is inside the valid port range for its usage
104+
func validatePort(port int) error {
105+
if port < 1024 || port > 65535 {
106+
return fmt.Errorf("port outside of valid port range [1024 - 65535]: %v", port)
107+
}
108+
return nil
109+
}

‎cmd/gateway/validation_test.go

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -294,3 +294,40 @@ func TestValidateIP(t *testing.T) {
294294
})
295295
}
296296
}
297+
298+
func TestValidatePort(t *testing.T) {
299+
tests := []struct {
300+
name string
301+
port int
302+
expErr bool
303+
}{
304+
{
305+
name: "port under minimum allowed value",
306+
port: 1023,
307+
expErr: true,
308+
},
309+
{
310+
name: "port over maximum allowed value",
311+
port: 65536,
312+
expErr: true,
313+
},
314+
{
315+
name: "valid port",
316+
port: 9113,
317+
expErr: false,
318+
},
319+
}
320+
321+
for _, tc := range tests {
322+
t.Run(tc.name, func(t *testing.T) {
323+
g := NewGomegaWithT(t)
324+
325+
err := validatePort(tc.port)
326+
if !tc.expErr {
327+
g.Expect(err).ToNot(HaveOccurred())
328+
} else {
329+
g.Expect(err).To(HaveOccurred())
330+
}
331+
})
332+
}
333+
}

‎conformance/provisioner/static-deployment.yaml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ spec:
2828
- --gateway-ctlr-name=gateway.nginx.org/nginx-gateway-controller
2929
- --gatewayclass=nginx
3030
- --config=nginx-gateway-config
31+
- --metrics-disable
3132
env:
3233
- name: POD_IP
3334
valueFrom:

‎deploy/helm-chart/README.md

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -142,3 +142,6 @@ The following tables lists the configurable parameters of the NGINX Kubernetes G
142142
| `service.externalTrafficPolicy` | The `externalTrafficPolicy` of the service. The value `Local` preserves the client source IP. | Local |
143143
| `service.annotations` | The `annotations` of the NGINX Kubernetes Gateway service. | {} |
144144
| `service.ports` | A list of ports to expose through the NGINX Kubernetes Gateway service. Update it to match the listener ports from your Gateway resource. Follows the conventional Kubernetes yaml syntax for service ports. | [ port: 80, targetPort: 80, protocol: TCP, name: http; port: 443, targetPort: 443, protocol: TCP, name: https ] |
145+
| `metrics.disable` | Disable exposing metrics in the Prometheus format. |false |
146+
| `metrics.port` | Set the port where the Prometheus metrics are exposed. Format: [1024 - 65535] |9113 |
147+
| `metrics.secure` | Enable serving metrics via https. By default metrics are served via http. Please note that this endpoint will be secured with a self-signed certificate. |false |

0 commit comments

Comments
 (0)