diff --git a/tests/scale/generate_manifests.go b/tests/scale/generate_manifests.go new file mode 100644 index 0000000000..025744344a --- /dev/null +++ b/tests/scale/generate_manifests.go @@ -0,0 +1,277 @@ +//go:build scale +// +build scale + +package scale + +import ( + "bytes" + "fmt" + "os" + "path/filepath" + "text/template" +) + +var gwTmplTxt = `apiVersion: gateway.networking.k8s.io/v1beta1 +kind: Gateway +metadata: + name: gateway +spec: + gatewayClassName: nginx + listeners: +{{- range $l := . }} + - name: {{ $l.Name }} + hostname: "{{ $l.HostnamePrefix }}.example.com"{{ if ne $l.SecretName "" }} + port: 443 + protocol: HTTPS + tls: + mode: Terminate + certificateRefs: + - kind: Secret + name: {{ $l.SecretName }}{{ else }} + port: 80 + protocol: HTTP + {{- end -}} +{{- end -}}` + +var hrTmplTxt = `apiVersion: gateway.networking.k8s.io/v1beta1 +kind: HTTPRoute +metadata: + name: {{ .Name }} +spec: + parentRefs: + - name: gateway + sectionName: {{ .ListenerName }} + hostnames: + - "{{ .HostnamePrefix }}.example.com" + rules: + - matches: + - path: + type: PathPrefix + value: / + backendRefs: + - name: {{ .BackendName }} + port: 80` + +// nolint:all +var secretTmplTxt = `apiVersion: v1 +kind: Secret +metadata: + name: {{ . }} +type: kubernetes.io/tls +data: + tls.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUNzakNDQVpvQ0NRQzdCdVdXdWRtRkNEQU5CZ2txaGtpRzl3MEJBUXNGQURBYk1Sa3dGd1lEVlFRRERCQmoKWVdabExtVjRZVzF3YkdVdVkyOXRNQjRYRFRJeU1EY3hOREl4TlRJek9Wb1hEVEl6TURjeE5ESXhOVEl6T1ZvdwpHekVaTUJjR0ExVUVBd3dRWTJGbVpTNWxlR0Z0Y0d4bExtTnZiVENDQVNJd0RRWUpLb1pJaHZjTkFRRUJCUUFECmdnRVBBRENDQVFvQ2dnRUJBTHFZMnRHNFc5aStFYzJhdnV4Q2prb2tnUUx1ek10U1Rnc1RNaEhuK3ZRUmxIam8KVzFLRnMvQVdlS25UUStyTWVKVWNseis4M3QwRGtyRThwUisxR2NKSE50WlNMb0NEYUlRN0Nhck5nY1daS0o4Qgo1WDNnVS9YeVJHZjI2c1REd2xzU3NkSEQ1U2U3K2Vab3NPcTdHTVF3K25HR2NVZ0VtL1Q1UEMvY05PWE0zZWxGClRPL051MStoMzROVG9BbDNQdTF2QlpMcDNQVERtQ0thaEROV0NWbUJQUWpNNFI4VERsbFhhMHQ5Z1o1MTRSRzUKWHlZWTNtdzZpUzIrR1dYVXllMjFuWVV4UEhZbDV4RHY0c0FXaGRXbElweHlZQlNCRURjczN6QlI2bFF1OWkxZAp0R1k4dGJ3blVmcUVUR3NZdWxzc05qcU95V1VEcFdJelhibHhJZVVDQXdFQUFUQU5CZ2txaGtpRzl3MEJBUXNGCkFBT0NBUUVBcjkrZWJ0U1dzSnhLTGtLZlRkek1ISFhOd2Y5ZXFVbHNtTXZmMGdBdWVKTUpUR215dG1iWjlpbXQKL2RnWlpYVE9hTElHUG9oZ3BpS0l5eVVRZVdGQ2F0NHRxWkNPVWRhbUloOGk0Q1h6QVJYVHNvcUNOenNNLzZMRQphM25XbFZyS2lmZHYrWkxyRi8vblc0VVNvOEoxaCtQeDljY0tpRDZZU0RVUERDRGh1RUtFWXcvbHpoUDJVOXNmCnl6cEJKVGQ4enFyM3paTjNGWWlITmgzYlRhQS82di9jU2lyamNTK1EwQXg4RWpzQzYxRjRVMTc4QzdWNWRCKzQKcmtPTy9QNlA0UFlWNTRZZHMvRjE2WkZJTHFBNENCYnExRExuYWRxamxyN3NPbzl2ZzNnWFNMYXBVVkdtZ2todAp6VlZPWG1mU0Z4OS90MDBHUi95bUdPbERJbWlXMGc9PQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== + tls.key: LS0tLS1CRUdJTiBQUklWQVRFIEtFWS0tLS0tCk1JSUV2UUlCQURBTkJna3Foa2lHOXcwQkFRRUZBQVNDQktjd2dnU2pBZ0VBQW9JQkFRQzZtTnJSdUZ2WXZoSE4KbXI3c1FvNUtKSUVDN3N6TFVrNExFeklSNS9yMEVaUjQ2RnRTaGJQd0ZuaXAwMFBxekhpVkhKYy92TjdkQTVLeApQS1VmdFJuQ1J6YldVaTZBZzJpRU93bXF6WUhGbVNpZkFlVjk0RlAxOGtSbjl1ckV3OEpiRXJIUncrVW51L25tCmFMRHF1eGpFTVBweGhuRklCSnYwK1R3djNEVGx6TjNwUlV6dnpidGZvZCtEVTZBSmR6N3Rid1dTNmR6MHc1Z2kKbW9RelZnbFpnVDBJek9FZkV3NVpWMnRMZllHZWRlRVJ1VjhtR041c09va3R2aGxsMU1udHRaMkZNVHgySmVjUQo3K0xBRm9YVnBTS2NjbUFVZ1JBM0xOOHdVZXBVTHZZdFhiUm1QTFc4SjFINmhFeHJHTHBiTERZNmpzbGxBNlZpCk0xMjVjU0hsQWdNQkFBRUNnZ0VBQnpaRE50bmVTdWxGdk9HZlFYaHRFWGFKdWZoSzJBenRVVVpEcUNlRUxvekQKWlV6dHdxbkNRNlJLczUyandWNTN4cU9kUU94bTNMbjNvSHdNa2NZcEliWW82MjJ2dUczYnkwaVEzaFlsVHVMVgpqQmZCcS9UUXFlL2NMdngvSkczQWhFNmJxdFRjZFlXeGFmTmY2eUtpR1dzZk11WVVXTWs4MGVJVUxuRmZaZ1pOCklYNTlSOHlqdE9CVm9Sa3hjYTVoMW1ZTDFsSlJNM3ZqVHNHTHFybmpOTjNBdWZ3ZGRpK1VDbGZVL2l0K1EvZkUKV216aFFoTlRpNVFkRWJLVStOTnYvNnYvb2JvandNb25HVVBCdEFTUE05cmxFemIralQ1WHdWQjgvLzRGY3VoSwoyVzNpcjhtNHVlQ1JHSVlrbGxlLzhuQmZ0eVhiVkNocVRyZFBlaGlPM1FLQmdRRGlrR3JTOTc3cjg3Y1JPOCtQClpoeXltNXo4NVIzTHVVbFNTazJiOTI1QlhvakpZL2RRZDVTdFVsSWE4OUZKZnNWc1JRcEhHaTFCYzBMaTY1YjIKazR0cE5xcVFoUmZ1UVh0UG9GYXRuQzlPRnJVTXJXbDVJN0ZFejZnNkNQMVBXMEg5d2hPemFKZUdpZVpNYjlYTQoybDdSSFZOcC9jTDlYbmhNMnN0Q1lua2Iwd0tCZ1FEUzF4K0crakEyUVNtRVFWNXA1RnRONGcyamsyZEFjMEhNClRIQ2tTazFDRjhkR0Z2UWtsWm5ZbUt0dXFYeXNtekJGcnZKdmt2eUhqbUNYYTducXlpajBEdDZtODViN3BGcVAKQWxtajdtbXI3Z1pUeG1ZMXBhRWFLMXY4SDNINGtRNVl3MWdrTWRybVJHcVAvaTBGaDVpaGtSZS9DOUtGTFVkSQpDcnJjTzhkUVp3S0JnSHA1MzRXVWNCMVZibzFlYStIMUxXWlFRUmxsTWlwRFM2TzBqeWZWSmtFb1BZSEJESnp2ClIrdzZLREJ4eFoyWmJsZ05LblV0YlhHSVFZd3lGelhNcFB5SGxNVHpiZkJhYmJLcDFyR2JVT2RCMXpXM09PRkgKcmppb21TUm1YNmxhaDk0SjRHU0lFZ0drNGw1SHhxZ3JGRDZ2UDd4NGRjUktJWFpLZ0w2dVJSSUpBb0dCQU1CVApaL2p5WStRNTBLdEtEZHUrYU9ORW4zaGxUN3hrNXRKN3NBek5rbWdGMU10RXlQUk9Xd1pQVGFJbWpRbk9qbHdpCldCZ2JGcXg0M2ZlQ1Z4ZXJ6V3ZEM0txaWJVbWpCTkNMTGtYeGh3ZEVteFQwVit2NzZGYzgwaTNNYVdSNnZZR08KditwVVovL0F6UXdJcWZ6dlVmV2ZxdStrMHlhVXhQOGNlcFBIRyt0bEFvR0FmQUtVVWhqeFU0Ym5vVzVwVUhKegpwWWZXZXZ5TW54NWZyT2VsSmRmNzlvNGMvMHhVSjh1eFBFWDFkRmNrZW96dHNpaVFTNkN6MENRY09XVWxtSkRwCnVrdERvVzM3VmNSQU1BVjY3NlgxQVZlM0UwNm5aL2g2Tkd4Z28rT042Q3pwL0lkMkJPUm9IMFAxa2RjY1NLT3kKMUtFZlNnb1B0c1N1eEpBZXdUZmxDMXc9Ci0tLS0tRU5EIFBSSVZBVEUgS0VZLS0tLS0K +` + +var appTmplTxt = `apiVersion: v1 +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ . }} +spec: + replicas: 1 + selector: + matchLabels: + app: {{ . }} + template: + metadata: + labels: + app: {{ . }} + spec: + containers: + - name: nginx + image: nginxdemos/nginx-hello:plain-text + ports: + - containerPort: 8080 +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ . }} +spec: + ports: + - port: 80 + targetPort: 8080 + protocol: TCP + name: http + selector: + app: {{ . }} +` + +var ( + gwTmpl = template.Must(template.New("gw").Parse(gwTmplTxt)) + hrTmpl = template.Must(template.New("hr").Parse(hrTmplTxt)) + secretTmpl = template.Must(template.New("secret").Parse(secretTmplTxt)) + appTmpl = template.Must(template.New("app").Parse(appTmplTxt)) +) + +type Listener struct { + Name string + HostnamePrefix string + SecretName string +} + +type Route struct { + Name string + ListenerName string + HostnamePrefix string + BackendName string +} + +func getPrereqDirName(manifestDir string) string { + return filepath.Join(manifestDir, "prereqs") +} + +func generateScaleListenerManifests(numListeners int, manifestDir string, tls bool) error { + listeners := make([]Listener, 0) + backends := make([]string, 0) + secrets := make([]string, 0) + + for i := 0; i < numListeners; i++ { + listenerName := fmt.Sprintf("listener-%d", i) + hostnamePrefix := fmt.Sprintf("%d", i) + backendName := fmt.Sprintf("backend-%d", i) + + var secretName string + if tls { + secretName = fmt.Sprintf("secret-%d", i) + secrets = append(secrets, secretName) + } + + listeners = append(listeners, Listener{ + Name: listenerName, + HostnamePrefix: hostnamePrefix, + SecretName: secretName, + }) + + route := Route{ + Name: fmt.Sprintf("route-%d", i), + ListenerName: listenerName, + HostnamePrefix: hostnamePrefix, + BackendName: backendName, + } + + backends = append(backends, backendName) + + if err := generateManifests(manifestDir, i, listeners, []Route{route}); err != nil { + return err + } + } + + if err := generateSecrets(getPrereqDirName(manifestDir), secrets); err != nil { + return err + } + + return generateBackendAppManifests(getPrereqDirName(manifestDir), backends) +} + +func generateSecrets(secretsDir string, secrets []string) error { + err := os.Mkdir(secretsDir, 0o750) + if err != nil && !os.IsExist(err) { + return err + } + + for _, secret := range secrets { + var buf bytes.Buffer + + if err = secretTmpl.Execute(&buf, secret); err != nil { + return err + } + + path := filepath.Join(secretsDir, fmt.Sprintf("%s.yaml", secret)) + + fmt.Println("Writing", path) + if err := os.WriteFile(path, buf.Bytes(), 0o600); err != nil { + return err + } + } + + return nil +} + +func generateScaleHTTPRouteManifests(numRoutes int, manifestDir string) error { + l := Listener{ + Name: "listener", + HostnamePrefix: "*", + } + + backendName := "backend" + + for i := 0; i < numRoutes; i++ { + + route := Route{ + Name: fmt.Sprintf("route-%d", i), + HostnamePrefix: fmt.Sprintf("%d", i), + ListenerName: "listener", + BackendName: backendName, + } + + var listeners []Listener + if i == 0 { + // only generate a Gateway on the first iteration + listeners = []Listener{l} + } + + if err := generateManifests(manifestDir, i, listeners, []Route{route}); err != nil { + return err + } + + } + + return generateBackendAppManifests(getPrereqDirName(manifestDir), []string{backendName}) +} + +func generateManifests(outDir string, version int, listeners []Listener, routes []Route) error { + var buf bytes.Buffer + + if len(listeners) > 0 { + if err := gwTmpl.Execute(&buf, listeners); err != nil { + return err + } + } + + for _, r := range routes { + if buf.Len() > 0 { + buf.Write([]byte("\n---\n")) + } + + if err := hrTmpl.Execute(&buf, r); err != nil { + return err + } + } + + err := os.Mkdir(outDir, 0o750) + if err != nil && !os.IsExist(err) { + return err + } + + filename := fmt.Sprintf("manifest-%d.yaml", version) + path := filepath.Join(outDir, filename) + + fmt.Println("Writing", path) + return os.WriteFile(path, buf.Bytes(), 0o600) +} + +func generateBackendAppManifests(outDir string, backends []string) error { + err := os.Mkdir(outDir, 0o750) + if err != nil && !os.IsExist(err) { + return err + } + + for _, backend := range backends { + var buf bytes.Buffer + + if err = appTmpl.Execute(&buf, backend); err != nil { + return err + } + + path := filepath.Join(outDir, fmt.Sprintf("%s.yaml", backend)) + + fmt.Println("Writing", path) + if err := os.WriteFile(path, buf.Bytes(), 0o600); err != nil { + return err + } + } + + return nil +} diff --git a/tests/scale/manifests/prom-clusterrole.yaml b/tests/scale/manifests/prom-clusterrole.yaml new file mode 100644 index 0000000000..f8aefdd36e --- /dev/null +++ b/tests/scale/manifests/prom-clusterrole.yaml @@ -0,0 +1,44 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: prom +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: prometheus + namespace: prom +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: prometheus + namespace: prom +rules: +- apiGroups: [""] + resources: + - nodes + - services + - endpoints + - pods + verbs: ["get", "list", "watch"] +- apiGroups: [""] + resources: + - configmaps + verbs: ["get"] +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: prometheus + namespace: prom +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: prometheus +subjects: +- kind: ServiceAccount + name: prometheus + namespace: prom diff --git a/tests/scale/manifests/scale-matches.yaml b/tests/scale/manifests/scale-matches.yaml new file mode 100644 index 0000000000..553b5ea88b --- /dev/null +++ b/tests/scale/manifests/scale-matches.yaml @@ -0,0 +1,233 @@ +apiVersion: gateway.networking.k8s.io/v1beta1 +kind: Gateway +metadata: + name: gateway +spec: + gatewayClassName: nginx + listeners: + - name: listener + port: 80 + protocol: HTTP +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: backend +spec: + replicas: 1 + selector: + matchLabels: + app: backend + template: + metadata: + labels: + app: backend + spec: + containers: + - name: nginx + image: nginxdemos/nginx-hello:plain-text + ports: + - containerPort: 8080 +--- +apiVersion: v1 +kind: Service +metadata: + name: backend +spec: + ports: + - port: 80 + targetPort: 8080 + protocol: TCP + name: http + selector: + app: backend +--- +apiVersion: gateway.networking.k8s.io/v1beta1 +kind: HTTPRoute +metadata: + name: route +spec: + parentRefs: + - name: gateway + sectionName: listener + hostnames: + - "cafe.example.com" + rules: + - matches: # Max of 8 matches per rule + - headers: + - name: header-1 + value: header-1-val + - headers: + - name: header-2 + value: header-2-val + - headers: + - name: header-3 + value: header-3-val + - headers: + - name: header-4 + value: header-4-val + - headers: + - name: header-5 + value: header-5-val + - headers: + - name: header-6 + value: header-6-val + - headers: + - name: header-7 + value: header-7-val + - headers: + - name: header-8 + value: header-8-val + backendRefs: + - name: backend + port: 80 + - matches: + - headers: + - name: header-9 + value: header-9-val + - headers: + - name: header-10 + value: header-10-val + - headers: + - name: header-11 + value: header-11-val + - headers: + - name: header-12 + value: header-12-val + - headers: + - name: header-13 + value: header-13-val + - headers: + - name: header-14 + value: header-14-val + - headers: + - name: header-15 + value: header-15-val + - headers: + - name: header-16 + value: header-16-val + backendRefs: + - name: backend + port: 80 + - matches: + - headers: + - name: header-17 + value: header-17-val + - headers: + - name: header-18 + value: header-18-val + - headers: + - name: header-19 + value: header-19-val + - headers: + - name: header-20 + value: header-20-val + - headers: + - name: header-21 + value: header-21-val + - headers: + - name: header-22 + value: header-22-val + - headers: + - name: header-23 + value: header-23-val + - headers: + - name: header-24 + value: header-24-val + backendRefs: + - name: backend + port: 80 + - matches: + - headers: + - name: header-25 + value: header-25-val + - headers: + - name: header-26 + value: header-26-val + - headers: + - name: header-27 + value: header-27-val + - headers: + - name: header-28 + value: header-28-val + - headers: + - name: header-29 + value: header-29-val + - headers: + - name: header-30 + value: header-30-val + - headers: + - name: header-31 + value: header-31-val + - headers: + - name: header-32 + value: header-32-val + backendRefs: + - name: backend + port: 80 + - matches: + - headers: + - name: header-33 + value: header-33-val + - headers: + - name: header-34 + value: header-34-val + - headers: + - name: header-35 + value: header-35-val + - headers: + - name: header-36 + value: header-36-val + - headers: + - name: header-37 + value: header-37-val + - headers: + - name: header-38 + value: header-38-val + - headers: + - name: header-39 + value: header-39-val + - headers: + - name: header-40 + value: header-40-val + backendRefs: + - name: backend + port: 80 + - matches: + - headers: + - name: header-41 + value: header-41-val + - headers: + - name: header-42 + value: header-42-val + - headers: + - name: header-43 + value: header-43-val + - headers: + - name: header-44 + value: header-44-val + - headers: + - name: header-45 + value: header-45-val + - headers: + - name: header-46 + value: header-46-val + - headers: + - name: header-47 + value: header-47-val + - headers: + - name: header-48 + value: header-48-val + backendRefs: + - name: backend + port: 80 + - matches: + - headers: + - name: header-49 + value: header-49-val + - headers: + - name: header-50 + value: header-50-val # max number of headers allowed for a single hostname/path b/c of nginx var limit + backendRefs: + - name: backend + port: 80 diff --git a/tests/scale/manifests/scale-upstreams.yaml b/tests/scale/manifests/scale-upstreams.yaml new file mode 100644 index 0000000000..298a4b876f --- /dev/null +++ b/tests/scale/manifests/scale-upstreams.yaml @@ -0,0 +1,63 @@ +apiVersion: gateway.networking.k8s.io/v1beta1 +kind: Gateway +metadata: + name: gateway +spec: + gatewayClassName: nginx + listeners: + - name: listener + hostname: "*.example.com" + port: 80 + protocol: HTTP +--- +apiVersion: gateway.networking.k8s.io/v1beta1 +kind: HTTPRoute +metadata: + name: route +spec: + parentRefs: + - name: gateway + sectionName: listener + hostnames: + - "*.example.com" + rules: + - matches: + - path: + type: PathPrefix + value: / + backendRefs: + - name: backend + port: 80 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: backend +spec: + replicas: 1 + selector: + matchLabels: + app: backend + template: + metadata: + labels: + app: backend + spec: + containers: + - name: nginx + image: nginxdemos/nginx-hello:plain-text + ports: + - containerPort: 8080 +--- +apiVersion: v1 +kind: Service +metadata: + name: backend +spec: + ports: + - port: 80 + targetPort: 8080 + protocol: TCP + name: http + selector: + app: backend diff --git a/tests/scale/results/1.0.0/1.0.0.md b/tests/scale/results/1.0.0/1.0.0.md new file mode 100644 index 0000000000..0d8b60ad62 --- /dev/null +++ b/tests/scale/results/1.0.0/1.0.0.md @@ -0,0 +1,187 @@ +# Results for v1.0.0 + + +- [Results for v1.0.0](#results-for-v100) + - [Versions](#versions) + - [Tests](#tests) + - [Scale Listeners](#scale-listeners) + - [Scale HTTPS Listeners](#scale-https-listeners) + - [Scale HTTPRoutes](#scale-httproutes) + - [Scale Upstream Servers](#scale-upstream-servers) + - [Scale HTTP Matches](#scale-http-matches) + - [Future Improvements](#future-improvements) + + +## Versions + +NGF version: + +```text +commit: "72b6c6ef8915c697626eeab88fdb6a3ce15b8da0" +date: "2023-10-04T22:22:09Z" +version: "edge" +``` + + +with NGINX: + +```text +nginx/1.25.2 +built by gcc 12.2.1 20220924 (Alpine 12.2.1_git20220924-r10) +OS: Linux 5.15.109+ +``` + +Kubernetes: + +```text +Server Version: version.Info{Major:"1", Minor:"27", +GitVersion:"v1.27.6-gke.1248000", +GitCommit:"85a90ed8e702b392003d6757917e4cc167776e03", +GitTreeState:"clean", BuildDate:"2023-09-21T22:16:57Z", +GoVersion:"go1.20.8 X:boringcrypto", Compiler:"gc", +Platform:"linux/amd64"} +``` + +## Tests + +### Scale Listeners + +| Total Reloads | Total Reload Errors | Average Reload Time (ms) | +|---------------|---------------------|--------------------------| +| 107 | 0 | 155.05374791154367 | + +**NGINX Errors**: None. + +**NGF Errors**: None. + +**Pod Restarts**: None. + +**CPU**: Steep linear increase as NGF processed all the Services. Dropped off during scaling of Listeners. +See [graph](/tests/scale/results/1.0.0/TestScale_Listeners/CPU.png). + +**Memory**: Gradual increase in memory. Topped out at 40MiB. +See [graph](/tests/scale/results/1.0.0/TestScale_Listeners/Memory.png). + +**Time To Ready**: Time to ready numbers consistently under 3s. 62nd Listener had longest TTR of 3.02s. +See [graph](/tests/scale/results/1.0.0/TestScale_Listeners/TTR.png). + +### Scale HTTPS Listeners + +| Total Reloads | Total Reload Errors | Average Reload Time (ms) | +|---------------|---------------------|--------------------------| +| 130 | 0 | 151.49397590361446ms | + +**NGINX Errors**: None. + +**NGF Errors**: None. + +**Pod Restarts**: None. + +**CPU**: Steep linear increase as NGF processed all the Services and Secrets. Dropped off during scaling of Listeners. +See [graph](/tests/scale/results/1.0.0/TestScale_HTTPSListeners/CPU.png). + +**Memory**: Mostly linear increase. Topping out at right under 50MiB. +See [graph](/tests/scale/results/1.0.0/TestScale_HTTPSListeners/Memory.png). + +**Time To Ready**: The time to ready numbers were pretty consistent (under 3 sec) except for one spike of 10s. I believe +this spike was client-side because the NGF logs indicated that the reload successfully happened under 3s. +See [graph](/tests/scale/results/1.0.0/TestScale_HTTPSListeners/TTR.png). + +### Scale HTTPRoutes + +| Total Reloads | Total Reload Errors | Average Reload Time (ms) | +|---------------|---------------------|--------------------------| +| 1001 | 0 | 354.3878787878788ms | + +**NGINX Errors**: None. + +**NGF Errors**: None. + +**Pod Restarts**: None. + +**CPU**: CPU mostly oscillated between .04 and .06. Several spikes over .06. +See [graph](/tests/scale/results/1.0.0/TestScale_HTTPRoutes/CPU.png). + +**Memory**: Memory usage gradually increased from 25 - 150MiB over course of the test with some spikes reaching up to +200MiB. See [graph](/tests/scale/results/1.0.0/TestScale_HTTPRoutes/Memory.png). + +**Time To Ready**: This time to ready graph is unique because there are three plotted lines: + +- Blue Line: 2-second delay after adding a new HTTPRoute. +- Red Line: No delay after adding a new HTTPRoute. +- Green Line: 10-second delay after adding a new HTTPRoute + +The Blue and Red lines are incomplete because the tests timed out. However, I think the implications are pretty clear. +The more time that passes between scaling events, the smaller the time to ready values are. This is because NGF +re-queues all the HTTPRoutes after updating their statuses. This is because the HTTPRoute has changed after we write its +status. This is compounded by the fact that NGF writes status for every HTTPRoute in the graph on every configuration +update. So if you add HTTPRoute 100, NGF will update the configuration with this new route and then update the status of +all 100 HTTPRoutes in the graph. + +Related issues: + +- https://github.com/nginxinc/nginx-gateway-fabric/issues/1013 +- https://github.com/nginxinc/nginx-gateway-fabric/issues/825 + +See [graph](/tests/scale/results/1.0.0/TestScale_HTTPRoutes/TTR.png). + +### Scale Upstream Servers + +| Start Time (UNIX) | End Time (UNIX) | Duration (s) | Total Reloads | Total Reload Errors | Average Reload Time (ms) | +|-------------------|-----------------|--------------|---------------|---------------------|--------------------------| +| 1696535183 | 1696535311 | 128 | 83 | 0 | 126.55555555555557 | + +**NGINX Errors**: None. + +**NGF Errors**: None. + +**Pod Restarts**: None. + +**CPU**: CPU steeply increases as NGF handles all the new Pods. Drops after they are processed. +See [graph](/tests/scale/results/1.0.0/TestScale_UpstreamServers/CPU.png). + +**Memory**: Memory stays relatively flat and under 40MiB. +See [graph](/tests/scale/results/1.0.0/TestScale_UpstreamServers/Memory.png). + +### Scale HTTP Matches + +**Results for the first match**: + +```text +Running 30s test @ http://cafe.example.com +2 threads and 10 connections +Thread Stats Avg Stdev Max +/- Stdev +Latency 47.64ms 13.87ms 217.49ms 97.52% +Req/Sec 107.84 17.47 151.00 79.76% +6410 requests in 30.09s, 0.95MB read +Requests/sec: 213.02 +Transfer/sec: 32.24KB +``` + +**Results for the last match**: + +```text +Running 30s test @ http://cafe.example.com +2 threads and 10 connections +Thread Stats Avg Stdev Max +/- Stdev +Latency 47.10ms 13.59ms 301.73ms 98.57% +Req/Sec 108.01 12.55 150.00 84.62% +6459 requests in 30.10s, 0.95MB read +Requests/sec: 214.61 +Transfer/sec: 32.49KB +``` + +**Findings**: + +- There's not a noticeable difference between the response times for the first match and last match. In +fact, the latency of the last match is slightly lower than the latency of the first match. +- If you add one more match to the [manifest](/tests/scale/manifests/scale-matches.yaml) NGINX will fail to reload + because the generate `http_matches` variable is too long. + +Issue Filed: https://github.com/nginxinc/nginx-gateway-fabric/issues/1107 + +## Future Improvements + +- Check that the statuses of the Gateway API resources are updated after each scaling event. +- Measure the time it takes for NGF to update the status of the Gateway API resources after creating or updating the resources. +- Record the reload time distributions per bucket after each test. diff --git a/tests/scale/results/1.0.0/TestScale_HTTPRoutes/CPU.png b/tests/scale/results/1.0.0/TestScale_HTTPRoutes/CPU.png new file mode 100644 index 0000000000..d881a9c70a Binary files /dev/null and b/tests/scale/results/1.0.0/TestScale_HTTPRoutes/CPU.png differ diff --git a/tests/scale/results/1.0.0/TestScale_HTTPRoutes/Memory.png b/tests/scale/results/1.0.0/TestScale_HTTPRoutes/Memory.png new file mode 100644 index 0000000000..615e8cc70e Binary files /dev/null and b/tests/scale/results/1.0.0/TestScale_HTTPRoutes/Memory.png differ diff --git a/tests/scale/results/1.0.0/TestScale_HTTPRoutes/TTR.png b/tests/scale/results/1.0.0/TestScale_HTTPRoutes/TTR.png new file mode 100644 index 0000000000..7079e762c5 Binary files /dev/null and b/tests/scale/results/1.0.0/TestScale_HTTPRoutes/TTR.png differ diff --git a/tests/scale/results/1.0.0/TestScale_HTTPRoutes/results-10s-delay.csv b/tests/scale/results/1.0.0/TestScale_HTTPRoutes/results-10s-delay.csv new file mode 100644 index 0000000000..4a153e5920 --- /dev/null +++ b/tests/scale/results/1.0.0/TestScale_HTTPRoutes/results-10s-delay.csv @@ -0,0 +1,541 @@ +# HTTPRoutes,Time to Ready (s),Error +1,0.377127053 +2,0.379007896 +3,0.384165516 +4,0.377453448 +5,0.376946431 +6,0.373370894 +7,0.37584378 +8,0.380403844 +9,0.372546743 +10,0.375794438 +11,0.380632329 +12,0.376386433 +13,0.378915807 +14,0.379204325 +15,0.37596693 +16,0.37958542 +17,0.377282514 +18,0.379717313 +19,0.378369917 +20,0.373178842 +21,0.376339261 +22,0.378602457 +23,0.377526567 +24,0.377001291 +25,0.376519688 +26,0.380761083 +27,0.376727453 +28,0.378244891 +29,0.376102086 +30,0.390081367 +31,0.379204727 +32,0.374149871 +33,0.374552792 +34,0.380052266 +35,0.375878862 +36,0.372185661 +37,0.376191911 +38,0.378403682 +39,0.376402242 +40,0.377471489 +41,0.379175334 +42,0.372701415 +43,0.378738686 +44,0.372680741 +45,0.382039848 +46,0.375800207 +47,0.37550487 +48,0.377931247 +49,0.379590986 +50,0.377128272 +51,0.373003265 +52,0.375941264 +53,0.373991868 +54,0.39105751 +55,0.377170034 +56,0.377030239 +57,0.381733031 +58,0.375338096 +59,0.373153098 +60,0.376193201 +61,0.388617968 +62,0.374054113 +63,0.385547646 +64,0.375017699 +65,0.377488533 +66,0.376849327 +67,0.379391472 +68,0.379960901 +69,0.379718635 +70,0.375880048 +71,0.381485872 +72,0.374207373 +73,0.379410339 +74,0.381847412 +75,0.387592821 +76,0.376002385 +77,0.39215576 +78,0.379137899 +79,0.374343388 +80,0.385782056 +81,0.39245103 +82,0.375190589 +83,0.38281925 +84,0.374855705 +85,0.37289945 +86,0.373501507 +87,0.37820994 +88,0.380233486 +89,0.380667334 +90,0.379121569 +91,0.375309614 +92,0.374657036 +93,0.375086793 +94,0.376036438 +95,0.376015178 +96,0.377403571 +97,0.379801309 +98,0.376607964 +99,0.377697555 +100,0.375681407 +101,0.373934892 +102,0.377706113 +103,0.376752193 +104,0.395867603 +105,0.378681264 +106,0.377698889 +107,0.377541842 +108,0.37642283 +109,0.380908328 +110,0.379868566 +111,0.374663195 +112,0.37438595 +113,0.375067552 +114,0.376440531 +115,0.378176649 +116,0.379116844 +117,0.375201091 +118,0.374259594 +119,0.3744267 +120,0.376328596 +121,0.380620307 +122,0.378390659 +123,0.379844766 +124,0.372985752 +125,0.412397255 +126,0.375595657 +127,0.376272551 +128,0.386955877 +129,0.37447506 +130,0.375448968 +131,0.380362639 +132,0.376260893 +133,0.373944286 +134,0.376313094 +135,0.379435093 +136,0.375936218 +137,0.381314325 +138,0.377391853 +139,0.373517889 +140,0.385729386 +141,0.373902254 +142,0.379373653 +143,0.375899537 +144,0.388511683 +145,0.376187254 +146,0.379788871 +147,0.375300752 +148,0.390292013 +149,0.381669126 +150,0.385621004 +151,0.378633965 +152,0.389065753 +153,0.377405501 +154,0.376104303 +155,0.374026043 +156,0.374427418 +157,0.37564084 +158,0.375753784 +159,0.378136721 +160,0.39491974 +161,0.380198391 +162,0.379450206 +163,0.376706612 +164,0.37725745 +165,0.377553209 +166,0.375263013 +167,0.374094095 +168,0.379675909 +169,0.377853894 +170,0.376477499 +171,0.377793114 +172,0.381162961 +173,0.384042619 +174,0.375727437 +175,0.379877203 +176,0.378395042 +177,0.379994525 +178,0.377707453 +179,0.380739601 +180,0.375452996 +181,0.380526888 +182,0.373074645 +183,0.378006164 +184,0.379156838 +185,0.378706723 +186,0.385387704 +187,0.376950635 +188,0.374606601 +189,0.37554332 +190,0.380540166 +191,0.383980703 +192,0.382047836 +193,0.378620724 +194,0.373282127 +195,0.3733034 +196,0.377690736 +197,0.37933952 +198,0.380632044 +199,0.375993717 +200,0.376418352 +201,0.379006358 +202,0.379446884 +203,0.377732479 +204,0.385805797 +205,0.374459096 +206,0.38552005 +207,0.375223678 +208,0.373540476 +209,0.378472994 +210,0.377177484 +211,0.389642047 +212,0.378047886 +213,0.377214844 +214,0.377815395 +215,0.380875079 +216,0.378271097 +217,0.379704667 +218,0.37518504 +219,0.372301831 +220,0.376468601 +221,0.397737433 +222,0.376489154 +223,0.384317974 +224,0.380045353 +225,0.379258414 +226,0.668478963 +227,0.955560771 +228,0.66530459 +229,0.952927996 +230,0.959922183 +231,0.9604619 +232,1.245051426 +233,0.965650032 +234,1.249752868 +235,1.244404307 +236,1.253600817 +237,1.5425500300000001 +238,1.244638903 +239,1.5327819919999999 +240,1.53215351 +241,1.242658652 +242,1.544132456 +243,1.832230929 +244,1.5336775070000002 +245,1.8136965250000001 +246,1.532000429 +247,1.822073813 +248,1.823666376 +249,1.834192744 +250,2.135217325 +251,2.116891406 +252,2.113768951 +253,2.11533498 +254,2.1031375900000002 +255,2.111154624 +256,2.394712224 +257,2.404185921 +258,2.403343508 +259,2.1192398040000002 +260,2.705668597 +261,2.425563373 +262,2.696382249 +263,2.419572901 +264,1.529554483 +265,2.693073514 +266,2.699861572 +267,2.700676579 +268,2.991834641 +269,2.98815476 +270,2.97689877 +271,2.98189013 +272,3.000758657 +273,3.2595185669999998 +274,2.979182525 +275,3.277262975 +276,3.267665804 +277,3.567651725 +278,3.267198553 +279,3.288899236 +280,3.563272702 +281,3.565613747 +282,3.583468852 +283,3.563264436 +284,3.582336183 +285,3.861246016 +286,3.837587939 +287,3.84217554 +288,3.842180473 +289,3.846408332 +290,4.161861515 +291,3.865296708 +292,4.129283475 +293,4.139912236 +294,4.142976956 +295,4.128571385 +296,4.430305149 +297,4.154271212 +298,4.44798042 +299,4.421583211 +300,4.445049136 +301,4.442084446 +302,4.719657902 +303,4.716392198 +304,4.429964955 +305,5.016602943 +306,4.712858757 +307,4.732749232 +308,5.011187843 +309,4.728635103 +310,5.009524091 +311,4.993745165 +312,4.999831959 +313,5.056810035 +314,5.299232009 +315,5.302893218 +316,5.289425198 +317,5.301798037 +318,5.316324266 +319,5.31578592 +320,5.583774038 +321,5.589489447 +322,5.585255978 +323,5.608319763 +324,5.58157805 +325,5.592130031 +326,5.88500775 +327,5.890452729 +328,5.878823069 +329,5.885211418 +330,5.888762908 +331,6.172425968 +332,6.162267084 +333,6.136797062 +334,6.188560178 +335,6.156708843 +336,6.173356479 +337,6.445751831 +338,6.466813572 +339,6.220670727 +340,6.4681506970000004 +341,6.738561065 +342,6.453701698 +343,6.739565414 +344,6.467595425 +345,6.756441375 +346,6.747759547 +347,7.021372621 +348,6.760149012 +349,7.027064959 +350,6.745440319 +351,7.03684921 +352,7.070410437 +353,7.345334938 +354,7.032605545 +355,7.314518003 +356,7.332957481 +357,7.320349864 +358,7.355330965 +359,7.324711409 +360,7.600443829 +361,7.342456916 +362,7.637212436 +363,7.61203082 +364,7.602042525 +365,7.921439749 +366,7.593254316 +367,7.8878549190000005 +368,7.878748981 +369,7.907913 +370,7.896839348 +371,8.175472822 +372,8.169038771 +373,7.900216592 +374,8.174884591 +375,8.46796053 +376,8.182866008 +377,8.173207242 +378,8.496276912 +379,8.478493881 +380,8.500768496 +381,8.458273396 +382,8.510114914 +383,8.766329095 +384,8.475009324 +385,8.768574381 +386,8.776868361 +387,8.753333382 +388,9.07963734 +389,8.777710038 +390,9.078463601 +391,8.804048374 +392,9.355398808 +393,9.063636003 +394,9.084499365 +395,9.051824166 +396,9.324349684 +397,9.347768059 +398,9.347283251 +399,9.330815998 +400,9.638099697 +401,9.345913202 +402,9.670871432 +403,9.627375821 +404,9.651899708 +405,9.651694779 +406,9.920753963 +407,9.634500355 +408,9.92951956 +409,9.979044 +410,10.009683329 +411,9.910567963 +412,10.242448294 +413,9.913652433 +414,10.229765714 +415,10.221528942 +416,10.209768128 +417,10.509570666 +418,10.178965779 +419,10.523132068 +420,10.492208191 +421,10.487077489 +422,10.561445748 +423,10.50907287 +424,10.806014193 +425,10.791503489 +426,10.776016601 +427,10.767349911 +428,10.829705152 +429,10.794738583 +430,11.100617561 +431,11.107683213 +432,11.082964287 +433,11.059634842 +434,11.134076954 +435,11.10971053 +436,11.351062358 +437,11.089311034 +438,11.673286887 +439,11.387633025 +440,11.385106315 +441,11.654051175 +442,11.379078685 +443,11.660465699 +444,11.653163049 +445,11.660672689 +446,11.942987076 +447,11.689186124 +448,11.998450883 +449,11.954422609 +450,11.93773603 +451,11.986909825 +452,11.944524062 +453,12.243822924 +454,12.225589002 +455,12.230073628 +456,12.244093371 +457,12.234125372 +458,12.290051373 +459,12.546050988 +460,12.516166401 +461,12.536425889 +462,12.58340908 +463,11.110792047 +464,12.533078231 +465,12.751411329 +466,12.638948481 +467,12.961321263 +468,12.919280084 +469,12.677135155 +470,12.955273026 +471,13.241579438 +472,12.914374355 +473,13.219126837 +474,13.246411301 +475,13.16525104 +476,13.141419195 +477,13.472758119 +478,13.147993635 +479,13.453752791 +480,13.44201971 +481,13.441499539 +482,13.685713753 +483,13.680974148 +484,13.67949153 +485,13.690136217 +486,13.680766192 +487,13.652653519 +488,13.946795614 +489,13.931486796 +490,13.660625045 +491,13.683905758 +492,13.961606635999999 +493,13.940987935 +494,13.96423573 +495,14.255488341 +496,14.268293577 +497,14.291670482 +498,14.308522077 +499,14.225553012 +500,14.52736599 +501,14.241945388 +502,14.554484094 +503,14.525298047 +504,14.541811771999999 +505,14.542870557 +506,14.831602849 +507,14.817246728 +508,14.858879123 +509,14.806777274 +510,15.113272545 +511,14.826928065 +512,15.14680419 +513,15.170079005 +514,15.127925279 +515,15.108157698 +516,15.124106934 +517,15.394785854 +518,15.456463428 +519,15.427569691 +520,15.415259891 +521,15.427105775 +522,15.758264989 +523,15.413347719 +524,15.681030381 +525,15.390239277 +526,15.725503161 +527,16.022467555 +528,15.66540755 +529,15.983720688 +530,15.693195366 +531,15.685172415 +532,16.258215425 +533,15.966483273 +534,15.995873056 +535,15.952287141 +536,16.302072662 +537,16.297618715 +538,16.269145592 +539,16.240918609 +540,16.258037758 diff --git a/tests/scale/results/1.0.0/TestScale_HTTPRoutes/results-no-delay.csv b/tests/scale/results/1.0.0/TestScale_HTTPRoutes/results-no-delay.csv new file mode 100644 index 0000000000..d36ab1e0a2 --- /dev/null +++ b/tests/scale/results/1.0.0/TestScale_HTTPRoutes/results-no-delay.csv @@ -0,0 +1,476 @@ +# HTTPRoutes,Time to Ready (s),Error +1,0.383770841 +2,0.380656052 +3,0.380485657 +4,0.097146779 +5,0.377347929 +6,0.377125682 +7,0.37841844 +8,0.385015529 +9,0.089883912 +10,0.390451446 +11,0.378943073 +12,0.37725258 +13,0.378384658 +14,0.381131982 +15,0.377787581 +16,0.380144847 +17,0.379475256 +18,0.389799799 +19,0.378165317 +20,0.380534746 +21,0.380772033 +22,0.3836257 +23,0.378608803 +24,0.375804601 +25,0.379483524 +26,0.864854344 +27,0.6795812 +28,0.977591645 +29,0.963147228 +30,0.969510444 +31,0.965014447 +32,0.962099045 +33,1.102333015 +34,0.985620344 +35,1.09272505 +36,0.381917363 +37,0.664688404 +38,0.675943838 +39,0.386166754 +40,1.242637282 +41,1.298552424 +42,1.543812998 +43,1.810062531 +44,1.5296734 +45,1.816329149 +46,1.816646567 +47,1.8099877100000001 +48,1.813467829 +49,1.819065259 +50,2.097738851 +51,1.808799378 +52,2.098034641 +53,2.097764522 +54,2.101585146 +55,2.400262015 +56,2.389255576 +57,2.106989278 +58,2.428372868 +59,2.383734736 +60,2.107034143 +61,2.671407464 +62,2.388532318 +63,2.692242566 +64,2.678131784 +65,2.67492656 +66,2.977667557 +67,2.673629509 +68,2.962085064 +69,2.676131184 +70,2.967315415 +71,3.004304221 +72,2.969081665 +73,3.269183992 +74,2.967210283 +75,3.258638326 +76,3.249844151 +77,3.244806883 +78,3.561841321 +79,3.262699845 +80,3.539880137 +81,3.557550544 +82,3.542547365 +83,3.529880477 +84,3.840924524 +85,3.560927228 +86,3.827644323 +87,3.84080638 +88,3.833997502 +89,3.844903735 +90,4.117603379 +91,3.825068601 +92,4.124596355 +93,4.113714081 +94,4.152235268 +95,0.380709799 +96,3.548207755 +97,4.426409231 +98,4.424787019 +99,4.419769537 +100,4.422898661 +101,4.407654049 +102,4.706503788 +103,4.411553937 +104,4.7018559490000005 +105,4.715109887 +106,4.694933353 +107,4.698910085 +108,4.968518843 +109,4.691070862 +110,4.989728545 +111,4.981128082 +112,5.268325019 +113,4.97037176 +114,4.988800755 +115,5.271450693 +116,5.295387155 +117,5.272393873 +118,5.557244961 +119,5.280333265 +120,0.945514043 +121,5.262105221 +122,5.595568334 +123,5.574083237 +124,5.589206743 +125,5.552453588 +126,5.851369792 +127,1.233311532 +128,5.855350395 +129,5.964516196 +130,5.8471394530000005 +131,5.848075205 +132,5.860991674 +133,6.133646201 +134,6.145907893 +135,6.262283384 +136,6.155964576 +137,6.131266442 +138,6.43505469 +139,6.419861492 +140,6.415178561 +141,6.427191442 +142,6.434567097 +143,6.420440836 +144,6.73294683 +145,6.730110685 +146,6.713098847 +147,6.72025458 +148,6.716271473 +149,6.99301375 +150,6.731237644 +151,6.9977301 +152,7.021136399 +153,6.986413808 +154,7.003035744 +155,7.310633121 +156,7.317694499 +157,7.269159095 +158,7.283305354 +159,7.2859621820000005 +160,7.271657811 +161,7.582609967 +162,7.606299408 +163,7.568498447 +164,7.575934876 +165,7.5890662330000005 +166,7.568851525 +167,7.897321315 +168,7.924148761 +169,7.881869427 +170,7.889892944 +171,7.852905591 +172,8.161059877 +173,7.865857095 +174,8.174832698 +175,8.155205304 +176,8.15236726 +177,8.186432332 +178,8.442840984 +179,8.460232955 +180,8.451031473 +181,8.460532762 +182,8.449290509 +183,8.454978046 +184,8.767023309 +185,8.455161403 +186,8.780258723 +187,8.770188478 +188,9.013137528 +189,8.77018534 +190,6.147018329 +191,9.018709577 +192,9.019548294 +193,8.984440027 +194,9.02189999 +195,9.319032523 +196,9.330832487 +197,9.018864846 +198,9.587344531 +199,9.30696642 +200,9.315142028 +201,9.586112288 +202,9.298035512 +203,9.617492623 +204,9.585508158 +205,9.912826088 +206,9.610537584 +207,9.847416084 +208,9.602136188 +209,9.893778128 +210,10.170237989 +211,9.595497944 +212,10.176073337 +213,9.894899045 +214,10.162996596 +215,9.952862936 +216,10.48106678 +217,10.162380194 +218,10.162266336 +219,10.488419733 +220,10.496234762 +221,10.470966589 +222,10.50118922 +223,10.499243832 +224,10.479238461 +225,10.749923528 +226,10.776629856 +227,10.743692394 +228,10.755004548 +229,10.771770892 +230,11.021130656 +231,10.747373535 +232,11.0750719 +233,11.041423527 +234,11.360049616 +235,11.058005117 +236,11.340463323 +237,11.050966395 +238,11.344113357 +239,11.337348929000001 +240,11.603600576 +241,11.346556197 +242,7.034123487 +243,11.636125892 +244,11.633910611 +245,11.603578323 +246,11.636285816000001 +247,11.897112891 +248,7.307798539 +249,11.918549641 +250,11.906533268 +251,11.8995394 +252,11.900474696 +253,12.201526993 +254,11.944296505 +255,12.190401503 +256,12.226729616 +257,12.230866968 +258,12.232820174 +259,12.47781475 +260,12.240100055 +261,12.520273912 +262,12.214461539 +263,12.522577857 +264,12.795913592 +265,12.494518731 +266,12.79278819 +267,12.759949243 +268,12.887502352 +269,12.770259288 +270,32.395021702 +271,0.374506815 +272,11.901244048 +273,13.075312528 +274,13.059521502 +275,13.077884811 +276,13.340200974 +277,13.146416903 +278,13.345507272 +279,13.37279045 +280,13.371691526 +281,13.669128218 +282,13.353862711 +283,13.678822998 +284,13.672502392 +285,13.651719506 +286,13.658850703 +287,13.661302626 +288,13.978328505 +289,13.940908396 +290,13.936206684 +291,13.947572959 +292,13.945947007000001 +293,14.224175465 +294,13.956113138 +295,14.206128572 +296,14.230369493 +297,14.206886939 +298,14.229910918 +299,14.239633901 +300,14.225149174 +301,14.506786921 +302,14.208907617 +303,14.234825371 +304,14.495108177 +305,14.524795147 +306,14.760110319 +307,14.822820149 +308,14.837508014 +309,14.80072829 +310,14.786506376 +311,15.092856733 +312,15.083235162 +313,14.806087759 +314,15.034910757 +315,15.056887306 +316,15.357751318 +317,15.065564849 +318,15.370088909 +319,15.353822604 +320,15.357593312 +321,15.40037503 +322,15.66120014 +323,15.362804896 +324,15.705511462 +325,15.645579946 +326,15.618289891 +327,13.942397622 +328,15.922382777 +329,15.947337829 +330,15.945278678 +331,15.943202858 +332,16.0073185 +333,15.927050813 +334,16.226592783 +335,16.187233433 +336,16.246480633 +337,16.237334103 +338,16.227720886 +339,15.946216830000001 +340,16.534781017 +341,16.507881368 +342,16.254285373 +343,16.576747317 +344,16.54642211 +345,16.545732963 +346,16.843086099 +347,16.232294914 +348,16.8136862 +349,16.820897867 +350,16.821277668 +351,16.816728191 +352,16.839415343 +353,17.153682019 +354,17.113154336 +355,17.144029836 +356,17.170722512 +357,17.092822527 +358,17.411367673 +359,17.46173771 +360,17.389891699 +361,17.43299391 +362,17.437457132 +363,17.414578314 +364,17.704094626 +365,17.721105545 +366,17.74807521 +367,17.726058221 +368,17.708324845 +369,17.737824773 +370,17.997769394 +371,17.754324679 +372,18.010773053 +373,18.012075537 +374,18.307598961 +375,18.0699199 +376,18.294276854 +377,18.316656373 +378,18.066631139 +379,18.405606592 +380,18.40470596 +381,18.629784072 +382,18.326288914 +383,18.641559759 +384,18.633411613 +385,18.328779618 +386,18.597437479 +387,18.704598885 +388,18.883604591 +389,18.89240392 +390,18.918688037 +391,18.917226774 +392,18.892069689 +393,18.95492239 +394,19.218288714 +395,19.169093596 +396,14.873706174 +397,19.180519994 +398,19.605025347 +399,19.178308424 +400,19.494217796 +401,19.446356302 +402,19.495733812 +403,19.803003043 +404,19.464744612 +405,19.800134155 +406,19.496257638 +407,20.016203759 +408,19.794462513 +409,15.441952501 +410,20.115836025 +411,19.796751609 +412,20.029376436 +413,20.171004803 +414,20.08184473 +415,20.325934549 +416,20.333374709 +417,20.093339258 +418,20.356575706 +419,20.411727088 +420,16.011443907 +421,20.69810284 +422,20.63230193 +423,20.598879281 +424,20.387827233 +425,20.761187109 +426,20.66080738 +427,20.629535227 +428,20.909389827 +429,20.923543042 +430,20.637488247 +431,21.245912296 +432,20.992789011 +433,20.979198568 +434,21.225965227 +435,21.205642512 +436,21.212557428 +437,21.242860302 +438,21.523383588 +439,21.241824455 +440,21.499328997 +441,21.549901038 +442,21.520652532 +443,21.780803876 +444,21.562871924 +445,21.829179892 +446,21.800966716 +447,21.809903338 +448,21.7910108 +449,21.83978298 +450,22.082463304 +451,22.126015909 +452,21.831557769 +453,22.248887912 +454,22.126643502 +455,48.365839369 +456,0.376811942 +457,21.215613992 +458,22.445884133 +459,22.380127349 +460,22.265444665 +461,22.725412845 +462,22.36771563 +463,22.659603907 +464,22.65619497 +465,22.723849525 +466,22.700845057 +467,22.719689394 +468,22.962046973 +469,22.789032688 +470,22.975549597 +471,22.986912774 +472,22.949533459 +473,23.289465192 +474,22.997424888 +475,23.240723 diff --git a/tests/scale/results/1.0.0/TestScale_HTTPRoutes/results.csv b/tests/scale/results/1.0.0/TestScale_HTTPRoutes/results.csv new file mode 100644 index 0000000000..36b989c8c4 --- /dev/null +++ b/tests/scale/results/1.0.0/TestScale_HTTPRoutes/results.csv @@ -0,0 +1,1001 @@ +# HTTPRoutes,Time to Ready (s),Error +1,0.385042 +2,0.382147622 +3,0.379821284 +4,0.379737071 +5,0.379980332 +6,0.380883635 +7,0.38194278 +8,0.384607201 +9,0.399597381 +10,0.383712896 +11,0.385982892 +12,0.381628865 +13,0.383431779 +14,0.380270067 +15,0.378141185 +16,0.387142744 +17,0.380637439 +18,0.383168174 +19,0.384756966 +20,0.384278116 +21,0.384212425 +22,0.38308232 +23,0.382446783 +24,0.388838492 +25,0.387291714 +26,0.380083607 +27,0.446689351 +28,0.385081587 +29,0.385361996 +30,0.387850995 +31,0.377779806 +32,0.39151114 +33,0.392356289 +34,0.427523592 +35,0.381354729 +36,0.384113814 +37,0.3801672 +38,0.379487191 +39,0.383137692 +40,0.381893477 +41,0.380364125 +42,0.381806096 +43,0.388057082 +44,0.384284274 +45,0.385182757 +46,0.382352598 +47,0.389805938 +48,0.400674567 +49,0.392630929 +50,0.384568685 +51,0.38524618 +52,0.379389459 +53,0.386951052 +54,0.384325197 +55,0.382886417 +56,0.389441321 +57,0.383999268 +58,0.384693315 +59,0.380554606 +60,0.38608117 +61,0.38488563 +62,0.382124047 +63,0.389271177 +64,0.387389756 +65,0.383198306 +66,0.682481106 +67,0.67444258 +68,0.970568223 +69,1.014999035 +70,0.973646955 +71,0.969847787 +72,0.991308799 +73,1.254453876 +74,0.985181179 +75,1.262578394 +76,1.260318123 +77,1.260580925 +78,1.263775268 +79,1.289057704 +80,1.271338337 +81,1.553105417 +82,1.559913882 +83,1.678703766 +84,1.591168978 +85,1.86223424 +86,1.844302761 +87,1.547678299 +88,1.847528396 +89,1.894979287 +90,2.144782547 +91,1.8621425139999999 +92,2.148959875 +93,2.129363923 +94,1.837864742 +95,1.846953274 +96,2.267352278 +97,2.155376881 +98,2.420211349 +99,2.414074736 +100,2.424816646 +101,2.721810069 +102,2.473841073 +103,2.748770853 +104,2.4316491239999998 +105,2.7178590160000002 +106,2.7861162520000002 +107,2.738862137 +108,3.041759396 +109,3.018813415 +110,2.708517359 +111,3.013564487 +112,3.2934272399999998 +113,3.009758894 +114,3.02650995 +115,3.2986881009999998 +116,3.141020256 +117,3.325437798 +118,3.583768892 +119,3.2999014 +120,3.595940144 +121,3.034143626 +122,3.605313412 +123,3.54850254 +124,3.617334108 +125,3.637293178 +126,3.9288259610000003 +127,3.881287462 +128,4.61811947 +129,3.288423539 +130,3.620755448 +131,3.934249756 +132,4.054183417 +133,4.213836762 +134,4.175238814 +135,4.175630634 +136,4.163063512 +137,4.202729586 +138,4.491626064 +139,4.263553526 +140,4.506628162 +141,4.489577994 +142,4.510760647 +143,4.452220136 +144,4.782550678 +145,4.77699404 +146,4.478480558 +147,5.038061317 +148,4.77519762 +149,4.769659284 +150,5.067124139 +151,5.057393805 +152,5.005089052 +153,5.119731288 +154,5.093113355 +155,5.183158537 +156,5.359393964 +157,5.057175814 +158,5.366343244 +159,5.405545673 +160,5.653089137 +161,5.355915223 +162,5.65874337 +163,5.637884689 +164,5.64887917 +165,5.615412919 +166,5.6381729069999995 +167,5.979529192 +168,5.9348313820000005 +169,5.648260129 +170,5.950106851 +171,6.066447468 +172,6.096931754 +173,5.764695952 +174,6.380062327 +175,5.66656968 +176,6.272783281 +177,6.221946278 +178,6.240302251 +179,6.414905571 +180,6.563974786 +181,6.522895801 +182,6.493242072 +183,6.443757982 +184,6.974011986 +185,6.52774633 +186,6.81419512 +187,6.790342793 +188,7.758369739 +189,5.937684343 +190,7.076457047 +191,7.123723743 +192,7.557060908 +193,6.5163503590000005 +194,7.108920551 +195,7.115321489 +196,7.40913478 +197,7.239179751 +198,7.395974213 +199,7.39538685 +200,7.517393048 +201,7.501897832 +202,7.814676309 +203,7.123856981 +204,7.700579924 +205,7.840622743 +206,7.722739872 +207,7.55302047 +208,7.687103622 +209,7.480004886 +210,8.156368151 +211,7.804533559 +212,8.288270125 +213,8.005565296 +214,8.004393888 +215,8.462546078999999 +216,8.024345326 +217,8.313862247 +218,8.344053023 +219,8.579421373 +220,8.289709383 +221,8.611967649 +222,8.585668337 +223,8.429041015 +224,8.580136978 +225,8.951947128 +226,8.578459164 +227,8.915455951 +228,8.857611337 +229,8.705042557 +230,9.170933217 +231,8.623892856 +232,8.921918869 +233,9.190434372 +234,9.159182995 +235,9.15488773 +236,9.471499052 +237,9.21651202 +238,9.233143149 +239,9.447514648 +240,9.586912402 +241,9.449483537 +242,9.490955118 +243,9.446495527 +244,9.726599567 +245,9.881813864 +246,9.584750159 +247,9.751090104 +248,10.009108372 +249,10.014152559 +250,9.758181883 +251,9.893224133 +252,10.106693395 +253,10.313508271 +254,10.072765931 +255,10.138212372 +256,9.296253551 +257,10.311199826 +258,10.128726716 +259,10.415380248 +260,10.443652387 +261,10.6420528 +262,10.614467204 +263,10.386039166 +264,10.726860816 +265,10.376580013 +266,10.740635426 +267,10.942521519 +268,10.901106595 +269,10.877931949 +270,11.012175771 +271,10.961585794 +272,10.931964082 +273,11.232116387 +274,11.210639484 +275,11.275499703 +276,11.210823213 +277,11.219041249 +278,11.533387578 +279,11.215886923 +280,11.546196755 +281,11.565836952 +282,11.571423236 +283,11.512723487 +284,11.528767818 +285,11.849987387 +286,11.864171536 +287,11.635332997999999 +288,11.889636363 +289,11.76892983 +290,12.207261147 +291,11.792182402 +292,12.335200656 +293,12.097684492 +294,12.103063918 +295,12.085841953 +296,12.469395154 +297,12.131030775 +298,12.405417177 +299,12.391954757 +300,12.379394268 +301,12.565762398 +302,12.408597337 +303,12.684903978 +304,12.708412852 +305,12.371383459 +306,12.693234396 +307,12.719631498 +308,13.040710355 +309,12.927983789 +310,12.977783187 +311,12.807163133 +312,13.379932757 +313,12.709859358 +314,13.291819766 +315,13.060006443 +316,13.303808197 +317,13.284253253 +318,13.29751204 +319,13.271607192 +320,13.417060694 +321,13.70029229 +322,13.377623602 +323,13.622685939 +324,13.528297491 +325,13.754196761 +326,13.875762092 +327,13.825902329 +328,13.931361 +329,13.758134165 +330,14.109854762 +331,14.010441761 +332,13.643428951 +333,14.144946622 +334,14.185458655 +335,14.392189885 +336,14.230359781 +337,14.32012791 +338,14.19807588 +339,14.256977214 +340,14.548063136 +341,14.465300027 +342,14.723959772 +343,14.339139374 +344,14.87078413 +345,14.443061959 +346,14.875051509 +347,14.87759207 +348,14.77773053 +349,14.769552148 +350,15.10367367 +351,15.117947396 +352,14.855381983000001 +353,15.017789933 +354,15.145974441 +355,15.32010491 +356,15.068143838 +357,15.275396618 +358,15.23281466 +359,15.189269566 +360,15.577918159 +361,15.372148684 +362,15.671111492 +363,15.303942189 +364,15.668559627 +365,15.797643447 +366,15.663171149 +367,15.933899011 +368,15.665407129 +369,15.852120998 +370,15.889898323 +371,15.963718218 +372,15.977373067 +373,16.163176669 +374,15.927281837 +375,14.736798144 +376,16.156760056 +377,16.468384748 +378,16.348876685 +379,16.398853451 +380,15.976793608 +381,16.514288568 +382,16.639794391 +383,16.453938836 +384,16.316995395 +385,16.743604474 +386,16.506053018 +387,16.783119188 +388,16.607363112 +389,17.008478779 +390,16.823745788 +391,17.060766644 +392,17.077873714 +393,17.046714215 +394,17.110553004 +395,17.047490045 +396,17.063928259 +397,17.373884652 +398,17.157657931 +399,17.489016202 +400,17.351843145 +401,17.47217862 +402,17.375319708 +403,17.459777531 +404,17.715900978 +405,17.646793539 +406,17.757515262 +407,17.606350828 +408,17.97744462 +409,17.740024189 +410,18.029429741 +411,18.008791589 +412,17.638611049 +413,18.059927561 +414,18.278911876 +415,17.95843105 +416,18.312597149 +417,18.225649801 +418,18.394198185 +419,18.520789779 +420,18.228401405 +421,18.595082022 +422,18.291398404 +423,18.588732109 +424,18.74791212 +425,18.599117429 +426,18.664008051 +427,18.564616431 +428,18.918485469 +429,18.92435889 +430,18.850794908 +431,19.141210012 +432,18.892189834 +433,19.117511665 +434,18.917876091 +435,19.161540682 +436,19.123684349 +437,19.162010725 +438,19.427617013 +439,19.521264909 +440,19.21184219 +441,19.554330723 +442,19.657042827 +443,19.452228143 +444,19.553777667 +445,19.680911127 +446,19.851861609 +447,19.672833393 +448,19.724328509 +449,19.616568579 +450,19.958856777 +451,19.765061333 +452,20.005162847 +453,19.757921307 +454,20.237635434 +455,20.31129502 +456,20.149187526 +457,20.413330424 +458,20.157038319 +459,20.360130697 +460,20.434425328 +461,20.420543853 +462,16.054128791 +463,20.270353233 +464,20.720230307 +465,20.755472527 +466,20.442446491 +467,21.142439627 +468,20.55118376 +469,20.729758839 +470,20.770440613 +471,20.867173747 +472,21.012146976 +473,21.143066767 +474,19.871434676 +475,20.98680626 +476,21.070800301 +477,21.389798069 +478,21.1654169 +479,21.494387617 +480,21.036759754 +481,21.112269574 +482,21.531892915 +483,21.337126066 +484,21.250978546 +485,20.509124166 +486,21.745883133 +487,21.103062782 +488,21.937275097 +489,21.920068816 +490,22.346367017 +491,21.292977791 +492,21.92061976 +493,22.092450352 +494,22.584700292 +495,21.458868392 +496,22.052234616 +497,22.454638002 +498,22.127435985 +499,22.522374324 +500,21.883213828 +501,21.982803364 +502,22.33556325 +503,22.415271362 +504,22.661676083 +505,22.806925196 +506,22.707423954 +507,22.340561781 +508,22.672490776 +509,22.631177416 +510,22.895915001 +511,22.713789195 +512,22.750381261 +513,22.957853464 +514,18.375919838 +515,22.860031114 +516,22.99797962 +517,23.299321407 +518,23.372230714 +519,23.269856979 +520,23.502336018 +521,23.20181453 +522,23.49991449 +523,23.64418039 +524,23.704222722 +525,23.769078018 +526,23.767130476 +527,23.738705835 +528,23.965922729 +529,23.869472989 +530,24.05009783 +531,23.824101502 +532,24.05573653 +533,23.989579074 +534,22.357893156 +535,23.18549222 +536,24.480278265 +537,24.026677194 +538,24.22988682 +539,24.386426265 +540,24.473809548 +541,24.674271787 +542,24.543278449 +543,24.700960874 +544,24.969447494 +545,24.333870883 +546,24.668663243 +547,24.666292367 +548,24.939866772 +549,24.946053397 +550,24.753292338 +551,24.820927632 +552,25.192553635 +553,25.075685951 +554,25.206709839 +555,25.248519888 +556,25.188306372 +557,25.192406035 +558,25.330387328 +559,25.471257659 +560,25.378147555 +561,25.726585109 +562,25.424377618 +563,25.790521544 +564,25.619911227 +565,25.536843057 +566,25.903676717 +567,25.87893036 +568,25.648297482 +569,25.87120104 +570,25.933177205 +571,26.192813251 +572,25.90081028 +573,26.214134206 +574,26.141011842 +575,26.246625585 +576,26.09809675 +577,26.340708263 +578,26.512834545 +579,26.240917631 +580,26.438277176 +581,26.764700643 +582,26.54277495 +583,26.498550859 +584,26.558657017 +585,26.815180342 +586,27.024340032 +587,26.431015433 +588,26.937928339 +589,26.791352087 +590,27.714522501 +591,26.201116306 +592,26.436610151 +593,27.102208878 +594,26.19530116 +595,27.118518584 +596,27.388642723 +597,27.480835235 +598,27.253742123 +599,27.324889889 +600,27.635807636 +601,27.254484846 +602,27.752763426 +603,27.48113274 +604,27.416303825 +605,27.408340749 +606,27.697793493 +607,27.796882583 +608,27.964799934 +609,27.823316641 +610,27.898344034 +611,28.209644515 +612,27.908292479 +613,28.218555957 +614,27.880263563 +615,27.867922688 +616,27.810873363 +617,28.684313227 +618,28.004002305 +619,28.350767599 +620,28.548666574 +621,28.256627827 +622,28.609898549 +623,28.448800978 +624,28.412905517 +625,28.763264369 +626,28.800030098 +627,28.918824589 +628,28.682836165 +629,28.970455229 +630,28.994155213 +631,28.920960888 +632,29.113591282 +633,29.261662013 +634,28.950615635 +635,29.243960904 +636,29.294680653 +637,29.33256142 +638,29.2900358 +639,29.697713659 +640,29.126868285 +641,29.734468814 +642,29.372990843 +643,29.594829808 +644,29.885468063 +645,29.587723895 +646,29.766930204 +647,29.828231551000002 +648,29.64953129 +649,29.966260729 +650,30.177644134 +651,29.972401615 +652,30.091857544 +653,30.093859102 +654,30.093775194 +655,30.206717448 +656,28.681422529 +657,30.219944058 +658,30.281830661 +659,30.666410314 +660,30.319944612 +661,30.532342413 +662,30.65329485 +663,30.646258714 +664,30.372179529 +665,30.937523534 +666,30.607411245 +667,30.845437066 +668,30.874295987 +669,30.786269839 +670,31.10182997 +671,31.055038211 +672,30.950370529 +673,30.997416668 +674,31.202138494 +675,31.252602837 +676,32.094032061 +677,30.515181124 +678,31.443896745 +679,31.451071023 +680,31.244367746 +681,31.522649979 +682,31.682840281 +683,31.368488661 +684,31.763013361 +685,31.608248307 +686,32.006498258 +687,31.703430811 +688,31.690898548 +689,32.036667949 +690,32.075787202 +691,31.783247357 +692,32.066928141 +693,32.236504458 +694,32.15537187 +695,32.208397621 +696,32.066546288 +697,32.536006169 +698,31.82230569 +699,32.434571521 +700,32.485642805 +701,32.698211289 +702,32.02567053 +703,32.613965368 +704,32.485869517 +705,32.828500744 +706,32.577307339 +707,32.333139515 +708,32.888567315 +709,32.614495432 +710,33.059036775 +711,33.216463344 +712,32.996813024 +713,32.954969642 +714,33.368942064 +715,33.146787957 +716,33.312489754 +717,33.029403738 +718,33.375930208 +719,33.487482005 +720,33.536680637 +721,33.307924586 +722,33.143614974 +723,33.52987231 +724,33.220851826 +725,33.656253523 +726,33.818601528 +727,33.589326889 +728,33.829996074 +729,33.895954039 +730,33.943975879 +731,34.038779124 +732,33.877155671 +733,34.253951991 +734,33.841181341 +735,34.021483843 +736,34.275737255 +737,34.160015773 +738,34.035368635 +739,34.280323753 +740,34.43637143 +741,34.407867884 +742,34.551317412 +743,34.666116549 +744,34.618400609 +745,34.675952834 +746,34.866508502 +747,34.632815747 +748,34.775578238 +749,35.50506069 +750,34.443795652 +751,34.89670591 +752,34.924520112 +753,35.180212727 +754,35.215338369 +755,34.987348077 +756,35.608344882 +757,33.874212088 +758,34.044744082 +759,35.574077401 +760,35.150056506 +761,35.72634878 +762,35.45079798 +763,35.865771632 +764,35.427604226 +765,35.667238458 +766,35.898466903 +767,35.804097476 +768,35.834170134 +769,35.896964583 +770,36.082061928 +771,35.836167064 +772,36.047323096 +773,36.202295966 +774,35.99166311 +775,36.412145704 +776,36.283841013 +777,35.983344695 +778,36.108361798 +779,36.508695673 +780,35.926579826 +781,36.500876559 +782,36.622011676 +783,36.550970389 +784,36.683771644 +785,36.872314682 +786,36.540312097 +787,36.783760094 +788,36.91872591 +789,37.096798082 +790,36.685051202 +791,37.20530758 +792,37.000189263 +793,37.099625874 +794,37.012226699 +795,37.448803223 +796,37.300768721 +797,37.223420776 +798,37.29162253 +799,37.435941317 +800,37.208434043 +801,37.2499197 +802,37.66582782 +803,37.654677445 +804,37.794832342 +805,37.454374325 +806,37.862383292 +807,37.93023652 +808,37.754437049 +809,38.038564907 +810,37.852294342 +811,37.99269785 +812,38.071864254 +813,38.17196806 +814,37.898905795 +815,38.407297019 +816,38.326513593 +817,38.24558466 +818,38.279104566 +819,38.409669597 +820,38.376204833 +821,38.128068756 +822,38.619899762 +823,37.983904743 +824,38.628779013 +825,38.911680247 +826,38.679771917 +827,38.714261932 +828,39.115790419 +829,38.641115 +830,38.937191293 +831,39.307233431 +832,38.824771817 +833,39.291982239 +834,38.68330027 +835,39.322497359 +836,39.090506468 +837,39.515033154 +838,39.253855315 +839,39.459788169 +840,38.920449303 +841,39.775861892 +842,39.686079355 +843,39.238609607 +844,39.723426937 +845,39.504629846 +846,39.289219859 +847,39.762318544 +848,38.778510905 +849,39.725515759 +850,39.987143133000004 +851,40.091022857 +852,40.349603916 +853,39.779709219 +854,40.195892127 +855,39.752316977 +856,40.311265409 +857,40.300417486 +858,40.315248098 +859,40.434328954 +860,40.19435715 +861,40.363335057 +862,41.669585049 +863,39.781274404 +864,40.354705579 +865,40.84619771 +866,40.728768619 +867,40.813227822 +868,40.964212391 +869,40.800967426 +870,41.000432604 +871,41.086572241 +872,40.821199926 +873,41.088048355 +874,41.370710885 +875,41.05639203 +876,41.416772172 +877,40.804547158 +878,41.370783479 +879,41.627574969 +880,40.853632539 +881,41.35864723 +882,41.645450331 +883,41.540242523 +884,41.794797105 +885,41.616038219 +886,41.920267929 +887,41.544109063 +888,42.411710046 +889,41.264691296 +890,41.743791518 +891,42.015610323 +892,41.848968881 +893,41.805073703 +894,42.027773351 +895,42.210190339 +896,42.076725874 +897,42.150341443 +898,42.315650886 +899,42.307507141 +900,42.217072285 +901,42.617782652 +902,42.241970775 +903,42.637235072 +904,42.493406701 +905,42.575583091 +906,42.736866699 +907,43.002907711 +908,42.634991819 +909,43.082808944 +910,42.908847193 +911,42.823499198 +912,43.132879978 +913,43.108164235 +914,43.058552019 +915,43.393716098 +916,42.691245144 +917,58.73102609 +918,28.066846227 +919,43.057087543 +920,43.147919358 +921,43.46887098 +922,43.663000603 +923,43.123878417 +924,43.766435259 +925,43.634602739 +926,43.516437928 +927,43.824348339 +928,43.447019554 +929,42.414165678 +930,43.808199767 +931,44.086641079 +932,44.070733436 +933,44.211348569 +934,43.920230992 +935,44.215260913 +936,44.179635296 +937,44.389108829 +938,44.214206702 +939,44.529518909 +940,44.387881114 +941,44.432899966 +942,44.307549914 +943,44.337567962 +944,44.641681413 +945,43.979862547 +946,44.914763536 +947,44.677172145 +948,44.909389865 +949,44.910000713 +950,44.719413381 +951,45.01212029 +952,44.898786641 +953,45.142755969 +954,44.859578864 +955,45.169217004 +956,44.870462995 +957,45.098422865 +958,45.468057985 +959,45.339417667 +960,45.277504668 +961,45.616167171 +962,45.406287131 +963,45.438691896 +964,45.801433717 +965,45.221075323 +966,45.770892791 +967,45.507180707 +968,45.670349562 +969,45.655492147 +970,45.881600854 +971,45.934390478 +972,45.924218752 +973,45.530736568 +974,46.092620072 +975,46.180485415 +976,46.246379199 +977,46.380686715 +978,46.202818149 +979,46.343931401 +980,46.355746266 +981,46.601280513 +982,46.462298547 +983,46.453345524 +984,46.581686759 +985,46.847757213 +986,46.617841844 +987,46.871250103 +988,46.653667552 +989,46.859011274 +990,46.951427338 +991,46.879632876 +992,47.099152468 +993,46.691930454 +994,47.226925086 +995,47.06015949 +996,47.288046477 +997,47.105833116 +998,47.400701356 +999,47.319450545 +1000,47.584592417 diff --git a/tests/scale/results/1.0.0/TestScale_HTTPSListeners/CPU.png b/tests/scale/results/1.0.0/TestScale_HTTPSListeners/CPU.png new file mode 100644 index 0000000000..55de292fa7 Binary files /dev/null and b/tests/scale/results/1.0.0/TestScale_HTTPSListeners/CPU.png differ diff --git a/tests/scale/results/1.0.0/TestScale_HTTPSListeners/Memory.png b/tests/scale/results/1.0.0/TestScale_HTTPSListeners/Memory.png new file mode 100644 index 0000000000..3ddf1f1601 Binary files /dev/null and b/tests/scale/results/1.0.0/TestScale_HTTPSListeners/Memory.png differ diff --git a/tests/scale/results/1.0.0/TestScale_HTTPSListeners/TTR.png b/tests/scale/results/1.0.0/TestScale_HTTPSListeners/TTR.png new file mode 100644 index 0000000000..14e9d920c6 Binary files /dev/null and b/tests/scale/results/1.0.0/TestScale_HTTPSListeners/TTR.png differ diff --git a/tests/scale/results/1.0.0/TestScale_HTTPSListeners/results.csv b/tests/scale/results/1.0.0/TestScale_HTTPSListeners/results.csv new file mode 100644 index 0000000000..1c30ee62f9 --- /dev/null +++ b/tests/scale/results/1.0.0/TestScale_HTTPSListeners/results.csv @@ -0,0 +1,65 @@ +# HTTPS Listeners,Time to Ready (s),Error +1,0.169995759 +2,0.316317206 +3,0.335810125 +4,0.35087607 +5,0.374128886 +6,0.786490003 +7,0.395602107 +8,0.335063415 +9,0.351511284 +10,0.306195938 +11,1.079579494 +12,1.085989951 +13,0.710736746 +14,0.706770998 +15,0.791338601 +16,0.910576504 +17,0.72462806 +18,0.891562537 +19,0.699489578 +20,0.989158916 +21,0.992727987 +22,1.080868064 +23,0.793088182 +24,0.691032678 +25,0.926785913 +26,1.308280347 +27,0.878097664 +28,1.29426 +29,1.306243568 +30,2.749318703 +31,1.091964274 +32,1.412320275 +33,1.581276201 +34,1.9931428759999998 +35,1.205374759 +36,1.723513203 +37,1.720674308 +38,1.55956355 +39,2.236604075 +40,1.67136963 +41,2.272512461 +42,1.944905419 +43,2.689681995 +44,1.462145122 +45,1.891378043 +46,2.483272037 +47,1.632890784 +48,1.225230643 +49,1.417660855 +50,0.330152469 +51,1.357993629 +52,1.9156297869999999 +53,1.4871654460000001 +54,0.612916622 +55,2.981474032 +56,10.003182785 +57,2.873364642 +58,2.174200222 +59,2.501841692 +60,2.342435315 +61,1.335872465 +62,2.8931696909999998 +63,3.182950872 +64,6.230650774 diff --git a/tests/scale/results/1.0.0/TestScale_Listeners/CPU.png b/tests/scale/results/1.0.0/TestScale_Listeners/CPU.png new file mode 100644 index 0000000000..b6028ec1db Binary files /dev/null and b/tests/scale/results/1.0.0/TestScale_Listeners/CPU.png differ diff --git a/tests/scale/results/1.0.0/TestScale_Listeners/Memory.png b/tests/scale/results/1.0.0/TestScale_Listeners/Memory.png new file mode 100644 index 0000000000..3e7800da30 Binary files /dev/null and b/tests/scale/results/1.0.0/TestScale_Listeners/Memory.png differ diff --git a/tests/scale/results/1.0.0/TestScale_Listeners/TTR.png b/tests/scale/results/1.0.0/TestScale_Listeners/TTR.png new file mode 100644 index 0000000000..67aeb8c23e Binary files /dev/null and b/tests/scale/results/1.0.0/TestScale_Listeners/TTR.png differ diff --git a/tests/scale/results/1.0.0/TestScale_Listeners/results.csv b/tests/scale/results/1.0.0/TestScale_Listeners/results.csv new file mode 100644 index 0000000000..89ef9e114b --- /dev/null +++ b/tests/scale/results/1.0.0/TestScale_Listeners/results.csv @@ -0,0 +1,65 @@ +# Listeners,Time to Ready (s),Error +1,0.389434458 +2,0.385901884 +3,0.386300635 +4,0.39178591 +5,0.382599518 +6,0.381957681 +7,0.381679551 +8,0.673451873 +9,0.38174132 +10,0.394320317 +11,0.386441444 +12,1.257764007 +13,0.678272427 +14,0.674835931 +15,0.961024281 +16,0.674262046 +17,0.678830512 +18,0.385906626 +19,0.679167754 +20,0.964288262 +21,1.254985027 +22,0.674975659 +23,0.678150708 +24,0.674943489 +25,1.2649119770000001 +26,1.2595624 +27,1.8420411049999998 +28,0.964987293 +29,0.963224162 +30,1.5548014540000001 +31,0.392973186 +32,1.591366332 +33,1.5760351959999999 +34,0.67539319 +35,1.836873 +36,0.390074267 +37,0.669736812 +38,2.159392649 +39,0.394423826 +40,2.265563849 +41,0.386839254 +42,2.145638601 +43,2.146480487 +44,0.395269441 +45,2.426101109 +46,0.966531362 +47,0.67866656 +48,1.577306721 +49,0.384150846 +50,0.385452621 +51,2.152355796 +52,0.380525188 +53,0.395430389 +54,0.387830157 +55,2.742531289 +56,0.966502151 +57,0.672086839 +58,0.674754009 +59,2.74438795 +60,0.378782959 +61,2.436529727 +62,3.019968911 +63,0.380224085 +64,0.383710448 diff --git a/tests/scale/results/1.0.0/TestScale_UpstreamServers/CPU.png b/tests/scale/results/1.0.0/TestScale_UpstreamServers/CPU.png new file mode 100644 index 0000000000..92eebb022f Binary files /dev/null and b/tests/scale/results/1.0.0/TestScale_UpstreamServers/CPU.png differ diff --git a/tests/scale/results/1.0.0/TestScale_UpstreamServers/Memory.png b/tests/scale/results/1.0.0/TestScale_UpstreamServers/Memory.png new file mode 100644 index 0000000000..6d035f3ed4 Binary files /dev/null and b/tests/scale/results/1.0.0/TestScale_UpstreamServers/Memory.png differ diff --git a/tests/scale/scale.md b/tests/scale/scale.md new file mode 100644 index 0000000000..7ee324f862 --- /dev/null +++ b/tests/scale/scale.md @@ -0,0 +1,441 @@ +# Scale Tests + +This document describes how we scale test NGF. + + +- [Scale Tests](#scale-tests) + - [Goals](#goals) + - [Test Environment](#test-environment) + - [Steps](#steps) + - [Setup](#setup) + - [Run the tests](#run-the-tests) + - [Scale Listeners to Max of 64](#scale-listeners-to-max-of-64) + - [Scale HTTPS Listeners to Max of 64](#scale-https-listeners-to-max-of-64) + - [Scale HTTPRoutes](#scale-httproutes) + - [Scale Upstream Servers](#scale-upstream-servers) + - [Scale HTTP Matches](#scale-http-matches) + - [Analyze](#analyze) + - [Results](#results) + + +## Goals + +- Measure how NGF performs when the number of Gateway API and referenced core Kubernetes resources are scaled. +- Test the following number of resources: + - Max number of HTTP and HTTPS Listeners (64) + - Max number of Upstream Servers (648) + - Max number of HTTPMatches + - 1000 HTTPRoutes + +## Test Environment + +For most of the tests, the following cluster will be sufficient: + +- A Kubernetes cluster with 4 nodes on GKE + - Node: n2d-standard-8 (8 vCPU, 32GB memory) + - Enabled GKE logging + +The Upstream Server scale test requires a bigger cluster to accommodate the large number of Pods. Those cluster details +are listed in the [Scale Upstream Servers](#scale-upstream-servers) test steps. + +## Steps + +### Setup + +- Install Gateway API Resources: + + ```console + kubectl apply -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v0.8.1/standard-install.yaml + ``` + +- Install edge NGF and save the Pod Name and LoadBalancer IP for tests: + + ```console + helm install scale-test oci://ghcr.io/nginxinc/charts/nginx-gateway-fabric --create-namespace --wait -n nginx-gateway --version=0.0.0-edge + ``` + + ```console + export NGF_IP=$(kubectl get svc -n nginx-gateway scale-test-nginx-gateway-fabric --output jsonpath='{.status.loadBalancer.ingress[0].ip}') + export NGF_POD=$(kubectl get pods -n nginx-gateway -l "app.kubernetes.io/name=nginx-gateway-fabric,app.kubernetes.io/instance=scale-test" -o jsonpath="{.items[0].metadata.name}") + ``` + +- Install Prometheus: + + ```console + kubectl apply -f manifets/prom-clusterrole.yaml + helm repo add prometheus-community https://prometheus-community.github.io/helm-charts + helm repo update + helm install prom prometheus-community/prometheus --set useExistingClusterRoleName=prometheus -n prom + ``` + +- Create a directory under [results](/tests/scale/results) and name it after the version of NGF you are testing. Then + create a file for the result summary, also named after the NGF version. For + example: [1.0.0.md](/tests/scale/results/1.0.0/1.0.0.md). + +### Run the tests + +#### Scale Listeners to Max of 64 + +Test Goal: Measure how NGF performs as the number of Listeners increases to the max of 64. + +Test Plan: + +- Scale up to 64 HTTP Listeners +- All Listeners are on a single Gateway +- Each Listener has 1 HTTPRoute attached +- Each HTTPRoute references 1 unique Service +- Services and Deployments are created before scaling Listeners. +- After each Listener + HTTPRoute is created, measure the time it takes to get a successful response from the new + route (time to ready). +- Record the time to ready in seconds in a csv file for each iteration. + +Total Resources Created: + +- 1 Gateway with 64 Listeners +- 64 HTTPRoutes +- 64 Services, Deployments, Pods + +Follow the steps below to run the test: + +- Run the test: + + ```console + go test -v -tags scale -run TestScale_Listeners -i 64 + ``` + +- [Analyze](#analyze) the results. + +- Clean up: + + Delete resources from cluster: + + ```console + kubectl delete -Rf TestScale_Listeners + ``` + + Delete generated manifests: + + ```console + rm -rf TestScale_Listeners + ``` + +- Check for any errors or restarts after cleanup. +- Check NGINX conf to make sure it looks correct. + +#### Scale HTTPS Listeners to Max of 64 + +Test Goal: Measure how NGF performs as the number of HTTPS Listeners increases to the max of 64. + +Test Plan: + +- Scale up to 64 HTTPS Listeners +- All Listeners are on a single Gateway +- Each Listener has 1 HTTPRoute attached +- Each Listener references a unique Secret +- Each HTTPRoute references 1 unique Service +- Services, Deployments, and Secrets are created before scaling Listeners +- After each Listener + HTTPRoute is created, measure the time it takes to get a successful response from the new + route (time to ready). +- Record the time to ready in seconds in a csv file for each iteration. + +Total Resources Created: + +- 1 Gateway with 64 HTTPS Listeners +- 64 Secrets +- 64 HTTPRoutes +- 64 Services, Deployments, Pods + +Follow the steps below to run the test: + +- Run the test: + + ```console + go test -v -tags scale -run TestScale_HTTPSListeners -i 64 + ``` + +- [Analyze](#analyze) the results. + +- Clean up: + + Delete resources from cluster: + + ```console + kubectl delete -Rf TestScale_HTTPSListeners + ``` + + Delete generated manifests: + + ```console + rm -rf TestScale_HTTPSListeners + ``` + +- Check for any errors or restarts after cleanup. +- Check NGINX conf to make sure it looks correct. + +#### Scale HTTPRoutes + +Test Goal: Measure how NGF performs as the number of HTTPRoutes increases to 1000. + +Test Plan: + +- Scale up to 1000 HTTPRoutes +- All HTTPRoutes attach to a single Gateway with one Listener +- Each HTTPRoute references the same Service +- Service and Deployment are created before scaling HTTPRoutes +- After each HTTPRoute is created, measure the time it takes to get a successful response from the new route (time to + ready). +- Record the time to ready in seconds in a csv file for each iteration. + +Total Resources Created: + +- 1 Gateway with 1 Listener +- 1000 HTTPRoutes +- 1 Service, Deployment, Pod + +This test takes around 7 hours to run, so I recommend running it on a VM, or overnight with the aid of +[caffeinate](https://www.theapplegeek.co.uk/blog/caffeinate) for MAC users. + +Follow the steps below to run the test: + +- Run the test: + + ```console + go test -v -tags scale -timeout 600m -run TestScale_HTTPRoutes -i 1000 -delay 2s + ``` + +- [Analyze](#analyze) the results. + +- Clean up: + + Delete resources from cluster: + + ```console + kubectl delete -Rf TestScale_HTTPRoutes + ``` + + Delete generated manifests: + + ```console + rm -rf TestScale_HTTPRoutes + ``` + +- Check for any errors or restarts after cleanup. +- Check NGINX conf to make sure it looks correct. + +#### Scale Upstream Servers + +Test Goal: Measure how NGF performs as the number of Upstream Servers increases to the max of 648. + +Test Plan: + +- Deploy a single Gateway with 1 Listener and attach one HTTPRoute that references a single Service +- Scale the deployment for that Service to 648 Pods (this is the limit that the upstream zone size allows) +- Gateway, HTTPRoute, Service, and Deployment with 1 replica are created before scaling up to 648 replicas. + +Total Resources Created: + +- 1 Gateway with 1 Listener +- 1 HTTPRoutes +- 1 Service, 1 Deployment, 648 Pods + +Test Environment: + +For this test you must use a much bigger cluster in order to create 648 Pods. + +- A Kubernetes cluster with 12 nodes on GKE + - Node: n2d-standard-16 (16 vCPU, 64GB memory) + - Enabled GKE logging + +Follow the steps below to run the test: + +- Apply manifest + + ```console + kubectl apply -f manifests/scale-upstreams.yaml + ``` + +- Check the status of the Gateway and HTTPRoute to make sure everything is OK before scaling. + + ```console + kubectl describe gateway gateway + kubectl describe httproute route + ``` + +- Get the start time as a UNIX timestamp and record it in the results. + + ```console + date +%s + ``` + + This will be used in the metrics query. + +- Open a new terminal window and start the following loop: + + ```console + for i in $(seq 1 150); do curl --resolve cafe.example.com:80:$NGF_IP http://cafe.example.com:80/; sleep 1; done >> requests.log + ``` + +- Back in your original terminal, scale the backend app: + + ```console + kubectl scale deploy backend --replicas 648 + ``` + +- Wait for all Pods to become available: + + ```console + watch kubectl get deploy backend + ``` + +- Check the NGINX config for 648 upstream servers: + + ```console + kubectl exec -it -n nginx-gateway $NGF_POD -c nginx -- nginx -T | grep -E "server (?:[0-9]{1,3}\.){3}[0-9]{1,3}:8080" | wc -l + ``` + +- Get the end time as a UNIX timestamp and make a note of it: + + ```console + date +%s + ``` + +- In the terminal you started the request loop, kill the loop if it's still running and check the request.log to see if + any of the requests failed. Record any failures in the results file. + +- [Analyze](#analyze) the results. Use the start time and end time you made note of earlier for the + queries. You can calculate the test duration in seconds by subtracting the start time from the end time. + +- Clean up: + + ```console + kubectl delete -f manifests/scale-upstreams.yaml + ``` + +- Check for any errors or restarts after cleanup. +- Check NGINX conf to make sure it looks correct. + +#### Scale HTTP Matches + +Test Goal: Find the difference in latency between the first match and last match for the max length of +the `http_matches` variable. + +Test Plan: + +- Deploy a single Gateway with 1 Listener and attach one HTTPRoute that references a single Service +- Within the HTTPRoute configure the max number of matches (max is determined by the length of the + generated `http_matches` variable (4096 characters)) +- Use `wrk` to send requests to the _first_ match in `http_matches` list and measure the latency +- Use `wrk` to send requests to the _last_ match in `http_matches` list and measure the latency + +Total Resources Created: + +- 1 Gateway with 1 Listener +- 1 HTTPRoute with 7 rules and 50 matches +- 1 Service, 1 Deployment, 1 Pod + +Follow these steps to run the test: + +- Download [wrk](https://github.com/wg/wrk) + +- Apply manifest: + + ```console + kubectl apply -f manifests/scale-matches.yaml + ``` + +- Check the status of the Gateway and HTTPRoute to make sure everything is OK before scaling. + + ```console + kubectl describe gateway gateway + kubectl describe httproute route + ``` + +- Test the first match: + + ```console + ./wrk -t2 -c10 -d30 http://cafe.example.com -H "header-1: header-1-val" + ``` + +- Test the last match: + + ```console + ./wrk -t2 -c10 -d30 http://cafe.example.com -H "header-50: header-50-val" + ``` + +- Copy and paste the results into the results file. + +- Clean up: + + ```console + kubectl delete -f manifests/scale-matches.yaml + ``` + +### Analyze + +- Query Prometheus for reload metrics. To access the Prometheus Server, run: + + ```console + export POD_NAME=$(kubectl get pods --namespace prom -l "app.kubernetes.io/name=prometheus,app.kubernetes.io/instance=prom" -o jsonpath="{.items[0].metadata.name}") + kubectl --namespace prom port-forward $POD_NAME 9090 + ``` + + To query Prometheus, you can either browse to localhost:9090 or use curl. The following instructions assume you are + using the prom GUI. + + > Note: + > For the tests that write to a csv file, the `Test Start`, `Test End + 10s`, and `Duration` are at the + > end of the results.csv file in the `results//` directory. + > We are using `Test End + 10s` in the Prometheus query to account for the 10s scraping interval. + + Total number of reloads: + + ```console + nginx_gateway_fabric_nginx_reloads_total - nginx_gateway_fabric_nginx_reloads_total @ + ``` + + Total number of reload errors: + + ```console + nginx_gateway_fabric_nginx_reload_errors_total - nginx_gateway_fabric_nginx_reload_errors_total @ + ``` + + Average reload time (ms): + + ```console + rate(nginx_gateway_fabric_nginx_reloads_milliseconds_sum[] @ ) / + rate(nginx_gateway_fabric_nginx_reloads_milliseconds_count[] @ ) + ``` + + Record these numbers in a table in the results file. + +- Take screenshots of memory and CPU usage in GKE Dashboard + + To Monitor memory and CPU usage, navigate to the Kubernetes Engine > Workloads > Filter by `nginx-gateway` namespace > + click on NGF Pod name. You should see graphs for CPU, Memory, and Disk. + + - Convert the `Start Time` and `End Time` UNIX timestamps to your local date time: + + ```console + date -r + ``` + + - Create a custom time frame for the graphs in GKE. + - Take a screenshot of the CPU and Memory graphs individually. Store them in the `results//` + directory. + +- If the test writes time to ready numbers to a csv, create a time to ready graph. + - Use https://chart-studio.plotly.com/create/#/ to plot the time to ready numbers on a graph. + - Remove the `"Test Start", "Test End", "Test End + 10s", "Duration"` rows from the bottom of the csv. + - Upload the csv file to plotly. + - Create a new `Trace`, select `Line` as the type. + - Set the Y axis to the Time to Ready column. + - Set the X axis to the number of resources column. + - Label the graph and take a screenshot. + - Store the graph in the `results//` directory. + +- Check for errors or restarts and record in the results file. File a bug if there's unexpected errors or restarts. +- Check NGINX conf and make sure it looks correct. File a bug if there is an issue. + +### Results + +- [1.0.0](/tests/scale/results/1.0.0/1.0.0.md) diff --git a/tests/scale/scale_test.go b/tests/scale/scale_test.go new file mode 100644 index 0000000000..cf18eaacd6 --- /dev/null +++ b/tests/scale/scale_test.go @@ -0,0 +1,252 @@ +//go:build scale +// +build scale + +package scale + +import ( + "context" + "crypto/tls" + "encoding/csv" + "flag" + "fmt" + "net/http" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "testing" + "time" + + "k8s.io/apimachinery/pkg/util/wait" +) + +// testing flags +var ( + numIterations = flag.Int("i", 1, "number of times to scale the resource") + delay = flag.Duration("delay", 0, "delay between each scaling iteration") + version = flag.String("version", "1.0", "version of NGF under test") +) + +func TestScale_Listeners(t *testing.T) { + ip := getIP(t) + url := fmt.Sprintf("http://%s/", ip) + + runScaleTest( + t, + []string{"# Listeners", "Time to Ready (s)", "Error"}, + func(dir string) error { + return generateScaleListenerManifests(*numIterations, dir, false /*non-tls*/) + }, + url, + ) +} + +func TestScale_HTTPSListeners(t *testing.T) { + ip := getIP(t) + url := fmt.Sprintf("https://%s/", ip) + + runScaleTest( + t, + []string{"# HTTPS Listeners", "Time to Ready (s)", "Error"}, + func(dir string) error { + return generateScaleListenerManifests(*numIterations, dir, true /*tls*/) + }, + url, + ) +} + +func TestScale_HTTPRoutes(t *testing.T) { + ip := getIP(t) + url := fmt.Sprintf("http://%s/", ip) + + runScaleTest( + t, + []string{"# HTTPRoutes", "Time to Ready (s)", "Error"}, + func(dir string) error { + return generateScaleHTTPRouteManifests(*numIterations, dir) + }, + url, + ) +} + +func runScaleTest( + t *testing.T, + resultHeaders []string, + generateManifests func(dir string) error, + url string, +) { + t.Helper() + manifestDir := t.Name() + + writer := newResultsWriter(t, t.Name(), resultHeaders...) + + if err := generateManifests(manifestDir); err != nil { + t.Fatalf("failed to generate manifests: %s", err) + } + + startTime := time.Now() + startUnix := fmt.Sprintf("%d", startTime.Unix()) + + if err := kubectlApply(getPrereqDirName(manifestDir)); err != nil { + t.Fatalf("failed to apply prerequisite resources: %s", err) + } + + t.Log("Waiting for all Pods to be Ready") + if err := kubectlWaitAllPodsReady(); err != nil { + t.Fatalf("failed to wait for all Pods to be Ready: %s", err) + } + + for i := 0; i < *numIterations; i++ { + t.Logf("Scaling up to %d resources", i) + + manifestFile := filepath.Join(manifestDir, fmt.Sprintf("manifest-%d.yaml", i)) + + if err := kubectlApply(manifestFile); err != nil { + t.Errorf("failed to scale up: %s", err) + } + + host := fmt.Sprintf("%d.example.com", i) + + t.Logf("Sending request to url %s with host %s...", url, host) + + ttr, err := waitForResponseForHost(url, host) + + seconds := ttr.Seconds() + record := []string{strconv.Itoa(i + 1), strconv.FormatFloat(seconds, 'f', -1, 64)} + if err != nil { + record = append(record, err.Error()) + } + + if err = writer.Write(record); err != nil { + t.Fatalf("failed to write time to ready to csv file: %s", err) + } + + time.Sleep(*delay) + } + + endTime := time.Now() + endUnix := fmt.Sprintf("%d", endTime.Unix()) + + // This accounts for prometheus 10s scraping window + endUnixPlusTen := fmt.Sprintf("%d", endTime.Add(10*time.Second).Unix()) + + records := [][]string{ + {"Test Start", "Test End", "Test End + 10s", "Duration"}, + {startUnix, endUnix, endUnixPlusTen, endTime.Sub(startTime).String()}, + } + + if err := writer.WriteAll(records); err != nil { + t.Logf("failed to write records to csv") + } +} + +func getIP(t *testing.T) string { + t.Helper() + + ip := os.Getenv("NGF_IP") + if ip == "" { + t.Fatalf("NGF_IP env var not set") + } + + return ip +} + +func newResultsWriter(t *testing.T, testName string, resultHeaders ...string) *csv.Writer { + t.Helper() + + versionDir := filepath.Join("results", *version) + if err := os.Mkdir(versionDir, 0o750); err != nil && !os.IsExist(err) { + t.Fatalf("failed to create results version directory: %s", err) + } + + dir := filepath.Join(versionDir, testName) + if err := os.Mkdir(dir, 0o750); err != nil { + t.Fatalf("failed to create results test directory: %s", err) + } + + file, err := os.Create(filepath.Join(dir, "results.csv")) + if err != nil { + t.Fatalf("failed to create results csv file: %s", err) + } + + writer := csv.NewWriter(file) + + if err = writer.Write(resultHeaders); err != nil { + t.Fatalf("failed to write headers to csv file: %s", err) + } + + t.Cleanup(func() { + writer.Flush() + _ = file.Close() + }) + + return writer +} + +func kubectlApply(filename string) error { + if err := kubectlExec("apply", "-f", filename); err != nil { + return fmt.Errorf("error applying %s: %w", filename, err) + } + + return nil +} + +func kubectlWaitAllPodsReady() error { + if err := kubectlExec("wait", "pod", "--all", "--for=condition=Ready"); err != nil { + return fmt.Errorf("error waiting for all pods to be ready:%w", err) + } + + return nil +} + +func kubectlExec(arg ...string) error { + cmd := exec.Command("kubectl", arg...) + return cmd.Err +} + +func waitForResponseForHost(url, host string) (time.Duration, error) { + client := &http.Client{} + + if strings.HasPrefix(url, "https") { + customTransport := http.DefaultTransport.(*http.Transport) + customTransport.TLSClientConfig = &tls.Config{ + InsecureSkipVerify: true, // nolint: gosec + ServerName: host, + } + client.Transport = customTransport + } + + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + + req, err := http.NewRequestWithContext(ctx, "GET", url, nil) + if err != nil { + return 0, err + } + + req.Host = host + + start := time.Now() + + err = wait.PollUntilContextCancel( + ctx, + 200*time.Millisecond, + true, + func(ctx context.Context) (done bool, err error) { + resp, err := client.Do(req) + if err != nil { + fmt.Println("Retrying GET request", "error", err) + return false, err + } + + if resp.StatusCode == http.StatusOK { + return true, nil + } + + fmt.Println("Retrying GET request", "host", host, "status", resp.Status) + return false, nil + }) + + return time.Since(start), err +}