perf: add readiness and health probes

Author: dimakis

cmd/kar-controllers/app/options/options.go

@@ -56,6 +56,7 @@ type ServerOption struct {
 	QuotaRestURL          string
 	HealthProbeListenAddr string
 	DispatchResourceReservationTimeout int64
+	ReadinessProbeListenAddr string
 }
 
 // NewServerOption creates a new CMServer with a default config.

cmd/kar-controllers/app/server.go

@@ -38,6 +38,7 @@ import (
 	"github.com/project-codeflare/multi-cluster-app-dispatcher/cmd/kar-controllers/app/options"
 	"github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/controller/queuejob"
 	"github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/health"
+	ready "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/ready"
 
 	_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
 )
@@ -79,10 +80,12 @@ func Run(opt *options.ServerOption) error {
 func listenHealthProbe(opt *options.ServerOption) error {
 	handler := http.NewServeMux()
 	handler.Handle("/healthz", &health.Handler{})
+	handler.Handle("/readyz", &ready.Handler{})
 	err := http.ListenAndServe(opt.HealthProbeListenAddr, handler)
 	if err != nil {
 		return err
 	}
 
 	return nil
 }
+

deployment/mcad-controller/templates/deployment.yaml

@@ -322,6 +322,18 @@ spec:
         imagePullPolicy: {{ .Values.httpServerImage.pullPolicy }}
         ports:
         - containerPort: 80
+        livesnessProbe:
+          httpGet:
+            path: /healthz
+            port: 80
+          initialDelaySeconds: 30
+          timeoutSeconds: 5
+        readinessProbe:
+          httpGet:
+            path: /readyz
+            port: 80
+          initialDelaySeconds: 30
+          timeoutSeconds: 5
       - name: "quota-management"
         image: "{{ .Values.httpImage.repository }}:{{ .Values.httpImage.tag }}"
         imagePullPolicy: {{ .Values.httpImage.pullPolicy }}
@@ -333,6 +345,18 @@ spec:
 #{{ if .Values.volumes.hostPath }}
           - name: agent-config-vol
             mountPath: /root/kubernetes
+        livesnessProbe:
+          httpGet:
+            path: /healthz
+            port: 80
+          initialDelaySeconds: 30
+          timeoutSeconds: 5
+        readinessProbe:
+          httpGet:
+            path: /readyz
+            port: 80
+          initialDelaySeconds: 30
+          timeoutSeconds: 5
 #{{ end }}
 #{{ end }}
       - name: {{ .Chart.Name }}
@@ -357,6 +381,18 @@ spec:
         - name: agent-config-vol
           mountPath: /root/kubernetes
 #{{ end }}
+        livesnessProbe:
+          httpGet:
+            path: /healthz
+            port: 80
+          initialDelaySeconds: 30
+          timeoutSeconds: 5
+        readinessProbe:
+          httpGet:
+            path: /readyz
+            port: 80
+          initialDelaySeconds: 30
+          timeoutSeconds: 5
 #{{ if .Values.configMap.name }}
         envFrom:
         - configMapRef:

pkg/ready/readiness_test.go

@@ -0,0 +1,27 @@
+package readiness
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"testing"
+)
+
+func TestReadinessProbe(t *testing.T) {
+	req, err := http.NewRequest("GET", "/readyz", nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	rr := httptest.NewRecorder()
+	handler := Handler{}
+	handler.ServeHTTP(rr, req)
+	if status := rr.Code; status != http.StatusOK {
+		t.Errorf("handler returned unexpected status: %v", status)
+	}
+
+	expected := "ok"
+	if rr.Body.String() != expected {
+		t.Errorf("handler returned unexpected body: got %v expected %v",
+			rr.Body.String(), expected)
+	}
+}