perf: add readiness and health probes

Author: dimakis

cmd/kar-controllers/app/server.go

@@ -38,6 +38,7 @@ import (
 	"github.com/project-codeflare/multi-cluster-app-dispatcher/cmd/kar-controllers/app/options"
 	"github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/controller/queuejob"
 	"github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/health"
+	ready "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/ready"
 
 	_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
 )
@@ -79,10 +80,12 @@ func Run(opt *options.ServerOption) error {
 func listenHealthProbe(opt *options.ServerOption) error {
 	handler := http.NewServeMux()
 	handler.Handle("/healthz", &health.Handler{})
+	handler.Handle("/readyz", &ready.Handler{})
 	err := http.ListenAndServe(opt.HealthProbeListenAddr, handler)
 	if err != nil {
 		return err
 	}
 
 	return nil
 }
+

deployment/mcad-controller/templates/deployment.yaml

@@ -357,6 +357,18 @@ spec:
         - name: agent-config-vol
           mountPath: /root/kubernetes
 #{{ end }}
+        livesnessProbe:
+          httpGet:
+            path: /healthz
+            port: 8081
+          initialDelaySeconds: 30
+          timeoutSeconds: 5
+        readinessProbe:
+          httpGet:
+            path: /readyz
+            port: 8081
+          initialDelaySeconds: 30
+          timeoutSeconds: 5
 #{{ if .Values.configMap.name }}
         envFrom:
         - configMapRef:

pkg/quotaplugins/quota-forest/quota-manager/main.go

@@ -82,3 +82,17 @@ func main() {
 
 	fmt.Println(quotaManager)
 }
+
+
+// Starts the health probe listener
+func listenHealthProbe(opt *options.ServerOption) error {
+	handler := http.NewServeMux()
+	handler.Handle("/healthz", &health.Handler{})
+	handler.Handle("/readyz", &ready.Handler{})
+	err := http.ListenAndServe(opt.HealthProbeListenAddr, handler)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}

pkg/ready/readiness.go

@@ -0,0 +1,14 @@
+package readiness
+
+import (
+	"fmt"
+	"net/http"
+)
+
+type Handler struct {
+}
+
+func (h *Handler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
+	fmt.Fprint(resp, "ok")
+}
+

pkg/ready/readiness_test.go

@@ -0,0 +1,27 @@
+package readiness
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"testing"
+)
+
+func TestReadinessProbe(t *testing.T) {
+	req, err := http.NewRequest("GET", "/readyz", nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	rr := httptest.NewRecorder()
+	handler := Handler{}
+	handler.ServeHTTP(rr, req)
+	if status := rr.Code; status != http.StatusOK {
+		t.Errorf("handler returned unexpected status: %v", status)
+	}
+
+	expected := "ok"
+	if rr.Body.String() != expected {
+		t.Errorf("handler returned unexpected body: got %v expected %v",
+			rr.Body.String(), expected)
+	}
+}