From 27aff06499c5c33433e50260b05f53b3e05c75fc Mon Sep 17 00:00:00 2001 From: Eoin Gallinagh Date: Tue, 16 Apr 2024 12:13:04 +0100 Subject: [PATCH 1/5] add: mutating webhook --- PROJECT | 4 + config/certmanager/certificate.yaml | 3 + config/certmanager/kustomization.yaml | 5 + config/certmanager/kustomizeconfig.yaml | 16 +++ config/default/kustomization.yaml | 11 +- config/default/manager_webhook_patch.yaml | 23 ++++ config/default/webhookcainjection_patch.yaml | 15 +++ config/webhook/kustomization.yaml | 6 + config/webhook/kustomizeconfig.yaml | 25 ++++ config/webhook/manifests.yaml | 27 ++++ config/webhook/service.yaml | 22 ++++ main.go | 3 + pkg/controllers/raycluster_webhook.go | 122 +++++++++++++++++++ 13 files changed, 279 insertions(+), 3 deletions(-) create mode 100644 config/certmanager/certificate.yaml create mode 100644 config/certmanager/kustomization.yaml create mode 100644 config/certmanager/kustomizeconfig.yaml create mode 100644 config/default/manager_webhook_patch.yaml create mode 100644 config/default/webhookcainjection_patch.yaml create mode 100644 config/webhook/kustomization.yaml create mode 100644 config/webhook/kustomizeconfig.yaml create mode 100644 config/webhook/manifests.yaml create mode 100644 config/webhook/service.yaml create mode 100644 pkg/controllers/raycluster_webhook.go diff --git a/PROJECT b/PROJECT index cb6903086..3e821aa02 100644 --- a/PROJECT +++ b/PROJECT @@ -15,5 +15,9 @@ resources: domain: codeflare.dev group: ray kind: RayCluster + path: github.com/project-codeflare/codeflare-operator/pkg/controllers version: v1 + webhooks: + defaulting: true + webhookVersion: v1 version: "3" diff --git a/config/certmanager/certificate.yaml b/config/certmanager/certificate.yaml new file mode 100644 index 000000000..451153488 --- /dev/null +++ b/config/certmanager/certificate.yaml @@ -0,0 +1,3 @@ +# The following manifests contain a self-signed issuer CR and a certificate CR. +# More document can be found at https://docs.cert-manager.io +# WARNING: Targets CertManager v1.0. Check https://cert-manager.io/docs/installation/upgrading/ for breaking changes. diff --git a/config/certmanager/kustomization.yaml b/config/certmanager/kustomization.yaml new file mode 100644 index 000000000..bebea5a59 --- /dev/null +++ b/config/certmanager/kustomization.yaml @@ -0,0 +1,5 @@ +resources: +- certificate.yaml + +configurations: +- kustomizeconfig.yaml diff --git a/config/certmanager/kustomizeconfig.yaml b/config/certmanager/kustomizeconfig.yaml new file mode 100644 index 000000000..e631f7773 --- /dev/null +++ b/config/certmanager/kustomizeconfig.yaml @@ -0,0 +1,16 @@ +# This configuration is for teaching kustomize how to update name ref and var substitution +nameReference: +- kind: Issuer + group: cert-manager.io + fieldSpecs: + - kind: Certificate + group: cert-manager.io + path: spec/issuerRef/name + +varReference: +- kind: Certificate + group: cert-manager.io + path: spec/commonName +- kind: Certificate + group: cert-manager.io + path: spec/dnsNames diff --git a/config/default/kustomization.yaml b/config/default/kustomization.yaml index 6e926aed2..fccc7164e 100644 --- a/config/default/kustomization.yaml +++ b/config/default/kustomization.yaml @@ -14,10 +14,15 @@ commonLabels: app.kubernetes.io/part-of: codeflare bases: -- ../rbac -- ../manager + - ../rbac + - ../manager + - ../webhook # [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'. # - ../prometheus resources: -- metrics_service.yaml + - metrics_service.yaml + +patches: +- path: manager_webhook_patch.yaml +- path: webhookcainjection_patch.yaml diff --git a/config/default/manager_webhook_patch.yaml b/config/default/manager_webhook_patch.yaml new file mode 100644 index 000000000..5d6b541dc --- /dev/null +++ b/config/default/manager_webhook_patch.yaml @@ -0,0 +1,23 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: manager + namespace: system +spec: + template: + spec: + containers: + - name: manager + ports: + - containerPort: 9443 + name: webhook-server + protocol: TCP + volumeMounts: + - mountPath: /tmp/k8s-webhook-server/serving-certs + name: cert + readOnly: true + volumes: + - name: cert + secret: + defaultMode: 420 + secretName: codeflare-operator-raycluster-webhook-cert diff --git a/config/default/webhookcainjection_patch.yaml b/config/default/webhookcainjection_patch.yaml new file mode 100644 index 000000000..2770f6a9c --- /dev/null +++ b/config/default/webhookcainjection_patch.yaml @@ -0,0 +1,15 @@ +# This patch add annotation to admission webhook config +apiVersion: admissionregistration.k8s.io/v1 +kind: MutatingWebhookConfiguration +metadata: + labels: + app.kubernetes.io/name: mutatingwebhookconfiguration + app.kubernetes.io/instance: mutating-webhook-configuration + app.kubernetes.io/component: webhook + app.kubernetes.io/created-by: codeflare-operator + app.kubernetes.io/part-of: codeflare-operator + app.kubernetes.io/managed-by: kustomize + name: mutating-webhook-configuration + annotations: + service.beta.openshift.io/inject-cabundle: "true" + service.beta.openshift.io/serving-cert-secret-name: codeflare-operator-raycluster-webhook-cert diff --git a/config/webhook/kustomization.yaml b/config/webhook/kustomization.yaml new file mode 100644 index 000000000..9cf26134e --- /dev/null +++ b/config/webhook/kustomization.yaml @@ -0,0 +1,6 @@ +resources: +- manifests.yaml +- service.yaml + +configurations: +- kustomizeconfig.yaml diff --git a/config/webhook/kustomizeconfig.yaml b/config/webhook/kustomizeconfig.yaml new file mode 100644 index 000000000..25e21e3c9 --- /dev/null +++ b/config/webhook/kustomizeconfig.yaml @@ -0,0 +1,25 @@ +# the following config is for teaching kustomize where to look at when substituting vars. +# It requires kustomize v2.1.0 or newer to work properly. +nameReference: +- kind: Service + version: v1 + fieldSpecs: + - kind: MutatingWebhookConfiguration + group: admissionregistration.k8s.io + path: webhooks/clientConfig/service/name + - kind: ValidatingWebhookConfiguration + group: admissionregistration.k8s.io + path: webhooks/clientConfig/service/name + +namespace: +- kind: MutatingWebhookConfiguration + group: admissionregistration.k8s.io + path: webhooks/clientConfig/service/namespace + create: true +- kind: ValidatingWebhookConfiguration + group: admissionregistration.k8s.io + path: webhooks/clientConfig/service/namespace + create: true + +varReference: +- path: metadata/annotations diff --git a/config/webhook/manifests.yaml b/config/webhook/manifests.yaml new file mode 100644 index 000000000..faf91696b --- /dev/null +++ b/config/webhook/manifests.yaml @@ -0,0 +1,27 @@ +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: MutatingWebhookConfiguration +metadata: + creationTimestamp: null + name: mutating-webhook-configuration +webhooks: +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: webhook-service + namespace: system + path: /mutate-ray-io-v1-raycluster + failurePolicy: Fail + name: mraycluster.kb.io + rules: + - apiGroups: + - ray.io + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - rayclusters + sideEffects: None diff --git a/config/webhook/service.yaml b/config/webhook/service.yaml new file mode 100644 index 000000000..9caf0911a --- /dev/null +++ b/config/webhook/service.yaml @@ -0,0 +1,22 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/name: service + app.kubernetes.io/part-of: codeflare-operator + app.kubernetes.io/instance: webhook-service + app.kubernetes.io/component: webhook + app.kubernetes.io/created-by: codeflare-operator + app.kubernetes.io/managed-by: kustomize + name: webhook-service + namespace: openshift-operators + annotations: + service.beta.openshift.io/serving-cert-secret-name: codeflare-operator-raycluster-webhook-cert +spec: + ports: + - port: 443 + protocol: TCP + targetPort: 9443 + selector: + app.kubernetes.io/part-of: codeflare + app.kubernetes.io/name: codeflare-operator diff --git a/main.go b/main.go index 09ebe2599..40f916a50 100644 --- a/main.go +++ b/main.go @@ -147,6 +147,9 @@ func main() { }) exitOnError(err, "unable to start manager") + rayClusterDefaulter := &controllers.RayClusterDefaulter{} + exitOnError(rayClusterDefaulter.SetupWebhookWithManager(mgr), "error setting up webhook") + ok, err := HasAPIResourceForGVK(kubeClient.DiscoveryClient, rayv1.GroupVersion.WithKind("RayCluster")) if ok { rayClusterController := controllers.RayClusterReconciler{Client: mgr.GetClient(), Scheme: mgr.GetScheme(), Config: cfg.KubeRay} diff --git a/pkg/controllers/raycluster_webhook.go b/pkg/controllers/raycluster_webhook.go new file mode 100644 index 000000000..5cabf2fae --- /dev/null +++ b/pkg/controllers/raycluster_webhook.go @@ -0,0 +1,122 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers + +import ( + "context" + rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" + ctrl "sigs.k8s.io/controller-runtime" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/webhook" +) + +// log is for logging in this package. +var rayclusterlog = logf.Log.WithName("raycluster-resource") + +func (r *RayClusterDefaulter) SetupWebhookWithManager(mgr ctrl.Manager) error { + return ctrl.NewWebhookManagedBy(mgr). + For(&rayv1.RayCluster{}). + WithDefaulter(r). + Complete() +} + +//+kubebuilder:webhook:path=/mutate-ray-io-v1-raycluster,mutating=true,failurePolicy=fail,sideEffects=None,groups=ray.io,resources=rayclusters,verbs=create;update,versions=v1,name=mraycluster.kb.io,admissionReviewVersions=v1 + +type RayClusterDefaulter struct{} + +var _ webhook.CustomDefaulter = &RayClusterDefaulter{} + +// Default implements webhook.Defaulter so a webhook will be registered for the type +func (r *RayClusterDefaulter) Default(ctx context.Context, obj runtime.Object) error { + raycluster := obj.(*rayv1.RayCluster) + + rayclusterlog.Info("default", "name", raycluster.Name) + // Check and add OAuth proxy if it does not exist. + alreadyExists := false + for _, container := range raycluster.Spec.HeadGroupSpec.Template.Spec.Containers { + if container.Name == "oauth-proxy" { + rayclusterlog.Info("OAuth sidecar already exists, no patch needed") + alreadyExists = true + break // exits the for loop + } + } + + if !alreadyExists { + rayclusterlog.Info("Adding OAuth sidecar container") + // definition of the new container + newOAuthSidecar := corev1.Container{ + Name: "oauth-proxy", + Image: "registry.redhat.io/openshift4/ose-oauth-proxy@sha256:1ea6a01bf3e63cdcf125c6064cbd4a4a270deaf0f157b3eabb78f60556840366", + Ports: []corev1.ContainerPort{ + {ContainerPort: 8443, Name: "oauth-proxy"}, + }, + Args: []string{ + "--https-address=:8443", + "--provider=openshift", + "--openshift-service-account=" + raycluster.Name + "-oauth-proxy", + "--upstream=http://localhost:8265", + "--tls-cert=/etc/tls/private/tls.crt", + "--tls-key=/etc/tls/private/tls.key", + "--cookie-secret=$(COOKIE_SECRET)", + "--openshift-delegate-urls={\"/\":{\"resource\":\"pods\",\"namespace\":\"default\",\"verb\":\"get\"}}", + "--added-label=True", + }, + Env: []corev1.EnvVar{ + { + Name: "COOKIE_SECRET", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: raycluster.Name + "-oauth-config", + }, + Key: "cookie_secret", + }, + }, + }, + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "proxy-tls-secret", + MountPath: "/etc/tls/private", + ReadOnly: true, + }, + }, + } + + // Adding the new OAuth sidecar container + raycluster.Spec.HeadGroupSpec.Template.Spec.Containers = append(raycluster.Spec.HeadGroupSpec.Template.Spec.Containers, newOAuthSidecar) + + tlsSecretVolume := corev1.Volume{ + Name: "proxy-tls-secret", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: raycluster.Name + "-proxy-tls-secret", + }, + }, + } + + raycluster.Spec.HeadGroupSpec.Template.Spec.Volumes = append(raycluster.Spec.HeadGroupSpec.Template.Spec.Volumes, tlsSecretVolume) + + // Ensure the service account is set + if raycluster.Spec.HeadGroupSpec.Template.Spec.ServiceAccountName == "" { + raycluster.Spec.HeadGroupSpec.Template.Spec.ServiceAccountName = raycluster.Name + "-oauth-proxy" + } + } + return nil +} From 08743f5788329427c013ca6386dfa83f2b411995 Mon Sep 17 00:00:00 2001 From: Eoin Gallinagh Date: Tue, 16 Apr 2024 14:32:42 +0100 Subject: [PATCH 2/5] remove: added label --- pkg/controllers/raycluster_webhook.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/controllers/raycluster_webhook.go b/pkg/controllers/raycluster_webhook.go index 5cabf2fae..63581907c 100644 --- a/pkg/controllers/raycluster_webhook.go +++ b/pkg/controllers/raycluster_webhook.go @@ -75,7 +75,6 @@ func (r *RayClusterDefaulter) Default(ctx context.Context, obj runtime.Object) e "--tls-key=/etc/tls/private/tls.key", "--cookie-secret=$(COOKIE_SECRET)", "--openshift-delegate-urls={\"/\":{\"resource\":\"pods\",\"namespace\":\"default\",\"verb\":\"get\"}}", - "--added-label=True", }, Env: []corev1.EnvVar{ { From 1e3bedc0a5a397d0c15dc22c49c24fb50e8847b7 Mon Sep 17 00:00:00 2001 From: Eoin Gallinagh Date: Tue, 16 Apr 2024 16:31:05 +0100 Subject: [PATCH 3/5] add: address feedback on pr --- config/certmanager/certificate.yaml | 3 --- config/certmanager/kustomization.yaml | 5 ----- config/certmanager/kustomizeconfig.yaml | 16 ---------------- config/default/webhookcainjection_patch.yaml | 10 +--------- config/webhook/service.yaml | 11 ++--------- pkg/controllers/raycluster_webhook.go | 4 +++- 6 files changed, 6 insertions(+), 43 deletions(-) delete mode 100644 config/certmanager/certificate.yaml delete mode 100644 config/certmanager/kustomization.yaml delete mode 100644 config/certmanager/kustomizeconfig.yaml diff --git a/config/certmanager/certificate.yaml b/config/certmanager/certificate.yaml deleted file mode 100644 index 451153488..000000000 --- a/config/certmanager/certificate.yaml +++ /dev/null @@ -1,3 +0,0 @@ -# The following manifests contain a self-signed issuer CR and a certificate CR. -# More document can be found at https://docs.cert-manager.io -# WARNING: Targets CertManager v1.0. Check https://cert-manager.io/docs/installation/upgrading/ for breaking changes. diff --git a/config/certmanager/kustomization.yaml b/config/certmanager/kustomization.yaml deleted file mode 100644 index bebea5a59..000000000 --- a/config/certmanager/kustomization.yaml +++ /dev/null @@ -1,5 +0,0 @@ -resources: -- certificate.yaml - -configurations: -- kustomizeconfig.yaml diff --git a/config/certmanager/kustomizeconfig.yaml b/config/certmanager/kustomizeconfig.yaml deleted file mode 100644 index e631f7773..000000000 --- a/config/certmanager/kustomizeconfig.yaml +++ /dev/null @@ -1,16 +0,0 @@ -# This configuration is for teaching kustomize how to update name ref and var substitution -nameReference: -- kind: Issuer - group: cert-manager.io - fieldSpecs: - - kind: Certificate - group: cert-manager.io - path: spec/issuerRef/name - -varReference: -- kind: Certificate - group: cert-manager.io - path: spec/commonName -- kind: Certificate - group: cert-manager.io - path: spec/dnsNames diff --git a/config/default/webhookcainjection_patch.yaml b/config/default/webhookcainjection_patch.yaml index 2770f6a9c..39db86347 100644 --- a/config/default/webhookcainjection_patch.yaml +++ b/config/default/webhookcainjection_patch.yaml @@ -2,14 +2,6 @@ apiVersion: admissionregistration.k8s.io/v1 kind: MutatingWebhookConfiguration metadata: - labels: - app.kubernetes.io/name: mutatingwebhookconfiguration - app.kubernetes.io/instance: mutating-webhook-configuration - app.kubernetes.io/component: webhook - app.kubernetes.io/created-by: codeflare-operator - app.kubernetes.io/part-of: codeflare-operator - app.kubernetes.io/managed-by: kustomize name: mutating-webhook-configuration annotations: - service.beta.openshift.io/inject-cabundle: "true" - service.beta.openshift.io/serving-cert-secret-name: codeflare-operator-raycluster-webhook-cert + service.beta.openshift.io/inject-cabundle: "true" diff --git a/config/webhook/service.yaml b/config/webhook/service.yaml index 9caf0911a..ada0e3281 100644 --- a/config/webhook/service.yaml +++ b/config/webhook/service.yaml @@ -1,22 +1,15 @@ apiVersion: v1 kind: Service metadata: - labels: - app.kubernetes.io/name: service - app.kubernetes.io/part-of: codeflare-operator - app.kubernetes.io/instance: webhook-service - app.kubernetes.io/component: webhook - app.kubernetes.io/created-by: codeflare-operator - app.kubernetes.io/managed-by: kustomize name: webhook-service namespace: openshift-operators annotations: - service.beta.openshift.io/serving-cert-secret-name: codeflare-operator-raycluster-webhook-cert + service.beta.openshift.io/serving-cert-secret-name: codeflare-operator-raycluster-webhook-cert spec: ports: - port: 443 protocol: TCP targetPort: 9443 selector: - app.kubernetes.io/part-of: codeflare + app.kubernetes.io/part-of: codeflare app.kubernetes.io/name: codeflare-operator diff --git a/pkg/controllers/raycluster_webhook.go b/pkg/controllers/raycluster_webhook.go index 63581907c..bce533222 100644 --- a/pkg/controllers/raycluster_webhook.go +++ b/pkg/controllers/raycluster_webhook.go @@ -18,7 +18,9 @@ package controllers import ( "context" + rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" ctrl "sigs.k8s.io/controller-runtime" @@ -32,7 +34,7 @@ var rayclusterlog = logf.Log.WithName("raycluster-resource") func (r *RayClusterDefaulter) SetupWebhookWithManager(mgr ctrl.Manager) error { return ctrl.NewWebhookManagedBy(mgr). For(&rayv1.RayCluster{}). - WithDefaulter(r). + WithDefaulter(&RayClusterDefaulter{}). Complete() } From cd5396fa2f220c51a10b5c7c7b41722096ad5b18 Mon Sep 17 00:00:00 2001 From: Eoin Gallinagh Date: Wed, 17 Apr 2024 11:26:03 +0100 Subject: [PATCH 4/5] add: check for isRayDashboardOAuthEnabledWebhook before applying patch --- pkg/controllers/raycluster_webhook.go | 128 ++++++++++++++------------ pkg/controllers/support.go | 7 ++ 2 files changed, 76 insertions(+), 59 deletions(-) diff --git a/pkg/controllers/raycluster_webhook.go b/pkg/controllers/raycluster_webhook.go index bce533222..3406cd9b6 100644 --- a/pkg/controllers/raycluster_webhook.go +++ b/pkg/controllers/raycluster_webhook.go @@ -26,6 +26,8 @@ import ( ctrl "sigs.k8s.io/controller-runtime" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/webhook" + + "github.com/project-codeflare/codeflare-operator/pkg/config" ) // log is for logging in this package. @@ -34,13 +36,19 @@ var rayclusterlog = logf.Log.WithName("raycluster-resource") func (r *RayClusterDefaulter) SetupWebhookWithManager(mgr ctrl.Manager) error { return ctrl.NewWebhookManagedBy(mgr). For(&rayv1.RayCluster{}). - WithDefaulter(&RayClusterDefaulter{}). + WithDefaulter(&RayClusterDefaulter{ + Config: r.Config, + rayDashboardOauthEnabled: r.isRayDashboardOAuthEnabledWebhook(), + }). Complete() } //+kubebuilder:webhook:path=/mutate-ray-io-v1-raycluster,mutating=true,failurePolicy=fail,sideEffects=None,groups=ray.io,resources=rayclusters,verbs=create;update,versions=v1,name=mraycluster.kb.io,admissionReviewVersions=v1 -type RayClusterDefaulter struct{} +type RayClusterDefaulter struct { + Config *config.KubeRayConfiguration + rayDashboardOauthEnabled bool +} var _ webhook.CustomDefaulter = &RayClusterDefaulter{} @@ -48,75 +56,77 @@ var _ webhook.CustomDefaulter = &RayClusterDefaulter{} func (r *RayClusterDefaulter) Default(ctx context.Context, obj runtime.Object) error { raycluster := obj.(*rayv1.RayCluster) - rayclusterlog.Info("default", "name", raycluster.Name) - // Check and add OAuth proxy if it does not exist. - alreadyExists := false - for _, container := range raycluster.Spec.HeadGroupSpec.Template.Spec.Containers { - if container.Name == "oauth-proxy" { - rayclusterlog.Info("OAuth sidecar already exists, no patch needed") - alreadyExists = true - break // exits the for loop + if r.rayDashboardOauthEnabled { + rayclusterlog.Info("default", "name", raycluster.Name) + // Check and add OAuth proxy if it does not exist. + alreadyExists := false + for _, container := range raycluster.Spec.HeadGroupSpec.Template.Spec.Containers { + if container.Name == "oauth-proxy" { + rayclusterlog.Info("OAuth sidecar already exists, no patch needed") + alreadyExists = true + break // exits the for loop + } } - } - if !alreadyExists { - rayclusterlog.Info("Adding OAuth sidecar container") - // definition of the new container - newOAuthSidecar := corev1.Container{ - Name: "oauth-proxy", - Image: "registry.redhat.io/openshift4/ose-oauth-proxy@sha256:1ea6a01bf3e63cdcf125c6064cbd4a4a270deaf0f157b3eabb78f60556840366", - Ports: []corev1.ContainerPort{ - {ContainerPort: 8443, Name: "oauth-proxy"}, - }, - Args: []string{ - "--https-address=:8443", - "--provider=openshift", - "--openshift-service-account=" + raycluster.Name + "-oauth-proxy", - "--upstream=http://localhost:8265", - "--tls-cert=/etc/tls/private/tls.crt", - "--tls-key=/etc/tls/private/tls.key", - "--cookie-secret=$(COOKIE_SECRET)", - "--openshift-delegate-urls={\"/\":{\"resource\":\"pods\",\"namespace\":\"default\",\"verb\":\"get\"}}", - }, - Env: []corev1.EnvVar{ - { - Name: "COOKIE_SECRET", - ValueFrom: &corev1.EnvVarSource{ - SecretKeyRef: &corev1.SecretKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: raycluster.Name + "-oauth-config", + if !alreadyExists { + rayclusterlog.Info("Adding OAuth sidecar container") + // definition of the new container + newOAuthSidecar := corev1.Container{ + Name: "oauth-proxy", + Image: "registry.redhat.io/openshift4/ose-oauth-proxy@sha256:1ea6a01bf3e63cdcf125c6064cbd4a4a270deaf0f157b3eabb78f60556840366", + Ports: []corev1.ContainerPort{ + {ContainerPort: 8443, Name: "oauth-proxy"}, + }, + Args: []string{ + "--https-address=:8443", + "--provider=openshift", + "--openshift-service-account=" + raycluster.Name + "-oauth-proxy", + "--upstream=http://localhost:8265", + "--tls-cert=/etc/tls/private/tls.crt", + "--tls-key=/etc/tls/private/tls.key", + "--cookie-secret=$(COOKIE_SECRET)", + "--openshift-delegate-urls={\"/\":{\"resource\":\"pods\",\"namespace\":\"default\",\"verb\":\"get\"}}", + }, + Env: []corev1.EnvVar{ + { + Name: "COOKIE_SECRET", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: raycluster.Name + "-oauth-config", + }, + Key: "cookie_secret", }, - Key: "cookie_secret", }, }, }, - }, - VolumeMounts: []corev1.VolumeMount{ - { - Name: "proxy-tls-secret", - MountPath: "/etc/tls/private", - ReadOnly: true, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "proxy-tls-secret", + MountPath: "/etc/tls/private", + ReadOnly: true, + }, }, - }, - } + } - // Adding the new OAuth sidecar container - raycluster.Spec.HeadGroupSpec.Template.Spec.Containers = append(raycluster.Spec.HeadGroupSpec.Template.Spec.Containers, newOAuthSidecar) + // Adding the new OAuth sidecar container + raycluster.Spec.HeadGroupSpec.Template.Spec.Containers = append(raycluster.Spec.HeadGroupSpec.Template.Spec.Containers, newOAuthSidecar) - tlsSecretVolume := corev1.Volume{ - Name: "proxy-tls-secret", - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: raycluster.Name + "-proxy-tls-secret", + tlsSecretVolume := corev1.Volume{ + Name: "proxy-tls-secret", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: raycluster.Name + "-proxy-tls-secret", + }, }, - }, - } + } - raycluster.Spec.HeadGroupSpec.Template.Spec.Volumes = append(raycluster.Spec.HeadGroupSpec.Template.Spec.Volumes, tlsSecretVolume) + raycluster.Spec.HeadGroupSpec.Template.Spec.Volumes = append(raycluster.Spec.HeadGroupSpec.Template.Spec.Volumes, tlsSecretVolume) - // Ensure the service account is set - if raycluster.Spec.HeadGroupSpec.Template.Spec.ServiceAccountName == "" { - raycluster.Spec.HeadGroupSpec.Template.Spec.ServiceAccountName = raycluster.Name + "-oauth-proxy" + // Ensure the service account is set + if raycluster.Spec.HeadGroupSpec.Template.Spec.ServiceAccountName == "" { + raycluster.Spec.HeadGroupSpec.Template.Spec.ServiceAccountName = raycluster.Name + "-oauth-proxy" + } } } return nil diff --git a/pkg/controllers/support.go b/pkg/controllers/support.go index 344e6215f..698bd8da3 100644 --- a/pkg/controllers/support.go +++ b/pkg/controllers/support.go @@ -155,3 +155,10 @@ func (r *RayClusterReconciler) isRayDashboardOAuthEnabled() bool { } return true } + +func (r *RayClusterDefaulter) isRayDashboardOAuthEnabledWebhook() bool { + if r.Config != nil && r.Config.RayDashboardOAuthEnabled != nil { + return *r.Config.RayDashboardOAuthEnabled + } + return true +} From b32e0a921d0ea51b0cd505c1bf2a9640c5dabc85 Mon Sep 17 00:00:00 2001 From: Eoin Gallinagh Date: Wed, 17 Apr 2024 12:47:55 +0100 Subject: [PATCH 5/5] update: placement of cookie secret in the mutating webhook for rayclusters --- pkg/controllers/raycluster_webhook.go | 30 +++++++++++++++------------ 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/pkg/controllers/raycluster_webhook.go b/pkg/controllers/raycluster_webhook.go index 3406cd9b6..d63616e13 100644 --- a/pkg/controllers/raycluster_webhook.go +++ b/pkg/controllers/raycluster_webhook.go @@ -87,19 +87,6 @@ func (r *RayClusterDefaulter) Default(ctx context.Context, obj runtime.Object) e "--cookie-secret=$(COOKIE_SECRET)", "--openshift-delegate-urls={\"/\":{\"resource\":\"pods\",\"namespace\":\"default\",\"verb\":\"get\"}}", }, - Env: []corev1.EnvVar{ - { - Name: "COOKIE_SECRET", - ValueFrom: &corev1.EnvVarSource{ - SecretKeyRef: &corev1.SecretKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: raycluster.Name + "-oauth-config", - }, - Key: "cookie_secret", - }, - }, - }, - }, VolumeMounts: []corev1.VolumeMount{ { Name: "proxy-tls-secret", @@ -112,6 +99,23 @@ func (r *RayClusterDefaulter) Default(ctx context.Context, obj runtime.Object) e // Adding the new OAuth sidecar container raycluster.Spec.HeadGroupSpec.Template.Spec.Containers = append(raycluster.Spec.HeadGroupSpec.Template.Spec.Containers, newOAuthSidecar) + cookieSecret := corev1.EnvVar{ + Name: "COOKIE_SECRET", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: raycluster.Name + "-oauth-config", + }, + Key: "cookie_secret", + }, + }, + } + + raycluster.Spec.HeadGroupSpec.Template.Spec.Containers[0].Env = append( + raycluster.Spec.HeadGroupSpec.Template.Spec.Containers[0].Env, + cookieSecret, + ) + tlsSecretVolume := corev1.Volume{ Name: "proxy-tls-secret", VolumeSource: corev1.VolumeSource{