From a062fb2fad555ac3b787355c80d659632653d42e Mon Sep 17 00:00:00 2001 From: ChristianZaccaria Date: Wed, 30 Aug 2023 14:16:32 +0100 Subject: [PATCH] Remove custom-metrics-api --- .../mcad-controller/templates/deployment.yaml | 93 -------- go.mod | 3 +- go.sum | 2 - pkg/controller/metrics/adapter/adapter.go | 12 +- .../metrics/adapter/provider/provider.go | 216 +----------------- 5 files changed, 10 insertions(+), 316 deletions(-) diff --git a/deployment/mcad-controller/templates/deployment.yaml b/deployment/mcad-controller/templates/deployment.yaml index 1ef381460..44da801fe 100644 --- a/deployment/mcad-controller/templates/deployment.yaml +++ b/deployment/mcad-controller/templates/deployment.yaml @@ -46,62 +46,10 @@ spec: insecureSkipTLSVerify: true groupPriorityMinimum: 100 versionPriority: 100 ---- -apiVersion: apiregistration.k8s.io/v1beta1 -kind: APIService -metadata: - name: v1beta1.custom.metrics.k8s.io -spec: - service: - name: custom-metrics-apiserver - namespace: kube-system - group: custom.metrics.k8s.io - version: v1beta1 - insecureSkipTLSVerify: true - groupPriorityMinimum: 100 - versionPriority: 100 #{{ end }} --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole -metadata: - name: custom-metrics-server-resources -rules: -- apiGroups: - - custom.metrics.k8s.io - resources: ["*"] - verbs: ["*"] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: custom-metrics-resource-reader -rules: -- apiGroups: - - "" - resources: - - namespaces - - pods - - services - verbs: - - get - - list ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: hpa-controller-custom-metrics -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: custom-metrics-server-resources -subjects: -- kind: ServiceAccount - name: horizontal-pod-autoscaler - namespace: kube-system ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole metadata: annotations: rbac.authorization.kubernetes.io/autoupdate: "true" @@ -193,46 +141,6 @@ metadata: #{{ if .Values.serviceAccount }} apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding -metadata: - name: custom-metrics:system:auth-delegator -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: system:auth-delegator -subjects: -- kind: ServiceAccount - name: {{ .Values.serviceAccount }} - namespace: kube-system ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: custom-metrics-auth-reader - namespace: kube-system -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: extension-apiserver-authentication-reader -subjects: -- kind: ServiceAccount - name: {{ .Values.serviceAccount }} - namespace: kube-system ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: custom-metrics-resource-reader -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: custom-metrics-resource-reader -subjects: -- kind: ServiceAccount - name: {{ .Values.serviceAccount }} - namespace: kube-system ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding metadata: annotations: rbac.authorization.kubernetes.io/autoupdate: "true" @@ -378,4 +286,3 @@ spec: #{{ end }} resources: {{ toYaml .Values.resources | indent 10 }} - diff --git a/go.mod b/go.mod index 249a797e1..2dd1bceae 100644 --- a/go.mod +++ b/go.mod @@ -4,8 +4,6 @@ go 1.19 require ( github.com/eapache/go-resiliency v1.3.0 - github.com/emicklei/go-restful v2.16.0+incompatible - github.com/gogo/protobuf v1.3.2 github.com/golang/protobuf v1.5.3 github.com/hashicorp/go-multierror v1.1.1 github.com/onsi/ginkgo v1.11.0 @@ -44,6 +42,7 @@ require ( github.com/go-openapi/jsonpointer v0.19.6 // indirect github.com/go-openapi/jsonreference v0.20.1 // indirect github.com/go-openapi/swag v0.22.3 // indirect + github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/google/cel-go v0.12.6 // indirect github.com/google/gnostic v0.6.9 // indirect diff --git a/go.sum b/go.sum index 2d69f1dba..477132092 100644 --- a/go.sum +++ b/go.sum @@ -88,8 +88,6 @@ github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/eapache/go-resiliency v1.3.0 h1:RRL0nge+cWGlxXbUzJ7yMcq6w2XBEr19dCN6HECGaT0= github.com/eapache/go-resiliency v1.3.0/go.mod h1:5yPzW0MIvSe0JDsv0v+DvcjEv2FyD6iZYSs1ZI+iQho= -github.com/emicklei/go-restful v2.16.0+incompatible h1:rgqiKNjTnFQA6kkhFe16D8epTksy9HQ1MyrbDXSdYhM= -github.com/emicklei/go-restful v2.16.0+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emicklei/go-restful/v3 v3.10.1 h1:rc42Y5YTp7Am7CS630D7JmhRjq4UlEUuEKfrDac4bSQ= github.com/emicklei/go-restful/v3 v3.10.1/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= diff --git a/pkg/controller/metrics/adapter/adapter.go b/pkg/controller/metrics/adapter/adapter.go index 57eaae36d..bf4677262 100644 --- a/pkg/controller/metrics/adapter/adapter.go +++ b/pkg/controller/metrics/adapter/adapter.go @@ -25,12 +25,11 @@ import ( openapinamer "k8s.io/apiserver/pkg/endpoints/openapi" genericapiserver "k8s.io/apiserver/pkg/server" - "github.com/emicklei/go-restful" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/rest" "k8s.io/klog/v2" - generatedcustommetrics "sigs.k8s.io/custom-metrics-apiserver/pkg/generated/openapi/custommetrics" + generatedexternalmetrics "sigs.k8s.io/custom-metrics-apiserver/pkg/generated/openapi/externalmetrics" adapterprov "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/controller/metrics/adapter/provider" "sigs.k8s.io/custom-metrics-apiserver/pkg/apiserver" @@ -52,7 +51,7 @@ type MetricsAdapter struct { Message string } -func (a *MetricsAdapter) makeProviderOrDie(clusterStateCache clusterstatecache.Cache) (provider.MetricsProvider, *restful.WebService) { +func (a *MetricsAdapter) makeProviderOrDie(clusterStateCache clusterstatecache.Cache) (provider.ExternalMetricsProvider) { klog.Infof("[makeProviderOrDie] Entered makeProviderOrDie()") client, err := a.DynamicClient() if err != nil { @@ -85,7 +84,7 @@ func newMetricsAdapter(serverOptions *options.ServerOption, config *rest.Config, cmd := &MetricsAdapter{} - cmd.OpenAPIConfig = genericapiserver.DefaultOpenAPIConfig(generatedcustommetrics.GetOpenAPIDefinitions, openapinamer.NewDefinitionNamer(apiserver.Scheme)) + cmd.OpenAPIConfig = genericapiserver.DefaultOpenAPIConfig(generatedexternalmetrics.GetOpenAPIDefinitions, openapinamer.NewDefinitionNamer(apiserver.Scheme)) cmd.OpenAPIConfig.Info.Title = "MetricsAdpater" cmd.OpenAPIConfig.Info.Version = "1.0.0" @@ -100,13 +99,10 @@ func newMetricsAdapter(serverOptions *options.ServerOption, config *rest.Config, portedArgs := covertServerOptionsToMetricsServerOptions(serverOptions) cmd.Flags().Parse(portedArgs) - testProvider, webService := cmd.makeProviderOrDie(clusterStateCache) - cmd.WithCustomMetrics(testProvider) + testProvider := cmd.makeProviderOrDie(clusterStateCache) cmd.WithExternalMetrics(testProvider) klog.Infof(cmd.Message) - // Set up POST endpoint for writing fake metric values - restful.DefaultContainer.Add(webService) go func() { // Open port for POSTing fake metrics klog.Fatal(http.ListenAndServe(":8080", nil)) diff --git a/pkg/controller/metrics/adapter/provider/provider.go b/pkg/controller/metrics/adapter/provider/provider.go index 96cf6abae..a140e89d9 100644 --- a/pkg/controller/metrics/adapter/provider/provider.go +++ b/pkg/controller/metrics/adapter/provider/provider.go @@ -18,28 +18,21 @@ package provider import ( "context" - "net/http" "strings" "sync" - "time" - "github.com/emicklei/go-restful" "k8s.io/klog/v2" - apierr "k8s.io/apimachinery/pkg/api/errors" apimeta "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" - "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/dynamic" - "k8s.io/metrics/pkg/apis/custom_metrics" "k8s.io/metrics/pkg/apis/external_metrics" clusterstatecache "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/controller/clusterstate/cache" "sigs.k8s.io/custom-metrics-apiserver/pkg/provider" - "sigs.k8s.io/custom-metrics-apiserver/pkg/provider/helpers" ) // CustomMetricResource wraps provider.CustomMetricInfo in a struct which stores the Name and Namespace of the resource @@ -117,7 +110,7 @@ type metricValue struct { value resource.Quantity } -// clusterMetricsProvider is a sample implementation of provider.MetricsProvider which stores a map of fake metrics +// clusterMetricsProvider is a sample implementation of provider.ExternalMetricsProvider type clusterMetricsProvider struct { client dynamic.Interface mapper apimeta.RESTMapper @@ -128,10 +121,10 @@ type clusterMetricsProvider struct { cache2 clusterstatecache.Cache } -var _ provider.MetricsProvider = (*clusterMetricsProvider)(nil) +var _ provider.ExternalMetricsProvider = (*clusterMetricsProvider)(nil) -// NewFakeProvider returns an instance of clusterMetricsProvider, along with its restful.WebService that opens endpoints to post new fake metrics -func NewFakeProvider(client dynamic.Interface, mapper apimeta.RESTMapper, clusterStateCache clusterstatecache.Cache) (provider.MetricsProvider, *restful.WebService) { +// NewFakeProvider returns an instance of clusterMetricsProvider +func NewFakeProvider(client dynamic.Interface, mapper apimeta.RESTMapper, clusterStateCache clusterstatecache.Cache) (provider.ExternalMetricsProvider) { klog.V(10).Infof("[NewFakeProvider] Entered NewFakeProvider()") provider := &clusterMetricsProvider{ client: client, @@ -140,206 +133,7 @@ func NewFakeProvider(client dynamic.Interface, mapper apimeta.RESTMapper, cluste externalMetrics: defaultValueExternalMetrics, cache2: clusterStateCache, } - return provider, provider.webService() -} - -// webService creates a restful.WebService with routes set up for receiving fake metrics -// These writing routes have been set up to be identical to the format of routes which metrics are read from. -// There are 3 metric types available: namespaced, root-scoped, and namespaces. -// (Note: Namespaces, we're assuming, are themselves namespaced resources, but for consistency with how metrics are retreived they have a separate route) -func (p *clusterMetricsProvider) webService() *restful.WebService { - klog.V(10).Infof("Entered webService()") - ws := new(restful.WebService) - - ws.Path("/write-metrics") - - // Namespaced resources - ws.Route(ws.POST("/namespaces/{namespace}/{resourceType}/{name}/{metric}").To(p.updateMetric). - Param(ws.BodyParameter("Value", "Value to set metric").DataType("integer").DefaultValue("0"))) - - // Root-scoped resources - ws.Route(ws.POST("/{resourceType}/{name}/{metric}").To(p.updateMetric). - Param(ws.BodyParameter("Value", "Value to set metric").DataType("integer").DefaultValue("0"))) - - // Namespaces, where {resourceType} == "namespaces" to match API - ws.Route(ws.POST("/{resourceType}/{name}/metrics/{metric}").To(p.updateMetric). - Param(ws.BodyParameter("Value", "Value to set metric").DataType("integer").DefaultValue("0"))) - return ws -} - -// updateMetric writes the metric provided by a restful request and stores it in memory -func (p *clusterMetricsProvider) updateMetric(request *restful.Request, response *restful.Response) { - p.valuesLock.Lock() - defer p.valuesLock.Unlock() - - namespace := request.PathParameter("namespace") - resourceType := request.PathParameter("resourceType") - namespaced := false - if len(namespace) > 0 || resourceType == "namespaces" { - namespaced = true - } - name := request.PathParameter("name") - metricName := request.PathParameter("metric") - - value := new(resource.Quantity) - err := request.ReadEntity(value) - if err != nil { - response.WriteErrorString(http.StatusBadRequest, err.Error()) - return - } - - groupResource := schema.ParseGroupResource(resourceType) - - metricLabels := labels.Set{} - sel := request.QueryParameter("labels") - if len(sel) > 0 { - metricLabels, err = labels.ConvertSelectorToLabelsMap(sel) - if err != nil { - response.WriteErrorString(http.StatusBadRequest, err.Error()) - return - } - } - - info := provider.CustomMetricInfo{ - GroupResource: groupResource, - Metric: metricName, - Namespaced: namespaced, - } - - info, _, err = info.Normalized(p.mapper) - if err != nil { - klog.Errorf("Error normalizing info: %s", err) - } - namespacedName := types.NamespacedName{ - Name: name, - Namespace: namespace, - } - - metricInfo := CustomMetricResource{ - CustomMetricInfo: info, - NamespacedName: namespacedName, - } - p.values[metricInfo] = metricValue{ - labels: metricLabels, - value: *value, - } -} - -// valueFor is a helper function to get just the value of a specific metric -func (p *clusterMetricsProvider) valueFor(info provider.CustomMetricInfo, name types.NamespacedName, metricSelector labels.Selector) (resource.Quantity, error) { - info, _, err := info.Normalized(p.mapper) - if err != nil { - return resource.Quantity{}, err - } - metricInfo := CustomMetricResource{ - CustomMetricInfo: info, - NamespacedName: name, - } - - value, found := p.values[metricInfo] - if !found { - return resource.Quantity{}, provider.NewMetricNotFoundForError(info.GroupResource, info.Metric, name.Name) - } - - if !metricSelector.Matches(value.labels) { - return resource.Quantity{}, provider.NewMetricNotFoundForSelectorError(info.GroupResource, info.Metric, name.Name, metricSelector) - } - - return value.value, nil -} - -// metricFor is a helper function which formats a value, metric, and object info into a MetricValue which can be returned by the metrics API -func (p *clusterMetricsProvider) metricFor(value resource.Quantity, name types.NamespacedName, selector labels.Selector, info provider.CustomMetricInfo, metricSelector labels.Selector) (*custom_metrics.MetricValue, error) { - objRef, err := helpers.ReferenceFor(p.mapper, name, info) - if err != nil { - return nil, err - } - - metric := &custom_metrics.MetricValue{ - DescribedObject: objRef, - Metric: custom_metrics.MetricIdentifier{ - Name: info.Metric, - }, - Timestamp: metav1.Time{Time: time.Now()}, - Value: value, - } - - if len(metricSelector.String()) > 0 { - sel, err := metav1.ParseToLabelSelector(metricSelector.String()) - if err != nil { - return nil, err - } - metric.Metric.Selector = sel - } - - return metric, nil -} - -// metricsFor is a wrapper used by GetMetricBySelector to format several metrics which match a resource selector -func (p *clusterMetricsProvider) metricsFor(namespace string, selector labels.Selector, info provider.CustomMetricInfo, metricSelector labels.Selector) (*custom_metrics.MetricValueList, error) { - names, err := helpers.ListObjectNames(p.mapper, p.client, namespace, selector, info) - if err != nil { - return nil, err - } - - res := make([]custom_metrics.MetricValue, 0, len(names)) - for _, name := range names { - namespacedName := types.NamespacedName{Name: name, Namespace: namespace} - value, err := p.valueFor(info, namespacedName, metricSelector) - if err != nil { - if apierr.IsNotFound(err) { - continue - } - return nil, err - } - - metric, err := p.metricFor(value, namespacedName, selector, info, metricSelector) - if err != nil { - return nil, err - } - res = append(res, *metric) - } - - return &custom_metrics.MetricValueList{ - Items: res, - }, nil -} - -func (p *clusterMetricsProvider) GetMetricByName(_ context.Context, name types.NamespacedName, info provider.CustomMetricInfo, metricSelector labels.Selector) (*custom_metrics.MetricValue, error) { - p.valuesLock.RLock() - defer p.valuesLock.RUnlock() - - value, err := p.valueFor(info, name, metricSelector) - if err != nil { - return nil, err - } - return p.metricFor(value, name, labels.Everything(), info, metricSelector) -} - -func (p *clusterMetricsProvider) GetMetricBySelector(_ context.Context, namespace string, selector labels.Selector, info provider.CustomMetricInfo, metricSelector labels.Selector) (*custom_metrics.MetricValueList, error) { - p.valuesLock.RLock() - defer p.valuesLock.RUnlock() - - return p.metricsFor(namespace, selector, info, metricSelector) -} - -func (p *clusterMetricsProvider) ListAllMetrics() []provider.CustomMetricInfo { - p.valuesLock.RLock() - defer p.valuesLock.RUnlock() - - // Get unique CustomMetricInfos from wrapper CustomMetricResources - infos := make(map[provider.CustomMetricInfo]struct{}) - for resource := range p.values { - infos[resource.CustomMetricInfo] = struct{}{} - } - - // Build slice of CustomMetricInfos to be returns - metrics := make([]provider.CustomMetricInfo, 0, len(infos)) - for info := range infos { - metrics = append(metrics, info) - } - - return metrics + return provider } func (p *clusterMetricsProvider) GetExternalMetric(_ context.Context, namespace string, metricSelector labels.Selector,