UPSTREAM: <carry>: kcp modifications

Signed-off-by: Dr. Stefan Schimanski <stefan.schimanski@gmail.com>

Author: xrstf

.prow.yaml

@@ -0,0 +1,13 @@
+presubmits:
+  - name: pull-controller-runtime-everything
+    always_run: true
+    decorate: true
+    clone_uri: "ssh://git@github.com/kcp-dev/controller-runtime.git"
+    labels:
+      preset-goproxy: "true"
+    spec:
+      containers:
+      - image: ghcr.io/kcp-dev/infra/build:1.22.2-1
+        command:
+        - make
+        - test

DOWNSTREAM_OWNERS

@@ -0,0 +1,5 @@
+approvers:
+  - sttts
+  - xrstf
+  - mjudeikis
+  - embik

DOWNSTREAM_OWNERS_ALIASES

@@ -31,6 +31,11 @@ require (
 	sigs.k8s.io/yaml v1.4.0
 )
 
+require (
+	github.com/kcp-dev/apimachinery/v2 v2.0.0
+	github.com/kcp-dev/logicalcluster/v3 v3.0.5
+)
+
 require (
 	cel.dev/expr v0.18.0 // indirect
 	github.com/antlr4-go/antlr/v4 v4.13.0 // indirect

go.mod

@@ -75,6 +75,10 @@ github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8Hm
 github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
 github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
 github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
+github.com/kcp-dev/apimachinery/v2 v2.0.0 h1:hQuhBBh+AvUYYMRG+nDzo1VXxNCdMAE95wSD2uB7nxw=
+github.com/kcp-dev/apimachinery/v2 v2.0.0/go.mod h1:cXCx7fku8/rYK23PNEBRLQ5ByoABoA+CZeJNC81TO0g=
+github.com/kcp-dev/logicalcluster/v3 v3.0.5 h1:JbYakokb+5Uinz09oTXomSUJVQsqfxEvU4RyHUYxHOU=
+github.com/kcp-dev/logicalcluster/v3 v3.0.5/go.mod h1:EWBUBxdr49fUB1cLMO4nOdBWmYifLbP1LfoL20KkXYY=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=

go.sum

@@ -23,6 +23,7 @@ import (
 	"net/http"
 	"slices"
 	"sort"
+	"strings"
 	"time"
 
 	corev1 "k8s.io/api/core/v1"
@@ -170,6 +171,10 @@ type Options struct {
 	// instead of `reconcile.Result{}`.
 	SyncPeriod *time.Duration
 
+	// Indexers is the indexers that the informers will be configured to use.
+	// Will always have the standard NamespaceIndex.
+	Indexers toolscache.Indexers
+
 	// ReaderFailOnMissingInformer configures the cache to return a ErrResourceNotCached error when a user
 	// requests, using Get() and List(), a resource the cache does not already have an informer for.
 	//
@@ -436,6 +441,7 @@ func newCache(restConfig *rest.Config, opts Options) newCacheFunc {
 				NewInformer:           opts.NewInformer,
 			}),
 			readerFailOnMissingInformer: opts.ReaderFailOnMissingInformer,
+			clusterIndexes:              strings.HasSuffix(restConfig.Host, "/clusters/*"),
 		}
 	}
 }
@@ -546,6 +552,10 @@ func defaultOpts(config *rest.Config, opts Options) (Options, error) {
 	if opts.SyncPeriod == nil {
 		opts.SyncPeriod = &defaultSyncPeriod
 	}
+
+	if opts.NewInformer == nil {
+		opts.NewInformer = toolscache.NewSharedIndexInformer
+	}
 	return opts, nil
 }
 

pkg/cache/cache.go

@@ -426,6 +426,9 @@ func TestDefaultOpts(t *testing.T) {
 				t.Fatal(err)
 			}
 
+			// We cannot reference kcp.NewInformerWithClusterIndexes due to import cycle.
+			defaulted.NewInformerFunc = nil
+
 			if diff := tc.verification(defaulted); diff != "" {
 				t.Errorf("expected config differs from actual: %s", diff)
 			}

pkg/cache/defaulting_test.go

@@ -31,6 +31,8 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/cache/internal"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/client/apiutil"
+
+	"github.com/kcp-dev/logicalcluster/v3"
 )
 
 var (
@@ -68,6 +70,7 @@ type informerCache struct {
 	scheme *runtime.Scheme
 	*internal.Informers
 	readerFailOnMissingInformer bool
+	clusterIndexes              bool
 }
 
 // Get implements Reader.
@@ -217,10 +220,10 @@ func (ic *informerCache) IndexField(ctx context.Context, obj client.Object, fiel
 	if err != nil {
 		return err
 	}
-	return indexByField(informer, field, extractValue)
+	return indexByField(informer, field, extractValue, ic.clusterIndexes)
 }
 
-func indexByField(informer Informer, field string, extractValue client.IndexerFunc) error {
+func indexByField(informer Informer, field string, extractValue client.IndexerFunc, clusterIndexes bool) error {
 	indexFunc := func(objRaw interface{}) ([]string, error) {
 		// TODO(directxman12): check if this is the correct type?
 		obj, isObj := objRaw.(client.Object)
@@ -233,6 +236,13 @@ func indexByField(informer Informer, field string, extractValue client.IndexerFu
 		}
 		ns := meta.GetNamespace()
 
+		keyFunc := internal.KeyToNamespacedKey
+		if clusterName := logicalcluster.From(obj); clusterIndexes && !clusterName.Empty() {
+			keyFunc = func(ns, val string) string {
+				return internal.KeyToClusteredKey(clusterName.String(), ns, val)
+			}
+		}
+
 		rawVals := extractValue(obj)
 		var vals []string
 		if ns == "" {
@@ -242,14 +252,15 @@ func indexByField(informer Informer, field string, extractValue client.IndexerFu
 			// if we need to add non-namespaced versions too, double the length
 			vals = make([]string, len(rawVals)*2)
 		}
+
 		for i, rawVal := range rawVals {
 			// save a namespaced variant, so that we can ask
 			// "what are all the object matching a given index *in a given namespace*"
-			vals[i] = internal.KeyToNamespacedKey(ns, rawVal)
+			vals[i] = keyFunc(ns, rawVal)
 			if ns != "" {
 				// if we have a namespace, also inject a special index key for listing
 				// regardless of the object namespace
-				vals[i+len(rawVals)] = internal.KeyToNamespacedKey("", rawVal)
+				vals[i+len(rawVals)] = keyFunc("", rawVal)
 			}
 		}
 

pkg/cache/informer_cache.go

@@ -21,6 +21,9 @@ import (
 	"fmt"
 	"reflect"
 
+	kcpcache "github.com/kcp-dev/apimachinery/v2/pkg/cache"
+	"github.com/kcp-dev/logicalcluster/v3"
+
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	apimeta "k8s.io/apimachinery/pkg/api/meta"
 	"k8s.io/apimachinery/pkg/fields"
@@ -31,6 +34,7 @@ import (
 
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/internal/field/selector"
+	"sigs.k8s.io/controller-runtime/pkg/kontext"
 )
 
 // CacheReader is a client.Reader.
@@ -54,12 +58,22 @@ type CacheReader struct {
 }
 
 // Get checks the indexer for the object and writes a copy of it if found.
-func (c *CacheReader) Get(_ context.Context, key client.ObjectKey, out client.Object, _ ...client.GetOption) error {
+func (c *CacheReader) Get(ctx context.Context, key client.ObjectKey, out client.Object, _ ...client.GetOption) error {
 	if c.scopeName == apimeta.RESTScopeNameRoot {
 		key.Namespace = ""
 	}
 	storeKey := objectKeyToStoreKey(key)
 
+	// create cluster-aware key for KCP
+	_, isClusterAware := c.indexer.GetIndexers()[kcpcache.ClusterAndNamespaceIndexName]
+	clusterName, _ := kontext.ClusterFrom(ctx)
+	if isClusterAware && clusterName.Empty() {
+		return fmt.Errorf("cluster-aware cache requires a cluster in context")
+	}
+	if isClusterAware {
+		storeKey = clusterName.String() + "|" + storeKey
+	}
+
 	// Lookup the object from the indexer cache
 	obj, exists, err := c.indexer.GetByKey(storeKey)
 	if err != nil {
@@ -105,7 +119,7 @@ func (c *CacheReader) Get(_ context.Context, key client.ObjectKey, out client.Ob
 }
 
 // List lists items out of the indexer and writes them to out.
-func (c *CacheReader) List(_ context.Context, out client.ObjectList, opts ...client.ListOption) error {
+func (c *CacheReader) List(ctx context.Context, out client.ObjectList, opts ...client.ListOption) error {
 	var objs []interface{}
 	var err error
 
@@ -116,6 +130,9 @@ func (c *CacheReader) List(_ context.Context, out client.ObjectList, opts ...cli
 		return fmt.Errorf("continue list option is not supported by the cache")
 	}
 
+	_, isClusterAware := c.indexer.GetIndexers()[kcpcache.ClusterAndNamespaceIndexName]
+	clusterName, _ := kontext.ClusterFrom(ctx)
+
 	switch {
 	case listOpts.FieldSelector != nil:
 		requiresExact := selector.RequiresExactMatch(listOpts.FieldSelector)
@@ -125,11 +142,19 @@ func (c *CacheReader) List(_ context.Context, out client.ObjectList, opts ...cli
 		// list all objects by the field selector. If this is namespaced and we have one, ask for the
 		// namespaced index key. Otherwise, ask for the non-namespaced variant by using the fake "all namespaces"
 		// namespace.
-		objs, err = byIndexes(c.indexer, listOpts.FieldSelector.Requirements(), listOpts.Namespace)
+		objs, err = byIndexes(c.indexer, listOpts.FieldSelector.Requirements(), clusterName, listOpts.Namespace)
 	case listOpts.Namespace != "":
-		objs, err = c.indexer.ByIndex(cache.NamespaceIndex, listOpts.Namespace)
+		if isClusterAware && !clusterName.Empty() {
+			objs, err = c.indexer.ByIndex(kcpcache.ClusterAndNamespaceIndexName, kcpcache.ClusterAndNamespaceIndexKey(clusterName, listOpts.Namespace))
+		} else {
+			objs, err = c.indexer.ByIndex(cache.NamespaceIndex, listOpts.Namespace)
+		}
 	default:
-		objs = c.indexer.List()
+		if isClusterAware && !clusterName.Empty() {
+			objs, err = c.indexer.ByIndex(kcpcache.ClusterIndexName, kcpcache.ClusterIndexKey(clusterName))
+		} else {
+			objs = c.indexer.List()
+		}
 	}
 	if err != nil {
 		return err
@@ -177,16 +202,22 @@ func (c *CacheReader) List(_ context.Context, out client.ObjectList, opts ...cli
 	return apimeta.SetList(out, runtimeObjs)
 }
 
-func byIndexes(indexer cache.Indexer, requires fields.Requirements, namespace string) ([]interface{}, error) {
+func byIndexes(indexer cache.Indexer, requires fields.Requirements, clusterName logicalcluster.Name, namespace string) ([]interface{}, error) {
 	var (
 		err  error
 		objs []interface{}
 		vals []string
 	)
 	indexers := indexer.GetIndexers()
+	_, isClusterAware := indexers[kcpcache.ClusterAndNamespaceIndexName]
 	for idx, req := range requires {
 		indexName := FieldIndexName(req.Field)
-		indexedValue := KeyToNamespacedKey(namespace, req.Value)
+		var indexedValue string
+		if isClusterAware {
+			indexedValue = KeyToClusteredKey(clusterName.String(), namespace, req.Value)
+		} else {
+			indexedValue = KeyToNamespacedKey(namespace, req.Value)
+		}
 		if idx == 0 {
 			// we use first require to get snapshot data
 			// TODO(halfcrazy): use complicated index when client-go provides byIndexes
@@ -253,3 +284,9 @@ func KeyToNamespacedKey(ns string, baseKey string) string {
 	}
 	return allNamespacesNamespace + "/" + baseKey
 }
+
+// KeyToClusteredKey prefixes the given index key with a cluster name
+// for use in field selector indexes.
+func KeyToClusteredKey(clusterName string, ns string, baseKey string) string {
+	return clusterName + "|" + KeyToNamespacedKey(ns, baseKey)
+}