Allow to remap key name when mirroring secret

sbenoistmics · sbenoistmics · commit 31bfc6822eee · 2025-02-24T09:13:20.000+01:00
Use reflector.v1.k8s.emberstack.com/reflection-key-mapping
and reflector.v1.k8s.emberstack.com/reflection-auto-key-mapping
to do the renaming

See README.md for details
diff --git a/README.md b/README.md
@@ -79,6 +79,7 @@ $ kubectl -n kube-system apply -f https://github.com/emberstack/kubernetes-refle
   Reflector can create mirrors with the same name in other namespaces automatically. The following annotations control if and how the mirrors are created:
   - Add `reflector.v1.k8s.emberstack.com/reflection-auto-enabled: "true"` to the resource annotations to automatically create mirrors in other namespaces. Note: Requires `reflector.v1.k8s.emberstack.com/reflection-allowed` to be `true` since mirrors need to able to reflect the source.
   - Add `reflector.v1.k8s.emberstack.com/reflection-auto-namespaces: "<list>"` to the resource annotations specify in which namespaces to automatically create mirrors. Note: If this annotation is omitted or is empty, all namespaces are allowed. Namespaces in this list will also be checked by `reflector.v1.k8s.emberstack.com/reflection-allowed-namespaces` since mirrors need to be in namespaces from where reflection is permitted.
+  - Optionally add `reflector.v1.k8s.emberstack.com/reflection-auto-key-mapping: "<list>"` to the resource annotations specify in the key mapping to use when automatically create mirrors. Expected format is a list of comma separated `src_key:dst_key`. 
 
   > Important: If the `source` is deleted, automatic mirrors are deleted. Also if either reflection or automirroring is turned off or the automatic mirror's namespace is no longer a valid match for the allowed namespaces, the automatic mirror is deleted.
 
@@ -109,10 +110,29 @@ $ kubectl -n kube-system apply -f https://github.com/emberstack/kubernetes-refle
   data:
     ...
   ```
+
+  Example source secret with key mapping:
+   ```yaml
+  apiVersion: v1
+  kind: Secret
+  metadata:
+    name: source-secret
+    annotations:
+      reflector.v1.k8s.emberstack.com/reflection-allowed: "true"
+      reflector.v1.k8s.emberstack.com/reflection-allowed-namespaces: "namespace-1,namespace-2,namespace-[0-9]*"
+      reflector.v1.k8s.emberstack.com/reflection-auto-enabled: "true"
+      reflector.v1.k8s.emberstack.com/reflection-auto-namespaces: "namespace-1"
+      reflector.v1.k8s.emberstack.com/reflection-auto-key-mapping: "user:username,pass:password"
+  data:
+    user: YWRtaW4K
+    pass: YWRtaW4K
+    ...
+  ```
   
 ### 2. Annotate the mirror secret or configmap
 
   - Add `reflector.v1.k8s.emberstack.com/reflects: "<source namespace>/<source name>"` to the mirror object. The value of the annotation is the full name of the source object in `namespace/name` format.
+  - Optionally add `reflector.v1.k8s.emberstack.com/reflection-key-mapping: "<list>"` to the resource annotations specify in the key mapping to use when mirroring. Expected format is a list of comma separated `src_key:dst_key`. All omitted source key will be copied as is. No warning will be issued if the source key does not exist.
 
   > Note: Add `reflector.v1.k8s.emberstack.com/reflected-version: ""` to the resource annotations when doing any manual changes to the mirror (for example when deploying with `helm` or re-applying the deployment script). This will reset the reflected version of the mirror.
   
@@ -140,6 +160,19 @@ $ kubectl -n kube-system apply -f https://github.com/emberstack/kubernetes-refle
     ...
   ```
 
+  Example mirror secret with key mapping :
+   ```yaml
+  apiVersion: v1
+  kind: Secret
+  metadata:
+    name: mirror-secret
+    annotations:
+      reflector.v1.k8s.emberstack.com/reflects: "default/source-secret"
+      reflector.v1.k8s.emberstack.com/reflection-key-mapping: "user:username,pass:password"
+  data:
+    ...
+  ```
+
 ### 3. Done!
   Reflector will monitor any changes done to the source objects and copy the following fields:
   - `data` for secrets
diff --git a/src/ES.Kubernetes.Reflector/Core/ConfigMapMirror.cs b/src/ES.Kubernetes.Reflector/Core/ConfigMapMirror.cs
@@ -23,10 +23,10 @@ protected override Task OnResourceApplyPatch(V1Patch patch, KubeRef refId)
         return Client.CoreV1.PatchNamespacedConfigMapAsync(patch, refId.Name, refId.Namespace);
     }
 
-    protected override Task OnResourceConfigurePatch(V1ConfigMap source, JsonPatchDocument<V1ConfigMap> patchDoc)
+    protected override Task OnResourceConfigurePatch(V1ConfigMap source, JsonPatchDocument<V1ConfigMap> patchDoc, Dictionary<string, string>? mapping)
     {
-        patchDoc.Replace(e => e.Data, source.Data);
-        patchDoc.Replace(e => e.BinaryData, source.BinaryData);
+        patchDoc.Replace(e => e.Data, MappedData(source.Data, mapping));
+        patchDoc.Replace(e => e.BinaryData, MappedData(source.BinaryData, mapping));
         return Task.CompletedTask;
     }
 
@@ -35,14 +35,14 @@ protected override Task OnResourceCreate(V1ConfigMap item, string ns)
         return Client.CoreV1.CreateNamespacedConfigMapAsync(item, ns);
     }
 
-    protected override Task<V1ConfigMap> OnResourceClone(V1ConfigMap sourceResource)
+    protected override Task<V1ConfigMap> OnResourceClone(V1ConfigMap sourceResource, Dictionary<string, string>? mapping)
     {
         return Task.FromResult(new V1ConfigMap
         {
             ApiVersion = sourceResource.ApiVersion,
             Kind = sourceResource.Kind,
-            Data = sourceResource.Data,
-            BinaryData = sourceResource.BinaryData
+            Data =  MappedData(sourceResource.Data, mapping),
+            BinaryData = MappedData(sourceResource.BinaryData, mapping)
         });
     }
 
diff --git a/src/ES.Kubernetes.Reflector/Core/Mirroring/Constants/Annotations.cs b/src/ES.Kubernetes.Reflector/Core/Mirroring/Constants/Annotations.cs
@@ -10,6 +10,8 @@ public static class Reflection
         public static string AllowedNamespaces => $"{Prefix}/reflection-allowed-namespaces";
         public static string AutoEnabled => $"{Prefix}/reflection-auto-enabled";
         public static string AutoNamespaces => $"{Prefix}/reflection-auto-namespaces";
+        public static string KeyMapping => $"{Prefix}/reflection-key-mapping";
+        public static string AutoKeyMapping => $"{Prefix}/reflection-auto-key-mapping";
         public static string Reflects => $"{Prefix}/reflects";
 
 
diff --git a/src/ES.Kubernetes.Reflector/Core/Mirroring/Extensions/ReflectorExtensions.cs b/src/ES.Kubernetes.Reflector/Core/Mirroring/Extensions/ReflectorExtensions.cs
@@ -35,6 +35,16 @@ public static ReflectorProperties GetReflectionProperties(this V1ObjectMeta meta
                 ? autoNamespaces ?? string.Empty
                 : string.Empty,
 
+            KeyMapping = metadata.SafeAnnotations()
+                .TryGet(Annotations.Reflection.KeyMapping, out string? keyMapping)
+                ? keyMapping ?? string.Empty
+                : string.Empty,
+
+            AutoKeyMapping = metadata.SafeAnnotations()
+                .TryGet(Annotations.Reflection.AutoKeyMapping, out string? autoKeyMapping)
+                ? autoKeyMapping ?? string.Empty
+                : string.Empty,
+
             Reflects = metadata.SafeAnnotations()
                 .TryGet(Annotations.Reflection.Reflects, out string? metaReflects)
                 ? string.IsNullOrWhiteSpace(metaReflects) ? KubeRef.Empty :
diff --git a/src/ES.Kubernetes.Reflector/Core/Mirroring/ReflectorProperties.cs b/src/ES.Kubernetes.Reflector/Core/Mirroring/ReflectorProperties.cs
@@ -9,6 +9,8 @@ public class ReflectorProperties
     public bool AutoEnabled { get; set; }
     public string AutoNamespaces { get; set; } = string.Empty;
     public KubeRef Reflects { get; set; } = KubeRef.Empty;
+    public string KeyMapping { get; set; } = string.Empty;
+    public string AutoKeyMapping { get; set; } = string.Empty;
 
     public string Version { get; set; } = string.Empty;
 
diff --git a/src/ES.Kubernetes.Reflector/Core/Mirroring/ResourceMirror.cs b/src/ES.Kubernetes.Reflector/Core/Mirroring/ResourceMirror.cs
@@ -149,6 +149,19 @@ public async Task Handle(WatcherEvent notification, CancellationToken cancellati
         }
     }
 
+    protected IDictionary<string, T> MappedData<T>(IDictionary<string, T>? data, Dictionary<string, string>? mapping = null)
+    {
+        mapping ??= new Dictionary<string, string>();
+        IDictionary<string, T> newData = new Dictionary<string, T>();
+        if (data != null)
+        {
+            foreach (var (key, value) in data)
+            {
+                newData.Add(mapping.GetValueOrDefault(key, key), value);
+            }
+        }
+        return newData;
+    }
 
     private async Task HandleUpsert(TResource resource, WatchEventType eventType, CancellationToken cancellationToken)
     {
@@ -428,25 +441,38 @@ private async Task ResourceReflect(KubeRef sourceId, KubeRef targetId, TResource
             [Annotations.Reflection.MetaReflectedVersion] = source.Metadata.ResourceVersion,
             [Annotations.Reflection.MetaReflectedAt] = JsonConvert.SerializeObject(DateTimeOffset.UtcNow)
         };
+        patchAnnotations[Annotations.Reflection.KeyMapping] = "";
+        if (autoReflection)
+        {
+            patchAnnotations[Annotations.Reflection.KeyMapping] = autoReflection ? source.GetReflectionProperties().AutoKeyMapping : string.Empty;
+        } else if (targetResource is not null)
+        {
+            patchAnnotations[Annotations.Reflection.KeyMapping] =
+                targetResource.Metadata.Annotations.TryGetValue(Annotations.Reflection.KeyMapping, out var keyMapping) ? keyMapping : string.Empty;
+        }
 
+        Dictionary<string, string> mapping = new Dictionary<string, string>();
+        try
+        {
+            mapping = Mapping(patchAnnotations[Annotations.Reflection.KeyMapping]);
+        }
+        catch (FormatException e)
+        {
+            Logger.LogError(e, e.Message);
+        }
 
         try
         {
             if (targetResource is null)
             {
-                var newResource = await OnResourceClone(source);
+                var newResource = await OnResourceClone(source, mapping);
                 newResource.Metadata ??= new V1ObjectMeta();
                 newResource.Metadata.Name = targetId.Name;
                 newResource.Metadata.NamespaceProperty = targetId.Namespace;
                 newResource.Metadata.Annotations ??= new Dictionary<string, string>();
                 var newResourceAnnotations = newResource.Metadata.Annotations;
                 foreach (var patchAnnotation in patchAnnotations)
                     newResourceAnnotations[patchAnnotation.Key] = patchAnnotation.Value;
-                newResourceAnnotations[Annotations.Reflection.MetaAutoReflects] = autoReflection.ToString();
-                newResourceAnnotations[Annotations.Reflection.Reflects] = sourceId.ToString();
-                newResourceAnnotations[Annotations.Reflection.MetaReflectedVersion] = source.Metadata.ResourceVersion;
-                newResourceAnnotations[Annotations.Reflection.MetaReflectedAt] =
-                    JsonConvert.SerializeObject(DateTimeOffset.UtcNow);
 
                 try
                 {
@@ -476,7 +502,7 @@ private async Task ResourceReflect(KubeRef sourceId, KubeRef targetId, TResource
                 annotations[patchAnnotation.Key] = patchAnnotation.Value;
             patchDoc.Replace(e => e.Metadata.Annotations, annotations);
 
-            await OnResourceConfigurePatch(source, patchDoc);
+            await OnResourceConfigurePatch(source, patchDoc, mapping);
 
             var patch = JsonConvert.SerializeObject(patchDoc, Formatting.Indented);
             await OnResourceApplyPatch(new V1Patch(patch, V1Patch.PatchType.JsonPatch), targetId);
@@ -488,11 +514,31 @@ private async Task ResourceReflect(KubeRef sourceId, KubeRef targetId, TResource
         }
     }
 
+    private Dictionary<string, string> Mapping(string? keyMapping)
+    {
+        var mappings = new Dictionary<string, string>();
+        if (string.IsNullOrEmpty(keyMapping)) return mappings;
+
+        foreach (var definition in keyMapping.Split(","))
+        {
+            var definitionDetail = definition.Split(":");
+            if (definitionDetail.Length == 2)
+            {
+                mappings.Add(definitionDetail[0], definitionDetail[1]);
+            }
+            else
+            {
+                throw new FormatException("Invalid key mapping, format is src_key:dst_key,src_other:dst_other. Received: " + definition);
+            }
+        }
+        return mappings;
+    }
+
 
     protected abstract Task OnResourceApplyPatch(V1Patch source, KubeRef refId);
-    protected abstract Task OnResourceConfigurePatch(TResource source, JsonPatchDocument<TResource> patchDoc);
+    protected abstract Task OnResourceConfigurePatch(TResource source, JsonPatchDocument<TResource> patchDoc, Dictionary<string, string> mapping);
     protected abstract Task OnResourceCreate(TResource item, string ns);
-    protected abstract Task<TResource> OnResourceClone(TResource sourceResource);
+    protected abstract Task<TResource> OnResourceClone(TResource sourceResource, Dictionary<string, string> mapping);
     protected abstract Task OnResourceDelete(KubeRef resourceId);
 
 
diff --git a/src/ES.Kubernetes.Reflector/Core/SecretMirror.cs b/src/ES.Kubernetes.Reflector/Core/SecretMirror.cs
@@ -1,4 +1,5 @@
 ﻿using ES.Kubernetes.Reflector.Core.Mirroring;
+using ES.Kubernetes.Reflector.Core.Mirroring.Extensions;
 using ES.Kubernetes.Reflector.Core.Resources;
 using k8s;
 using k8s.Models;
@@ -23,9 +24,9 @@ protected override Task OnResourceApplyPatch(V1Patch patch, KubeRef refId)
         return Client.CoreV1.PatchNamespacedSecretWithHttpMessagesAsync(patch, refId.Name, refId.Namespace);
     }
 
-    protected override Task OnResourceConfigurePatch(V1Secret source, JsonPatchDocument<V1Secret> patchDoc)
+    protected override Task OnResourceConfigurePatch(V1Secret source, JsonPatchDocument<V1Secret> patchDoc, Dictionary<string, string> mapping)
     {
-        patchDoc.Replace(e => e.Data, source.Data);
+        patchDoc.Replace(e => e.Data, MappedData(source.Data, mapping));
         return Task.CompletedTask;
     }
 
@@ -34,14 +35,14 @@ protected override Task OnResourceCreate(V1Secret item, string ns)
         return Client.CoreV1.CreateNamespacedSecretAsync(item, ns);
     }
 
-    protected override Task<V1Secret> OnResourceClone(V1Secret sourceResource)
+    protected override Task<V1Secret> OnResourceClone(V1Secret sourceResource, Dictionary<string, string> mapping) 
     {
         return Task.FromResult(new V1Secret
         {
             ApiVersion = sourceResource.ApiVersion,
             Kind = sourceResource.Kind,
             Type = sourceResource.Type,
-            Data = sourceResource.Data
+            Data = MappedData(sourceResource.Data, mapping)
         });
     }
 

Original file line number	Diff line number	Diff line change
`@@ -23,10 +23,10 @@ protected override Task OnResourceApplyPatch(V1Patch patch, KubeRef refId)`
`23`	`23`	`return Client.CoreV1.PatchNamespacedConfigMapAsync(patch, refId.Name, refId.Namespace);`
`24`	`24`	`}`
`25`	`25`
`26`		`- protected override Task OnResourceConfigurePatch(V1ConfigMap source, JsonPatchDocument<V1ConfigMap> patchDoc)`
	`26`	`+ protected override Task OnResourceConfigurePatch(V1ConfigMap source, JsonPatchDocument<V1ConfigMap> patchDoc, Dictionary<string, string>? mapping)`
`27`	`27`	`{`
`28`		`- patchDoc.Replace(e => e.Data, source.Data);`
`29`		`- patchDoc.Replace(e => e.BinaryData, source.BinaryData);`
	`28`	`+ patchDoc.Replace(e => e.Data, MappedData(source.Data, mapping));`
	`29`	`+ patchDoc.Replace(e => e.BinaryData, MappedData(source.BinaryData, mapping));`
`30`	`30`	`return Task.CompletedTask;`
`31`	`31`	`}`
`32`	`32`
`@@ -35,14 +35,14 @@ protected override Task OnResourceCreate(V1ConfigMap item, string ns)`
`35`	`35`	`return Client.CoreV1.CreateNamespacedConfigMapAsync(item, ns);`
`36`	`36`	`}`
`37`	`37`
`38`		`- protected override Task<V1ConfigMap> OnResourceClone(V1ConfigMap sourceResource)`
	`38`	`+ protected override Task<V1ConfigMap> OnResourceClone(V1ConfigMap sourceResource, Dictionary<string, string>? mapping)`
`39`	`39`	`{`
`40`	`40`	`return Task.FromResult(new V1ConfigMap`
`41`	`41`	`{`
`42`	`42`	`ApiVersion = sourceResource.ApiVersion,`
`43`	`43`	`Kind = sourceResource.Kind,`
`44`		`- Data = sourceResource.Data,`
`45`		`- BinaryData = sourceResource.BinaryData`
	`44`	`+ Data = MappedData(sourceResource.Data, mapping),`
	`45`	`+ BinaryData = MappedData(sourceResource.BinaryData, mapping)`
`46`	`46`	`});`
`47`	`47`	`}`
`48`	`48`
Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,5 @@`
`1`	`1`	`using ES.Kubernetes.Reflector.Core.Mirroring;`
	`2`	`+using ES.Kubernetes.Reflector.Core.Mirroring.Extensions;`
`2`	`3`	`using ES.Kubernetes.Reflector.Core.Resources;`
`3`	`4`	`using k8s;`
`4`	`5`	`using k8s.Models;`
`@@ -23,9 +24,9 @@ protected override Task OnResourceApplyPatch(V1Patch patch, KubeRef refId)`
`23`	`24`	`return Client.CoreV1.PatchNamespacedSecretWithHttpMessagesAsync(patch, refId.Name, refId.Namespace);`
`24`	`25`	`}`
`25`	`26`
`26`		`- protected override Task OnResourceConfigurePatch(V1Secret source, JsonPatchDocument<V1Secret> patchDoc)`
	`27`	`+ protected override Task OnResourceConfigurePatch(V1Secret source, JsonPatchDocument<V1Secret> patchDoc, Dictionary<string, string> mapping)`
`27`	`28`	`{`
`28`		`- patchDoc.Replace(e => e.Data, source.Data);`
	`29`	`+ patchDoc.Replace(e => e.Data, MappedData(source.Data, mapping));`
`29`	`30`	`return Task.CompletedTask;`
`30`	`31`	`}`
`31`	`32`
`@@ -34,14 +35,14 @@ protected override Task OnResourceCreate(V1Secret item, string ns)`
`34`	`35`	`return Client.CoreV1.CreateNamespacedSecretAsync(item, ns);`
`35`	`36`	`}`
`36`	`37`
`37`		`- protected override Task<V1Secret> OnResourceClone(V1Secret sourceResource)`
	`38`	`+ protected override Task<V1Secret> OnResourceClone(V1Secret sourceResource, Dictionary<string, string> mapping)`
`38`	`39`	`{`
`39`	`40`	`return Task.FromResult(new V1Secret`
`40`	`41`	`{`
`41`	`42`	`ApiVersion = sourceResource.ApiVersion,`
`42`	`43`	`Kind = sourceResource.Kind,`
`43`	`44`	`Type = sourceResource.Type,`
`44`		`- Data = sourceResource.Data`
	`45`	`+ Data = MappedData(sourceResource.Data, mapping)`
`45`	`46`	`});`
`46`	`47`	`}`
`47`	`48`