[Java] Add support for transforming the schema to generate code for older versions.

Author: mikeb01

.gitignore

@@ -113,3 +113,5 @@ rust/Cargo.lock
 # Mac
 .DS_Store
 /sbe-tool/src/main/golang/uk_co_real_logic_sbe_ir_generated/
+
+/generated/

sbe-tool/src/main/java/uk/co/real_logic/sbe/SbeTool.java

@@ -18,12 +18,20 @@
 import org.agrona.DirectBuffer;
 import org.agrona.MutableDirectBuffer;
 import org.xml.sax.InputSource;
-import uk.co.real_logic.sbe.generation.*;
-import uk.co.real_logic.sbe.ir.*;
+import uk.co.real_logic.sbe.generation.CodeGenerator;
+import uk.co.real_logic.sbe.generation.TargetCodeGenerator;
+import uk.co.real_logic.sbe.generation.TargetCodeGeneratorLoader;
+import uk.co.real_logic.sbe.ir.Ir;
+import uk.co.real_logic.sbe.ir.IrDecoder;
+import uk.co.real_logic.sbe.ir.IrEncoder;
 import uk.co.real_logic.sbe.xml.*;
 
-import java.io.*;
-import java.nio.file.*;
+import java.io.BufferedInputStream;
+import java.io.File;
+import java.io.InputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
 
 /**
  * A tool for running the SBE parser, validator, and code generator.
@@ -176,6 +184,17 @@ public class SbeTool
      */
     public static final String DECODE_UNKNOWN_ENUM_VALUES = "sbe.decode.unknown.enum.values";
 
+    /**
+     * Configuration option used to manage sinceVersion based transformations. When set, parsed schemas will be
+     * transformed to discard messages and types higher than the specified version. This can be useful when needing
+     * to generate older versions of a schema to do version compatibility testing.
+     * <p>
+     * This field can contain a list of ordered pairs in the form:
+     * <code>((&lt;schema id&gt; | '*') ':' &lt;schema id&gt;)(',' ((&lt;schema id&gt; | '*') ':' &lt;schema id&gt;))*</code>.
+     * E.g. <code>123:5,*:6</code> which means transform schema with id = 123 to version 5, all others to version 6.
+     */
+    public static final String SCHEMA_TRANSFORM_VERSION = "sbe.schema.transform.version";
+
     /**
      * Main entry point for the SBE Tool.
      *
@@ -201,7 +220,10 @@ public static void main(final String[] args) throws Exception
                     validateAgainstSchema(fileName, xsdFilename);
                 }
 
-                ir = new IrGenerator().generate(parseSchema(fileName), System.getProperty(TARGET_NAMESPACE));
+                final MessageSchema schema = parseSchema(fileName);
+                final SchemaTransformer transformer = new SchemaTransformerFactory(
+                    System.getProperty(SCHEMA_TRANSFORM_VERSION));
+                ir = new IrGenerator().generate(transformer.transform(schema), System.getProperty(TARGET_NAMESPACE));
             }
             else if (fileName.endsWith(".sbeir"))
             {

sbe-tool/src/main/java/uk/co/real_logic/sbe/xml/IdKeyedSchemaTransformer.java

@@ -0,0 +1,42 @@
+/*
+ * Copyright 2013-2022 Real Logic Limited.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package uk.co.real_logic.sbe.xml;
+
+import java.util.Map;
+
+class IdKeyedSchemaTransformer implements SchemaTransformer
+{
+    private final Map<Integer, SchemaTransformer> transformerBySchemaId;
+    private final SchemaTransformer defaultTransformer;
+
+    IdKeyedSchemaTransformer(
+        final Map<Integer, SchemaTransformer> transformerBySchemaId,
+        final SchemaTransformer defaultTransformer)
+    {
+        this.transformerBySchemaId = transformerBySchemaId;
+        this.defaultTransformer = defaultTransformer;
+    }
+
+    public MessageSchema transform(final MessageSchema originalSchema)
+    {
+        return lookupTransformer(originalSchema.id()).transform(originalSchema);
+    }
+
+    SchemaTransformer lookupTransformer(final int schemaId)
+    {
+        return transformerBySchemaId.getOrDefault(schemaId, defaultTransformer);
+    }
+}

sbe-tool/src/main/java/uk/co/real_logic/sbe/xml/MessageSchema.java

@@ -15,8 +15,8 @@
  */
 package uk.co.real_logic.sbe.xml;
 
-import org.w3c.dom.Node;
 import org.agrona.Verify;
+import org.w3c.dom.Node;
 
 import java.nio.ByteOrder;
 import java.util.*;
@@ -62,6 +62,28 @@ public class MessageSchema
         ((CompositeType)typeByNameMap.get(this.headerType)).checkForWellFormedMessageHeader(schemaNode);
     }
 
+    MessageSchema(
+        final String packageName,
+        final String description,
+        final int id,
+        final int version,
+        final String semanticVersion,
+        final ByteOrder byteOrder,
+        final String headerType,
+        final Map<String, Type> typeByNameMap,
+        final Map<Long, Message> messageByIdMap)
+    {
+        this.packageName = packageName;
+        this.description = description;
+        this.id = id;
+        this.version = version;
+        this.semanticVersion = semanticVersion;
+        this.byteOrder = byteOrder;
+        this.headerType = headerType;
+        this.typeByNameMap = typeByNameMap;
+        this.messageByIdMap = messageByIdMap;
+    }
+
     /**
      * The Schema headerType for message headers. This should be a {@link CompositeType}.
      *
@@ -154,6 +176,16 @@ public Collection<Message> messages()
         return messageByIdMap.values();
     }
 
+    /**
+     * Get the {@link Collection} of {@link Type}s for this Schema.
+     *
+     * @return the {@link Collection} of {@link Type}s for this Schema.
+     */
+    public Collection<Type> types()
+    {
+        return typeByNameMap.values();
+    }
+
     /**
      * Return the byte order specified by the messageSchema
      *

sbe-tool/src/main/java/uk/co/real_logic/sbe/xml/SchemaTransformer.java

@@ -0,0 +1,35 @@
+/*
+ * Copyright 2013-2022 Real Logic Limited.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package uk.co.real_logic.sbe.xml;
+
+/**
+ * An abstraction that allows for general transformations of the IR.
+ */
+public interface SchemaTransformer
+{
+    /**
+     * Does an identity transform of the incoming schema.
+     */
+    SchemaTransformer IDENTITY_TRANSFORMER = originalSchema -> originalSchema;
+
+    /**
+     * Return a new MessageSchema that has been transformed from the original.
+     *
+     * @param originalSchema to be transformed
+     * @return resulting transformed schema.
+     */
+    MessageSchema transform(final MessageSchema originalSchema);
+}

sbe-tool/src/main/java/uk/co/real_logic/sbe/xml/SchemaTransformerFactory.java

@@ -0,0 +1,88 @@
+/*
+ * Copyright 2013-2022 Real Logic Limited.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package uk.co.real_logic.sbe.xml;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Entry point for schema transformations, will check all incoming configuration/properties and create the appropriate
+ * transformers as required.
+ */
+public class SchemaTransformerFactory implements SchemaTransformer
+{
+    private final SchemaTransformer transformer;
+
+    public SchemaTransformerFactory(final String schemaTransformConfig)
+    {
+        transformer = parse(schemaTransformConfig);
+    }
+
+    /**
+     * {@inheritDoc}
+     */
+    public MessageSchema transform(final MessageSchema originalSchema)
+    {
+        return transformer.transform(originalSchema);
+    }
+
+    static SchemaTransformer parse(final String configuration)
+    {
+        if (null == configuration || configuration.isEmpty())
+        {
+            return IDENTITY_TRANSFORMER;
+        }
+
+        final String[] split = configuration.split(",");
+        if (0 == split.length)
+        {
+            return IDENTITY_TRANSFORMER;
+        }
+
+        final HashMap<Integer, SchemaTransformer> transformerBySchemaId = new HashMap<>();
+        parseComponents(split, transformerBySchemaId);
+
+        SchemaTransformer defaultTransformer = transformerBySchemaId.remove(-1);
+        defaultTransformer = null != defaultTransformer ? defaultTransformer : IDENTITY_TRANSFORMER;
+
+        return transformerBySchemaId.isEmpty() ?
+            defaultTransformer : new IdKeyedSchemaTransformer(transformerBySchemaId, defaultTransformer);
+    }
+
+    private static void parseComponents(
+        final String[] configuration,
+        final Map<Integer, SchemaTransformer> transformerBySchemaId)
+    {
+        for (String field : configuration)
+        {
+            final String[] fieldParts = field.split(":");
+
+            if (2 != fieldParts.length)
+            {
+                throw new IllegalArgumentException("version transformation property part '" + field + "' is invalid");
+            }
+
+            final int schemaId = "*".equals(fieldParts[0]) ? -1 : Integer.parseInt(fieldParts[0].trim());
+            final int sinceVersion = Integer.parseInt(fieldParts[1].trim());
+            transformerBySchemaId.put(schemaId, new SinceVersionSchemaTransformer(sinceVersion));
+        }
+    }
+
+    SchemaTransformer delegate()
+    {
+        return transformer;
+    }
+}

sbe-tool/src/main/java/uk/co/real_logic/sbe/xml/SinceVersionSchemaTransformer.java

@@ -0,0 +1,70 @@
+/*
+ * Copyright 2013-2022 Real Logic Limited.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package uk.co.real_logic.sbe.xml;
+
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
+
+class SinceVersionSchemaTransformer implements SchemaTransformer
+{
+    private final int sinceVersion;
+
+    public SinceVersionSchemaTransformer(final int sinceVersion)
+    {
+        this.sinceVersion = sinceVersion;
+    }
+
+    public MessageSchema transform(final MessageSchema originalSchema)
+    {
+        final Collection<Type> types = originalSchema.types();
+        final Map<String, Type> newTypes = new HashMap<>();
+
+        for (Type type : types)
+        {
+            if (type.sinceVersion() <= this.sinceVersion)
+            {
+                newTypes.put(type.name(), type);
+            }
+        }
+
+        final Collection<Message> messages = originalSchema.messages();
+        final Map<Long, Message> newMessages = new HashMap<>();
+        for (Message message : messages)
+        {
+            if (message.sinceVersion() <= this.sinceVersion)
+            {
+                newMessages.put((long)message.id(), message);
+            }
+        }
+
+        return new MessageSchema(
+            originalSchema.packageName(),
+            originalSchema.description(),
+            originalSchema.id(),
+            sinceVersion,
+            originalSchema.semanticVersion(),
+            originalSchema.byteOrder(),
+            originalSchema.messageHeader().name(),
+            newTypes,
+            newMessages);
+    }
+
+    int sinceVersion()
+    {
+        return sinceVersion;
+    }
+}