Adding support for the package attribute on the types element. (#904)

* Adding support for the package attribute on the types element.

* checkstyle.

* Revert accidental delete.

* Change parameter order for Token. Add Javadoc.

* Code style fixes.

* Make QualifiedYieldTest use the new constructor.

* Revert changes to test files (using custom version of StringWriterOutputManager). Should be replaced with a new version of Agrona.

* Use agrone 1.17.1

* Remove @OverRide

* make checkstyle happy.

* Correct inherit directive.

Author: ratcashdev

sbe-tool/src/main/java/uk/co/real_logic/sbe/SbeTool.java

@@ -68,6 +68,8 @@
  * <li><b>sbe.keyword.append.token</b>: Token to be appended to keywords.</li>
  * <li><b>sbe.decode.unknown.enum.values</b>: Support unknown decoded enum values. Defaults to false.</li>
  * <li><b>sbe.xinclude.aware</b>: Is XInclude supported for the schema. Defaults to false.</li>
+ * <li><b>sbe.type.package.override</b>: Is a package attribute for types element supported (only for JAVA). Defaults to
+ * false.</li>
  * </ul>
  */
 public class SbeTool
@@ -117,6 +119,12 @@ public class SbeTool
      */
     public static final String XINCLUDE_AWARE = "sbe.xinclude.aware";
 
+    /**
+     * Boolean system property to control the support of package names in {@code <types>} elements.
+     * Part of SBE v2-rc2. Defaults to false.
+     */
+    public static final String TYPE_PACKAGE_OVERRIDE = "sbe.type.package.override";
+
     /**
      * Target language for generated code.
      */

sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/TargetCodeGeneratorLoader.java

@@ -15,7 +15,7 @@
  */
 package uk.co.real_logic.sbe.generation;
 
-import org.agrona.generation.PackageOutputManager;
+import uk.co.real_logic.sbe.generation.java.JavaOutputManager;
 import uk.co.real_logic.sbe.generation.c.CGenerator;
 import uk.co.real_logic.sbe.generation.c.COutputManager;
 import uk.co.real_logic.sbe.generation.cpp.CppGenerator;
@@ -52,7 +52,8 @@ public CodeGenerator newInstance(final Ir ir, final String outputDir)
                 "true".equals(System.getProperty(JAVA_GROUP_ORDER_ANNOTATION)),
                 "true".equals(System.getProperty(JAVA_GENERATE_INTERFACES)),
                 "true".equals(System.getProperty(DECODE_UNKNOWN_ENUM_VALUES)),
-                new PackageOutputManager(outputDir, ir.applicableNamespace()));
+                "true".equals(System.getProperty(TYPE_PACKAGE_OVERRIDE)),
+                new JavaOutputManager(outputDir, ir.applicableNamespace()));
         }
     },
 

sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/java/JavaGenerator.java

@@ -19,7 +19,7 @@
 import org.agrona.MutableDirectBuffer;
 import org.agrona.Strings;
 import org.agrona.Verify;
-import org.agrona.generation.OutputManager;
+import org.agrona.generation.DynamicPackageOutputManager;
 import org.agrona.sbe.*;
 import uk.co.real_logic.sbe.PrimitiveType;
 import uk.co.real_logic.sbe.generation.CodeGenerator;
@@ -30,7 +30,9 @@
 import java.io.Writer;
 import java.util.ArrayList;
 import java.util.Formatter;
+import java.util.HashSet;
 import java.util.List;
+import java.util.Set;
 import java.util.function.Function;
 
 import static uk.co.real_logic.sbe.SbeTool.JAVA_INTERFACE_PACKAGE;
@@ -60,14 +62,40 @@ enum CodecType
     private static final String INDENT = "    ";
 
     private final Ir ir;
-    private final OutputManager outputManager;
+    private final DynamicPackageOutputManager outputManager;
     private final String fqMutableBuffer;
     private final String mutableBuffer;
     private final String fqReadOnlyBuffer;
     private final String readOnlyBuffer;
     private final boolean shouldGenerateGroupOrderAnnotation;
     private final boolean shouldGenerateInterfaces;
     private final boolean shouldDecodeUnknownEnumValues;
+    private final boolean shouldSupportTypePackages;
+    private final Set<String> typePackages = new HashSet<>();
+
+    /**
+     * Create a new Java language {@link CodeGenerator}. Generator support for types in their own package is disabled.
+     *
+     * @param ir                                 for the messages and types.
+     * @param mutableBuffer                      implementation used for mutating underlying buffers.
+     * @param readOnlyBuffer                     implementation used for reading underlying buffers.
+     * @param shouldGenerateGroupOrderAnnotation in the codecs.
+     * @param shouldGenerateInterfaces           for common methods.
+     * @param shouldDecodeUnknownEnumValues      generate support for unknown enum values when decoding.
+     * @param outputManager                      for generating the codecs to.
+     */
+    public JavaGenerator(
+        final Ir ir,
+        final String mutableBuffer,
+        final String readOnlyBuffer,
+        final boolean shouldGenerateGroupOrderAnnotation,
+        final boolean shouldGenerateInterfaces,
+        final boolean shouldDecodeUnknownEnumValues,
+        final DynamicPackageOutputManager outputManager)
+    {
+        this(ir, mutableBuffer, readOnlyBuffer, shouldGenerateGroupOrderAnnotation, shouldGenerateInterfaces,
+            shouldDecodeUnknownEnumValues, false, outputManager);
+    }
 
     /**
      * Create a new Java language {@link CodeGenerator}.
@@ -78,6 +106,7 @@ enum CodecType
      * @param shouldGenerateGroupOrderAnnotation in the codecs.
      * @param shouldGenerateInterfaces           for common methods.
      * @param shouldDecodeUnknownEnumValues      generate support for unknown enum values when decoding.
+     * @param shouldSupportTypePackages          generator support for types in their own package
      * @param outputManager                      for generating the codecs to.
      */
     public JavaGenerator(
@@ -87,12 +116,14 @@ public JavaGenerator(
         final boolean shouldGenerateGroupOrderAnnotation,
         final boolean shouldGenerateInterfaces,
         final boolean shouldDecodeUnknownEnumValues,
-        final OutputManager outputManager)
+        final boolean shouldSupportTypePackages,
+        final DynamicPackageOutputManager outputManager)
     {
         Verify.notNull(ir, "ir");
         Verify.notNull(outputManager, "outputManager");
 
         this.ir = ir;
+        this.shouldSupportTypePackages = shouldSupportTypePackages;
         this.outputManager = outputManager;
 
         this.mutableBuffer = validateBufferImplementation(mutableBuffer, MutableDirectBuffer.class);
@@ -144,11 +175,30 @@ public void generateTypeStubs() throws IOException
         }
     }
 
+    /**
+     * Register the the type's explicit package - if it's set and should be supported.
+     *
+     * @param token the 0-th token of the type
+     * @param ir the intermediate representation
+     * @return the overriden package name of the type if set and supported, or {@link Ir#applicableNamespace() }
+     */
+    private String registerTypePackage(final Token token, final Ir ir)
+    {
+        if (shouldSupportTypePackages && token.packageName() != null)
+        {
+            typePackages.add(token.packageName());
+            outputManager.setPackageName(token.packageName());
+            return token.packageName();
+        }
+        return ir.applicableNamespace();
+    }
+
     /**
      * {@inheritDoc}
      */
     public void generate() throws IOException
     {
+        typePackages.clear();
         generatePackageInfo();
         generateTypeStubs();
         generateMessageHeaderStub();
@@ -1188,6 +1238,7 @@ private void generateBitSet(final List<Token> tokens) throws IOException
         final List<Token> choiceList = tokens.subList(1, tokens.size() - 1);
         final String implementsString = implementsInterface(Flyweight.class.getSimpleName());
 
+        registerTypePackage(token, ir);
         try (Writer out = outputManager.createOutput(decoderName))
         {
             final Encoding encoding = token.encoding();
@@ -1208,6 +1259,7 @@ private void generateBitSet(final List<Token> tokens) throws IOException
             out.append("}\n");
         }
 
+        registerTypePackage(token, ir);
         try (Writer out = outputManager.createOutput(encoderName))
         {
             generateFixedFlyweightHeader(out, token, encoderName, implementsString, mutableBuffer, fqMutableBuffer);
@@ -1225,7 +1277,8 @@ private void generateFixedFlyweightHeader(
         final String buffer,
         final String fqBuffer) throws IOException
     {
-        out.append(generateFileHeader(ir.applicableNamespace(), fqBuffer));
+        final String packageName = registerTypePackage(token, ir);
+        out.append(generateFileHeader(packageName, fqBuffer));
         out.append(generateDeclaration(typeName, implementsString, token));
         out.append(generateFixedFlyweightCode(typeName, token.encodedLength(), buffer));
     }
@@ -1238,7 +1291,8 @@ private void generateCompositeFlyweightHeader(
         final String fqBuffer,
         final String implementsString) throws IOException
     {
-        out.append(generateFileHeader(ir.applicableNamespace(), fqBuffer));
+        final String packageName = registerTypePackage(token, ir);
+        out.append(generateFileHeader(packageName, fqBuffer));
         out.append(generateDeclaration(typeName, implementsString, token));
         out.append(generateFixedFlyweightCode(typeName, token.encodedLength(), buffer));
     }
@@ -1250,9 +1304,10 @@ private void generateEnum(final List<Token> tokens) throws IOException
         final Encoding encoding = enumToken.encoding();
         final String nullVal = encoding.applicableNullValue().toString();
 
+        final String packageName = registerTypePackage(enumToken, ir);
         try (Writer out = outputManager.createOutput(enumName))
         {
-            out.append(generateEnumFileHeader(ir.applicableNamespace()));
+            out.append(generateEnumFileHeader(packageName));
             out.append(generateEnumDeclaration(enumName, enumToken));
 
             final List<Token> valuesList = tokens.subList(1, tokens.size() - 1);
@@ -1272,6 +1327,7 @@ private void generateComposite(final List<Token> tokens) throws IOException
         final String decoderName = decoderName(compositeName);
         final String encoderName = encoderName(compositeName);
 
+        registerTypePackage(token, ir);
         try (Writer out = outputManager.createOutput(decoderName))
         {
             final String implementsString = implementsInterface(CompositeDecoderFlyweight.class.getSimpleName());
@@ -1320,6 +1376,7 @@ private void generateComposite(final List<Token> tokens) throws IOException
             out.append("}\n");
         }
 
+        registerTypePackage(token, ir);
         try (Writer out = outputManager.createOutput(encoderName))
         {
             final String implementsString = implementsInterface(CompositeEncoderFlyweight.class.getSimpleName());
@@ -1572,13 +1629,22 @@ private CharSequence generateFileHeader(final String packageName, final String f
     private CharSequence generateMainHeader(
         final String packageName, final CodecType codecType, final boolean hasVarData)
     {
+        final StringBuffer packageImports = new StringBuffer();
+        for (final String typePackage : typePackages)
+        {
+            packageImports.append("import ");
+            packageImports.append(typePackage);
+            packageImports.append(".*;\n");
+        }
+
         if (fqMutableBuffer.equals(fqReadOnlyBuffer))
         {
             return
                 "/* Generated SBE (Simple Binary Encoding) message codec. */\n" +
                 "package " + packageName + ";\n\n" +
                 "import " + fqMutableBuffer + ";\n" +
-                interfaceImportLine();
+                interfaceImportLine() +
+                packageImports.toString();
         }
         else
         {
@@ -1590,7 +1656,8 @@ private CharSequence generateMainHeader(
                 "package " + packageName + ";\n\n" +
                 (hasMutableBuffer ? "import " + fqMutableBuffer + ";\n" : "") +
                 (hasReadOnlyBuffer ? "import " + fqReadOnlyBuffer + ";\n" : "") +
-                interfaceImportLine();
+                interfaceImportLine() +
+                packageImports.toString();
         }
     }
 

sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/java/JavaOutputManager.java

@@ -0,0 +1,81 @@
+/*
+ * Copyright 2013-2022 Real Logic Limited.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package uk.co.real_logic.sbe.generation.java;
+
+import java.io.FilterWriter;
+import java.io.IOException;
+import java.io.Writer;
+import org.agrona.collections.Object2NullableObjectHashMap;
+import org.agrona.collections.Object2ObjectHashMap;
+import org.agrona.generation.DynamicPackageOutputManager;
+import org.agrona.generation.PackageOutputManager;
+
+/**
+ * Implementation of {@link DynamicPackageOutputManager} for Java.
+ */
+public class JavaOutputManager implements DynamicPackageOutputManager
+{
+    private final String baseDirName;
+    private final PackageOutputManager basePackageOutputManager;
+    private PackageOutputManager actingPackageOutputManager;
+    private final Object2ObjectHashMap<String, PackageOutputManager> outputManagerCache
+        = new Object2NullableObjectHashMap<>();
+
+    /**
+     * Constructor.
+     * @param baseDirName the target directory
+     * @param packageName the initial package name
+     */
+    public JavaOutputManager(final String baseDirName, final String packageName)
+    {
+        basePackageOutputManager = new PackageOutputManager(baseDirName, packageName);
+        actingPackageOutputManager = basePackageOutputManager;
+        this.baseDirName = baseDirName;
+    }
+
+    /**
+     * {@inheritDoc}
+     */
+    public void setPackageName(final String packageName)
+    {
+        actingPackageOutputManager = outputManagerCache.get(packageName);
+        if (actingPackageOutputManager == null)
+        {
+            actingPackageOutputManager = new PackageOutputManager(baseDirName, packageName);
+            outputManagerCache.put(packageName, actingPackageOutputManager);
+        }
+    }
+
+    private void resetPackage()
+    {
+        actingPackageOutputManager = basePackageOutputManager;
+    }
+
+    /**
+     * {@inheritDoc}
+     */
+    public Writer createOutput(final String name) throws IOException
+    {
+        return new FilterWriter(actingPackageOutputManager.createOutput(name))
+        {
+            public void close() throws IOException
+            {
+                super.close();
+                resetPackage();
+            }
+        };
+    }
+}