From 8e3fb5ddd16228a7bd51a5a5b1eff07016e0da6e Mon Sep 17 00:00:00 2001 From: Russell Gold Date: Thu, 10 Dec 2020 10:55:44 -0500 Subject: [PATCH 1/6] Create plugin for running bash unit tests --- .../kubernetes/json/mojo/FileSystem.java | 90 -- .../pom.xml | 4 +- .../kubernetes/json/mojo/ExternalSchema.java | 0 .../kubernetes/json/mojo/JsonSchemaMojo.java | 10 + .../oracle/kubernetes/json/mojo/Main.java | 0 .../oracle/kubernetes/json/mojo/MainImpl.java | 0 .../kubernetes/mojo/shunit2/AnsiUtils.java | 63 + .../mojo/shunit2/BashProcessBuilder.java | 60 + .../kubernetes/mojo/shunit2/ShUnit2Mojo.java | 261 ++++ .../kubernetes/mojo/shunit2/TestSuite.java | 191 +++ .../shunit2/TestSuiteFailedException.java | 11 + .../kubernetes/mojosupport/FileSystem.java | 90 ++ .../json/mojo/JsonSchemaMojoTest.java | 286 +--- .../oracle/kubernetes/json/mojo/TestMain.java | 0 .../mojo/shunit2/AnsiUtilsTest.java | 56 + .../mojo/shunit2/ShUnit2MojoTest.java | 450 ++++++ .../kubernetes/mojosupport/MojoTestBase.java | 349 +++++ .../mojosupport}/TestFileSystem.java | 83 +- operator/pom.xml | 13 +- .../src/test/resources/shunit2/2.1.8/shunit2 | 1343 +++++++++++++++++ pom.xml | 2 +- 21 files changed, 2977 insertions(+), 385 deletions(-) delete mode 100644 json-schema-maven-plugin/src/main/java/oracle/kubernetes/json/mojo/FileSystem.java rename {json-schema-maven-plugin => operator-build-maven-plugin}/pom.xml (96%) rename {json-schema-maven-plugin => operator-build-maven-plugin}/src/main/java/oracle/kubernetes/json/mojo/ExternalSchema.java (100%) rename {json-schema-maven-plugin => operator-build-maven-plugin}/src/main/java/oracle/kubernetes/json/mojo/JsonSchemaMojo.java (93%) rename {json-schema-maven-plugin => operator-build-maven-plugin}/src/main/java/oracle/kubernetes/json/mojo/Main.java (100%) rename {json-schema-maven-plugin => operator-build-maven-plugin}/src/main/java/oracle/kubernetes/json/mojo/MainImpl.java (100%) create mode 100644 operator-build-maven-plugin/src/main/java/oracle/kubernetes/mojo/shunit2/AnsiUtils.java create mode 100644 operator-build-maven-plugin/src/main/java/oracle/kubernetes/mojo/shunit2/BashProcessBuilder.java create mode 100644 operator-build-maven-plugin/src/main/java/oracle/kubernetes/mojo/shunit2/ShUnit2Mojo.java create mode 100644 operator-build-maven-plugin/src/main/java/oracle/kubernetes/mojo/shunit2/TestSuite.java create mode 100644 operator-build-maven-plugin/src/main/java/oracle/kubernetes/mojo/shunit2/TestSuiteFailedException.java create mode 100644 operator-build-maven-plugin/src/main/java/oracle/kubernetes/mojosupport/FileSystem.java rename {json-schema-maven-plugin => operator-build-maven-plugin}/src/test/java/oracle/kubernetes/json/mojo/JsonSchemaMojoTest.java (54%) rename {json-schema-maven-plugin => operator-build-maven-plugin}/src/test/java/oracle/kubernetes/json/mojo/TestMain.java (100%) create mode 100644 operator-build-maven-plugin/src/test/java/oracle/kubernetes/mojo/shunit2/AnsiUtilsTest.java create mode 100644 operator-build-maven-plugin/src/test/java/oracle/kubernetes/mojo/shunit2/ShUnit2MojoTest.java create mode 100644 operator-build-maven-plugin/src/test/java/oracle/kubernetes/mojosupport/MojoTestBase.java rename {json-schema-maven-plugin/src/test/java/oracle/kubernetes/json/mojo => operator-build-maven-plugin/src/test/java/oracle/kubernetes/mojosupport}/TestFileSystem.java (62%) create mode 100755 operator/src/test/resources/shunit2/2.1.8/shunit2 diff --git a/json-schema-maven-plugin/src/main/java/oracle/kubernetes/json/mojo/FileSystem.java b/json-schema-maven-plugin/src/main/java/oracle/kubernetes/json/mojo/FileSystem.java deleted file mode 100644 index 8a021ffbd7e..00000000000 --- a/json-schema-maven-plugin/src/main/java/oracle/kubernetes/json/mojo/FileSystem.java +++ /dev/null @@ -1,90 +0,0 @@ -// Copyright (c) 2018, 2020, Oracle Corporation and/or its affiliates. -// Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. - -package oracle.kubernetes.json.mojo; - -import java.io.File; -import java.io.FileReader; -import java.io.FileWriter; -import java.io.FilenameFilter; -import java.io.IOException; -import java.io.Reader; -import java.io.Writer; -import java.net.MalformedURLException; -import java.net.URL; - -abstract class FileSystem { - - static final FileSystem LIVE_FILE_SYSTEM = new LiveFileSystem(); - - abstract URL toUrl(File file) throws MalformedURLException; - - abstract File[] listFiles(File directory); - - abstract File[] listFiles(File directory, FilenameFilter filter); - - abstract boolean exists(File file); - - abstract boolean isDirectory(File file); - - abstract boolean isWritable(File directory); - - abstract void createDirectory(File directory); - - abstract Writer createWriter(File file) throws IOException; - - abstract Reader createReader(File file) throws IOException; - - abstract long getLastModified(File file); - - private static class LiveFileSystem extends FileSystem { - - @Override - URL toUrl(File file) throws MalformedURLException { - return file.toURI().toURL(); - } - - File[] listFiles(File directory) { - return directory.listFiles(); - } - - File[] listFiles(File directory, FilenameFilter filter) { - return directory.listFiles(filter); - } - - @Override - boolean exists(File file) { - return file.exists(); - } - - @Override - boolean isDirectory(File file) { - return file.isDirectory(); - } - - @Override - boolean isWritable(File directory) { - return directory.canWrite(); - } - - @Override - void createDirectory(File directory) { - directory.mkdirs(); - } - - @Override - Writer createWriter(File file) throws IOException { - return new FileWriter(file); - } - - @Override - Reader createReader(File file) throws IOException { - return new FileReader(file); - } - - @Override - long getLastModified(File file) { - return file.lastModified(); - } - } -} diff --git a/json-schema-maven-plugin/pom.xml b/operator-build-maven-plugin/pom.xml similarity index 96% rename from json-schema-maven-plugin/pom.xml rename to operator-build-maven-plugin/pom.xml index 4b4a955e309..44aebec84aa 100644 --- a/json-schema-maven-plugin/pom.xml +++ b/operator-build-maven-plugin/pom.xml @@ -10,9 +10,9 @@ 3.2.0 - jsonschema-maven-plugin + operator-build-maven-plugin maven-plugin - jsonschema-maven-plugin Maven Mojo + Operator Build Maven Plugin diff --git a/json-schema-maven-plugin/src/main/java/oracle/kubernetes/json/mojo/ExternalSchema.java b/operator-build-maven-plugin/src/main/java/oracle/kubernetes/json/mojo/ExternalSchema.java similarity index 100% rename from json-schema-maven-plugin/src/main/java/oracle/kubernetes/json/mojo/ExternalSchema.java rename to operator-build-maven-plugin/src/main/java/oracle/kubernetes/json/mojo/ExternalSchema.java diff --git a/json-schema-maven-plugin/src/main/java/oracle/kubernetes/json/mojo/JsonSchemaMojo.java b/operator-build-maven-plugin/src/main/java/oracle/kubernetes/json/mojo/JsonSchemaMojo.java similarity index 93% rename from json-schema-maven-plugin/src/main/java/oracle/kubernetes/json/mojo/JsonSchemaMojo.java rename to operator-build-maven-plugin/src/main/java/oracle/kubernetes/json/mojo/JsonSchemaMojo.java index 0d6838ead94..9c5ff8b618c 100644 --- a/json-schema-maven-plugin/src/main/java/oracle/kubernetes/json/mojo/JsonSchemaMojo.java +++ b/operator-build-maven-plugin/src/main/java/oracle/kubernetes/json/mojo/JsonSchemaMojo.java @@ -12,6 +12,7 @@ import java.util.Map; import java.util.Optional; +import oracle.kubernetes.mojosupport.FileSystem; import org.apache.maven.plugin.AbstractMojo; import org.apache.maven.plugin.MojoExecutionException; import org.apache.maven.plugins.annotations.LifecyclePhase; @@ -26,12 +27,21 @@ public class JsonSchemaMojo extends AbstractMojo { private static final String DOT = "\\."; + + @SuppressWarnings("FieldMayBeFinal") // must be non-final so unit tests can set it private static Main main = new MainImpl(); + + @SuppressWarnings("FieldMayBeFinal") // must be non-final so unit tests can set it private static FileSystem fileSystem = FileSystem.LIVE_FILE_SYSTEM; + + @SuppressWarnings("unused") // set by Maven @Parameter(defaultValue = "${project.compileClasspathElements}", readonly = true, required = true) private List compileClasspathElements; + + @SuppressWarnings("unused") // set by Maven @Parameter(defaultValue = "${project.build.outputDirectory}/schema") private String targetDir; + @Parameter private String kubernetesVersion; @Parameter private final List externalSchemas = Collections.emptyList(); @Parameter(required = true) diff --git a/json-schema-maven-plugin/src/main/java/oracle/kubernetes/json/mojo/Main.java b/operator-build-maven-plugin/src/main/java/oracle/kubernetes/json/mojo/Main.java similarity index 100% rename from json-schema-maven-plugin/src/main/java/oracle/kubernetes/json/mojo/Main.java rename to operator-build-maven-plugin/src/main/java/oracle/kubernetes/json/mojo/Main.java diff --git a/json-schema-maven-plugin/src/main/java/oracle/kubernetes/json/mojo/MainImpl.java b/operator-build-maven-plugin/src/main/java/oracle/kubernetes/json/mojo/MainImpl.java similarity index 100% rename from json-schema-maven-plugin/src/main/java/oracle/kubernetes/json/mojo/MainImpl.java rename to operator-build-maven-plugin/src/main/java/oracle/kubernetes/json/mojo/MainImpl.java diff --git a/operator-build-maven-plugin/src/main/java/oracle/kubernetes/mojo/shunit2/AnsiUtils.java b/operator-build-maven-plugin/src/main/java/oracle/kubernetes/mojo/shunit2/AnsiUtils.java new file mode 100644 index 00000000000..720192bcd56 --- /dev/null +++ b/operator-build-maven-plugin/src/main/java/oracle/kubernetes/mojo/shunit2/AnsiUtils.java @@ -0,0 +1,63 @@ +// Copyright (c) 2020, Oracle Corporation and/or its affiliates. +// Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. + +package oracle.kubernetes.mojo.shunit2; + +import java.util.Arrays; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Utilities related to ANSI-formatting of strings sent to a terminal. + */ +class AnsiUtils { + + private static final Pattern ANSI_ESCAPE_CHARS = Pattern.compile("(\\x9B|\\x1B\\[)[0-?]*[ -\\/]*[@-~]"); + + static String withoutAnsiEscapeChars(String input) { + final Matcher matcher = ANSI_ESCAPE_CHARS.matcher(input); + return matcher.replaceAll(""); + } + + public static AnsiFormatter createFormatter(Format... formats) { + return new AnsiFormatter(formats); + } + + static class AnsiFormatter { + + private final Format[] formats; + + AnsiFormatter(Format... formats) { + this.formats = formats; + } + + String format(String string) { + return startCodes() + string + endCodes(); + } + + private String startCodes() { + return formats.length == 0 ? "" : sequence(Arrays.stream(formats).map(Format::getFormat).toArray(String[]::new)); + } + + private String endCodes() { + return formats.length == 0 ? "" : sequence("0"); + } + + String sequence(String... formatCodes) { + return "\u001B[" + String.join(";", formatCodes) + "m"; + } + } + + static enum Format { + BOLD(1), RED_FG(31), BLUE_FG(34), GREEN_FG(32); + + private final String format; + Format(int format) { + this.format = Integer.toString(format); + } + + public String getFormat() { + return format; + } + } +} diff --git a/operator-build-maven-plugin/src/main/java/oracle/kubernetes/mojo/shunit2/BashProcessBuilder.java b/operator-build-maven-plugin/src/main/java/oracle/kubernetes/mojo/shunit2/BashProcessBuilder.java new file mode 100644 index 00000000000..22938f4b76e --- /dev/null +++ b/operator-build-maven-plugin/src/main/java/oracle/kubernetes/mojo/shunit2/BashProcessBuilder.java @@ -0,0 +1,60 @@ +// Copyright (c) 2020, Oracle Corporation and/or its affiliates. +// Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. + +package oracle.kubernetes.mojo.shunit2; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import java.util.function.BiFunction; +import javax.annotation.Nonnull; + +/** + * A wrapper for ProcessBuilder, in order to allow unit testing. + */ +class BashProcessBuilder { + + private final String commands; + private final Map environmentVariables = new HashMap<>(); + private final BiFunction,Process> processBiFunction; + + /** + * Constructs a builder. + * @param commands the commands to be issued to a new bash process + */ + BashProcessBuilder(String commands) { + this(BashProcessBuilder::createProcess, commands); + } + + BashProcessBuilder(BiFunction, Process> processBiFunction, String commands) { + this.commands = commands; + this.processBiFunction = processBiFunction; + } + + /** + * Updates the builder by adding an environment variable to be set in the process. + * @param name the environment variable name + * @param value the environment variable value + */ + void addEnvironmentVariable(String name, String value) { + environmentVariables.put(name, value); + } + + /** + * Starts the specified process and returns a Process object to control it. + */ + public Process build() { + return processBiFunction.apply(commands, environmentVariables); + } + + @Nonnull + protected static Process createProcess(String command, Map environmentVariables) { + try { + ProcessBuilder pb = new ProcessBuilder("bash", command); + pb.environment().putAll(environmentVariables); + return pb.start(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } +} diff --git a/operator-build-maven-plugin/src/main/java/oracle/kubernetes/mojo/shunit2/ShUnit2Mojo.java b/operator-build-maven-plugin/src/main/java/oracle/kubernetes/mojo/shunit2/ShUnit2Mojo.java new file mode 100644 index 00000000000..56fa0f99634 --- /dev/null +++ b/operator-build-maven-plugin/src/main/java/oracle/kubernetes/mojo/shunit2/ShUnit2Mojo.java @@ -0,0 +1,261 @@ +// Copyright (c) 2020, Oracle Corporation and/or its affiliates. +// Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. + +package oracle.kubernetes.mojo.shunit2; + +import java.io.BufferedReader; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.function.Function; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import javax.annotation.Nonnull; + +import oracle.kubernetes.mojosupport.FileSystem; +import org.apache.maven.plugin.AbstractMojo; +import org.apache.maven.plugin.MojoExecutionException; +import org.apache.maven.plugin.MojoFailureException; +import org.apache.maven.plugins.annotations.LifecyclePhase; +import org.apache.maven.plugins.annotations.Mojo; +import org.apache.maven.plugins.annotations.Parameter; +import org.apache.maven.plugins.annotations.ResolutionScope; + +import static oracle.kubernetes.mojo.shunit2.AnsiUtils.Format.BLUE_FG; +import static oracle.kubernetes.mojo.shunit2.AnsiUtils.Format.BOLD; + +/* + Will run all shunit2 tests in the testSourceDirectory named "test*" or "*test" with environment variables: + SHUNIT2_PATH - pointing to the shunit2 script to include + SCRIPTPATH - pointing to the sourceDirectory that contains scripts to test + + This mojo will run as part of the build in which it is configured. To run from the command line, + + 1. add the following to ~/.m2/settings.xml: + + + oracle.kubernetes + + + 2. execute: + + mvn operator-build:shunit2 -pl + + where is the name of the subdirectory containing the module on which the plugin is to be run. + */ + + +@Mojo( + name = "shunit2", + defaultPhase = LifecyclePhase.TEST, + requiresDependencyResolution = ResolutionScope.NONE) +public class ShUnit2Mojo extends AbstractMojo { + + static final String SHUNIT2_PATH = "SHUNIT2_PATH"; + static final String SCRIPTPATH = "SCRIPTPATH"; + + private static final String BASH_TOO_OLD_MESSAGE = + "Bash %d.%d is too old to run unit tests." + + " Install a later version with Homebrew: " + + AnsiUtils.createFormatter(BOLD, BLUE_FG).format("brew install bash") + + "."; + private static final Pattern DIGITS = Pattern.compile("\\d+"); + private static final Pattern VERSION_PATTERN = Pattern.compile("GNU bash, version (\\d+)\\.(\\d+)"); + private static final String SHUNIT2_SCRIPT_ROOT = "shunit2"; + private static final int MINIMUM_SUPPORTED_BASH_MAJOR_VERSION = 4; + private static final int MINIMUM_SUPPORTED_BASH_MINOR_VERSION = 2; + + @SuppressWarnings("FieldMayBeFinal") // not final to allow unit test to change it + private static FileSystem fileSystem = FileSystem.LIVE_FILE_SYSTEM; + + @SuppressWarnings("FieldMayBeFinal") // not final to allow unit test to change it + private static Function builderFunction = BashProcessBuilder::new; + + /** The directory into which the mojo will copy the shunit2 script. */ + @SuppressWarnings("unused") // set by Maven + @Parameter(defaultValue = "${project.build.testOutputDirectory}", readonly = true, required = true) + private File outputDirectory; + + /** The directory containing the scripts to be tested. Test scripts will find SOURCE_DIR set with this value. */ + @SuppressWarnings("unused") // set by Maven + @Parameter(defaultValue = "${project.basedir}/src/main/sh", readonly = true, required = true) + private File sourceDirectory; + + /** The director this mojo will search for tests to execute. */ + @Parameter(defaultValue = "${project.basedir}/src/test/sh", readonly = true, required = true) + @SuppressWarnings("unused") // set by Maven + private File testSourceDirectory; + + private Map environmentVariables; + private List testSuites; + + @Override + public void execute() throws MojoFailureException, MojoExecutionException { + if (isEnvironmentNotSupported()) { + return; + } + + environmentVariables = getEnvironmentVariables(); + testSuites = Arrays.stream(getScriptPaths()).map(this::createTestSuite).collect(Collectors.toList()); + + testSuites.forEach(TestSuite::run); + if ((totalNumFailures() + totalNumErrors()) != 0) { + throw new MojoFailureException(String.format("%d failures, %d errors", totalNumFailures(), totalNumErrors())); + } + } + + private boolean isEnvironmentNotSupported() throws MojoExecutionException { + return isMacOSX() && isBashTooOld(getBashVersion()); + } + + private boolean isMacOSX() { + return System.getProperty("os.name").startsWith("Mac "); + } + + private boolean isBashTooOld(int[] bashVersion) { + if (isBashVersionTooOld(bashVersion)) { + getLog().warn(createVersionTooOldMessage(bashVersion)); + return true; + } + + return false; + } + + private boolean isBashVersionTooOld(int[] version) { + return version[0] < MINIMUM_SUPPORTED_BASH_MAJOR_VERSION + || (version[0] == MINIMUM_SUPPORTED_BASH_MAJOR_VERSION && version[1] < MINIMUM_SUPPORTED_BASH_MINOR_VERSION); + } + + private int[] getBashVersion() throws MojoExecutionException { + try { + final Process process = builderFunction.apply("-version").build(); + process.waitFor(); + return extractVersionFromResponse(process); + } catch (InterruptedException | IOException e) { + throw new MojoExecutionException("Unable to check bash version", e); + } + } + + private int[] extractVersionFromResponse(Process process) throws IOException { + try (final InputStream inputStream = process.getInputStream()) { + return new BufferedReader(new InputStreamReader(inputStream)).lines() + .map(this::parseVersionNumber) + .filter(Objects::nonNull) + .findAny() + .orElseThrow(() -> new IOException("No bash version detected")); + } + } + + private int[] parseVersionNumber(String message) { + final Matcher matcher = VERSION_PATTERN.matcher(message); + if (!matcher.find()) { + return null; + } else { + return new int[] {Integer.parseInt(matcher.group(1)), Integer.parseInt(matcher.group(2))}; + } + } + + private String createVersionTooOldMessage(int[] version) { + return String.format(BASH_TOO_OLD_MESSAGE, version[0], version[1]); + } + + + private int totalNumFailures() { + return testSuites.stream().mapToInt(TestSuite::numFailures).sum(); + } + + private int totalNumErrors() { + return testSuites.stream().mapToInt(TestSuite::getNumErrors).sum(); + } + + private TestSuite createTestSuite(String scriptPath) { + return new TestSuite(builderFunction, scriptPath, getLog(), environmentVariables); + } + + List getTestSuites() { + return testSuites; + } + + private Map getEnvironmentVariables() throws MojoExecutionException { + return Map.of( + SHUNIT2_PATH, getEffectiveShUnit2Directory() + "/shunit2", + SCRIPTPATH, sourceDirectory.getAbsolutePath()); + } + + private String[] getScriptPaths() { + return Arrays.stream(fileSystem.listFiles(testSourceDirectory, this::isTestScript)) + .map(File::getAbsolutePath) + .toArray(String[]::new); + } + + private boolean isTestScript(File directory, String fileName) { + return isTestName(fileName.toLowerCase().split("\\.")[0]); + } + + private boolean isTestName(String baseName) { + return baseName.startsWith("test") || baseName.endsWith("test"); + } + + File getEffectiveShUnit2Directory() throws MojoExecutionException { + return lookupShUnit2Install(); + } + + private File lookupShUnit2Install() throws MojoExecutionException { + return Optional.ofNullable(lookupLatestShUnit2Install()) + .orElseThrow(() -> new MojoExecutionException("Cannot find shunit2 installation.")); + } + + // It is possible that we have more than one version of shunit2 built into the plugin, which the copy-resources + // phase will copy into the classes directory. This method iterates through them, selects the highest version, + // looking only at those which actually contain a 'shunit2' script, and returns the selected install directory. + private File lookupLatestShUnit2Install() { + return Optional.of(getShUnitRootDirectory()) + .filter(this::exists) + .map(this::getVersionSubdirectories).orElse(Stream.empty()) + .max(this::compare) + .orElse(null); + } + + private File getShUnitRootDirectory() { + return new File(outputDirectory, SHUNIT2_SCRIPT_ROOT); + } + + private boolean exists(File file) { + return fileSystem.exists(file); + } + + @Nonnull + private Stream getVersionSubdirectories(File rootDirectory) { + return Arrays.stream(fileSystem.listFiles(rootDirectory, this::hasShUnit2Install)); + } + + boolean hasShUnit2Install(File directory, String fileName) { + return fileSystem.exists(new File(directory, String.join(File.separator, fileName, SHUNIT2_SCRIPT_ROOT))); + } + + // The last element of each file is expected to be a version in the form ... This comparator + // sorts the lowest version first, so that "1.2.4" compared with "1.3.1" will return -1. + int compare(File first, File second) { + return Long.compare(toLong(first), toLong(second)); + } + + // Given a File representing a path to a version directory (consisting of numbers and periods), converts it to a long. + private long toLong(File versionFile) { + long result = 0; + final Matcher m = DIGITS.matcher(versionFile.getName()); + while (m.find()) { + result = (result * 100) + Long.parseLong(m.group()); + } + return result; + } + + +} diff --git a/operator-build-maven-plugin/src/main/java/oracle/kubernetes/mojo/shunit2/TestSuite.java b/operator-build-maven-plugin/src/main/java/oracle/kubernetes/mojo/shunit2/TestSuite.java new file mode 100644 index 00000000000..6663c2cbc2e --- /dev/null +++ b/operator-build-maven-plugin/src/main/java/oracle/kubernetes/mojo/shunit2/TestSuite.java @@ -0,0 +1,191 @@ +// Copyright (c) 2020, Oracle Corporation and/or its affiliates. +// Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. + +package oracle.kubernetes.mojo.shunit2; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.util.Map; +import java.util.function.Function; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import oracle.kubernetes.mojo.shunit2.AnsiUtils.AnsiFormatter; +import org.apache.maven.plugin.logging.Log; + +import static oracle.kubernetes.mojo.shunit2.AnsiUtils.Format.BOLD; +import static oracle.kubernetes.mojo.shunit2.AnsiUtils.Format.GREEN_FG; +import static oracle.kubernetes.mojo.shunit2.AnsiUtils.Format.RED_FG; + +class TestSuite { + private static final Pattern TESTS_RUN_PATTERN = Pattern.compile("Ran (\\d+) tests."); + private static final Pattern TEST_FAILED_PATTERN = Pattern.compile("ASSERT:(.*)"); + + private static final AnsiFormatter SUCCESS_PREFIX_STYLE = AnsiUtils.createFormatter(BOLD, GREEN_FG); + private static final AnsiFormatter RUN_PROBLEM_STYLE = AnsiUtils.createFormatter(BOLD); + private static final AnsiFormatter PLAIN_STYLE = AnsiUtils.createFormatter(); + private static final AnsiFormatter FAILURE_STYLE = AnsiUtils.createFormatter(BOLD, RED_FG); + + + @SuppressWarnings("FieldMayBeFinal") // not final to allow unit test to change it + private Function builderFunction; + private final String scriptPath; + private final Log log; + private final Map environmentVariables; + + private int numTestsRun; + private int numFailures; + private int numErrors; + + TestSuite(Function builderFunction, + String scriptPath, Log log, Map environmentVariables) { + this.builderFunction = builderFunction; + this.scriptPath = scriptPath; + this.log = log; + this.environmentVariables = environmentVariables; + } + + void run() { + try { + final Process process = createProcess(); + process.waitFor(); + processResults(process); + } catch (IOException | InterruptedException e) { + throw new TestSuiteFailedException("Unable to run test script " + scriptPath, e); + } + } + + int numTestsRun() { + return numTestsRun; + } + + int numFailures() { + return numFailures; + } + + public int getNumErrors() { + return numErrors; + } + + private Process createProcess() { + final BashProcessBuilder builder = builderFunction.apply(scriptPath); + for (Map.Entry variable : environmentVariables.entrySet()) { + builder.addEnvironmentVariable(variable.getKey(), variable.getValue()); + } + return builder.build(); + } + + private void processResults(Process process) throws IOException { + processOutputMessages(process); + processErrorMessages(process); + + logSummaryLine(); + } + + private void logSummaryLine() { + if (wasSuccess()) { + log.info(createSummaryLine()); + } else { + log.error(createSummaryLine()); + } + } + + private boolean wasSuccess() { + return numErrors == 0 && numFailures == 0; + } + + String createSummaryLine() { + return getLineStart() + getTestsRun() + getFailures() + getErrors() + getLineEnd(); + } + + private String getLineStart() { + return getTestsFormat().format("Tests "); + } + + private AnsiFormatter getTestsFormat() { + return wasSuccess() ? SUCCESS_PREFIX_STYLE : FAILURE_STYLE; + } + + private String getTestsRun() { + return getRunFormat().format("run: " + numTestsRun); + } + + private AnsiFormatter getRunFormat() { + return wasSuccess() ? SUCCESS_PREFIX_STYLE : RUN_PROBLEM_STYLE; + } + + private String getFailures() { + return ", " + getFailureFormat().format("Failures: " + numFailures); + } + + private AnsiFormatter getFailureFormat() { + return numFailures == 0 ? PLAIN_STYLE : FAILURE_STYLE; + } + + private String getErrors() { + return ", " + getErrorFormat().format("Errors: " + numErrors); + } + + private AnsiFormatter getErrorFormat() { + return numErrors == 0 ? PLAIN_STYLE : FAILURE_STYLE; + } + + private String getLineEnd() { + return wasSuccess() ? getLineEndText() : FAILURE_STYLE.format(" <<< FAILURE!" + getLineEndText()); + } + + private String getLineEndText() { + return " - in " + scriptPath; + } + + private void processOutputMessages(Process process) throws IOException { + try (final InputStream inputStream = process.getInputStream()) { + new BufferedReader(new InputStreamReader(inputStream)).lines() + .map(AnsiUtils::withoutAnsiEscapeChars) + .filter(this::isAllowedOutput) + .forEach(this::processOutputLine); + } + } + + // Returns true if the line should be logged as output. + private boolean isAllowedOutput(String outputLine) { + final Matcher testsRunMatcher = TESTS_RUN_PATTERN.matcher(outputLine); + if (testsRunMatcher.find()) { + numTestsRun = Integer.parseInt(testsRunMatcher.group(1)); + return false; + } else { + return !outputLine.contains("FAILED "); + } + } + + private void processOutputLine(String line) { + final Matcher testFailedMatcher = TEST_FAILED_PATTERN.matcher(line); + if (testFailedMatcher.find()) { + numFailures++; + line = AnsiUtils.createFormatter(RED_FG, BOLD).format("FAILED: ") + testFailedMatcher.group(1); + } + + log.info(line); + } + + private void processErrorMessages(Process process) throws IOException { + try (final InputStream errorStream = process.getErrorStream()) { + new BufferedReader(new InputStreamReader(errorStream)).lines() + .map(AnsiUtils::withoutAnsiEscapeChars) + .filter(this::allowedError) + .forEach(this::logError); + } + } + + private void logError(String line) { + numErrors++; + log.error(line); + } + + private boolean allowedError(String errorLine) { + return !errorLine.contains("returned non-zero return code"); + } + +} diff --git a/operator-build-maven-plugin/src/main/java/oracle/kubernetes/mojo/shunit2/TestSuiteFailedException.java b/operator-build-maven-plugin/src/main/java/oracle/kubernetes/mojo/shunit2/TestSuiteFailedException.java new file mode 100644 index 00000000000..845ac8f568a --- /dev/null +++ b/operator-build-maven-plugin/src/main/java/oracle/kubernetes/mojo/shunit2/TestSuiteFailedException.java @@ -0,0 +1,11 @@ +// Copyright (c) 2020, Oracle Corporation and/or its affiliates. +// Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. + +package oracle.kubernetes.mojo.shunit2; + +class TestSuiteFailedException extends RuntimeException { + + public TestSuiteFailedException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/operator-build-maven-plugin/src/main/java/oracle/kubernetes/mojosupport/FileSystem.java b/operator-build-maven-plugin/src/main/java/oracle/kubernetes/mojosupport/FileSystem.java new file mode 100644 index 00000000000..19d1f882327 --- /dev/null +++ b/operator-build-maven-plugin/src/main/java/oracle/kubernetes/mojosupport/FileSystem.java @@ -0,0 +1,90 @@ +// Copyright (c) 2018, 2020, Oracle Corporation and/or its affiliates. +// Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. + +package oracle.kubernetes.mojosupport; + +import java.io.File; +import java.io.FileReader; +import java.io.FileWriter; +import java.io.FilenameFilter; +import java.io.IOException; +import java.io.Reader; +import java.io.Writer; +import java.net.MalformedURLException; +import java.net.URL; + +public abstract class FileSystem { + + public static final FileSystem LIVE_FILE_SYSTEM = new LiveFileSystem(); + + public abstract URL toUrl(File file) throws MalformedURLException; + + public abstract File[] listFiles(File directory); + + public abstract File[] listFiles(File directory, FilenameFilter filter); + + public abstract boolean exists(File file); + + public abstract boolean isDirectory(File file); + + public abstract boolean isWritable(File directory); + + public abstract void createDirectory(File directory); + + public abstract Writer createWriter(File file) throws IOException; + + public abstract Reader createReader(File file) throws IOException; + + public abstract long getLastModified(File file); + + private static class LiveFileSystem extends FileSystem { + + @Override + public URL toUrl(File file) throws MalformedURLException { + return file.toURI().toURL(); + } + + public File[] listFiles(File directory) { + return directory.listFiles(); + } + + public File[] listFiles(File directory, FilenameFilter filter) { + return directory.listFiles(filter); + } + + @Override + public boolean exists(File file) { + return file.exists(); + } + + @Override + public boolean isDirectory(File file) { + return file.isDirectory(); + } + + @Override + public boolean isWritable(File directory) { + return directory.canWrite(); + } + + @Override + public void createDirectory(File directory) { + directory.mkdirs(); + } + + @Override + public Writer createWriter(File file) throws IOException { + return new FileWriter(file); + } + + @Override + public Reader createReader(File file) throws IOException { + return new FileReader(file); + } + + @Override + public long getLastModified(File file) { + return file.lastModified(); + } + } +} diff --git a/json-schema-maven-plugin/src/test/java/oracle/kubernetes/json/mojo/JsonSchemaMojoTest.java b/operator-build-maven-plugin/src/test/java/oracle/kubernetes/json/mojo/JsonSchemaMojoTest.java similarity index 54% rename from json-schema-maven-plugin/src/test/java/oracle/kubernetes/json/mojo/JsonSchemaMojoTest.java rename to operator-build-maven-plugin/src/test/java/oracle/kubernetes/json/mojo/JsonSchemaMojoTest.java index b041b6b7a8d..5ea39a6b145 100644 --- a/json-schema-maven-plugin/src/test/java/oracle/kubernetes/json/mojo/JsonSchemaMojoTest.java +++ b/operator-build-maven-plugin/src/test/java/oracle/kubernetes/json/mojo/JsonSchemaMojoTest.java @@ -4,97 +4,53 @@ package oracle.kubernetes.json.mojo; import java.io.File; -import java.lang.annotation.Annotation; import java.lang.reflect.Field; -import java.net.MalformedURLException; -import java.net.URISyntaxException; import java.net.URL; -import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; -import java.util.HashMap; import java.util.List; -import java.util.Map; import com.google.common.collect.ImmutableMap; -import com.meterware.simplestub.Memento; import com.meterware.simplestub.StaticStubSupport; -import org.apache.maven.plugin.AbstractMojo; +import oracle.kubernetes.mojosupport.MojoTestBase; +import oracle.kubernetes.mojosupport.TestFileSystem; import org.apache.maven.plugin.MojoExecutionException; -import org.apache.maven.plugin.logging.Log; import org.apache.maven.plugins.annotations.Mojo; import org.apache.maven.plugins.annotations.Parameter; -import org.junit.After; import org.junit.Before; import org.junit.Test; -import org.objectweb.asm.AnnotationVisitor; import org.objectweb.asm.ClassReader; -import org.objectweb.asm.ClassVisitor; -import org.objectweb.asm.FieldVisitor; -import org.objectweb.asm.TypePath; -import static com.meterware.simplestub.Stub.createNiceStub; import static java.util.Collections.singletonList; import static org.apache.maven.plugins.annotations.LifecyclePhase.PROCESS_CLASSES; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.arrayContaining; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasKey; -import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.is; -import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; import static org.hamcrest.Matchers.sameInstance; -import static org.objectweb.asm.Opcodes.ASM7; @SuppressWarnings("SameParameterValue") -public class JsonSchemaMojoTest { +public class JsonSchemaMojoTest extends MojoTestBase { private static final List EMPTY_CLASSPATH = new ArrayList<>(); private static final String TARGET_DIR = "/target/dir"; private static final String TEST_ROOT_CLASS = "a.b.c.D"; - private static final String DOT = "\\."; private static final File SCHEMA_FILE = createFile(TARGET_DIR, TEST_ROOT_CLASS, ".json"); private static final File MARKDOWN_FILE = createFile(TARGET_DIR, TEST_ROOT_CLASS, ".md"); private static final File CLASS_FILE = createFile("/classes", TEST_ROOT_CLASS, ".class"); private static final String SPECIFIED_FILE_NAME = "specifiedFile.json"; private static final File SPECIFIED_FILE = new File(TARGET_DIR + "/" + SPECIFIED_FILE_NAME); - private JsonSchemaMojo mojo = new JsonSchemaMojo(); - private Map classAnnotations = new HashMap<>(); + private final TestFileSystem fileSystem = new TestFileSystem(); - // a map of fields to their annotations - private Map> fieldAnnotations = new HashMap<>(); - private List mementos = new ArrayList<>(); private TestMain main; - private TestFileSystem fileSystem = new TestFileSystem(); - @SuppressWarnings("SameParameterValue") - private static File createFile(String dir, String className, String extension) { - return new File(dir + File.separator + classNameToPath(className) + extension); + public JsonSchemaMojoTest() { + super(new JsonSchemaMojo()); } - private static String classNameToPath(String className) { - return className.replaceAll(DOT, File.separator); - } - - private static File getTargetDir(Class aaClass) throws URISyntaxException { - File dir = getPackageDir(aaClass); - while (dir.getParent() != null && !dir.getName().equals("target")) { - dir = dir.getParentFile(); - } - return dir; - } - - private static File getPackageDir(Class aaClass) throws URISyntaxException { - URL url = aaClass.getResource(aaClass.getSimpleName() + ".class"); - return Paths.get(url.toURI()).toFile().getParentFile(); - } - - /** - * Setup test. - * @throws Exception on failure - */ @Before public void setUp() throws Exception { ClassReader classReader = new ClassReader(JsonSchemaMojo.class.getName()); @@ -112,50 +68,14 @@ public void setUp() throws Exception { mementos.add(StaticStubSupport.install(JsonSchemaMojo.class, "fileSystem", fileSystem)); } - private void silenceMojoLog() { - mojo.setLog(createNiceStub(Log.class)); - } - - private void setMojoParameter(String fieldName, Object value) throws Exception { - Field field = mojo.getClass().getDeclaredField(fieldName); - field.setAccessible(true); - field.set(mojo, value); - } - - private Object getMojoParameter(String fieldName) throws Exception { - Field field = mojo.getClass().getDeclaredField(fieldName); - field.setAccessible(true); - return field.get(mojo); - } - - /** - * Tear down test. - */ - @After - public void tearDown() { - for (Memento memento : mementos) { - memento.revert(); - } - } - - @Test - public void mojoExtendsBaseClass() { - assertThat(mojo, instanceOf(AbstractMojo.class)); - } - - @Test - public void mojoHasGoalAnnotation() { - assertThat(getClassAnnotation(Mojo.class), notNullValue()); - } - @Test public void mojoAnnotatedWithName() { - assertThat(getClassAnnotation(Mojo.class).fields.get("name"), equalTo("generate")); + assertThat(getClassAnnotation(Mojo.class).getField("name"), equalTo("generate")); } @Test public void mojoAnnotatedWithDefaultPhase() { - assertThat(getClassAnnotation(Mojo.class).fields.get("defaultPhase"), equalTo(PROCESS_CLASSES)); + assertThat(getClassAnnotation(Mojo.class).getField("defaultPhase"), equalTo(PROCESS_CLASSES)); } @Test @@ -163,7 +83,7 @@ public void hasClasspathElementsField_withAnnotation() throws NoSuchFieldExcepti Field classPathField = JsonSchemaMojo.class.getDeclaredField("compileClasspathElements"); assertThat(classPathField.getType(), equalTo(List.class)); assertThat( - getFieldAnnotation(classPathField, Parameter.class).fields.get("defaultValue"), + getFieldAnnotation(classPathField, Parameter.class).getField("defaultValue"), equalTo("${project.compileClasspathElements}")); } @@ -172,7 +92,7 @@ public void hasTargetDirField_withAnnotation() throws NoSuchFieldException { Field targetDirField = JsonSchemaMojo.class.getDeclaredField("targetDir"); assertThat(targetDirField.getType(), equalTo(String.class)); assertThat( - getFieldAnnotation(targetDirField, Parameter.class).fields.get("defaultValue"), + getFieldAnnotation(targetDirField, Parameter.class).getField("defaultValue"), equalTo("${project.build.outputDirectory}/schema")); } @@ -188,7 +108,7 @@ public void hasRootClassNameField_withAnnotation() throws NoSuchFieldException { Field rootClassField = JsonSchemaMojo.class.getDeclaredField("rootClass"); assertThat(rootClassField.getType(), equalTo(String.class)); assertThat( - getFieldAnnotation(rootClassField, Parameter.class).fields.get("required"), is(true)); + getFieldAnnotation(rootClassField, Parameter.class).getField("required"), is(true)); } @Test @@ -241,7 +161,7 @@ public void hasOutputFileField_withAnnotation() throws Exception { public void whenKubernetesVersionSpecified_passToGenerator() throws Exception { setMojoParameter("kubernetesVersion", "1.9.0"); - mojo.execute(); + executeMojo(); assertThat(main.getKubernetesVersion(), equalTo("1.9.0")); } @@ -250,7 +170,7 @@ public void whenKubernetesVersionSpecified_passToGenerator() throws Exception { public void whenKubernetesVersionNotSpecified_passToGenerator() throws Exception { setMojoParameter("kubernetesVersion", null); - mojo.execute(); + executeMojo(); assertThat(main.getKubernetesVersion(), nullValue()); } @@ -261,7 +181,7 @@ public void whenExternalSchemaSpecified_passToGenerator() throws Exception { "externalSchemas", singletonList(new ExternalSchema("http://schema.json", "src/cache/schema.json"))); - mojo.execute(); + executeMojo(); assertThat( main.getCacheFor(new URL("http://schema.json")), @@ -274,18 +194,18 @@ public void whenUnableToUseDefineSchema_haltTheBuild() throws Exception { "externalSchemas", singletonList(new ExternalSchema("abcd://schema.json", "src/cache/schema.json"))); - mojo.execute(); + executeMojo(); } @Test(expected = MojoExecutionException.class) public void whenNoClassSpecified_haltTheBuild() throws Exception { setMojoParameter("rootClass", null); - mojo.execute(); + executeMojo(); } @Test public void whenLookingForClassFile_specifyRelativeFilePath() throws Exception { - mojo.execute(); + executeMojo(); assertThat(main.getResourceName(), equalTo(classNameToPath(TEST_ROOT_CLASS) + ".class")); } @@ -293,7 +213,7 @@ public void whenLookingForClassFile_specifyRelativeFilePath() throws Exception { @Test(expected = MojoExecutionException.class) public void whenRootClassNotFound_haltTheBuild() throws Exception { main.setClasspathResource(null); - mojo.execute(); + executeMojo(); } @Test @@ -305,21 +225,21 @@ public void useSpecifiedClasspath() throws Exception { fileSystem.defineUrl(new File(classpathElements[i]), classPathUrls[i]); } - mojo.execute(); + executeMojo(); assertThat(main.getClasspath(), arrayContaining(classPathUrls)); } @Test public void generateToExpectedLocation() throws Exception { - mojo.execute(); + executeMojo(); assertThat(main.getSchemaFile(), equalTo(SCHEMA_FILE)); } @Test public void whenGenerateMarkdownNotSpecified_dontGenerateMarkdown() throws Exception { - mojo.execute(); + executeMojo(); assertThat(main.getMarkdownFile(), nullValue()); } @@ -328,7 +248,7 @@ public void whenGenerateMarkdownNotSpecified_dontGenerateMarkdown() throws Excep public void whenGenerateMarkdownSpecified_generateMarkdown() throws Exception { setMojoParameter("generateMarkdown", true); - mojo.execute(); + executeMojo(); assertThat(main.getMarkdownFile(), equalTo(MARKDOWN_FILE)); } @@ -339,7 +259,7 @@ public void whenGenerateMarkdownSpecified_useGeneratedSchemaForMarkdown() throws main.setGeneratedSchema(generatedSchema); setMojoParameter("generateMarkdown", true); - mojo.execute(); + executeMojo(); assertThat(main.getMarkdownSchema(), sameInstance(generatedSchema)); } @@ -350,7 +270,7 @@ public void whenSchemaMoreRecentThanClassFile_dontGenerateNewSchema() throws Exc fileSystem.defineFileContents(SCHEMA_FILE, ""); fileSystem.touch(SCHEMA_FILE); - mojo.execute(); + executeMojo(); assertThat(main.getSchemaFile(), nullValue()); } @@ -361,7 +281,7 @@ public void whenClassFileMoreRecentThanSchema_generateNewSchema() throws Excepti fileSystem.defineFileContents(SCHEMA_FILE, ""); fileSystem.touch(CLASS_FILE); - mojo.execute(); + executeMojo(); assertThat(main.getSchemaFile(), equalTo(SCHEMA_FILE)); } @@ -369,7 +289,7 @@ public void whenClassFileMoreRecentThanSchema_generateNewSchema() throws Excepti @Test public void whenOutputFileSpecified_generateToIt() throws Exception { setMojoParameter("outputFile", SPECIFIED_FILE_NAME); - mojo.execute(); + executeMojo(); assertThat(main.getSchemaFile(), equalTo(SPECIFIED_FILE)); } @@ -378,7 +298,7 @@ public void whenOutputFileSpecified_generateToIt() throws Exception { public void whenIncludeAdditionalPropertiesSet_setOnMain() throws Exception { setMojoParameter("includeAdditionalProperties", true); - mojo.execute(); + executeMojo(); assertThat(main.isIncludeAdditionalProperties(), is(true)); } @@ -387,159 +307,9 @@ public void whenIncludeAdditionalPropertiesSet_setOnMain() throws Exception { public void whenSupportObjectReferencesSet_setOnMain() throws Exception { setMojoParameter("supportObjectReferences", true); - mojo.execute(); + executeMojo(); assertThat(main.isSupportObjectReferences(), is(true)); } - @SuppressWarnings("SameParameterValue") - private AnnotationInfo getClassAnnotation(Class annotationClass) { - return classAnnotations.get(toDescription(annotationClass)); - } - - @SuppressWarnings("SameParameterValue") - private AnnotationInfo getFieldAnnotation(Field field, Class annotation) { - return fieldAnnotations.get(field).get(toDescription(annotation)); - } - - private Map getOrCreateAnnotationMap(Field field) { - Map map = fieldAnnotations.get(field); - return map != null ? map : createAnnotationMap(field); - } - - private Map createAnnotationMap(Field field) { - Map map = new HashMap<>(); - fieldAnnotations.put(field, map); - return map; - } - - private String toDescription(Class aaClass) { - return "L" + aaClass.getName().replaceAll(DOT, "/") + ';'; - } - - private AnnotationInfo getOrCreateAnnotationInfo( - String description, Map map) { - AnnotationInfo info = map.get(description); - return info != null ? info : createAnnotationInfo(map, description); - } - - private AnnotationInfo createAnnotationInfo(Map map, String description) { - AnnotationInfo info = new AnnotationInfo(); - map.put(description, info); - return info; - } - - private URL toModuleUrl(String path) throws URISyntaxException, MalformedURLException { - return new File(getModuleDir(), path).toURI().toURL(); - } - - private File getModuleDir() throws URISyntaxException { - return getTargetDir(getClass()).getParentFile(); - } - - private class Visitor extends ClassVisitor { - private Class theClass; - - Visitor(Class theClass) { - super(ASM7); - this.theClass = theClass; - } - - @Override - public AnnotationVisitor visitTypeAnnotation( - int typeRef, TypePath typePath, String desc, boolean visible) { - return super.visitTypeAnnotation(typeRef, typePath, desc, visible); - } - - @Override - public AnnotationVisitor visitAnnotation(String desc, boolean visible) { - return new ClassAnnotationVisitor(desc); - } - - @Override - public FieldVisitor visitField(int flags, String fieldName, String desc, String s, Object v) { - try { - return new MojoFieldVisitor(getField(fieldName)); - } catch (NoSuchFieldException e) { - return super.visitField(flags, fieldName, desc, s, v); - } - } - - private Field getField(String fieldName) throws NoSuchFieldException { - return theClass.getDeclaredField(fieldName); - } - } - - private abstract class MojoAnnotationVisitor extends AnnotationVisitor { - private String annotationClassDesc; - private Map annotations; - - MojoAnnotationVisitor(Map annotations, String desc) { - super(ASM7); - this.annotations = annotations; - annotationClassDesc = desc; - annotations.put(desc, new AnnotationInfo()); - } - - @Override - public void visit(String name, Object value) { - getOrCreateAnnotationInfo(annotationClassDesc, annotations).fields.put(name, value); - } - - @Override - public void visitEnum(String name, String enumDesc, String value) { - getOrCreateAnnotationInfo(annotationClassDesc, annotations) - .fields - .put(name, getEnumConstant(getEnumClass(enumDesc), value)); - } - - Class getEnumClass(String desc) { - try { - String className = desc.substring(1, desc.length() - 1).replaceAll("/", DOT); - return Class.forName(className); - } catch (ClassNotFoundException e) { - throw new RuntimeException(e.toString()); - } - } - - private Object getEnumConstant(Class enumClass, String value) { - for (Object constant : enumClass.getEnumConstants()) { - if (value.equalsIgnoreCase(constant.toString())) { - return constant; - } - } - throw new RuntimeException("No enum constant " + value + " in " + enumClass); - } - } - - private class ClassAnnotationVisitor extends MojoAnnotationVisitor { - - ClassAnnotationVisitor(String annotationDescriptor) { - super(classAnnotations, annotationDescriptor); - } - } - - private class FieldAnnotationVisitor extends MojoAnnotationVisitor { - FieldAnnotationVisitor(Map annotationMap, String desc) { - super(annotationMap, desc); - } - } - - private class MojoFieldVisitor extends FieldVisitor { - private final Map annotationMap; - - MojoFieldVisitor(Field field) { - super(ASM7); - this.annotationMap = getOrCreateAnnotationMap(field); - } - - @Override - public AnnotationVisitor visitAnnotation(String desc, boolean visible) { - return new FieldAnnotationVisitor(annotationMap, desc); - } - } - - private class AnnotationInfo { - private Map fields = new HashMap<>(); - } } diff --git a/json-schema-maven-plugin/src/test/java/oracle/kubernetes/json/mojo/TestMain.java b/operator-build-maven-plugin/src/test/java/oracle/kubernetes/json/mojo/TestMain.java similarity index 100% rename from json-schema-maven-plugin/src/test/java/oracle/kubernetes/json/mojo/TestMain.java rename to operator-build-maven-plugin/src/test/java/oracle/kubernetes/json/mojo/TestMain.java diff --git a/operator-build-maven-plugin/src/test/java/oracle/kubernetes/mojo/shunit2/AnsiUtilsTest.java b/operator-build-maven-plugin/src/test/java/oracle/kubernetes/mojo/shunit2/AnsiUtilsTest.java new file mode 100644 index 00000000000..e608d2c664b --- /dev/null +++ b/operator-build-maven-plugin/src/test/java/oracle/kubernetes/mojo/shunit2/AnsiUtilsTest.java @@ -0,0 +1,56 @@ +// Copyright (c) 2020, Oracle Corporation and/or its affiliates. +// Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. + +package oracle.kubernetes.mojo.shunit2; + +import org.junit.Test; + +import static oracle.kubernetes.mojo.shunit2.AnsiUtils.Format.BLUE_FG; +import static oracle.kubernetes.mojo.shunit2.AnsiUtils.Format.BOLD; +import static oracle.kubernetes.mojo.shunit2.AnsiUtils.Format.GREEN_FG; +import static oracle.kubernetes.mojo.shunit2.AnsiUtils.Format.RED_FG; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; + +public class AnsiUtilsTest { + + + @Test + public void removeAnsiEscapeCharacters() { + assertThat(AnsiUtils.withoutAnsiEscapeChars("\u001B[1;31mASSERT:\u001B[0mIt didn't work"), + equalTo("ASSERT:It didn't work")); + assertThat(AnsiUtils.withoutAnsiEscapeChars("Ran \u001B[1;36m2\u001B[0m tests."), + equalTo("Ran 2 tests.")); + assertThat(AnsiUtils.withoutAnsiEscapeChars("\u001B[1;31mFAILED\u001B[0m (\u001B[1;31mfailures=1\u001B[0m)"), + equalTo("FAILED (failures=1)")); + } + + @Test + public void formatBoldTexts() { + assertThat(AnsiUtils.createFormatter(BOLD).format("sample"), + equalTo("\u001B[1msample\u001B[0m")); + } + + @Test + public void formatBoldText() { + assertThat(AnsiUtils.createFormatter(BOLD).format("sample"), equalTo("\u001B[1msample\u001B[0m")); + } + + @Test + public void formatBoldRedText() { + assertThat(AnsiUtils.createFormatter(BOLD, RED_FG).format("sample"), + equalTo("\u001B[1;31msample\u001B[0m")); + } + + @Test + public void formatBlueText() { + assertThat(AnsiUtils.createFormatter(BLUE_FG).format("sample"), + equalTo("\u001B[34msample\u001B[0m")); + } + + @Test + public void formatGreenText() { + assertThat(AnsiUtils.createFormatter(GREEN_FG).format("sample"), + equalTo("\u001B[32msample\u001B[0m")); + } +} \ No newline at end of file diff --git a/operator-build-maven-plugin/src/test/java/oracle/kubernetes/mojo/shunit2/ShUnit2MojoTest.java b/operator-build-maven-plugin/src/test/java/oracle/kubernetes/mojo/shunit2/ShUnit2MojoTest.java new file mode 100644 index 00000000000..a14c162b90a --- /dev/null +++ b/operator-build-maven-plugin/src/test/java/oracle/kubernetes/mojo/shunit2/ShUnit2MojoTest.java @@ -0,0 +1,450 @@ +// Copyright (c) 2020, Oracle Corporation and/or its affiliates. +// Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. + +package oracle.kubernetes.mojo.shunit2; + +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.InputStream; +import java.lang.reflect.Field; +import java.nio.charset.StandardCharsets; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.function.BiFunction; +import java.util.function.Function; +import java.util.stream.Collectors; +import javax.annotation.Nonnull; + +import com.meterware.simplestub.StaticStubSupport; +import com.meterware.simplestub.SystemPropertySupport; +import oracle.kubernetes.mojosupport.MojoTestBase; +import oracle.kubernetes.mojosupport.TestFileSystem; +import org.apache.maven.plugin.MojoExecutionException; +import org.apache.maven.plugin.MojoFailureException; +import org.apache.maven.plugins.annotations.Mojo; +import org.apache.maven.plugins.annotations.Parameter; +import org.junit.Before; +import org.junit.Test; +import org.objectweb.asm.ClassReader; + +import static com.meterware.simplestub.Stub.createNiceStub; +import static oracle.kubernetes.mojo.shunit2.AnsiUtils.Format.BLUE_FG; +import static oracle.kubernetes.mojo.shunit2.AnsiUtils.Format.BOLD; +import static oracle.kubernetes.mojo.shunit2.AnsiUtils.Format.GREEN_FG; +import static oracle.kubernetes.mojo.shunit2.AnsiUtils.Format.RED_FG; +import static org.apache.maven.plugins.annotations.LifecyclePhase.TEST; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.arrayContainingInAnyOrder; +import static org.hamcrest.Matchers.both; +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.empty; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasItem; +import static org.junit.Assert.fail; + +@SuppressWarnings("UnconstructableJUnitTestCase") // mistaken warning due to private constructor +public class ShUnit2MojoTest extends MojoTestBase { + + private static final String TEST_SCRIPT = "shunit2"; + private static final String OS_NAME_PROPERTY = "os.name"; + private static final String[] INSTALLED_OSX_BASH_VERSION = { + "GNU bash, version 3.2.57(1)-release (x86_64-apple-darwin19)", + "Copyright (C) 2007 Free Software Foundation, Inc." + }; + + private static final String[] HOMEBREW_BASH_VERSION = { + "GNU bash, version 5.0.18(1)-release (x86_64-apple-darwin19.6.0)", + "Copyright (C) 2019 Free Software Foundation, Inc.", + "License GPLv3+: GNU GPL version 3 or later " + }; + + private final ShUnit2Mojo mojo; + private final Function builderFunction = this::createProcessBuilder; + private final TestDelegate delegate = new TestDelegate(); + private final TestFileSystem fileSystem = new TestFileSystem(); + private static final File TEST_CLASSES_DIRECTORY = new File("/test-classes"); + private static final File LATEST_SHUNIT2_DIRECTORY = new File(TEST_CLASSES_DIRECTORY, "shunit2/2.1.8"); + private static final File EARLIER_SHUNIT2_DIRECTORY = new File(TEST_CLASSES_DIRECTORY, "shunit2/2.1.6"); + private static final File SOURCE_DIRECTORY = new File("/sources"); + private static final File TEST_SOURCE_DIRECTORY = new File("/tests"); + + public ShUnit2MojoTest() { + this(new ShUnit2Mojo()); + } + + private ShUnit2MojoTest(ShUnit2Mojo mojo) { + super(mojo); + this.mojo = mojo; + } + + @Before + public void setUp() throws Exception { + ClassReader classReader = new ClassReader(ShUnit2Mojo.class.getName()); + classReader.accept(new Visitor(ShUnit2Mojo.class), 0); + + mementos.add(StaticStubSupport.install(ShUnit2Mojo.class, "fileSystem", fileSystem)); + mementos.add(StaticStubSupport.install(ShUnit2Mojo.class, "builderFunction", builderFunction)); + mementos.add(SystemPropertySupport.install(OS_NAME_PROPERTY, "Linux")); + + setMojoParameter("outputDirectory", TEST_CLASSES_DIRECTORY); + setMojoParameter("sourceDirectory", SOURCE_DIRECTORY); + setMojoParameter("testSourceDirectory", TEST_SOURCE_DIRECTORY); + silenceMojoLog(); + + fileSystem.defineFileContents(new File(LATEST_SHUNIT2_DIRECTORY, TEST_SCRIPT), ""); + fileSystem.defineFileContents(new File(TEST_SOURCE_DIRECTORY, "test1.sh"), ""); + } + + BashProcessBuilder createProcessBuilder(String command) { + return new BashProcessBuilder(delegate, command); + } + + @Test + public void mojoAnnotatedWithName() { + assertThat(getClassAnnotation(Mojo.class).getField("name"), equalTo("shunit2")); + } + + @Test + public void mojoAnnotatedWithDefaultPhase() { + assertThat(getClassAnnotation(Mojo.class).getField("defaultPhase"), equalTo(TEST)); + } + + @Test + public void hasOutputDirectoryField_withAnnotation() throws NoSuchFieldException { + Field classPathField = ShUnit2Mojo.class.getDeclaredField("outputDirectory"); + assertThat(classPathField.getType(), equalTo(File.class)); + assertThat( + getFieldAnnotation(classPathField, Parameter.class).getField("defaultValue"), + equalTo("${project.build.testOutputDirectory}")); + } + + @Test + public void hasTestSourceDirectoryField_withAnnotation() throws NoSuchFieldException { + Field classPathField = ShUnit2Mojo.class.getDeclaredField("testSourceDirectory"); + assertThat(classPathField.getType(), equalTo(File.class)); + assertThat( + getFieldAnnotation(classPathField, Parameter.class).getField("defaultValue"), + equalTo("${project.basedir}/src/test/sh")); + } + + @Test + public void hasSourceDirectoryField_withAnnotation() throws NoSuchFieldException { + Field classPathField = ShUnit2Mojo.class.getDeclaredField("sourceDirectory"); + assertThat(classPathField.getType(), equalTo(File.class)); + assertThat( + getFieldAnnotation(classPathField, Parameter.class).getField("defaultValue"), + equalTo("${project.basedir}/src/main/sh")); + } + + @Test + public void useCopiedShUnit2Directory() throws MojoExecutionException { + fileSystem.defineFileContents(new File(LATEST_SHUNIT2_DIRECTORY, TEST_SCRIPT), ""); + + assertThat(mojo.getEffectiveShUnit2Directory(), equalTo(LATEST_SHUNIT2_DIRECTORY)); + } + + @Test + public void whenMultipleShUnit2VersionsInstalled_selectLatest() throws MojoExecutionException { + fileSystem.defineFileContents(new File(EARLIER_SHUNIT2_DIRECTORY, TEST_SCRIPT), ""); + fileSystem.defineFileContents(new File(LATEST_SHUNIT2_DIRECTORY, TEST_SCRIPT), ""); + + assertThat(mojo.getEffectiveShUnit2Directory(), equalTo(LATEST_SHUNIT2_DIRECTORY)); + } + + @Test + public void whenLatestShUnit2VersionsMissing_selectPrior() throws MojoExecutionException { + fileSystem.clear(); + fileSystem.defineFileContents(new File(EARLIER_SHUNIT2_DIRECTORY, TEST_SCRIPT), ""); + fileSystem.defineFileContents(LATEST_SHUNIT2_DIRECTORY, ""); + + assertThat(mojo.getEffectiveShUnit2Directory(), equalTo(EARLIER_SHUNIT2_DIRECTORY)); + } + + @Test(expected = MojoExecutionException.class) + public void whenShUnit2NotInstalled_reportFailure() throws MojoFailureException, MojoExecutionException { + fileSystem.clear(); + fileSystem.defineFileContents(TEST_CLASSES_DIRECTORY, ""); + + executeMojo(); + } + + @Test + public void onMacOS_warnIfBashIsOld() throws MojoFailureException, MojoExecutionException { + System.setProperty(OS_NAME_PROPERTY, "Mac OS X"); + defineExecution().withOutputs(INSTALLED_OSX_BASH_VERSION); + + executeMojo(); + + assertThat(getWarningLines(), + contains("Bash 3.2 is too old to run unit tests. Install a later version with Homebrew: " + + AnsiUtils.createFormatter(BOLD, BLUE_FG).format("brew install bash") + ".")); + } + + @Test + public void onMacOS_dontRunTestsIfBashIsOld() throws MojoFailureException, MojoExecutionException { + System.setProperty(OS_NAME_PROPERTY, "Mac OS X"); + defineExecution().withOutputs(INSTALLED_OSX_BASH_VERSION); + defineExecution().withOutputs("This is an example", "and here is another", "Ran 2 tests."); + + executeMojo(); + + assertThat(getInfoLines(), empty()); + } + + @Test + public void onMacOS_dontWarnIfBashVersionIsSupported() throws MojoFailureException, MojoExecutionException { + System.setProperty(OS_NAME_PROPERTY, "Mac OS X"); + defineExecution().withOutputs(HOMEBREW_BASH_VERSION); + + executeMojo(); + + assertThat(getWarningLines(), empty()); + } + + @Test + public void onMacOS_runTestsIfBashVersionIsSupported() throws MojoFailureException, MojoExecutionException { + System.setProperty(OS_NAME_PROPERTY, "Mac OS X"); + defineExecution().withOutputs(HOMEBREW_BASH_VERSION); + defineExecution().withOutputs("This is an example", "and here is another", "Ran 2 tests."); + + executeMojo(); + + assertThat(getInfoLines(), contains("This is an example", "and here is another", + createExpectedSuccessSummary(2, "test1.sh"))); + } + + @Test + public void onExecution_specifyTheSelectedShUnit2ScriptPath() throws MojoFailureException, MojoExecutionException { + executeMojo(); + + assertThat(delegate.getShUnit2ScriptPath(), equalTo(LATEST_SHUNIT2_DIRECTORY + "/shunit2")); + } + + @Test + public void onExecution_specifyPathToSourceScripts() throws MojoFailureException, MojoExecutionException { + executeMojo(); + + assertThat(delegate.getSourceScriptDir(), equalTo(SOURCE_DIRECTORY.getAbsolutePath())); + } + + @Test + public void onExecution_specifyTestScripts() throws MojoFailureException, MojoExecutionException { + fileSystem.defineFileContents(new File(TEST_SOURCE_DIRECTORY, "test1.sh"), ""); + fileSystem.defineFileContents(new File(TEST_SOURCE_DIRECTORY, "test3.sh"), ""); + fileSystem.defineFileContents(new File(TEST_SOURCE_DIRECTORY, "nothing.sh"), ""); + fileSystem.defineFileContents(new File(TEST_SOURCE_DIRECTORY, "2ndtest.sh"), ""); + fileSystem.defineFileContents(new File(TEST_SOURCE_DIRECTORY, "4thtest"), ""); + + executeMojo(); + + assertThat(delegate.getScriptPaths(), + arrayContainingInAnyOrder("/tests/test1.sh", "/tests/2ndtest.sh", "/tests/test3.sh", "/tests/4thtest")); + } + + @Test + public void onExecution_logOutputs() throws MojoFailureException, MojoExecutionException { + defineExecution().withOutputs("This is an example", "and here is another", "Ran 2 tests."); + + executeMojo(); + + assertThat(getInfoLines(), contains("This is an example", "and here is another", + createExpectedSuccessSummary(2, "test1.sh"))); + } + + protected ProcessStub defineExecution() { + return delegate.defineScriptExecution(); + } + + @SuppressWarnings("SameParameterValue") + private String createExpectedSuccessSummary(int numTestsRun, String testScript) { + return AnsiUtils.createFormatter(BOLD, GREEN_FG).format("Tests ") + + AnsiUtils.createFormatter(BOLD, GREEN_FG).format("run: " + numTestsRun) + + String.format(", Failures: 0, Errors: 0 - in /tests/%s", testScript); + } + + @Test + public void whenErrorDetected_reportInSummary() throws MojoExecutionException { + defineExecution().withErrors("This is an example", "and here is another").withOutputs("Ran 3 tests."); + + try { + executeMojo(); + fail("Should have thrown an exception"); + } catch (MojoFailureException ignored) { + assertThat(getErrorLines(), + contains("This is an example", "and here is another", + createExpectedErrorSummary(3, 2, "test1.sh"))); + } + } + + @SuppressWarnings("SameParameterValue") + private String createExpectedErrorSummary(int numTestsRun, int numErrors, String testScript) { + return AnsiUtils.createFormatter(BOLD, RED_FG).format("Tests ") + + AnsiUtils.createFormatter(BOLD).format("run: " + numTestsRun) + + ", Failures: 0, " + + AnsiUtils.createFormatter(BOLD, RED_FG).format("Errors: " + numErrors) + + AnsiUtils.createFormatter(BOLD, RED_FG).format(" <<< FAILURE! - in /tests/" + testScript); + } + + @Test + public void onExecution_logErrors() throws MojoExecutionException { + defineExecution().withErrors("This is an example", "and here is another"); + + try { + executeMojo(); + fail("Should have thrown an exception"); + } catch (MojoFailureException ignored) { + assertThat(getErrorLines(), both(hasItem("This is an example")).and(hasItem("and here is another"))); + } + } + + @Test + public void onExecution_ignoreNonZeroReturnCodeErrors() throws MojoExecutionException { + defineExecution().withErrors("This is an example", + "\u001B[1;31mERROR:\u001B[0m testPartyLikeItIs1999() returned non-zero return code."); + + try { + executeMojo(); + fail("Should have thrown an exception"); + } catch (MojoFailureException ignored) { + assertThat(getErrorLines(), hasItem("This is an example")); + } + } + + @Test + public void onExecution_ignoreFailureMessage() throws MojoFailureException, MojoExecutionException { + defineExecution().withOutputs("This is an example", "FAILED (failures=2)"); + + executeMojo(); + + assertThat(getErrorLines(), empty()); + } + + @Test + public void onExecution_recordReportedNumberOfTests() throws MojoFailureException, MojoExecutionException { + fileSystem.defineFileContents(new File(TEST_SOURCE_DIRECTORY, "test1.sh"), ""); + fileSystem.defineFileContents(new File(TEST_SOURCE_DIRECTORY, "test2.sh"), ""); + final AnsiUtils.AnsiFormatter shUnit2RunCountFormat = AnsiUtils.createFormatter(BOLD, BLUE_FG); + defineExecution().withOutputs(String.format("Ran %s tests.", shUnit2RunCountFormat.format("3"))); + defineExecution().withOutputs(String.format("Ran %s tests.", shUnit2RunCountFormat.format("2"))); + + executeMojo(); + + assertThat(mojo.getTestSuites().stream().map(TestSuite::numTestsRun).collect(Collectors.toList()), contains(3, 2)); + } + + @Test + public void onExecution_recordReportedNumberOfFailures() throws MojoExecutionException { + fileSystem.defineFileContents(new File(TEST_SOURCE_DIRECTORY, "test1.sh"), ""); + fileSystem.defineFileContents(new File(TEST_SOURCE_DIRECTORY, "test2.sh"), ""); + defineExecution() + .withOutputs("test1", "test2", createExpectedTestFailure("expected up but was down")) + .withErrors("test2 returned non-zero return code."); + defineExecution() + .withOutputs("test3", createExpectedTestFailure("expected blue but was red"), + "test4", createExpectedTestFailure("expected left but was right")); + + try { + executeMojo(); + } catch (MojoFailureException e) { + assertThat(getFailuresByTestSuite(), contains(1, 2)); + } + } + + @Nonnull + protected List getFailuresByTestSuite() { + return mojo.getTestSuites().stream().map(TestSuite::numFailures).collect(Collectors.toList()); + } + + private String createExpectedTestFailure(String explanation) { + return AnsiUtils.createFormatter(BOLD, RED_FG).format("ASSERT:") + explanation; + } + + @Test(expected = MojoFailureException.class) + public void whenAnyTestsFail_mojoThrowsException() throws MojoFailureException, MojoExecutionException { + fileSystem.defineFileContents(new File(TEST_SOURCE_DIRECTORY, "test1.sh"), ""); + fileSystem.defineFileContents(new File(TEST_SOURCE_DIRECTORY, "test2.sh"), ""); + defineExecution() + .withOutputs("test1", "test2", createExpectedTestFailure("expected up but was down")) + .withErrors("test2 returned non-zero return code."); + defineExecution() + .withOutputs("test3", createExpectedTestFailure("expected blue but was red"), + "test4", createExpectedTestFailure("expected left but was right")); + + executeMojo(); + } + + // todo print tests run, failures at end of each testsuite + // todo print total tests run, total failures across multiple tests + + static class TestDelegate implements BiFunction, Process> { + private final ArrayDeque processStubs = new ArrayDeque<>(); + private final List commands = new ArrayList<>(); + private Map environmentVariables; + + ProcessStub defineScriptExecution() { + final ProcessStub processStub = createNiceStub(ProcessStub.class); + processStubs.add(processStub); + return processStub; + } + + String getShUnit2ScriptPath() { + return environmentVariables.get(ShUnit2Mojo.SHUNIT2_PATH); + } + + String getSourceScriptDir() { + return environmentVariables.get(ShUnit2Mojo.SCRIPTPATH); + } + + String[] getScriptPaths() { + return commands.toArray(new String[0]); + } + + @Override + public Process apply(String command, Map environmentVariables) { + this.commands.add(command); + this.environmentVariables = environmentVariables; + return Optional.ofNullable(processStubs.pollFirst()).orElseGet(this::defineScriptExecution); + } + } + + + abstract static class ProcessStub extends Process { + private final List outputLines = new ArrayList<>(); + private final List errorLines = new ArrayList<>(); + + ProcessStub withOutputs(String... lines) { + outputLines.addAll(Arrays.asList(lines)); + return this; + } + + ProcessStub withErrors(String... lines) { + errorLines.addAll(Arrays.asList(lines)); + return this; + } + + @Override + public InputStream getInputStream() { + return new StringListInputStream(outputLines); + } + + @Override + public InputStream getErrorStream() { + return new StringListInputStream(errorLines); + } + } + + static class StringListInputStream extends ByteArrayInputStream { + + public StringListInputStream(List inputs) { + super(toByteArray(inputs)); + } + + private static byte[] toByteArray(List inputs) { + return String.join(System.lineSeparator(), inputs).getBytes(StandardCharsets.UTF_8); + } + } +} diff --git a/operator-build-maven-plugin/src/test/java/oracle/kubernetes/mojosupport/MojoTestBase.java b/operator-build-maven-plugin/src/test/java/oracle/kubernetes/mojosupport/MojoTestBase.java new file mode 100644 index 00000000000..71ca3047b3f --- /dev/null +++ b/operator-build-maven-plugin/src/test/java/oracle/kubernetes/mojosupport/MojoTestBase.java @@ -0,0 +1,349 @@ +// Copyright (c) 2020, Oracle Corporation and/or its affiliates. +// Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. + +package oracle.kubernetes.mojosupport; + +import java.io.File; +import java.lang.annotation.Annotation; +import java.lang.reflect.Field; +import java.net.MalformedURLException; +import java.net.URISyntaxException; +import java.net.URL; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import javax.annotation.Nonnull; + +import com.meterware.simplestub.Memento; +import org.apache.maven.plugin.AbstractMojo; +import org.apache.maven.plugin.MojoExecutionException; +import org.apache.maven.plugin.MojoFailureException; +import org.apache.maven.plugin.logging.Log; +import org.apache.maven.plugins.annotations.Mojo; +import org.junit.After; +import org.junit.Test; +import org.objectweb.asm.AnnotationVisitor; +import org.objectweb.asm.ClassVisitor; +import org.objectweb.asm.FieldVisitor; +import org.objectweb.asm.TypePath; + +import static com.meterware.simplestub.Stub.createNiceStub; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.notNullValue; +import static org.objectweb.asm.Opcodes.ASM7; + +public abstract class MojoTestBase { + + private final AbstractMojo mojo; + private static final String DOT = "\\."; + // a map of fields to their annotations + protected Map> fieldAnnotations = new HashMap<>(); + protected List mementos = new ArrayList<>(); + private final Map classAnnotations = new HashMap<>(); + private final LogStub logStub = createNiceStub(LogStub.class); + + @SuppressWarnings("SameParameterValue") + protected static File createFile(String dir, String className, String extension) { + return new File(dir + File.separator + MojoTestBase.classNameToPath(className) + extension); + } + + protected static String classNameToPath(String className) { + return className.replaceAll(DOT, File.separator); + } + + private static File getTargetDir(Class aaClass) throws URISyntaxException { + File dir = MojoTestBase.getPackageDir(aaClass); + while (dir.getParent() != null && !dir.getName().equals("target")) { + dir = dir.getParentFile(); + } + return dir; + } + + private static File getPackageDir(Class aaClass) throws URISyntaxException { + URL url = aaClass.getResource(aaClass.getSimpleName() + ".class"); + return Paths.get(url.toURI()).toFile().getParentFile(); + } + + public MojoTestBase(AbstractMojo mojo) { + this.mojo = mojo; + } + + public void executeMojo() throws MojoFailureException, MojoExecutionException { + mojo.execute(); + } + + protected List getInfoLines() { + return logStub.infoMessages; + } + + protected List getWarningLines() { + return logStub.warningMessages; + } + + protected List getErrorLines() { + return logStub.errorMessages; + } + + protected void silenceMojoLog() { + mojo.setLog(logStub); + } + + protected void setMojoParameter(String fieldName, Object value) throws Exception { + Field field = mojo.getClass().getDeclaredField(fieldName); + field.setAccessible(true); + field.set(mojo, value); + } + + protected Object getMojoParameter(String fieldName) throws Exception { + Field field = mojo.getClass().getDeclaredField(fieldName); + field.setAccessible(true); + return field.get(mojo); + } + + /** + * Tear down test. + */ + @After + public void tearDown() { + for (Memento memento : mementos) { + memento.revert(); + } + } + + @Test + public void mojoExtendsBaseClass() { + assertThat(mojo, instanceOf(AbstractMojo.class)); + } + + @Test + public void mojoHasGoalAnnotation() { + assertThat(getClassAnnotation(Mojo.class), notNullValue()); + } + + @SuppressWarnings("SameParameterValue") + protected AnnotationInfo getClassAnnotation(Class annotationClass) { + return classAnnotations.get(toDescription(annotationClass)); + } + + @SuppressWarnings("SameParameterValue") + protected AnnotationInfo getFieldAnnotation(Field field, Class annotation) { + return fieldAnnotations.get(field).get(toDescription(annotation)); + } + + private Map getOrCreateAnnotationMap(Field field) { + Map map = fieldAnnotations.get(field); + return map != null ? map : createAnnotationMap(field); + } + + private Map createAnnotationMap(Field field) { + Map map = new HashMap<>(); + fieldAnnotations.put(field, map); + return map; + } + + protected String toDescription(Class aaClass) { + return "L" + aaClass.getName().replaceAll(DOT, "/") + ';'; + } + + private AnnotationInfo getOrCreateAnnotationInfo( + String description, Map map) { + AnnotationInfo info = map.get(description); + return info != null ? info : createAnnotationInfo(map, description); + } + + private AnnotationInfo createAnnotationInfo(Map map, String description) { + AnnotationInfo info = new AnnotationInfo(); + map.put(description, info); + return info; + } + + @SuppressWarnings("SameParameterValue") + protected URL toModuleUrl(String path) throws URISyntaxException, MalformedURLException { + return new File(getModuleDir(), path).toURI().toURL(); + } + + protected File getModuleDir() throws URISyntaxException { + return MojoTestBase.getTargetDir(getClass()).getParentFile(); + } + + protected class Visitor extends ClassVisitor { + private final Class theClass; + + public Visitor(Class theClass) { + super(ASM7); + this.theClass = theClass; + } + + @Override + public AnnotationVisitor visitTypeAnnotation( + int typeRef, TypePath typePath, String desc, boolean visible) { + return super.visitTypeAnnotation(typeRef, typePath, desc, visible); + } + + @Override + public AnnotationVisitor visitAnnotation(String desc, boolean visible) { + return new ClassAnnotationVisitor(desc); + } + + @Override + public FieldVisitor visitField(int flags, String fieldName, String desc, String s, Object v) { + try { + return new MojoFieldVisitor(getField(fieldName)); + } catch (NoSuchFieldException e) { + return super.visitField(flags, fieldName, desc, s, v); + } + } + + private Field getField(String fieldName) throws NoSuchFieldException { + return theClass.getDeclaredField(fieldName); + } + } + + private abstract class MojoAnnotationVisitor extends AnnotationVisitor { + + private final String annotationClassDesc; + private final Map annotations; + + MojoAnnotationVisitor(Map annotations, String desc) { + super(ASM7); + this.annotations = annotations; + annotationClassDesc = desc; + annotations.put(desc, new AnnotationInfo()); + } + + @Override + public void visit(String name, Object value) { + getOrCreateAnnotationInfo(annotationClassDesc, annotations).fields.put(name, value); + } + + @Override + public void visitEnum(String name, String enumDesc, String value) { + getOrCreateAnnotationInfo(annotationClassDesc, annotations) + .fields + .put(name, getEnumConstant(getEnumClass(enumDesc), value)); + } + + Class getEnumClass(String desc) { + try { + String className = desc.substring(1, desc.length() - 1).replaceAll("/", DOT); + return Class.forName(className); + } catch (ClassNotFoundException e) { + throw new RuntimeException(e.toString()); + } + } + + private Object getEnumConstant(Class enumClass, String value) { + for (Object constant : enumClass.getEnumConstants()) { + if (value.equalsIgnoreCase(constant.toString())) { + return constant; + } + } + throw new RuntimeException("No enum constant " + value + " in " + enumClass); + } + } + + private class ClassAnnotationVisitor extends MojoAnnotationVisitor { + + ClassAnnotationVisitor(String annotationDescriptor) { + super(classAnnotations, annotationDescriptor); + } + } + + private class FieldAnnotationVisitor extends MojoAnnotationVisitor { + + FieldAnnotationVisitor(Map annotationMap, String desc) { + super(annotationMap, desc); + } + } + + private class MojoFieldVisitor extends FieldVisitor { + + private final Map annotationMap; + + MojoFieldVisitor(Field field) { + super(ASM7); + this.annotationMap = getOrCreateAnnotationMap(field); + } + + @Override + public AnnotationVisitor visitAnnotation(String desc, boolean visible) { + return new FieldAnnotationVisitor(annotationMap, desc); + } + } + + public static class AnnotationInfo { + private final Map fields = new HashMap<>(); + + public Object getField(String name) { + return fields.get(name); + } + } + + abstract static class LogStub implements Log { + private final List infoMessages = new ArrayList<>(); + private final List warningMessages = new ArrayList<>(); + private final List errorMessages = new ArrayList<>(); + + @Override + public boolean isInfoEnabled() { + return true; + } + + @Override + public void info(CharSequence content) { + info(content, null); + } + + @Override + public void info(CharSequence content, Throwable error) { + infoMessages.add(createLogMessage(content, error)); + } + + @Override + public void info(Throwable error) { + info(null, error); + } + + @Nonnull + public static String createLogMessage(CharSequence content, Throwable error) { + final List builder = new ArrayList<>(); + Optional.ofNullable(content).map(CharSequence::toString).ifPresent(builder::add); + Optional.ofNullable(error).map(Throwable::toString).ifPresent(builder::add); + return String.join(System.lineSeparator(), builder); + } + + @Override + public boolean isWarnEnabled() { + return true; + } + + @Override + public void warn(CharSequence content) { + warningMessages.add(createLogMessage(content, null)); + } + + @Override + public boolean isErrorEnabled() { + return true; + } + + @Override + public void error(CharSequence content) { + error(content, null); + } + + @Override + public void error(CharSequence content, Throwable error) { + errorMessages.add(createLogMessage(content, error)); + } + + @Override + public void error(Throwable error) { + error(null, error); + } + } +} diff --git a/json-schema-maven-plugin/src/test/java/oracle/kubernetes/json/mojo/TestFileSystem.java b/operator-build-maven-plugin/src/test/java/oracle/kubernetes/mojosupport/TestFileSystem.java similarity index 62% rename from json-schema-maven-plugin/src/test/java/oracle/kubernetes/json/mojo/TestFileSystem.java rename to operator-build-maven-plugin/src/test/java/oracle/kubernetes/mojosupport/TestFileSystem.java index 9cf3056321f..c14edf9ad11 100644 --- a/json-schema-maven-plugin/src/test/java/oracle/kubernetes/json/mojo/TestFileSystem.java +++ b/operator-build-maven-plugin/src/test/java/oracle/kubernetes/mojosupport/TestFileSystem.java @@ -1,7 +1,7 @@ // Copyright (c) 2018, 2020, Oracle Corporation and/or its affiliates. // Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. -package oracle.kubernetes.json.mojo; +package oracle.kubernetes.mojosupport; import java.io.File; import java.io.FilenameFilter; @@ -9,7 +9,6 @@ import java.io.Reader; import java.io.StringReader; import java.io.Writer; -import java.net.MalformedURLException; import java.net.URL; import java.util.ArrayList; import java.util.HashMap; @@ -17,15 +16,29 @@ import java.util.List; import java.util.Map; import java.util.Set; +import javax.annotation.Nonnull; +@SuppressWarnings("unused") public class TestFileSystem extends FileSystem { + protected static final File[] NO_FILES = new File[0]; long lastModificationTime = 0; - private Map> directories = new HashMap>(); - private Map contents = new HashMap(); - private Set writeOnlyFiles = new HashSet(); - private Map lastModified = new HashMap(); - private Map urls = new HashMap<>(); + private final Map> directories = new HashMap<>(); + private final Map contents = new HashMap<>(); + private final Set writeOnlyFiles = new HashSet<>(); + private final Map lastModified = new HashMap<>(); + private final Map urls = new HashMap<>(); + + /** + * Clear all defined test files. + */ + public void clear() { + directories.clear(); + contents.clear(); + writeOnlyFiles.clear(); + lastModified.clear(); + urls.clear(); + } public void touch(File file) { setLastModified(file, ++lastModificationTime); @@ -57,7 +70,7 @@ private void addDirectoryIfNotDefined(File dir) { return; } addToParent(dir); - directories.put(dir, new ArrayList()); + directories.put(dir, new ArrayList<>()); } public void defineFileContents(File file, String data) { @@ -74,62 +87,66 @@ public void makeWriteOnly(File file) { writeOnlyFiles.add(file); } - void defineUrl(File file, URL url) { + public void defineUrl(File file, URL url) { urls.put(file, url); } @Override - URL toUrl(File file) throws MalformedURLException { + public URL toUrl(File file) { return urls.get(file); } - File[] listFiles(File directory) { - return isDirectory(directory) ? getDirectoryContents(directory, null) : null; + public File[] listFiles(File directory) { + return isDirectory(directory) ? getDirectoryContents(directory, null) : NO_FILES; } - File[] listFiles(File directory, FilenameFilter filter) { - return isDirectory(directory) ? getDirectoryContents(directory, filter) : null; + public File[] listFiles(File directory, FilenameFilter filter) { + return isDirectory(directory) ? getDirectoryContents(directory, filter) : NO_FILES; } private File[] getDirectoryContents(File directory, FilenameFilter filter) { - List files = new ArrayList(); + List files = new ArrayList<>(); for (File file : directories.get(directory)) { if (filter == null || filter.accept(file.getParentFile(), file.getName())) { files.add(file); } } - return files.toArray(new File[files.size()]); + return toArray(files); } private File[] toArray(List files) { - return files.toArray(new File[files.size()]); + return files.toArray(NO_FILES); } - boolean exists(File file) { + public boolean exists(File file) { return contents.containsKey(file) || isDirectory(file); } - boolean isDirectory(File file) { + public boolean isDirectory(File file) { return directories.containsKey(file); } - boolean isWritable(File file) { + public boolean isWritable(File file) { return !writeOnlyFiles.contains(file); } - void createDirectory(File directory) { + /** + * Creates the specified directory. + * @param directory a file which describes a directory + */ + public void createDirectory(File directory) { if (!isDirectory(directory)) { - directories.put(directory, new ArrayList()); + directories.put(directory, new ArrayList<>()); } } @Override - long getLastModified(File file) { + public long getLastModified(File file) { return lastModified.containsKey(file) ? lastModified.get(file) : 0; } @Override - Writer createWriter(File file) throws IOException { + public Writer createWriter(File file) throws IOException { if (!exists(file.getParentFile())) { throw new IOException("Parent directory " + file.getParentFile() + " does not exist"); } @@ -137,46 +154,46 @@ Writer createWriter(File file) throws IOException { } @Override - Reader createReader(File file) throws IOException { + public Reader createReader(File file) { return new TestFileReader(file); } class TestFileWriter extends Writer { - private StringBuilder sb = new StringBuilder(); - private File file; + private final StringBuilder sb = new StringBuilder(); + private final File file; TestFileWriter(File file) { this.file = file; } @Override - public void write(char[] cbuf, int off, int len) throws IOException { + public void write(@Nonnull char[] cbuf, int off, int len) { sb.append(cbuf, off, len); } @Override - public void flush() throws IOException { + public void flush() { } @Override - public void close() throws IOException { + public void close() { contents.put(file, sb.toString()); } } class TestFileReader extends Reader { - private StringReader reader; + private final StringReader reader; TestFileReader(File file) { reader = new StringReader(contents.get(file)); } @Override - public void close() throws IOException { + public void close() { } @Override - public int read(char[] cbuf, int off, int len) throws IOException { + public int read(@Nonnull char[] cbuf, int off, int len) throws IOException { return reader.read(cbuf, off, len); } } diff --git a/operator/pom.xml b/operator/pom.xml index 8e8438a5a4e..ddc3e635505 100644 --- a/operator/pom.xml +++ b/operator/pom.xml @@ -158,10 +158,11 @@ ${project.groupId} - jsonschema-maven-plugin + operator-build-maven-plugin ${project.version} + generate-schema process-classes generate @@ -172,6 +173,16 @@ true + + default-cli + test + + shunit2 + + + src/main/resources/scripts + + diff --git a/operator/src/test/resources/shunit2/2.1.8/shunit2 b/operator/src/test/resources/shunit2/2.1.8/shunit2 new file mode 100755 index 00000000000..6239683af16 --- /dev/null +++ b/operator/src/test/resources/shunit2/2.1.8/shunit2 @@ -0,0 +1,1343 @@ +#! /bin/sh +# vim:et:ft=sh:sts=2:sw=2 +# +# Copyright 2008-2020 Kate Ward. All Rights Reserved. +# Released under the Apache 2.0 license. +# http://www.apache.org/licenses/LICENSE-2.0 +# +# shUnit2 -- Unit testing framework for Unix shell scripts. +# https://github.com/kward/shunit2 +# +# Author: kate.ward@forestent.com (Kate Ward) +# +# shUnit2 is a xUnit based unit test framework for Bourne shell scripts. It is +# based on the popular JUnit unit testing framework for Java. +# +# $() are not fully portable (POSIX != portable). +# shellcheck disable=SC2006 +# expr may be antiquated, but it is the only solution in some cases. +# shellcheck disable=SC2003 + +# Return if shunit2 already loaded. +command [ -n "${SHUNIT_VERSION:-}" ] && exit 0 +SHUNIT_VERSION='2.1.8' + +# Return values that scripts can use. +SHUNIT_TRUE=0 +SHUNIT_FALSE=1 +SHUNIT_ERROR=2 + +# Logging functions. +_shunit_warn() { + ${__SHUNIT_CMD_ECHO_ESC} \ + "${__shunit_ansi_yellow}shunit2:WARN${__shunit_ansi_none} $*" >&2 +} +_shunit_error() { + ${__SHUNIT_CMD_ECHO_ESC} \ + "${__shunit_ansi_red}shunit2:ERROR${__shunit_ansi_none} $*" >&2 +} +_shunit_fatal() { + ${__SHUNIT_CMD_ECHO_ESC} \ + "${__shunit_ansi_red}shunit2:FATAL${__shunit_ansi_none} $*" >&2 + exit ${SHUNIT_ERROR} +} + +# Determine some reasonable command defaults. +__SHUNIT_CMD_ECHO_ESC='echo -e' +# shellcheck disable=SC2039 +command [ "`echo -e test`" = '-e test' ] && __SHUNIT_CMD_ECHO_ESC='echo' + +__SHUNIT_UNAME_S=`uname -s` +case "${__SHUNIT_UNAME_S}" in + BSD) __SHUNIT_CMD_EXPR='gexpr' ;; + *) __SHUNIT_CMD_EXPR='expr' ;; +esac +__SHUNIT_CMD_TPUT='tput' + +# Commands a user can override if needed. +SHUNIT_CMD_EXPR=${SHUNIT_CMD_EXPR:-${__SHUNIT_CMD_EXPR}} +SHUNIT_CMD_TPUT=${SHUNIT_CMD_TPUT:-${__SHUNIT_CMD_TPUT}} + +# Enable color output. Options are 'never', 'always', or 'auto'. +SHUNIT_COLOR=${SHUNIT_COLOR:-auto} + +# Specific shell checks. +if command [ -n "${ZSH_VERSION:-}" ]; then + setopt |grep "^shwordsplit$" >/dev/null + if command [ $? -ne ${SHUNIT_TRUE} ]; then + _shunit_fatal 'zsh shwordsplit option is required for proper operation' + fi + if command [ -z "${SHUNIT_PARENT:-}" ]; then + _shunit_fatal "zsh does not pass \$0 through properly. please declare \ +\"SHUNIT_PARENT=\$0\" before calling shUnit2" + fi +fi + +# +# Constants +# + +__SHUNIT_MODE_SOURCED='sourced' +__SHUNIT_MODE_STANDALONE='standalone' +__SHUNIT_PARENT=${SHUNIT_PARENT:-$0} + +# User provided test prefix to display in front of the name of the test being +# executed. Define by setting the SHUNIT_TEST_PREFIX variable. +__SHUNIT_TEST_PREFIX=${SHUNIT_TEST_PREFIX:-} + +# ANSI colors. +__SHUNIT_ANSI_NONE='\033[0m' +__SHUNIT_ANSI_RED='\033[1;31m' +__SHUNIT_ANSI_GREEN='\033[1;32m' +__SHUNIT_ANSI_YELLOW='\033[1;33m' +__SHUNIT_ANSI_CYAN='\033[1;36m' + +# Set the constants readonly. +__shunit_constants=`set |grep '^__SHUNIT_' |cut -d= -f1` +echo "${__shunit_constants}" |grep '^Binary file' >/dev/null && \ + __shunit_constants=`set |grep -a '^__SHUNIT_' |cut -d= -f1` +for __shunit_const in ${__shunit_constants}; do + if command [ -z "${ZSH_VERSION:-}" ]; then + readonly "${__shunit_const}" + else + case ${ZSH_VERSION} in + [123].*) readonly "${__shunit_const}" ;; + *) readonly -g "${__shunit_const}" # Declare readonly constants globally. + esac + fi +done +unset __shunit_const __shunit_constants + +# +# Internal variables. +# + +# Variables. +__shunit_lineno='' # Line number of executed test. +__shunit_mode=${__SHUNIT_MODE_SOURCED} # Operating mode. +__shunit_reportGenerated=${SHUNIT_FALSE} # Is report generated. +__shunit_script='' # Filename of unittest script (standalone mode). +__shunit_skip=${SHUNIT_FALSE} # Is skipping enabled. +__shunit_suite='' # Suite of tests to execute. +__shunit_clean=${SHUNIT_FALSE} # _shunit_cleanup() was already called. + +# ANSI colors (populated by _shunit_configureColor()). +__shunit_ansi_none='' +__shunit_ansi_red='' +__shunit_ansi_green='' +__shunit_ansi_yellow='' +__shunit_ansi_cyan='' + +# Counts of tests. +__shunit_testSuccess=${SHUNIT_TRUE} +__shunit_testsTotal=0 +__shunit_testsPassed=0 +__shunit_testsFailed=0 + +# Counts of asserts. +__shunit_assertsTotal=0 +__shunit_assertsPassed=0 +__shunit_assertsFailed=0 +__shunit_assertsSkipped=0 + +# +# Macros. +# + +# shellcheck disable=SC2016,SC2089 +_SHUNIT_LINENO_='eval __shunit_lineno=""; if command [ "${1:-}" = "--lineno" ]; then command [ -n "$2" ] && __shunit_lineno="[$2] "; shift 2; fi' + +#----------------------------------------------------------------------------- +# Assertion functions. +# + +# Assert that two values are equal to one another. +# +# Args: +# message: string: failure message [optional] +# expected: string: expected value +# actual: string: actual value +# Returns: +# integer: success (TRUE/FALSE/ERROR constant) +assertEquals() { + # shellcheck disable=SC2090 + ${_SHUNIT_LINENO_} + if command [ $# -lt 2 -o $# -gt 3 ]; then + _shunit_error "assertEquals() requires two or three arguments; $# given" + _shunit_assertFail + return ${SHUNIT_ERROR} + fi + _shunit_shouldSkip && return ${SHUNIT_TRUE} + + shunit_message_=${__shunit_lineno} + if command [ $# -eq 3 ]; then + shunit_message_="${shunit_message_}$1" + shift + fi + shunit_expected_=$1 + shunit_actual_=$2 + + shunit_return=${SHUNIT_TRUE} + if command [ "${shunit_expected_}" = "${shunit_actual_}" ]; then + _shunit_assertPass + else + failNotEquals "${shunit_message_}" "${shunit_expected_}" "${shunit_actual_}" + shunit_return=${SHUNIT_FALSE} + fi + + unset shunit_message_ shunit_expected_ shunit_actual_ + return ${shunit_return} +} +# shellcheck disable=SC2016,SC2034 +_ASSERT_EQUALS_='eval assertEquals --lineno "${LINENO:-}"' + +# Assert that two values are not equal to one another. +# +# Args: +# message: string: failure message [optional] +# expected: string: expected value +# actual: string: actual value +# Returns: +# integer: success (TRUE/FALSE/ERROR constant) +assertNotEquals() { + # shellcheck disable=SC2090 + ${_SHUNIT_LINENO_} + if command [ $# -lt 2 -o $# -gt 3 ]; then + _shunit_error "assertNotEquals() requires two or three arguments; $# given" + _shunit_assertFail + return ${SHUNIT_ERROR} + fi + _shunit_shouldSkip && return ${SHUNIT_TRUE} + + shunit_message_=${__shunit_lineno} + if command [ $# -eq 3 ]; then + shunit_message_="${shunit_message_}$1" + shift + fi + shunit_expected_=$1 + shunit_actual_=$2 + + shunit_return=${SHUNIT_TRUE} + if command [ "${shunit_expected_}" != "${shunit_actual_}" ]; then + _shunit_assertPass + else + failSame "${shunit_message_}" "${shunit_expected_}" "${shunit_actual_}" + shunit_return=${SHUNIT_FALSE} + fi + + unset shunit_message_ shunit_expected_ shunit_actual_ + return ${shunit_return} +} +# shellcheck disable=SC2016,SC2034 +_ASSERT_NOT_EQUALS_='eval assertNotEquals --lineno "${LINENO:-}"' + +# Assert that a container contains a content. +# +# Args: +# message: string: failure message [optional] +# container: string: container to analyze +# content: string: content to find +# Returns: +# integer: success (TRUE/FALSE/ERROR constant) +assertContains() { + # shellcheck disable=SC2090 + ${_SHUNIT_LINENO_} + if command [ $# -lt 2 -o $# -gt 3 ]; then + _shunit_error "assertContains() requires two or three arguments; $# given" + _shunit_assertFail + return ${SHUNIT_ERROR} + fi + _shunit_shouldSkip && return ${SHUNIT_TRUE} + + shunit_message_=${__shunit_lineno} + if command [ $# -eq 3 ]; then + shunit_message_="${shunit_message_}$1" + shift + fi + shunit_container_=$1 + shunit_content_=$2 + + shunit_return=${SHUNIT_TRUE} + if echo "$shunit_container_" | grep -F -- "$shunit_content_" > /dev/null; then + _shunit_assertPass + else + failNotFound "${shunit_message_}" "${shunit_content_}" + shunit_return=${SHUNIT_FALSE} + fi + + unset shunit_message_ shunit_container_ shunit_content_ + return ${shunit_return} +} +# shellcheck disable=SC2016,SC2034 +_ASSERT_CONTAINS_='eval assertContains --lineno "${LINENO:-}"' + +# Assert that a container does not contain a content. +# +# Args: +# message: string: failure message [optional] +# container: string: container to analyze +# content: string: content to look for +# Returns: +# integer: success (TRUE/FALSE/ERROR constant) +assertNotContains() { + # shellcheck disable=SC2090 + ${_SHUNIT_LINENO_} + if command [ $# -lt 2 -o $# -gt 3 ]; then + _shunit_error "assertNotContains() requires two or three arguments; $# given" + _shunit_assertFail + return ${SHUNIT_ERROR} + fi + _shunit_shouldSkip && return ${SHUNIT_TRUE} + + shunit_message_=${__shunit_lineno} + if command [ $# -eq 3 ]; then + shunit_message_="${shunit_message_}$1" + shift + fi + shunit_container_=$1 + shunit_content_=$2 + + shunit_return=${SHUNIT_TRUE} + if echo "$shunit_container_" | grep -F -- "$shunit_content_" > /dev/null; then + failFound "${shunit_message_}" "${shunit_content_}" + shunit_return=${SHUNIT_FALSE} + else + _shunit_assertPass + fi + + unset shunit_message_ shunit_container_ shunit_content_ + return ${shunit_return} +} +# shellcheck disable=SC2016,SC2034 +_ASSERT_NOT_CONTAINS_='eval assertNotContains --lineno "${LINENO:-}"' + +# Assert that a value is null (i.e. an empty string) +# +# Args: +# message: string: failure message [optional] +# actual: string: actual value +# Returns: +# integer: success (TRUE/FALSE/ERROR constant) +assertNull() { + # shellcheck disable=SC2090 + ${_SHUNIT_LINENO_} + if command [ $# -lt 1 -o $# -gt 2 ]; then + _shunit_error "assertNull() requires one or two arguments; $# given" + _shunit_assertFail + return ${SHUNIT_ERROR} + fi + _shunit_shouldSkip && return ${SHUNIT_TRUE} + + shunit_message_=${__shunit_lineno} + if command [ $# -eq 2 ]; then + shunit_message_="${shunit_message_}$1" + shift + fi + assertTrue "${shunit_message_}" "[ -z '$1' ]" + shunit_return=$? + + unset shunit_message_ + return ${shunit_return} +} +# shellcheck disable=SC2016,SC2034 +_ASSERT_NULL_='eval assertNull --lineno "${LINENO:-}"' + +# Assert that a value is not null (i.e. a non-empty string) +# +# Args: +# message: string: failure message [optional] +# actual: string: actual value +# Returns: +# integer: success (TRUE/FALSE/ERROR constant) +assertNotNull() { + # shellcheck disable=SC2090 + ${_SHUNIT_LINENO_} + if command [ $# -gt 2 ]; then # allowing 0 arguments as $1 might actually be null + _shunit_error "assertNotNull() requires one or two arguments; $# given" + _shunit_assertFail + return ${SHUNIT_ERROR} + fi + _shunit_shouldSkip && return ${SHUNIT_TRUE} + + shunit_message_=${__shunit_lineno} + if command [ $# -eq 2 ]; then + shunit_message_="${shunit_message_}$1" + shift + fi + shunit_actual_=`_shunit_escapeCharactersInString "${1:-}"` + test -n "${shunit_actual_}" + assertTrue "${shunit_message_}" $? + shunit_return=$? + + unset shunit_actual_ shunit_message_ + return ${shunit_return} +} +# shellcheck disable=SC2016,SC2034 +_ASSERT_NOT_NULL_='eval assertNotNull --lineno "${LINENO:-}"' + +# Assert that two values are the same (i.e. equal to one another). +# +# Args: +# message: string: failure message [optional] +# expected: string: expected value +# actual: string: actual value +# Returns: +# integer: success (TRUE/FALSE/ERROR constant) +assertSame() { + # shellcheck disable=SC2090 + ${_SHUNIT_LINENO_} + if command [ $# -lt 2 -o $# -gt 3 ]; then + _shunit_error "assertSame() requires two or three arguments; $# given" + _shunit_assertFail + return ${SHUNIT_ERROR} + fi + _shunit_shouldSkip && return ${SHUNIT_TRUE} + + shunit_message_=${__shunit_lineno} + if command [ $# -eq 3 ]; then + shunit_message_="${shunit_message_}$1" + shift + fi + assertEquals "${shunit_message_}" "$1" "$2" + shunit_return=$? + + unset shunit_message_ + return ${shunit_return} +} +# shellcheck disable=SC2016,SC2034 +_ASSERT_SAME_='eval assertSame --lineno "${LINENO:-}"' + +# Assert that two values are not the same (i.e. not equal to one another). +# +# Args: +# message: string: failure message [optional] +# expected: string: expected value +# actual: string: actual value +# Returns: +# integer: success (TRUE/FALSE/ERROR constant) +assertNotSame() { + # shellcheck disable=SC2090 + ${_SHUNIT_LINENO_} + if command [ $# -lt 2 -o $# -gt 3 ]; then + _shunit_error "assertNotSame() requires two or three arguments; $# given" + _shunit_assertFail + return ${SHUNIT_ERROR} + fi + _shunit_shouldSkip && return ${SHUNIT_TRUE} + + shunit_message_=${__shunit_lineno} + if command [ $# -eq 3 ]; then + shunit_message_="${shunit_message_:-}$1" + shift + fi + assertNotEquals "${shunit_message_}" "$1" "$2" + shunit_return=$? + + unset shunit_message_ + return ${shunit_return} +} +# shellcheck disable=SC2016,SC2034 +_ASSERT_NOT_SAME_='eval assertNotSame --lineno "${LINENO:-}"' + +# Assert that a value or shell test condition is true. +# +# In shell, a value of 0 is true and a non-zero value is false. Any integer +# value passed can thereby be tested. +# +# Shell supports much more complicated tests though, and a means to support +# them was needed. As such, this function tests that conditions are true or +# false through evaluation rather than just looking for a true or false. +# +# The following test will succeed: +# assertTrue 0 +# assertTrue "[ 34 -gt 23 ]" +# The following test will fail with a message: +# assertTrue 123 +# assertTrue "test failed" "[ -r '/non/existent/file' ]" +# +# Args: +# message: string: failure message [optional] +# condition: string: integer value or shell conditional statement +# Returns: +# integer: success (TRUE/FALSE/ERROR constant) +assertTrue() { + # shellcheck disable=SC2090 + ${_SHUNIT_LINENO_} + if command [ $# -lt 1 -o $# -gt 2 ]; then + _shunit_error "assertTrue() takes one or two arguments; $# given" + _shunit_assertFail + return ${SHUNIT_ERROR} + fi + _shunit_shouldSkip && return ${SHUNIT_TRUE} + + shunit_message_=${__shunit_lineno} + if command [ $# -eq 2 ]; then + shunit_message_="${shunit_message_}$1" + shift + fi + shunit_condition_=$1 + + # See if condition is an integer, i.e. a return value. + shunit_match_=`expr "${shunit_condition_}" : '\([0-9]*\)'` + shunit_return=${SHUNIT_TRUE} + if command [ -z "${shunit_condition_}" ]; then + # Null condition. + shunit_return=${SHUNIT_FALSE} + elif command [ -n "${shunit_match_}" -a "${shunit_condition_}" = "${shunit_match_}" ] + then + # Possible return value. Treating 0 as true, and non-zero as false. + command [ "${shunit_condition_}" -ne 0 ] && shunit_return=${SHUNIT_FALSE} + else + # Hopefully... a condition. + ( eval "${shunit_condition_}" ) >/dev/null 2>&1 + command [ $? -ne 0 ] && shunit_return=${SHUNIT_FALSE} + fi + + # Record the test. + if command [ ${shunit_return} -eq ${SHUNIT_TRUE} ]; then + _shunit_assertPass + else + _shunit_assertFail "${shunit_message_}" + fi + + unset shunit_message_ shunit_condition_ shunit_match_ + return ${shunit_return} +} +# shellcheck disable=SC2016,SC2034 +_ASSERT_TRUE_='eval assertTrue --lineno "${LINENO:-}"' + +# Assert that a value or shell test condition is false. +# +# In shell, a value of 0 is true and a non-zero value is false. Any integer +# value passed can thereby be tested. +# +# Shell supports much more complicated tests though, and a means to support +# them was needed. As such, this function tests that conditions are true or +# false through evaluation rather than just looking for a true or false. +# +# The following test will succeed: +# assertFalse 1 +# assertFalse "[ 'apples' = 'oranges' ]" +# The following test will fail with a message: +# assertFalse 0 +# assertFalse "test failed" "[ 1 -eq 1 -a 2 -eq 2 ]" +# +# Args: +# message: string: failure message [optional] +# condition: string: integer value or shell conditional statement +# Returns: +# integer: success (TRUE/FALSE/ERROR constant) +assertFalse() { + # shellcheck disable=SC2090 + ${_SHUNIT_LINENO_} + if command [ $# -lt 1 -o $# -gt 2 ]; then + _shunit_error "assertFalse() requires one or two arguments; $# given" + _shunit_assertFail + return ${SHUNIT_ERROR} + fi + _shunit_shouldSkip && return ${SHUNIT_TRUE} + + shunit_message_=${__shunit_lineno} + if command [ $# -eq 2 ]; then + shunit_message_="${shunit_message_}$1" + shift + fi + shunit_condition_=$1 + + # See if condition is an integer, i.e. a return value. + shunit_match_=`expr "${shunit_condition_}" : '\([0-9]*\)'` + shunit_return=${SHUNIT_TRUE} + if command [ -z "${shunit_condition_}" ]; then + # Null condition. + shunit_return=${SHUNIT_FALSE} + elif command [ -n "${shunit_match_}" -a "${shunit_condition_}" = "${shunit_match_}" ] + then + # Possible return value. Treating 0 as true, and non-zero as false. + command [ "${shunit_condition_}" -eq 0 ] && shunit_return=${SHUNIT_FALSE} + else + # Hopefully... a condition. + ( eval "${shunit_condition_}" ) >/dev/null 2>&1 + command [ $? -eq 0 ] && shunit_return=${SHUNIT_FALSE} + fi + + # Record the test. + if command [ "${shunit_return}" -eq "${SHUNIT_TRUE}" ]; then + _shunit_assertPass + else + _shunit_assertFail "${shunit_message_}" + fi + + unset shunit_message_ shunit_condition_ shunit_match_ + return "${shunit_return}" +} +# shellcheck disable=SC2016,SC2034 +_ASSERT_FALSE_='eval assertFalse --lineno "${LINENO:-}"' + +#----------------------------------------------------------------------------- +# Failure functions. +# + +# Records a test failure. +# +# Args: +# message: string: failure message [optional] +# Returns: +# integer: success (TRUE/FALSE/ERROR constant) +fail() { + # shellcheck disable=SC2090 + ${_SHUNIT_LINENO_} + if command [ $# -gt 1 ]; then + _shunit_error "fail() requires zero or one arguments; $# given" + return ${SHUNIT_ERROR} + fi + _shunit_shouldSkip && return ${SHUNIT_TRUE} + + shunit_message_=${__shunit_lineno} + if command [ $# -eq 1 ]; then + shunit_message_="${shunit_message_}$1" + shift + fi + + _shunit_assertFail "${shunit_message_}" + + unset shunit_message_ + return ${SHUNIT_FALSE} +} +# shellcheck disable=SC2016,SC2034 +_FAIL_='eval fail --lineno "${LINENO:-}"' + +# Records a test failure, stating two values were not equal. +# +# Args: +# message: string: failure message [optional] +# expected: string: expected value +# actual: string: actual value +# Returns: +# integer: success (TRUE/FALSE/ERROR constant) +failNotEquals() { + # shellcheck disable=SC2090 + ${_SHUNIT_LINENO_} + if command [ $# -lt 2 -o $# -gt 3 ]; then + _shunit_error "failNotEquals() requires one or two arguments; $# given" + return ${SHUNIT_ERROR} + fi + _shunit_shouldSkip && return ${SHUNIT_TRUE} + + shunit_message_=${__shunit_lineno} + if command [ $# -eq 3 ]; then + shunit_message_="${shunit_message_}$1" + shift + fi + shunit_expected_=$1 + shunit_actual_=$2 + + shunit_message_=${shunit_message_%% } + _shunit_assertFail "${shunit_message_:+${shunit_message_} }expected:<${shunit_expected_}> but was:<${shunit_actual_}>" + + unset shunit_message_ shunit_expected_ shunit_actual_ + return ${SHUNIT_FALSE} +} +# shellcheck disable=SC2016,SC2034 +_FAIL_NOT_EQUALS_='eval failNotEquals --lineno "${LINENO:-}"' + +# Records a test failure, stating a value was found. +# +# Args: +# message: string: failure message [optional] +# content: string: found value +# Returns: +# integer: success (TRUE/FALSE/ERROR constant) +failFound() { + # shellcheck disable=SC2090 + ${_SHUNIT_LINENO_} + if command [ $# -lt 1 -o $# -gt 2 ]; then + _shunit_error "failFound() requires one or two arguments; $# given" + return ${SHUNIT_ERROR} + fi + _shunit_shouldSkip && return ${SHUNIT_TRUE} + + shunit_message_=${__shunit_lineno} + if command [ $# -eq 2 ]; then + shunit_message_="${shunit_message_}$1" + shift + fi + + shunit_message_=${shunit_message_%% } + _shunit_assertFail "${shunit_message_:+${shunit_message_} }Found" + + unset shunit_message_ + return ${SHUNIT_FALSE} +} +# shellcheck disable=SC2016,SC2034 +_FAIL_FOUND_='eval failFound --lineno "${LINENO:-}"' + +# Records a test failure, stating a content was not found. +# +# Args: +# message: string: failure message [optional] +# content: string: content not found +# Returns: +# integer: success (TRUE/FALSE/ERROR constant) +failNotFound() { + # shellcheck disable=SC2090 + ${_SHUNIT_LINENO_} + if command [ $# -lt 1 -o $# -gt 2 ]; then + _shunit_error "failNotFound() requires one or two arguments; $# given" + return ${SHUNIT_ERROR} + fi + _shunit_shouldSkip && return ${SHUNIT_TRUE} + + shunit_message_=${__shunit_lineno} + if command [ $# -eq 2 ]; then + shunit_message_="${shunit_message_}$1" + shift + fi + shunit_content_=$1 + + shunit_message_=${shunit_message_%% } + _shunit_assertFail "${shunit_message_:+${shunit_message_} }Not found:<${shunit_content_}>" + + unset shunit_message_ shunit_content_ + return ${SHUNIT_FALSE} +} +# shellcheck disable=SC2016,SC2034 +_FAIL_NOT_FOUND_='eval failNotFound --lineno "${LINENO:-}"' + +# Records a test failure, stating two values should have been the same. +# +# Args: +# message: string: failure message [optional] +# expected: string: expected value +# actual: string: actual value +# Returns: +# integer: success (TRUE/FALSE/ERROR constant) +failSame() +{ + # shellcheck disable=SC2090 + ${_SHUNIT_LINENO_} + if command [ $# -lt 2 -o $# -gt 3 ]; then + _shunit_error "failSame() requires two or three arguments; $# given" + return ${SHUNIT_ERROR} + fi + _shunit_shouldSkip && return ${SHUNIT_TRUE} + + shunit_message_=${__shunit_lineno} + if command [ $# -eq 3 ]; then + shunit_message_="${shunit_message_}$1" + shift + fi + + shunit_message_=${shunit_message_%% } + _shunit_assertFail "${shunit_message_:+${shunit_message_} }expected not same" + + unset shunit_message_ + return ${SHUNIT_FALSE} +} +# shellcheck disable=SC2016,SC2034 +_FAIL_SAME_='eval failSame --lineno "${LINENO:-}"' + +# Records a test failure, stating two values were not equal. +# +# This is functionally equivalent to calling failNotEquals(). +# +# Args: +# message: string: failure message [optional] +# expected: string: expected value +# actual: string: actual value +# Returns: +# integer: success (TRUE/FALSE/ERROR constant) +failNotSame() { + # shellcheck disable=SC2090 + ${_SHUNIT_LINENO_} + if command [ $# -lt 2 -o $# -gt 3 ]; then + _shunit_error "failNotSame() requires one or two arguments; $# given" + return ${SHUNIT_ERROR} + fi + _shunit_shouldSkip && return ${SHUNIT_TRUE} + + shunit_message_=${__shunit_lineno} + if command [ $# -eq 3 ]; then + shunit_message_="${shunit_message_}$1" + shift + fi + failNotEquals "${shunit_message_}" "$1" "$2" + shunit_return=$? + + unset shunit_message_ + return ${shunit_return} +} +# shellcheck disable=SC2016,SC2034 +_FAIL_NOT_SAME_='eval failNotSame --lineno "${LINENO:-}"' + +#----------------------------------------------------------------------------- +# Skipping functions. +# + +# Force remaining assert and fail functions to be "skipped". +# +# This function forces the remaining assert and fail functions to be "skipped", +# i.e. they will have no effect. Each function skipped will be recorded so that +# the total of asserts and fails will not be altered. +# +# Args: +# None +startSkipping() { __shunit_skip=${SHUNIT_TRUE}; } + +# Resume the normal recording behavior of assert and fail calls. +# +# Args: +# None +endSkipping() { __shunit_skip=${SHUNIT_FALSE}; } + +# Returns the state of assert and fail call skipping. +# +# Args: +# None +# Returns: +# boolean: (TRUE/FALSE constant) +isSkipping() { return ${__shunit_skip}; } + +#----------------------------------------------------------------------------- +# Suite functions. +# + +# Stub. This function should contains all unit test calls to be made. +# +# DEPRECATED (as of 2.1.0) +# +# This function can be optionally overridden by the user in their test suite. +# +# If this function exists, it will be called when shunit2 is sourced. If it +# does not exist, shunit2 will search the parent script for all functions +# beginning with the word 'test', and they will be added dynamically to the +# test suite. +# +# This function should be overridden by the user in their unit test suite. +# Note: see _shunit_mktempFunc() for actual implementation +# +# Args: +# None +#suite() { :; } # DO NOT UNCOMMENT THIS FUNCTION + +# Adds a function name to the list of tests schedule for execution. +# +# This function should only be called from within the suite() function. +# +# Args: +# function: string: name of a function to add to current unit test suite +suite_addTest() { + shunit_func_=${1:-} + + __shunit_suite="${__shunit_suite:+${__shunit_suite} }${shunit_func_}" + __shunit_testsTotal=`expr ${__shunit_testsTotal} + 1` + + unset shunit_func_ +} + +# Stub. This function will be called once before any tests are run. +# +# Common one-time environment preparation tasks shared by all tests can be +# defined here. +# +# This function should be overridden by the user in their unit test suite. +# Note: see _shunit_mktempFunc() for actual implementation +# +# Args: +# None +#oneTimeSetUp() { :; } # DO NOT UNCOMMENT THIS FUNCTION + +# Stub. This function will be called once after all tests are finished. +# +# Common one-time environment cleanup tasks shared by all tests can be defined +# here. +# +# This function should be overridden by the user in their unit test suite. +# Note: see _shunit_mktempFunc() for actual implementation +# +# Args: +# None +#oneTimeTearDown() { :; } # DO NOT UNCOMMENT THIS FUNCTION + +# Stub. This function will be called before each test is run. +# +# Common environment preparation tasks shared by all tests can be defined here. +# +# This function should be overridden by the user in their unit test suite. +# Note: see _shunit_mktempFunc() for actual implementation +# +# Args: +# None +#setUp() { :; } # DO NOT UNCOMMENT THIS FUNCTION + +# Note: see _shunit_mktempFunc() for actual implementation +# Stub. This function will be called after each test is run. +# +# Common environment cleanup tasks shared by all tests can be defined here. +# +# This function should be overridden by the user in their unit test suite. +# Note: see _shunit_mktempFunc() for actual implementation +# +# Args: +# None +#tearDown() { :; } # DO NOT UNCOMMENT THIS FUNCTION + +#------------------------------------------------------------------------------ +# Internal shUnit2 functions. +# + +# Create a temporary directory to store various run-time files in. +# +# This function is a cross-platform temporary directory creation tool. Not all +# OSes have the `mktemp` function, so one is included here. +# +# Args: +# None +# Outputs: +# string: the temporary directory that was created +_shunit_mktempDir() { + # Try the standard `mktemp` function. + ( exec mktemp -dqt shunit.XXXXXX 2>/dev/null ) && return + + # The standard `mktemp` didn't work. Use our own. + # shellcheck disable=SC2039 + if command [ -r '/dev/urandom' -a -x '/usr/bin/od' ]; then + _shunit_random_=`/usr/bin/od -vAn -N4 -tx4 "${_shunit_file_}" +#! /bin/sh +exit ${SHUNIT_TRUE} +EOF + command chmod +x "${_shunit_file_}" + done + + unset _shunit_file_ +} + +# Final cleanup function to leave things as we found them. +# +# Besides removing the temporary directory, this function is in charge of the +# final exit code of the unit test. The exit code is based on how the script +# was ended (e.g. normal exit, or via Ctrl-C). +# +# Args: +# name: string: name of the trap called (specified when trap defined) +_shunit_cleanup() { + _shunit_name_=$1 + + case "${_shunit_name_}" in + EXIT) ;; + INT) _shunit_signal_=130 ;; # 2+128 + TERM) _shunit_signal_=143 ;; # 15+128 + *) + _shunit_error "unrecognized trap value (${_shunit_name_})" + _shunit_signal_=0 + ;; + esac + if command [ "${_shunit_name_}" != 'EXIT' ]; then + _shunit_warn "trapped and now handling the (${_shunit_name_}) signal" + fi + + # Do our work. + if command [ ${__shunit_clean} -eq ${SHUNIT_FALSE} ]; then + # Ensure tear downs are only called once. + __shunit_clean=${SHUNIT_TRUE} + + tearDown + command [ $? -eq ${SHUNIT_TRUE} ] \ + || _shunit_warn "tearDown() returned non-zero return code." + oneTimeTearDown + command [ $? -eq ${SHUNIT_TRUE} ] \ + || _shunit_warn "oneTimeTearDown() returned non-zero return code." + + command rm -fr "${__shunit_tmpDir}" + fi + + if command [ "${_shunit_name_}" != 'EXIT' ]; then + # Handle all non-EXIT signals. + trap - 0 # Disable EXIT trap. + exit ${_shunit_signal_} + elif command [ ${__shunit_reportGenerated} -eq ${SHUNIT_FALSE} ]; then + _shunit_assertFail 'unknown failure encountered running a test' + _shunit_generateReport + exit ${SHUNIT_ERROR} + fi + + unset _shunit_name_ _shunit_signal_ +} + +# configureColor based on user color preference. +# +# Args: +# color: string: color mode (one of `always`, `auto`, or `none`). +_shunit_configureColor() { + _shunit_color_=${SHUNIT_FALSE} # By default, no color. + case $1 in + 'always') _shunit_color_=${SHUNIT_TRUE} ;; + 'auto') + command [ "`_shunit_colors`" -ge 8 ] && _shunit_color_=${SHUNIT_TRUE} + ;; + 'none') ;; + *) _shunit_fatal "unrecognized color option '$1'" ;; + esac + + case ${_shunit_color_} in + ${SHUNIT_TRUE}) + __shunit_ansi_none=${__SHUNIT_ANSI_NONE} + __shunit_ansi_red=${__SHUNIT_ANSI_RED} + __shunit_ansi_green=${__SHUNIT_ANSI_GREEN} + __shunit_ansi_yellow=${__SHUNIT_ANSI_YELLOW} + __shunit_ansi_cyan=${__SHUNIT_ANSI_CYAN} + ;; + ${SHUNIT_FALSE}) + __shunit_ansi_none='' + __shunit_ansi_red='' + __shunit_ansi_green='' + __shunit_ansi_yellow='' + __shunit_ansi_cyan='' + ;; + esac + + unset _shunit_color_ _shunit_tput_ +} + +# colors returns the number of supported colors for the TERM. +_shunit_colors() { + _shunit_tput_=`${SHUNIT_CMD_TPUT} colors 2>/dev/null` + if command [ $? -eq 0 ]; then + echo "${_shunit_tput_}" + else + echo 16 + fi + unset _shunit_tput_ +} + +# The actual running of the tests happens here. +# +# Args: +# None +_shunit_execSuite() { + for _shunit_test_ in ${__shunit_suite}; do + __shunit_testSuccess=${SHUNIT_TRUE} + + # Disable skipping. + endSkipping + + # Execute the per-test setup function. + setUp + command [ $? -eq ${SHUNIT_TRUE} ] \ + || _shunit_fatal "setup() returned non-zero return code." + + # Execute the test. + echo "${__SHUNIT_TEST_PREFIX}${_shunit_test_}" + eval "${_shunit_test_}" + if command [ $? -ne ${SHUNIT_TRUE} ]; then + _shunit_error "${_shunit_test_}() returned non-zero return code." + __shunit_testSuccess=${SHUNIT_ERROR} + _shunit_incFailedCount + fi + + # Execute the per-test tear-down function. + tearDown + command [ $? -eq ${SHUNIT_TRUE} ] \ + || _shunit_fatal "tearDown() returned non-zero return code." + + # Update stats. + if command [ ${__shunit_testSuccess} -eq ${SHUNIT_TRUE} ]; then + __shunit_testsPassed=`expr ${__shunit_testsPassed} + 1` + else + __shunit_testsFailed=`expr ${__shunit_testsFailed} + 1` + fi + done + + unset _shunit_test_ +} + +# Generates the user friendly report with appropriate OK/FAILED message. +# +# Args: +# None +# Output: +# string: the report of successful and failed tests, as well as totals. +_shunit_generateReport() { + command [ "${__shunit_reportGenerated}" -eq ${SHUNIT_TRUE} ] && return + + _shunit_ok_=${SHUNIT_TRUE} + + # If no exit code was provided, determine an appropriate one. + command [ "${__shunit_testsFailed}" -gt 0 \ + -o ${__shunit_testSuccess} -eq ${SHUNIT_FALSE} ] \ + && _shunit_ok_=${SHUNIT_FALSE} + + echo + _shunit_msg_="Ran ${__shunit_ansi_cyan}${__shunit_testsTotal}${__shunit_ansi_none}" + if command [ "${__shunit_testsTotal}" -eq 1 ]; then + ${__SHUNIT_CMD_ECHO_ESC} "${_shunit_msg_} test." + else + ${__SHUNIT_CMD_ECHO_ESC} "${_shunit_msg_} tests." + fi + + if command [ ${_shunit_ok_} -eq ${SHUNIT_TRUE} ]; then + _shunit_msg_="${__shunit_ansi_green}OK${__shunit_ansi_none}" + command [ "${__shunit_assertsSkipped}" -gt 0 ] \ + && _shunit_msg_="${_shunit_msg_} (${__shunit_ansi_yellow}skipped=${__shunit_assertsSkipped}${__shunit_ansi_none})" + else + _shunit_msg_="${__shunit_ansi_red}FAILED${__shunit_ansi_none}" + _shunit_msg_="${_shunit_msg_} (${__shunit_ansi_red}failures=${__shunit_assertsFailed}${__shunit_ansi_none}" + command [ "${__shunit_assertsSkipped}" -gt 0 ] \ + && _shunit_msg_="${_shunit_msg_},${__shunit_ansi_yellow}skipped=${__shunit_assertsSkipped}${__shunit_ansi_none}" + _shunit_msg_="${_shunit_msg_})" + fi + + echo + ${__SHUNIT_CMD_ECHO_ESC} "${_shunit_msg_}" + __shunit_reportGenerated=${SHUNIT_TRUE} + + unset _shunit_msg_ _shunit_ok_ +} + +# Test for whether a function should be skipped. +# +# Args: +# None +# Returns: +# boolean: whether the test should be skipped (TRUE/FALSE constant) +_shunit_shouldSkip() { + command [ ${__shunit_skip} -eq ${SHUNIT_FALSE} ] && return ${SHUNIT_FALSE} + _shunit_assertSkip +} + +# Records a successful test. +# +# Args: +# None +_shunit_assertPass() { + __shunit_assertsPassed=`expr ${__shunit_assertsPassed} + 1` + __shunit_assertsTotal=`expr ${__shunit_assertsTotal} + 1` +} + +# Records a test failure. +# +# Args: +# message: string: failure message to provide user +_shunit_assertFail() { + __shunit_testSuccess=${SHUNIT_FALSE} + _shunit_incFailedCount + + \[ $# -gt 0 ] && ${__SHUNIT_CMD_ECHO_ESC} \ + "${__shunit_ansi_red}ASSERT:${__shunit_ansi_none}$*" +} + +# Increment the count of failed asserts. +# +# Args: +# none +_shunit_incFailedCount() { + __shunit_assertsFailed=`expr "${__shunit_assertsFailed}" + 1` + __shunit_assertsTotal=`expr "${__shunit_assertsTotal}" + 1` +} + + +# Records a skipped test. +# +# Args: +# None +_shunit_assertSkip() { + __shunit_assertsSkipped=`expr "${__shunit_assertsSkipped}" + 1` + __shunit_assertsTotal=`expr "${__shunit_assertsTotal}" + 1` +} + +# Prepare a script filename for sourcing. +# +# Args: +# script: string: path to a script to source +# Returns: +# string: filename prefixed with ./ (if necessary) +_shunit_prepForSourcing() { + _shunit_script_=$1 + case "${_shunit_script_}" in + /*|./*) echo "${_shunit_script_}" ;; + *) echo "./${_shunit_script_}" ;; + esac + unset _shunit_script_ +} + +# Escape a character in a string. +# +# Args: +# c: string: unescaped character +# s: string: to escape character in +# Returns: +# string: with escaped character(s) +_shunit_escapeCharInStr() { + command [ -n "$2" ] || return # No point in doing work on an empty string. + + # Note: using shorter variable names to prevent conflicts with + # _shunit_escapeCharactersInString(). + _shunit_c_=$1 + _shunit_s_=$2 + + # Escape the character. + # shellcheck disable=SC1003,SC2086 + echo ''${_shunit_s_}'' |command sed 's/\'${_shunit_c_}'/\\\'${_shunit_c_}'/g' + + unset _shunit_c_ _shunit_s_ +} + +# Escape a character in a string. +# +# Args: +# str: string: to escape characters in +# Returns: +# string: with escaped character(s) +_shunit_escapeCharactersInString() { + command [ -n "$1" ] || return # No point in doing work on an empty string. + + _shunit_str_=$1 + + # Note: using longer variable names to prevent conflicts with + # _shunit_escapeCharInStr(). + for _shunit_char_ in '"' '$' "'" '`'; do + _shunit_str_=`_shunit_escapeCharInStr "${_shunit_char_}" "${_shunit_str_}"` + done + + echo "${_shunit_str_}" + unset _shunit_char_ _shunit_str_ +} + +# Extract list of functions to run tests against. +# +# Args: +# script: string: name of script to extract functions from +# Returns: +# string: of function names +_shunit_extractTestFunctions() { + _shunit_script_=$1 + + # Extract the lines with test function names, strip of anything besides the + # function name, and output everything on a single line. + _shunit_regex_='^\s*((function test[A-Za-z0-9_-]*)|(test[A-Za-z0-9_-]* *\(\)))' + # shellcheck disable=SC2196 + egrep "${_shunit_regex_}" "${_shunit_script_}" \ + |command sed 's/^[^A-Za-z0-9_-]*//;s/^function //;s/\([A-Za-z0-9_-]*\).*/\1/g' \ + |xargs + + unset _shunit_regex_ _shunit_script_ +} + +#------------------------------------------------------------------------------ +# Main. +# + +# Determine the operating mode. +if command [ $# -eq 0 -o "${1:-}" = '--' ]; then + __shunit_script=${__SHUNIT_PARENT} + __shunit_mode=${__SHUNIT_MODE_SOURCED} +else + __shunit_script=$1 + command [ -r "${__shunit_script}" ] || \ + _shunit_fatal "unable to read from ${__shunit_script}" + __shunit_mode=${__SHUNIT_MODE_STANDALONE} +fi + +# Create a temporary storage location. +__shunit_tmpDir=`_shunit_mktempDir` + +# Provide a public temporary directory for unit test scripts. +# TODO(kward): document this. +SHUNIT_TMPDIR="${__shunit_tmpDir}/tmp" +command mkdir "${SHUNIT_TMPDIR}" + +# Setup traps to clean up after ourselves. +trap '_shunit_cleanup EXIT' 0 +trap '_shunit_cleanup INT' 2 +trap '_shunit_cleanup TERM' 15 + +# Create phantom functions to work around issues with Cygwin. +_shunit_mktempFunc +PATH="${__shunit_tmpDir}:${PATH}" + +# Make sure phantom functions are executable. This will bite if `/tmp` (or the +# current `$TMPDIR`) points to a path on a partition that was mounted with the +# 'noexec' option. The noexec command was created with `_shunit_mktempFunc()`. +noexec 2>/dev/null || _shunit_fatal \ + 'Please declare TMPDIR with path on partition with exec permission.' + +# We must manually source the tests in standalone mode. +if command [ "${__shunit_mode}" = "${__SHUNIT_MODE_STANDALONE}" ]; then + # shellcheck disable=SC1090 + command . "`_shunit_prepForSourcing \"${__shunit_script}\"`" +fi + +# Configure default output coloring behavior. +_shunit_configureColor "${SHUNIT_COLOR}" + +# Execute the oneTimeSetUp function (if it exists). +oneTimeSetUp +command [ $? -eq ${SHUNIT_TRUE} ] \ + || _shunit_fatal "oneTimeSetUp() returned non-zero return code." + +# Command line selected tests or suite selected tests +if command [ "$#" -ge 2 ]; then + # Argument $1 is either the filename of tests or '--'; either way, skip it. + shift + # Remaining arguments ($2 .. $#) are assumed to be test function names. + # Interate through all remaining args in "$@" in a POSIX (likely portable) way. + # Helpful tip: https://unix.stackexchange.com/questions/314032/how-to-use-arguments-like-1-2-in-a-for-loop + for _shunit_arg_ do + suite_addTest "${_shunit_arg_}" + done + unset _shunit_arg_ +else + # Execute the suite function defined in the parent test script. + # DEPRECATED as of 2.1.0. + suite +fi + +# If no tests or suite specified, dynamically build a list of functions. +if command [ -z "${__shunit_suite}" ]; then + shunit_funcs_=`_shunit_extractTestFunctions "${__shunit_script}"` + for shunit_func_ in ${shunit_funcs_}; do + suite_addTest "${shunit_func_}" + done +fi +unset shunit_func_ shunit_funcs_ + +# Execute the suite of unit tests. +_shunit_execSuite + +# Execute the oneTimeTearDown function (if it exists). +oneTimeTearDown +command [ $? -eq ${SHUNIT_TRUE} ] \ + || _shunit_fatal "oneTimeTearDown() returned non-zero return code." + +# Generate a report summary. +_shunit_generateReport + +# That's it folks. +command [ "${__shunit_testsFailed}" -eq 0 ] +exit $? diff --git a/pom.xml b/pom.xml index 3c98f5ae19b..084f34ced1e 100644 --- a/pom.xml +++ b/pom.xml @@ -14,7 +14,7 @@ swagger integration-tests kubernetes - json-schema-maven-plugin + operator-build-maven-plugin json-schema buildtime-reports From 34e966a47aefc4cc44559c82e4a126b2ade6e838 Mon Sep 17 00:00:00 2001 From: Russell Gold Date: Thu, 10 Dec 2020 11:01:07 -0500 Subject: [PATCH 2/6] Support splitting encoded zips across multiple config maps --- .../IntrospectorConfigMapConstants.java | 66 ++++++ .../operator/IntrospectorConfigMapKeys.java | 31 --- .../operator/KubernetesConstants.java | 1 - .../operator/helpers/ConfigMapHelper.java | 190 +++++++++++++--- .../operator/helpers/ConfigMapSplitter.java | 154 +++++++++++++ .../operator/helpers/FileGroupReader.java | 2 +- .../operator/helpers/JobHelper.java | 4 +- .../operator/helpers/JobStepContext.java | 7 +- .../operator/helpers/PodDefaults.java | 45 ++-- .../operator/helpers/PodStepContext.java | 36 ++- .../operator/helpers/SplitterTarget.java | 25 +++ .../helpers/StepContextConstants.java | 2 +- .../main/resources/scripts/modelInImage.sh | 68 +++++- .../src/main/resources/scripts/startServer.sh | 15 +- .../operator/DomainProcessorTest.java | 13 +- .../oracle/kubernetes/operator/MainTest.java | 8 - .../operator/helpers/ConfigMapHelperTest.java | 28 ++- .../helpers/ConfigMapSplitterTest.java | 189 ++++++++++++++++ .../operator/helpers/FileGroupReaderTest.java | 15 +- .../helpers/IntrospectorConfigMapTest.java | 208 +++++++++++++----- .../kubernetes/operator/helpers/Matchers.java | 68 +++++- .../operator/helpers/PodHelperTestBase.java | 63 +++++- .../operator/utils/InMemoryFileSystem.java | 19 +- operator/src/test/sh/modelInImageTest.sh | 207 +++++++++++++++++ 24 files changed, 1237 insertions(+), 227 deletions(-) create mode 100644 operator/src/main/java/oracle/kubernetes/operator/IntrospectorConfigMapConstants.java delete mode 100644 operator/src/main/java/oracle/kubernetes/operator/IntrospectorConfigMapKeys.java create mode 100644 operator/src/main/java/oracle/kubernetes/operator/helpers/ConfigMapSplitter.java create mode 100644 operator/src/main/java/oracle/kubernetes/operator/helpers/SplitterTarget.java create mode 100644 operator/src/test/java/oracle/kubernetes/operator/helpers/ConfigMapSplitterTest.java create mode 100644 operator/src/test/sh/modelInImageTest.sh diff --git a/operator/src/main/java/oracle/kubernetes/operator/IntrospectorConfigMapConstants.java b/operator/src/main/java/oracle/kubernetes/operator/IntrospectorConfigMapConstants.java new file mode 100644 index 00000000000..3a55d22fd12 --- /dev/null +++ b/operator/src/main/java/oracle/kubernetes/operator/IntrospectorConfigMapConstants.java @@ -0,0 +1,66 @@ +// Copyright (c) 2020, Oracle Corporation and/or its affiliates. +// Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. + +package oracle.kubernetes.operator; + +import javax.annotation.Nonnull; + +/** + * Constants for generating introspector config maps. + */ +public interface IntrospectorConfigMapConstants { + + /** The topology generated from the WebLogic domain. */ + String TOPOLOGY_YAML = "topology.yaml"; + + /** An MD5 has of the Model-in-Image secrets. */ + String SECRETS_MD_5 = "secrets.md5"; + + /** A hash computed from the WebLogic domain. */ + String DOMAINZIP_HASH = "domainzip_hash"; + + /** The last value of the restartVersion field from the domain resource. */ + String DOMAIN_RESTART_VERSION = "weblogic.domainRestartVersion"; + + /** A hash of the Model-in-Image inputs. */ + String DOMAIN_INPUTS_HASH = "weblogic.domainInputsHash"; + + /** The number of config maps required to hold the encoded domains. */ + String NUM_CONFIG_MAPS = "numConfigMaps"; + + /** + * The prefix for a number of keys which may appear in the introspector config map. + * They are not preserved from one update to another. + */ + String SIT_CONFIG_FILE_PREFIX = "Sit-Cfg"; + + /** The suffix for naming introspector config maps. */ + String INTROSPECTOR_CONFIG_MAP_NAME_SUFFIX = "-weblogic-domain-introspect-cm"; + + /** + * Returns the name of an introspector config map. + * @param domainUid the unique UID for the domain containing the map + * @param index the index of the config map + */ + static String getIntrospectorConfigMapName(String domainUid, int index) { + return getIntrospectorConfigMapNamePrefix(domainUid) + suffix(index); + } + + static String getIntrospectorConfigMapNamePrefix(String uid) { + return uid + INTROSPECTOR_CONFIG_MAP_NAME_SUFFIX; + } + + /** A (possibly empty) suffix for introspector config maps. */ + static String suffix(int index) { + return index == 0 ? "" : "_" + index; + } + + /** + * Returns the mount path for an introspector volume mount. + * @param index the index of the mount / config map + */ + @Nonnull + static String getIntrospectorVolumePath(int index) { + return "/weblogic-operator/introspector" + suffix(index); + } +} diff --git a/operator/src/main/java/oracle/kubernetes/operator/IntrospectorConfigMapKeys.java b/operator/src/main/java/oracle/kubernetes/operator/IntrospectorConfigMapKeys.java deleted file mode 100644 index 3a55d9df5dd..00000000000 --- a/operator/src/main/java/oracle/kubernetes/operator/IntrospectorConfigMapKeys.java +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright (c) 2020, Oracle Corporation and/or its affiliates. -// Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. - -package oracle.kubernetes.operator; - -/** - * Keys in the generated introspector config map. - */ -public interface IntrospectorConfigMapKeys { - - /** The topology generated from the WebLogic domain. */ - String TOPOLOGY_YAML = "topology.yaml"; - - /** An MD5 has of the Model-in-Image secrets. */ - String SECRETS_MD_5 = "secrets.md5"; - - /** A hash computed from the WebLogic domain. */ - String DOMAINZIP_HASH = "domainzip_hash"; - - /** The last value of the restartVersion field from the domain resource. */ - String DOMAIN_RESTART_VERSION = "weblogic.domainRestartVersion"; - - /** A hash of the Model-in-Image inputs. */ - String DOMAIN_INPUTS_HASH = "weblogic.domainInputsHash"; - - /** - * The prefix for a number of keys which may appear in the introspector config map. - * They are not preserved from one update to another. - */ - String SIT_CONFIG_FILE_PREFIX = "Sit-Cfg"; -} diff --git a/operator/src/main/java/oracle/kubernetes/operator/KubernetesConstants.java b/operator/src/main/java/oracle/kubernetes/operator/KubernetesConstants.java index 496c89db6a2..8640dfc13ef 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/KubernetesConstants.java +++ b/operator/src/main/java/oracle/kubernetes/operator/KubernetesConstants.java @@ -35,7 +35,6 @@ public interface KubernetesConstants { String SCRIPT_CONFIG_MAP_NAME = "weblogic-scripts-cm"; String DOMAIN_DEBUG_CONFIG_MAP_SUFFIX = "-weblogic-domain-debug-cm"; - String INTROSPECTOR_CONFIG_MAP_NAME_SUFFIX = "-weblogic-domain-introspect-cm"; String GRACEFUL_SHUTDOWNTYPE = ShutdownType.Graceful.name(); diff --git a/operator/src/main/java/oracle/kubernetes/operator/helpers/ConfigMapHelper.java b/operator/src/main/java/oracle/kubernetes/operator/helpers/ConfigMapHelper.java index 8b8232ce65f..c331373cbe3 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/helpers/ConfigMapHelper.java +++ b/operator/src/main/java/oracle/kubernetes/operator/helpers/ConfigMapHelper.java @@ -6,12 +6,17 @@ import java.io.BufferedReader; import java.io.IOException; import java.io.StringReader; +import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +import javax.annotation.Nonnull; import javax.json.Json; import javax.json.JsonPatchBuilder; import javax.json.JsonValue; @@ -19,11 +24,11 @@ import io.kubernetes.client.custom.V1Patch; import io.kubernetes.client.openapi.models.V1ConfigMap; +import io.kubernetes.client.openapi.models.V1ConfigMapList; import io.kubernetes.client.openapi.models.V1DeleteOptions; import io.kubernetes.client.openapi.models.V1ObjectMeta; import oracle.kubernetes.operator.DomainStatusUpdater; -import oracle.kubernetes.operator.IntrospectorConfigMapKeys; -import oracle.kubernetes.operator.KubernetesConstants; +import oracle.kubernetes.operator.IntrospectorConfigMapConstants; import oracle.kubernetes.operator.LabelConstants; import oracle.kubernetes.operator.ProcessingConstants; import oracle.kubernetes.operator.calls.CallResponse; @@ -43,11 +48,12 @@ import static java.lang.System.lineSeparator; import static oracle.kubernetes.operator.DomainStatusUpdater.BAD_TOPOLOGY; -import static oracle.kubernetes.operator.IntrospectorConfigMapKeys.DOMAINZIP_HASH; -import static oracle.kubernetes.operator.IntrospectorConfigMapKeys.DOMAIN_INPUTS_HASH; -import static oracle.kubernetes.operator.IntrospectorConfigMapKeys.DOMAIN_RESTART_VERSION; -import static oracle.kubernetes.operator.IntrospectorConfigMapKeys.SECRETS_MD_5; -import static oracle.kubernetes.operator.IntrospectorConfigMapKeys.SIT_CONFIG_FILE_PREFIX; +import static oracle.kubernetes.operator.IntrospectorConfigMapConstants.DOMAINZIP_HASH; +import static oracle.kubernetes.operator.IntrospectorConfigMapConstants.DOMAIN_INPUTS_HASH; +import static oracle.kubernetes.operator.IntrospectorConfigMapConstants.DOMAIN_RESTART_VERSION; +import static oracle.kubernetes.operator.IntrospectorConfigMapConstants.NUM_CONFIG_MAPS; +import static oracle.kubernetes.operator.IntrospectorConfigMapConstants.SECRETS_MD_5; +import static oracle.kubernetes.operator.IntrospectorConfigMapConstants.SIT_CONFIG_FILE_PREFIX; import static oracle.kubernetes.operator.KubernetesConstants.SCRIPT_CONFIG_MAP_NAME; import static oracle.kubernetes.operator.LabelConstants.INTROSPECTION_STATE_LABEL; import static oracle.kubernetes.operator.ProcessingConstants.DOMAIN_VALIDATION_ERRORS; @@ -144,12 +150,12 @@ public static int getModelInImageSpecHash(String imageName) { } /** - * Returns the standard name for the generated domain config map. + * Returns the standard name for the introspector config map. * @param domainUid the unique ID of the domain * @return map name */ public static String getIntrospectorConfigMapName(String domainUid) { - return domainUid + KubernetesConstants.INTROSPECTOR_CONFIG_MAP_NAME_SUFFIX; + return IntrospectorConfigMapConstants.getIntrospectorConfigMapName(domainUid, 0); } abstract static class ConfigMapComparator { @@ -227,6 +233,10 @@ abstract static class ConfigMapContext extends StepContextBase { void recordCurrentMap(Packet packet, V1ConfigMap configMap) { } + void setContentValue(String key, String value) { + contents.put(key, value); + } + protected String getName() { return name; } @@ -352,7 +362,7 @@ private Step patchCurrentMap(V1ConfigMap currentMap, Step next) { return new CallBuilder() .patchConfigMapAsync(name, namespace, - getDomainUidLabel(Optional.ofNullable(currentMap).map(V1ConfigMap::getMetadata).orElse(null)), + getDomainUidLabel(Optional.of(currentMap).map(V1ConfigMap::getMetadata).orElse(null)), new V1Patch(patchBuilder.build().toString()), createPatchResponseStep(next)); } @@ -488,6 +498,7 @@ private long timeSinceJobStart(Packet packet) { } static class IntrospectionLoader { + private final Packet packet; private final Step conflictStep; private final DomainPresenceInfo info; @@ -509,7 +520,7 @@ private void parseIntrospectorResult() { LOGGER.fine(data.toString()); LOGGER.fine("================"); - wlsDomainConfig = Optional.ofNullable(data.get(IntrospectorConfigMapKeys.TOPOLOGY_YAML)) + wlsDomainConfig = Optional.ofNullable(data.get(IntrospectorConfigMapConstants.TOPOLOGY_YAML)) .map(this::getDomainTopology) .map(DomainTopology::getDomain) .orElse(null); @@ -532,16 +543,42 @@ private void updatePacket() { private Step createIntrospectionVersionUpdateStep() { return DomainValidationSteps.createValidateDomainTopologyStep( - createIntrospectorConfigMapContext(conflictStep).patchOnly().verifyConfigMap(conflictStep.getNext())); + createIntrospectorConfigMapContext().patchOnly().verifyConfigMap(conflictStep.getNext())); } private Step createValidationStep() { - return DomainValidationSteps.createValidateDomainTopologyStep( - createIntrospectorConfigMapContext(conflictStep).verifyConfigMap(conflictStep.getNext())); + return Step.chain( + DomainValidationSteps.createValidateDomainTopologyStep(null), + new IntrospectionConfigMapStep(data, conflictStep.getNext())); + } + + private class IntrospectionConfigMapStep extends Step { + private final Map data; + private final ConfigMapSplitter splitter; + + IntrospectionConfigMapStep(Map data, Step next) { + super(next); + this.splitter = new ConfigMapSplitter<>(IntrospectionLoader.this::createIntrospectorConfigMapContext); + this.data = data; + } + + @Override + public NextAction apply(Packet packet) { + Collection startDetails = splitter.split(data).stream() + .map(c -> c.createStepAndPacket(packet)) + .collect(Collectors.toList()); + packet.put(NUM_CONFIG_MAPS, Integer.toString(startDetails.size())); + return doForkJoin(getNext(), packet, startDetails); + } + } - private IntrospectorConfigMapContext createIntrospectorConfigMapContext(Step conflictStep) { - return new IntrospectorConfigMapContext(conflictStep, info.getDomain(), data, info); + private IntrospectorConfigMapContext createIntrospectorConfigMapContext() { + return createIntrospectorConfigMapContext(data, 0); + } + + private IntrospectorConfigMapContext createIntrospectorConfigMapContext(Map data, int index) { + return new IntrospectorConfigMapContext(conflictStep, info, data, index); } private String getModelInImageSpecHash() { @@ -604,19 +641,40 @@ private String perLine(List errors) { } } - public static class IntrospectorConfigMapContext extends ConfigMapContext { - final String domainUid; + public static class IntrospectorConfigMapContext extends ConfigMapContext implements SplitterTarget { + + private static final Pattern ENCODED_ZIP_PATTERN = Pattern.compile("([A-Za-z_]+)\\.secure"); + private boolean patchOnly; - IntrospectorConfigMapContext( - Step conflictStep, - Domain domain, - Map data, - DomainPresenceInfo info) { - super(conflictStep, getIntrospectorConfigMapName(domain.getDomainUid()), domain.getNamespace(), data, info); + IntrospectorConfigMapContext(Step conflictStep, DomainPresenceInfo info, Map data, int index) { + super(conflictStep, getConfigMapName(info, index), info.getNamespace(), data, info); - this.domainUid = domain.getDomainUid(); - addLabel(LabelConstants.DOMAINUID_LABEL, domainUid); + addLabel(LabelConstants.DOMAINUID_LABEL, info.getDomainUid()); + } + + private static String getConfigMapName(DomainPresenceInfo info, int index) { + return IntrospectorConfigMapConstants.getIntrospectorConfigMapName(info.getDomainUid(), index); + } + + @Override + public void recordNumTargets(int numTargets) { + setContentValue(NUM_CONFIG_MAPS, Integer.toString(numTargets)); + } + + @Override + public void recordEntryLocation(String key, int firstTarget, int lastTarget) { + if (isEncodedZip(key)) { + setContentValue(createRangeName(key), firstTarget + " " + lastTarget); + } + } + + private boolean isEncodedZip(String key) { + return ENCODED_ZIP_PATTERN.matcher(key).matches(); + } + + private String createRangeName(String key) { + return key + ".range"; } IntrospectorConfigMapContext patchOnly() { @@ -643,6 +701,9 @@ private boolean isRemovableKey(String key) { return key.startsWith(SIT_CONFIG_FILE_PREFIX); } + public Step.StepAndPacket createStepAndPacket(Packet packet) { + return new Step.StepAndPacket(verifyConfigMap(null), packet.clone()); + } } /** @@ -654,19 +715,84 @@ private boolean isRemovableKey(String key) { * @return the created step */ public static Step deleteIntrospectorConfigMapStep(String domainUid, String namespace, Step next) { - return new DeleteIntrospectorConfigMapStep(domainUid, namespace, next); + return new DeleteIntrospectorConfigMapsStep(domainUid, namespace, next); } - private static class DeleteIntrospectorConfigMapStep extends Step { + private static class DeleteIntrospectorConfigMapsStep extends Step { private final String domainUid; private final String namespace; - DeleteIntrospectorConfigMapStep(String domainUid, String namespace, Step next) { + private DeleteIntrospectorConfigMapsStep(String domainUid, String namespace, Step next) { super(next); this.domainUid = domainUid; this.namespace = namespace; } + @Override + public NextAction apply(Packet packet) { + Step step = new CallBuilder() + .withLabelSelectors(LabelConstants.getCreatedbyOperatorSelector()) + .listConfigMapsAsync(namespace, new SelectConfigMapsToDeleteStep(domainUid, namespace)); + + return doNext(step, packet); + } + } + + private static class SelectConfigMapsToDeleteStep extends DefaultResponseStep { + private final String domainUid; + private final String namespace; + + public SelectConfigMapsToDeleteStep(String domainUid, String namespace) { + this.domainUid = domainUid; + this.namespace = namespace; + } + + @Override + public NextAction onSuccess(Packet packet, CallResponse callResponse) { + final List configMapNames = getIntrospectorConfigMapNames(callResponse.getResult()); + if (configMapNames.isEmpty()) { + return doNext(packet); + } else { + Collection startDetails = new ArrayList<>(); + for (String configMapName : configMapNames) { + startDetails.add(new StepAndPacket( + new DeleteIntrospectorConfigMapStep(domainUid, namespace, configMapName), packet)); + } + return doForkJoin(getNext(), packet, startDetails); + } + } + + @Nonnull + protected List getIntrospectorConfigMapNames(V1ConfigMapList list) { + return list.getItems().stream() + .map(this::getName) + .filter(this::isIntrospectorConfigMapName) + .collect(Collectors.toList()); + } + + private boolean isIntrospectorConfigMapName(String name) { + return name.startsWith(IntrospectorConfigMapConstants.getIntrospectorConfigMapNamePrefix(domainUid)); + } + + @Nonnull + private String getName(V1ConfigMap configMap) { + return Optional.ofNullable(configMap.getMetadata()) + .map(V1ObjectMeta::getName).orElse(""); + } + } + + private static class DeleteIntrospectorConfigMapStep extends Step { + private final String domainUid; + private final String namespace; + private final String configMapName; + + + DeleteIntrospectorConfigMapStep(String domainUid, String namespace, String configMapName) { + this.domainUid = domainUid; + this.namespace = namespace; + this.configMapName = configMapName; + } + @Override public NextAction apply(Packet packet) { return doNext(deleteIntrospectorConfigMap(getNext()), packet); @@ -682,9 +808,8 @@ protected void logConfigMapDeleted() { private Step deleteIntrospectorConfigMap(Step next) { logConfigMapDeleted(); - String configMapName = getIntrospectorConfigMapName(this.domainUid); return new CallBuilder() - .deleteConfigMapAsync(configMapName, namespace, this.domainUid, + .deleteConfigMapAsync(configMapName, namespace, domainUid, new V1DeleteOptions(), new DefaultResponseStep<>(next)); } } @@ -718,6 +843,7 @@ public NextAction onSuccess(Packet packet, CallResponse callRespons copyMapEntryToPacket(result, packet, DOMAINZIP_HASH); copyMapEntryToPacket(result, packet, DOMAIN_RESTART_VERSION); copyMapEntryToPacket(result, packet, DOMAIN_INPUTS_HASH); + copyMapEntryToPacket(result, packet, NUM_CONFIG_MAPS); DomainTopology domainTopology = Optional.ofNullable(result) @@ -735,7 +861,7 @@ public NextAction onSuccess(Packet packet, CallResponse callRespons } private String getTopologyYaml(Map data) { - return data.get(IntrospectorConfigMapKeys.TOPOLOGY_YAML); + return data.get(IntrospectorConfigMapConstants.TOPOLOGY_YAML); } private void recordTopology(Packet packet, DomainPresenceInfo info, DomainTopology domainTopology) { diff --git a/operator/src/main/java/oracle/kubernetes/operator/helpers/ConfigMapSplitter.java b/operator/src/main/java/oracle/kubernetes/operator/helpers/ConfigMapSplitter.java new file mode 100644 index 00000000000..11ba655b85b --- /dev/null +++ b/operator/src/main/java/oracle/kubernetes/operator/helpers/ConfigMapSplitter.java @@ -0,0 +1,154 @@ +// Copyright (c) 2020, Oracle Corporation and/or its affiliates. +// Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. + +package oracle.kubernetes.operator.helpers; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.function.BiFunction; +import java.util.stream.Collectors; +import javax.annotation.Nonnull; + +/** + * A Kubernetes ConfigMap has a hard size limit; attempts to create one larger will fail. This is a + * problem when we need to store more data in a config map. Our solution is to split the data among multiple maps. + * + * @param the kind of target object to create, which will ultimately be used to create config maps + */ +public class ConfigMapSplitter { + + // The limit for a Kubernetes Config Map is 1MB, including all components of the map. We use a data limit a bit + // below that to ensure that the map structures, including the keys, metadata and the results of JSON encoding, don't + // accidentally put us over the limit. + @SuppressWarnings({"FieldMayBeFinal", "FieldCanBeLocal"}) // not private or local so that unit tests can set it. + private static int DATA_LIMIT = 900_000; + + private final BiFunction, Integer, T> factory; + + private final List result = new ArrayList<>(); + private Map current; + private int remainingRoom; + private final Map locations = new HashMap<>(); + + /** + * Constructs a splitter object. + * + * @param factory a function that the splitter should use to create its target objects. + */ + public ConfigMapSplitter(BiFunction, Integer, T> factory) { + this.factory = factory; + } + + /** + * Given a map, splits it so that no map has more total data than the specified limit, and returns a list of + * target objects built from the resultant maps. This may result in some maps receiving partial value for the largest + * items. If the target type implements CountRecorder, the 'recordCount' method of the first target will be invoked + * with the number of targets created. + * + * @param data the map to split. + */ + public List split(Map data) { + startSplitResult(); + for (DataEntry dataEntry : getSortedEntrySizes(data)) { + addToSplitResult(dataEntry); + } + recordSplitResult(); + + recordTargetInfo(result.get(0), result.size()); + return result; + } + + @Nonnull + private List getSortedEntrySizes(Map data) { + return data.entrySet().stream().map(DataEntry::new).sorted().collect(Collectors.toList()); + } + + private void startSplitResult() { + current = new HashMap<>(); + remainingRoom = DATA_LIMIT; + } + + /** + * Adds the specified data entry to one or more split results, recording its location if it is not wholly + * in the first split result. + * @param entry a data entry + */ + private void addToSplitResult(DataEntry entry) { + int startIndex = result.size(); + while (entry.getRemainingLength() > 0) { + remainingRoom -= entry.addToMap(current, remainingRoom); + if (remainingRoom == 0) { + recordSplitResult(); + startSplitResult(); + } + if (!result.isEmpty()) { + locations.put(entry.key, createLocation(entry, startIndex)); + } + } + } + + private void recordSplitResult() { + result.add(factory.apply(current, result.size())); + } + + private void recordTargetInfo(T target, int size) { + target.recordNumTargets(size); + + for (Location location : locations.values()) { + target.recordEntryLocation(location.key, location.first, location.last); + } + } + + private Location createLocation(DataEntry entry, int startIndex) { + return new Location(entry, startIndex, result.size()); + } + + static class DataEntry implements Comparable { + private final String key; + private String value; + + public DataEntry(Map.Entry mapEntry) { + key = mapEntry.getKey(); + value = mapEntry.getValue(); + } + + /** + * Adds to the specified map, as much of this entry as will fit, removing it from the string + * still to be added. Returns the number of characters added. + * @param map the map to update + * @param limit the maximum number of characters to add + */ + int addToMap(Map map, int limit) { + final int numCharsAdded = Math.min(limit, value.length()); + map.put(key, value.substring(0, numCharsAdded)); + value = value.substring(numCharsAdded); + + return numCharsAdded; + } + + private int getRemainingLength() { + return value.length(); + } + + @Override + public int compareTo(@Nonnull DataEntry o) { + return Integer.compare(getRemainingLength(), o.getRemainingLength()); + } + } + + static class Location { + private final String key; + private final int first; + private final int last; + + Location(DataEntry entry, int first, int last) { + this.key = entry.key; + this.first = first; + this.last = last; + } + + } + +} diff --git a/operator/src/main/java/oracle/kubernetes/operator/helpers/FileGroupReader.java b/operator/src/main/java/oracle/kubernetes/operator/helpers/FileGroupReader.java index 3c22bbdbe3a..2a5ddfe03bd 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/helpers/FileGroupReader.java +++ b/operator/src/main/java/oracle/kubernetes/operator/helpers/FileGroupReader.java @@ -46,7 +46,7 @@ class FileGroupReader { * Given a file path, loads the contents of the files into a map. * * @param rootDir the path to the top-level directory - * @return a map of file paths to string contents. + * @return a map of file names to string contents. * @throws IOException if an error occurs during the read */ static Map loadContents(Path rootDir) throws IOException { diff --git a/operator/src/main/java/oracle/kubernetes/operator/helpers/JobHelper.java b/operator/src/main/java/oracle/kubernetes/operator/helpers/JobHelper.java index 2efe8aad1c3..278fb6d8c10 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/helpers/JobHelper.java +++ b/operator/src/main/java/oracle/kubernetes/operator/helpers/JobHelper.java @@ -19,7 +19,7 @@ import io.kubernetes.client.openapi.models.V1Volume; import io.kubernetes.client.openapi.models.V1VolumeMount; import oracle.kubernetes.operator.DomainStatusUpdater; -import oracle.kubernetes.operator.IntrospectorConfigMapKeys; +import oracle.kubernetes.operator.IntrospectorConfigMapConstants; import oracle.kubernetes.operator.JobWatcher; import oracle.kubernetes.operator.LabelConstants; import oracle.kubernetes.operator.MakeRightDomainOperation; @@ -113,7 +113,7 @@ private static String getCurrentImageSpecHash(DomainPresenceInfo info) { } private static String getIntrospectionImageSpecHash(Packet packet) { - return (String) packet.get(IntrospectorConfigMapKeys.DOMAIN_INPUTS_HASH); + return (String) packet.get(IntrospectorConfigMapConstants.DOMAIN_INPUTS_HASH); } private static int runningServersCount(DomainPresenceInfo info) { diff --git a/operator/src/main/java/oracle/kubernetes/operator/helpers/JobStepContext.java b/operator/src/main/java/oracle/kubernetes/operator/helpers/JobStepContext.java index 5140ed0384f..fa74961bd33 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/helpers/JobStepContext.java +++ b/operator/src/main/java/oracle/kubernetes/operator/helpers/JobStepContext.java @@ -23,6 +23,7 @@ import io.kubernetes.client.openapi.models.V1VolumeMount; import oracle.kubernetes.operator.DomainSourceType; import oracle.kubernetes.operator.DomainStatusUpdater; +import oracle.kubernetes.operator.IntrospectorConfigMapConstants; import oracle.kubernetes.operator.KubernetesConstants; import oracle.kubernetes.operator.LabelConstants; import oracle.kubernetes.operator.ProcessingConstants; @@ -277,7 +278,7 @@ protected V1PodSpec createPodSpec(TuningParameters tuningParameters) { new V1Volume().name(SCRIPTS_VOLUME).configMap(getConfigMapVolumeSource())) .addVolumesItem( new V1Volume() - .name("mii" + KubernetesConstants.INTROSPECTOR_CONFIG_MAP_NAME_SUFFIX) + .name("mii" + IntrospectorConfigMapConstants.INTROSPECTOR_CONFIG_MAP_NAME_SUFFIX) .configMap(getIntrospectMD5VolumeSource())); if (getOpssWalletPasswordSecretVolume() != null) { podSpec.addVolumesItem(new V1Volume().name(OPSS_KEYPASSPHRASE_VOLUME).secret( @@ -342,7 +343,7 @@ protected V1Container createContainer(TuningParameters tuningParameters) { .addVolumeMountsItem(readOnlyVolumeMount(SCRIPTS_VOLUME, SCRIPTS_MOUNTS_PATH)) .addVolumeMountsItem( volumeMount( - "mii" + KubernetesConstants.INTROSPECTOR_CONFIG_MAP_NAME_SUFFIX, + "mii" + IntrospectorConfigMapConstants.INTROSPECTOR_CONFIG_MAP_NAME_SUFFIX, "/weblogic-operator/introspectormii") .readOnly(false)); @@ -446,7 +447,7 @@ private V1ConfigMapVolumeSource getConfigMapVolumeSource() { protected V1ConfigMapVolumeSource getIntrospectMD5VolumeSource() { V1ConfigMapVolumeSource result = new V1ConfigMapVolumeSource() - .name(getDomainUid() + KubernetesConstants.INTROSPECTOR_CONFIG_MAP_NAME_SUFFIX) + .name(getDomainUid() + IntrospectorConfigMapConstants.INTROSPECTOR_CONFIG_MAP_NAME_SUFFIX) .defaultMode(ALL_READ_AND_EXECUTE); result.setOptional(true); return result; diff --git a/operator/src/main/java/oracle/kubernetes/operator/helpers/PodDefaults.java b/operator/src/main/java/oracle/kubernetes/operator/helpers/PodDefaults.java index 4b79b768598..a70c6de9f98 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/helpers/PodDefaults.java +++ b/operator/src/main/java/oracle/kubernetes/operator/helpers/PodDefaults.java @@ -5,30 +5,35 @@ import java.util.ArrayList; import java.util.List; +import java.util.Objects; import io.kubernetes.client.openapi.models.V1ConfigMapVolumeSource; import io.kubernetes.client.openapi.models.V1Volume; import io.kubernetes.client.openapi.models.V1VolumeMount; +import oracle.kubernetes.operator.IntrospectorConfigMapConstants; +import static oracle.kubernetes.operator.IntrospectorConfigMapConstants.getIntrospectorConfigMapName; +import static oracle.kubernetes.operator.IntrospectorConfigMapConstants.getIntrospectorVolumePath; import static oracle.kubernetes.operator.KubernetesConstants.DOMAIN_DEBUG_CONFIG_MAP_SUFFIX; import static oracle.kubernetes.operator.KubernetesConstants.SCRIPT_CONFIG_MAP_NAME; import static oracle.kubernetes.operator.helpers.StepContextConstants.ALL_READ_AND_EXECUTE; import static oracle.kubernetes.operator.helpers.StepContextConstants.DEBUG_CM_MOUNTS_PATH; import static oracle.kubernetes.operator.helpers.StepContextConstants.DEBUG_CM_VOLUME; +import static oracle.kubernetes.operator.helpers.StepContextConstants.INTROSPECTOR_VOLUME; import static oracle.kubernetes.operator.helpers.StepContextConstants.SCRIPTS_MOUNTS_PATH; import static oracle.kubernetes.operator.helpers.StepContextConstants.SCRIPTS_VOLUME; -import static oracle.kubernetes.operator.helpers.StepContextConstants.SIT_CONFIG_MAP_VOLUME; -import static oracle.kubernetes.operator.helpers.StepContextConstants.WDT_CONFIG_MAP_VOLUME; class PodDefaults { static final String K8S_SERVICE_ACCOUNT_MOUNT_PATH = "/var/run/secrets/kubernetes.io/serviceaccount"; - static List getStandardVolumes(String domainUid) { + static List getStandardVolumes(String domainUid, int numIntrospectorVolumes) { List volumes = new ArrayList<>(); volumes.add(createScriptsVolume()); volumes.add(createDebugCmVolume(domainUid)); - volumes.add(createSitConfigVolume(domainUid)); + for (int i = 0; i < numIntrospectorVolumes; i++) { + volumes.add(createIntrospectorVolume(domainUid, i)); + } return volumes; } @@ -45,31 +50,25 @@ private static V1Volume createVolume(String volumeName, String configMapName) { private static V1Volume createDebugCmVolume(String domainUid) { V1Volume volume = createVolume(DEBUG_CM_VOLUME, domainUid + DOMAIN_DEBUG_CONFIG_MAP_SUFFIX); - volume.getConfigMap().setOptional(true); + Objects.requireNonNull(volume.getConfigMap()).setOptional(true); return volume; } - private static V1Volume createSitConfigVolume(String domainUid) { - return createVolume(getSitConfigMapVolumeName(domainUid), ConfigMapHelper.getIntrospectorConfigMapName(domainUid)); + private static V1Volume createIntrospectorVolume(String domainUid, int index) { + return createVolume(getIntrospectorVolumeName(index), getIntrospectorConfigMapName(domainUid, index)); } - private static String getSitConfigMapVolumeName(String domainUid) { - return SIT_CONFIG_MAP_VOLUME; + static String getIntrospectorVolumeName(int index) { + return INTROSPECTOR_VOLUME + IntrospectorConfigMapConstants.suffix(index); } - private static V1Volume createWdtConfigMapVolume(String domainUid) { - return createVolume(getWdtConfigMapVolumeName(domainUid), ConfigMapHelper.getIntrospectorConfigMapName(domainUid)); - } - - private static String getWdtConfigMapVolumeName(String domainUid) { - return WDT_CONFIG_MAP_VOLUME; - } - - static List getStandardVolumeMounts(String domainUid) { + static List getStandardVolumeMounts(String domainUid, int numIntrospectorVolumes) { List mounts = new ArrayList<>(); mounts.add(createScriptsVolumeMount()); mounts.add(createDebugCmVolumeMount()); - mounts.add(createSitConfigVolumeMount(domainUid)); + for (int i = 0; i < numIntrospectorVolumes; i++) { + mounts.add(createIntrospectorVolumeMount(i)); + } return mounts; } @@ -81,12 +80,8 @@ private static V1VolumeMount createDebugCmVolumeMount() { return readOnlyVolumeMount(DEBUG_CM_VOLUME, DEBUG_CM_MOUNTS_PATH); } - private static V1VolumeMount createSitConfigVolumeMount(String domainUid) { - return volumeMount(getSitConfigMapVolumeName(domainUid), "/weblogic-operator/introspector"); - } - - private static V1VolumeMount createWdtConfigVolumeMount(String domainUid) { - return volumeMount(getWdtConfigMapVolumeName(domainUid), "/weblogic-operator/introspector"); + private static V1VolumeMount createIntrospectorVolumeMount(int index) { + return volumeMount(getIntrospectorVolumeName(index), getIntrospectorVolumePath(index)); } private static V1VolumeMount readOnlyVolumeMount(String volumeName, String mountPath) { diff --git a/operator/src/main/java/oracle/kubernetes/operator/helpers/PodStepContext.java b/operator/src/main/java/oracle/kubernetes/operator/helpers/PodStepContext.java index 13b64f4b216..3898f0da145 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/helpers/PodStepContext.java +++ b/operator/src/main/java/oracle/kubernetes/operator/helpers/PodStepContext.java @@ -13,6 +13,7 @@ import java.util.Objects; import java.util.Optional; import java.util.stream.Collectors; +import javax.annotation.Nonnull; import javax.json.Json; import javax.json.JsonPatchBuilder; @@ -36,7 +37,7 @@ import io.kubernetes.client.openapi.models.V1VolumeMount; import oracle.kubernetes.operator.DomainSourceType; import oracle.kubernetes.operator.DomainStatusUpdater; -import oracle.kubernetes.operator.IntrospectorConfigMapKeys; +import oracle.kubernetes.operator.IntrospectorConfigMapConstants; import oracle.kubernetes.operator.KubernetesConstants; import oracle.kubernetes.operator.LabelConstants; import oracle.kubernetes.operator.PodAwaiterStepFactory; @@ -55,11 +56,13 @@ import oracle.kubernetes.operator.work.Packet; import oracle.kubernetes.operator.work.Step; import oracle.kubernetes.weblogic.domain.model.Domain; +import oracle.kubernetes.weblogic.domain.model.DomainStatus; import oracle.kubernetes.weblogic.domain.model.ServerEnvVars; import oracle.kubernetes.weblogic.domain.model.ServerSpec; import oracle.kubernetes.weblogic.domain.model.Shutdown; import org.apache.commons.lang3.builder.EqualsBuilder; +import static oracle.kubernetes.operator.IntrospectorConfigMapConstants.NUM_CONFIG_MAPS; import static oracle.kubernetes.operator.LabelConstants.INTROSPECTION_STATE_LABEL; public abstract class PodStepContext extends BasePodStepContext { @@ -73,6 +76,8 @@ public abstract class PodStepContext extends BasePodStepContext { private static final String READINESS_PATH = "/weblogic/ready"; final WlsServerConfig scan; + @Nonnull + private final Packet packet; private final WlsDomainConfig domainTopology; private final Step conflictStep; private V1Pod podModel; @@ -84,10 +89,11 @@ public abstract class PodStepContext extends BasePodStepContext { super(packet.getSpi(DomainPresenceInfo.class)); this.conflictStep = conflictStep; domainTopology = (WlsDomainConfig) packet.get(ProcessingConstants.DOMAIN_TOPOLOGY); - miiModelSecretsHash = (String)packet.get(IntrospectorConfigMapKeys.SECRETS_MD_5); - miiDomainZipHash = (String)packet.get(IntrospectorConfigMapKeys.DOMAINZIP_HASH); - domainRestartVersion = (String)packet.get(IntrospectorConfigMapKeys.DOMAIN_RESTART_VERSION); + miiModelSecretsHash = (String)packet.get(IntrospectorConfigMapConstants.SECRETS_MD_5); + miiDomainZipHash = (String)packet.get(IntrospectorConfigMapConstants.DOMAINZIP_HASH); + domainRestartVersion = (String)packet.get(IntrospectorConfigMapConstants.DOMAIN_RESTART_VERSION); scan = (WlsServerConfig) packet.get(ProcessingConstants.SERVER_SCAN); + this.packet = packet; } private static boolean isPatchableItem(Map.Entry entry) { @@ -118,6 +124,22 @@ String getNamespace() { return info.getNamespace(); } + private int getNumIntrospectorConfigMaps() { + return Optional.ofNullable(getSpecifiedNumConfigMaps()).orElse(1); + } + + private Integer getSpecifiedNumConfigMaps() { + return Optional.ofNullable(packet.getValue(NUM_CONFIG_MAPS)).map(this::parseOrNull).orElse(null); + } + + private Integer parseOrNull(String count) { + try { + return Integer.parseInt(count); + } catch (NumberFormatException e) { + return null; + } + } + // ------------------------ data methods ---------------------------- String getDomainUid() { @@ -614,7 +636,7 @@ private List getReadinessGates() { } private List getVolumes(String domainUid) { - List volumes = PodDefaults.getStandardVolumes(domainUid); + List volumes = PodDefaults.getStandardVolumes(domainUid, getNumIntrospectorConfigMaps()); volumes.addAll(getServerSpec().getAdditionalVolumes()); if (getDomainHomeSourceType() == DomainSourceType.FromModel) { volumes.add(createRuntimeEncryptionSecretVolume()); @@ -657,7 +679,7 @@ protected List getContainers() { } private List getVolumeMounts() { - List mounts = PodDefaults.getStandardVolumeMounts(getDomainUid()); + List mounts = PodDefaults.getStandardVolumeMounts(getDomainUid(), getNumIntrospectorConfigMaps()); mounts.addAll(getServerSpec().getAdditionalVolumeMounts()); if (getDomainHomeSourceType() == DomainSourceType.FromModel) { mounts.add(createRuntimeEncryptionSecretVolumeMount()); @@ -865,7 +887,7 @@ public NextAction apply(Packet packet) { Optional.ofNullable(packet.getSpi(DomainPresenceInfo.class)) .map(DomainPresenceInfo::getDomain) .map(Domain::getStatus) - .ifPresent(a -> a.resetIntrospectJobFailureCount()); + .ifPresent(DomainStatus::resetIntrospectJobFailureCount); if (currentPod == null) { return doNext(createNewPod(getNext()), packet); diff --git a/operator/src/main/java/oracle/kubernetes/operator/helpers/SplitterTarget.java b/operator/src/main/java/oracle/kubernetes/operator/helpers/SplitterTarget.java new file mode 100644 index 00000000000..e007b5ce694 --- /dev/null +++ b/operator/src/main/java/oracle/kubernetes/operator/helpers/SplitterTarget.java @@ -0,0 +1,25 @@ +// Copyright (c) 2020, Oracle Corporation and/or its affiliates. +// Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. + +package oracle.kubernetes.operator.helpers; + +/** + * An interface for objects created by the ConfigMapSplitter. After the split operation creates its list of targets, + * the splitter will invoke these methods on the first target to indicate what happened. + */ +public interface SplitterTarget { + + /** + * Records the total number of targets created by the split operation. + * @param numTargets the number of created targets + */ + void recordNumTargets(int numTargets); + + /** + * Records the location of a entry that was split. + * @param key the key of the split entry + * @param firstTarget the index of first target in which the entry was recorded + * @param lastTarget the index of the last target in which the entry was recorded + */ + void recordEntryLocation(String key, int firstTarget, int lastTarget); +} diff --git a/operator/src/main/java/oracle/kubernetes/operator/helpers/StepContextConstants.java b/operator/src/main/java/oracle/kubernetes/operator/helpers/StepContextConstants.java index aadbfbff892..fcf9b2d723c 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/helpers/StepContextConstants.java +++ b/operator/src/main/java/oracle/kubernetes/operator/helpers/StepContextConstants.java @@ -11,7 +11,7 @@ public interface StepContextConstants { static final String SECRETS_VOLUME = "weblogic-credentials-volume"; static final String SCRIPTS_VOLUME = "weblogic-scripts-cm-volume"; static final String DEBUG_CM_VOLUME = "weblogic-domain-debug-cm-volume"; - static final String SIT_CONFIG_MAP_VOLUME = "weblogic-domain-introspect-cm-volume"; + static final String INTROSPECTOR_VOLUME = "weblogic-domain-introspect-cm-volume"; static final String WDT_CONFIG_MAP_VOLUME = "weblogic-domain-introspect-wdt-cm-volume"; static final String RUNTIME_ENCRYPTION_SECRET_VOLUME = "weblogic-domain-runtime-encryption-volume"; static final String STORAGE_VOLUME = "weblogic-domain-storage-volume"; diff --git a/operator/src/main/resources/scripts/modelInImage.sh b/operator/src/main/resources/scripts/modelInImage.sh index d8a753585cc..1d9b4f605a3 100755 --- a/operator/src/main/resources/scripts/modelInImage.sh +++ b/operator/src/main/resources/scripts/modelInImage.sh @@ -8,6 +8,7 @@ source ${SCRIPTPATH}/utils.sh WDT_MINIMUM_VERSION="1.7.3" +OPERATOR_ROOT=${TEST_OPERATOR_ROOT:-/weblogic-operator} INTROSPECTCM_IMAGE_MD5="/weblogic-operator/introspectormii/inventory_image.md5" INTROSPECTCM_CM_MD5="/weblogic-operator/introspectormii/inventory_cm.md5" INTROSPECTCM_PASSPHRASE_MD5="/weblogic-operator/introspectormii/inventory_passphrase.md5" @@ -15,7 +16,6 @@ INTROSPECTCM_MERGED_MODEL="/weblogic-operator/introspectormii/merged_model.json" INTROSPECTCM_WLS_VERSION="/weblogic-operator/introspectormii/wls.version" INTROSPECTCM_JDK_PATH="/weblogic-operator/introspectormii/jdk.path" INTROSPECTCM_SECRETS_AND_ENV_MD5="/weblogic-operator/introspectormii/secrets_and_env.md5" -DOMAIN_ZIPPED="/weblogic-operator/introspectormii/domainzip.secure" PRIMORDIAL_DOMAIN_ZIPPED="/weblogic-operator/introspectormii/primordial_domainzip.secure" INTROSPECTJOB_IMAGE_MD5="/tmp/inventory_image.md5" INTROSPECTJOB_CM_MD5="/tmp/inventory_cm.md5" @@ -25,6 +25,7 @@ LOCAL_PRIM_DOMAIN_TAR="/tmp/prim_domain.tar" NEW_MERGED_MODEL="/tmp/new_merged_model.json" WDT_CONFIGMAP_ROOT="/weblogic-operator/wdt-config-map" RUNTIME_ENCRYPTION_SECRET_PASSWORD="/weblogic-operator/model-runtime-secret/password" + # we export the opss password file location because it's also used by introspectDomain.py export OPSS_KEY_PASSPHRASE="/weblogic-operator/opss-walletkey-secret/walletPassword" OPSS_KEY_B64EWALLET="/weblogic-operator/opss-walletfile-secret/walletFile" @@ -514,6 +515,71 @@ function createModelDomain() { trace "Exiting createModelDomain" } + +# Expands into the root directory the MII domain configuration, stored in one or more config maps +function restoreDomainConfig() { + restoreEncodedTar "domainzip.secure" || return 1 + + chmod +x ${DOMAIN_HOME}/bin/*.sh ${DOMAIN_HOME}/*.sh || return 1 +} + +# Expands into the root directory the MII primordial domain, stored in one or more config maps +function restorePrimordialDomain() { + restoreEncodedTar "primordial_domainzip.secure" || return 1 +} + +# Restores the specified directory, targz'ed and stored in one or more config maps after base 64 encoding +# args: +# $1 the name of the encoded file in the config map +function restoreEncodedTar() { + cd / || return 1 + indexRange="$(getIndexRange $1)" + # shellcheck disable=SC2046 + cat $(buildConfigMapElements $1 $indexRange) > /tmp/domain.secure + base64 -d "/tmp/domain.secure" > /tmp/domain.tar.gz || return 1 + + tar -xzf /tmp/domain.tar.gz || return 1 +} + +# Returns the index range for an encoded file in an introspector config map +# args: +# $1 the key of the encoded file in the map +function getIndexRange() { + rangeFile="$OPERATOR_ROOT/introspector/$1.range" + if [ -f "$rangeFile" ]; then + cat "$rangeFile" + else + echo "0 0" + fi +} + +# Creates a string containing entries from one or more config maps +# args: +# $1 the key of the entry in each map +# $2 the index of the first configmap containing the entry key +# $3 the index of the last configmap containing the entry key +function buildConfigMapElements() { + result="" + + for ((i=$2;i<=$3;i++)); do + result="$result ${OPERATOR_ROOT}/introspector$(getSuffix $i)/${1}" + done + + echo $result +} + +# Returns the suffix for a config map mount address +# args: +# $1 the index of the map (0 means no suffix) +function getSuffix() { + if [ "$1" -eq 0 ]; then + echo "" + else + echo "_$1" + fi +} + + function diff_model() { trace "Entering diff_model" diff --git a/operator/src/main/resources/scripts/startServer.sh b/operator/src/main/resources/scripts/startServer.sh index 322f722ff57..21f0cb62039 100755 --- a/operator/src/main/resources/scripts/startServer.sh +++ b/operator/src/main/resources/scripts/startServer.sh @@ -8,7 +8,11 @@ # This is the script WebLogic Operator WLS Pods use to start their WL Server. # -SCRIPTPATH="$( cd "$(dirname "$0")" > /dev/null 2>&1 ; pwd -P )" +if [ -z ${SCRIPTPATH+x} ]; then + SCRIPTPATH="$( cd "$(dirname "$0")" > /dev/null 2>&1 ; pwd -P )" +fi + +echo "script path is ${SCRIPTPATH}" source ${SCRIPTPATH}/utils.sh [ $? -ne 0 ] && echo "[SEVERE] Missing file ${SCRIPTPATH}/utils.sh" && exitOrLoop @@ -183,9 +187,7 @@ function prepareMIIServer() { fi trace "Model-in-Image: Restoring primordial domain" - cd / || return 1 - base64 -d /weblogic-operator/introspector/primordial_domainzip.secure > /tmp/domain.tar.gz || return 1 - tar -xzf /tmp/domain.tar.gz || return 1 + restorePrimordialDomain || return 1 trace "Model-in-Image: Restore domain secret" # decrypt the SerializedSystemIni first @@ -201,10 +203,7 @@ function prepareMIIServer() { # restore the config zip # trace "Model-in-Image: Restore domain config" - cd / || return 1 - base64 -d /weblogic-operator/introspector/domainzip.secure > /tmp/domain.tar.gz || return 1 - tar -xzf /tmp/domain.tar.gz || return 1 - chmod +x ${DOMAIN_HOME}/bin/*.sh ${DOMAIN_HOME}/*.sh || return 1 + restoreDomainConfig || return 1 # restore the archive apps and libraries # diff --git a/operator/src/test/java/oracle/kubernetes/operator/DomainProcessorTest.java b/operator/src/test/java/oracle/kubernetes/operator/DomainProcessorTest.java index 3034cae2a6b..367cbaa49c0 100644 --- a/operator/src/test/java/oracle/kubernetes/operator/DomainProcessorTest.java +++ b/operator/src/test/java/oracle/kubernetes/operator/DomainProcessorTest.java @@ -66,6 +66,7 @@ import static oracle.kubernetes.operator.DomainSourceType.FromModel; import static oracle.kubernetes.operator.DomainSourceType.Image; import static oracle.kubernetes.operator.DomainSourceType.PersistentVolume; +import static oracle.kubernetes.operator.IntrospectorConfigMapConstants.INTROSPECTOR_CONFIG_MAP_NAME_SUFFIX; import static oracle.kubernetes.operator.LabelConstants.CREATEDBYOPERATOR_LABEL; import static oracle.kubernetes.operator.LabelConstants.DOMAINNAME_LABEL; import static oracle.kubernetes.operator.LabelConstants.DOMAINUID_LABEL; @@ -163,7 +164,9 @@ public void setUp() throws Exception { } @After - public void tearDown() { + public void tearDown() throws Exception { + testSupport.throwOnCompletionFailure(); + mementos.forEach(Memento::revert); } @@ -558,7 +561,7 @@ public void whenExternalServiceNameSuffixConfigured_externalServiceNameContainsS private static final String OLD_INTROSPECTION_STATE = "123"; private static final String NEW_INTROSPECTION_STATE = "124"; - private static final String INTROSPECTOR_MAP_NAME = UID + KubernetesConstants.INTROSPECTOR_CONFIG_MAP_NAME_SUFFIX; + private static final String INTROSPECTOR_MAP_NAME = UID + INTROSPECTOR_CONFIG_MAP_NAME_SUFFIX; @Test public void whenDomainHasRunningServersAndExistingTopology_dontRunIntrospectionJob() throws JsonProcessingException { @@ -635,8 +638,8 @@ private String getCurrentImageSpecHash() { private V1ConfigMap createIntrospectorConfigMap(String introspectionDoneValue) throws JsonProcessingException { return new V1ConfigMap() .metadata(createIntrospectorConfigMapMeta(introspectionDoneValue)) - .data(new HashMap<>(Map.of(IntrospectorConfigMapKeys.TOPOLOGY_YAML, defineTopology(), - IntrospectorConfigMapKeys.DOMAIN_INPUTS_HASH, getCurrentImageSpecHash()))); + .data(new HashMap<>(Map.of(IntrospectorConfigMapConstants.TOPOLOGY_YAML, defineTopology(), + IntrospectorConfigMapConstants.DOMAIN_INPUTS_HASH, getCurrentImageSpecHash()))); } private V1ObjectMeta createIntrospectorConfigMapMeta(@Nullable String introspectionDoneValue) { @@ -823,7 +826,7 @@ private void cacheChangedDomainInputsHash() { .filter(this::isIntrospectorConfigMap) .findFirst() .map(V1ConfigMap::getData) - .ifPresent(data -> data.put(IntrospectorConfigMapKeys.DOMAIN_INPUTS_HASH, "changedHash")); + .ifPresent(data -> data.put(IntrospectorConfigMapConstants.DOMAIN_INPUTS_HASH, "changedHash")); } private boolean isIntrospectorConfigMap(V1ConfigMap map) { diff --git a/operator/src/test/java/oracle/kubernetes/operator/MainTest.java b/operator/src/test/java/oracle/kubernetes/operator/MainTest.java index 914a0e4be25..e8bd8652d81 100644 --- a/operator/src/test/java/oracle/kubernetes/operator/MainTest.java +++ b/operator/src/test/java/oracle/kubernetes/operator/MainTest.java @@ -63,9 +63,7 @@ import static oracle.kubernetes.utils.LogMatcher.containsInfo; import static oracle.kubernetes.utils.LogMatcher.containsSevere; import static oracle.kubernetes.utils.LogMatcher.containsWarning; -import static org.hamcrest.Matchers.allOf; import static org.hamcrest.Matchers.contains; -import static org.hamcrest.Matchers.hasKey; import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; @@ -162,12 +160,6 @@ public void tearDown() throws Exception { mementos.forEach(Memento::revert); } - @Test - public void mainHasAccessToBuildProperties() { - assertThat(Main.getBuildProperties(), allOf( - hasKey(GIT_BRANCH_KEY), hasKey(GIT_COMMIT_KEY), hasKey(GIT_BUILD_TIME_KEY), hasKey(GIT_BUILD_VERSION_KEY))); - } - @Test public void whenOperatorCreated_logStartupMessage() { loggerControl.withLogLevel(Level.INFO).collectLogMessages(logRecords, OPERATOR_STARTED); diff --git a/operator/src/test/java/oracle/kubernetes/operator/helpers/ConfigMapHelperTest.java b/operator/src/test/java/oracle/kubernetes/operator/helpers/ConfigMapHelperTest.java index af9e8e948a6..e61550e01cd 100644 --- a/operator/src/test/java/oracle/kubernetes/operator/helpers/ConfigMapHelperTest.java +++ b/operator/src/test/java/oracle/kubernetes/operator/helpers/ConfigMapHelperTest.java @@ -3,12 +3,15 @@ package oracle.kubernetes.operator.helpers; +import java.net.URI; +import java.nio.file.Path; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.function.Function; import java.util.logging.Level; import java.util.logging.LogRecord; import java.util.stream.Collectors; @@ -21,6 +24,7 @@ import io.kubernetes.client.openapi.models.V1ObjectMeta; import oracle.kubernetes.operator.LabelConstants; import oracle.kubernetes.operator.calls.FailureStatusSourceException; +import oracle.kubernetes.operator.utils.InMemoryFileSystem; import oracle.kubernetes.operator.work.Packet; import oracle.kubernetes.operator.work.Step; import oracle.kubernetes.utils.TestUtils; @@ -77,6 +81,8 @@ public class ConfigMapHelperTest { private final KubernetesTestSupport testSupport = new KubernetesTestSupport(); private final List mementos = new ArrayList<>(); private final List logRecords = new ArrayList<>(); + private final InMemoryFileSystem fileSystem = InMemoryFileSystem.createInstance(); + private final Function pathFunction = fileSystem::getPath; @SuppressWarnings("SameParameterValue") @@ -108,10 +114,6 @@ private V1ObjectMeta createMetadata() { .putLabelsItem(LabelConstants.CREATEDBYOPERATOR_LABEL, "true"); } - /** - * Setup test. - * @throws Exception on failure - */ @Before public void setUp() throws Exception { mementos.add( @@ -120,17 +122,21 @@ public void setUp() throws Exception { .withLogLevel(Level.FINE)); mementos.add(testSupport.install()); mementos.add(TestComparator.install()); + mementos.add(StaticStubSupport.install(FileGroupReader.class, "uriToPath", pathFunction)); + + defineInMemoryFiles(SCRIPT_NAMES); + } + + private void defineInMemoryFiles(String... scriptNames) { + final String scriptRoot = getClass().getResource("/scripts/").getPath(); + for (String scriptName : scriptNames) { + fileSystem.defineFile(scriptRoot + scriptName, ""); + } } - /** - * Tear down test. - * @throws Exception on failure - */ @After public void tearDown() throws Exception { - for (Memento memento : mementos) { - memento.revert(); - } + mementos.forEach(Memento::revert); testSupport.throwOnCompletionFailure(); } diff --git a/operator/src/test/java/oracle/kubernetes/operator/helpers/ConfigMapSplitterTest.java b/operator/src/test/java/oracle/kubernetes/operator/helpers/ConfigMapSplitterTest.java new file mode 100644 index 00000000000..a3d32b7e3cf --- /dev/null +++ b/operator/src/test/java/oracle/kubernetes/operator/helpers/ConfigMapSplitterTest.java @@ -0,0 +1,189 @@ +// Copyright (c) 2020, Oracle Corporation and/or its affiliates. +// Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. + +package oracle.kubernetes.operator.helpers; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +import com.meterware.simplestub.Memento; +import com.meterware.simplestub.StaticStubSupport; +import org.apache.commons.lang3.tuple.ImmutablePair; +import org.apache.commons.lang3.tuple.Pair; +import org.hamcrest.Description; +import org.hamcrest.Matchers; +import org.hamcrest.TypeSafeDiagnosingMatcher; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import static oracle.kubernetes.operator.helpers.ConfigMapSplitterTest.TargetMatcher.isTarget; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.junit.MatcherAssert.assertThat; + +public class ConfigMapSplitterTest { + + private static final int TEST_DATA_LIMIT = 1000; + private static final String UNIT_DATA = "123456789"; + private static final String LARGE_DATA_VALUE = createLargeData(2.5); // require three maps, including other data + + private final ConfigMapSplitter splitter = new ConfigMapSplitter<>(TestTarget::new); + private final Map data = new HashMap<>(); + private final List mementos = new ArrayList<>(); + + private static String createLargeData(double fraction) { + final int numRepeats = (int) Math.round(fraction * TEST_DATA_LIMIT / UNIT_DATA.length()); + return UNIT_DATA.repeat(numRepeats); + } + + @Before + public void setUp() throws Exception { + mementos.add(StaticStubSupport.install(ConfigMapSplitter.class, "DATA_LIMIT", TEST_DATA_LIMIT)); + } + + @After + public void tearDown() throws Exception { + mementos.forEach(Memento::revert); + } + + @Test + public void whenDataWithinLimits_createOneTargetObject() { + data.put("a", "aValue"); + data.put("b", "bValue"); + + final List result = splitter.split(data); + + assertThat(result, Matchers.hasSize(1)); + assertThat(result.get(0), isTarget(0, "a", "b")); + } + + @Test + public void whenDataTooLarge_createMultipleTargetObjects() { + data.put("a", "aValue"); + data.put("b", LARGE_DATA_VALUE); + + final List result = splitter.split(data); + + assertThat(result, Matchers.hasSize(3)); + assertThat(result.get(0), isTarget(0, "a", "b")); + assertThat(result.get(1), isTarget(1, "b")); + assertThat(result.get(2), isTarget(2, "b")); + } + + @Test + public void whenDataTooLarge_canReconstituteSplitValue() { + data.put("a", "aValue"); + data.put("b", "123456789".repeat(250)); + + final List result = splitter.split(data); + + final String reassembled = result.stream().map(TestTarget::getB).collect(Collectors.joining()); + assertThat(reassembled, equalTo(data.get("b"))); + } + + @Test + public void whenDataSplit_recordNumTargetsInFirstResult() { + data.put("a", "aValue"); + data.put("b", "123456789".repeat(250)); + + final List result = splitter.split(data); + + assertThat(result.get(0).numTargets, equalTo(3)); + assertThat(result.get(1).numTargets, equalTo(0)); + assertThat(result.get(2).numTargets, equalTo(0)); + } + + @Test + public void whenDataSplit_recordLocationOfSplitEntry() { + data.put("a", "aValue"); + data.put("b", "123456789".repeat(250)); + + final List result = splitter.split(data); + + assertThat(result.get(0).splitEntries.get("b"), equalTo(new ImmutablePair<>(0, 2))); + } + + @Test + public void recordLocationsOfNonSplitItemsPastFirstMap() { + data.put("a", createLargeData(0.5)); + data.put("b", createLargeData(0.6)); + data.put("c", createLargeData(0.7)); + + final List result = splitter.split(data); + + assertThat(result.get(0).splitEntries.get("c"), equalTo(new ImmutablePair<>(1, 1))); + } + + private static class TestTarget implements SplitterTarget { + private final Map data; + private final Map> splitEntries = new HashMap<>(); + private final int index; + private int numTargets; + + TestTarget(Map data, int index) { + this.data = data; + this.index = index; + } + + @Override + public void recordNumTargets(int numTargets) { + this.numTargets = numTargets; + } + + @Override + public void recordEntryLocation(String key, int firstTarget, int lastTarget) { + splitEntries.put(key, new ImmutablePair<>(firstTarget, lastTarget)); + } + + private String getB() { + return data.get("b"); + } + } + + @SuppressWarnings("unused") + static class TargetMatcher extends TypeSafeDiagnosingMatcher { + + private final int expectedIndex; + private final String[] expectedKeys; + + private TargetMatcher(int expectedIndex, String... expectedKeys) { + this.expectedIndex = expectedIndex; + this.expectedKeys = expectedKeys; + } + + static TargetMatcher isTarget(int expectedIndex, String... expectedKeys) { + return new TargetMatcher(expectedIndex, expectedKeys); + } + + @Override + protected boolean matchesSafely(TestTarget item, Description mismatchDescription) { + if (isExpectedTarget(item)) { + return true; + } else { + mismatchDescription.appendText("TestTarget with index ").appendValue(item.index) + .appendValueList("and data keys [", ",", "]", item.data.keySet()); + return false; + } + } + + boolean isExpectedTarget(TestTarget testTarget) { + return testTarget.index == expectedIndex && testTarget.data.keySet().equals(expectedKeySet()); + } + + private Set expectedKeySet() { + return new HashSet<>(Arrays.asList(expectedKeys)); + } + + @Override + public void describeTo(Description description) { + description.appendText("TestTarget with index ").appendValue(expectedIndex) + .appendValueList("and data keys [", ",", "]", expectedKeys); + } + } +} diff --git a/operator/src/test/java/oracle/kubernetes/operator/helpers/FileGroupReaderTest.java b/operator/src/test/java/oracle/kubernetes/operator/helpers/FileGroupReaderTest.java index 071ff1a3e50..b2462b86729 100644 --- a/operator/src/test/java/oracle/kubernetes/operator/helpers/FileGroupReaderTest.java +++ b/operator/src/test/java/oracle/kubernetes/operator/helpers/FileGroupReaderTest.java @@ -10,21 +10,12 @@ import oracle.kubernetes.operator.utils.InMemoryFileSystem; import org.junit.Test; -import static oracle.kubernetes.operator.helpers.ConfigMapHelperTest.SCRIPT_NAMES; -import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.hasEntry; import static org.hamcrest.junit.MatcherAssert.assertThat; public class FileGroupReaderTest { - private static InMemoryFileSystem fileSystem = InMemoryFileSystem.createInstance(); - private final FileGroupReader scriptReader = ConfigMapHelper.getScriptReader(); - - @Test - public void afterLoadScriptsFromClasspath_haveScriptNamesAsKeys() { - Map scripts = scriptReader.loadFilesFromClasspath(); - assertThat(scripts.keySet(), containsInAnyOrder(SCRIPT_NAMES)); - } + private static final InMemoryFileSystem fileSystem = InMemoryFileSystem.createInstance(); @Test public void loadFilesFromMemory() throws IOException { @@ -34,7 +25,7 @@ public void loadFilesFromMemory() throws IOException { Path p = fileSystem.getPath("group"); Map map = FileGroupReader.loadContents(p); - assertThat(map, hasEntry("group/a.b", "1234")); - assertThat(map, hasEntry("group/x/c.d", "5678")); + assertThat(map, hasEntry("a.b", "1234")); + assertThat(map, hasEntry("c.d", "5678")); } } diff --git a/operator/src/test/java/oracle/kubernetes/operator/helpers/IntrospectorConfigMapTest.java b/operator/src/test/java/oracle/kubernetes/operator/helpers/IntrospectorConfigMapTest.java index dfb78aca794..8c30b8434ca 100644 --- a/operator/src/test/java/oracle/kubernetes/operator/helpers/IntrospectorConfigMapTest.java +++ b/operator/src/test/java/oracle/kubernetes/operator/helpers/IntrospectorConfigMapTest.java @@ -10,14 +10,16 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.stream.Collectors; import javax.annotation.Nonnull; import com.meterware.simplestub.Memento; +import com.meterware.simplestub.StaticStubSupport; import io.kubernetes.client.openapi.models.V1ConfigMap; import io.kubernetes.client.openapi.models.V1ObjectMeta; import oracle.kubernetes.operator.DomainProcessorTestSetup; import oracle.kubernetes.operator.DomainSourceType; -import oracle.kubernetes.operator.IntrospectorConfigMapKeys; +import oracle.kubernetes.operator.IntrospectorConfigMapConstants; import oracle.kubernetes.operator.ProcessingConstants; import oracle.kubernetes.operator.rest.ScanCacheStub; import oracle.kubernetes.operator.wlsconfig.WlsDomainConfig; @@ -34,33 +36,46 @@ import static java.lang.System.lineSeparator; import static oracle.kubernetes.operator.DomainProcessorTestSetup.NS; import static oracle.kubernetes.operator.DomainProcessorTestSetup.UID; -import static oracle.kubernetes.operator.IntrospectorConfigMapKeys.DOMAINZIP_HASH; -import static oracle.kubernetes.operator.IntrospectorConfigMapKeys.DOMAIN_INPUTS_HASH; -import static oracle.kubernetes.operator.IntrospectorConfigMapKeys.DOMAIN_RESTART_VERSION; -import static oracle.kubernetes.operator.IntrospectorConfigMapKeys.SECRETS_MD_5; -import static oracle.kubernetes.operator.IntrospectorConfigMapKeys.TOPOLOGY_YAML; +import static oracle.kubernetes.operator.IntrospectorConfigMapConstants.DOMAINZIP_HASH; +import static oracle.kubernetes.operator.IntrospectorConfigMapConstants.DOMAIN_INPUTS_HASH; +import static oracle.kubernetes.operator.IntrospectorConfigMapConstants.DOMAIN_RESTART_VERSION; +import static oracle.kubernetes.operator.IntrospectorConfigMapConstants.NUM_CONFIG_MAPS; +import static oracle.kubernetes.operator.IntrospectorConfigMapConstants.SECRETS_MD_5; +import static oracle.kubernetes.operator.IntrospectorConfigMapConstants.TOPOLOGY_YAML; +import static oracle.kubernetes.operator.IntrospectorConfigMapConstants.getIntrospectorConfigMapNamePrefix; +import static oracle.kubernetes.operator.LabelConstants.CREATEDBYOPERATOR_LABEL; import static oracle.kubernetes.operator.LabelConstants.INTROSPECTION_STATE_LABEL; import static oracle.kubernetes.operator.ProcessingConstants.DOMAIN_TOPOLOGY; import static oracle.kubernetes.operator.helpers.DomainStatusMatcher.hasStatus; import static oracle.kubernetes.weblogic.domain.DomainConfiguratorFactory.forDomain; import static org.hamcrest.Matchers.allOf; +import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasEntry; import static org.hamcrest.Matchers.hasKey; +import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.nullValue; import static org.hamcrest.junit.MatcherAssert.assertThat; public class IntrospectorConfigMapTest { + private static final int TEST_DATA_LIMIT = 1000; + private static final int DATA_ALLOWANCE = 500; // assumed size of data that will not be split + private static final int NUM_MAPS_TO_CREATE = 3; + private static final String NUM_MAPS_STRING = Integer.toString(NUM_MAPS_TO_CREATE); + private static final int SPLITTABLE_DATA_SIZE = NUM_MAPS_TO_CREATE * TEST_DATA_LIMIT - DATA_ALLOWANCE; + private static final String UNIT_DATA = "123456789"; + private static final String LARGE_DATA_VALUE = UNIT_DATA.repeat(SPLITTABLE_DATA_SIZE / UNIT_DATA.length()); + private static final String LARGE_DATA_KEY = "domainzip.encoded"; private static final String TOPOLOGY_VALUE = "domainValid: true\ndomain:\n name: sample"; private static final String DOMAIN_HASH_VALUE = "MII_domain_hash"; private static final String INPUTS_HASH_VALUE = "MII_inputs_hash"; private static final String MD5_SECRETS = "md5-secrets"; private static final String RESTART_VERSION = "123"; - private static final String OVERRIDES_VALUE = "a[]"; private final KubernetesTestSupport testSupport = new KubernetesTestSupport(); private final List mementos = new ArrayList<>(); private final TerminalStep terminalStep = new TerminalStep(); @@ -73,6 +88,7 @@ public void setUp() throws Exception { mementos.add(TestUtils.silenceOperatorLogger()); mementos.add(testSupport.install()); mementos.add(ScanCacheStub.install()); + mementos.add(StaticStubSupport.install(ConfigMapSplitter.class, "DATA_LIMIT", TEST_DATA_LIMIT)); testSupport.defineResources(domain); testSupport.addDomainPresenceInfo(info); @@ -80,7 +96,9 @@ public void setUp() throws Exception { } @After - public void tearDown() { + public void tearDown() throws Exception { + testSupport.throwOnCompletionFailure(); + mementos.forEach(Memento::revert); } @@ -107,7 +125,7 @@ void addToPacket() { @Test public void whenNoTopologySpecified_continueProcessing() { testSupport.defineResources( - createIntrospectorConfigMap(Map.of(TOPOLOGY_YAML, TOPOLOGY_VALUE, SECRETS_MD_5, MD5_SECRETS))); + createIntrospectorConfigMap(0, Map.of(TOPOLOGY_YAML, TOPOLOGY_VALUE, SECRETS_MD_5, MD5_SECRETS))); introspectResult.defineFile(SECRETS_MD_5, "not telling").addToPacket(); testSupport.runSteps(ConfigMapHelper.createIntrospectorConfigMapStep(terminalStep)); @@ -118,7 +136,7 @@ public void whenNoTopologySpecified_continueProcessing() { @Test public void whenNoTopologySpecified_dontUpdateConfigMap() { testSupport.defineResources( - createIntrospectorConfigMap(Map.of(TOPOLOGY_YAML, TOPOLOGY_VALUE, SECRETS_MD_5, MD5_SECRETS))); + createIntrospectorConfigMap(0, Map.of(TOPOLOGY_YAML, TOPOLOGY_VALUE, SECRETS_MD_5, MD5_SECRETS))); introspectResult.defineFile(SECRETS_MD_5, "not telling").addToPacket(); testSupport.runSteps(ConfigMapHelper.createIntrospectorConfigMapStep(terminalStep)); @@ -130,7 +148,7 @@ public void whenNoTopologySpecified_dontUpdateConfigMap() { public void whenNoTopologySpecified_addIntrospectionVersionLabel() { forDomain(domain).withIntrospectVersion("4"); testSupport.defineResources( - createIntrospectorConfigMap(Map.of(TOPOLOGY_YAML, TOPOLOGY_VALUE, SECRETS_MD_5, MD5_SECRETS))); + createIntrospectorConfigMap(0, Map.of(TOPOLOGY_YAML, TOPOLOGY_VALUE, SECRETS_MD_5, MD5_SECRETS))); introspectResult.defineFile(SECRETS_MD_5, "not telling").addToPacket(); testSupport.runSteps(ConfigMapHelper.createIntrospectorConfigMapStep(terminalStep)); @@ -154,26 +172,6 @@ private String getIntrospectionVersion() { .orElse(null); } - @Nonnull - private Optional getIntrospectionConfigMap() { - return testSupport.getResources(KubernetesTestSupport.CONFIG_MAP) - .stream() - .filter(this::isInstrospectConfigMap) - .findFirst(); - } - - private boolean isInstrospectConfigMap(V1ConfigMap configMap) { - return Optional.ofNullable(configMap) - .map(V1ConfigMap::getMetadata) - .map(V1ObjectMeta::getName) - .filter(name -> name.equals(getIntrospectorConfigMapName())) - .isPresent(); - } - - private static String getIntrospectorConfigMapName() { - return ConfigMapHelper.getIntrospectorConfigMapName(UID); - } - @Test public void whenTopologyNotValid_reportInDomainStatus() { introspectResult.defineFile(TOPOLOGY_YAML, @@ -250,26 +248,34 @@ public void whenTopologyAndDomainZipHashPresent_addToConfigMap() { assertThat(getIntrospectorConfigMapData(), hasEntry(DOMAINZIP_HASH, DOMAIN_HASH_VALUE)); } - public Map getIntrospectorConfigMapData() { - return getIntrospectorConfigMapData(testSupport); + private Map getIntrospectorConfigMapData() { + return getIntrospectionConfigMap() + .map(V1ConfigMap::getData) + .orElseGet(Collections::emptyMap); } - /** - * Returns the data portion of the introspector config map for the test domain. - * @param testSupport the instance of KubernetesTestSupport holding the data - */ @Nonnull - public static Map getIntrospectorConfigMapData(KubernetesTestSupport testSupport) { - return testSupport.getResources(KubernetesTestSupport.CONFIG_MAP).stream() - .map(V1ConfigMap.class::cast) + private Optional getIntrospectionConfigMap() { + return testSupport.getResources(KubernetesTestSupport.CONFIG_MAP) + .stream() + .filter(this::isOperatorResource) .filter(IntrospectorConfigMapTest::isIntrospectorConfigMap) - .map(V1ConfigMap::getData) - .findFirst() - .orElseGet(Collections::emptyMap); + .findFirst(); + } + + private boolean isOperatorResource(V1ConfigMap configMap) { + return Optional.ofNullable(configMap.getMetadata()) + .map(V1ObjectMeta::getLabels) + .map(this::hasCreatedByOperatorLabel) + .orElse(false); + } + + private boolean hasCreatedByOperatorLabel(Map m) { + return "true".equals(m.get(CREATEDBYOPERATOR_LABEL)); } private static boolean isIntrospectorConfigMap(V1ConfigMap configMap) { - return getIntrospectorConfigMapName().equals(getConfigMapName(configMap)); + return getConfigMapName(configMap).startsWith(getIntrospectorConfigMapNamePrefix(UID)); } private static String getConfigMapName(V1ConfigMap configMap) { @@ -300,6 +306,42 @@ public void whenTopologyAndMIISecretsHashPresent_addToConfigMap() { assertThat(getIntrospectorConfigMapData(), hasEntry(SECRETS_MD_5, MD5_SECRETS)); } + @Test + public void whenDataTooLargeForSingleConfigMap_recordCountInMap() { + introspectResult + .defineFile(TOPOLOGY_YAML, "domainValid: true", "domain:", " name: \"sample\"") + .defineFile(LARGE_DATA_KEY, LARGE_DATA_VALUE) + .addToPacket(); + + testSupport.runSteps(ConfigMapHelper.createIntrospectorConfigMapStep(terminalStep)); + + assertThat(getIntrospectorConfigMapData(), hasEntry(NUM_CONFIG_MAPS, NUM_MAPS_STRING)); + } + + @Test + public void whenDataTooLargeForSingleConfigMap_recordCountInPacket() { + introspectResult + .defineFile(TOPOLOGY_YAML, "domainValid: true", "domain:", " name: \"sample\"") + .defineFile(LARGE_DATA_KEY, LARGE_DATA_VALUE) + .addToPacket(); + + final Packet packet = testSupport.runSteps(ConfigMapHelper.createIntrospectorConfigMapStep(terminalStep)); + + assertThat(packet.getValue(NUM_CONFIG_MAPS), equalTo(NUM_MAPS_STRING)); + } + + @Test + public void whenDataTooLargeForSingleConfigMap_createMultipleMaps() { + introspectResult + .defineFile(TOPOLOGY_YAML, "domainValid: true", "domain:", " name: \"sample\"") + .defineFile(LARGE_DATA_KEY, LARGE_DATA_VALUE) + .addToPacket(); + + testSupport.runSteps(ConfigMapHelper.createIntrospectorConfigMapStep(terminalStep)); + + assertThat(getIntrospectionConfigMaps(), hasSize(NUM_MAPS_TO_CREATE)); + } + @Test public void whenDomainHasRestartVersion_addToPacket() { configureDomain().withRestartVersion(RESTART_VERSION); @@ -309,7 +351,7 @@ public void whenDomainHasRestartVersion_addToPacket() { Packet packet = testSupport.runSteps(ConfigMapHelper.createIntrospectorConfigMapStep(terminalStep)); - assertThat(packet.get(IntrospectorConfigMapKeys.DOMAIN_RESTART_VERSION), equalTo(RESTART_VERSION)); + assertThat(packet.get(IntrospectorConfigMapConstants.DOMAIN_RESTART_VERSION), equalTo(RESTART_VERSION)); } private DomainConfigurator configureDomain() { @@ -328,20 +370,58 @@ public void whenDomainIsModelInImage_addImageSpecHashToPacket() { assertThat(packet.get(DOMAIN_INPUTS_HASH), notNullValue()); } - private V1ConfigMap createIntrospectorConfigMap(Map entries) { + @Test + public void whenDomainIsModelInImage_dontAddRangesForZipsThatFitInMainConfigMap() { + configureDomain().withDomainHomeSourceType(DomainSourceType.FromModel); + introspectResult + .defineFile(TOPOLOGY_YAML, "domainValid: true", "domain:", " name: \"sample\"") + .defineFile("domainzip.secure", "abcdefg") + .defineFile("primordial_domainzip.secure", "hijklmno") + .addToPacket(); + + testSupport.runSteps(ConfigMapHelper.createIntrospectorConfigMapStep(terminalStep)); + + assertThat(getIntrospectorConfigMapValue("domainzip.secure.range"), nullValue()); + assertThat(getIntrospectorConfigMapValue("primordial_domainzip.secure.range"), nullValue()); + } + + private V1ConfigMap createIntrospectorConfigMap(int mapIndex, Map entries) { return new V1ConfigMap() - .metadata(new V1ObjectMeta().name(getIntrospectorConfigMapName()).namespace(NS)) + .metadata(createOperatorMetadata().name(getIntrospectorConfigMapName(mapIndex)).namespace(NS)) .data(new HashMap<>(entries)); } + @Test + public void whenDomainIsModelInImageAndEncodedZipTooLargeForSingleMap_reportRange() { + configureDomain().withDomainHomeSourceType(DomainSourceType.FromModel); + introspectResult + .defineFile(TOPOLOGY_YAML, "domainValid: true", "domain:", " name: \"sample\"") + .defineFile("domainzip.secure", LARGE_DATA_VALUE) + .addToPacket(); + + testSupport.runSteps(ConfigMapHelper.createIntrospectorConfigMapStep(terminalStep)); + + assertThat(getIntrospectorConfigMapValue("domainzip.secure.range"), equalTo("0 2")); + } + + @Nonnull + private V1ObjectMeta createOperatorMetadata() { + return new V1ObjectMeta().putLabelsItem(CREATEDBYOPERATOR_LABEL, "true"); + } + + private static String getIntrospectorConfigMapName(int mapIndex) { + return IntrospectorConfigMapConstants.getIntrospectorConfigMapName(UID, mapIndex); + } + @Test public void loadExistingEntriesFromIntrospectorConfigMap() { - testSupport.defineResources(createIntrospectorConfigMap(Map.of( + testSupport.defineResources(createIntrospectorConfigMap(0, Map.of( TOPOLOGY_YAML, TOPOLOGY_VALUE, SECRETS_MD_5, MD5_SECRETS, DOMAINZIP_HASH, DOMAIN_HASH_VALUE, DOMAIN_RESTART_VERSION, RESTART_VERSION, - DOMAIN_INPUTS_HASH, INPUTS_HASH_VALUE))); + DOMAIN_INPUTS_HASH, INPUTS_HASH_VALUE, + NUM_CONFIG_MAPS, NUM_MAPS_STRING))); Packet packet = testSupport.runSteps(ConfigMapHelper.readExistingIntrospectorConfigMap(NS, UID)); @@ -350,6 +430,7 @@ public void loadExistingEntriesFromIntrospectorConfigMap() { assertThat(packet.get(DOMAIN_RESTART_VERSION), equalTo(RESTART_VERSION)); assertThat(packet.get(DOMAIN_INPUTS_HASH), equalTo(INPUTS_HASH_VALUE)); assertThat(packet.get(DOMAIN_TOPOLOGY), equalTo(getParsedDomain(TOPOLOGY_VALUE))); + assertThat(packet.get(NUM_CONFIG_MAPS), equalTo(NUM_MAPS_STRING)); } @SuppressWarnings("SameParameterValue") @@ -362,7 +443,7 @@ private WlsDomainConfig getParsedDomain(String topologyYaml) { @Test public void whenOrdinaryEntriesMissingFromIntrospectionResult_doNotRemoveFromConfigMap() { - testSupport.defineResources(createIntrospectorConfigMap(Map.of( + testSupport.defineResources(createIntrospectorConfigMap(0, Map.of( TOPOLOGY_YAML, TOPOLOGY_VALUE, "oldEntry1", "value1", "oldEntry2", "value2"))); @@ -377,7 +458,7 @@ public void whenOrdinaryEntriesMissingFromIntrospectionResult_doNotRemoveFromCon @Test public void whenSitConfigEntriesMissingFromIntrospectionResult_removeFromConfigMap() { - testSupport.defineResources(createIntrospectorConfigMap(Map.of( + testSupport.defineResources(createIntrospectorConfigMap(0, Map.of( TOPOLOGY_YAML, TOPOLOGY_VALUE, "Sit-Cfg-1", "value1", "Sit-Cfg-2", "value2"))); @@ -393,11 +474,34 @@ public void whenSitConfigEntriesMissingFromIntrospectionResult_removeFromConfigM @Test public void whenNoTopologySpecified_dontRemoveSitConfigEntries() { testSupport.defineResources( - createIntrospectorConfigMap(Map.of(TOPOLOGY_YAML, TOPOLOGY_VALUE, "Sit-Cfg-1", "value1"))); + createIntrospectorConfigMap(0, Map.of(TOPOLOGY_YAML, TOPOLOGY_VALUE, "Sit-Cfg-1", "value1"))); introspectResult.defineFile(SECRETS_MD_5, "not telling").addToPacket(); testSupport.runSteps(ConfigMapHelper.createIntrospectorConfigMapStep(terminalStep)); assertThat(getIntrospectorConfigMapValue("Sit-Cfg-1"), equalTo("value1")); } + + @Test + public void whenRequested_deleteAllIntrospectorConfigMaps() { + testSupport.defineResources( + createIntrospectorConfigMap(0, Map.of()), + createIntrospectorConfigMap(1, Map.of()), + createIntrospectorConfigMap(2, Map.of()) + ); + + testSupport.runSteps(ConfigMapHelper.deleteIntrospectorConfigMapStep(UID, NS, null)); + + assertThat(getIntrospectionConfigMaps(), empty()); + } + + @Nonnull + private List getIntrospectionConfigMaps() { + return testSupport.getResources(KubernetesTestSupport.CONFIG_MAP) + .stream() + .filter(this::isOperatorResource) + .filter(IntrospectorConfigMapTest::isIntrospectorConfigMap) + .collect(Collectors.toList()); + } + } diff --git a/operator/src/test/java/oracle/kubernetes/operator/helpers/Matchers.java b/operator/src/test/java/oracle/kubernetes/operator/helpers/Matchers.java index d6c6f6d388e..c7c060e6d00 100644 --- a/operator/src/test/java/oracle/kubernetes/operator/helpers/Matchers.java +++ b/operator/src/test/java/oracle/kubernetes/operator/helpers/Matchers.java @@ -5,12 +5,15 @@ import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Optional; import javax.annotation.Nonnull; import io.kubernetes.client.custom.Quantity; +import io.kubernetes.client.openapi.models.V1ConfigMapVolumeSource; import io.kubernetes.client.openapi.models.V1Container; import io.kubernetes.client.openapi.models.V1EnvVar; import io.kubernetes.client.openapi.models.V1HostPathVolumeSource; @@ -85,7 +88,7 @@ private static V1Container createInitContainer(String name, String image, String private static V1Container createInitContainerWithEnvVar(String name, String image, String serverName, V1EnvVar envVar, String... command) { - List envVars = new ArrayList<>(Arrays.asList(envVar)); + List envVars = new ArrayList<>(Collections.singletonList(envVar)); PodHelperTestBase.getPredefinedEnvVariables(serverName).forEach(predefEnvVar -> addIfMissing(envVars, predefEnvVar.getName(), predefEnvVar.getValue())); return new V1Container().name(name).image(image).command(Arrays.asList(command)) @@ -111,9 +114,53 @@ protected static void addIfMissing(List vars, String name, String valu } } + public static class VolumeMatcher extends TypeSafeDiagnosingMatcher { + private final String expectedName; + private final String expectedConfigMapName; + + private VolumeMatcher(String expectedName, String expectedConfigMapName) { + this.expectedName = expectedName; + this.expectedConfigMapName = expectedConfigMapName; + } + + public static VolumeMatcher volume(String expectedName, String expectedConfigMapName) { + return new VolumeMatcher(expectedName, expectedConfigMapName); + } + + @Override + protected boolean matchesSafely(V1Volume item, Description mismatchDescription) { + if (isExpectedVolume(item)) { + return true; + } + + describe(mismatchDescription, item.getName(), getConfigMapName(item)); + return false; + } + + private void describe(Description description, String name, String configMapName) { + description.appendText("volume with name: ").appendValue(name); + if (expectedConfigMapName != null) { + description.appendText(", config map name: ").appendValue(configMapName); + } + } + + private boolean isExpectedVolume(V1Volume volume) { + return expectedName.equals(volume.getName()) + && expectedConfigMapName.equals(getConfigMapName(volume)); + } + + private String getConfigMapName(V1Volume volume) { + return Optional.ofNullable(volume.getConfigMap()).map(V1ConfigMapVolumeSource::getName).orElse(null); + } + + @Override + public void describeTo(Description description) { + describe(description, expectedName, expectedConfigMapName); + } + } + @SuppressWarnings("unused") - public static class VolumeMountMatcher - extends org.hamcrest.TypeSafeDiagnosingMatcher { + public static class VolumeMountMatcher extends TypeSafeDiagnosingMatcher { private final String expectedName; private final String expectedPath; private final boolean readOnly; @@ -214,6 +261,7 @@ public void describeTo(Description description) { } } + @SuppressWarnings("unused") static class EnvVarMatcher extends TypeSafeDiagnosingMatcher { private static final String DONTCARE = "SENTINEL_DONT_CARE"; private final String expectedName; @@ -239,21 +287,19 @@ static EnvVarMatcher envVarWithNameAndValue(@Nonnull String name, String value) @Override protected boolean matchesSafely(V1EnvVar item, Description mismatchDescription) { - if (expectedValueRegEx == DONTCARE) { - if (expectedName.equals(item.getName())) { + if (expectedValueRegEx == null) { + if (expectedName.equals(item.getName()) && item.getValue() == null) { return true; } mismatchDescription.appendText("EnvVar with name ").appendValue(item.getName()); return false; - } - if (expectedValueRegEx == null) { - if (expectedName.equals(item.getName()) && item.getValue() == null) { + } else if (expectedValueRegEx.equals(DONTCARE)) { + if (expectedName.equals(item.getName())) { return true; } mismatchDescription.appendText("EnvVar with name ").appendValue(item.getName()); return false; - } - if (expectedName.equals(item.getName()) + } else if (expectedName.equals(item.getName()) && item.getValue() != null && item.getValue().matches(expectedValueRegEx)) { return true; @@ -269,7 +315,7 @@ protected boolean matchesSafely(V1EnvVar item, Description mismatchDescription) @Override public void describeTo(Description description) { description.appendText("EnvVar with name=").appendValue(expectedName); - if (expectedValueRegEx != DONTCARE) { + if (!expectedValueRegEx.equals(DONTCARE)) { description.appendText(" value=").appendValue(expectedValueRegEx); } } diff --git a/operator/src/test/java/oracle/kubernetes/operator/helpers/PodHelperTestBase.java b/operator/src/test/java/oracle/kubernetes/operator/helpers/PodHelperTestBase.java index 440b5c05aae..ba08c0f4de5 100644 --- a/operator/src/test/java/oracle/kubernetes/operator/helpers/PodHelperTestBase.java +++ b/operator/src/test/java/oracle/kubernetes/operator/helpers/PodHelperTestBase.java @@ -48,7 +48,7 @@ import io.kubernetes.client.openapi.models.V1VolumeMount; import io.kubernetes.client.openapi.models.V1WeightedPodAffinityTerm; import oracle.kubernetes.operator.DomainSourceType; -import oracle.kubernetes.operator.IntrospectorConfigMapKeys; +import oracle.kubernetes.operator.IntrospectorConfigMapConstants; import oracle.kubernetes.operator.KubernetesConstants; import oracle.kubernetes.operator.LabelConstants; import oracle.kubernetes.operator.MakeRightDomainOperation; @@ -81,9 +81,12 @@ import static com.meterware.simplestub.Stub.createStrictStub; import static com.meterware.simplestub.Stub.createStub; +import static oracle.kubernetes.operator.IntrospectorConfigMapConstants.INTROSPECTOR_CONFIG_MAP_NAME_SUFFIX; +import static oracle.kubernetes.operator.IntrospectorConfigMapConstants.NUM_CONFIG_MAPS; import static oracle.kubernetes.operator.KubernetesConstants.ALWAYS_IMAGEPULLPOLICY; import static oracle.kubernetes.operator.KubernetesConstants.CONTAINER_NAME; import static oracle.kubernetes.operator.KubernetesConstants.DEFAULT_IMAGE; +import static oracle.kubernetes.operator.KubernetesConstants.DOMAIN_DEBUG_CONFIG_MAP_SUFFIX; import static oracle.kubernetes.operator.KubernetesConstants.IFNOTPRESENT_IMAGEPULLPOLICY; import static oracle.kubernetes.operator.KubernetesConstants.SCRIPT_CONFIG_MAP_NAME; import static oracle.kubernetes.operator.ProcessingConstants.MAKE_RIGHT_DOMAIN_OPERATION; @@ -93,6 +96,7 @@ import static oracle.kubernetes.operator.helpers.KubernetesTestSupport.DOMAIN; import static oracle.kubernetes.operator.helpers.KubernetesTestSupport.POD; import static oracle.kubernetes.operator.helpers.Matchers.ProbeMatcher.hasExpectedTuning; +import static oracle.kubernetes.operator.helpers.Matchers.VolumeMatcher.volume; import static oracle.kubernetes.operator.helpers.Matchers.VolumeMountMatcher.readOnlyVolumeMount; import static oracle.kubernetes.operator.helpers.Matchers.VolumeMountMatcher.writableVolumeMount; import static oracle.kubernetes.operator.helpers.Matchers.hasEnvVar; @@ -100,9 +104,11 @@ import static oracle.kubernetes.operator.helpers.Matchers.hasResourceQuantity; import static oracle.kubernetes.operator.helpers.Matchers.hasVolume; import static oracle.kubernetes.operator.helpers.Matchers.hasVolumeMount; +import static oracle.kubernetes.operator.helpers.StepContextConstants.DEBUG_CM_VOLUME; +import static oracle.kubernetes.operator.helpers.StepContextConstants.INTROSPECTOR_VOLUME; import static oracle.kubernetes.operator.helpers.StepContextConstants.RUNTIME_ENCRYPTION_SECRET_MOUNT_PATH; import static oracle.kubernetes.operator.helpers.StepContextConstants.RUNTIME_ENCRYPTION_SECRET_VOLUME; -import static oracle.kubernetes.operator.helpers.StepContextConstants.SIT_CONFIG_MAP_VOLUME; +import static oracle.kubernetes.operator.helpers.StepContextConstants.SCRIPTS_VOLUME; import static oracle.kubernetes.operator.helpers.TuningParametersStub.LIVENESS_INITIAL_DELAY; import static oracle.kubernetes.operator.helpers.TuningParametersStub.LIVENESS_PERIOD; import static oracle.kubernetes.operator.helpers.TuningParametersStub.LIVENESS_TIMEOUT; @@ -170,7 +176,7 @@ public abstract class PodHelperTestBase extends DomainValidationBaseTest { } Domain getDomain() { - return (Domain) testSupport.getResourceWithName(DOMAIN, DOMAIN_NAME); + return testSupport.getResourceWithName(DOMAIN, DOMAIN_NAME); } String getPodName() { @@ -388,7 +394,7 @@ public void whenPodCreated_withNoPvc_image_containerHasExpectedVolumeMounts() { getCreatedPodSpecContainer().getVolumeMounts(), containsInAnyOrder( writableVolumeMount( - SIT_CONFIG_MAP_VOLUME, "/weblogic-operator/introspector"), + INTROSPECTOR_VOLUME, "/weblogic-operator/introspector"), readOnlyVolumeMount("weblogic-domain-debug-cm-volume", "/weblogic-operator/debug"), readOnlyVolumeMount("weblogic-scripts-cm-volume", "/weblogic-operator/scripts"))); } @@ -402,13 +408,26 @@ public void whenPodCreated_withNoPvc_fromModel_containerHasExpectedVolumeMounts( getCreatedPodSpecContainer().getVolumeMounts(), containsInAnyOrder( writableVolumeMount( - SIT_CONFIG_MAP_VOLUME, "/weblogic-operator/introspector"), + INTROSPECTOR_VOLUME, "/weblogic-operator/introspector"), readOnlyVolumeMount("weblogic-domain-debug-cm-volume", "/weblogic-operator/debug"), readOnlyVolumeMount("weblogic-scripts-cm-volume", "/weblogic-operator/scripts"), readOnlyVolumeMount(RUNTIME_ENCRYPTION_SECRET_VOLUME, RUNTIME_ENCRYPTION_SECRET_MOUNT_PATH))); } + @Test + public void whenIntrospectionCreatesMultipleConfigMaps_createCorrespondingVolumeMounts() { + testSupport.addToPacket(NUM_CONFIG_MAPS, "3"); + + assertThat( + getCreatedPodSpecContainer().getVolumeMounts(), + allOf( + hasItem(writableVolumeMount(INTROSPECTOR_VOLUME, "/weblogic-operator/introspector")), + hasItem(writableVolumeMount(INTROSPECTOR_VOLUME + "_1", "/weblogic-operator/introspector_1")), + hasItem(writableVolumeMount(INTROSPECTOR_VOLUME + "_2", "/weblogic-operator/introspector_2")) + )); + } + public void reportInspectionWasRun() { testSupport.addToPacket(MAKE_RIGHT_DOMAIN_OPERATION, reportIntrospectionRun()); } @@ -606,7 +625,7 @@ private V1Pod getPatchedPod() { assertThat(logRecords, containsInfo(getPatchedMessageKey())); - return (V1Pod) testSupport.getResourceWithName(KubernetesTestSupport.POD, getPodName()); + return testSupport.getResourceWithName(KubernetesTestSupport.POD, getPodName()); } protected abstract ServerConfigurator configureServer( @@ -1038,20 +1057,20 @@ public void whenServerAddsNap_replacePod() { @Test public void whenMiiSecretsHashChanged_replacePod() { - testSupport.addToPacket(IntrospectorConfigMapKeys.SECRETS_MD_5, "originalSecret"); + testSupport.addToPacket(IntrospectorConfigMapConstants.SECRETS_MD_5, "originalSecret"); initializeExistingPod(); - testSupport.addToPacket(IntrospectorConfigMapKeys.SECRETS_MD_5, "newSecret"); + testSupport.addToPacket(IntrospectorConfigMapConstants.SECRETS_MD_5, "newSecret"); verifyPodReplaced(); } @Test public void whenMiiDomainZipHashChanged_replacePod() { - testSupport.addToPacket(IntrospectorConfigMapKeys.DOMAINZIP_HASH, "originalSecret"); + testSupport.addToPacket(IntrospectorConfigMapConstants.DOMAINZIP_HASH, "originalSecret"); initializeExistingPod(); - testSupport.addToPacket(IntrospectorConfigMapKeys.DOMAINZIP_HASH, "newSecret"); + testSupport.addToPacket(IntrospectorConfigMapConstants.DOMAINZIP_HASH, "newSecret"); verifyPodReplaced(); } @@ -1139,7 +1158,7 @@ V1Container createPodSpecContainer() { .addPortsItem( new V1ContainerPort().name("default").containerPort(listenPort).protocol("TCP")) .lifecycle(createLifecycle()) - .volumeMounts(PodDefaults.getStandardVolumeMounts(UID)) + .volumeMounts(PodDefaults.getStandardVolumeMounts(UID, 1)) .command(createStartCommand()) .addEnvItem(envItem("DOMAIN_NAME", DOMAIN_NAME)) .addEnvItem(envItem("DOMAIN_HOME", "/u01/oracle/user_projects/domains")) @@ -1168,7 +1187,7 @@ V1PodSpec createPodSpec() { .securityContext(new V1PodSecurityContext()) .containers(Collections.singletonList(createPodSpecContainer())) .nodeSelector(Collections.emptyMap()) - .volumes(PodDefaults.getStandardVolumes(UID)); + .volumes(PodDefaults.getStandardVolumes(UID, 1)); } static V1PodSecurityContext createPodSecurityContext(long runAsGroup) { @@ -1237,6 +1256,26 @@ public void whenDomainPresenceInfoLacksImageName_createdPodUsesDefaultImage() { assertThat(getCreatedPodSpecContainer().getImage(), equalTo(DEFAULT_IMAGE)); } + @Test + public void verifyStandardVolumes() { + assertThat( + getCreatedPod().getSpec().getVolumes(), + allOf(hasItem(volume(INTROSPECTOR_VOLUME, UID + INTROSPECTOR_CONFIG_MAP_NAME_SUFFIX)), + hasItem(volume(DEBUG_CM_VOLUME, UID + DOMAIN_DEBUG_CONFIG_MAP_SUFFIX)), + hasItem(volume(SCRIPTS_VOLUME, SCRIPT_CONFIG_MAP_NAME)))); + } + + @Test + public void whenIntrospectionCreatesMultipleConfigMaps_createCorrespondingVolumes() { + testSupport.addToPacket(NUM_CONFIG_MAPS, "3"); + + assertThat( + getCreatedPod().getSpec().getVolumes(), + allOf(hasItem(volume(INTROSPECTOR_VOLUME, UID + INTROSPECTOR_CONFIG_MAP_NAME_SUFFIX)), + hasItem(volume(INTROSPECTOR_VOLUME + "_1", UID + INTROSPECTOR_CONFIG_MAP_NAME_SUFFIX + "_1")), + hasItem(volume(INTROSPECTOR_VOLUME + "_2", UID + INTROSPECTOR_CONFIG_MAP_NAME_SUFFIX + "_2")))); + } + @Test public void whenDomainHasAdditionalVolumes_createPodWithThem() { getConfigurator() diff --git a/operator/src/test/java/oracle/kubernetes/operator/utils/InMemoryFileSystem.java b/operator/src/test/java/oracle/kubernetes/operator/utils/InMemoryFileSystem.java index 26a44376e47..db041609e88 100644 --- a/operator/src/test/java/oracle/kubernetes/operator/utils/InMemoryFileSystem.java +++ b/operator/src/test/java/oracle/kubernetes/operator/utils/InMemoryFileSystem.java @@ -3,6 +3,7 @@ package oracle.kubernetes.operator.utils; +import java.io.File; import java.io.FileNotFoundException; import java.net.URI; import java.nio.ByteBuffer; @@ -40,12 +41,12 @@ public void defineFile(String filePath, String contents) { @Nonnull public Path getPath(@Nonnull String first, @Nonnull String... more) { - return createStrictStub(PathStub.class, createPathString(first, more)); + return PathStub.createPathStub(createPathString(first, more)); } @Nonnull public Path getPath(@Nonnull URI uri) { - return createStrictStub(PathStub.class, createPathString(uri.getPath(), new String[0])); + return PathStub.createPathStub(createPathString(uri.getPath(), new String[0])); } private String createPathString(String first, String[] more) { @@ -91,6 +92,10 @@ abstract static class PathStub implements Path { this.filePath = filePath; } + static PathStub createPathStub(String pathString) { + return createStrictStub(PathStub.class, pathString); + } + @Override @Nonnull public FileSystem getFileSystem() { return instance; @@ -98,7 +103,13 @@ abstract static class PathStub implements Path { @Override public Path getFileName() { - return this; + return createPathStub(getLastElement()); + } + + @Nonnull + protected String getLastElement() { + final int beginIndex = filePath.lastIndexOf(File.separatorChar); + return beginIndex < 0 ? filePath : filePath.substring(beginIndex + 1); } @Override @@ -190,7 +201,7 @@ abstract static class DirectoryStreamStub implements DirectoryStream { public DirectoryStreamStub(FileSystemProviderStub parent, String root) { for (String key : parent.fileContents.keySet()) { if (key.startsWith(root + "/")) { - paths.add(createStrictStub(PathStub.class, key)); + paths.add(PathStub.createPathStub(key)); } } } diff --git a/operator/src/test/sh/modelInImageTest.sh b/operator/src/test/sh/modelInImageTest.sh new file mode 100644 index 00000000000..85cabd64a9e --- /dev/null +++ b/operator/src/test/sh/modelInImageTest.sh @@ -0,0 +1,207 @@ +#!/usr/bin/env bash +# Copyright (c) 2020, Oracle Corporation and/or its affiliates. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. + +TEST_OPERATOR_ROOT=/tmp/test/weblogic-operator +setUp() { + DISALLOW= + PWD=/no/where/special + DOMAIN_HOME=/domain/home + + INTROSPECTOR_MAP=${TEST_OPERATOR_ROOT}/introspector + rm -fR ${TEST_OPERATOR_ROOT} + mkdir -p ${TEST_OPERATOR_ROOT}/introspector + echo "" > $INTROSPECTOR_MAP/domainzip.secure + echo "" > $INTROSPECTOR_MAP/primordial_domainzip.secure +} + +testIndexRangeWhenRangeFileMissing() { + actual=$(getIndexRange "domainzip.secure") + expected="0 0" + + assertEquals "$expected" "$actual" +} + +testIndexRangeWhenRangeFilePresent() { + echo "0 2" > $INTROSPECTOR_MAP/domainzip.secure.range + actual=$(getIndexRange "domainzip.secure") + expected="0 2" + + assertEquals "$expected" "$actual" +} + +testBuildConfigMapMultipleElements() { + actual=$(buildConfigMapElements "domain.secure" 0 2) + expected="$INTROSPECTOR_MAP/domain.secure ${INTROSPECTOR_MAP}_1/domain.secure ${INTROSPECTOR_MAP}_2/domain.secure" + + assertEquals "$expected" "$actual" +} + +testBuildConfigMapOneElementAtZero() { + actual=$(buildConfigMapElements "domain.secure" 0 0) + expected="$INTROSPECTOR_MAP/domain.secure" + + assertEquals "$expected" "$actual" +} + +testBuildConfigMapOneElementAfterZero() { + actual=$(buildConfigMapElements "domain.secure" 1 1) + expected="${INTROSPECTOR_MAP}_1/domain.secure" + + assertEquals "$expected" "$actual" +} + +testRestoreDomainConfig_failsIfUnableToCDToRoot() { + DISALLOW="CD" + + restoreDomainConfig + + assertEquals "should have failed to cd to /" '1' "$?" +} + +testRestoreDomainConfig_failsIfUnableToDecodeDomainZip() { + DISALLOW="BASE64" + + restoreDomainConfig + + assertEquals "should have failed to run decode domainzip" '1' "$?" +} + +testRestoreDomainConfig_failsIfUnableToUnTarDomain() { + DISALLOW="TAR" + + restoreDomainConfig + + assertEquals "should have failed to run tar" '1' "$?" +} + +testOnRestoreDomainConfig_useRootDirectory() { + restoreDomainConfig + + assertEquals "should be at '/'" "/" "$PWD" +} + +testOnRestoreDomainConfig_whenNoIndexesDefinedCatSingleFile() { + echo -n "abc" > $INTROSPECTOR_MAP/domainzip.secure + + restoreDomainConfig + + expected="abc" + actual="$(cat /tmp/domain.secure)" + assertEquals "$expected" "$actual" +} + +testOnRestoreDomainConfig_whenIndexesDefinedCatMultipleFiles() { + mkdir ${INTROSPECTOR_MAP}_1 + mkdir ${INTROSPECTOR_MAP}_2 + echo "0 2" > $INTROSPECTOR_MAP/domainzip.secure.range + echo -n "abc" > $INTROSPECTOR_MAP/domainzip.secure + echo -n "def" > ${INTROSPECTOR_MAP}_1/domainzip.secure + echo -n "ghi" > ${INTROSPECTOR_MAP}_2/domainzip.secure + + restoreDomainConfig + + expected="abcdefghi" + actual="$(cat /tmp/domain.secure)" + assertEquals "$expected" "$actual" +} + +testOnRestoreDomainConfig_base64DecodeZip() { + rm /tmp/domain.tar.gz + + restoreDomainConfig + + actual="$(cat /tmp/domain.tar.gz)" + assertEquals "/tmp/domain.secure" $actual +} + +testOnRestoreDomainConfig_unTarDomain() { + restoreDomainConfig + + assertEquals "TAR command arguments" "-xzf /tmp/domain.tar.gz" "$TAR_ARGS" +} + +testOnRestoreDomainConfig_makeScriptsExecutable() { + restoreDomainConfig + + assertEquals "CD command arguments" "+x ${DOMAIN_HOME}/bin/*.sh ${DOMAIN_HOME}/*.sh" "$CHMOD_ARGS" +} + +testOnRestorePrimordialDomain_useRootDirectory() { + restorePrimordialDomain + + assertEquals "should be at '/'" "/" "$PWD" +} + +testOnRestorePrimordialDomain_base64DecodeZip() { + rm /tmp/domain.tar.gz + + restorePrimordialDomain + + actual="$(cat /tmp/domain.tar.gz)" + assertEquals "/tmp/domain.secure" $actual +} + +testOnRestoreDomainConfig_whenNoIndexesDefinedCatSingleFile() { + echo -n "abc" > $INTROSPECTOR_MAP/primordial_domainzip.secure + + restorePrimordialDomain + + expected="abc" + actual="$(cat /tmp/domain.secure)" + assertEquals "$expected" "$actual" +} + +testOnRestorePrimordialDomain_unTarDomain() { + restorePrimordialDomain + + assertEquals "TAR command arguments" "-xzf /tmp/domain.tar.gz" "$TAR_ARGS" +} + +######################### Mocks for the tests ############### + +# simulates the shell 'cd' command. Will fail on CD to forbidden location, or set PWD +# otherwise +cd() { + if [ "$DISALLOW" = "CD" ]; then + return 1 + else + PWD=$1 + fi +} + +base64() { + if [ "$DISALLOW" = "BASE64" ]; then + return 1 + elif [ "$1" != "-d" ]; then + return 1 + else + echo "$2" + fi +} + +source() { + if [ "$DISALLOW" = "SOURCE" ]; then + return 1 + else + SOURCE_ARGS="$*" + fi +} + +tar() { + if [ "$DISALLOW" = "TAR" ]; then + return 1 + else + TAR_ARGS="$*" + fi +} + +chmod() { + CHMOD_ARGS="$*" +} + +# shellcheck source=src/main/resources/scripts/modelInImage.sh +. ${SCRIPTPATH}/modelInImage.sh + +# shellcheck source=target/classes/shunit/shunit2 +. ${SHUNIT2_PATH} \ No newline at end of file From 1d2cb146476385e6996d54c36830b96914e126b8 Mon Sep 17 00:00:00 2001 From: Russell Gold Date: Thu, 10 Dec 2020 14:51:45 -0500 Subject: [PATCH 3/6] Update implementation docs to mention additional config maps --- .../userguide/managing-domains/configoverrides/_index.md | 4 ++-- .../userguide/managing-domains/model-in-image/overview.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs-source/content/userguide/managing-domains/configoverrides/_index.md b/docs-source/content/userguide/managing-domains/configoverrides/_index.md index 1f9fcd57660..c5898489627 100644 --- a/docs-source/content/userguide/managing-domains/configoverrides/_index.md +++ b/docs-source/content/userguide/managing-domains/configoverrides/_index.md @@ -451,8 +451,8 @@ when `spec.logHome` is configured and `spec.logHomeEnabled` is true. * The operator runtime: * Reads the expanded configuration overrides files or errors from the introspector. * And, if the introspector reported no errors, it: - * Puts configuration overrides files in a ConfigMap named `DOMAIN_UID-weblogic-domain-introspect-cm`. - * Mounts this ConfigMap into the WebLogic Server instance Pods. + * Puts configuration overrides files in one or more ConfigMaps whose names start with `DOMAIN_UID-weblogic-domain-introspect-cm`. + * Mounts these ConfigMaps into the WebLogic Server instance Pods. * Otherwise, if the introspector reported errors, it: * Logs warning, error, or severe messages. * Will not start WebLogic Server instance Pods; however, any already running Pods are preserved. diff --git a/docs-source/content/userguide/managing-domains/model-in-image/overview.md b/docs-source/content/userguide/managing-domains/model-in-image/overview.md index 6b2c995e3f0..ece66c16a0c 100644 --- a/docs-source/content/userguide/managing-domains/model-in-image/overview.md +++ b/docs-source/content/userguide/managing-domains/model-in-image/overview.md @@ -47,7 +47,7 @@ When you deploy a Model in Image Domain YAML file: - Packages the domain home and passes it to the operator. - After the introspector job completes: - - The operator creates a ConfigMap named `DOMAIN_UID-weblogic-domain-introspect-cm` and puts the packaged domain home in it. + - The operator creates a ConfigMap named `DOMAIN_UID-weblogic-domain-introspect-cm` (possibly with some additional maps distinguished serial names) and puts the packaged domain home in it. - The operator subsequently boots your domain's WebLogic Server pods. - The pods will obtain their domain home from the ConfigMap. From ccd44f38811ea7bc7fe92c15da7114c7512e377a Mon Sep 17 00:00:00 2001 From: Russell Gold Date: Fri, 11 Dec 2020 11:59:27 -0500 Subject: [PATCH 4/6] simplify joining split archives --- .../main/resources/scripts/modelInImage.sh | 4 +-- operator/src/test/sh/modelInImageTest.sh | 36 ------------------- 2 files changed, 1 insertion(+), 39 deletions(-) diff --git a/operator/src/main/resources/scripts/modelInImage.sh b/operator/src/main/resources/scripts/modelInImage.sh index 1d9b4f605a3..45b0ccb9fdb 100755 --- a/operator/src/main/resources/scripts/modelInImage.sh +++ b/operator/src/main/resources/scripts/modelInImage.sh @@ -533,9 +533,7 @@ function restorePrimordialDomain() { # $1 the name of the encoded file in the config map function restoreEncodedTar() { cd / || return 1 - indexRange="$(getIndexRange $1)" - # shellcheck disable=SC2046 - cat $(buildConfigMapElements $1 $indexRange) > /tmp/domain.secure + cat ${OPERATOR_ROOT}/introspector*/${1} > /tmp/domain.secure base64 -d "/tmp/domain.secure" > /tmp/domain.tar.gz || return 1 tar -xzf /tmp/domain.tar.gz || return 1 diff --git a/operator/src/test/sh/modelInImageTest.sh b/operator/src/test/sh/modelInImageTest.sh index 85cabd64a9e..2873bdf894e 100644 --- a/operator/src/test/sh/modelInImageTest.sh +++ b/operator/src/test/sh/modelInImageTest.sh @@ -15,42 +15,6 @@ setUp() { echo "" > $INTROSPECTOR_MAP/primordial_domainzip.secure } -testIndexRangeWhenRangeFileMissing() { - actual=$(getIndexRange "domainzip.secure") - expected="0 0" - - assertEquals "$expected" "$actual" -} - -testIndexRangeWhenRangeFilePresent() { - echo "0 2" > $INTROSPECTOR_MAP/domainzip.secure.range - actual=$(getIndexRange "domainzip.secure") - expected="0 2" - - assertEquals "$expected" "$actual" -} - -testBuildConfigMapMultipleElements() { - actual=$(buildConfigMapElements "domain.secure" 0 2) - expected="$INTROSPECTOR_MAP/domain.secure ${INTROSPECTOR_MAP}_1/domain.secure ${INTROSPECTOR_MAP}_2/domain.secure" - - assertEquals "$expected" "$actual" -} - -testBuildConfigMapOneElementAtZero() { - actual=$(buildConfigMapElements "domain.secure" 0 0) - expected="$INTROSPECTOR_MAP/domain.secure" - - assertEquals "$expected" "$actual" -} - -testBuildConfigMapOneElementAfterZero() { - actual=$(buildConfigMapElements "domain.secure" 1 1) - expected="${INTROSPECTOR_MAP}_1/domain.secure" - - assertEquals "$expected" "$actual" -} - testRestoreDomainConfig_failsIfUnableToCDToRoot() { DISALLOW="CD" From aafd19d61e3ad5fb69405fca1302a720e6d19c32 Mon Sep 17 00:00:00 2001 From: Russell Gold Date: Fri, 11 Dec 2020 12:00:42 -0500 Subject: [PATCH 5/6] delete unusued functions --- .../main/resources/scripts/modelInImage.sh | 38 ------------------- 1 file changed, 38 deletions(-) diff --git a/operator/src/main/resources/scripts/modelInImage.sh b/operator/src/main/resources/scripts/modelInImage.sh index 45b0ccb9fdb..45bcc24ba7f 100755 --- a/operator/src/main/resources/scripts/modelInImage.sh +++ b/operator/src/main/resources/scripts/modelInImage.sh @@ -539,44 +539,6 @@ function restoreEncodedTar() { tar -xzf /tmp/domain.tar.gz || return 1 } -# Returns the index range for an encoded file in an introspector config map -# args: -# $1 the key of the encoded file in the map -function getIndexRange() { - rangeFile="$OPERATOR_ROOT/introspector/$1.range" - if [ -f "$rangeFile" ]; then - cat "$rangeFile" - else - echo "0 0" - fi -} - -# Creates a string containing entries from one or more config maps -# args: -# $1 the key of the entry in each map -# $2 the index of the first configmap containing the entry key -# $3 the index of the last configmap containing the entry key -function buildConfigMapElements() { - result="" - - for ((i=$2;i<=$3;i++)); do - result="$result ${OPERATOR_ROOT}/introspector$(getSuffix $i)/${1}" - done - - echo $result -} - -# Returns the suffix for a config map mount address -# args: -# $1 the index of the map (0 means no suffix) -function getSuffix() { - if [ "$1" -eq 0 ]; then - echo "" - else - echo "_$1" - fi -} - function diff_model() { trace "Entering diff_model" From 56b8abc94be48029e7f3d1d7b6b0be18859a2025 Mon Sep 17 00:00:00 2001 From: Russell Gold Date: Fri, 11 Dec 2020 12:32:29 -0500 Subject: [PATCH 6/6] Remove unused functionality --- .../operator/helpers/ConfigMapHelper.java | 7 ----- .../operator/helpers/ConfigMapSplitter.java | 26 ------------------ .../operator/helpers/SplitterTarget.java | 8 ------ .../helpers/ConfigMapSplitterTest.java | 27 ------------------- .../helpers/IntrospectorConfigMapTest.java | 13 --------- 5 files changed, 81 deletions(-) diff --git a/operator/src/main/java/oracle/kubernetes/operator/helpers/ConfigMapHelper.java b/operator/src/main/java/oracle/kubernetes/operator/helpers/ConfigMapHelper.java index c331373cbe3..a3f51639365 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/helpers/ConfigMapHelper.java +++ b/operator/src/main/java/oracle/kubernetes/operator/helpers/ConfigMapHelper.java @@ -662,13 +662,6 @@ public void recordNumTargets(int numTargets) { setContentValue(NUM_CONFIG_MAPS, Integer.toString(numTargets)); } - @Override - public void recordEntryLocation(String key, int firstTarget, int lastTarget) { - if (isEncodedZip(key)) { - setContentValue(createRangeName(key), firstTarget + " " + lastTarget); - } - } - private boolean isEncodedZip(String key) { return ENCODED_ZIP_PATTERN.matcher(key).matches(); } diff --git a/operator/src/main/java/oracle/kubernetes/operator/helpers/ConfigMapSplitter.java b/operator/src/main/java/oracle/kubernetes/operator/helpers/ConfigMapSplitter.java index 11ba655b85b..ef12ee1848c 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/helpers/ConfigMapSplitter.java +++ b/operator/src/main/java/oracle/kubernetes/operator/helpers/ConfigMapSplitter.java @@ -30,7 +30,6 @@ public class ConfigMapSplitter { private final List result = new ArrayList<>(); private Map current; private int remainingRoom; - private final Map locations = new HashMap<>(); /** * Constructs a splitter object. @@ -76,16 +75,12 @@ private void startSplitResult() { * @param entry a data entry */ private void addToSplitResult(DataEntry entry) { - int startIndex = result.size(); while (entry.getRemainingLength() > 0) { remainingRoom -= entry.addToMap(current, remainingRoom); if (remainingRoom == 0) { recordSplitResult(); startSplitResult(); } - if (!result.isEmpty()) { - locations.put(entry.key, createLocation(entry, startIndex)); - } } } @@ -95,14 +90,6 @@ private void recordSplitResult() { private void recordTargetInfo(T target, int size) { target.recordNumTargets(size); - - for (Location location : locations.values()) { - target.recordEntryLocation(location.key, location.first, location.last); - } - } - - private Location createLocation(DataEntry entry, int startIndex) { - return new Location(entry, startIndex, result.size()); } static class DataEntry implements Comparable { @@ -138,17 +125,4 @@ public int compareTo(@Nonnull DataEntry o) { } } - static class Location { - private final String key; - private final int first; - private final int last; - - Location(DataEntry entry, int first, int last) { - this.key = entry.key; - this.first = first; - this.last = last; - } - - } - } diff --git a/operator/src/main/java/oracle/kubernetes/operator/helpers/SplitterTarget.java b/operator/src/main/java/oracle/kubernetes/operator/helpers/SplitterTarget.java index e007b5ce694..f8cc5715707 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/helpers/SplitterTarget.java +++ b/operator/src/main/java/oracle/kubernetes/operator/helpers/SplitterTarget.java @@ -14,12 +14,4 @@ public interface SplitterTarget { * @param numTargets the number of created targets */ void recordNumTargets(int numTargets); - - /** - * Records the location of a entry that was split. - * @param key the key of the split entry - * @param firstTarget the index of first target in which the entry was recorded - * @param lastTarget the index of the last target in which the entry was recorded - */ - void recordEntryLocation(String key, int firstTarget, int lastTarget); } diff --git a/operator/src/test/java/oracle/kubernetes/operator/helpers/ConfigMapSplitterTest.java b/operator/src/test/java/oracle/kubernetes/operator/helpers/ConfigMapSplitterTest.java index a3d32b7e3cf..91828093c7a 100644 --- a/operator/src/test/java/oracle/kubernetes/operator/helpers/ConfigMapSplitterTest.java +++ b/operator/src/test/java/oracle/kubernetes/operator/helpers/ConfigMapSplitterTest.java @@ -14,7 +14,6 @@ import com.meterware.simplestub.Memento; import com.meterware.simplestub.StaticStubSupport; -import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.commons.lang3.tuple.Pair; import org.hamcrest.Description; import org.hamcrest.Matchers; @@ -99,27 +98,6 @@ public void whenDataSplit_recordNumTargetsInFirstResult() { assertThat(result.get(2).numTargets, equalTo(0)); } - @Test - public void whenDataSplit_recordLocationOfSplitEntry() { - data.put("a", "aValue"); - data.put("b", "123456789".repeat(250)); - - final List result = splitter.split(data); - - assertThat(result.get(0).splitEntries.get("b"), equalTo(new ImmutablePair<>(0, 2))); - } - - @Test - public void recordLocationsOfNonSplitItemsPastFirstMap() { - data.put("a", createLargeData(0.5)); - data.put("b", createLargeData(0.6)); - data.put("c", createLargeData(0.7)); - - final List result = splitter.split(data); - - assertThat(result.get(0).splitEntries.get("c"), equalTo(new ImmutablePair<>(1, 1))); - } - private static class TestTarget implements SplitterTarget { private final Map data; private final Map> splitEntries = new HashMap<>(); @@ -136,11 +114,6 @@ public void recordNumTargets(int numTargets) { this.numTargets = numTargets; } - @Override - public void recordEntryLocation(String key, int firstTarget, int lastTarget) { - splitEntries.put(key, new ImmutablePair<>(firstTarget, lastTarget)); - } - private String getB() { return data.get("b"); } diff --git a/operator/src/test/java/oracle/kubernetes/operator/helpers/IntrospectorConfigMapTest.java b/operator/src/test/java/oracle/kubernetes/operator/helpers/IntrospectorConfigMapTest.java index 8c30b8434ca..293df046170 100644 --- a/operator/src/test/java/oracle/kubernetes/operator/helpers/IntrospectorConfigMapTest.java +++ b/operator/src/test/java/oracle/kubernetes/operator/helpers/IntrospectorConfigMapTest.java @@ -391,19 +391,6 @@ private V1ConfigMap createIntrospectorConfigMap(int mapIndex, Map(entries)); } - @Test - public void whenDomainIsModelInImageAndEncodedZipTooLargeForSingleMap_reportRange() { - configureDomain().withDomainHomeSourceType(DomainSourceType.FromModel); - introspectResult - .defineFile(TOPOLOGY_YAML, "domainValid: true", "domain:", " name: \"sample\"") - .defineFile("domainzip.secure", LARGE_DATA_VALUE) - .addToPacket(); - - testSupport.runSteps(ConfigMapHelper.createIntrospectorConfigMapStep(terminalStep)); - - assertThat(getIntrospectorConfigMapValue("domainzip.secure.range"), equalTo("0 2")); - } - @Nonnull private V1ObjectMeta createOperatorMetadata() { return new V1ObjectMeta().putLabelsItem(CREATEDBYOPERATOR_LABEL, "true");