From 0779b2eb24bcc04f02f40afb69350db8c55d38d8 Mon Sep 17 00:00:00 2001 From: amandelpie Date: Thu, 21 Jul 2022 15:08:20 +0300 Subject: [PATCH 1/2] Handled the bad case with the empty paths --- .../summary/clustering/ExecutionMetric.kt | 15 ++++++++++- .../summary/clustering/ExecutionMetricTest.kt | 26 +++++++++++++++++++ 2 files changed, 40 insertions(+), 1 deletion(-) create mode 100644 utbot-summary/src/test/kotlin/org/utbot/summary/clustering/ExecutionMetricTest.kt diff --git a/utbot-summary/src/main/kotlin/org/utbot/summary/clustering/ExecutionMetric.kt b/utbot-summary/src/main/kotlin/org/utbot/summary/clustering/ExecutionMetric.kt index 318368d207..44043d9136 100644 --- a/utbot-summary/src/main/kotlin/org/utbot/summary/clustering/ExecutionMetric.kt +++ b/utbot-summary/src/main/kotlin/org/utbot/summary/clustering/ExecutionMetric.kt @@ -9,6 +9,9 @@ class ExecutionMetric : Metric> { * Minimum Edit Distance */ private fun compareTwoPaths(path1: Iterable, path2: Iterable): Double { + require(path1.count() > 0) { "Two paths can not be compared: path1 is empty!"} + require(path2.count() > 0) { "Two paths can not be compared: path2 is empty!"} + val distances = Array(path1.count()) { i -> Array(path2.count()) { j -> i + j } } for (i in 1 until path1.count()) { @@ -22,7 +25,17 @@ class ExecutionMetric : Metric> { distances[i][j] = minOf(d1, d2, d3) } } - return distances.last().last().toDouble() + + if (distances.isNotEmpty()) { + val last = distances.last() + if (last.isNotEmpty()) { + return last.last().toDouble() + } else { + throw IllegalStateException("Last row in the distance matrix has 0 columns. It should contain more or equal 1 column.") + } + } else { + throw IllegalStateException("Distance matrix has 0 rows. It should contain more or equal 1 row.") + } } private fun distance(stmt1: Step, stmt2: Step): Int { diff --git a/utbot-summary/src/test/kotlin/org/utbot/summary/clustering/ExecutionMetricTest.kt b/utbot-summary/src/test/kotlin/org/utbot/summary/clustering/ExecutionMetricTest.kt new file mode 100644 index 0000000000..853260f360 --- /dev/null +++ b/utbot-summary/src/test/kotlin/org/utbot/summary/clustering/ExecutionMetricTest.kt @@ -0,0 +1,26 @@ +package org.utbot.summary.clustering + +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.Test + +import org.utbot.framework.plugin.api.Step +import java.lang.IllegalArgumentException + +internal class ExecutionMetricTest { + @Test + fun computeWithTwoEmptySteps() { + val executionMetric = ExecutionMetric() + val object1 = listOf() + val object2 = listOf() + + + val exception = Assertions.assertThrows(IllegalArgumentException::class.java) { + executionMetric.compute(object1 = object1, object2 = object2) + } + + Assertions.assertEquals( + "Two paths can not be compared: path1 is empty!", + exception.message + ) + } +} \ No newline at end of file From 6437fea73b5fbc403c6e5dd7317b1174176af032 Mon Sep 17 00:00:00 2001 From: amandelpie Date: Thu, 21 Jul 2022 17:37:31 +0300 Subject: [PATCH 2/2] Improved test messages in the generated meta matchers --- .../examples/SummaryTestCaseGeneratorTest.kt | 186 ++++++++++++++++-- .../src/test/kotlin/math/SummaryOfMathTest.kt | 6 +- 2 files changed, 176 insertions(+), 16 deletions(-) diff --git a/utbot-summary-tests/src/test/kotlin/examples/SummaryTestCaseGeneratorTest.kt b/utbot-summary-tests/src/test/kotlin/examples/SummaryTestCaseGeneratorTest.kt index 7dcc3df548..d6260e17d8 100644 --- a/utbot-summary-tests/src/test/kotlin/examples/SummaryTestCaseGeneratorTest.kt +++ b/utbot-summary-tests/src/test/kotlin/examples/SummaryTestCaseGeneratorTest.kt @@ -25,6 +25,10 @@ import kotlin.reflect.KFunction3 import kotlin.reflect.KFunction4 +private const val NEW_LINE = "\n" +private const val POINT_IN_THE_LIST = " * " +private const val COMMENT_SEPARATOR = "-------------------------------------------------------------" + @Disabled open class SummaryTestCaseGeneratorTest( testClass: KClass<*>, @@ -120,18 +124,51 @@ open class SummaryTestCaseGeneratorTest( return result } - fun List.checkMatchersWithTextSummary( - summaryTextKeys: List, + comments: List, ) { - if (summaryTextKeys.isEmpty()) { + if (comments.isEmpty()) { return } val notMatchedExecutions = this.filter { execution -> - summaryTextKeys.none { summaryKey -> val normalize = execution.summary?.toString()?.normalize() - normalize?.contains(summaryKey.normalize()) == true } + comments.none { comment -> + val normalize = execution.summary?.toString()?.normalize() + normalize?.contains(comment.normalize()) == true + } + } + + val notMatchedComments = comments.filter { comment -> + this.none { execution -> + val normalize = execution.summary?.toString()?.normalize() + normalize?.contains(comment.normalize()) == true + } + } + + Assertions.assertTrue(notMatchedExecutions.isEmpty() && notMatchedComments.isEmpty()) { + buildString { + if (notMatchedExecutions.isNotEmpty()) { + append( + "\nThe following comments were produced by the UTBot, " + + "but were not found in the list of comments passed in the check() method:\n\n${ + commentsFromExecutions( + notMatchedExecutions + ) + }" + ) + } + + if (notMatchedComments.isNotEmpty()) { + append( + "\nThe following comments were passed in the check() method, " + + "but were not found in the list of comments produced by the UTBot:\n\n${ + comments( + notMatchedComments + ) + }" + ) + } + } } - Assertions.assertTrue(notMatchedExecutions.isEmpty()) { "Not matched comments ${summaries(notMatchedExecutions)}" } } fun List.checkMatchersWithMethodNames( @@ -143,7 +180,36 @@ open class SummaryTestCaseGeneratorTest( val notMatchedExecutions = this.filter { execution -> methodNames.none { methodName -> execution.testMethodName?.equals(methodName) == true } } - Assertions.assertTrue(notMatchedExecutions.isEmpty()) { "Not matched test names ${summaries(notMatchedExecutions)}" } + + val notMatchedMethodNames = methodNames.filter { methodName -> + this.none { execution -> execution.testMethodName?.equals(methodName) == true } + } + + Assertions.assertTrue(notMatchedExecutions.isEmpty() && notMatchedMethodNames.isEmpty()) { + buildString { + if (notMatchedExecutions.isNotEmpty()) { + append( + "\nThe following method names were produced by the UTBot, " + + "but were not found in the list of method names passed in the check() method:\n\n${ + methodNamesFromExecutions( + notMatchedExecutions + ) + }" + ) + } + + if (notMatchedMethodNames.isNotEmpty()) { + append( + "\nThe following method names were passed in the check() method, " + + "but were not found in the list of method names produced by the UTBot:\n\n${ + methodNames( + notMatchedMethodNames + ) + }" + ) + } + } + } } fun List.checkMatchersWithDisplayNames( @@ -155,14 +221,108 @@ open class SummaryTestCaseGeneratorTest( val notMatchedExecutions = this.filter { execution -> displayNames.none { displayName -> execution.displayName?.equals(displayName) == true } } - Assertions.assertTrue(notMatchedExecutions.isEmpty()) { "Not matched display names ${summaries(notMatchedExecutions)}" } + + val notMatchedDisplayNames = displayNames.filter { displayName -> + this.none { execution -> execution.displayName?.equals(displayName) == true } + } + + Assertions.assertTrue(notMatchedExecutions.isEmpty() && notMatchedDisplayNames.isEmpty()) { + buildString { + if (notMatchedExecutions.isNotEmpty()) { + append( + "\nThe following display names were produced by the UTBot, " + + "but were not found in the list of display names passed in the check() method:\n\n${ + displayNamesFromExecutions( + notMatchedExecutions + ) + }" + ) + } + + if (notMatchedDisplayNames.isNotEmpty()) { + append( + "\nThe following display names were passed in the check() method, " + + "but were not found in the list of display names produced by the UTBot:\n\n${ + displayNames( + notMatchedDisplayNames + ) + }" + ) + } + } + } } - private fun summaries(executions: List): String { - var result = "" - executions.forEach { - result += it.summary?.joinToString(separator = "", postfix = "\n") + private fun commentsFromExecutions(executions: List): String { + return buildString { + append(COMMENT_SEPARATOR) + executions.forEach { + append(NEW_LINE) + append(NEW_LINE) + append(it.summary?.joinToString(separator = "", postfix = NEW_LINE)) + append(COMMENT_SEPARATOR) + append(NEW_LINE) + } + append(NEW_LINE) + } + } + + private fun comments(comments: List): String { + return buildString { + append(COMMENT_SEPARATOR) + comments.forEach { + append(NEW_LINE) + append(NEW_LINE) + append(it) + append(NEW_LINE) + append(COMMENT_SEPARATOR) + append(NEW_LINE) + } + append(NEW_LINE) + } + } + + private fun displayNamesFromExecutions(executions: List): String { + return buildString { + executions.forEach { + append(POINT_IN_THE_LIST) + append(it.displayName) + append(NEW_LINE) + } + append(NEW_LINE) + } + } + + private fun displayNames(displayNames: List): String { + return buildString { + displayNames.forEach { + append(POINT_IN_THE_LIST) + append(it) + append(NEW_LINE) + } + append(NEW_LINE) + } + } + + private fun methodNamesFromExecutions(executions: List): String { + return buildString { + executions.forEach { + append(POINT_IN_THE_LIST) + append(it.testMethodName) + append(NEW_LINE) + } + append(NEW_LINE) + } + } + + private fun methodNames(methodNames: List): String { + return buildString { + methodNames.forEach { + append(POINT_IN_THE_LIST) + append(it) + append(NEW_LINE) + } + append(NEW_LINE) } - return result } } \ No newline at end of file diff --git a/utbot-summary-tests/src/test/kotlin/math/SummaryOfMathTest.kt b/utbot-summary-tests/src/test/kotlin/math/SummaryOfMathTest.kt index 9200310a58..6f7dcd4d29 100644 --- a/utbot-summary-tests/src/test/kotlin/math/SummaryOfMathTest.kt +++ b/utbot-summary-tests/src/test/kotlin/math/SummaryOfMathTest.kt @@ -143,13 +143,13 @@ class SummaryOfMathTest : SummaryTestCaseGeneratorTest( "Test then returns from: return acummulator.snapshot();\n" val summary6 = "Test calls {@link guava.examples.math.StatsAccumulator#addAll(double[])},\n" + " there it iterates the loop for(double value: values) twice,\n" + - " inside this loop, the test calls StatsAccumulator::add,\n" + + " inside this loop, the test calls {@link guava.examples.math.StatsAccumulator#add(double)},\n" + " there it executes conditions:\n" + " (!isFinite(value)): True\n" + - "Test afterwards calls {@link guava.examples.math.StatsAccumulator#snapshot()},\n" + + "Test then calls {@link guava.examples.math.StatsAccumulator#snapshot()},\n" + " there it returns from: return new Stats(count, mean, sumOfSquaresOfDeltas, min, max);\n" + " \n" + - "Test then returns from: return acummulator.snapshot();\n" + "Test afterwards returns from: return acummulator.snapshot();\n" val summary7 = "Test calls {@link guava.examples.math.StatsAccumulator#addAll(double[])},\n" + " there it iterates the loop for(double value: values) twice,\n" + " inside this loop, the test calls {@link guava.examples.math.StatsAccumulator#add(double)},\n" +