From 0779b2eb24bcc04f02f40afb69350db8c55d38d8 Mon Sep 17 00:00:00 2001 From: amandelpie Date: Thu, 21 Jul 2022 15:08:20 +0300 Subject: [PATCH 1/4] Handled the bad case with the empty paths --- .../summary/clustering/ExecutionMetric.kt | 15 ++++++++++- .../summary/clustering/ExecutionMetricTest.kt | 26 +++++++++++++++++++ 2 files changed, 40 insertions(+), 1 deletion(-) create mode 100644 utbot-summary/src/test/kotlin/org/utbot/summary/clustering/ExecutionMetricTest.kt diff --git a/utbot-summary/src/main/kotlin/org/utbot/summary/clustering/ExecutionMetric.kt b/utbot-summary/src/main/kotlin/org/utbot/summary/clustering/ExecutionMetric.kt index 318368d207..44043d9136 100644 --- a/utbot-summary/src/main/kotlin/org/utbot/summary/clustering/ExecutionMetric.kt +++ b/utbot-summary/src/main/kotlin/org/utbot/summary/clustering/ExecutionMetric.kt @@ -9,6 +9,9 @@ class ExecutionMetric : Metric> { * Minimum Edit Distance */ private fun compareTwoPaths(path1: Iterable, path2: Iterable): Double { + require(path1.count() > 0) { "Two paths can not be compared: path1 is empty!"} + require(path2.count() > 0) { "Two paths can not be compared: path2 is empty!"} + val distances = Array(path1.count()) { i -> Array(path2.count()) { j -> i + j } } for (i in 1 until path1.count()) { @@ -22,7 +25,17 @@ class ExecutionMetric : Metric> { distances[i][j] = minOf(d1, d2, d3) } } - return distances.last().last().toDouble() + + if (distances.isNotEmpty()) { + val last = distances.last() + if (last.isNotEmpty()) { + return last.last().toDouble() + } else { + throw IllegalStateException("Last row in the distance matrix has 0 columns. It should contain more or equal 1 column.") + } + } else { + throw IllegalStateException("Distance matrix has 0 rows. It should contain more or equal 1 row.") + } } private fun distance(stmt1: Step, stmt2: Step): Int { diff --git a/utbot-summary/src/test/kotlin/org/utbot/summary/clustering/ExecutionMetricTest.kt b/utbot-summary/src/test/kotlin/org/utbot/summary/clustering/ExecutionMetricTest.kt new file mode 100644 index 0000000000..853260f360 --- /dev/null +++ b/utbot-summary/src/test/kotlin/org/utbot/summary/clustering/ExecutionMetricTest.kt @@ -0,0 +1,26 @@ +package org.utbot.summary.clustering + +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.Test + +import org.utbot.framework.plugin.api.Step +import java.lang.IllegalArgumentException + +internal class ExecutionMetricTest { + @Test + fun computeWithTwoEmptySteps() { + val executionMetric = ExecutionMetric() + val object1 = listOf() + val object2 = listOf() + + + val exception = Assertions.assertThrows(IllegalArgumentException::class.java) { + executionMetric.compute(object1 = object1, object2 = object2) + } + + Assertions.assertEquals( + "Two paths can not be compared: path1 is empty!", + exception.message + ) + } +} \ No newline at end of file From 7b6b09c5f0b3f4b0960036335ea9b0b72417738d Mon Sep 17 00:00:00 2001 From: amandelpie Date: Mon, 25 Jul 2022 15:11:50 +0300 Subject: [PATCH 2/4] Added the ability to differ tests produced by fuzzing and symbolic engine --- .../org/utbot/framework/plugin/api/Api.kt | 16 ++++- .../org/utbot/engine/UtBotSymbolicEngine.kt | 21 +++--- .../org/utbot/external/api/UtBotJavaApi.kt | 14 ++-- .../kotlin/org/utbot/summary/Summarization.kt | 66 ++++++++++++++----- .../kotlin/org/utbot/summary/TagGenerator.kt | 2 +- 5 files changed, 86 insertions(+), 33 deletions(-) diff --git a/utbot-framework-api/src/main/kotlin/org/utbot/framework/plugin/api/Api.kt b/utbot-framework-api/src/main/kotlin/org/utbot/framework/plugin/api/Api.kt index bfd98844e7..45a7e53068 100644 --- a/utbot-framework-api/src/main/kotlin/org/utbot/framework/plugin/api/Api.kt +++ b/utbot-framework-api/src/main/kotlin/org/utbot/framework/plugin/api/Api.kt @@ -139,6 +139,8 @@ sealed class UtResult * - static fields changed during execution; * - required instrumentation details (such as randoms, time, static methods). * - coverage information (instructions) if this execution was obtained from the concrete execution. + * - the engine type that created this execution. + * - comments, method names and display names created by utbot-summary module. */ data class UtExecution( val stateBefore: EnvironmentModels, @@ -148,6 +150,7 @@ data class UtExecution( val path: MutableList, val fullPath: List, val coverage: Coverage? = null, + val createdBy: UtExecutionCreator? = null, var summary: List? = null, var testMethodName: String? = null, var displayName: String? = null, @@ -1172,7 +1175,14 @@ private fun StringBuilder.appendOptional(name: String, value: Map<*, *>) { } /** - * Entity that represents cluster information that should appear in the comment + * Enum that represents different type of engines that produce tests. + */ +enum class UtExecutionCreator { + FUZZER, SYMBOLIC_ENGINE +} + +/** + * Entity that represents cluster information that should appear in the comment. */ data class UtClusterInfo( val header: String? = null, @@ -1180,13 +1190,13 @@ data class UtClusterInfo( ) /** - * Entity that represents cluster of executions + * Entity that represents cluster of executions. */ data class UtExecutionCluster(val clusterInfo: UtClusterInfo, val executions: List) /** - * Entity that represents various types of statements in comments + * Entity that represents various types of statements in comments. */ sealed class DocStatement diff --git a/utbot-framework/src/main/kotlin/org/utbot/engine/UtBotSymbolicEngine.kt b/utbot-framework/src/main/kotlin/org/utbot/engine/UtBotSymbolicEngine.kt index c5b76a69b7..b7de8dbfef 100644 --- a/utbot-framework/src/main/kotlin/org/utbot/engine/UtBotSymbolicEngine.kt +++ b/utbot-framework/src/main/kotlin/org/utbot/engine/UtBotSymbolicEngine.kt @@ -68,6 +68,7 @@ import org.utbot.framework.plugin.api.UtAssembleModel import org.utbot.framework.plugin.api.UtConcreteExecutionFailure import org.utbot.framework.plugin.api.UtError import org.utbot.framework.plugin.api.UtExecution +import org.utbot.framework.plugin.api.UtExecutionCreator import org.utbot.framework.plugin.api.UtInstrumentation import org.utbot.framework.plugin.api.UtMethod import org.utbot.framework.plugin.api.UtNullModel @@ -302,7 +303,8 @@ class UtBotSymbolicEngine( instrumentation, mutableListOf(), listOf(), - concreteExecutionResult.coverage + concreteExecutionResult.coverage, + UtExecutionCreator.SYMBOLIC_ENGINE ) emit(concreteUtExecution) @@ -485,6 +487,7 @@ class UtBotSymbolicEngine( path = mutableListOf(), fullPath = emptyList(), coverage = concreteExecutionResult.coverage, + createdBy = UtExecutionCreator.FUZZER, testMethodName = testMethodName?.testName, displayName = testMethodName?.takeIf { hasMethodUnderTestParametersToFuzz }?.displayName ) @@ -509,7 +512,8 @@ class UtBotSymbolicEngine( result = UtConcreteExecutionFailure(e), instrumentation = emptyList(), path = mutableListOf(), - fullPath = listOf() + fullPath = listOf(), + createdBy = UtExecutionCreator.SYMBOLIC_ENGINE, ) emit(failedConcreteExecution) @@ -544,12 +548,13 @@ class UtBotSymbolicEngine( require(stateBefore.parameters.size == stateAfter.parameters.size) val symbolicUtExecution = UtExecution( - stateBefore, - stateAfter, - symbolicExecutionResult, - instrumentation, - entryMethodPath(state), - state.fullPath() + stateBefore = stateBefore, + stateAfter = stateAfter, + result = symbolicExecutionResult, + instrumentation = instrumentation, + path = entryMethodPath(state), + fullPath = state.fullPath(), + createdBy = UtExecutionCreator.SYMBOLIC_ENGINE, ) globalGraph.traversed(state) diff --git a/utbot-framework/src/main/kotlin/org/utbot/external/api/UtBotJavaApi.kt b/utbot-framework/src/main/kotlin/org/utbot/external/api/UtBotJavaApi.kt index 6d5b69f26a..ce099096a8 100644 --- a/utbot-framework/src/main/kotlin/org/utbot/external/api/UtBotJavaApi.kt +++ b/utbot-framework/src/main/kotlin/org/utbot/external/api/UtBotJavaApi.kt @@ -18,6 +18,7 @@ import org.utbot.framework.plugin.api.MockFramework import org.utbot.framework.plugin.api.MockStrategyApi import org.utbot.framework.plugin.api.TestCaseGenerator import org.utbot.framework.plugin.api.UtExecution +import org.utbot.framework.plugin.api.UtExecutionCreator import org.utbot.framework.plugin.api.UtMethod import org.utbot.framework.plugin.api.UtPrimitiveModel import org.utbot.framework.plugin.api.UtMethodTestSet @@ -240,12 +241,13 @@ object UtBotJavaApi { } val utExecution = UtExecution( - testInfo.initialState, - testInfo.initialState, // it seems ok for concrete execution - utExecutionResult, - emptyList(), - mutableListOf(), - listOf() + stateBefore = testInfo.initialState, + stateAfter = testInfo.initialState, // it seems ok for concrete execution + result = utExecutionResult, + instrumentation = emptyList(), + path = mutableListOf(), + fullPath = listOf(), + createdBy = UtExecutionCreator.SYMBOLIC_ENGINE ) val utMethod = UtMethod(methodCallable, containingClass.kotlin) diff --git a/utbot-summary/src/main/kotlin/org/utbot/summary/Summarization.kt b/utbot-summary/src/main/kotlin/org/utbot/summary/Summarization.kt index cab135ea21..7f65338ee1 100644 --- a/utbot-summary/src/main/kotlin/org/utbot/summary/Summarization.kt +++ b/utbot-summary/src/main/kotlin/org/utbot/summary/Summarization.kt @@ -5,6 +5,7 @@ import org.utbot.framework.UtSettings import org.utbot.framework.plugin.api.UtClusterInfo import org.utbot.framework.plugin.api.UtExecution import org.utbot.framework.plugin.api.UtExecutionCluster +import org.utbot.framework.plugin.api.UtExecutionCreator import org.utbot.framework.plugin.api.UtMethodTestSet import org.utbot.instrumentation.instrumentation.instrumenter.Instrumenter import org.utbot.summary.SummarySentenceConstants.NEW_LINE @@ -75,29 +76,50 @@ class Summarization(val sourceFile: File?, val invokeDescriptions: List() val clustersToReturn = mutableListOf() - // TODO: Now it excludes tests generated by Fuzzer, handle it properly, related to the https://github.com/UnitTestBot/UTBotJava/issues/428 - val executionsProducedByFuzzer = getExecutionsWithEmptyPath(testSet) + // handles tests produced by fuzzing + val executionsProducedByFuzzer = testSet.executions.filter { it.createdBy == UtExecutionCreator.FUZZER } if (executionsProducedByFuzzer.isNotEmpty()) { executionsProducedByFuzzer.forEach { logger.info { - "The path for test ${it.testMethodName} " + + "Test is created by Fuzzing. The path for test ${it.testMethodName} " + "for method ${testSet.method.clazz.qualifiedName} is empty and summaries could not be generated." } } clustersToReturn.add( UtExecutionCluster( - UtClusterInfo(), + UtClusterInfo(), // TODO: add something https://github.com/UnitTestBot/UTBotJava/issues/430 executionsProducedByFuzzer ) ) } + // handles tests produced by symbolic engine, but with empty paths + val executionsWithEmptyPaths = getExecutionsCreatedBySymbolicEngineWithEmptyPath(testSet) + + if (executionsWithEmptyPaths.isNotEmpty()) { + executionsWithEmptyPaths.forEach { + logger.info { + "Test is created by Symbolic Engine. The path for test ${it.testMethodName} " + + "for method ${testSet.method.clazz.qualifiedName} is empty and summaries could not be generated." + } + } + + clustersToReturn.add( + UtExecutionCluster( + UtClusterInfo(), // TODO: https://github.com/UnitTestBot/UTBotJava/issues/430 + executionsWithEmptyPaths + ) + ) + } + + val testSetForAnalysis = prepareTestSetForByteCodeAnalysis(testSet) + // analyze if (jimpleBody != null && sootToAST != null) { val methodUnderTest = jimpleBody.method - val clusteredTags = tagGenerator.testSetToTags(testSet) + val clusteredTags = tagGenerator.testSetToTags(testSetForAnalysis) jimpleBodyAnalysis.traceStructuralAnalysis(jimpleBody, clusteredTags, methodUnderTest, invokeDescriptions) val numberOfSuccessfulClusters = clusteredTags.filter { it.isSuccessful }.size for (clusterTraceTags in clusteredTags) { @@ -108,21 +130,22 @@ class Summarization(val sourceFile: File?, val invokeDescriptions: List 1 // add if there is more than 1 execution ) { SimpleClusterCommentBuilder(clusterTraceTags.commonStepsTraceTag, sootToAST) - .buildString(methodUnderTest) - .takeIf { it.isNotBlank() } - ?.let { - buildString { - append("${NEW_LINE}Common steps:") - append("$NEW_LINE$it") - } + .buildString(methodUnderTest) + .takeIf { it.isNotBlank() } + ?.let { + buildString { + append("${NEW_LINE}Common steps:") + append("$NEW_LINE$it") } + } } else { null } for (traceTags in clusterTraceTags.traceTags) { if (GENERATE_COMMENTS) { - traceTags.execution.summary = SimpleCommentBuilder(traceTags, sootToAST).buildDocStmts(methodUnderTest) + traceTags.execution.summary = + SimpleCommentBuilder(traceTags, sootToAST).buildDocStmts(methodUnderTest) } if (GENERATE_DISPLAY_NAMES || GENERATE_NAMES) { @@ -164,8 +187,21 @@ class Summarization(val sourceFile: File?, val invokeDescriptions: List, splitSteps: Spl * @return clustered executions */ private fun toClusterExecutions(testSet: UtMethodTestSet): List { - val methodExecutions = testSet.executions.filter { it.path.isNotEmpty() } // TODO: Now it excludes tests generated by Fuzzer, handle it properly, related to the https://github.com/UnitTestBot/UTBotJava/issues/428 + val methodExecutions = testSet.executions.filter { it.path.isNotEmpty() } val clusters = mutableListOf() val commentPostfix = "for method ${testSet.method.displayName}" From fc94bed945d7e427dd1d54ad9ecc078ce6481deb Mon Sep 17 00:00:00 2001 From: amandelpie Date: Mon, 25 Jul 2022 15:13:46 +0300 Subject: [PATCH 3/4] Fixed a bug --- utbot-summary/src/main/kotlin/org/utbot/summary/TagGenerator.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utbot-summary/src/main/kotlin/org/utbot/summary/TagGenerator.kt b/utbot-summary/src/main/kotlin/org/utbot/summary/TagGenerator.kt index 0c231dc86a..b757c70ecf 100644 --- a/utbot-summary/src/main/kotlin/org/utbot/summary/TagGenerator.kt +++ b/utbot-summary/src/main/kotlin/org/utbot/summary/TagGenerator.kt @@ -95,7 +95,7 @@ private fun generateExecutionTags(executions: List, splitSteps: Spl * @return clustered executions */ private fun toClusterExecutions(testSet: UtMethodTestSet): List { - val methodExecutions = testSet.executions.filter { it.path.isNotEmpty() } + val methodExecutions = testSet.executions val clusters = mutableListOf() val commentPostfix = "for method ${testSet.method.displayName}" From babcec830ad424cecc561cd42f09d5024ba3a879 Mon Sep 17 00:00:00 2001 From: amandelpie Date: Tue, 26 Jul 2022 12:11:00 +0300 Subject: [PATCH 4/4] Fixed a bug --- .../src/main/kotlin/org/utbot/external/api/UtBotJavaApi.kt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/utbot-framework/src/main/kotlin/org/utbot/external/api/UtBotJavaApi.kt b/utbot-framework/src/main/kotlin/org/utbot/external/api/UtBotJavaApi.kt index ce099096a8..7df4ff771e 100644 --- a/utbot-framework/src/main/kotlin/org/utbot/external/api/UtBotJavaApi.kt +++ b/utbot-framework/src/main/kotlin/org/utbot/external/api/UtBotJavaApi.kt @@ -246,8 +246,7 @@ object UtBotJavaApi { result = utExecutionResult, instrumentation = emptyList(), path = mutableListOf(), - fullPath = listOf(), - createdBy = UtExecutionCreator.SYMBOLIC_ENGINE + fullPath = listOf() ) val utMethod = UtMethod(methodCallable, containingClass.kotlin)