From 3c39682de8ae56ab3527f44881e4c4d9a4db3cb8 Mon Sep 17 00:00:00 2001 From: Maksim Pelevin Date: Fri, 8 Jul 2022 18:12:58 +0300 Subject: [PATCH 1/2] Minimize UtExecution number produced by fuzzing and collect coverage statistic --- .../org/utbot/engine/UtBotSymbolicEngine.kt | 11 +- .../src/main/kotlin/org/utbot/fuzzer/Trie.kt | 130 ++++++++++++++++++ .../utbot/framework/plugin/api/TrieTest.kt | 114 +++++++++++++++ 3 files changed, 252 insertions(+), 3 deletions(-) create mode 100644 utbot-fuzzers/src/main/kotlin/org/utbot/fuzzer/Trie.kt create mode 100644 utbot-fuzzers/src/test/kotlin/org/utbot/framework/plugin/api/TrieTest.kt diff --git a/utbot-framework/src/main/kotlin/org/utbot/engine/UtBotSymbolicEngine.kt b/utbot-framework/src/main/kotlin/org/utbot/engine/UtBotSymbolicEngine.kt index 19811fb3c5..06d13d5070 100644 --- a/utbot-framework/src/main/kotlin/org/utbot/engine/UtBotSymbolicEngine.kt +++ b/utbot-framework/src/main/kotlin/org/utbot/engine/UtBotSymbolicEngine.kt @@ -81,7 +81,9 @@ import org.utbot.framework.plugin.api.util.utContext import org.utbot.framework.plugin.api.util.description import org.utbot.fuzzer.FallbackModelProvider import org.utbot.fuzzer.FuzzedMethodDescription +import org.utbot.fuzzer.FuzzedValue import org.utbot.fuzzer.ModelProvider +import org.utbot.fuzzer.Trie import org.utbot.fuzzer.collectConstantsForFuzzer import org.utbot.fuzzer.defaultModelProviders import org.utbot.fuzzer.fuzz @@ -408,7 +410,8 @@ class UtBotSymbolicEngine( parameterNameMap = { index -> names?.getOrNull(index) } } val modelProviderWithFallback = modelProvider(defaultModelProviders { nextDefaultModelId++ }).withFallback(fallbackModelProvider::toModel) - val coveredInstructionTracker = mutableSetOf() + val coveredInstructionTracker = Trie(Instruction::id) + val coveredInstructorValues = mutableMapOf, List>() var attempts = UtSettings.fuzzingMaxAttempts fuzz(methodUnderTestDescription, modelProviderWithFallback).forEach { values -> if (System.currentTimeMillis() >= until) { @@ -431,12 +434,14 @@ class UtBotSymbolicEngine( } } - if (!coveredInstructionTracker.addAll(concreteExecutionResult.coverage.coveredInstructions)) { + val count = coveredInstructionTracker.add(concreteExecutionResult.coverage.coveredInstructions) + if (count.count > 1) { if (--attempts < 0) { return@flow } + return@forEach } - + coveredInstructorValues[count] = values val nameSuggester = sequenceOf(ModelBasedNameSuggester(), MethodBasedNameSuggester()) val testMethodName = try { nameSuggester.flatMap { it.suggest(methodUnderTestDescription, values, concreteExecutionResult.result) }.firstOrNull() diff --git a/utbot-fuzzers/src/main/kotlin/org/utbot/fuzzer/Trie.kt b/utbot-fuzzers/src/main/kotlin/org/utbot/fuzzer/Trie.kt new file mode 100644 index 0000000000..b80e79f072 --- /dev/null +++ b/utbot-fuzzers/src/main/kotlin/org/utbot/fuzzer/Trie.kt @@ -0,0 +1,130 @@ +package org.utbot.fuzzer + +fun trieOf(vararg values: Iterable): Trie = IdentityTrie().apply { + values.forEach(this::add) +} + +fun stringTrieOf(vararg values: String): StringTrie = StringTrie().apply { + values.forEach(this::add) +} + +class StringTrie : IdentityTrie() { + fun add(string: String) = super.add(string.toCharArray().asIterable()) + fun remove(string: String) = super.remove(string.toCharArray().asIterable()) + operator fun get(string: String) = super.get(string.toCharArray().asIterable()) + fun collect() = asSequence().map { String(it.toCharArray()) }.toSet() +} + +open class IdentityTrie : Trie({it}) + +open class Trie( + private val keyExtractor: (T) -> K +) : Iterable> { + + private val roots = HashMap>() + private val implementations = HashMap, NodeImpl>() + + fun add(values: Iterable): Node { + val root = try { values.first() } catch (e: NoSuchElementException) { error("Empty list are not allowed") } + var key = keyExtractor(root) + var node = roots.computeIfAbsent(key) { NodeImpl(root, null) } + values.asSequence().drop(1).forEach { value -> + key = keyExtractor(value) + node = node.children.computeIfAbsent(key) { NodeImpl(value, node) } + } + node.count++ + implementations[node] = node + return node + } + + fun remove(values: Iterable): Node? { + val node = findImpl(values) ?: return null + if (node.count > 0 && node.children.isEmpty()) { + var n: NodeImpl? = node + while (n != null) { + val key = keyExtractor(n.data) + n = n.parent + if (n == null) { + val removed = roots.remove(key) + check(removed != null) + } else { + val removed = n.children.remove(key) + check(removed != null) + if (n.count != 0) { + break + } + } + } + } + return if (node.count > 0) { + node.count = 0 + implementations.remove(node) + node + } else { + null + } + } + + operator fun get(values: Iterable): Node? { + return findImpl(values) + } + + operator fun get(node: Node): List? { + return implementations[node]?.let(this::buildValue) + } + + private fun findImpl(values: Iterable): NodeImpl? { + val root = try { values.first() } catch (e: NoSuchElementException) { return null } + var key = keyExtractor(root) + var node = roots[key] ?: return null + values.asSequence().drop(1).forEach { value -> + key = keyExtractor(value) + node = node.children[key] ?: return null + } + return node.takeIf { it.count > 0 } + } + + override fun iterator(): Iterator> { + return iterator { + roots.values.forEach { node -> + traverseImpl(node) + } + } + } + + private suspend fun SequenceScope>.traverseImpl(node: NodeImpl) { + val stack = ArrayDeque>() + stack.addLast(node) + while (stack.isNotEmpty()) { + val n = stack.removeLast() + if (n.count > 0) { + yield(buildValue(n)) + } + n.children.values.forEach(stack::addLast) + } + } + + private fun buildValue(node: NodeImpl): List { + return generateSequence(node) { it.parent }.map { it.data }.toList().asReversed() + } + + interface Node{ + val data: T + val count: Int + } + + /** + * Trie node + * + * @param data data to be stored + * @param parent reference to the previous element of the value + * @param count number of value insertions + * @param children list of children mapped by their key + */ + private class NodeImpl( + override val data: T, + val parent: NodeImpl?, + override var count: Int = 0, + val children: MutableMap> = HashMap(), + ) : Node +} \ No newline at end of file diff --git a/utbot-fuzzers/src/test/kotlin/org/utbot/framework/plugin/api/TrieTest.kt b/utbot-fuzzers/src/test/kotlin/org/utbot/framework/plugin/api/TrieTest.kt new file mode 100644 index 0000000000..6ae2b4df50 --- /dev/null +++ b/utbot-fuzzers/src/test/kotlin/org/utbot/framework/plugin/api/TrieTest.kt @@ -0,0 +1,114 @@ +package org.utbot.framework.plugin.api + +import org.junit.jupiter.api.Assertions.* +import org.junit.jupiter.api.Test +import org.utbot.fuzzer.Trie +import org.utbot.fuzzer.stringTrieOf +import org.utbot.fuzzer.trieOf + +class TrieTest { + + @Test + fun simpleTest() { + val trie = stringTrieOf() + assertThrows(java.lang.IllegalStateException::class.java) { + trie.add(emptyList()) + } + assertEquals(1, trie.add("Tree").count) + assertEquals(2, trie.add("Tree").count) + assertEquals(1, trie.add("Trees").count) + assertEquals(1, trie.add("Treespss").count) + assertEquals(1, trie.add("Game").count) + assertEquals(1, trie.add("Gamer").count) + assertEquals(1, trie.add("Games").count) + assertEquals(2, trie["Tree"]?.count) + assertEquals(1, trie["Trees"]?.count) + assertEquals(1, trie["Gamer"]?.count) + assertNull(trie["Treesp"]) + assertNull(trie["Treessss"]) + + assertEquals(setOf("Tree", "Trees", "Treespss", "Game", "Gamer", "Games"), trie.collect()) + } + + @Test + fun testSingleElement() { + val trie = trieOf(listOf(1)) + assertEquals(1, trie.toList().size) + } + + @Test + fun testRemoval() { + val trie = stringTrieOf() + trie.add("abc") + assertEquals(1, trie.toList().size) + trie.add("abcd") + assertEquals(2, trie.toList().size) + trie.add("abcd") + assertEquals(2, trie.toList().size) + trie.add("abcde") + assertEquals(3, trie.toList().size) + + assertNotNull(trie.remove("abcd")) + assertEquals(2, trie.toList().size) + + assertNull(trie.remove("ffff")) + assertEquals(2, trie.toList().size) + + assertNotNull(trie.remove("abcde")) + assertEquals(1, trie.toList().size) + + assertNotNull(trie.remove("abc")) + assertEquals(0, trie.toList().size) + } + + @Test + fun testTraverse() { + val trie = Trie(Data::id).apply { + add((1..10).map { Data(it.toLong(), it) }) + add((1..10).mapIndexed { index, it -> if (index == 5) Data(3L, it) else Data(it.toLong(), it) }) + } + + val paths = trie.toList() + assertEquals(2, paths.size) + assertNotEquals(paths[0], paths[1]) + } + + @Test + fun testNoDuplications() { + val trie = trieOf( + (1..10), + (1..10), + (1..10), + (1..10), + (1..10), + ) + + assertEquals(1, trie.toList().size) + assertEquals(5, trie[(1..10)]!!.count) + } + + @Test + fun testAcceptsNulls() { + val trie = trieOf( + listOf(null), + listOf(null, null), + listOf(null, null, null), + ) + + assertEquals(3, trie.toList().size) + for (i in 1 .. 3) { + assertEquals(1, trie[(1..i).map { null }]!!.count) + } + } + + @Test + fun testAddPrefixAfterWord() { + val trie = stringTrieOf() + trie.add("Hello, world!") + trie.add("Hello") + + assertEquals(setOf("Hello, world!", "Hello"), trie.collect()) + } + + data class Data(val id: Long, val number: Int) +} \ No newline at end of file From efacd494c9d664d0b89e0cec2b3d128ec67927d9 Mon Sep 17 00:00:00 2001 From: Maksim Pelevin Date: Mon, 11 Jul 2022 11:59:00 +0300 Subject: [PATCH 2/2] Improve documentation and add removeCompletely method --- .../org/utbot/engine/UtBotSymbolicEngine.kt | 4 +- .../src/main/kotlin/org/utbot/fuzzer/Trie.kt | 46 ++++++++++++++++++- .../utbot/framework/plugin/api/TrieTest.kt | 25 ++++++++-- 3 files changed, 67 insertions(+), 8 deletions(-) diff --git a/utbot-framework/src/main/kotlin/org/utbot/engine/UtBotSymbolicEngine.kt b/utbot-framework/src/main/kotlin/org/utbot/engine/UtBotSymbolicEngine.kt index 06d13d5070..4313ddd5fd 100644 --- a/utbot-framework/src/main/kotlin/org/utbot/engine/UtBotSymbolicEngine.kt +++ b/utbot-framework/src/main/kotlin/org/utbot/engine/UtBotSymbolicEngine.kt @@ -411,7 +411,7 @@ class UtBotSymbolicEngine( } val modelProviderWithFallback = modelProvider(defaultModelProviders { nextDefaultModelId++ }).withFallback(fallbackModelProvider::toModel) val coveredInstructionTracker = Trie(Instruction::id) - val coveredInstructorValues = mutableMapOf, List>() + val coveredInstructionValues = mutableMapOf, List>() var attempts = UtSettings.fuzzingMaxAttempts fuzz(methodUnderTestDescription, modelProviderWithFallback).forEach { values -> if (System.currentTimeMillis() >= until) { @@ -441,7 +441,7 @@ class UtBotSymbolicEngine( } return@forEach } - coveredInstructorValues[count] = values + coveredInstructionValues[count] = values val nameSuggester = sequenceOf(ModelBasedNameSuggester(), MethodBasedNameSuggester()) val testMethodName = try { nameSuggester.flatMap { it.suggest(methodUnderTestDescription, values, concreteExecutionResult.result) }.firstOrNull() diff --git a/utbot-fuzzers/src/main/kotlin/org/utbot/fuzzer/Trie.kt b/utbot-fuzzers/src/main/kotlin/org/utbot/fuzzer/Trie.kt index b80e79f072..fb3c8d6345 100644 --- a/utbot-fuzzers/src/main/kotlin/org/utbot/fuzzer/Trie.kt +++ b/utbot-fuzzers/src/main/kotlin/org/utbot/fuzzer/Trie.kt @@ -10,13 +10,17 @@ fun stringTrieOf(vararg values: String): StringTrie = StringTrie().apply { class StringTrie : IdentityTrie() { fun add(string: String) = super.add(string.toCharArray().asIterable()) + fun removeCompletely(string: String) = super.removeCompletely(string.toCharArray().asIterable()) fun remove(string: String) = super.remove(string.toCharArray().asIterable()) operator fun get(string: String) = super.get(string.toCharArray().asIterable()) fun collect() = asSequence().map { String(it.toCharArray()) }.toSet() } -open class IdentityTrie : Trie({it}) +open class IdentityTrie : Trie({ it }) +/** + * Implementation of a trie for any iterable values. + */ open class Trie( private val keyExtractor: (T) -> K ) : Iterable> { @@ -24,6 +28,14 @@ open class Trie( private val roots = HashMap>() private val implementations = HashMap, NodeImpl>() + /** + * Adds value into a trie. + * + * If value already exists then do nothing except increasing internal counter of added values. + * The counter can be returned by [Node.count]. + * + * @return corresponding [Node] of the last element in the `values` + */ fun add(values: Iterable): Node { val root = try { values.first() } catch (e: NoSuchElementException) { error("Empty list are not allowed") } var key = keyExtractor(root) @@ -37,7 +49,37 @@ open class Trie( return node } + /** + * Decreases node counter value or removes the value completely if `counter == 1`. + * + * Use [removeCompletely] to remove the value from the trie regardless of counter value. + * + * @return removed node if value exists. + */ fun remove(values: Iterable): Node? { + val node = findImpl(values) ?: return null + return when { + node.count == 1 -> removeCompletely(values) + node.count > 1 -> node.apply { count-- } + else -> throw IllegalStateException("count should be 1 or greater") + } + } + + /** + * Removes value from a trie. + * + * The value is removed completely from the trie. Thus, the next code is true: + * + * ``` + * trie.remove(someValue) + * trie.get(someValue) == null + * ``` + * + * Use [remove] to decrease counter value instead of removal. + * + * @return removed node if value exists + */ + fun removeCompletely(values: Iterable): Node? { val node = findImpl(values) ?: return null if (node.count > 0 && node.children.isEmpty()) { var n: NodeImpl? = node @@ -108,7 +150,7 @@ open class Trie( return generateSequence(node) { it.parent }.map { it.data }.toList().asReversed() } - interface Node{ + interface Node { val data: T val count: Int } diff --git a/utbot-fuzzers/src/test/kotlin/org/utbot/framework/plugin/api/TrieTest.kt b/utbot-fuzzers/src/test/kotlin/org/utbot/framework/plugin/api/TrieTest.kt index 6ae2b4df50..a5cd8f62ab 100644 --- a/utbot-fuzzers/src/test/kotlin/org/utbot/framework/plugin/api/TrieTest.kt +++ b/utbot-fuzzers/src/test/kotlin/org/utbot/framework/plugin/api/TrieTest.kt @@ -48,19 +48,36 @@ class TrieTest { trie.add("abcde") assertEquals(3, trie.toList().size) - assertNotNull(trie.remove("abcd")) + assertNotNull(trie.removeCompletely("abcd")) assertEquals(2, trie.toList().size) - assertNull(trie.remove("ffff")) + assertNull(trie.removeCompletely("ffff")) assertEquals(2, trie.toList().size) - assertNotNull(trie.remove("abcde")) + assertNotNull(trie.removeCompletely("abcde")) assertEquals(1, trie.toList().size) - assertNotNull(trie.remove("abc")) + assertNotNull(trie.removeCompletely("abc")) assertEquals(0, trie.toList().size) } + @Test + fun testSearchingAfterDeletion() { + val trie = stringTrieOf("abc", "abc", "abcde") + assertEquals(2, trie.toList().size) + assertEquals(2, trie["abc"]?.count) + + val removed1 = trie.remove("abc") + assertNotNull(removed1) + + val find = trie["abc"] + assertNotNull(find) + assertEquals(1, find!!.count) + + val removed2 = trie.remove("abc") + assertNotNull(removed2) + } + @Test fun testTraverse() { val trie = Trie(Data::id).apply {