diff --git a/utbot-fuzzers/src/main/kotlin/org/utbot/fuzzer/CartesianProduct.kt b/utbot-fuzzers/src/main/kotlin/org/utbot/fuzzer/CartesianProduct.kt index ec26297d01..0dd190f1d1 100644 --- a/utbot-fuzzers/src/main/kotlin/org/utbot/fuzzer/CartesianProduct.kt +++ b/utbot-fuzzers/src/main/kotlin/org/utbot/fuzzer/CartesianProduct.kt @@ -1,5 +1,6 @@ package org.utbot.fuzzer +import kotlin.jvm.Throws import kotlin.random.Random /** @@ -10,18 +11,59 @@ class CartesianProduct( private val random: Random? = null ): Iterable> { - fun asSequence(): Sequence> = iterator().asSequence() + /** + * Estimated number of all combinations. + */ + val estimatedSize: Long + get() = Combinations(*lists.map { it.size }.toIntArray()).size - override fun iterator(): Iterator> { + @Throws(TooManyCombinationsException::class) + fun asSequence(): Sequence> { val combinations = Combinations(*lists.map { it.size }.toIntArray()) val sequence = if (random != null) { - val permutation = PseudoShuffledIntProgression(combinations.size, random) - (0 until combinations.size).asSequence().map { combinations[permutation[it]] } + sequence { + forEachChunk(Int.MAX_VALUE, combinations.size) { startIndex, combinationSize, _ -> + val permutation = PseudoShuffledIntProgression(combinationSize, random) + val temp = IntArray(size = lists.size) + for (it in 0 until combinationSize) { + yield(combinations[permutation[it] + startIndex, temp]) + } + } + } } else { combinations.asSequence() } return sequence.map { combination -> - combination.mapIndexedTo(mutableListOf()) { element, value -> lists[element][value] } - }.iterator() + combination.mapIndexedTo(ArrayList(combination.size)) { index, value -> lists[index][value] } + } + } + + override fun iterator(): Iterator> = asSequence().iterator() + + companion object { + /** + * Consumer for processing blocks of input larger block. + * + * If source example is sized to 12 and every block is sized to 5 then consumer should be called 3 times with these values: + * + * 1. start = 0, size = 5, remain = 7 + * 2. start = 5, size = 5, remain = 2 + * 3. start = 10, size = 2, remain = 0 + * + * The sum of start, size and remain should be equal to source block size. + */ + internal inline fun forEachChunk( + chunkSize: Int, + totalSize: Long, + block: (start: Long, size: Int, remain: Long) -> Unit + ) { + val iterationsCount = totalSize / chunkSize + if (totalSize % chunkSize == 0L) 0 else 1 + (0L until iterationsCount).forEach { iteration -> + val start = iteration * chunkSize + val size = minOf(chunkSize.toLong(), totalSize - start).toInt() + val remain = totalSize - size - start + block(start, size, remain) + } + } } } \ No newline at end of file diff --git a/utbot-fuzzers/src/main/kotlin/org/utbot/fuzzer/Combinations.kt b/utbot-fuzzers/src/main/kotlin/org/utbot/fuzzer/Combinations.kt index 35134d6785..c724022ae6 100644 --- a/utbot-fuzzers/src/main/kotlin/org/utbot/fuzzer/Combinations.kt +++ b/utbot-fuzzers/src/main/kotlin/org/utbot/fuzzer/Combinations.kt @@ -63,8 +63,8 @@ class Combinations(vararg elementNumbers: Int): Iterable { * * The total count of all possible combinations is therefore `count[0]`. */ - private val count: IntArray - val size: Int + private val count: LongArray + val size: Long get() = if (count.isEmpty()) 0 else count[0] init { @@ -72,9 +72,13 @@ class Combinations(vararg elementNumbers: Int): Iterable { if (badValue >= 0) { throw IllegalArgumentException("Max value must be at least 1 to build combinations, but ${elementNumbers[badValue]} is found at position $badValue (list: $elementNumbers)") } - count = IntArray(elementNumbers.size) { elementNumbers[it] } + count = LongArray(elementNumbers.size) { elementNumbers[it].toLong() } for (i in count.size - 2 downTo 0) { - count[i] = count[i] * count[i + 1] + try { + count[i] = StrictMath.multiplyExact(count[i], count[i + 1]) + } catch (e: ArithmeticException) { + throw TooManyCombinationsException("Long overflow: ${count[i]} * ${count[i + 1]}") + } } } @@ -94,7 +98,7 @@ class Combinations(vararg elementNumbers: Int): Iterable { * } * ``` */ - operator fun get(value: Int, target: IntArray = IntArray(count.size)): IntArray { + operator fun get(value: Long, target: IntArray = IntArray(count.size)): IntArray { if (value >= size) { throw java.lang.IllegalArgumentException("Only $size values allowed") } @@ -104,13 +108,20 @@ class Combinations(vararg elementNumbers: Int): Iterable { var rem = value for (i in target.indices) { target[i] = if (i < target.size - 1) { - val res = rem / count[i + 1] + val res = checkBoundsAndCast(rem / count[i + 1]) rem %= count[i + 1] res } else { - rem + checkBoundsAndCast(rem) } } return target } -} \ No newline at end of file + + private fun checkBoundsAndCast(value: Long): Int { + check(value >= 0 && value < Int.MAX_VALUE) { "Value is out of bounds: $value" } + return value.toInt() + } +} + +class TooManyCombinationsException(msg: String) : RuntimeException(msg) \ No newline at end of file diff --git a/utbot-fuzzers/src/main/kotlin/org/utbot/fuzzer/Fuzzer.kt b/utbot-fuzzers/src/main/kotlin/org/utbot/fuzzer/Fuzzer.kt index 52775f12ec..33f0179095 100644 --- a/utbot-fuzzers/src/main/kotlin/org/utbot/fuzzer/Fuzzer.kt +++ b/utbot-fuzzers/src/main/kotlin/org/utbot/fuzzer/Fuzzer.kt @@ -11,13 +11,18 @@ import org.utbot.fuzzer.providers.CollectionModelProvider import org.utbot.fuzzer.providers.PrimitiveDefaultsModelProvider import org.utbot.fuzzer.providers.EnumModelProvider import org.utbot.fuzzer.providers.PrimitiveWrapperModelProvider +import java.lang.IllegalArgumentException import java.util.concurrent.atomic.AtomicInteger import java.util.function.IntSupplier import kotlin.random.Random -private val logger = KotlinLogging.logger {} +private val logger by lazy { KotlinLogging.logger {} } fun fuzz(description: FuzzedMethodDescription, vararg modelProviders: ModelProvider): Sequence> { + if (modelProviders.isEmpty()) { + throw IllegalArgumentException("At least one model provider is required") + } + val values = List>(description.parameters.size) { mutableListOf() } modelProviders.forEach { fuzzingProvider -> fuzzingProvider.generate(description).forEach { (index, model) -> diff --git a/utbot-fuzzers/src/main/kotlin/org/utbot/fuzzer/providers/ObjectModelProvider.kt b/utbot-fuzzers/src/main/kotlin/org/utbot/fuzzer/providers/ObjectModelProvider.kt index 332dda6117..7dadefe0cb 100644 --- a/utbot-fuzzers/src/main/kotlin/org/utbot/fuzzer/providers/ObjectModelProvider.kt +++ b/utbot-fuzzers/src/main/kotlin/org/utbot/fuzzer/providers/ObjectModelProvider.kt @@ -1,5 +1,6 @@ package org.utbot.fuzzer.providers +import mu.KotlinLogging import org.utbot.framework.plugin.api.ClassId import org.utbot.framework.plugin.api.ConstructorId import org.utbot.framework.plugin.api.FieldId @@ -20,6 +21,7 @@ import org.utbot.fuzzer.FuzzedParameter import org.utbot.fuzzer.FuzzedValue import org.utbot.fuzzer.ModelProvider import org.utbot.fuzzer.ModelProvider.Companion.yieldValue +import org.utbot.fuzzer.TooManyCombinationsException import org.utbot.fuzzer.exceptIsInstance import org.utbot.fuzzer.fuzz import org.utbot.fuzzer.objectModelProviders @@ -31,6 +33,8 @@ import java.lang.reflect.Method import java.lang.reflect.Modifier.* import java.util.function.IntSupplier +private val logger by lazy { KotlinLogging.logger {} } + /** * Creates [UtAssembleModel] for objects which have public constructors with primitives types and String as parameters. */ @@ -170,7 +174,12 @@ class ObjectModelProvider : ModelProvider { ).apply { this.packageName = this@fuzzParameters.packageName } - return fuzz(fuzzedMethod, *modelProviders) + return try { + fuzz(fuzzedMethod, *modelProviders) + } catch (t: TooManyCombinationsException) { + logger.warn(t) { "Number of combination of ${parameters.size} parameters is huge. Fuzzing is skipped for $name" } + emptySequence() + } } private fun assembleModel(id: Int, constructorId: ConstructorId, params: List): FuzzedValue { diff --git a/utbot-fuzzers/src/test/kotlin/org/utbot/framework/plugin/api/CombinationsTest.kt b/utbot-fuzzers/src/test/kotlin/org/utbot/framework/plugin/api/CombinationsTest.kt index 87e8a502a0..bddde87d50 100644 --- a/utbot-fuzzers/src/test/kotlin/org/utbot/framework/plugin/api/CombinationsTest.kt +++ b/utbot-fuzzers/src/test/kotlin/org/utbot/framework/plugin/api/CombinationsTest.kt @@ -4,9 +4,17 @@ import org.utbot.fuzzer.CartesianProduct import org.utbot.fuzzer.Combinations import org.junit.jupiter.api.Assertions.assertArrayEquals import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertFalse import org.junit.jupiter.api.Assertions.assertThrows +import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.Test +import org.junit.jupiter.api.assertDoesNotThrow +import org.junit.jupiter.params.ParameterizedTest +import org.junit.jupiter.params.provider.ValueSource +import org.utbot.fuzzer.TooManyCombinationsException +import java.util.BitSet import kotlin.math.pow +import kotlin.random.Random class CombinationsTest { @@ -55,11 +63,11 @@ class CombinationsTest { val array = intArrayOf(10, 10, 10) val combinations = Combinations(*array) combinations.forEachIndexed { i, c -> - var actual = 0 + var actual = 0L for (pos in array.indices) { actual += c[pos] * (10.0.pow(array.size - 1.0 - pos).toInt()) } - assertEquals(i, actual) + assertEquals(i.toLong(), actual) } } @@ -105,4 +113,161 @@ class CombinationsTest { } } + @ParameterizedTest(name = "testAllLongValues{arguments}") + @ValueSource(ints = [1, 100, Int.MAX_VALUE]) + fun testAllLongValues(value: Int) { + val combinations = Combinations(value, value, 2) + assertEquals(2L * value * value, combinations.size) + val array = combinations[combinations.size - 1] + assertEquals(value - 1, array[0]) + assertEquals(value - 1, array[1]) + assertEquals(1, array[2]) + } + + @Test + fun testCartesianFindsAllValues() { + val radix = 4 + val product = createIntCartesianProduct(radix, 10) + val total = product.estimatedSize + assertTrue(total < Int.MAX_VALUE) { "This test should generate less than Int.MAX_VALUE values but has $total" } + + val set = BitSet((total / 64).toInt()) + val updateSet: (List) -> Unit = { + val value = it.joinToString("").toLong(radix).toInt() + assertFalse(set[value]) + set.set(value) + } + val realCount = product.onEach(updateSet).count() + assertEquals(total, realCount.toLong()) + + for (i in 0 until total) { + assertTrue(set[i.toInt()]) { "Values is not listed for index = $i" } + } + for (i in total until set.size()) { + assertFalse(set[i.toInt()]) + } + } + + /** + * Creates all numbers from 0 to `radix^repeat`. + * + * For example: + * + * radix = 2, repeat = 2 -> {'0', '0'}, {'0', '1'}, {'1', '0'}, {'1', '1'} + * radix = 16, repeat = 1 -> {'0'}, {'1'}, {'2'}, {'3'}, {'4'}, {'5'}, {'6'}, {'7'}, {'8'}, {'9'}, {'a'}, {'b'}, {'c'}, {'d'}, {'e'}, {'f'} + */ + private fun createIntCartesianProduct(radix: Int, repeat: Int) = + CartesianProduct( + lists = (1..repeat).map { + Array(radix) { it.toString(radix) }.toList() + }, + random = Random(0) + ).apply { + assertEquals((1L..repeat).fold(1L) { acc, _ -> acc * radix }, estimatedSize) + } + + @Test + fun testCanCreateCartesianProductWithSizeGreaterThanMaxInt() { + val product = createIntCartesianProduct(5, 15) + assertTrue(product.estimatedSize > Int.MAX_VALUE) { "This test should generate more than Int.MAX_VALUE values but has ${product.estimatedSize}" } + assertDoesNotThrow { + product.first() + } + } + + @Test + fun testIterationWithChunksIsCorrect() { + val expected = mutableListOf( + Triple(0L, 5, 7L), + Triple(5L, 5, 2L), + Triple(10L, 2, 0L), + ) + CartesianProduct.forEachChunk(5, 12) { start, chunk, remain -> + assertEquals(expected.removeFirst(), Triple(start, chunk, remain)) + } + assertTrue(expected.isEmpty()) + } + + @Test + fun testIterationWithChunksIsCorrectWhenChunkIsIntMax() { + val total = 12 + val expected = mutableListOf( + Triple(0L, total, 0L) + ) + CartesianProduct.forEachChunk(Int.MAX_VALUE, total.toLong()) { start, chunk, remain -> + assertEquals(expected.removeFirst(), Triple(start, chunk, remain)) + } + assertTrue(expected.isEmpty()) + } + + @ParameterizedTest(name = "testIterationWithChunksIsCorrectWhenChunkIs{arguments}") + @ValueSource(ints = [1, 2, 3, 4, 6, 12]) + fun testIterationWithChunksIsCorrectWhenChunk(chunkSize: Int) { + val total = 12 + assertTrue(total % chunkSize == 0) { "Test requires values that are dividers of the total = $total, but it is not true for $chunkSize" } + val expected = (0 until total step chunkSize).map { it.toLong() }.map { + Triple(it, chunkSize, total - it - chunkSize) + }.toMutableList() + CartesianProduct.forEachChunk(chunkSize, total.toLong()) { start, chunk, remain -> + assertEquals(expected.removeFirst(), Triple(start, chunk, remain)) + } + assertTrue(expected.isEmpty()) + } + + @ParameterizedTest(name = "testIterationsWithChunksThroughLongWithRemainingIs{arguments}") + @ValueSource(longs = [1L, 200L, 307, Int.MAX_VALUE - 1L, Int.MAX_VALUE.toLong()]) + fun testIterationsWithChunksThroughLongTotal(remaining: Long) { + val expected = mutableListOf( + Triple(0L, Int.MAX_VALUE, Int.MAX_VALUE + remaining), + Triple(Int.MAX_VALUE.toLong(), Int.MAX_VALUE, remaining), + Triple(Int.MAX_VALUE * 2L, remaining.toInt(), 0L), + ) + CartesianProduct.forEachChunk(Int.MAX_VALUE, Int.MAX_VALUE * 2L + remaining) { start, chunk, remain -> + assertEquals(expected.removeFirst(), Triple(start, chunk, remain)) + } + assertTrue(expected.isEmpty()) + } + + @Test + fun testCartesianProductDoesNotThrowsExceptionBeforeOverflow() { + // We assume that a standard method has no more than 7 parameters. + // In this case every parameter can accept up to 511 values without Long overflow. + // CartesianProduct throws exception + val values = Array(511) { it }.toList() + val parameters = Array(7) { values }.toList() + assertDoesNotThrow { + CartesianProduct(parameters, Random(0)).asSequence() + } + } + + @Test + fun testCartesianProductThrowsExceptionOnOverflow() { + // We assume that a standard method has no more than 7 parameters. + // In this case every parameter can accept up to 511 values without Long overflow. + // CartesianProduct throws exception + val values = Array(512) { it }.toList() + val parameters = Array(7) { values }.toList() + assertThrows(TooManyCombinationsException::class.java) { + CartesianProduct(parameters, Random(0)).asSequence() + } + } + + @ParameterizedTest(name = "testCombinationHasValue{arguments}") + @ValueSource(ints = [1, Int.MAX_VALUE]) + fun testCombinationHasValue(value: Int) { + val combinations = Combinations(value) + assertEquals(value.toLong(), combinations.size) + assertEquals(value - 1, combinations[value - 1L][0]) + } + + @Test + fun testNoFailWhenMixedValues() { + val combinations = Combinations(2, Int.MAX_VALUE) + assertEquals(2 * Int.MAX_VALUE.toLong(), combinations.size) + assertArrayEquals(intArrayOf(0, 0), combinations[0L]) + assertArrayEquals(intArrayOf(0, Int.MAX_VALUE - 1), combinations[Int.MAX_VALUE - 1L]) + assertArrayEquals(intArrayOf(1, 0), combinations[Int.MAX_VALUE.toLong()]) + assertArrayEquals(intArrayOf(1, 1), combinations[Int.MAX_VALUE + 1L]) + assertArrayEquals(intArrayOf(1, Int.MAX_VALUE - 1), combinations[Int.MAX_VALUE * 2L - 1]) + } } \ No newline at end of file diff --git a/utbot-fuzzers/src/test/kotlin/org/utbot/framework/plugin/api/FuzzerTest.kt b/utbot-fuzzers/src/test/kotlin/org/utbot/framework/plugin/api/FuzzerTest.kt new file mode 100644 index 0000000000..da4e5135c0 --- /dev/null +++ b/utbot-fuzzers/src/test/kotlin/org/utbot/framework/plugin/api/FuzzerTest.kt @@ -0,0 +1,160 @@ +package org.utbot.framework.plugin.api + +import org.junit.jupiter.api.Assertions.* +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.Timeout +import org.junit.jupiter.api.assertThrows +import org.utbot.framework.plugin.api.util.booleanClassId +import org.utbot.framework.plugin.api.util.byteClassId +import org.utbot.framework.plugin.api.util.charClassId +import org.utbot.framework.plugin.api.util.doubleClassId +import org.utbot.framework.plugin.api.util.floatClassId +import org.utbot.framework.plugin.api.util.id +import org.utbot.framework.plugin.api.util.intClassId +import org.utbot.framework.plugin.api.util.longClassId +import org.utbot.framework.plugin.api.util.shortClassId +import org.utbot.framework.plugin.api.util.stringClassId +import org.utbot.framework.plugin.api.util.voidClassId +import org.utbot.fuzzer.FuzzedConcreteValue +import org.utbot.fuzzer.FuzzedMethodDescription +import org.utbot.fuzzer.FuzzedParameter +import org.utbot.fuzzer.ModelProvider +import org.utbot.fuzzer.fuzz +import org.utbot.fuzzer.providers.ConstantsModelProvider +import org.utbot.fuzzer.providers.NullModelProvider +import org.utbot.fuzzer.providers.PrimitiveDefaultsModelProvider +import org.utbot.fuzzer.providers.PrimitiveWrapperModelProvider.fuzzed +import java.lang.IllegalArgumentException +import java.util.concurrent.TimeUnit + +class FuzzerTest { + + private val testProvider = ModelProvider.of(PrimitiveDefaultsModelProvider, NullModelProvider) + + @Test + fun `error when no provider is passed`() { + assertThrows { + fuzz(newDescription(emptyList())) + } + } + + @Test + fun `zero values for empty input`() { + val fuzz = fuzz(newDescription(emptyList()), testProvider) + assertNull(fuzz.firstOrNull()) + } + + @Test + fun `single value for every type`() { + val fuzz = fuzz( + newDescription( + defaultTypes() + ), + testProvider + ) + assertEquals(1, fuzz.count()) { "Default provider should create 1 value for every type, but have ${fuzz.count()}" } + assertEquals(listOf( + UtPrimitiveModel(false), + UtPrimitiveModel(0.toByte()), + UtPrimitiveModel('\u0000'), + UtPrimitiveModel(0.toShort()), + UtPrimitiveModel(0), + UtPrimitiveModel(0L), + UtPrimitiveModel(0.0f), + UtPrimitiveModel(0.0), + UtNullModel(Any::class.java.id) + ), fuzz.first().map { it.model }) + } + + @Test + fun `concrete values are created`() { + val concreteValues = listOf( + FuzzedConcreteValue(intClassId, 1), + FuzzedConcreteValue(intClassId, 2), + FuzzedConcreteValue(intClassId, 3), + ) + val fuzz = fuzz(newDescription(listOf(intClassId), concreteValues), ConstantsModelProvider) + assertEquals(concreteValues.size, fuzz.count()) + assertEquals(setOf( + UtPrimitiveModel(1), + UtPrimitiveModel(2), + UtPrimitiveModel(3), + ), fuzz.map { it.first().model }.toSet()) + } + + @Test + fun `concrete values are created but filtered`() { + val concreteValues = listOf( + FuzzedConcreteValue(intClassId, 1), + FuzzedConcreteValue(intClassId, 2), + FuzzedConcreteValue(intClassId, 3), + ) + val fuzz = fuzz(newDescription(listOf(charClassId), concreteValues), ConstantsModelProvider) + assertEquals(0, fuzz.count()) + } + + @Test + fun `all combinations is found`() { + val fuzz = fuzz(newDescription(listOf(booleanClassId, intClassId)), ModelProvider { + sequenceOf( + FuzzedParameter(0, UtPrimitiveModel(true).fuzzed()), + FuzzedParameter(0, UtPrimitiveModel(false).fuzzed()), + FuzzedParameter(1, UtPrimitiveModel(-1).fuzzed()), + FuzzedParameter(1, UtPrimitiveModel(0).fuzzed()), + FuzzedParameter(1, UtPrimitiveModel(1).fuzzed()), + ) + }) + assertEquals(6, fuzz.count()) + assertEquals(setOf( + listOf(UtPrimitiveModel(true), UtPrimitiveModel(-1)), + listOf(UtPrimitiveModel(false), UtPrimitiveModel(-1)), + listOf(UtPrimitiveModel(true), UtPrimitiveModel(0)), + listOf(UtPrimitiveModel(false), UtPrimitiveModel(0)), + listOf(UtPrimitiveModel(true), UtPrimitiveModel(1)), + listOf(UtPrimitiveModel(false), UtPrimitiveModel(1)), + ), fuzz.map { arguments -> arguments.map { fuzzedValue -> fuzzedValue.model } }.toSet()) + } + + // Because of Long limitation fuzzer can process no more than 511 values for method with 7 parameters + @Test + @Timeout(1, unit = TimeUnit.SECONDS) + fun `the worst case works well`() { + assertDoesNotThrow { + val values = (0 until 511).map { UtPrimitiveModel(it).fuzzed() }.asSequence() + val provider = ModelProvider { descr -> + (0 until descr.parameters.size).asSequence() + .flatMap { index -> values.map { FuzzedParameter(index, it) } } + } + val parameters = (0 until 7).mapTo(mutableListOf()) { intClassId } + val fuzz = fuzz(newDescription(parameters), provider) + val first10 = fuzz.take(10).toList() + assertEquals(10, first10.size) + } + } + + private fun defaultTypes(includeStringId: Boolean = false): List { + val result = mutableListOf( + booleanClassId, + byteClassId, + charClassId, + shortClassId, + intClassId, + longClassId, + floatClassId, + doubleClassId, + ) + if (includeStringId) { + result += stringClassId + } + result += Any::class.java.id + return result + } + + private fun newDescription( + parameters: List, + concreteValues: Collection = emptyList() + ): FuzzedMethodDescription { + return FuzzedMethodDescription("testMethod", voidClassId, parameters, concreteValues) + } + +} \ No newline at end of file