Overflow in fuzzing #588 (#604)

Markoutte · web-flow · commit 45b71650489d · 2022-07-28T14:33:12.000+03:00
diff --git a/utbot-fuzzers/src/main/kotlin/org/utbot/fuzzer/CartesianProduct.kt b/utbot-fuzzers/src/main/kotlin/org/utbot/fuzzer/CartesianProduct.kt
@@ -1,5 +1,6 @@
 package org.utbot.fuzzer
 
+import kotlin.jvm.Throws
 import kotlin.random.Random
 
 /**
@@ -10,18 +11,59 @@ class CartesianProduct<T>(
     private val random: Random? = null
 ): Iterable<List<T>> {
 
-    fun asSequence(): Sequence<List<T>> = iterator().asSequence()
+    /**
+     * Estimated number of all combinations.
+     */
+    val estimatedSize: Long
+        get() = Combinations(*lists.map { it.size }.toIntArray()).size
 
-    override fun iterator(): Iterator<List<T>> {
+    @Throws(TooManyCombinationsException::class)
+    fun asSequence(): Sequence<List<T>> {
         val combinations = Combinations(*lists.map { it.size }.toIntArray())
         val sequence = if (random != null) {
-            val permutation = PseudoShuffledIntProgression(combinations.size, random)
-            (0 until combinations.size).asSequence().map { combinations[permutation[it]] }
+            sequence {
+                forEachChunk(Int.MAX_VALUE, combinations.size) { startIndex, combinationSize, _ ->
+                    val permutation = PseudoShuffledIntProgression(combinationSize, random)
+                    val temp = IntArray(size = lists.size)
+                    for (it in 0 until combinationSize) {
+                        yield(combinations[permutation[it] + startIndex, temp])
+                    }
+                }
+            }
         } else {
             combinations.asSequence()
         }
         return sequence.map { combination ->
-            combination.mapIndexedTo(mutableListOf()) { element, value -> lists[element][value] }
-        }.iterator()
+            combination.mapIndexedTo(ArrayList(combination.size)) { index, value -> lists[index][value] }
+        }
+    }
+
+    override fun iterator(): Iterator<List<T>> = asSequence().iterator()
+
+    companion object {
+        /**
+         * Consumer for processing blocks of input larger block.
+         *
+         * If source example is sized to 12 and every block is sized to 5 then consumer should be called 3 times with these values:
+         *
+         * 1. start = 0, size = 5, remain = 7
+         * 2. start = 5, size = 5, remain = 2
+         * 3. start = 10, size = 2, remain = 0
+         *
+         * The sum of start, size and remain should be equal to source block size.
+         */
+        internal inline fun forEachChunk(
+            chunkSize: Int,
+            totalSize: Long,
+            block: (start: Long, size: Int, remain: Long) -> Unit
+        ) {
+            val iterationsCount = totalSize / chunkSize + if (totalSize % chunkSize == 0L) 0 else 1
+            (0L until iterationsCount).forEach { iteration ->
+                val start = iteration * chunkSize
+                val size = minOf(chunkSize.toLong(), totalSize - start).toInt()
+                val remain = totalSize - size - start
+                block(start, size, remain)
+            }
+        }
     }
 }
diff --git a/utbot-fuzzers/src/main/kotlin/org/utbot/fuzzer/Combinations.kt b/utbot-fuzzers/src/main/kotlin/org/utbot/fuzzer/Combinations.kt
@@ -63,18 +63,22 @@ class Combinations(vararg elementNumbers: Int): Iterable<IntArray> {
      *
      * The total count of all possible combinations is therefore `count[0]`.
      */
-    private val count: IntArray
-    val size: Int
+    private val count: LongArray
+    val size: Long
         get() = if (count.isEmpty()) 0 else count[0]
 
     init {
         val badValue = elementNumbers.indexOfFirst { it <= 0 }
         if (badValue >= 0) {
             throw IllegalArgumentException("Max value must be at least 1 to build combinations, but ${elementNumbers[badValue]} is found at position $badValue (list: $elementNumbers)")
         }
-        count = IntArray(elementNumbers.size) { elementNumbers[it] }
+        count = LongArray(elementNumbers.size) { elementNumbers[it].toLong() }
         for (i in count.size - 2 downTo 0) {
-            count[i] = count[i] * count[i + 1]
+            try {
+                count[i] = StrictMath.multiplyExact(count[i], count[i + 1])
+            } catch (e: ArithmeticException) {
+                throw TooManyCombinationsException("Long overflow: ${count[i]} * ${count[i + 1]}")
+            }
         }
     }
 
@@ -94,7 +98,7 @@ class Combinations(vararg elementNumbers: Int): Iterable<IntArray> {
      * }
      * ```
      */
-    operator fun get(value: Int, target: IntArray = IntArray(count.size)): IntArray {
+    operator fun get(value: Long, target: IntArray = IntArray(count.size)): IntArray {
         if (value >= size) {
             throw java.lang.IllegalArgumentException("Only $size values allowed")
         }
@@ -104,13 +108,20 @@ class Combinations(vararg elementNumbers: Int): Iterable<IntArray> {
         var rem = value
         for (i in target.indices) {
             target[i] = if (i < target.size - 1) {
-                val res = rem / count[i + 1]
+                val res = checkBoundsAndCast(rem / count[i + 1])
                 rem %= count[i + 1]
                 res
             } else {
-                rem
+                checkBoundsAndCast(rem)
             }
         }
         return target
     }
-}
+
+    private fun checkBoundsAndCast(value: Long): Int {
+        check(value >= 0 && value < Int.MAX_VALUE) { "Value is out of bounds: $value" }
+        return value.toInt()
+    }
+}
+
+class TooManyCombinationsException(msg: String) : RuntimeException(msg)
diff --git a/utbot-fuzzers/src/main/kotlin/org/utbot/fuzzer/Fuzzer.kt b/utbot-fuzzers/src/main/kotlin/org/utbot/fuzzer/Fuzzer.kt
@@ -11,13 +11,18 @@ import org.utbot.fuzzer.providers.CollectionModelProvider
 import org.utbot.fuzzer.providers.PrimitiveDefaultsModelProvider
 import org.utbot.fuzzer.providers.EnumModelProvider
 import org.utbot.fuzzer.providers.PrimitiveWrapperModelProvider
+import java.lang.IllegalArgumentException
 import java.util.concurrent.atomic.AtomicInteger
 import java.util.function.IntSupplier
 import kotlin.random.Random
 
-private val logger = KotlinLogging.logger {}
+private val logger by lazy { KotlinLogging.logger {} }
 
 fun fuzz(description: FuzzedMethodDescription, vararg modelProviders: ModelProvider): Sequence<List<FuzzedValue>> {
+    if (modelProviders.isEmpty()) {
+        throw IllegalArgumentException("At least one model provider is required")
+    }
+
     val values = List<MutableList<FuzzedValue>>(description.parameters.size) { mutableListOf() }
     modelProviders.forEach { fuzzingProvider ->
         fuzzingProvider.generate(description).forEach { (index, model) ->
diff --git a/utbot-fuzzers/src/main/kotlin/org/utbot/fuzzer/providers/ObjectModelProvider.kt b/utbot-fuzzers/src/main/kotlin/org/utbot/fuzzer/providers/ObjectModelProvider.kt
@@ -1,5 +1,6 @@
 package org.utbot.fuzzer.providers
 
+import mu.KotlinLogging
 import org.utbot.framework.plugin.api.ClassId
 import org.utbot.framework.plugin.api.ConstructorId
 import org.utbot.framework.plugin.api.FieldId
@@ -20,6 +21,7 @@ import org.utbot.fuzzer.FuzzedParameter
 import org.utbot.fuzzer.FuzzedValue
 import org.utbot.fuzzer.ModelProvider
 import org.utbot.fuzzer.ModelProvider.Companion.yieldValue
+import org.utbot.fuzzer.TooManyCombinationsException
 import org.utbot.fuzzer.exceptIsInstance
 import org.utbot.fuzzer.fuzz
 import org.utbot.fuzzer.objectModelProviders
@@ -31,6 +33,8 @@ import java.lang.reflect.Method
 import java.lang.reflect.Modifier.*
 import java.util.function.IntSupplier
 
+private val logger by lazy { KotlinLogging.logger {} }
+
 /**
  * Creates [UtAssembleModel] for objects which have public constructors with primitives types and String as parameters.
  */
@@ -170,7 +174,12 @@ class ObjectModelProvider : ModelProvider {
             ).apply {
                 this.packageName = this@fuzzParameters.packageName
             }
-            return fuzz(fuzzedMethod, *modelProviders)
+            return try {
+                fuzz(fuzzedMethod, *modelProviders)
+            } catch (t: TooManyCombinationsException) {
+                logger.warn(t) { "Number of combination of ${parameters.size} parameters is huge. Fuzzing is skipped for $name" }
+                emptySequence()
+            }
         }
 
         private fun assembleModel(id: Int, constructorId: ConstructorId, params: List<FuzzedValue>): FuzzedValue {
diff --git a/utbot-fuzzers/src/test/kotlin/org/utbot/framework/plugin/api/CombinationsTest.kt b/utbot-fuzzers/src/test/kotlin/org/utbot/framework/plugin/api/CombinationsTest.kt
@@ -4,9 +4,17 @@ import org.utbot.fuzzer.CartesianProduct
 import org.utbot.fuzzer.Combinations
 import org.junit.jupiter.api.Assertions.assertArrayEquals
 import org.junit.jupiter.api.Assertions.assertEquals
+import org.junit.jupiter.api.Assertions.assertFalse
 import org.junit.jupiter.api.Assertions.assertThrows
+import org.junit.jupiter.api.Assertions.assertTrue
 import org.junit.jupiter.api.Test
+import org.junit.jupiter.api.assertDoesNotThrow
+import org.junit.jupiter.params.ParameterizedTest
+import org.junit.jupiter.params.provider.ValueSource
+import org.utbot.fuzzer.TooManyCombinationsException
+import java.util.BitSet
 import kotlin.math.pow
+import kotlin.random.Random
 
 class CombinationsTest {
 
@@ -55,11 +63,11 @@ class CombinationsTest {
         val array = intArrayOf(10, 10, 10)
         val combinations = Combinations(*array)
         combinations.forEachIndexed { i, c ->
-            var actual = 0
+            var actual = 0L
             for (pos in array.indices) {
                 actual += c[pos] * (10.0.pow(array.size - 1.0 - pos).toInt())
             }
-            assertEquals(i, actual)
+            assertEquals(i.toLong(), actual)
         }
     }
 
@@ -105,4 +113,161 @@ class CombinationsTest {
         }
     }
 
+    @ParameterizedTest(name = "testAllLongValues{arguments}")
+    @ValueSource(ints = [1, 100, Int.MAX_VALUE])
+    fun testAllLongValues(value: Int) {
+        val combinations = Combinations(value, value, 2)
+        assertEquals(2L * value * value, combinations.size)
+        val array = combinations[combinations.size - 1]
+        assertEquals(value - 1, array[0])
+        assertEquals(value - 1, array[1])
+        assertEquals(1, array[2])
+    }
+
+    @Test
+    fun testCartesianFindsAllValues() {
+        val radix = 4
+        val product = createIntCartesianProduct(radix, 10)
+        val total = product.estimatedSize
+        assertTrue(total < Int.MAX_VALUE) { "This test should generate less than Int.MAX_VALUE values but has $total" }
+
+        val set = BitSet((total / 64).toInt())
+        val updateSet: (List<String>) -> Unit = {
+            val value = it.joinToString("").toLong(radix).toInt()
+            assertFalse(set[value])
+            set.set(value)
+        }
+        val realCount = product.onEach(updateSet).count()
+        assertEquals(total, realCount.toLong())
+
+        for (i in 0 until total) {
+            assertTrue(set[i.toInt()]) { "Values is not listed for index = $i" }
+        }
+        for (i in total until set.size()) {
+            assertFalse(set[i.toInt()])
+        }
+    }
+
+    /**
+     * Creates all numbers from 0 to `radix^repeat`.
+     *
+     * For example:
+     *
+     * radix = 2, repeat = 2 -> {'0', '0'}, {'0', '1'}, {'1', '0'}, {'1', '1'}
+     * radix = 16, repeat = 1 -> {'0'}, {'1'}, {'2'}, {'3'}, {'4'}, {'5'}, {'6'}, {'7'}, {'8'}, {'9'}, {'a'}, {'b'}, {'c'}, {'d'}, {'e'}, {'f'}
+     */
+    private fun createIntCartesianProduct(radix: Int, repeat: Int) =
+        CartesianProduct(
+            lists = (1..repeat).map {
+                Array(radix) { it.toString(radix) }.toList()
+            },
+            random = Random(0)
+        ).apply {
+            assertEquals((1L..repeat).fold(1L) { acc, _ -> acc * radix }, estimatedSize)
+        }
+
+    @Test
+    fun testCanCreateCartesianProductWithSizeGreaterThanMaxInt() {
+        val product = createIntCartesianProduct(5, 15)
+        assertTrue(product.estimatedSize > Int.MAX_VALUE) { "This test should generate more than Int.MAX_VALUE values but has ${product.estimatedSize}" }
+        assertDoesNotThrow {
+            product.first()
+        }
+    }
+
+    @Test
+    fun testIterationWithChunksIsCorrect() {
+        val expected = mutableListOf(
+            Triple(0L, 5, 7L),
+            Triple(5L, 5, 2L),
+            Triple(10L, 2, 0L),
+        )
+        CartesianProduct.forEachChunk(5, 12) { start, chunk, remain ->
+            assertEquals(expected.removeFirst(), Triple(start, chunk, remain))
+        }
+        assertTrue(expected.isEmpty())
+    }
+
+    @Test
+    fun testIterationWithChunksIsCorrectWhenChunkIsIntMax() {
+        val total = 12
+        val expected = mutableListOf(
+            Triple(0L, total, 0L)
+        )
+        CartesianProduct.forEachChunk(Int.MAX_VALUE, total.toLong()) { start, chunk, remain ->
+            assertEquals(expected.removeFirst(), Triple(start, chunk, remain))
+        }
+        assertTrue(expected.isEmpty())
+    }
+
+    @ParameterizedTest(name = "testIterationWithChunksIsCorrectWhenChunkIs{arguments}")
+    @ValueSource(ints = [1, 2, 3, 4, 6, 12])
+    fun testIterationWithChunksIsCorrectWhenChunk(chunkSize: Int) {
+        val total = 12
+        assertTrue(total % chunkSize == 0) { "Test requires values that are dividers of the total = $total, but it is not true for $chunkSize" }
+        val expected = (0 until total step chunkSize).map { it.toLong() }.map {
+            Triple(it, chunkSize, total - it - chunkSize)
+        }.toMutableList()
+        CartesianProduct.forEachChunk(chunkSize, total.toLong()) { start, chunk, remain ->
+            assertEquals(expected.removeFirst(), Triple(start, chunk, remain))
+        }
+        assertTrue(expected.isEmpty())
+    }
+
+    @ParameterizedTest(name = "testIterationsWithChunksThroughLongWithRemainingIs{arguments}")
+    @ValueSource(longs = [1L, 200L, 307, Int.MAX_VALUE - 1L, Int.MAX_VALUE.toLong()])
+    fun testIterationsWithChunksThroughLongTotal(remaining: Long) {
+        val expected = mutableListOf(
+            Triple(0L, Int.MAX_VALUE, Int.MAX_VALUE + remaining),
+            Triple(Int.MAX_VALUE.toLong(), Int.MAX_VALUE, remaining),
+            Triple(Int.MAX_VALUE * 2L, remaining.toInt(), 0L),
+        )
+        CartesianProduct.forEachChunk(Int.MAX_VALUE, Int.MAX_VALUE * 2L + remaining) { start, chunk, remain ->
+            assertEquals(expected.removeFirst(), Triple(start, chunk, remain))
+        }
+        assertTrue(expected.isEmpty())
+    }
+
+    @Test
+    fun testCartesianProductDoesNotThrowsExceptionBeforeOverflow() {
+        // We assume that a standard method has no more than 7 parameters.
+        // In this case every parameter can accept up to 511 values without Long overflow.
+        // CartesianProduct throws exception
+        val values = Array(511) { it }.toList()
+        val parameters = Array(7) { values }.toList()
+        assertDoesNotThrow {
+            CartesianProduct(parameters, Random(0)).asSequence()
+        }
+    }
+
+    @Test
+    fun testCartesianProductThrowsExceptionOnOverflow() {
+        // We assume that a standard method has no more than 7 parameters.
+        // In this case every parameter can accept up to 511 values without Long overflow.
+        // CartesianProduct throws exception
+        val values = Array(512) { it }.toList()
+        val parameters = Array(7) { values }.toList()
+        assertThrows(TooManyCombinationsException::class.java) {
+            CartesianProduct(parameters, Random(0)).asSequence()
+        }
+    }
+
+    @ParameterizedTest(name = "testCombinationHasValue{arguments}")
+    @ValueSource(ints = [1, Int.MAX_VALUE])
+    fun testCombinationHasValue(value: Int) {
+        val combinations = Combinations(value)
+        assertEquals(value.toLong(), combinations.size)
+        assertEquals(value - 1, combinations[value - 1L][0])
+    }
+
+    @Test
+    fun testNoFailWhenMixedValues() {
+        val combinations = Combinations(2, Int.MAX_VALUE)
+        assertEquals(2 * Int.MAX_VALUE.toLong(), combinations.size)
+        assertArrayEquals(intArrayOf(0, 0), combinations[0L])
+        assertArrayEquals(intArrayOf(0, Int.MAX_VALUE - 1), combinations[Int.MAX_VALUE - 1L])
+        assertArrayEquals(intArrayOf(1, 0), combinations[Int.MAX_VALUE.toLong()])
+        assertArrayEquals(intArrayOf(1, 1), combinations[Int.MAX_VALUE + 1L])
+        assertArrayEquals(intArrayOf(1, Int.MAX_VALUE - 1), combinations[Int.MAX_VALUE * 2L - 1])
+    }
 }
diff --git a/utbot-fuzzers/src/test/kotlin/org/utbot/framework/plugin/api/FuzzerTest.kt b/utbot-fuzzers/src/test/kotlin/org/utbot/framework/plugin/api/FuzzerTest.kt
