diff --git a/utbot-cli-python/src/main/kotlin/org/utbot/cli/language/python/PythonCliProcessor.kt b/utbot-cli-python/src/main/kotlin/org/utbot/cli/language/python/PythonCliProcessor.kt index d49b2895a9..4ded0545ac 100644 --- a/utbot-cli-python/src/main/kotlin/org/utbot/cli/language/python/PythonCliProcessor.kt +++ b/utbot-cli-python/src/main/kotlin/org/utbot/cli/language/python/PythonCliProcessor.kt @@ -10,6 +10,7 @@ class PythonCliProcessor( private val output: String, private val logger: KLogger, private val coverageOutput: String?, + private val executionCounterOutput: String?, ) : PythonTestGenerationProcessor() { override fun saveTests(testsCode: String) { @@ -22,9 +23,18 @@ class PythonCliProcessor( ) } + private fun getExecutionsNumber(testSets: List): Int { + return testSets.sumOf { it.executionsNumber } + } + override fun processCoverageInfo(testSets: List) { - val coverageReport = getStringCoverageInfo(testSets) - val output = coverageOutput ?: return - writeToFileAndSave(output, coverageReport) + coverageOutput?.let { output -> + val coverageReport = getStringCoverageInfo(testSets) + writeToFileAndSave(output, coverageReport) + } + executionCounterOutput?.let { executionOutput -> + val executionReport = "{\"executions\": ${getExecutionsNumber(testSets)}}" + writeToFileAndSave(executionOutput, executionReport) + } } } \ No newline at end of file diff --git a/utbot-cli-python/src/main/kotlin/org/utbot/cli/language/python/PythonGenerateTestsCommand.kt b/utbot-cli-python/src/main/kotlin/org/utbot/cli/language/python/PythonGenerateTestsCommand.kt index 981d2f9b4b..89448aebb9 100644 --- a/utbot-cli-python/src/main/kotlin/org/utbot/cli/language/python/PythonGenerateTestsCommand.kt +++ b/utbot-cli-python/src/main/kotlin/org/utbot/cli/language/python/PythonGenerateTestsCommand.kt @@ -76,6 +76,11 @@ class PythonGenerateTestsCommand : CliktCommand( help = "File to write coverage report." ) + private val executionCounterOutput by option( + "--executions-counter", + help = "File to write number of executions." + ) + private val installRequirementsIfMissing by option( "--install-requirements", help = "Install Python requirements if missing." @@ -255,6 +260,7 @@ class PythonGenerateTestsCommand : CliktCommand( output.toAbsolutePath(), logger, coverageOutput?.toAbsolutePath(), + executionCounterOutput?.toAbsolutePath(), ) logger.info("Loading information about Python types...") @@ -270,7 +276,8 @@ class PythonGenerateTestsCommand : CliktCommand( testSet.executions.filterNot { it.result is UtExecutionSuccess }, testSet.errors, testSet.mypyReport, - testSet.classId + testSet.classId, + testSet.executionsNumber ) } } diff --git a/utbot-python-executor/src/main/python/utbot_executor/pyproject.toml b/utbot-python-executor/src/main/python/utbot_executor/pyproject.toml index 74249105ff..13b3063e09 100644 --- a/utbot-python-executor/src/main/python/utbot_executor/pyproject.toml +++ b/utbot-python-executor/src/main/python/utbot_executor/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "utbot-executor" -version = "1.4.44" +version = "1.7.0" description = "" authors = ["Vyacheslav Tamarin "] readme = "README.md" diff --git a/utbot-python-executor/src/main/resources/utbot_executor_version b/utbot-python-executor/src/main/resources/utbot_executor_version index b0a831507c..9dbb0c0052 100644 --- a/utbot-python-executor/src/main/resources/utbot_executor_version +++ b/utbot-python-executor/src/main/resources/utbot_executor_version @@ -1 +1 @@ -1.4.44 \ No newline at end of file +1.7.0 \ No newline at end of file diff --git a/utbot-python/samples/cli_utbot_tests/generated_tests__arithmetic.py b/utbot-python/samples/cli_utbot_tests/generated_tests__arithmetic.py deleted file mode 100644 index fe6ecb2585..0000000000 --- a/utbot-python/samples/cli_utbot_tests/generated_tests__arithmetic.py +++ /dev/null @@ -1,40 +0,0 @@ -import sys -sys.path.append('samples') -import builtins -import arithmetic -import unittest - - -class TestTopLevelFunctions(unittest.TestCase): - # region Test suites for executable arithmetic.calculate_function_value - # region - def test_calculate_function_value(self): - actual = arithmetic.calculate_function_value(1, 101) - - self.assertEqual(11886.327847992769, actual) - - def test_calculate_function_value1(self): - actual = arithmetic.calculate_function_value(4294967296, 101) - - self.assertEqual(65535.99845886229, actual) - - def test_calculate_function_value2(self): - actual = arithmetic.calculate_function_value(float('nan'), 4294967296) - - self.assertTrue(isinstance(actual, builtins.float)) - - def test_calculate_function_value_throws_t(self): - arithmetic.calculate_function_value(0, 101) - - # raises builtins.ZeroDivisionError - - def test_calculate_function_value_throws_t1(self): - arithmetic.calculate_function_value(101, 101) - - # raises builtins.ValueError - - # endregion - - # endregion - - diff --git a/utbot-python/samples/cli_utbot_tests/generated_tests__deep_equals.py b/utbot-python/samples/cli_utbot_tests/generated_tests__deep_equals.py deleted file mode 100644 index 3f04ba5a47..0000000000 --- a/utbot-python/samples/cli_utbot_tests/generated_tests__deep_equals.py +++ /dev/null @@ -1,87 +0,0 @@ -import sys -sys.path.append('samples') -import builtins -import deep_equals -import copyreg -import types -import unittest - - -class TestTopLevelFunctions(unittest.TestCase): - # region Test suites for executable deep_equals.comparable_list - # region - def test_comparable_list(self): - actual = deep_equals.comparable_list(4294967296) - - comparable_class = copyreg._reconstructor(deep_equals.ComparableClass, builtins.object, None) - comparable_class.x = 0 - comparable_class1 = copyreg._reconstructor(deep_equals.ComparableClass, builtins.object, None) - comparable_class1.x = 1 - comparable_class2 = copyreg._reconstructor(deep_equals.ComparableClass, builtins.object, None) - comparable_class2.x = 2 - comparable_class3 = copyreg._reconstructor(deep_equals.ComparableClass, builtins.object, None) - comparable_class3.x = 3 - comparable_class4 = copyreg._reconstructor(deep_equals.ComparableClass, builtins.object, None) - comparable_class4.x = 4 - comparable_class5 = copyreg._reconstructor(deep_equals.ComparableClass, builtins.object, None) - comparable_class5.x = 5 - comparable_class6 = copyreg._reconstructor(deep_equals.ComparableClass, builtins.object, None) - comparable_class6.x = 6 - comparable_class7 = copyreg._reconstructor(deep_equals.ComparableClass, builtins.object, None) - comparable_class7.x = 7 - comparable_class8 = copyreg._reconstructor(deep_equals.ComparableClass, builtins.object, None) - comparable_class8.x = 8 - comparable_class9 = copyreg._reconstructor(deep_equals.ComparableClass, builtins.object, None) - comparable_class9.x = 9 - - self.assertEqual([comparable_class, comparable_class1, comparable_class2, comparable_class3, comparable_class4, comparable_class5, comparable_class6, comparable_class7, comparable_class8, comparable_class9], actual) - - # endregion - - # endregion - - # region Test suites for executable deep_equals.incomparable_list - # region - def test_incomparable_list(self): - actual = deep_equals.incomparable_list(4294967296) - - incomparable_class = copyreg._reconstructor(deep_equals.IncomparableClass, builtins.object, None) - incomparable_class.x = 0 - incomparable_class1 = copyreg._reconstructor(deep_equals.IncomparableClass, builtins.object, None) - incomparable_class1.x = 1 - incomparable_class2 = copyreg._reconstructor(deep_equals.IncomparableClass, builtins.object, None) - incomparable_class2.x = 2 - incomparable_class3 = copyreg._reconstructor(deep_equals.IncomparableClass, builtins.object, None) - incomparable_class3.x = 3 - incomparable_class4 = copyreg._reconstructor(deep_equals.IncomparableClass, builtins.object, None) - incomparable_class4.x = 4 - incomparable_class5 = copyreg._reconstructor(deep_equals.IncomparableClass, builtins.object, None) - incomparable_class5.x = 5 - incomparable_class6 = copyreg._reconstructor(deep_equals.IncomparableClass, builtins.object, None) - incomparable_class6.x = 6 - incomparable_class7 = copyreg._reconstructor(deep_equals.IncomparableClass, builtins.object, None) - incomparable_class7.x = 7 - incomparable_class8 = copyreg._reconstructor(deep_equals.IncomparableClass, builtins.object, None) - incomparable_class8.x = 8 - incomparable_class9 = copyreg._reconstructor(deep_equals.IncomparableClass, builtins.object, None) - incomparable_class9.x = 9 - expected_list = [incomparable_class, incomparable_class1, incomparable_class2, incomparable_class3, incomparable_class4, incomparable_class5, incomparable_class6, incomparable_class7, incomparable_class8, incomparable_class9] - expected_length = len(expected_list) - actual_length = len(actual) - - self.assertEqual(expected_length, actual_length) - - index = None - for index in range(0, expected_length, 1): - expected_element = expected_list[index] - actual_element = actual[index] - actual_x = actual_element.x - expected_x = expected_element.x - - self.assertEqual(expected_x, actual_x) - - # endregion - - # endregion - - diff --git a/utbot-python/samples/cli_utbot_tests/generated_tests__deque.py b/utbot-python/samples/cli_utbot_tests/generated_tests__deque.py deleted file mode 100644 index fa4f36dee3..0000000000 --- a/utbot-python/samples/cli_utbot_tests/generated_tests__deque.py +++ /dev/null @@ -1,35 +0,0 @@ -import sys -sys.path.append('samples') -import builtins -import deque -import collections -import unittest - - -class TestTopLevelFunctions(unittest.TestCase): - # region Test suites for executable deque.generate_people_deque - # region - def test_generate_people_deque(self): - actual = deque.generate_people_deque(4294967297) - - deque1 = collections.deque() - deque1.append('Alex') - deque1.append('Bob') - deque1.append('Cate') - deque1.append('Daisy') - deque1.append('Ed') - - self.assertEqual(deque1, actual) - - def test_generate_people_deque1(self): - actual = deque.generate_people_deque(0) - - deque1 = collections.deque() - - self.assertEqual(deque1, actual) - - # endregion - - # endregion - - diff --git a/utbot-python/samples/cli_utbot_tests/generated_tests__dicts.py b/utbot-python/samples/cli_utbot_tests/generated_tests__dicts.py deleted file mode 100644 index aba188aa11..0000000000 --- a/utbot-python/samples/cli_utbot_tests/generated_tests__dicts.py +++ /dev/null @@ -1,44 +0,0 @@ -import sys -sys.path.append('samples') -import builtins -import types -import dicts -import unittest - - -class TestTopLevelFunctions(unittest.TestCase): - # region Test suites for executable dicts.keys - - # region - - def test_keys(self): - word = dicts.Word({str(-123456789): str(), str(1.5 + 3.5j): str(), str(b'\x80'): str(), str(): str(1e+300 * 1e+300), str('unicode remains unicode'): str(), }) - - actual = word.keys() - - self.assertEqual(['-123456789', '(1.5+3.5j)', "b'\\x80'", '', 'unicode remains unicode'], actual) - # endregion - - # endregion - - # region Test suites for executable dicts.translate - - # region - - def test_translate(self): - dictionary = dicts.Dictionary([str(b'\xf0\xa3\x91\x96', 'utf-8'), str(id), str(1e+300 * 1e+300)], []) - - actual = dictionary.translate(str(id), str(1.5 + 3.5j)) - - self.assertEqual(None, actual) - - def test_translate_throws_t(self): - dictionary = dicts.Dictionary([], [{str(): str(), str(1e+300 * 1e+300): str(1e+300 * 1e+300), str(b'\x80'): str(), str(1.5 + 3.5j): str(), }, {str(-123456789): str(), str(id): str(), str(): str(), str(-1234567890): str(), }, {str(1.5 + 3.5j): str(), str(1e+300 * 1e+300): str(), str(-1234567890): str(), str(): str(1e+300 * 1e+300), }]) - - dictionary.translate(str('unicode remains unicode'), str(1.5 + 3.5j)) - - # raises builtins.KeyError - # endregion - - # endregion - diff --git a/utbot-python/samples/cli_utbot_tests/generated_tests__dummy_with_eq.py b/utbot-python/samples/cli_utbot_tests/generated_tests__dummy_with_eq.py deleted file mode 100644 index f6eed2d5ab..0000000000 --- a/utbot-python/samples/cli_utbot_tests/generated_tests__dummy_with_eq.py +++ /dev/null @@ -1,39 +0,0 @@ -import sys -sys.path.append('samples') -import dummy_with_eq -import builtins -import copyreg -import types -import unittest - - -class TestDummy(unittest.TestCase): - # region Test suites for executable dummy_with_eq.propagate - # region - def test_propagate(self): - dummy = dummy_with_eq.Dummy(1) - - actual = dummy.propagate() - - dummy1 = copyreg._reconstructor(dummy_with_eq.Dummy, builtins.object, None) - dummy1.field = 1 - expected_list = [dummy1, dummy1] - expected_length = len(expected_list) - actual_length = len(actual) - - self.assertEqual(expected_length, actual_length) - - index = None - for index in range(0, expected_length, 1): - expected_element = expected_list[index] - actual_element = actual[index] - actual_field = actual_element.field - expected_field = expected_element.field - - self.assertEqual(expected_field, actual_field) - - # endregion - - # endregion - - diff --git a/utbot-python/samples/cli_utbot_tests/generated_tests__dummy_without_eq.py b/utbot-python/samples/cli_utbot_tests/generated_tests__dummy_without_eq.py deleted file mode 100644 index 3a38c1f6b7..0000000000 --- a/utbot-python/samples/cli_utbot_tests/generated_tests__dummy_without_eq.py +++ /dev/null @@ -1,36 +0,0 @@ -import sys -sys.path.append('samples') -import dummy_without_eq -import builtins -import copyreg -import types -import unittest - - -class TestDummy(unittest.TestCase): - # region Test suites for executable dummy_without_eq.propagate - # region - def test_propagate(self): - dummy = dummy_without_eq.Dummy() - - actual = dummy.propagate() - - dummy1 = copyreg._reconstructor(dummy_without_eq.Dummy, builtins.object, None) - expected_list = [dummy1, dummy1] - expected_length = len(expected_list) - actual_length = len(actual) - - self.assertEqual(expected_length, actual_length) - - index = None - for index in range(0, expected_length, 1): - expected_element = expected_list[index] - actual_element = actual[index] - - self.assertTrue(isinstance(actual_element, dummy_without_eq.Dummy)) - - # endregion - - # endregion - - diff --git a/utbot-python/samples/cli_utbot_tests/generated_tests__graph.py b/utbot-python/samples/cli_utbot_tests/generated_tests__graph.py deleted file mode 100644 index c383980fb6..0000000000 --- a/utbot-python/samples/cli_utbot_tests/generated_tests__graph.py +++ /dev/null @@ -1,36 +0,0 @@ -import sys -sys.path.append('samples') -import unittest -import builtins -import graph -import copyreg -import types - - -class TestTopLevelFunctions(unittest.TestCase): - # region Test suites for executable graph.bfs - # region - def test_bfs(self): - actual = graph.bfs([graph.Node(str(1e+300 * 1e+300), []), graph.Node(str(id), []), graph.Node(str('unicode remains unicode'), [])]) - - node = copyreg._reconstructor(graph.Node, builtins.object, None) - node.name = 'unicode remains unicode' - node.children = [] - node1 = copyreg._reconstructor(graph.Node, builtins.object, None) - node1.name = '' - node1.children = [] - node2 = copyreg._reconstructor(graph.Node, builtins.object, None) - node2.name = 'inf' - node2.children = [] - self.assertEqual([node, node1, node2], actual) - - def test_bfs1(self): - actual = graph.bfs([]) - - self.assertEqual([], actual) - - # endregion - - # endregion - - diff --git a/utbot-python/samples/cli_utbot_tests/generated_tests__list_of_datetime.py b/utbot-python/samples/cli_utbot_tests/generated_tests__list_of_datetime.py deleted file mode 100644 index 4339e26c0e..0000000000 --- a/utbot-python/samples/cli_utbot_tests/generated_tests__list_of_datetime.py +++ /dev/null @@ -1,32 +0,0 @@ -import sys -sys.path.append('samples') -import builtins -import list_of_datetime -import types -import datetime -import unittest - - -class TestTopLevelFunctions(unittest.TestCase): - # region Test suites for executable list_of_datetime.get_data_labels - # region - def test_get_data_labels(self): - actual = list_of_datetime.get_data_labels({}) - - self.assertEqual(None, actual) - - def test_get_data_labels1(self): - actual = list_of_datetime.get_data_labels([datetime.time(0), datetime.time(microsecond=40), datetime.time(18, 45, 3, 1234), datetime.time(12, 0)]) - - self.assertEqual(['00:00', '00:00', '18:45', '12:00'], actual) - - def test_get_data_labels2(self): - actual = list_of_datetime.get_data_labels([datetime.time(microsecond=40), datetime.time()]) - - self.assertEqual(['1900-01-01', '1900-01-01'], actual) - - # endregion - - # endregion - - diff --git a/utbot-python/samples/cli_utbot_tests/generated_tests__lists.py b/utbot-python/samples/cli_utbot_tests/generated_tests__lists.py deleted file mode 100644 index 38d5289f30..0000000000 --- a/utbot-python/samples/cli_utbot_tests/generated_tests__lists.py +++ /dev/null @@ -1,21 +0,0 @@ -import sys -sys.path.append('samples') -import builtins -import lists -import datetime -import unittest - - -class TestTopLevelFunctions(unittest.TestCase): - # region Test suites for executable lists.find_articles_with_author - # region - def test_find_articles_with_author(self): - actual = lists.find_articles_with_author([lists.Article(str(-123456789), str(b'\xf0\xa3\x91\x96', 'utf-8'), str('unicode remains unicode'), datetime.datetime(2015, 4, 5, 1, 45)), lists.Article(str(-123456789), str(b'\xf0\xa3\x91\x96', 'utf-8'), str('unicode remains unicode'), datetime.datetime(2011, 1, 1)), lists.Article(str(-123456789), str(b'\xf0\xa3\x91\x96', 'utf-8'), str(), datetime.datetime(1, 2, 3, 4, 5, 6, 7)), lists.Article(str(-123456789), str(b'\xf0\xa3\x91\x96', 'utf-8'), str(id), datetime.datetime(1, 2, 3, 4, 5, 6, 7)), lists.Article(str(-123456789), str(b'\xf0\xa3\x91\x96', 'utf-8'), str(id), datetime.datetime(2014, 11, 2, 1, 30)), lists.Article(str(-123456789), str(b'\xf0\xa3\x91\x96', 'utf-8'), str(id), datetime.datetime(1, 2, 3, 4, 5, 6, 7))], str('unicode remains unicode')) - - self.assertEqual([], actual) - - # endregion - - # endregion - - diff --git a/utbot-python/samples/cli_utbot_tests/generated_tests__longest_subsequence.py b/utbot-python/samples/cli_utbot_tests/generated_tests__longest_subsequence.py deleted file mode 100644 index 510171cbbc..0000000000 --- a/utbot-python/samples/cli_utbot_tests/generated_tests__longest_subsequence.py +++ /dev/null @@ -1,25 +0,0 @@ -import sys -sys.path.append('samples') -import builtins -import longest_subsequence -import unittest - - -class TestTopLevelFunctions(unittest.TestCase): - # region Test suites for executable longest_subsequence.longest_subsequence - # region - def test_longest_subsequence(self): - actual = longest_subsequence.longest_subsequence([1, 83]) - - self.assertEqual([1, 83], actual) - - def test_longest_subsequence1(self): - actual = longest_subsequence.longest_subsequence([2, -1, 4294967296]) - - self.assertEqual([-1, 4294967296], actual) - - # endregion - - # endregion - - diff --git a/utbot-python/samples/cli_utbot_tests/generated_tests__matrix.py b/utbot-python/samples/cli_utbot_tests/generated_tests__matrix.py deleted file mode 100644 index db4d57758e..0000000000 --- a/utbot-python/samples/cli_utbot_tests/generated_tests__matrix.py +++ /dev/null @@ -1,41 +0,0 @@ -import sys -sys.path.append('samples') -import matrix -import builtins -import types -import copyreg -import unittest - - -class TestMatrix(unittest.TestCase): - # region Test suites for executable matrix.__add__ - - # region - - def test__add__(self): - matrix1 = matrix.Matrix([[float('nan'), 0.0, float(1970), 7.3, float('nan')], [float(10 ** 23), float('1.4'), float(-1), float(-1), float('nan'), float('nan'), float(1970)], [float('nan'), 0.0, float(1970), 7.3, float('nan')], [float(314), float(-1), float('nan'), float(1970), 7.3, float(-1), float(-1)], [float('nan'), 0.0, float(1970), 7.3, float('nan')], [float('nan')]]) - self1 = matrix.Matrix([[float('nan'), 0.0, float(1970), 7.3, float('nan')], [float(10 ** 23), float('1.4'), float(-1), float(-1), float('nan'), float('nan'), float(1970)], [float('nan'), 0.0, float(1970), 7.3, float('nan')], [float(314), float(-1), float('nan'), float(1970), 7.3, float(-1), float(-1)], [float('nan'), 0.0, float(1970), 7.3, float('nan')], [float('nan')]]) - - actual = matrix1.__add__(self1) - - matrix2 = copyreg._reconstructor(matrix.Matrix, builtins.object, None) - matrix2.dim = (6, 7) - matrix2.elements = [[float('nan'), 0.0, 3940.0, 14.6, float('nan'), 0, 0], [2e+23, 2.8, -2.0, -2.0, float('nan'), float('nan'), 3940.0], [float('nan'), 0.0, 3940.0, 14.6, float('nan'), 0, 0], [628.0, -2.0, float('nan'), 3940.0, 14.6, -2.0, -2.0], [float('nan'), 0.0, 3940.0, 14.6, float('nan'), 0, 0], [float('nan'), 0, 0, 0, 0, 0, 0]] - actual_dim = actual.dim - expected_dim = matrix2.dim - - self.assertEqual(expected_dim, actual_dim) - actual_elements = actual.elements - expected_elements = matrix2.elements - expected_list = expected_elements - expected_length = len(expected_list) - actual_length = len(actual_elements) - - self.assertEqual(expected_length, actual_length) - - self.assertTrue(isinstance(actual_elements, builtins.list)) - # endregion - - # endregion - - diff --git a/utbot-python/samples/cli_utbot_tests/generated_tests__primitive_types.py b/utbot-python/samples/cli_utbot_tests/generated_tests__primitive_types.py deleted file mode 100644 index 16f8f8423f..0000000000 --- a/utbot-python/samples/cli_utbot_tests/generated_tests__primitive_types.py +++ /dev/null @@ -1,40 +0,0 @@ -import sys -sys.path.append('samples') -import builtins -import primitive_types -import unittest - - -class TestTopLevelFunctions(unittest.TestCase): - # region Test suites for executable primitive_types.pretty_print - # region - def test_pretty_print(self): - actual = primitive_types.pretty_print(object()) - - self.assertEqual('I do not have any variants', actual) - - def test_pretty_print1(self): - actual = primitive_types.pretty_print(str(b'\x80')) - - self.assertEqual("It is string.\nValue <>", actual) - - def test_pretty_print2(self): - actual = primitive_types.pretty_print((1 << 100)) - - self.assertEqual('It is integer.\nValue 1267650600228229401496703205376', actual) - - def test_pretty_print3(self): - actual = primitive_types.pretty_print(complex(float('inf'), float('inf'))) - - self.assertEqual('It is complex.\nValue (inf + infi)', actual) - - def test_pretty_print4(self): - actual = primitive_types.pretty_print([]) - - self.assertEqual('It is list.\nValue []', actual) - - # endregion - - # endregion - - diff --git a/utbot-python/samples/cli_utbot_tests/generated_tests__quick_sort.py b/utbot-python/samples/cli_utbot_tests/generated_tests__quick_sort.py deleted file mode 100644 index ea0cf1a849..0000000000 --- a/utbot-python/samples/cli_utbot_tests/generated_tests__quick_sort.py +++ /dev/null @@ -1,30 +0,0 @@ -import sys -sys.path.append('samples') -import builtins -import quick_sort -import unittest - - -class TestTopLevelFunctions(unittest.TestCase): - # region Test suites for executable quick_sort.quick_sort - # region - def test_quick_sort(self): - actual = quick_sort.quick_sort([4294967297, 83, (1 << 100), 4294967297, (1 << 100), 0, -3]) - - self.assertEqual([-3, 0, 83, 4294967297, 4294967297, 1267650600228229401496703205376, 1267650600228229401496703205376], actual) - - def test_quick_sort1(self): - actual = quick_sort.quick_sort([83, 123]) - - self.assertEqual([83, 123], actual) - - def test_quick_sort2(self): - actual = quick_sort.quick_sort([]) - - self.assertEqual([], actual) - - # endregion - - # endregion - - diff --git a/utbot-python/samples/cli_utbot_tests/generated_tests__test_coverage.py b/utbot-python/samples/cli_utbot_tests/generated_tests__test_coverage.py deleted file mode 100644 index 02bcda08b6..0000000000 --- a/utbot-python/samples/cli_utbot_tests/generated_tests__test_coverage.py +++ /dev/null @@ -1,35 +0,0 @@ -import sys -sys.path.append('samples') -import builtins -import test_coverage -import unittest - - -class TestTopLevelFunctions(unittest.TestCase): - # region Test suites for executable test_coverage.hard_function - # region - def test_hard_function(self): - actual = test_coverage.hard_function(83) - - self.assertEqual(2, actual) - - def test_hard_function1(self): - actual = test_coverage.hard_function(0) - - self.assertEqual(1, actual) - - def test_hard_function2(self): - actual = test_coverage.hard_function(4294967296) - - self.assertEqual(3, actual) - - def test_hard_function3(self): - actual = test_coverage.hard_function(float('nan')) - - self.assertEqual(4, actual) - - # endregion - - # endregion - - diff --git a/utbot-python/samples/cli_utbot_tests/generated_tests__type_inference.py b/utbot-python/samples/cli_utbot_tests/generated_tests__type_inference.py deleted file mode 100644 index 450f3f0488..0000000000 --- a/utbot-python/samples/cli_utbot_tests/generated_tests__type_inference.py +++ /dev/null @@ -1,20 +0,0 @@ -import sys -sys.path.append('samples') -import builtins -import type_inference -import unittest - - -class TestTopLevelFunctions(unittest.TestCase): - # region Test suites for executable type_inference.type_inference - # region - def test_type_inference_by_fuzzer(self): - actual = type_inference.type_inference(0, str(), str(b'\x80'), [], {}) - - self.assertEqual([0, 0, 0, 0, 0, 0, 0, 0, 0, ''], actual) - - # endregion - - # endregion - - diff --git a/utbot-python/samples/cli_utbot_tests/generated_tests__using_collections.py b/utbot-python/samples/cli_utbot_tests/generated_tests__using_collections.py deleted file mode 100644 index f9fe1c7d4a..0000000000 --- a/utbot-python/samples/cli_utbot_tests/generated_tests__using_collections.py +++ /dev/null @@ -1,23 +0,0 @@ -import sys -sys.path.append('samples') -import builtins -import using_collections -import collections -import unittest - - -class TestTopLevelFunctions(unittest.TestCase): - # region Test suites for executable using_collections.generate_collections - # region - def test_generate_collections(self): - actual = using_collections.generate_collections({}) - - counter = collections.Counter({0: 100, }) - - self.assertEqual([{0: 100, }, counter, [(0, 100)]], actual) - - # endregion - - # endregion - - diff --git a/utbot-python/samples/generate_test_samples.sh b/utbot-python/samples/generate_test_samples.sh deleted file mode 100644 index 73c765a72b..0000000000 --- a/utbot-python/samples/generate_test_samples.sh +++ /dev/null @@ -1,24 +0,0 @@ -# Usage: -# ./generate_test_samples.sh - -python_path=$1 -java_path=$2 - -$java_path -jar utbot-cli.jar generate_python samples/arithmetic.py -p $python_path -o cli_utbot_tests/generated_tests__arithmetic.py -s samples/ --timeout-for-run 500 --visit-only-specified-source --timeout 10000 --do-not-check-requirements -$java_path -jar utbot-cli.jar generate_python samples/deep_equals.py -p $python_path -o cli_utbot_tests/generated_tests__deep_equals.py -s samples/ --timeout-for-run 500 --visit-only-specified-source --timeout 10000 --do-not-check-requirements -$java_path -jar utbot-cli.jar generate_python samples/dicts.py -p $python_path -o cli_utbot_tests/generated_tests__dicts.py -s samples/ --timeout-for-run 500 --visit-only-specified-source -c Dictionary -m translate --timeout 10000 --do-not-check-requirements -$java_path -jar utbot-cli.jar generate_python samples/deque.py -p $python_path -o cli_utbot_tests/generated_tests__deque.py -s samples/ --timeout-for-run 500 --visit-only-specified-source --timeout 10000 --do-not-check-requirements -$java_path -jar utbot-cli.jar generate_python samples/dummy_with_eq.py -p $python_path -o cli_utbot_tests/generated_tests__dummy_with_eq.py -s samples/ --timeout-for-run 500 --visit-only-specified-source -c Dummy -m propogate --timeout 10000 --do-not-check-requirements -$java_path -jar utbot-cli.jar generate_python samples/dummy_without_eq.py -p $python_path -o cli_utbot_tests/generated_tests__dummy_without_eq.py -s samples/ --timeout-for-run 500 --visit-only-specified-source -c Dummy -m propogate --timeout 10000 --do-not-check-requirements -$java_path -jar utbot-cli.jar generate_python samples/lists.py -p $python_path -o cli_utbot_tests/generated_tests__lists.py -s samples/ --timeout-for-run 500 --visit-only-specified-source --timeout 10000 --do-not-check-requirements -$java_path -jar utbot-cli.jar generate_python samples/list_of_datetime.py -p $python_path -o cli_utbot_tests/generated_tests__list_of_datetime.py -s samples/ --timeout-for-run 500 --visit-only-specified-source --timeout 10000 --do-not-check-requirements -$java_path -jar utbot-cli.jar generate_python samples/longest_subsequence.py -p $python_path -o cli_utbot_tests/generated_tests__longest_subsequence.py -s samples/ --timeout-for-run 500 --visit-only-specified-source --timeout 10000 --do-not-check-requirements -$java_path -jar utbot-cli.jar generate_python samples/matrix.py -p $python_path -o cli_utbot_tests/generated_tests__matrix.py -s samples/ --timeout-for-run 500 --visit-only-specified-source -c Matrix -m __add__,__mul__,__matmul__ --timeout 10000 --do-not-check-requirements -$java_path -jar utbot-cli.jar generate_python samples/primitive_types.py -p $python_path -o cli_utbot_tests/generated_tests__primitive_types.py -s samples/ --timeout-for-run 500 --visit-only-specified-source --timeout 10000 --do-not-check-requirements -$java_path -jar utbot-cli.jar generate_python samples/quick_sort.py -p $python_path -o cli_utbot_tests/generated_tests__quick_sort.py -s samples/ --timeout-for-run 500 --visit-only-specified-source --timeout 10000 --do-not-check-requirements -$java_path -jar utbot-cli.jar generate_python samples/test_coverage.py -p $python_path -o cli_utbot_tests/generated_tests__test_coverage.py -s samples/ --timeout-for-run 500 --visit-only-specified-source --timeout 10000 --do-not-check-requirements -$java_path -jar utbot-cli.jar generate_python samples/type_inference.py -p $python_path -o cli_utbot_tests/generated_tests__type_inference.py -s samples/ --timeout-for-run 500 --visit-only-specified-source --timeout 10000 --do-not-check-requirements -$java_path -jar utbot-cli.jar generate_python samples/using_collections.py -p $python_path -o cli_utbot_tests/generated_tests__using_collections.py -s samples/ --timeout-for-run 500 --visit-only-specified-source --timeout 10000 --do-not-check-requirements -$java_path -jar utbot-cli.jar generate_python samples/dummy_without_eq.py -p $python_path -o cli_utbot_tests/generated_tests__dummy_without_eq.py -s samples/ --timeout-for-run 500 --visit-only-specified-source --timeout 10000 -c Dummy -m propagate --do-not-check-requirements -$java_path -jar utbot-cli.jar generate_python samples/dummy_with_eq.py -p $python_path -o cli_utbot_tests/generated_tests__dummy_with_eq.py -s samples/ --timeout-for-run 500 --visit-only-specified-source --timeout 10000 -c Dummy -m propagate --do-not-check-requirements -$java_path -jar utbot-cli.jar generate_python samples/list_of_datetime.py -p $python_path -o cli_utbot_tests/generated_tests__list_of_datetime.py -s samples/ --timeout-for-run 500 --visit-only-specified-source --timeout 10000 --do-not-check-requirements diff --git a/utbot-python/samples/run_test_samples.sh b/utbot-python/samples/run_test_samples.sh deleted file mode 100755 index bbbf1bb171..0000000000 --- a/utbot-python/samples/run_test_samples.sh +++ /dev/null @@ -1,24 +0,0 @@ -# Usage: -# ./run_test_samples.sh - -python_path=$1 -java_path=$2 - -# $java_path -jar utbot-cli.jar --verbosity DEBUG run_python cli_utbot_tests/generated_tests__arithmetic.py -p $python_path -# $java_path -jar utbot-cli.jar --verbosity DEBUG run_python cli_utbot_tests/generated_tests__deep_equals.py -p $python_path -# $java_path -jar utbot-cli.jar --verbosity DEBUG run_python cli_utbot_tests/generated_tests__dicts.py -p $python_path -# $java_path -jar utbot-cli.jar --verbosity DEBUG run_python cli_utbot_tests/generated_tests__deque.py -p $python_path -# $java_path -jar utbot-cli.jar --verbosity DEBUG run_python cli_utbot_tests/generated_tests__dummy_with_eq.py -p $python_path -# $java_path -jar utbot-cli.jar --verbosity DEBUG run_python cli_utbot_tests/generated_tests__dummy_without_eq.py -p $python_path -# $java_path -jar utbot-cli.jar --verbosity DEBUG run_python cli_utbot_tests/generated_tests__lists.py -p $python_path -# $java_path -jar utbot-cli.jar --verbosity DEBUG run_python cli_utbot_tests/generated_tests__list_of_datetime.py -p $python_path -# $java_path -jar utbot-cli.jar --verbosity DEBUG run_python cli_utbot_tests/generated_tests__longest_subsequence.py -p $python_path -$java_path -jar utbot-cli.jar --verbosity DEBUG run_python cli_utbot_tests/generated_tests__matrix.py -p $python_path -# $java_path -jar utbot-cli.jar --verbosity DEBUG run_python cli_utbot_tests/generated_tests__primitive_types.py -p $python_path -# $java_path -jar utbot-cli.jar --verbosity DEBUG run_python cli_utbot_tests/generated_tests__quick_sort.py -p $python_path -# $java_path -jar utbot-cli.jar --verbosity DEBUG run_python cli_utbot_tests/generated_tests__test_coverage.py -p $python_path -# $java_path -jar utbot-cli.jar --verbosity DEBUG run_python cli_utbot_tests/generated_tests__type_inference.py -p $python_path -# $java_path -jar utbot-cli.jar --verbosity DEBUG run_python cli_utbot_tests/generated_tests__using_collections.py -p $python_path -# $java_path -jar utbot-cli.jar --verbosity DEBUG run_python cli_utbot_tests/generated_tests__dummy_with_eq.py -p $python_path -# $java_path -jar utbot-cli.jar --verbosity DEBUG run_python cli_utbot_tests/generated_tests__dummy_without_eq.py -p $python_path -# $java_path -jar utbot-cli.jar --verbosity DEBUG run_python cli_utbot_tests/generated_tests__list_of_datetime.py -p $python_path diff --git a/utbot-python/samples/samples/collection/using_collections.py b/utbot-python/samples/samples/collection/using_collections.py index debdc31bef..e8a5e9e1ae 100644 --- a/utbot-python/samples/samples/collection/using_collections.py +++ b/utbot-python/samples/samples/collection/using_collections.py @@ -3,6 +3,8 @@ def generate_collections(collection): collection[0] = 100 + if isinstance(collection, collections.UserDict): + return collection.data elements = list(collection.items()) return [ collection, diff --git a/utbot-python/samples/samples/named_arguments/method_named_arguments.py b/utbot-python/samples/samples/named_arguments/method_named_arguments.py index 10a61f8f63..f86c3d623a 100644 --- a/utbot-python/samples/samples/named_arguments/method_named_arguments.py +++ b/utbot-python/samples/samples/named_arguments/method_named_arguments.py @@ -1,4 +1,5 @@ -from samples.named_arguments.named_arguments import g +def g(x): + return x ** 2 class A: diff --git a/utbot-python/samples/testing_utils/collect_executions.py b/utbot-python/samples/testing_utils/collect_executions.py new file mode 100644 index 0000000000..81404aa023 --- /dev/null +++ b/utbot-python/samples/testing_utils/collect_executions.py @@ -0,0 +1,34 @@ +import json +import pathlib +import sys +import typing + + +def get_all_files(folder: pathlib.Path) -> typing.List[pathlib.Path]: + if folder.is_dir: + return folder.glob("*.executions") + return [] + + +def get_excecutions_number(file: pathlib.Path) -> int: + with open(file, 'r', encoding='utf-8') as fin: + return int(json.loads(fin.readline().strip())['executions']) + + +def save_data(data: typing.Dict[pathlib.Path, int]) -> None: + with open('data_executions.json', 'w', encoding='utf-8') as fout: + print(json.dumps(data, indent=1), file=fout) + + +def main(folder: pathlib.Path) -> None: + files = get_all_files(folder) + data = { + str(file): get_excecutions_number(file) + for file in files + } + save_data(data) + + +if __name__ == '__main__': + main(pathlib.Path(sys.argv[1])) + diff --git a/utbot-python/samples/testing_utils/collect_timeouts.py b/utbot-python/samples/testing_utils/collect_timeouts.py new file mode 100644 index 0000000000..bbcb8241a2 --- /dev/null +++ b/utbot-python/samples/testing_utils/collect_timeouts.py @@ -0,0 +1,41 @@ +import json +import pathlib +import sys +import typing + +def read_test_config(path: pathlib.Path, coverage_dir: pathlib.Path) -> typing.Dict[str, int]: + with open(path, 'r', encoding='utf-8') as fin: + data = json.loads(fin.read()) + + res = {} + for part in data['parts']: + for file in part['files']: + for group in file['groups']: + file_suffix = f"{part['path'].replace('/', '_')}_{file['name']}" + executions_output_file = pathlib.Path(coverage_dir, f"coverage_{file_suffix}.json.executions") + timeout = group['timeout'] + res[executions_output_file] = timeout + return res + + +def read_executions(path: pathlib.Path): + with open(path, 'r', encoding='utf-8') as fin: + data = json.loads(fin.read()) + return data + + +def main(config_path: pathlib.Path, executions_path: pathlib.Path, coverage_dir: pathlib.Path): + timeouts = read_test_config(config_path, coverage_dir) + executions = read_executions(executions_path) + + for f, timeout in timeouts.items(): + executions[str(f)] /= timeout + + with open('speeds.json', 'w') as fout: + print(json.dumps(executions, indent=1), file=fout) + + +if __name__ == '__main__': + main(*sys.argv[1:]) + + diff --git a/utbot-python/samples/testing_utils/compare.py b/utbot-python/samples/testing_utils/compare.py new file mode 100644 index 0000000000..cb45d0cc49 --- /dev/null +++ b/utbot-python/samples/testing_utils/compare.py @@ -0,0 +1,25 @@ +import json +import matplotlib.pyplot as plt +import pathlib + + +def read_stats(path: pathlib.Path): + with open(path, 'r') as fin: + return json.loads(fin.read()) + + +data1 = read_stats('speeds_basic.json') +data2 = read_stats('speeds_forks.json') +data3 = read_stats('speeds_forks2.json') +data = [list(d.values()) for d in [data1, data2, data3]] + +fig = plt.figure() +ax = fig.add_subplot(1, 1, 1) +ax.boxplot( + data, + labels=['basic', 'forks', 'forks2'], + vert=True, + patch_artist=True + ) +ax.grid(True) +plt.show() diff --git a/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt b/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt index 6899ad1265..cc0730bffe 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt @@ -20,8 +20,13 @@ import org.utbot.python.framework.api.python.PythonUtExecution import org.utbot.python.fuzzing.* import org.utbot.python.newtyping.PythonTypeHintsStorage import org.utbot.python.newtyping.general.UtType +import org.utbot.python.newtyping.inference.InvalidTypeFeedback +import org.utbot.python.newtyping.inference.SuccessFeedback +import org.utbot.python.newtyping.inference.baseline.BaselineAlgorithm import org.utbot.python.newtyping.pythonModules import org.utbot.python.newtyping.pythonTypeRepresentation +import org.utbot.python.utils.ExecutionWithTimoutMode +import org.utbot.python.utils.TestGenerationLimitManager import org.utbot.python.utils.camelToSnakeCase import org.utbot.summary.fuzzer.names.TestSuggestedInfo import java.net.ServerSocket @@ -120,6 +125,7 @@ class PythonEngine( ) return ValidExecution(utFuzzedExecution) } + private fun handleSuccessResult( arguments: List, types: List, @@ -203,8 +209,7 @@ class PythonEngine( .filterNot { it.startsWith(moduleToImport) } val localAdditionalModules = (additionalModules + argumentModules + moduleToImport).toSet() - val (thisObject, modelList) = - if (methodUnderTest.hasThisArgument) + val (thisObject, modelList) = if (methodUnderTest.hasThisArgument) Pair(argumentValues[0], argumentValues.drop(1)) else Pair(null, argumentValues) @@ -219,11 +224,13 @@ class PythonEngine( return when (val evaluationResult = manager.runWithCoverage(functionArguments, localAdditionalModules, coverageId)) { is PythonEvaluationError -> { + val stackTraceMessage = evaluationResult.stackTrace.joinToString("\n") val utError = UtError( - "Error evaluation: ${evaluationResult.status}, ${evaluationResult.message}", - Throwable(evaluationResult.stackTrace.joinToString("\n")) + "Error evaluation: ${evaluationResult.status}, ${evaluationResult.message}\n${stackTraceMessage}", + Throwable(stackTraceMessage) ) - logger.debug(evaluationResult.stackTrace.joinToString("\n")) + description.limitManager.addInvalidExecution() + logger.debug(stackTraceMessage) PythonExecutionResult(InvalidExecution(utError), PythonFeedback(control = Control.PASS)) } @@ -237,33 +244,39 @@ class PythonEngine( Trie.emptyNode() else description.tracer.add(coveredInstructions) + description.limitManager.addInvalidExecution() PythonExecutionResult( utTimeoutException, - PythonFeedback(control = Control.PASS, result = trieNode) + PythonFeedback(control = Control.PASS, result = trieNode, SuccessFeedback) ) } is PythonEvaluationSuccess -> { val coveredInstructions = evaluationResult.coverage.coveredInstructions - when (val result = handleSuccessResult( - arguments, - parameters, - evaluationResult, - description, - )) { + val result = handleSuccessResult( + arguments, + parameters, + evaluationResult, + description, + ) + val typeInferenceFeedback = if (result is ValidExecution) SuccessFeedback else InvalidTypeFeedback + when (result) { is ValidExecution -> { val trieNode: Trie.Node = description.tracer.add(coveredInstructions) + description.limitManager.addSuccessExecution() PythonExecutionResult( result, - PythonFeedback(control = Control.CONTINUE, result = trieNode) + PythonFeedback(Control.CONTINUE, trieNode, typeInferenceFeedback) ) } is InvalidExecution -> { - PythonExecutionResult(result, PythonFeedback(control = Control.CONTINUE)) + description.limitManager.addInvalidExecution() + PythonExecutionResult(result, PythonFeedback(control = Control.CONTINUE, typeInferenceFeedback = typeInferenceFeedback)) } else -> { - PythonExecutionResult(result, PythonFeedback(control = Control.PASS)) + description.limitManager.addInvalidExecution() + PythonExecutionResult(result, PythonFeedback(control = Control.PASS, typeInferenceFeedback = typeInferenceFeedback)) } } } @@ -274,7 +287,12 @@ class PythonEngine( } } - fun fuzzing(parameters: List, isCancelled: () -> Boolean, until: Long): Flow = flow { + fun fuzzing( + parameters: List, + typeInferenceAlgorithm: BaselineAlgorithm, + isCancelled: () -> Boolean, + until: Long + ): Flow = flow { ServerSocket(0).use { serverSocket -> logger.debug { "Server port: ${serverSocket.localPort}" } val manager = try { @@ -295,6 +313,8 @@ class PythonEngine( pythonTypeStorage, Trie(Instruction::id), Random(0), + TestGenerationLimitManager(ExecutionWithTimoutMode, until, isRootManager = true), + methodUnderTest.definition.type, ) try { @@ -305,7 +325,7 @@ class PythonEngine( } } else { try { - PythonFuzzing(pmd.pythonTypeStorage) { description, arguments -> + PythonFuzzing(pythonTypeStorage, typeInferenceAlgorithm) { description, arguments -> if (isCancelled()) { logger.debug { "Fuzzing process was interrupted" } manager.disconnect() @@ -319,6 +339,7 @@ class PythonEngine( if (arguments.any { PythonTree.containsFakeNode(it.tree) }) { logger.debug { "FakeNode in Python model" } + description.limitManager.addFakeNodeExecutions() emit(FakeNodeFeedback) return@PythonFuzzing PythonFeedback(control = Control.CONTINUE) } @@ -326,9 +347,10 @@ class PythonEngine( val pair = Pair(description, arguments.map { PythonTreeWrapper(it.tree) }) val mem = cache.get(pair) if (mem != null) { - logger.debug("Repeat in fuzzing") + logger.debug("Repeat in fuzzing ${arguments.map {it.tree}}") + description.limitManager.addSuccessExecution() emit(CachedExecutionFeedback(mem.fuzzingExecutionFeedback)) - return@PythonFuzzing mem.fuzzingPlatformFeedback + return@PythonFuzzing mem.fuzzingPlatformFeedback.fromCache() } val result = fuzzingResultHandler(description, arguments, parameters, manager) if (result == null) { // timeout diff --git a/utbot-python/src/main/kotlin/org/utbot/python/PythonTestCaseGenerator.kt b/utbot-python/src/main/kotlin/org/utbot/python/PythonTestCaseGenerator.kt index 1500a6b801..f6ad5c1419 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/PythonTestCaseGenerator.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/PythonTestCaseGenerator.kt @@ -14,9 +14,6 @@ import org.utbot.python.newtyping.ast.visitor.Visitor import org.utbot.python.newtyping.ast.visitor.constants.ConstantCollector import org.utbot.python.newtyping.ast.visitor.hints.HintCollector import org.utbot.python.newtyping.general.* -import org.utbot.python.newtyping.inference.InferredTypeFeedback -import org.utbot.python.newtyping.inference.InvalidTypeFeedback -import org.utbot.python.newtyping.inference.SuccessFeedback import org.utbot.python.newtyping.inference.baseline.BaselineAlgorithm import org.utbot.python.newtyping.mypy.GlobalNamesStorage import org.utbot.python.newtyping.mypy.MypyInfoBuild @@ -24,7 +21,6 @@ import org.utbot.python.newtyping.mypy.MypyReportLine import org.utbot.python.newtyping.mypy.getErrorNumber import org.utbot.python.newtyping.utils.getOffsetLine import org.utbot.python.newtyping.utils.isRequired -import org.utbot.python.utils.ExecutionWithTimeoutMode import org.utbot.python.utils.TestGenerationLimitManager import org.utbot.python.utils.PriorityCartesianProduct import org.utbot.python.utils.TimeoutMode @@ -134,7 +130,7 @@ class PythonTestCaseGenerator( additionalVars: String = "", ): Set? { // returns missing lines val limitManager = TestGenerationLimitManager( - ExecutionWithTimeoutMode, + TimeoutMode, until, ) var missingLines = initMissingLines @@ -150,52 +146,66 @@ class PythonTestCaseGenerator( PythonFuzzedConcreteValue(type, value) } - inferAnnotations( + val engine = PythonEngine( method, - mypyStorage, - typeStorage, - hintCollector, - mypyReportLine, - limitManager, - additionalVars - ) { functionType -> - val args = (functionType as FunctionType).arguments - - logger.debug { "Inferred annotations: ${args.joinToString { it.pythonTypeRepresentation() }}" } - - val engine = PythonEngine( - method, - directoriesForSysPath, - curModule, - pythonPath, - constants, - timeoutForRun, - PythonTypeHintsStorage.get(mypyStorage) - ) + directoriesForSysPath, + curModule, + pythonPath, + constants, + timeoutForRun, + PythonTypeHintsStorage.get(mypyStorage) + ) + val namesInModule = mypyStorage.names + .getOrDefault(curModule, emptyList()) + .map { it.name } + .filter { + it.length < 4 || !it.startsWith("__") || !it.endsWith("__") + } - var feedback: InferredTypeFeedback = SuccessFeedback + val algo = BaselineAlgorithm( + typeStorage, + hintCollector.result, + pythonPath, + method, + directoriesForSysPath, + curModule, + namesInModule, + getErrorNumber( + mypyReportLine, + fileOfMethod, + getOffsetLine(sourceFileContent, method.ast.beginOffset), + getOffsetLine(sourceFileContent, method.ast.endOffset) + ), + mypyStorage.buildRoot.configFile, + additionalVars, + randomTypeFrequency = RANDOM_TYPE_FREQUENCY, + dMypyTimeout = timeoutForRun, + ) - val fuzzerCancellation = { isCancelled() || limitManager.isCancelled() } + val fuzzerCancellation = { isCancelled() || limitManager.isCancelled() } - engine.fuzzing(args, fuzzerCancellation, until).collect { + val initFunctionType = method.definition.type.arguments + runBlocking { + engine.fuzzing( + initFunctionType, + algo, + fuzzerCancellation, + until + ).collect { when (it) { is ValidExecution -> { executions += it.utFuzzedExecution missingLines = updateMissingLines(it.utFuzzedExecution, coveredLines, missingLines) - feedback = SuccessFeedback limitManager.addSuccessExecution() } is InvalidExecution -> { errors += it.utError - feedback = InvalidTypeFeedback limitManager.addInvalidExecution() } is ArgumentsTypeErrorFeedback -> { - feedback = InvalidTypeFeedback limitManager.addInvalidExecution() } is TypeErrorFeedback -> { - feedback = InvalidTypeFeedback limitManager.addInvalidExecution() } is CachedExecutionFeedback -> { @@ -209,14 +219,13 @@ class PythonTestCaseGenerator( } } is FakeNodeFeedback -> { - limitManager.addFakeNodeExecutions() + limitManager.addFakeNodeExecutions() } } limitManager.missedLines = missingLines?.size } - limitManager.restart() - feedback } + return missingLines } @@ -270,7 +279,8 @@ class PythonTestCaseGenerator( else coverageExecutions + emptyCoverageExecutions.take(MAX_EMPTY_COVERAGE_TESTS), errors, - storageForMypyMessages + storageForMypyMessages, + executionsNumber = executions.size, ) } @@ -291,58 +301,6 @@ class PythonTestCaseGenerator( return if (missingLines == null) curMissing else missingLines intersect curMissing } - private fun inferAnnotations( - method: PythonMethod, - mypyStorage: MypyInfoBuild, - typeStorage: PythonTypeHintsStorage, - hintCollector: HintCollector, - report: List, - limitManager: TestGenerationLimitManager, - additionalVars: String, - annotationHandler: suspend (UtType) -> InferredTypeFeedback, - ) { - val namesInModule = mypyStorage.names - .getOrDefault(curModule, emptyList()) - .map { it.name } - .filter { - it.length < 4 || !it.startsWith("__") || !it.endsWith("__") - } - val typeInferenceCancellation = { isCancelled() || limitManager.isCancelled() } - - val algo = BaselineAlgorithm( - typeStorage, - pythonPath, - method, - directoriesForSysPath, - curModule, - namesInModule, - getErrorNumber( - report, - fileOfMethod, - getOffsetLine(sourceFileContent, method.ast.beginOffset), - getOffsetLine(sourceFileContent, method.ast.endOffset) - ), - mypyStorage.buildRoot.configFile, - additionalVars, - randomTypeFrequency = RANDOM_TYPE_FREQUENCY, - dMypyTimeout = timeoutForRun - ) - - runBlocking breaking@{ - if (typeInferenceCancellation()) { - return@breaking - } - - val iterationNumber = algo.run(hintCollector.result, typeInferenceCancellation, annotationHandler) - - if (iterationNumber == 1) { // Initial annotation can't be substituted - limitManager.mode = TimeoutMode - val existsAnnotation = method.definition.type - annotationHandler(existsAnnotation) - } - } - } - companion object { fun createShortForm(method: PythonMethod): Pair? { val meta = method.definition.type.pythonDescription() as PythonCallableTypeDescription diff --git a/utbot-python/src/main/kotlin/org/utbot/python/UTPythonAPI.kt b/utbot-python/src/main/kotlin/org/utbot/python/UTPythonAPI.kt index 4b02bfa386..57da4cc17d 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/UTPythonAPI.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/UTPythonAPI.kt @@ -71,6 +71,7 @@ data class PythonTestSet( val errors: List, val mypyReport: List, val classId: PythonClassId? = null, + val executionsNumber: Int = 0, ) data class FunctionArguments( diff --git a/utbot-python/src/main/kotlin/org/utbot/python/framework/codegen/model/constructor/tree/PythonTestFrameworkManager.kt b/utbot-python/src/main/kotlin/org/utbot/python/framework/codegen/model/constructor/tree/PythonTestFrameworkManager.kt index 6cbfbb5096..8726c9af13 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/framework/codegen/model/constructor/tree/PythonTestFrameworkManager.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/framework/codegen/model/constructor/tree/PythonTestFrameworkManager.kt @@ -56,13 +56,13 @@ internal class PytestManager(context: CgContext) : TestFrameworkManager(context) override fun disableTestMethod(reason: String) { require(testFramework is Pytest) { "According to settings, Pytest was expected, but got: $testFramework" } - context.importIfNeeded(testFramework.skipDecoratorClassId) + context.importIfNeeded(Pytest.skipDecoratorClassId) val reasonArgument = CgNamedAnnotationArgument( name = "reason", value = CgPythonRepr(pythonStrClassId, "'${reason.replace("\"", "'")}'"), ) statementConstructor.addAnnotation( - classId = testFramework.skipDecoratorClassId, + classId = Pytest.skipDecoratorClassId, namedArguments = listOf(reasonArgument), target = Method ) @@ -145,11 +145,10 @@ internal class UnittestManager(context: CgContext) : TestFrameworkManager(contex value = CgPythonRepr(pythonStrClassId, "'${reason.replace("\"", "'")}'"), ) statementConstructor.addAnnotation( - classId = testFramework.skipDecoratorClassId, + classId = Unittest.skipDecoratorClassId, namedArguments = listOf(reasonArgument), target = Method, - - ) + ) } fun assertIsinstance(types: List, actual: CgVariable) { diff --git a/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/PythonApi.kt b/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/PythonApi.kt index 6f6f5c98ad..00c3fb48e2 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/PythonApi.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/PythonApi.kt @@ -11,7 +11,15 @@ import org.utbot.python.framework.api.python.PythonUtExecution import org.utbot.python.fuzzing.provider.* import org.utbot.python.fuzzing.provider.utils.isAny import org.utbot.python.newtyping.* +import org.utbot.python.newtyping.general.FunctionType import org.utbot.python.newtyping.general.UtType +import org.utbot.python.newtyping.inference.InferredTypeFeedback +import org.utbot.python.newtyping.inference.InvalidTypeFeedback +import org.utbot.python.newtyping.inference.SuccessFeedback +import org.utbot.python.newtyping.inference.baseline.BaselineAlgorithm +import org.utbot.python.utils.ExecutionWithTimoutMode +import org.utbot.python.utils.TestGenerationLimitManager +import org.utbot.python.utils.TimeoutMode import kotlin.random.Random private val logger = KotlinLogging.logger {} @@ -29,6 +37,8 @@ class PythonMethodDescription( val pythonTypeStorage: PythonTypeHintsStorage, val tracer: Trie, val random: Random, + val limitManager: TestGenerationLimitManager, + val type: FunctionType, ) : Description(parameters) sealed interface FuzzingExecutionFeedback @@ -47,7 +57,18 @@ data class PythonExecutionResult( data class PythonFeedback( override val control: Control = Control.CONTINUE, val result: Trie.Node = Trie.emptyNode(), -) : Feedback + val typeInferenceFeedback: InferredTypeFeedback = InvalidTypeFeedback, + val fromCache: Boolean = false, +) : Feedback { + fun fromCache(): PythonFeedback { + return PythonFeedback( + control = control, + result = result, + typeInferenceFeedback = typeInferenceFeedback, + fromCache = true, + ) + } +} class PythonFuzzedValue( val tree: PythonTree.PythonTreeNode, @@ -76,8 +97,21 @@ fun pythonDefaultValueProviders(typeStorage: PythonTypeHintsStorage) = listOf( SubtypeValueProvider(typeStorage) ) +fun pythonAnyTypeValueProviders() = listOf( + NoneValueProvider, + BoolValueProvider, + IntValueProvider, + FloatValueProvider, + ComplexValueProvider, + StrValueProvider, + BytesValueProvider, + BytearrayValueProvider, + ConstantValueProvider, +) + class PythonFuzzing( private val pythonTypeStorage: PythonTypeHintsStorage, + private val typeInferenceAlgorithm: BaselineAlgorithm, val execute: suspend (description: PythonMethodDescription, values: List) -> PythonFeedback, ) : Fuzzing { @@ -103,6 +137,50 @@ class PythonFuzzing( } override suspend fun handle(description: PythonMethodDescription, values: List): PythonFeedback { - return execute(description, values) + val result = execute(description, values) + if (result.typeInferenceFeedback is SuccessFeedback && !result.fromCache) { + typeInferenceAlgorithm.laudType(description.type) + } + if (description.limitManager.isCancelled()) { + typeInferenceAlgorithm.feedbackState(description.type, result.typeInferenceFeedback) + } + return result + } + + private suspend fun forkType(description: PythonMethodDescription, stats: Statistic) { + val type: UtType? = typeInferenceAlgorithm.expandState() + if (type != null) { + val newTypes = (type as FunctionType).arguments + val d = PythonMethodDescription( + description.name, + newTypes, + description.concreteValues, + description.pythonTypeStorage, + description.tracer, + description.random, + TestGenerationLimitManager(ExecutionWithTimoutMode, description.limitManager.until), + type + ) + if (!d.limitManager.isCancelled()) { + logger.debug { "Fork new type" } + fork(d, stats) + } + logger.debug { "Fork ended" } + } else { + description.limitManager.mode = TimeoutMode + } + } + + override suspend fun isCancelled( + description: PythonMethodDescription, + stats: Statistic + ): Boolean { + if (description.limitManager.isCancelled() || description.parameters.any { it.isAny() }) { + forkType(description, stats) + if (description.limitManager.isRootManager) { + return TimeoutMode.isCancelled(description.limitManager) + } + } + return description.limitManager.isCancelled() } } diff --git a/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/BoolValueProvider.kt b/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/BoolValueProvider.kt index 929ff35dae..17dbf7c2ef 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/BoolValueProvider.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/BoolValueProvider.kt @@ -9,13 +9,12 @@ import org.utbot.python.framework.api.python.util.pythonBoolClassId import org.utbot.python.fuzzing.PythonFuzzedValue import org.utbot.python.fuzzing.PythonMethodDescription import org.utbot.python.fuzzing.provider.utils.generateSummary -import org.utbot.python.fuzzing.provider.utils.isAny import org.utbot.python.newtyping.general.UtType import org.utbot.python.newtyping.pythonTypeName object BoolValueProvider : ValueProvider{ override fun accept(type: UtType): Boolean { - return type.pythonTypeName() == pythonBoolClassId.canonicalName || type.isAny() + return type.pythonTypeName() == pythonBoolClassId.canonicalName } override fun generate(description: PythonMethodDescription, type: UtType) = sequence { diff --git a/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/ComplexValueProvider.kt b/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/ComplexValueProvider.kt index c8e910e61a..c756cd20eb 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/ComplexValueProvider.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/ComplexValueProvider.kt @@ -7,22 +7,28 @@ import org.utbot.python.framework.api.python.PythonTree import org.utbot.python.framework.api.python.util.pythonComplexClassId import org.utbot.python.fuzzing.PythonFuzzedValue import org.utbot.python.fuzzing.PythonMethodDescription -import org.utbot.python.fuzzing.provider.utils.isAny +import org.utbot.python.newtyping.createPythonUnionType import org.utbot.python.newtyping.general.UtType import org.utbot.python.newtyping.pythonTypeName import org.utbot.python.newtyping.pythonTypeRepresentation object ComplexValueProvider : ValueProvider { override fun accept(type: UtType): Boolean { - return type.pythonTypeName() == pythonComplexClassId.canonicalName || type.isAny() + return type.pythonTypeName() == pythonComplexClassId.canonicalName } override fun generate(description: PythonMethodDescription, type: UtType) = sequence { + val numberType = createPythonUnionType( + listOf( + description.pythonTypeStorage.pythonFloat, + description.pythonTypeStorage.pythonInt + ) + ) yield(Seed.Recursive( construct = Routine.Create( listOf( - description.pythonTypeStorage.pythonFloat, - description.pythonTypeStorage.pythonFloat, + numberType, + numberType ) ) { v -> val real = v[0].tree as PythonTree.PrimitiveNode diff --git a/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/ConstantValueProvider.kt b/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/ConstantValueProvider.kt index 2194d13106..794c9c02d5 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/ConstantValueProvider.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/ConstantValueProvider.kt @@ -6,14 +6,13 @@ import org.utbot.python.framework.api.python.PythonClassId import org.utbot.python.framework.api.python.PythonTree import org.utbot.python.fuzzing.PythonFuzzedValue import org.utbot.python.fuzzing.PythonMethodDescription -import org.utbot.python.fuzzing.provider.utils.isAny import org.utbot.python.newtyping.general.UtType import org.utbot.python.newtyping.pythonTypeName import org.utbot.python.fuzzing.value.TypesFromJSONStorage object ConstantValueProvider : ValueProvider { override fun accept(type: UtType): Boolean { - return TypesFromJSONStorage.getTypesFromJsonStorage().containsKey(type.pythonTypeName()) || type.isAny() + return TypesFromJSONStorage.getTypesFromJsonStorage().containsKey(type.pythonTypeName()) } override fun generate(description: PythonMethodDescription, type: UtType): Sequence> = diff --git a/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/FloatValueProvider.kt b/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/FloatValueProvider.kt index 94ed272496..496ac4b67a 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/FloatValueProvider.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/FloatValueProvider.kt @@ -10,7 +10,6 @@ import org.utbot.python.fuzzing.PythonFuzzedConcreteValue import org.utbot.python.fuzzing.PythonFuzzedValue import org.utbot.python.fuzzing.PythonMethodDescription import org.utbot.python.fuzzing.provider.utils.generateSummary -import org.utbot.python.fuzzing.provider.utils.isAny import org.utbot.python.newtyping.general.UtType import org.utbot.python.newtyping.pythonTypeName import java.math.BigDecimal @@ -18,7 +17,7 @@ import java.math.BigInteger object FloatValueProvider : ValueProvider { override fun accept(type: UtType): Boolean { - return type.pythonTypeName() == pythonFloatClassId.canonicalName || type.isAny() + return type.pythonTypeName() == pythonFloatClassId.canonicalName } private fun getFloatConstants(concreteValues: Collection): List { @@ -44,7 +43,7 @@ object FloatValueProvider : ValueProvider> = sequence { val floatConstants = getFloatConstants(description.concreteValues) val intConstants = getIntConstants(description.concreteValues) - val constants = floatConstants + intConstants + val constants = floatConstants + intConstants + listOf(0, 1).map { IEEE754Value.fromValue(it.toDouble()) } constants.asSequence().forEach { value -> yield(Seed.Known(value) { diff --git a/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/IntValueProvider.kt b/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/IntValueProvider.kt index 38cb19a008..cf4f6d36dc 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/IntValueProvider.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/IntValueProvider.kt @@ -13,7 +13,6 @@ import org.utbot.python.fuzzing.PythonFuzzedConcreteValue import org.utbot.python.fuzzing.PythonFuzzedValue import org.utbot.python.fuzzing.PythonMethodDescription import org.utbot.python.fuzzing.provider.utils.generateSummary -import org.utbot.python.fuzzing.provider.utils.isAny import org.utbot.python.newtyping.general.UtType import org.utbot.python.newtyping.pythonTypeName import java.math.BigInteger @@ -24,7 +23,7 @@ object IntValueProvider : ValueProvider Unit): BitVectorValue { diff --git a/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/NoneValueProvider.kt b/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/NoneValueProvider.kt index 7cd515d3f8..9d16d2db3f 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/NoneValueProvider.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/NoneValueProvider.kt @@ -5,13 +5,12 @@ import org.utbot.fuzzing.ValueProvider import org.utbot.python.framework.api.python.PythonTree import org.utbot.python.fuzzing.PythonFuzzedValue import org.utbot.python.fuzzing.PythonMethodDescription -import org.utbot.python.fuzzing.provider.utils.isAny import org.utbot.python.newtyping.PythonNoneTypeDescription import org.utbot.python.newtyping.general.UtType object NoneValueProvider : ValueProvider { override fun accept(type: UtType): Boolean { - return type.isAny() || type.meta is PythonNoneTypeDescription + return type.meta is PythonNoneTypeDescription } override fun generate(description: PythonMethodDescription, type: UtType): Sequence> = sequence { diff --git a/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/ReduceValueProvider.kt b/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/ReduceValueProvider.kt index 0af926c17c..5bcf23358a 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/ReduceValueProvider.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/ReduceValueProvider.kt @@ -8,6 +8,7 @@ import org.utbot.python.framework.api.python.PythonTree import org.utbot.python.framework.api.python.util.* import org.utbot.python.fuzzing.PythonFuzzedValue import org.utbot.python.fuzzing.PythonMethodDescription +import org.utbot.python.fuzzing.provider.utils.isAny import org.utbot.python.fuzzing.provider.utils.isCallable import org.utbot.python.fuzzing.provider.utils.isConcreteType import org.utbot.python.fuzzing.provider.utils.isMagic @@ -37,7 +38,7 @@ object ReduceValueProvider : ValueProvider Boolean, annotationHandler: suspend (UtType) -> InferredTypeFeedback, ): Int diff --git a/utbot-python/src/main/kotlin/org/utbot/python/newtyping/inference/TypeInferenceProcessor.kt b/utbot-python/src/main/kotlin/org/utbot/python/newtyping/inference/TypeInferenceProcessor.kt index 0947489929..be372bde78 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/newtyping/inference/TypeInferenceProcessor.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/newtyping/inference/TypeInferenceProcessor.kt @@ -90,6 +90,7 @@ class TypeInferenceProcessor( val algo = BaselineAlgorithm( typeStorage, + collector.result, pythonPath, pythonMethod, directoriesForSysPath, @@ -109,7 +110,7 @@ class TypeInferenceProcessor( startingTypeInferenceAction() val annotations = emptyList().toMutableList() runBlocking { - algo.run(collector.result, cancel) { + algo.run(cancel) { annotations.add(it) processSignature(it) SuccessFeedback diff --git a/utbot-python/src/main/kotlin/org/utbot/python/newtyping/inference/baseline/BaselineAlgorithm.kt b/utbot-python/src/main/kotlin/org/utbot/python/newtyping/inference/baseline/BaselineAlgorithm.kt index 278c1f0af8..5272f0ed40 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/newtyping/inference/baseline/BaselineAlgorithm.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/newtyping/inference/baseline/BaselineAlgorithm.kt @@ -25,6 +25,7 @@ private val logger = KotlinLogging.logger {} class BaselineAlgorithm( private val storage: PythonTypeHintsStorage, + private val hintCollectorResult: HintCollectorResult, private val pythonPath: String, private val pythonMethodCopy: PythonMethod, private val directoriesForSysPath: Set, @@ -38,20 +39,98 @@ class BaselineAlgorithm( ) : TypeInferenceAlgorithm() { private val random = Random(0) + private val generalRating = createGeneralTypeRating(hintCollectorResult, storage) + private val initialState = getInitialState(hintCollectorResult, generalRating) + private val states: MutableList = mutableListOf(initialState) + private val fileForMypyRuns = TemporaryFileManager.assignTemporaryFile(tag = "mypy.py") + private var iterationCounter = 0 + private var randomTypeCounter = 0 + + private val simpleTypes = simplestTypes(storage) + private val mixtureType = createPythonUnionType(simpleTypes) + + private val openedStates: MutableMap> = mutableMapOf() + private val statistic: MutableMap = mutableMapOf() + + private fun getRandomType(): UtType? { + val weights = states.map { 1.0 / (it.anyNodes.size * it.anyNodes.size + 1) } + val state = weightedRandom(states, weights, random) + val newState = expandState(state, storage, state.anyNodes.map { mixtureType }) + if (newState != null) { + logger.info("Random type: ${newState.signature.pythonTypeRepresentation()}") + openedStates[newState.signature] = newState to state + return newState.signature + } + return null + } + + private fun getLaudedType(): UtType? { + if (statistic.isEmpty()) return null + val sum = statistic.values.sum() + val weights = statistic.values.map { it.toDouble() / sum } + val newType = weightedRandom(statistic.keys.toList(), weights, random) + logger.info("Lauded type: ${newType.pythonTypeRepresentation()}") + return newType + } + + fun expandState(): UtType? { + if (states.isEmpty()) return null + + logger.debug("State number: ${states.size}") + iterationCounter++ + + if (randomTypeFrequency > 0 && iterationCounter % randomTypeFrequency == 0) { + randomTypeCounter++ + if (randomTypeCounter % 2 == 0) { + val laudedType = getLaudedType() + if (laudedType != null) return laudedType + } + val randomType = getRandomType() + if (randomType != null) return randomType + } + + val state = chooseState(states) + val newState = expandState(state, storage) + if (newState != null) { + logger.info("Checking ${newState.signature.pythonTypeRepresentation()}") + if (checkSignature(newState.signature as FunctionType, fileForMypyRuns, configFile)) { + logger.debug("Found new state!") + openedStates[newState.signature] = newState to state + return newState.signature + } + } else if (state.anyNodes.isEmpty()) { + logger.info("Checking ${state.signature.pythonTypeRepresentation()}") + if (checkSignature(state.signature as FunctionType, fileForMypyRuns, configFile)) { + return state.signature + } else { + states.remove(state) + } + } else { + states.remove(state) + } + return expandState() + } + + fun feedbackState(signature: UtType, feedback: InferredTypeFeedback) { + val stateInfo = openedStates[signature] + if (stateInfo != null) { + val (newState, parent) = stateInfo + when (feedback) { + SuccessFeedback -> { + states.add(newState) + parent.children += 1 + } + InvalidTypeFeedback -> {} + } + openedStates.remove(signature) + } + } + + override suspend fun run( - hintCollectorResult: HintCollectorResult, isCancelled: () -> Boolean, annotationHandler: suspend (UtType) -> InferredTypeFeedback, ): Int { - val generalRating = createGeneralTypeRating(hintCollectorResult, storage) - val initialState = getInitialState(hintCollectorResult, generalRating) - val states: MutableList = mutableListOf(initialState) - val fileForMypyRuns = TemporaryFileManager.assignTemporaryFile(tag = "mypy.py") - var iterationCounter = 0 - - val simpleTypes = simplestTypes(storage) - val mixtureType = createPythonUnionType(simpleTypes) - run breaking@ { while (states.isNotEmpty()) { if (isCancelled()) @@ -78,8 +157,6 @@ class BaselineAlgorithm( logger.info("Checking ${newState.signature.pythonTypeRepresentation()}") if (checkSignature(newState.signature as FunctionType, fileForMypyRuns, configFile)) { logger.debug("Found new state!") -// annotationHandler(newState.signature) -// states.add(newState) when (annotationHandler(newState.signature)) { SuccessFeedback -> { states.add(newState) @@ -143,4 +220,8 @@ class BaselineAlgorithm( } return BaselineAlgorithmState(allNodes, generalRating, storage) } + + fun laudType(type: FunctionType) { + statistic[type] = statistic[type]?.plus(1) ?: 1 + } } \ No newline at end of file diff --git a/utbot-python/src/main/kotlin/org/utbot/python/utils/TestGenerationLimitManager.kt b/utbot-python/src/main/kotlin/org/utbot/python/utils/TestGenerationLimitManager.kt index b731bf3ffd..77b3354b58 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/utils/TestGenerationLimitManager.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/utils/TestGenerationLimitManager.kt @@ -8,17 +8,21 @@ class TestGenerationLimitManager( // local settings: one type inference iteration var executions: Int = 150, var invalidExecutions: Int = 10, - var fakeNodeExecutions: Int = 20, + var cacheNodeExecutions: Int = 20, + var fakeNodeExecutions: Int = 1, var missedLines: Int? = null, + val isRootManager: Boolean = false, ) { private val initExecution = executions private val initInvalidExecutions = invalidExecutions + private val initCacheNodeExecutions = cacheNodeExecutions private val initFakeNodeExecutions = fakeNodeExecutions private val initMissedLines = missedLines fun restart() { executions = initExecution invalidExecutions = initInvalidExecutions + cacheNodeExecutions = initCacheNodeExecutions fakeNodeExecutions = initFakeNodeExecutions missedLines = initMissedLines } @@ -58,7 +62,7 @@ object TimeoutMode : LimitManagerMode { object ExecutionMode : LimitManagerMode { override fun isCancelled(manager: TestGenerationLimitManager): Boolean { - return manager.invalidExecutions <= 0 || manager.executions <= 0 || manager.fakeNodeExecutions <= 0 + return manager.invalidExecutions <= 0 || manager.executions <= 0 || manager.fakeNodeExecutions <= 0 || manager.cacheNodeExecutions <= 0 } } @@ -68,7 +72,7 @@ object MaxCoverageWithTimeoutMode : LimitManagerMode { } } -object ExecutionWithTimeoutMode : LimitManagerMode { +object ExecutionWithTimoutMode : LimitManagerMode { override fun isCancelled(manager: TestGenerationLimitManager): Boolean { return ExecutionMode.isCancelled(manager) || TimeoutMode.isCancelled(manager) }