diff --git a/utbot-cli-python/src/main/kotlin/org/utbot/cli/language/python/PythonGenerateTestsCommand.kt b/utbot-cli-python/src/main/kotlin/org/utbot/cli/language/python/PythonGenerateTestsCommand.kt index 89448aebb9..43b081d4a7 100644 --- a/utbot-cli-python/src/main/kotlin/org/utbot/cli/language/python/PythonGenerateTestsCommand.kt +++ b/utbot-cli-python/src/main/kotlin/org/utbot/cli/language/python/PythonGenerateTestsCommand.kt @@ -10,12 +10,14 @@ import org.parsers.python.PythonParser import org.utbot.framework.codegen.domain.RuntimeExceptionTestsBehaviour import org.utbot.framework.codegen.domain.TestFramework import org.utbot.framework.plugin.api.UtExecutionSuccess +import org.utbot.python.coverage.CoverageOutputFormat import org.utbot.python.PythonMethodHeader import org.utbot.python.PythonTestGenerationConfig import org.utbot.python.PythonTestSet -import org.utbot.python.utils.RequirementsInstaller import org.utbot.python.TestFileInformation +import org.utbot.python.utils.RequirementsInstaller import org.utbot.python.code.PythonCode +import org.utbot.python.coverage.PythonCoverageMode import org.utbot.python.framework.api.python.PythonClassId import org.utbot.python.framework.codegen.model.Pytest import org.utbot.python.framework.codegen.model.Unittest @@ -114,9 +116,20 @@ class PythonGenerateTestsCommand : CliktCommand( .choice("PASS", "FAIL") .default("FAIL") - private val doNotGenerateRegressionSuite by option("--do-not-generate-regression-suite", help = "Do not generate regression test suite") + private val doNotGenerateRegressionSuite by option("--do-not-generate-regression-suite", help = "Do not generate regression test suite.") + .flag(default = false) + + private val coverageMeasureMode by option("--coverage-measure-mode", help = "Use LINES or INSTRUCTIONS for coverage measurement.") + .choice("INSTRUCTIONS", "LINES") + .default("INSTRUCTIONS") + + private val doNotSendCoverageContinuously by option("--do-not-send-coverage-continuously", help = "Do not send coverage during execution.") .flag(default = false) + private val coverageOutputFormat by option("--coverage-output-format", help = "Use LINES, INSTRUCTIONS (only from function frame).") + .choice("INSTRUCTIONS", "LINES") + .default("LINES") + private val testFramework: TestFramework get() = when (testFrameworkAsString) { @@ -252,7 +265,10 @@ class PythonGenerateTestsCommand : CliktCommand( testSourceRootPath = Paths.get(output.toAbsolutePath()).parent.toAbsolutePath(), withMinimization = !doNotMinimize, isCanceled = { false }, - runtimeExceptionTestsBehaviour = RuntimeExceptionTestsBehaviour.valueOf(runtimeExceptionTestsBehaviour) + runtimeExceptionTestsBehaviour = RuntimeExceptionTestsBehaviour.valueOf(runtimeExceptionTestsBehaviour), + coverageMeasureMode = PythonCoverageMode.parse(coverageMeasureMode), + sendCoverageContinuously = !doNotSendCoverageContinuously, + coverageOutputFormat = CoverageOutputFormat.parse(coverageOutputFormat), ) val processor = PythonCliProcessor( diff --git a/utbot-framework-api/src/main/kotlin/org/utbot/framework/plugin/api/CoverageApi.kt b/utbot-framework-api/src/main/kotlin/org/utbot/framework/plugin/api/CoverageApi.kt index 06c4c9350f..3c3f31cb08 100644 --- a/utbot-framework-api/src/main/kotlin/org/utbot/framework/plugin/api/CoverageApi.kt +++ b/utbot-framework-api/src/main/kotlin/org/utbot/framework/plugin/api/CoverageApi.kt @@ -10,7 +10,7 @@ package org.utbot.framework.plugin.api * * @see Test minimization */ -data class Instruction( +open class Instruction( val internalName: String, val methodSignature: String, val lineNumber: Int, diff --git a/utbot-python-executor/src/main/python/utbot_executor/pyproject.toml b/utbot-python-executor/src/main/python/utbot_executor/pyproject.toml index 13b3063e09..a1b7b52fd2 100644 --- a/utbot-python-executor/src/main/python/utbot_executor/pyproject.toml +++ b/utbot-python-executor/src/main/python/utbot_executor/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "utbot-executor" -version = "1.7.0" +version = "1.8.0" description = "" authors = ["Vyacheslav Tamarin "] readme = "README.md" diff --git a/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/__main__.py b/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/__main__.py index ab1ce3fb60..c9356cdcdb 100644 --- a/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/__main__.py +++ b/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/__main__.py @@ -2,35 +2,49 @@ import logging from utbot_executor.listener import PythonExecuteServer +from utbot_executor.utils import TraceMode -def main(hostname: str, port: int, coverage_hostname: str, coverage_port: str): - server = PythonExecuteServer(hostname, port, coverage_hostname, coverage_port) +def main(hostname: str, port: int, coverage_hostname: str, coverage_port: int, trace_mode: TraceMode, send_coverage: bool): + server = PythonExecuteServer(hostname, port, coverage_hostname, coverage_port, trace_mode, send_coverage) server.run() -if __name__ == '__main__': +if __name__ == "__main__": parser = argparse.ArgumentParser( - prog='UtBot Python Executor', - description='Listen socket stream and execute function value', - ) - parser.add_argument('hostname') - parser.add_argument('port', type=int) - parser.add_argument('--logfile', default=None) + prog="UtBot Python Executor", + description="Listen socket stream and execute function value", + ) + parser.add_argument("hostname") + parser.add_argument("port", type=int) + parser.add_argument("--logfile", default=None) parser.add_argument( - '--loglevel', - choices=["DEBUG", "INFO", "ERROR"], - default="ERROR", - ) - parser.add_argument('coverage_hostname') - parser.add_argument('coverage_port', type=int) + "--loglevel", + choices=["DEBUG", "INFO", "WARNING", "ERROR"], + default="ERROR", + ) + parser.add_argument("coverage_hostname") + parser.add_argument("coverage_port", type=int) + parser.add_argument( + "--coverage_type", choices=["lines", "instructions"], default="instructions" + ) + parser.add_argument( + "--send_coverage", action=argparse.BooleanOptionalAction + ) args = parser.parse_args() - loglevel = {"DEBUG": logging.DEBUG, "INFO": logging.INFO, "ERROR": logging.ERROR}[args.loglevel] + loglevel = { + "DEBUG": logging.DEBUG, + "INFO": logging.INFO, + "WARNING": logging.WARNING, + "ERROR": logging.ERROR, + }[args.loglevel] logging.basicConfig( - filename=args.logfile, - format='%(asctime)s | %(levelname)s | %(funcName)s - %(message)s', - datefmt='%m/%d/%Y %H:%M:%S', - level=loglevel, - ) - main(args.hostname, args.port, args.coverage_hostname, args.coverage_port) + filename=args.logfile, + format="%(asctime)s | %(levelname)s | %(funcName)s - %(message)s", + datefmt="%m/%d/%Y %H:%M:%S", + level=loglevel, + ) + trace_mode = TraceMode.Lines if args.coverage_type == "lines" else TraceMode.Instructions + send_coverage = args.send_coverage + main(args.hostname, args.port, args.coverage_hostname, args.coverage_port, trace_mode, send_coverage) diff --git a/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/executor.py b/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/executor.py index a1e10ec57e..01fdb290d4 100644 --- a/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/executor.py +++ b/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/executor.py @@ -4,10 +4,9 @@ import inspect import logging import pathlib -import socket import sys import traceback -import typing +import types from typing import Any, Callable, Dict, Iterable, List, Tuple from utbot_executor.deep_serialization.deep_serialization import serialize_memory_dump, \ @@ -17,8 +16,13 @@ from utbot_executor.deep_serialization.utils import PythonId, getattr_by_path from utbot_executor.memory_compressor import compress_memory from utbot_executor.parser import ExecutionRequest, ExecutionResponse, ExecutionFailResponse, ExecutionSuccessResponse -from utbot_executor.ut_tracer import UtTracer -from utbot_executor.utils import suppress_stdout as __suppress_stdout +from utbot_executor.ut_tracer import UtTracer, UtCoverageSender +from utbot_executor.utils import ( + suppress_stdout as __suppress_stdout, + get_instructions, + filter_instructions, + TraceMode, UtInstruction, +) __all__ = ['PythonExecutor'] @@ -41,9 +45,11 @@ def _load_objects(objs: List[Any]) -> MemoryDump: class PythonExecutor: - def __init__(self, coverage_hostname: str, coverage_port: int): + def __init__(self, coverage_hostname: str, coverage_port: int, trace_mode: TraceMode, send_coverage: bool): self.coverage_hostname = coverage_hostname self.coverage_port = coverage_port + self.trace_mode = trace_mode + self.send_coverage = send_coverage @staticmethod def add_syspaths(syspaths: Iterable[str]): @@ -91,7 +97,7 @@ def run_function(self, request: ExecutionRequest) -> ExecutionResponse: importlib.import_module(request.function_module), request.function_name ) - if not callable(function): + if not isinstance(function, types.FunctionType): return ExecutionFailResponse( "fail", f"Invalid function path {request.function_module}.{request.function_name}" @@ -111,23 +117,26 @@ def run_function(self, request: ExecutionRequest) -> ExecutionResponse: state_init = _update_states(loader.reload_id(), state_init_memory) serialized_state_init = serialize_memory_dump(state_init) - def _coverage_sender(info: typing.Tuple[str, int]): - if pathlib.Path(info[0]) == pathlib.Path(request.filepath): - sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) - logging.debug("Coverage message: %s:%d", request.coverage_id, info[1]) - logging.debug("Port: %d", self.coverage_port) - message = bytes(f'{request.coverage_id}:{info[1]}', encoding='utf-8') - sock.sendto(message, (self.coverage_hostname, self.coverage_port)) - logging.debug("ID: %s, Coverage: %s", request.coverage_id, info) + _coverage_sender = UtCoverageSender( + request.coverage_id, + self.coverage_hostname, + self.coverage_port, + send_coverage=self.send_coverage, + ) value = _run_calculate_function_value( - function, - args, - kwargs, - request.filepath, - serialized_state_init, - tracer=UtTracer(_coverage_sender) - ) + function, + args, + kwargs, + request.filepath, + serialized_state_init, + tracer=UtTracer( + pathlib.Path(request.filepath), + [sys.prefix, sys.exec_prefix], + _coverage_sender, + self.trace_mode, + ), + ) except Exception as _: logging.debug("Error \n%s", traceback.format_exc()) return ExecutionFailResponse("fail", traceback.format_exc()) @@ -157,7 +166,7 @@ def _serialize_state( def _run_calculate_function_value( - function: Callable, + function: types.FunctionType, args: List[Any], kwargs: Dict[str, Any], fullpath: str, @@ -172,10 +181,8 @@ def _run_calculate_function_value( __is_exception = False - (__sources, __start, ) = inspect.getsourcelines(function) - __not_empty_lines = [i for i, line in enumerate(__sources, __start) if len(line.strip()) != 0] - logging.debug("Not empty lines %s", __not_empty_lines) - __end = __start + len(__sources) + _, __start = inspect.getsourcelines(function) + __all_code_stmts = filter_instructions(get_instructions(function.__code__), tracer.mode) __tracer = tracer @@ -189,24 +196,23 @@ def _run_calculate_function_value( logging.debug("Coverage: %s", __tracer.counts) logging.debug("Fullpath: %s", fullpath) - module_path = pathlib.Path(fullpath) - __stmts = [x[1] for x in __tracer.counts if pathlib.Path(x[0]) == module_path] - __stmts_filtered = [x for x in __not_empty_lines if x in __stmts] - __stmts_filtered_with_def = [__start] + __stmts_filtered - __missed_filtered = [x for x in __not_empty_lines if x not in __stmts_filtered_with_def] - logging.debug("Covered lines: %s", __stmts_filtered_with_def) + __stmts_with_def = [UtInstruction(__start, 0, True)] + list(__tracer.counts.keys()) + __missed_filtered = [x for x in __all_code_stmts if x not in __stmts_with_def] + logging.debug("Covered lines: %s", __stmts_with_def) logging.debug("Missed lines: %s", __missed_filtered) + __str_statements = [x.serialize() for x in __stmts_with_def] + __str_missed_statements = [x.serialize() for x in __missed_filtered] + args_ids, kwargs_ids, result_id, state_after, serialized_state_after = _serialize_state(args, kwargs, __result) ids = args_ids + list(kwargs_ids.values()) - # state_before, state_after = compress_memory(ids, state_before, state_after) diff_ids = compress_memory(ids, state_before, state_after) return ExecutionSuccessResponse( status="success", is_exception=__is_exception, - statements=__stmts_filtered_with_def, - missed_statements=__missed_filtered, + statements=__str_statements, + missed_statements=__str_missed_statements, state_init=state_init, state_before=serialized_state_before, state_after=serialized_state_after, diff --git a/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/listener.py b/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/listener.py index 78df1a507c..a1c91e8c00 100644 --- a/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/listener.py +++ b/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/listener.py @@ -6,7 +6,7 @@ from utbot_executor.deep_serialization.memory_objects import PythonSerializer from utbot_executor.parser import parse_request, serialize_response, ExecutionFailResponse from utbot_executor.executor import PythonExecutor - +from utbot_executor.utils import TraceMode RECV_SIZE = 2**15 @@ -17,12 +17,14 @@ def __init__( hostname: str, port: int, coverage_hostname: str, - coverage_port: str, + coverage_port: int, + trace_mode: TraceMode, + send_coverage: bool ): logging.info('PythonExecutor is creating...') self.clientsocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self.clientsocket.connect((hostname, port)) - self.executor = PythonExecutor(coverage_hostname, coverage_port) + self.executor = PythonExecutor(coverage_hostname, coverage_port, trace_mode, send_coverage) def run(self) -> None: logging.info('PythonExecutor is ready...') @@ -80,9 +82,9 @@ def handler(self) -> None: response_size = str(len(bytes_data)) self.clientsocket.send((response_size + os.linesep).encode()) - sended_size = 0 - while len(bytes_data) > sended_size: - sended_size += self.clientsocket.send(bytes_data[sended_size:]) + sent_size = 0 + while len(bytes_data) > sent_size: + sent_size += self.clientsocket.send(bytes_data[sent_size:]) logging.debug('Sent all data') logging.info('All done...') diff --git a/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/parser.py b/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/parser.py index 838711d305..d247c28290 100644 --- a/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/parser.py +++ b/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/parser.py @@ -1,6 +1,6 @@ import dataclasses import json -from typing import Dict, List, Union +from typing import Dict, List, Union, Tuple @dataclasses.dataclass @@ -24,8 +24,8 @@ class ExecutionResponse: class ExecutionSuccessResponse(ExecutionResponse): status: str is_exception: bool - statements: List[int] - missed_statements: List[int] + statements: List[str] + missed_statements: List[str] state_init: str state_before: str state_after: str diff --git a/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/ut_tracer.py b/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/ut_tracer.py index d761c02d74..eb0d6d5b41 100644 --- a/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/ut_tracer.py +++ b/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/ut_tracer.py @@ -1,6 +1,15 @@ +import dis +import inspect +import logging import os +import pathlib +import queue +import socket import sys import typing +from concurrent.futures import ThreadPoolExecutor + +from utbot_executor.utils import TraceMode, UtInstruction def _modname(path): @@ -9,17 +18,70 @@ def _modname(path): return filename +class UtCoverageSender: + def __init__(self, coverage_id: str, host: str, port: int, use_thread: bool = False, send_coverage: bool = True): + self.coverage_id = coverage_id + self.host = host + self.port = port + self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + self.message_queue = queue.Queue() + self.send_coverage = send_coverage + + self.use_thread = use_thread + if use_thread: + self.thread = ThreadPoolExecutor(max_workers=4) + + def send_loop(self): + try: + while True: + self.send_message_thread() + except Exception as _: + self.send_loop() + + def send_message(self, message: bytes): + if self.send_coverage: + logging.debug(f"SEND {message}") + self.sock.sendto(message, (self.host, self.port)) + + def send_message_thread(self): + message = self.message_queue.get() + self.send_message(message) + + def put_message(self, key: str): + message = bytes(f"{self.coverage_id}:{key}", encoding="utf-8") + logging.debug(f"PUT {message}") + if self.use_thread: + self.message_queue.put((message, (self.host, self.port))) + self.thread.submit(self.send_message_thread) + else: + self.send_message(message) + + +class PureSender(UtCoverageSender): + def __init__(self): + super().__init__("000000", "localhost", 0, use_thread=False, send_coverage=False) + + class UtTracer: - def __init__(self, sender: typing.Callable[[typing.Tuple[str, int]], None]): - self.globaltrace = self.globaltrace_lt - self.counts = {} + def __init__( + self, + tested_file: pathlib.Path, + ignore_dirs: typing.List[str], + sender: UtCoverageSender, + mode: TraceMode = TraceMode.Instructions, + ): + self.tested_file = tested_file + self.counts: dict[UtInstruction, int] = {} self.localtrace = self.localtrace_count self.globaltrace = self.globaltrace_lt + self.ignore_dirs = ignore_dirs self.sender = sender + self.mode = mode def runfunc(self, func, /, *args, **kw): result = None sys.settrace(self.globaltrace) + self.f_code = func.__code__ try: result = func(*args, **kw) finally: @@ -31,13 +93,18 @@ def coverage(self, filename: str) -> typing.List[int]: return [line for file, line in self.counts.keys() if file == filename] def localtrace_count(self, frame, why, arg): - if why == "line": - filename = frame.f_code.co_filename - lineno = frame.f_lineno - key = filename, lineno + filename = frame.f_code.co_filename + lineno = frame.f_lineno + if pathlib.Path(filename) == self.tested_file and lineno is not None: + if self.mode == TraceMode.Instructions and frame.f_lasti is not None: + offset = frame.f_lasti + else: + offset = 0 + key = UtInstruction(lineno, offset, frame.f_code == self.f_code) if key not in self.counts: + message = key.serialize() try: - self.sender(key) + self.sender.put_message(message) except Exception: pass self.counts[key] = self.counts.get(key, 0) + 1 @@ -45,8 +112,11 @@ def localtrace_count(self, frame, why, arg): def globaltrace_lt(self, frame, why, arg): if why == 'call': + if self.mode == TraceMode.Instructions: + frame.f_trace_opcodes = True + frame.f_trace_lines = False filename = frame.f_globals.get('__file__', None) - if filename: + if filename and all(not filename.startswith(d + os.sep) for d in self.ignore_dirs): modulename = _modname(filename) if modulename is not None: return self.localtrace @@ -60,3 +130,20 @@ def __init__(self): def runfunc(self, func, /, *args, **kw): return func(*args, **kw) + + +def g1(x): + return x * 2 + + +def f(x): + def g(x): + xs = [[j for j in range(i)] for i in range(10)] + return x * 2 + return g1(x) * g(x) + 2 + + +if __name__ == "__main__": + tracer = UtTracer(pathlib.Path(__file__), [], PureSender()) + tracer.runfunc(f, 2) + print(tracer.counts) \ No newline at end of file diff --git a/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/utils.py b/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/utils.py index 35c10d0f83..d3f6d14700 100644 --- a/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/utils.py +++ b/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/utils.py @@ -1,6 +1,29 @@ +from __future__ import annotations +import dataclasses +import enum import os import sys +import typing from contextlib import contextmanager +from types import CodeType + + +class TraceMode(enum.Enum): + Lines = 1 + Instructions = 2 + + +@dataclasses.dataclass +class UtInstruction: + line: int + offset: int + from_main_frame: bool + + def serialize(self) -> str: + return ":".join(map(str, [self.line, self.offset, int(self.from_main_frame)])) + + def __hash__(self): + return hash((self.line, self.offset, self.from_main_frame)) @contextmanager @@ -12,3 +35,20 @@ def suppress_stdout(): yield finally: sys.stdout = old_stdout + + +def get_instructions(obj: CodeType) -> list[UtInstruction]: + return [UtInstruction(line, start_offset, True) for start_offset, _, line in obj.co_lines() if None not in {start_offset, line}] + + +def filter_instructions( + instructions: typing.Iterable[UtInstruction], + mode: TraceMode = TraceMode.Instructions, +) -> list[UtInstruction]: + if mode == TraceMode.Lines: + return list({UtInstruction(it.line, 0, True) for it in instructions}) + return list(instructions) + + +def get_lines(instructions: typing.Iterable[UtInstruction]) -> list[int]: + return [instruction.line for instruction in filter_instructions(instructions)] diff --git a/utbot-python-executor/src/main/resources/utbot_executor_version b/utbot-python-executor/src/main/resources/utbot_executor_version index 9dbb0c0052..afa2b3515e 100644 --- a/utbot-python-executor/src/main/resources/utbot_executor_version +++ b/utbot-python-executor/src/main/resources/utbot_executor_version @@ -1 +1 @@ -1.7.0 \ No newline at end of file +1.8.0 \ No newline at end of file diff --git a/utbot-python/samples/run_tests.py b/utbot-python/samples/run_tests.py index fd4064d1f7..c426fed41c 100644 --- a/utbot-python/samples/run_tests.py +++ b/utbot-python/samples/run_tests.py @@ -7,34 +7,39 @@ -c """ import argparse +import contextlib import json import os import shutil +from subprocess import Popen, PIPE +import sys +import threading import typing +import tqdm +from tqdm.contrib import DummyTqdmFile import pathlib def parse_arguments(): parser = argparse.ArgumentParser( - prog='UtBot Python test', - description='Generate tests for example files' + prog="UtBot Python test", description="Generate tests for example files" ) subparsers = parser.add_subparsers(dest="command") - parser_generate = subparsers.add_parser('generate', help='Generate tests') - parser_generate.add_argument('java') - parser_generate.add_argument('jar') - parser_generate.add_argument('path_to_test_dir') - parser_generate.add_argument('-c', '--config_file', required=True) - parser_generate.add_argument('-p', '--python_path', required=True) - parser_generate.add_argument('-o', '--output_dir', required=True) - parser_generate.add_argument('-i', '--coverage_output_dir', required=True) - parser_run = subparsers.add_parser('run', help='Run tests') - parser_run.add_argument('-p', '--python_path', required=True) - parser_run.add_argument('-t', '--test_directory', required=True) - parser_run.add_argument('-c', '--code_directory', required=True) - parser_coverage = subparsers.add_parser('check_coverage', help='Check coverage') - parser_coverage.add_argument('-i', '--coverage_output_dir', required=True) - parser_coverage.add_argument('-c', '--config_file', required=True) + parser_generate = subparsers.add_parser("generate", help="Generate tests") + parser_generate.add_argument("java") + parser_generate.add_argument("jar") + parser_generate.add_argument("path_to_test_dir") + parser_generate.add_argument("-c", "--config_file", required=True) + parser_generate.add_argument("-p", "--python_path", required=True) + parser_generate.add_argument("-o", "--output_dir", required=True) + parser_generate.add_argument("-i", "--coverage_output_dir", required=True) + parser_run = subparsers.add_parser("run", help="Run tests") + parser_run.add_argument("-p", "--python_path", required=True) + parser_run.add_argument("-t", "--test_directory", required=True) + parser_run.add_argument("-c", "--code_directory", required=True) + parser_coverage = subparsers.add_parser("check_coverage", help="Check coverage") + parser_coverage.add_argument("-i", "--coverage_output_dir", required=True) + parser_coverage.add_argument("-c", "--config_file", required=True) return parser.parse_args() @@ -43,62 +48,97 @@ def parse_config(config_path: str): return json.loads(fin.read()) +@contextlib.contextmanager +def std_out_err_redirect_tqdm(): + orig_out_err = sys.stdout, sys.stderr + try: + sys.stdout, sys.stderr = map(DummyTqdmFile, orig_out_err) + yield orig_out_err[0] + # Relay exceptions + except Exception as exc: + raise exc + # Always restore sys.stdout/err if necessary + finally: + sys.stdout, sys.stderr = orig_out_err + + def generate_tests( - java: str, - jar_path: str, - sys_paths: list[str], - python_path: str, - file_under_test: str, - timeout: int, - output: str, - coverage_output: str, - class_names: typing.Optional[list[str]] = None, - method_names: typing.Optional[list[str]] = None - ): + java: str, + jar_path: str, + sys_paths: list[str], + python_path: str, + file_under_test: str, + timeout: int, + output: str, + coverage_output: str, + class_names: typing.Optional[list[str]] = None, + method_names: typing.Optional[list[str]] = None, +): command = f"{java} -jar {jar_path} generate_python {file_under_test}.py -p {python_path} -o {output} -s {' '.join(sys_paths)} --timeout {timeout * 1000} --install-requirements --runtime-exception-behaviour PASS --coverage={coverage_output}" if class_names is not None: command += f" -c {','.join(class_names)}" if method_names is not None: command += f" -m {','.join(method_names)}" - print(command) - code = os.system(command) - return code + tqdm.tqdm.write("\n" + command) + + def stdout_printer(p): + for line in p.stdout: + tqdm.tqdm.write(line.rstrip().decode()) + + p = Popen(command.split(), stdout=PIPE) + t = threading.Thread(target=stdout_printer, args=(p,)) + t.run() def run_tests( - python_path: str, - tests_dir: str, - samples_dir: str, + python_path: str, + tests_dir: str, + samples_dir: str, ): command = f'{python_path} -m coverage run --source={samples_dir} -m unittest discover -p "utbot_*" {tests_dir}' - print(command) + tqdm.tqdm.write(command) code = os.system(command) return code def check_coverage( - config_file: str, - coverage_output_dir: str, + config_file: str, + coverage_output_dir: str, ): config = parse_config(config_file) report: typing.Dict[str, bool] = {} coverage: typing.Dict[str, typing.Tuple[float, float]] = {} - for part in config['parts']: - for file in part['files']: - for group in file['groups']: - expected_coverage = group.get('coverage', 0) + for part in config["parts"]: + for file in part["files"]: + for i, group in enumerate(file["groups"]): + if i > 0: + suffix = f"_{i}" + else: + suffix = "" + + expected_coverage = group.get("coverage", 0) - file_suffix = f"{part['path'].replace('/', '_')}_{file['name']}" - coverage_output_file = pathlib.Path(coverage_output_dir, f"coverage_{file_suffix}.json") + file_suffix = f"{part['path'].replace('/', '_')}_{file['name']}{suffix}" + coverage_output_file = pathlib.Path( + coverage_output_dir, f"coverage_{file_suffix}.json" + ) if coverage_output_file.exists(): with open(coverage_output_file, "rt") as fin: actual_coverage_json = json.loads(fin.readline()) - actual_covered = sum(lines['end'] - lines['start'] + 1 for lines in actual_coverage_json['covered']) - actual_not_covered = sum(lines['end'] - lines['start'] + 1 for lines in actual_coverage_json['notCovered']) + actual_covered = sum( + lines["end"] - lines["start"] + 1 + for lines in actual_coverage_json["covered"] + ) + actual_not_covered = sum( + lines["end"] - lines["start"] + 1 + for lines in actual_coverage_json["notCovered"] + ) if actual_covered + actual_not_covered == 0: actual_coverage = 0 else: - actual_coverage = round(actual_covered / (actual_not_covered + actual_covered) * 100) + actual_coverage = round( + actual_covered / (actual_not_covered + actual_covered) * 100 + ) else: actual_coverage = 0 @@ -119,31 +159,51 @@ def main_test_generation(args): config = parse_config(args.config_file) if pathlib.Path(args.coverage_output_dir).exists(): shutil.rmtree(args.coverage_output_dir) - for part in config['parts']: - for file in part['files']: - for group in file['groups']: - full_name = pathlib.PurePath(args.path_to_test_dir, part['path'], file['name']) - output_file = pathlib.PurePath(args.output_dir, f"utbot_tests_{part['path'].replace('/', '_')}_{file['name']}.py") - coverage_output_file = pathlib.PurePath(args.coverage_output_dir, f"coverage_{part['path'].replace('/', '_')}_{file['name']}.json") - generate_tests( - args.java, - args.jar, - [args.path_to_test_dir], - args.python_path, - str(full_name), - group['timeout'], - str(output_file), - str(coverage_output_file), - group['classes'], - group['methods'] - ) - - -if __name__ == '__main__': + with std_out_err_redirect_tqdm() as orig_stdout: + for part in tqdm.tqdm( + config["parts"], file=orig_stdout, dynamic_ncols=True, desc="Progress" + ): + for file in tqdm.tqdm( + part["files"], file=orig_stdout, dynamic_ncols=True, desc=part["path"] + ): + for i, group in enumerate(file["groups"]): + if i > 0: + suffix = f"_{i}" + else: + suffix = "" + + full_name = pathlib.PurePath( + args.path_to_test_dir, part["path"], file["name"] + ) + output_file = pathlib.PurePath( + args.output_dir, + f"utbot_tests_{part['path'].replace('/', '_')}_{file['name']}{suffix}.py", + ) + coverage_output_file = pathlib.PurePath( + args.coverage_output_dir, + f"coverage_{part['path'].replace('/', '_')}_{file['name']}{suffix}.json", + ) + generate_tests( + args.java, + args.jar, + [args.path_to_test_dir], + args.python_path, + str(full_name), + group["timeout"], + str(output_file), + str(coverage_output_file), + group["classes"], + group["methods"], + ) + + +if __name__ == "__main__": arguments = parse_arguments() - if arguments.command == 'generate': + if arguments.command == "generate": main_test_generation(arguments) - elif arguments.command == 'run': - run_tests(arguments.python_path, arguments.test_directory, arguments.code_directory) - elif arguments.command == 'check_coverage': + elif arguments.command == "run": + run_tests( + arguments.python_path, arguments.test_directory, arguments.code_directory + ) + elif arguments.command == "check_coverage": check_coverage(arguments.config_file, arguments.coverage_output_dir) diff --git a/utbot-python/samples/samples/classes/field.py b/utbot-python/samples/samples/classes/field.py deleted file mode 100644 index 55a4676041..0000000000 --- a/utbot-python/samples/samples/classes/field.py +++ /dev/null @@ -1,10 +0,0 @@ -class NoTestsProblem: - def __init__(self): - self.board = [] - - def set_position(self, row, col, symbol): - self.board[row][col] = symbol - return symbol - - def start(self): - self.set_position(1, 2, "O") diff --git a/utbot-python/samples/samples/controlflow/multi_conditions.py b/utbot-python/samples/samples/controlflow/multi_conditions.py new file mode 100644 index 0000000000..b7ba4bce58 --- /dev/null +++ b/utbot-python/samples/samples/controlflow/multi_conditions.py @@ -0,0 +1,14 @@ +def check_interval(x: float, left: float, right: float) -> str: + if left < x < right or right < x < left: + return "between" + elif x < left and x < right: + return "less" + elif x > left and x > right: + return "more" + elif left == right: + return "all equals" + elif x == left: + return "left" + elif x == right: + return "right" + return "what?" diff --git a/utbot-python/samples/samples/structures/matrix.py b/utbot-python/samples/samples/structures/matrix.py index 300851f49c..b9b284d334 100644 --- a/utbot-python/samples/samples/structures/matrix.py +++ b/utbot-python/samples/samples/structures/matrix.py @@ -10,15 +10,16 @@ def __init__(self, description): class Matrix: def __init__(self, elements: List[List[float]]): - self.dim = ( - len(elements), - max(len(elements[i]) for i in range(len(elements))) - if len(elements) > 0 else 0 + assert all(len(elements[i-1]) == len(row) for i, row in enumerate(elements)) + self.elements = elements + + @property + def dim(self) -> tuple[int, int]: + return ( + len(self.elements), + max(len(self.elements[i]) for i in range(len(self.elements))) + if len(self.elements) > 0 else 0 ) - self.elements = [ - row + [0] * (self.dim[1] - len(row)) - for row in elements - ] def __repr__(self): return str(self.elements) @@ -49,7 +50,7 @@ def __mul__(self, other): else: raise MatrixException("Wrong Type") - def __matmul__(self, other): + def __matmul__(self, other: Matrix): if isinstance(other, Matrix): if self.dim[1] == other.dim[0]: result = [[0 for _ in range(self.dim[0])] * other.dim[1]] @@ -59,6 +60,8 @@ def __matmul__(self, other): for k in range(self.dim[1]) ) return Matrix(result) + else: + MatrixException("Wrong dimensions") else: raise MatrixException("Wrong Type") diff --git a/utbot-python/samples/test_configuration.json b/utbot-python/samples/test_configuration.json index 37f4d5645c..2051d5515a 100644 --- a/utbot-python/samples/test_configuration.json +++ b/utbot-python/samples/test_configuration.json @@ -10,7 +10,7 @@ "classes": null, "methods": null, "timeout": 10, - "coverage": 100 + "coverage": 92 } ] }, @@ -69,8 +69,8 @@ { "classes": ["Dictionary"], "methods": ["translate"], - "timeout": 10, - "coverage": 89 + "timeout": 30, + "coverage": 100 } ] }, @@ -96,17 +96,6 @@ } ] }, - { - "name": "field", - "groups": [ - { - "classes": ["NoTestsProblem"], - "methods": null, - "timeout": 10, - "coverage": 100 - } - ] - }, { "name": "inner_class", "groups": [ @@ -151,7 +140,7 @@ { "classes": null, "methods": null, - "timeout": 10, + "timeout": 20, "coverage": 100 } ] @@ -196,7 +185,7 @@ "classes": null, "methods": null, "timeout": 10, - "coverage": 100 + "coverage": 75 } ] }, @@ -233,8 +222,8 @@ { "classes": null, "methods": null, - "timeout": 30, - "coverage": 72 + "timeout": 180, + "coverage": 83 } ] }, @@ -248,6 +237,17 @@ "coverage": 75 } ] + }, + { + "name": "multi_conditions", + "groups": [ + { + "classes": null, + "methods": null, + "timeout": 15, + "coverage": 93 + } + ] } ] }, @@ -332,7 +332,7 @@ "classes": null, "methods": null, "timeout": 180, - "coverage": 100 + "coverage": 97 } ] } @@ -454,9 +454,24 @@ "groups": [ { "classes": null, - "methods": null, + "methods": [ + "concat", + "concat_pair", + "string_constants", + "contains", + "const_contains", + "to_str", + "starts_with", + "join_str" + ], "timeout": 160, "coverage": 100 + }, + { + "classes": null, + "methods": ["separated_str"], + "timeout": 60, + "coverage": 100 } ] } @@ -488,7 +503,7 @@ "classes": ["Graph"], "methods": null, "timeout": 150, - "coverage": 100 + "coverage": 64 } ] }, @@ -510,7 +525,7 @@ "classes": null, "methods": null, "timeout": 120, - "coverage": 100 + "coverage": 96 } ] }, @@ -519,9 +534,15 @@ "groups": [ { "classes": ["Matrix"], - "methods": null, - "timeout": 240, + "methods": ["__repr__", "__eq__", "__add__", "__mul__", "is_diagonal"], + "timeout": 120, "coverage": 100 + }, + { + "classes": ["Matrix"], + "methods": ["__matmul__"], + "timeout": 80, + "coverage": 90 } ] }, @@ -532,7 +553,7 @@ "classes": null, "methods": null, "timeout": 180, - "coverage": 100 + "coverage": 62 } ] } diff --git a/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt b/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt index cc0730bffe..076194a87e 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt @@ -8,11 +8,20 @@ import org.utbot.fuzzing.Control import org.utbot.fuzzing.NoSeedValueException import org.utbot.fuzzing.fuzz import org.utbot.fuzzing.utils.Trie -import org.utbot.python.evaluation.* +import org.utbot.python.evaluation.EvaluationCache +import org.utbot.python.evaluation.PythonCodeExecutor +import org.utbot.python.evaluation.PythonCodeSocketExecutor +import org.utbot.python.evaluation.PythonEvaluationError +import org.utbot.python.evaluation.PythonEvaluationSuccess +import org.utbot.python.evaluation.PythonEvaluationTimeout +import org.utbot.python.evaluation.PythonWorker +import org.utbot.python.evaluation.PythonWorkerManager +import org.utbot.python.coverage.CoverageIdGenerator +import org.utbot.python.coverage.PyInstruction +import org.utbot.python.coverage.PythonCoverageMode +import org.utbot.python.coverage.buildCoverage import org.utbot.python.evaluation.serialization.MemoryDump import org.utbot.python.evaluation.serialization.toPythonTree -import org.utbot.python.evaluation.utils.CoverageIdGenerator -import org.utbot.python.evaluation.utils.coveredLinesToInstructions import org.utbot.python.framework.api.python.PythonTree import org.utbot.python.framework.api.python.PythonTreeModel import org.utbot.python.framework.api.python.PythonTreeWrapper @@ -42,6 +51,8 @@ class PythonEngine( private val fuzzedConcreteValues: List, private val timeoutForRun: Long, private val pythonTypeStorage: PythonTypeHintsStorage, + private val coverageMode: PythonCoverageMode = PythonCoverageMode.Instructions, + private val sendCoverageContinuously: Boolean = true, ) { private val cache = EvaluationCache() @@ -92,7 +103,7 @@ class PythonEngine( private fun handleTimeoutResult( arguments: List, methodUnderTestDescription: PythonMethodDescription, - coveredLines: Collection, + coveredInstructions: List, ): FuzzingExecutionFeedback { val summary = arguments .zip(methodUnderTest.arguments) @@ -109,7 +120,6 @@ class PythonEngine( val beforeThisObject = beforeThisObjectTree?.let { PythonTreeModel(it.tree) } val beforeModelList = beforeModelListTree.map { PythonTreeModel(it.tree) } - val coveredInstructions = coveredLinesToInstructions(coveredLines, methodUnderTest) val coverage = Coverage(coveredInstructions) val utFuzzedExecution = PythonUtExecution( stateInit = EnvironmentModels(beforeThisObject, beforeModelList, emptyMap(), executableToCall = null), @@ -134,7 +144,8 @@ class PythonEngine( ): FuzzingExecutionFeedback { val prohibitedExceptions = listOf( "builtins.AttributeError", - "builtins.TypeError" + "builtins.TypeError", + "builtins.NotImplementedError", ) val summary = arguments @@ -173,7 +184,7 @@ class PythonEngine( stateAfter = EnvironmentModels(afterThisObject, afterModelList, emptyMap(), executableToCall = null), diffIds = evaluationResult.diffIds, result = executionResult, - coverage = evaluationResult.coverage, + coverage = buildCoverage(evaluationResult.coveredStatements, evaluationResult.missedStatements), testMethodName = testMethodName.testName?.camelToSnakeCase(), displayName = testMethodName.displayName, summary = summary.map { DocRegularStmt(it) }, @@ -235,11 +246,10 @@ class PythonEngine( } is PythonEvaluationTimeout -> { - val coveredLines = - manager.coverageReceiver.coverageStorage.getOrDefault(coverageId, mutableSetOf()) - val utTimeoutException = handleTimeoutResult(arguments, description, coveredLines) - val coveredInstructions = coveredLinesToInstructions(coveredLines, methodUnderTest) - val trieNode: Trie.Node = + val coveredInstructions = + manager.coverageReceiver.coverageStorage.getOrDefault(coverageId, mutableListOf()) + val utTimeoutException = handleTimeoutResult(arguments, description, coveredInstructions) + val trieNode: Trie.Node = if (coveredInstructions.isEmpty()) Trie.emptyNode() else @@ -252,7 +262,7 @@ class PythonEngine( } is PythonEvaluationSuccess -> { - val coveredInstructions = evaluationResult.coverage.coveredInstructions + val coveredInstructions = evaluationResult.coveredStatements val result = handleSuccessResult( arguments, @@ -263,7 +273,7 @@ class PythonEngine( val typeInferenceFeedback = if (result is ValidExecution) SuccessFeedback else InvalidTypeFeedback when (result) { is ValidExecution -> { - val trieNode: Trie.Node = description.tracer.add(coveredInstructions) + val trieNode: Trie.Node = description.tracer.add(coveredInstructions) description.limitManager.addSuccessExecution() PythonExecutionResult( result, @@ -300,6 +310,8 @@ class PythonEngine( serverSocket, pythonPath, until, + coverageMode, + sendCoverageContinuously, ) { constructEvaluationInput(it) } } catch (_: TimeoutException) { return@flow @@ -311,7 +323,7 @@ class PythonEngine( parameters, fuzzedConcreteValues, pythonTypeStorage, - Trie(Instruction::id), + Trie(PyInstruction::id), Random(0), TestGenerationLimitManager(ExecutionWithTimoutMode, until, isRootManager = true), methodUnderTest.definition.type, @@ -347,7 +359,7 @@ class PythonEngine( val pair = Pair(description, arguments.map { PythonTreeWrapper(it.tree) }) val mem = cache.get(pair) if (mem != null) { - logger.debug("Repeat in fuzzing ${arguments.map {it.tree}}") + logger.debug { "Repeat in fuzzing ${arguments.map {it.tree}}" } description.limitManager.addSuccessExecution() emit(CachedExecutionFeedback(mem.fuzzingExecutionFeedback)) return@PythonFuzzing mem.fuzzingPlatformFeedback.fromCache() diff --git a/utbot-python/src/main/kotlin/org/utbot/python/PythonTestCaseGenerator.kt b/utbot-python/src/main/kotlin/org/utbot/python/PythonTestCaseGenerator.kt index f6ad5c1419..cd43648659 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/PythonTestCaseGenerator.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/PythonTestCaseGenerator.kt @@ -6,6 +6,7 @@ import org.utbot.framework.minimization.minimizeExecutions import org.utbot.framework.plugin.api.UtError import org.utbot.framework.plugin.api.UtExecution import org.utbot.framework.plugin.api.UtExecutionSuccess +import org.utbot.python.coverage.PythonCoverageMode import org.utbot.python.framework.api.python.PythonUtExecution import org.utbot.python.framework.api.python.util.pythonStrClassId import org.utbot.python.fuzzing.* @@ -40,7 +41,9 @@ class PythonTestCaseGenerator( private val timeoutForRun: Long = 0, private val sourceFileContent: String, private val mypyStorage: MypyInfoBuild, - private val mypyReportLine: List + private val mypyReportLine: List, + private val coverageMode: PythonCoverageMode = PythonCoverageMode.Instructions, + private val sendCoverageContinuously: Boolean = true, ) { private val storageForMypyMessages: MutableList = mutableListOf() @@ -153,7 +156,9 @@ class PythonTestCaseGenerator( pythonPath, constants, timeoutForRun, - PythonTypeHintsStorage.get(mypyStorage) + PythonTypeHintsStorage.get(mypyStorage), + coverageMode, + sendCoverageContinuously, ) val namesInModule = mypyStorage.names .getOrDefault(curModule, emptyList()) diff --git a/utbot-python/src/main/kotlin/org/utbot/python/PythonTestGenerationConfig.kt b/utbot-python/src/main/kotlin/org/utbot/python/PythonTestGenerationConfig.kt index 405a1abf0d..99a9500e18 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/PythonTestGenerationConfig.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/PythonTestGenerationConfig.kt @@ -2,6 +2,8 @@ package org.utbot.python import org.utbot.framework.codegen.domain.RuntimeExceptionTestsBehaviour import org.utbot.framework.codegen.domain.TestFramework +import org.utbot.python.coverage.CoverageOutputFormat +import org.utbot.python.coverage.PythonCoverageMode import java.nio.file.Path data class TestFileInformation( @@ -22,4 +24,7 @@ class PythonTestGenerationConfig( val withMinimization: Boolean, val isCanceled: () -> Boolean, val runtimeExceptionTestsBehaviour: RuntimeExceptionTestsBehaviour, + val coverageMeasureMode: PythonCoverageMode = PythonCoverageMode.Instructions, + val sendCoverageContinuously: Boolean = true, + val coverageOutputFormat: CoverageOutputFormat = CoverageOutputFormat.Lines, ) \ No newline at end of file diff --git a/utbot-python/src/main/kotlin/org/utbot/python/PythonTestGenerationProcessor.kt b/utbot-python/src/main/kotlin/org/utbot/python/PythonTestGenerationProcessor.kt index 59821778f5..cd8c835e45 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/PythonTestGenerationProcessor.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/PythonTestGenerationProcessor.kt @@ -1,7 +1,5 @@ package org.utbot.python -import com.squareup.moshi.Moshi -import com.squareup.moshi.kotlin.reflect.KotlinJsonAdapterFactory import mu.KotlinLogging import org.parsers.python.PythonParser import org.utbot.framework.codegen.domain.HangingTestsTimeout @@ -12,6 +10,13 @@ import org.utbot.framework.plugin.api.UtExecutionSuccess import org.utbot.framework.plugin.api.util.UtContext import org.utbot.framework.plugin.api.util.withUtContext import org.utbot.python.code.PythonCode +import org.utbot.python.coverage.CoverageFormat +import org.utbot.python.coverage.CoverageInfo +import org.utbot.python.coverage.CoverageOutputFormat +import org.utbot.python.coverage.PyInstruction +import org.utbot.python.coverage.filterMissedLines +import org.utbot.python.coverage.getInstructionsList +import org.utbot.python.coverage.getLinesList import org.utbot.python.framework.api.python.PythonClassId import org.utbot.python.framework.api.python.PythonMethodId import org.utbot.python.framework.api.python.PythonModel @@ -67,7 +72,9 @@ abstract class PythonTestGenerationProcessor { timeoutForRun = configuration.timeoutForRun, sourceFileContent = configuration.testFileInformation.testedFileContent, mypyStorage = mypyStorage, - mypyReportLine = emptyList() + mypyReportLine = emptyList(), + coverageMode = configuration.coverageMeasureMode, + sendCoverageContinuously = configuration.sendCoverageContinuously, ) val until = startTime + configuration.timeout @@ -253,58 +260,38 @@ abstract class PythonTestGenerationProcessor { paths } - data class InstructionSet( - val start: Int, - val end: Int - ) - - data class CoverageInfo( - val covered: List, - val notCovered: List - ) - - private val moshi: Moshi = Moshi.Builder().addLast(KotlinJsonAdapterFactory()).build() - private val jsonAdapter = moshi.adapter(CoverageInfo::class.java) - - private fun getInstructionSetList(instructions: Collection): List = - instructions.sorted().fold(emptyList()) { acc, lineNumber -> - if (acc.isEmpty()) - return@fold listOf(InstructionSet(lineNumber, lineNumber)) - val elem = acc.last() - if (elem.end + 1 == lineNumber) - acc.dropLast(1) + listOf(InstructionSet(elem.start, lineNumber)) - else - acc + listOf(InstructionSet(lineNumber, lineNumber)) - } - protected fun getCoverageInfo(testSets: List): CoverageInfo { - val covered = mutableSetOf() - val missed = mutableSetOf>() + private fun getCoverageInfo(testSets: List): CoverageInfo { + val covered = mutableSetOf() + val missed = mutableSetOf() testSets.forEach { testSet -> testSet.executions.forEach inner@{ execution -> val coverage = execution.coverage ?: return@inner - coverage.coveredInstructions.forEach { covered.add(it.lineNumber) } - missed.add(coverage.missedInstructions.map { it.lineNumber }.toSet()) + covered.addAll(coverage.coveredInstructions.filterIsInstance()) + missed.addAll(coverage.missedInstructions.filterIsInstance()) } } - val coveredInstructionSets = getInstructionSetList(covered) - val missedInstructionSets = - if (missed.isEmpty()) - emptyList() - else - getInstructionSetList(missed.reduce { a, b -> a intersect b }) - - return CoverageInfo( - coveredInstructionSets, - missedInstructionSets - ) + missed -= covered + val info = when (this.configuration.coverageOutputFormat) { + CoverageOutputFormat.Lines -> { + val coveredLines = getLinesList(covered) + val filteredMissed = filterMissedLines(coveredLines, missed) + val missedLines = getLinesList(filteredMissed) + CoverageInfo(coveredLines, missedLines) + } + CoverageOutputFormat.Instructions -> CoverageInfo( + getInstructionsList(covered), + getInstructionsList(missed) + ) + } + return CoverageInfo(info.covered.toSet().toList(), info.notCovered.toSet().toList()) } protected fun getStringCoverageInfo(testSets: List): String { - return jsonAdapter.toJson( - getCoverageInfo(testSets) - ) + val coverageInfo = getCoverageInfo(testSets) + val covered = coverageInfo.covered.map { it.toJson() } + val notCovered = coverageInfo.notCovered.map { it.toJson() } + return "{\"covered\": [${covered.joinToString(", ")}], \"notCovered\": [${notCovered.joinToString(", ")}]}" } - } data class SelectedMethodIsNotAFunctionDefinition(val methodName: String): Exception() \ No newline at end of file diff --git a/utbot-python/src/main/kotlin/org/utbot/python/coverage/CoverageApi.kt b/utbot-python/src/main/kotlin/org/utbot/python/coverage/CoverageApi.kt new file mode 100644 index 0000000000..34403ad42a --- /dev/null +++ b/utbot-python/src/main/kotlin/org/utbot/python/coverage/CoverageApi.kt @@ -0,0 +1,82 @@ +package org.utbot.python.coverage + +import org.utbot.framework.plugin.api.Coverage +import org.utbot.framework.plugin.api.Instruction + +enum class PythonCoverageMode { + Lines { + override fun toString() = "lines" + }, + + Instructions { + override fun toString() = "instructions" + }; + + companion object { + fun parse(name: String): PythonCoverageMode { + return PythonCoverageMode.values().first { + it.name.lowercase() == name.lowercase() + } + } + } +} + +data class PyInstruction( + val pyLineNumber: Int, + val offset: Long, + val fromMainFrame: Boolean, +) : Instruction( + "", + "", + pyLineNumber, + (pyLineNumber.toLong() to offset).toCoverageId() * 2 + fromMainFrame.toLong()) { + override fun toString(): String = listOf(lineNumber, offset, fromMainFrame).joinToString(":") + + constructor(lineNumber: Int) : this(lineNumber, lineNumber.toLong(), true) + constructor(lineNumber: Int, id: Long) : this(lineNumber, id.floorDiv(2).toPair().second, id % 2 == 1L) +} + +fun Boolean.toLong() = if (this) 1L else 0L + +fun String.toPyInstruction(): PyInstruction? { + val data = this.split(":") + when (data.size) { + 3 -> { + val line = data[0].toInt() + val offset = data[1].toLong() + val fromMainFrame = data[2].toInt() != 0 + return PyInstruction(line, offset, fromMainFrame) + } + 2 -> { + val line = data[0].toInt() + val offset = data[1].toLong() + return PyInstruction(line, offset, true) + } + 1 -> { + val line = data[0].toInt() + return PyInstruction(line) + } + else -> return null + } +} + +fun buildCoverage(coveredStatements: List, missedStatements: List): Coverage { + return Coverage( + coveredInstructions = coveredStatements, + instructionsCount = (coveredStatements.size + missedStatements.size).toLong(), + missedInstructions = missedStatements + ) +} + +enum class CoverageOutputFormat { + Lines, + Instructions; + + companion object { + fun parse(name: String): CoverageOutputFormat { + return CoverageOutputFormat.values().first { + it.name.lowercase() == name.lowercase() + } + } + } +} diff --git a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/utils/CoverageIdGenerator.kt b/utbot-python/src/main/kotlin/org/utbot/python/coverage/CoverageIdGenerator.kt similarity index 52% rename from utbot-python/src/main/kotlin/org/utbot/python/evaluation/utils/CoverageIdGenerator.kt rename to utbot-python/src/main/kotlin/org/utbot/python/coverage/CoverageIdGenerator.kt index 0e13ce16b7..15aafacb54 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/utils/CoverageIdGenerator.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/coverage/CoverageIdGenerator.kt @@ -1,11 +1,11 @@ -package org.utbot.python.evaluation.utils +package org.utbot.python.coverage import java.util.concurrent.atomic.AtomicLong object CoverageIdGenerator { - private const val lower_bound: Long = 1500_000_000 + private const val LOWER_BOUND: Long = 1500_000_000 - private val lastId: AtomicLong = AtomicLong(lower_bound) + private val lastId: AtomicLong = AtomicLong(LOWER_BOUND) fun createId(): String { return lastId.incrementAndGet().toString(radix = 16) diff --git a/utbot-python/src/main/kotlin/org/utbot/python/coverage/CoverageOutput.kt b/utbot-python/src/main/kotlin/org/utbot/python/coverage/CoverageOutput.kt new file mode 100644 index 0000000000..0bd4055da4 --- /dev/null +++ b/utbot-python/src/main/kotlin/org/utbot/python/coverage/CoverageOutput.kt @@ -0,0 +1,41 @@ +package org.utbot.python.coverage + +sealed interface CoverageFormat { + fun toJson(): String +} +data class LineCoverage(val start: Int, val end: Int) : CoverageFormat { + override fun toJson(): String = "{\"start\": ${start}, \"end\": ${end}}" +} + +data class InstructionCoverage( + val line: Int, + val offset: Long, + val fromMainFrame: Boolean +) : CoverageFormat { + override fun toJson(): String = "{\"line\": ${line}, \"offset\": ${offset}, \"fromMainFrame\": ${fromMainFrame}}" +} + +data class CoverageInfo( + val covered: List, + val notCovered: List, +) + +fun getLinesList(instructions: Collection): List = + instructions + .map { it.lineNumber } + .sorted() + .fold(emptyList()) { acc, lineNumber -> + if (acc.isEmpty()) + return@fold listOf(LineCoverage(lineNumber, lineNumber)) + val elem = acc.last() + if (elem.end + 1 == lineNumber || elem.end == lineNumber ) + acc.dropLast(1) + listOf(LineCoverage(elem.start, lineNumber)) + else + acc + listOf(LineCoverage(lineNumber, lineNumber)) + } + +fun filterMissedLines(covered: Collection, missed: Collection): List = + missed.filterNot { missedInstruction -> covered.any { it.start <= missedInstruction.lineNumber && missedInstruction.lineNumber <= it.end } } + +fun getInstructionsList(instructions: Collection): List = + instructions.map { InstructionCoverage(it.lineNumber, it.offset, it.fromMainFrame) }.toSet().toList() diff --git a/utbot-python/src/main/kotlin/org/utbot/python/coverage/Utils.kt b/utbot-python/src/main/kotlin/org/utbot/python/coverage/Utils.kt new file mode 100644 index 0000000000..37e04a747d --- /dev/null +++ b/utbot-python/src/main/kotlin/org/utbot/python/coverage/Utils.kt @@ -0,0 +1,22 @@ +package org.utbot.python.coverage + +import kotlin.math.ceil +import kotlin.math.max +import kotlin.math.min +import kotlin.math.sqrt + +fun Long.toPair(): Pair { + val n = ceil(sqrt(this + 2.0)).toLong() - 1 + val k = this - (n * n - 1) + return if (k <= n + 1) { + n + 1 to k + } else { + k to n + 1 + } +} + +fun Pair.toCoverageId(): Long { + val n = max(this.first, this.second) - 1 + val k = min(this.first, this.second) + return (n * n - 1) + k +} \ No newline at end of file diff --git a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/CodeEvaluationApi.kt b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/CodeEvaluationApi.kt index 3030ed6870..7e6eed438a 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/CodeEvaluationApi.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/CodeEvaluationApi.kt @@ -1,9 +1,9 @@ package org.utbot.python.evaluation -import org.utbot.framework.plugin.api.Coverage import org.utbot.python.FunctionArguments import org.utbot.python.PythonMethod import org.utbot.python.evaluation.serialization.MemoryDump +import org.utbot.python.coverage.PyInstruction interface PythonCodeExecutor { val method: PythonMethod @@ -40,7 +40,8 @@ data class PythonEvaluationTimeout( data class PythonEvaluationSuccess( val isException: Boolean, - val coverage: Coverage, + val coveredStatements: List, + val missedStatements: List, val stateInit: MemoryDump, val stateBefore: MemoryDump, val stateAfter: MemoryDump, diff --git a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCodeSocketExecutor.kt b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCodeSocketExecutor.kt index e4c53f02d6..f30872204a 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCodeSocketExecutor.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCodeSocketExecutor.kt @@ -1,8 +1,6 @@ package org.utbot.python.evaluation import mu.KotlinLogging -import org.utbot.framework.plugin.api.Coverage -import org.utbot.framework.plugin.api.Instruction import org.utbot.python.FunctionArguments import org.utbot.python.PythonMethod import org.utbot.python.evaluation.serialization.ExecutionRequest @@ -12,17 +10,14 @@ import org.utbot.python.evaluation.serialization.FailExecution import org.utbot.python.evaluation.serialization.PythonExecutionResult import org.utbot.python.evaluation.serialization.SuccessExecution import org.utbot.python.evaluation.serialization.serializeObjects -import org.utbot.python.evaluation.utils.CoverageIdGenerator -import org.utbot.python.framework.api.python.util.pythonAnyClassId +import org.utbot.python.coverage.CoverageIdGenerator +import org.utbot.python.coverage.toPyInstruction import org.utbot.python.newtyping.PythonCallableTypeDescription import org.utbot.python.newtyping.pythonDescription import org.utbot.python.newtyping.pythonTypeName -import org.utbot.python.newtyping.pythonTypeRepresentation import org.utbot.python.newtyping.utils.isNamed import java.net.SocketException -private val logger = KotlinLogging.logger {} - class PythonCodeSocketExecutor( override val method: PythonMethod, override val moduleToImport: String, @@ -132,9 +127,12 @@ class PythonCodeSocketExecutor( val stateBefore = ExecutionResultDeserializer.parseMemoryDump(executionResult.stateBefore) ?: return parsingException val stateAfter = ExecutionResultDeserializer.parseMemoryDump(executionResult.stateAfter) ?: return parsingException val diffIds = executionResult.diffIds.map {it.toLong()} + val statements = executionResult.statements.mapNotNull { it.toPyInstruction() } + val missedStatements = executionResult.missedStatements.mapNotNull { it.toPyInstruction() } PythonEvaluationSuccess( executionResult.isException, - calculateCoverage(executionResult.statements, executionResult.missedStatements), + statements, + missedStatements, stateInit, stateBefore, stateAfter, @@ -151,29 +149,6 @@ class PythonCodeSocketExecutor( } } - private fun calculateCoverage(statements: List, missedStatements: List): Coverage { - val covered = statements.filter { it !in missedStatements } - return Coverage( - coveredInstructions=covered.map { - Instruction( - method.containingPythonClass?.pythonTypeRepresentation() ?: pythonAnyClassId.name, - method.methodSignature(), - it, - it.toLong() - ) - }, - instructionsCount = statements.size.toLong(), - missedInstructions = missedStatements.map { - Instruction( - method.containingPythonClass?.pythonTypeRepresentation() ?: pythonAnyClassId.name, - method.methodSignature(), - it, - it.toLong() - ) - } - ) - } - override fun stop() { pythonWorker.stopServer() } diff --git a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCoverageReceiver.kt b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCoverageReceiver.kt index 413ff85aa4..e28cc2654b 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCoverageReceiver.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCoverageReceiver.kt @@ -1,6 +1,8 @@ package org.utbot.python.evaluation import mu.KotlinLogging +import org.utbot.python.coverage.PyInstruction +import org.utbot.python.coverage.toPyInstruction import java.io.IOException import java.net.DatagramPacket import java.net.DatagramSocket @@ -11,7 +13,7 @@ import kotlin.math.max class PythonCoverageReceiver( val until: Long, ) : Thread() { - val coverageStorage = mutableMapOf>() + val coverageStorage = mutableMapOf>() private val socket = DatagramSocket() private val logger = KotlinLogging.logger {} @@ -39,11 +41,13 @@ class PythonCoverageReceiver( val buf = ByteArray(256) val request = DatagramPacket(buf, buf.size) socket.receive(request) - val requestData = request.data.decodeToString().take(request.length).split(":") + val requestData = request.data.decodeToString().take(request.length).split(":", limit=2) if (requestData.size == 2) { val (id, line) = requestData - val lineNumber = line.toInt() - coverageStorage.getOrPut(id) { mutableSetOf() }.add(lineNumber) + val instruction = line.toPyInstruction() + if (instruction != null) { + coverageStorage.getOrPut(id) { mutableListOf() }.add(instruction) + } } } } catch (ex: SocketException) { diff --git a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonWorkerManager.kt b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonWorkerManager.kt index 2ef2e2eeaf..0841e1c36a 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonWorkerManager.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonWorkerManager.kt @@ -11,6 +11,7 @@ import java.net.ServerSocket import java.net.Socket import java.net.SocketTimeoutException import org.apache.logging.log4j.LogManager +import org.utbot.python.coverage.PythonCoverageMode private val logger = KotlinLogging.logger {} @@ -18,6 +19,8 @@ class PythonWorkerManager( private val serverSocket: ServerSocket, val pythonPath: String, val until: Long, + private val coverageMeasureMode: PythonCoverageMode = PythonCoverageMode.Instructions, + private val sendCoverageContinuously: Boolean = true, val pythonCodeExecutorConstructor: (PythonWorker) -> PythonCodeExecutor, ) { var timeout: Long = 0 @@ -47,6 +50,8 @@ class PythonWorkerManager( coverageReceiver.address().second, "--logfile", logfile.absolutePath, "--loglevel", logLevel, // "DEBUG", "INFO", "WARNING", "ERROR" + "--coverage_type", coverageMeasureMode.toString(), // "lines", "instructions" + sendCoverageContinuously.toSendCoverageContinuouslyString(), // "--send_coverage", "--no-send_coverage" )) timeout = max(until - processStartTime, 0) if (this::workerSocket.isInitialized && !workerSocket.isClosed) { @@ -120,5 +125,13 @@ class PythonWorkerManager( companion object { val logfile = TemporaryFileManager.createTemporaryFile("", "utbot_executor.log", "log", true) + + fun Boolean.toSendCoverageContinuouslyString(): String { + return if (this) { + "--send_coverage" + } else { + "--no-send_coverage" + } + } } } \ No newline at end of file diff --git a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/serialization/ExecutionResultDeserializer.kt b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/serialization/ExecutionResultDeserializer.kt index 2909143096..9cc140a12a 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/serialization/ExecutionResultDeserializer.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/serialization/ExecutionResultDeserializer.kt @@ -43,8 +43,8 @@ sealed class PythonExecutionResult data class SuccessExecution( val isException: Boolean, - val statements: List, - val missedStatements: List, + val statements: List, + val missedStatements: List, val stateInit: String, val stateBefore: String, val stateAfter: String, diff --git a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/utils/Utils.kt b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/utils/Utils.kt deleted file mode 100644 index 51d8a11e31..0000000000 --- a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/utils/Utils.kt +++ /dev/null @@ -1,17 +0,0 @@ -package org.utbot.python.evaluation.utils - -import org.utbot.framework.plugin.api.Instruction -import org.utbot.python.PythonMethod -import org.utbot.python.framework.api.python.util.pythonAnyClassId -import org.utbot.python.newtyping.pythonTypeRepresentation - -fun coveredLinesToInstructions(coveredLines: Collection, method: PythonMethod): List { - return coveredLines.map { - Instruction( - method.containingPythonClass?.pythonTypeRepresentation() ?: pythonAnyClassId.name, - method.methodSignature(), - it, - it.toLong() - ) - } -} diff --git a/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/PythonApi.kt b/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/PythonApi.kt index 00c3fb48e2..2d7f63ac35 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/PythonApi.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/PythonApi.kt @@ -1,11 +1,11 @@ package org.utbot.python.fuzzing import mu.KotlinLogging -import org.utbot.framework.plugin.api.Instruction import org.utbot.framework.plugin.api.UtError import org.utbot.fuzzer.FuzzedContext import org.utbot.fuzzing.* import org.utbot.fuzzing.utils.Trie +import org.utbot.python.coverage.PyInstruction import org.utbot.python.framework.api.python.PythonTree import org.utbot.python.framework.api.python.PythonUtExecution import org.utbot.python.fuzzing.provider.* @@ -35,7 +35,7 @@ class PythonMethodDescription( parameters: List, val concreteValues: Collection = emptyList(), val pythonTypeStorage: PythonTypeHintsStorage, - val tracer: Trie, + val tracer: Trie, val random: Random, val limitManager: TestGenerationLimitManager, val type: FunctionType, @@ -56,7 +56,7 @@ data class PythonExecutionResult( data class PythonFeedback( override val control: Control = Control.CONTINUE, - val result: Trie.Node = Trie.emptyNode(), + val result: Trie.Node = Trie.emptyNode(), val typeInferenceFeedback: InferredTypeFeedback = InvalidTypeFeedback, val fromCache: Boolean = false, ) : Feedback { diff --git a/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/DictValueProvider.kt b/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/DictValueProvider.kt index 6d26c6f517..22accbbf78 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/DictValueProvider.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/DictValueProvider.kt @@ -39,10 +39,9 @@ object DictValueProvider : ValueProvider - val items = mapOf(v[0].tree to v[1].tree).toMutableMap() + construct = Routine.Create(emptyList()) { v -> PythonFuzzedValue( - PythonTree.DictNode(items), + PythonTree.DictNode(mutableMapOf()), "%var% = ${type.pythonTypeRepresentation()}" ) }, diff --git a/utbot-python/src/main/kotlin/org/utbot/python/newtyping/inference/baseline/BaselineAlgorithm.kt b/utbot-python/src/main/kotlin/org/utbot/python/newtyping/inference/baseline/BaselineAlgorithm.kt index 5272f0ed40..fd1173ce17 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/newtyping/inference/baseline/BaselineAlgorithm.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/newtyping/inference/baseline/BaselineAlgorithm.kt @@ -52,6 +52,8 @@ class BaselineAlgorithm( private val openedStates: MutableMap> = mutableMapOf() private val statistic: MutableMap = mutableMapOf() + private val checkedSignatures: MutableSet = mutableSetOf() + private fun getRandomType(): UtType? { val weights = states.map { 1.0 / (it.anyNodes.size * it.anyNodes.size + 1) } val state = weightedRandom(states, weights, random) @@ -92,15 +94,19 @@ class BaselineAlgorithm( val state = chooseState(states) val newState = expandState(state, storage) if (newState != null) { - logger.info("Checking ${newState.signature.pythonTypeRepresentation()}") + logger.info("Checking new state ${newState.signature.pythonTypeRepresentation()}") if (checkSignature(newState.signature as FunctionType, fileForMypyRuns, configFile)) { logger.debug("Found new state!") openedStates[newState.signature] = newState to state return newState.signature } } else if (state.anyNodes.isEmpty()) { + if (state.signature in checkedSignatures) { + return state.signature + } logger.info("Checking ${state.signature.pythonTypeRepresentation()}") if (checkSignature(state.signature as FunctionType, fileForMypyRuns, configFile)) { + checkedSignatures.add(state.signature) return state.signature } else { states.remove(state)