From 8fc286f2e35c05e0bbfa27b2b669b737fc3ea8bc Mon Sep 17 00:00:00 2001 From: dmitrii-kriukov Date: Mon, 24 Jan 2022 18:37:54 +0300 Subject: [PATCH 1/7] report configs in runner --- report_generator/xpu_report_gen_config.json | 31 +++++++++++++++++++++ runner.py | 12 ++++---- 2 files changed, 38 insertions(+), 5 deletions(-) create mode 100644 report_generator/xpu_report_gen_config.json diff --git a/report_generator/xpu_report_gen_config.json b/report_generator/xpu_report_gen_config.json new file mode 100644 index 000000000..f717f107b --- /dev/null +++ b/report_generator/xpu_report_gen_config.json @@ -0,0 +1,31 @@ +{ + "header": [ + "algorithm", + "stage", + "device", + "input_data:data_order", + "input_data:data_type", + "input_data:dataset_name", + "input_data:classes", + "algorithm_parameters:tol", + "algorithm_parameters:max_iter", + "algorithm_parameters:solver", + "algorithm_parameters:C", + "algorithm_parameters:kernel", + "algorithm_parameters:nu", + "algorithm_parameters:eps", + "algorithm_parameters:n_neighbors", + "algorithm_parameters:n_estimators", + "algorithm_parameters:n_clusters", + "algorithm_parameters:min_samples", + "algorithm_parameters:fit_intercept", + "algorithm_parameters:max_depth", + "algorithm_parameters:max_features" + ], + "comparison_method": { + "default": "2 / 1" + }, + "aggregation_metrics": [ + "geomean" + ] +} diff --git a/runner.py b/runner.py index 980e40b87..a495aa7a1 100755 --- a/runner.py +++ b/runner.py @@ -66,9 +66,10 @@ def get_configs(path: Path) -> List[str]: parser.add_argument('--verbose', default='INFO', type=str, choices=("ERROR", "WARNING", "INFO", "DEBUG"), help='Print additional information during benchmarks running') - parser.add_argument('--report', default=False, action='store_true', - help='Create an Excel report based on benchmarks results. ' - 'Need "openpyxl" library') + parser.add_argument('--report-config', default=None, metavar='ConfigPath', type=str, + help='config file for the reporter, if None the reporter will not be started' + 'Create an Excel report based on benchmarks results. ' + 'Need "openpyxl" library') args = parser.parse_args() logging.basicConfig( @@ -263,12 +264,13 @@ class GenerationArgs: json.dump(json_result, args.output_file, indent=4) name_result_file = args.output_file.name args.output_file.close() + basename_result_file = os.path.splitext(name_result_file)[0] if args.report: command = 'python report_generator/report_generator.py ' \ + f'--result-files {name_result_file} ' \ - + f'--report-file {name_result_file}.xlsx ' \ - + '--generation-config report_generator/default_report_gen_config.json' + + f'--report-file {basename_result_file}.xlsx ' \ + + '--generation-config ' + args.report logging.info(command) stdout, stderr = utils.read_output_from_command(command) if stderr != '': From f11ae1d5e351c83b8dfca79a87deb6a6ab98cbb8 Mon Sep 17 00:00:00 2001 From: dmitrii-kriukov Date: Mon, 24 Jan 2022 18:42:00 +0300 Subject: [PATCH 2/7] space in help --- runner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runner.py b/runner.py index a495aa7a1..02c899a04 100755 --- a/runner.py +++ b/runner.py @@ -67,7 +67,7 @@ def get_configs(path: Path) -> List[str]: choices=("ERROR", "WARNING", "INFO", "DEBUG"), help='Print additional information during benchmarks running') parser.add_argument('--report-config', default=None, metavar='ConfigPath', type=str, - help='config file for the reporter, if None the reporter will not be started' + help='config file for the reporter, if None the reporter will not be started ' 'Create an Excel report based on benchmarks results. ' 'Need "openpyxl" library') args = parser.parse_args() From 1717b454ebb07915622cd565d3020bbf00364017 Mon Sep 17 00:00:00 2001 From: dmitrii-kriukov Date: Thu, 27 Jan 2022 11:52:36 +0300 Subject: [PATCH 3/7] default and const value --- runner.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/runner.py b/runner.py index 02c899a04..c8349227e 100755 --- a/runner.py +++ b/runner.py @@ -66,7 +66,8 @@ def get_configs(path: Path) -> List[str]: parser.add_argument('--verbose', default='INFO', type=str, choices=("ERROR", "WARNING", "INFO", "DEBUG"), help='Print additional information during benchmarks running') - parser.add_argument('--report-config', default=None, metavar='ConfigPath', type=str, + parser.add_argument('--report', nargs='?', const='report_generator/default_report_gen_config.json', + default=None, metavar='ConfigPath', type=str, help='config file for the reporter, if None the reporter will not be started ' 'Create an Excel report based on benchmarks results. ' 'Need "openpyxl" library') From 402bcc9282bc87e695fc72d9d7c3b46b5c9afc69 Mon Sep 17 00:00:00 2001 From: dmitrii-kriukov Date: Thu, 27 Jan 2022 12:13:02 +0300 Subject: [PATCH 4/7] pep8 --- runner.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/runner.py b/runner.py index c8349227e..c46be964b 100755 --- a/runner.py +++ b/runner.py @@ -66,9 +66,10 @@ def get_configs(path: Path) -> List[str]: parser.add_argument('--verbose', default='INFO', type=str, choices=("ERROR", "WARNING", "INFO", "DEBUG"), help='Print additional information during benchmarks running') - parser.add_argument('--report', nargs='?', const='report_generator/default_report_gen_config.json', - default=None, metavar='ConfigPath', type=str, - help='config file for the reporter, if None the reporter will not be started ' + parser.add_argument('--report', nargs='?', default=None, metavar='ConfigPath', type=str, + const='report_generator/default_report_gen_config.json', + help='config file for the reporter, ' + 'if None the reporter will not be started ' 'Create an Excel report based on benchmarks results. ' 'Need "openpyxl" library') args = parser.parse_args() From 5cdd56c9afc0119693dbc5c467a36c950461b380 Mon Sep 17 00:00:00 2001 From: dmitrii-kriukov Date: Mon, 31 Jan 2022 12:05:32 +0300 Subject: [PATCH 5/7] retrigger checks From b70dc8c3f83c0b6c91af383b3967191ed95ec978 Mon Sep 17 00:00:00 2001 From: dmitrii-kriukov Date: Mon, 31 Jan 2022 14:10:23 +0300 Subject: [PATCH 6/7] old name --- runner.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/runner.py b/runner.py index c46be964b..14cf6fd5c 100755 --- a/runner.py +++ b/runner.py @@ -266,12 +266,11 @@ class GenerationArgs: json.dump(json_result, args.output_file, indent=4) name_result_file = args.output_file.name args.output_file.close() - basename_result_file = os.path.splitext(name_result_file)[0] if args.report: command = 'python report_generator/report_generator.py ' \ + f'--result-files {name_result_file} ' \ - + f'--report-file {basename_result_file}.xlsx ' \ + + f'--report-file {name_result_file}.xlsx ' \ + '--generation-config ' + args.report logging.info(command) stdout, stderr = utils.read_output_from_command(command) From 57e72cb03a5a72fc83113c5cbcab5007b558ae45 Mon Sep 17 00:00:00 2001 From: dmitrii-kriukov Date: Wed, 2 Feb 2022 11:54:41 +0300 Subject: [PATCH 7/7] update help --- runner.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/runner.py b/runner.py index 14cf6fd5c..5fcf3f0ea 100755 --- a/runner.py +++ b/runner.py @@ -68,9 +68,10 @@ def get_configs(path: Path) -> List[str]: help='Print additional information during benchmarks running') parser.add_argument('--report', nargs='?', default=None, metavar='ConfigPath', type=str, const='report_generator/default_report_gen_config.json', - help='config file for the reporter, ' - 'if None the reporter will not be started ' - 'Create an Excel report based on benchmarks results. ' + help='Create an Excel report based on benchmarks results. ' + 'If the parameter is not set, the reporter will not be launched. ' + 'If the parameter is set and the config is not specified, ' + 'the default config will be used. ' 'Need "openpyxl" library') args = parser.parse_args()