diff --git a/bench.py b/bench.py index cd26c166e..52f25ca7d 100644 --- a/bench.py +++ b/bench.py @@ -338,20 +338,47 @@ def columnwise_score(y, yp, score_func): return score_func(y, yp) -def accuracy_score(y, yp): - return columnwise_score(y, yp, lambda y1, y2: np.mean(y1 == y2)) +def accuracy_score(y_true, y_pred): + return columnwise_score(y_true, y_pred, lambda y1, y2: np.mean(y1 == y2)) -def log_loss(y, yp): +def log_loss(y_true, y_pred): from sklearn.metrics import log_loss as sklearn_log_loss - y = convert_to_numpy(y) - yp = convert_to_numpy(yp) - return sklearn_log_loss(y, yp) + y_true = convert_to_numpy(y_true) + y_pred = convert_to_numpy(y_pred) + return sklearn_log_loss(y_true, y_pred) + +def roc_auc_score(y_true, y_pred, multi_class='ovr'): + from sklearn.metrics import roc_auc_score as sklearn_roc_auc + y_true = convert_to_numpy(y_true) + y_pred = convert_to_numpy(y_pred) + if y_pred.shape[1] == 2: # binary case + y_pred = y_pred[:, 1] + return sklearn_roc_auc(y_true, y_pred, multi_class=multi_class) -def rmse_score(y, yp): + +def rmse_score(y_true, y_pred): return columnwise_score( - y, yp, lambda y1, y2: float(np.sqrt(np.mean((y1 - y2)**2)))) + y_true, y_pred, lambda y1, y2: float(np.sqrt(np.mean((y1 - y2)**2)))) + + +def r2_score(y_true, y_pred): + from sklearn.metrics import r2_score as sklearn_r2_score + y_true = convert_to_numpy(y_true) + y_pred = convert_to_numpy(y_pred) + return sklearn_r2_score(y_true, y_pred) + + +def davies_bouldin_score(X, labels): + from sklearn.metrics.cluster import davies_bouldin_score as sklearn_dbs + X = convert_to_numpy(X) + labels = convert_to_numpy(labels) + try: + res = sklearn_dbs(X, labels) + except ValueError as ex: + res = ex + return res def convert_data(data, dtype, data_order, data_format): @@ -488,7 +515,7 @@ def gen_basic_dict(library, algorithm, stage, params, data, alg_instance=None, def print_output(library, algorithm, stages, params, functions, - times, accuracy_type, accuracies, data, alg_instance=None, + times, metric_type, metrics, data, alg_instance=None, alg_params=None): if params.output_format == 'json': output = [] @@ -496,8 +523,13 @@ def print_output(library, algorithm, stages, params, functions, result = gen_basic_dict(library, algorithm, stages[i], params, data[i], alg_instance, alg_params) result.update({'time[s]': times[i]}) - if accuracy_type is not None: - result.update({f'{accuracy_type}': accuracies[i]}) + if metric_type is not None: + if isinstance(metric_type, str): + result.update({f'{metric_type}': metrics[i]}) + elif isinstance(metric_type, list): + for ind, val in enumerate(metric_type): + if metrics[ind][i] is not None: + result.update({f'{val}': metrics[ind][i]}) if hasattr(params, 'n_classes'): result['input_data'].update({'classes': params.n_classes}) if hasattr(params, 'n_clusters'): diff --git a/configs/blogs/skl_2021_3.json b/configs/blogs/skl_2021_3.json index 30ced64e4..c3e2f409b 100644 --- a/configs/blogs/skl_2021_3.json +++ b/configs/blogs/skl_2021_3.json @@ -307,7 +307,7 @@ } ], "nu": [0.25], - "kernel": ["sigmoid"] + "kernel": ["poly"] }, { "algorithm": "svr", diff --git a/cuml_bench/dbscan.py b/cuml_bench/dbscan.py index 663a2fc10..03ecfc2e4 100644 --- a/cuml_bench/dbscan.py +++ b/cuml_bench/dbscan.py @@ -48,5 +48,5 @@ bench.print_output(library='cuml', algorithm='dbscan', stages=['training'], params=params, functions=['DBSCAN'], times=[time], - accuracies=[acc], accuracy_type='davies_bouldin_score', data=[X], + metrics=[acc], metric_type='davies_bouldin_score', data=[X], alg_instance=dbscan) diff --git a/cuml_bench/df_clsf.py b/cuml_bench/df_clsf.py index e3b133544..e4d265954 100755 --- a/cuml_bench/df_clsf.py +++ b/cuml_bench/df_clsf.py @@ -97,6 +97,6 @@ def predict(X): bench.print_output(library='cuml', algorithm='decision_forest_classification', stages=['training', 'prediction'], params=params, functions=['df_clsf.fit', 'df_clsf.predict'], - times=[fit_time, predict_time], accuracy_type='accuracy[%]', - accuracies=[train_acc, test_acc], data=[X_train, X_test], + times=[fit_time, predict_time], metric_type='accuracy[%]', + metrics=[train_acc, test_acc], data=[X_train, X_test], alg_instance=clf) diff --git a/cuml_bench/df_regr.py b/cuml_bench/df_regr.py index 62040791b..9e7298882 100644 --- a/cuml_bench/df_regr.py +++ b/cuml_bench/df_regr.py @@ -93,6 +93,6 @@ def predict(X): bench.print_output(library='cuml', algorithm='decision_forest_regression', stages=['training', 'prediction'], params=params, functions=['df_regr.fit', 'df_regr.predict'], - times=[fit_time, predict_time], accuracy_type='rmse', - accuracies=[train_rmse, test_rmse], data=[X_train, X_test], + times=[fit_time, predict_time], metric_type='rmse', + metrics=[train_rmse, test_rmse], data=[X_train, X_test], alg_instance=regr) diff --git a/cuml_bench/elasticnet.py b/cuml_bench/elasticnet.py index bd0684a09..2f4e3dd5e 100755 --- a/cuml_bench/elasticnet.py +++ b/cuml_bench/elasticnet.py @@ -56,6 +56,6 @@ bench.print_output(library='cuml', algorithm='elastic-net', stages=['training', 'prediction'], params=params, functions=['ElasticNet.fit', 'ElasticNet.predict'], - times=[fit_time, predict_time], accuracy_type='rmse', - accuracies=[train_rmse, test_rmse], data=[X_train, X_train], + times=[fit_time, predict_time], metric_type='rmse', + metrics=[train_rmse, test_rmse], data=[X_train, X_train], alg_instance=regr) diff --git a/cuml_bench/kmeans.py b/cuml_bench/kmeans.py index d0192ba4d..2e3e9d9ff 100644 --- a/cuml_bench/kmeans.py +++ b/cuml_bench/kmeans.py @@ -88,6 +88,6 @@ def kmeans_fit(X): bench.print_output(library='cuml', algorithm='kmeans', stages=['training', 'prediction'], params=params, functions=['KMeans.fit', 'KMeans.predict'], - times=[fit_time, predict_time], accuracy_type='davies_bouldin_score', - accuracies=[acc_train, acc_test], data=[X_train, X_test], + times=[fit_time, predict_time], metric_type='davies_bouldin_score', + metrics=[acc_train, acc_test], data=[X_train, X_test], alg_instance=kmeans) diff --git a/cuml_bench/knn_clsf.py b/cuml_bench/knn_clsf.py index 0460346bf..6ccf3aa47 100755 --- a/cuml_bench/knn_clsf.py +++ b/cuml_bench/knn_clsf.py @@ -68,7 +68,7 @@ stages=['training', 'prediction'], params=params, functions=['knn_clsf.fit', 'knn_clsf.predict'], times=[train_time, predict_time], - accuracies=[train_acc, test_acc], accuracy_type='accuracy[%]', + metrics=[train_acc, test_acc], metric_type='accuracy[%]', data=[X_train, X_test], alg_instance=knn_clsf) else: bench.print_output(library='cuml', @@ -76,5 +76,5 @@ stages=['training', 'search'], params=params, functions=['knn_clsf.fit', 'knn_clsf.kneighbors'], times=[train_time, predict_time], - accuracies=[], accuracy_type=None, + metrics=[], metric_type=None, data=[X_train, X_test], alg_instance=knn_clsf) diff --git a/cuml_bench/lasso.py b/cuml_bench/lasso.py index 373ea3f19..9dc9e9e1c 100755 --- a/cuml_bench/lasso.py +++ b/cuml_bench/lasso.py @@ -53,6 +53,6 @@ bench.print_output(library='sklearn', algorithm='lasso', stages=['training', 'prediction'], params=params, functions=['Lasso.fit', 'Lasso.predict'], - times=[fit_time, predict_time], accuracy_type='rmse', - accuracies=[train_rmse, test_rmse], data=[X_train, X_test], + times=[fit_time, predict_time], metric_type='rmse', + metrics=[train_rmse, test_rmse], data=[X_train, X_test], alg_instance=regr) diff --git a/cuml_bench/linear.py b/cuml_bench/linear.py index f80434917..bfe81991f 100644 --- a/cuml_bench/linear.py +++ b/cuml_bench/linear.py @@ -50,6 +50,6 @@ bench.print_output(library='cuml', algorithm='linear_regression', stages=['training', 'prediction'], params=params, functions=['Linear.fit', 'Linear.predict'], - times=[fit_time, predict_time], accuracy_type='rmse', - accuracies=[train_rmse, test_rmse], data=[X_train, X_test], + times=[fit_time, predict_time], metric_type='rmse', + metrics=[train_rmse, test_rmse], data=[X_train, X_test], alg_instance=regr) diff --git a/cuml_bench/log_reg.py b/cuml_bench/log_reg.py index f8e143d9b..599b1bfdf 100644 --- a/cuml_bench/log_reg.py +++ b/cuml_bench/log_reg.py @@ -61,6 +61,6 @@ bench.print_output(library='cuml', algorithm='logistic_regression', stages=['training', 'prediction'], params=params, functions=['LogReg.fit', 'LogReg.predict'], - times=[fit_time, predict_time], accuracy_type='accuracy[%]', - accuracies=[train_acc, test_acc], data=[X_train, X_test], + times=[fit_time, predict_time], metric_type='accuracy[%]', + metrics=[train_acc, test_acc], data=[X_train, X_test], alg_instance=clf) diff --git a/cuml_bench/pca.py b/cuml_bench/pca.py index c43f569ae..35f20f3b6 100644 --- a/cuml_bench/pca.py +++ b/cuml_bench/pca.py @@ -51,6 +51,6 @@ bench.print_output(library='cuml', algorithm='pca', stages=['training', 'transformation'], params=params, functions=['PCA.fit', 'PCA.transform'], - times=[fit_time, transform_time], accuracy_type=None, - accuracies=[None, None], data=[X_train, X_test], + times=[fit_time, transform_time], metric_type=None, + metrics=[None, None], data=[X_train, X_test], alg_instance=pca) diff --git a/cuml_bench/ridge.py b/cuml_bench/ridge.py index 6c1696ae7..d6d488673 100644 --- a/cuml_bench/ridge.py +++ b/cuml_bench/ridge.py @@ -52,6 +52,6 @@ bench.print_output(library='cuml', algorithm='ridge_regression', stages=['training', 'prediction'], params=params, functions=['Ridge.fit', 'Ridge.predict'], - times=[fit_time, predict_time], accuracy_type='rmse', - accuracies=[train_rmse, test_rmse], data=[X_train, X_test], + times=[fit_time, predict_time], metric_type='rmse', + metrics=[train_rmse, test_rmse], data=[X_train, X_test], alg_instance=regr) diff --git a/cuml_bench/svm.py b/cuml_bench/svm.py index 3067b175e..112427397 100644 --- a/cuml_bench/svm.py +++ b/cuml_bench/svm.py @@ -57,14 +57,14 @@ if params.probability: state_predict = 'predict_proba' - accuracy_type = 'log_loss' + metric_type = 'log_loss' clf_predict = clf.predict_proba def metric_call(x, y): return bench.log_loss(x, y) else: state_predict = 'prediction' - accuracy_type = 'accuracy[%]' + metric_type = 'accuracy[%]' clf_predict = clf.predict def metric_call(x, y): @@ -82,6 +82,6 @@ def metric_call(x, y): bench.print_output(library='cuml', algorithm='svc', stages=['training', state_predict], params=params, functions=['SVM.fit', 'SVM.predict'], - times=[fit_time, predict_train_time], accuracy_type=accuracy_type, - accuracies=[train_acc, test_acc], data=[X_train, X_train], + times=[fit_time, predict_train_time], metric_type=metric_type, + metrics=[train_acc, test_acc], data=[X_train, X_train], alg_instance=clf) diff --git a/cuml_bench/svr.py b/cuml_bench/svr.py index 9644a9362..7560ff103 100644 --- a/cuml_bench/svr.py +++ b/cuml_bench/svr.py @@ -66,6 +66,6 @@ bench.print_output(library='cuml', algorithm='svr', stages=['training', 'prediction'], params=params, functions=['SVR.fit', 'SVR.predict'], - times=[fit_time, predict_train_time], accuracy_type='rmse', - accuracies=[train_rmse, test_rmse], data=[X_train, X_train], + times=[fit_time, predict_train_time], metric_type='rmse', + metrics=[train_rmse, test_rmse], data=[X_train, X_train], alg_instance=regr) diff --git a/cuml_bench/train_test_split.py b/cuml_bench/train_test_split.py index d9e74ad94..d8f70f7e6 100644 --- a/cuml_bench/train_test_split.py +++ b/cuml_bench/train_test_split.py @@ -44,5 +44,5 @@ bench.print_output(library='cuml', algorithm='train_test_split', stages=['training'], params=params, - functions=['train_test_split'], times=[time], accuracies=[None], - accuracy_type=None, data=[X], alg_params=tts_params) + functions=['train_test_split'], times=[time], metrics=[None], + metric_type=None, data=[X], alg_params=tts_params) diff --git a/daal4py_bench/dbscan.py b/daal4py_bench/dbscan.py index 64a19d813..d5010cfd0 100644 --- a/daal4py_bench/dbscan.py +++ b/daal4py_bench/dbscan.py @@ -51,4 +51,4 @@ def test_dbscan(X): bench.print_output(library='daal4py', algorithm='dbscan', stages=['training'], params=params, functions=['DBSCAN'], times=[time], - accuracies=[None], accuracy_type=None, data=[X]) + metrics=[None], metric_type=None, data=[X]) diff --git a/daal4py_bench/df_clsf.py b/daal4py_bench/df_clsf.py index 0c149291f..0e32a1185 100644 --- a/daal4py_bench/df_clsf.py +++ b/daal4py_bench/df_clsf.py @@ -126,5 +126,5 @@ def df_clsf_predict(X, training_result, n_classes, verbose=False): bench.print_output(library='daal4py', algorithm='decision_forest_classification', stages=['training', 'prediction'], params=params, functions=['df_clsf.fit', 'df_clsf.predict'], - times=[fit_time, predict_time], accuracy_type='accuracy[%]', - accuracies=[train_acc, test_acc], data=[X_train, X_test]) + times=[fit_time, predict_time], metric_type='accuracy[%]', + metrics=[train_acc, test_acc], data=[X_train, X_test]) diff --git a/daal4py_bench/df_regr.py b/daal4py_bench/df_regr.py index 628e159fd..5ff2beb9b 100644 --- a/daal4py_bench/df_regr.py +++ b/daal4py_bench/df_regr.py @@ -123,5 +123,5 @@ def df_regr_predict(X, training_result): bench.print_output(library='daal4py', algorithm='decision_forest_regression', stages=['training', 'prediction'], params=params, functions=['df_regr.fit', 'df_regr.predict'], - times=[fit_time, predict_time], accuracy_type='rmse', - accuracies=[train_rmse, test_rmse], data=[X_train, X_test]) + times=[fit_time, predict_time], metric_type='rmse', + metrics=[train_rmse, test_rmse], data=[X_train, X_test]) diff --git a/daal4py_bench/distances.py b/daal4py_bench/distances.py index ed6896024..70408856d 100644 --- a/daal4py_bench/distances.py +++ b/daal4py_bench/distances.py @@ -43,5 +43,5 @@ def compute_distances(pairwise_distances, X): bench.print_output(library='daal4py', algorithm='distances', stages=['computation'], params=params, functions=[params.metric.capitalize()], times=[time], - accuracy_type=None, accuracies=[None], data=[X], + metric_type=None, metrics=[None], data=[X], alg_params={'metric': params.metric}) diff --git a/daal4py_bench/kmeans.py b/daal4py_bench/kmeans.py index 9a224713f..bc9b1afe4 100644 --- a/daal4py_bench/kmeans.py +++ b/daal4py_bench/kmeans.py @@ -87,5 +87,5 @@ def test_predict(X, X_init): bench.print_output(library='daal4py', algorithm='kmeans', stages=['training', 'prediction'], params=params, functions=['KMeans.fit', 'KMeans.predict'], - times=[fit_time, predict_time], accuracy_type='inertia', - accuracies=[train_inertia, test_inertia], data=[X_train, X_test]) + times=[fit_time, predict_time], metric_type='inertia', + metrics=[train_inertia, test_inertia], data=[X_train, X_test]) diff --git a/daal4py_bench/linear.py b/daal4py_bench/linear.py index 0b62a42a5..8cf076427 100644 --- a/daal4py_bench/linear.py +++ b/daal4py_bench/linear.py @@ -68,5 +68,5 @@ def test_predict(Xp, model): bench.print_output(library='daal4py', algorithm='linear_regression', stages=['training', 'prediction'], params=params, functions=['Linear.fit', 'Linear.predict'], - times=[fit_time, predict_time], accuracy_type='rmse', - accuracies=[train_rmse, test_rmse], data=[X_train, X_test]) + times=[fit_time, predict_time], metric_type='rmse', + metrics=[train_rmse, test_rmse], data=[X_train, X_test]) diff --git a/daal4py_bench/pca.py b/daal4py_bench/pca.py index 81161d1bd..98fb12e61 100644 --- a/daal4py_bench/pca.py +++ b/daal4py_bench/pca.py @@ -142,7 +142,7 @@ def test_transform(Xp, pca_result, eigenvalues, eigenvectors): bench.print_output(library='daal4py', algorithm='pca', stages=['training', 'transformation'], params=params, functions=['PCA.fit', 'PCA.transform'], - times=[fit_time, transform_time], accuracy_type=None, - accuracies=[None, None], data=[X_train, X_test], + times=[fit_time, transform_time], metric_type=None, + metrics=[None, None], data=[X_train, X_test], alg_params={'svd_solver': params.svd_solver, 'n_components': params.n_components}) diff --git a/daal4py_bench/ridge.py b/daal4py_bench/ridge.py index 1a04edf78..38722cb60 100644 --- a/daal4py_bench/ridge.py +++ b/daal4py_bench/ridge.py @@ -64,5 +64,5 @@ def test_predict(Xp, model): bench.print_output(library='daal4py', algorithm='ridge_regression', stages=['training', 'prediction'], params=params, functions=['Ridge.fit', 'Ridge.predict'], - times=[fit_time, predict_time], accuracy_type='rmse', - accuracies=[train_rmse, test_rmse], data=[X_train, X_test]) + times=[fit_time, predict_time], metric_type='rmse', + metrics=[train_rmse, test_rmse], data=[X_train, X_test]) diff --git a/datasets/loader_classification.py b/datasets/loader_classification.py index 5c8a91121..5a5d9df74 100644 --- a/datasets/loader_classification.py +++ b/datasets/loader_classification.py @@ -68,9 +68,11 @@ def airline(dataset_dir: Path) -> bool: Airline dataset http://kt.ijs.si/elena_ikonomovska/data.html - TaskType:binclass - NumberOfFeatures:13 - NumberOfInstances:115M + Classification task. n_classes = 2. + airline X train dataset (92055213, 13) + airline y train dataset (92055213, 1) + airline X test dataset (23013804, 13) + airline y test dataset (23013804, 1) """ dataset_name = 'airline' os.makedirs(dataset_dir, exist_ok=True) @@ -126,9 +128,12 @@ def airline(dataset_dir: Path) -> bool: def airline_ohe(dataset_dir: Path) -> bool: """ Dataset from szilard benchmarks: https://github.com/szilard/GBM-perf - TaskType:binclass - NumberOfFeatures:700 - NumberOfInstances:10100000 + + Classification task. n_classes = 2. + airline-ohe X train dataset (1000000, 692) + airline-ohe y train dataset (1000000, 1) + airline-ohe X test dataset (100000, 692) + airline-ohe y test dataset (100000, 1) """ dataset_name = 'airline-ohe' os.makedirs(dataset_dir, exist_ok=True) @@ -289,9 +294,11 @@ def epsilon(dataset_dir: Path) -> bool: Epsilon dataset https://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/binary.html - TaskType:binclass - NumberOfFeatures:2000 - NumberOfInstances:500K + Classification task. n_classes = 2. + epsilon X train dataset (400000, 2000) + epsilon y train dataset (400000, 1) + epsilon X test dataset (100000, 2000) + epsilon y test dataset (100000, 1) """ dataset_name = 'epsilon' os.makedirs(dataset_dir, exist_ok=True) @@ -444,9 +451,11 @@ def higgs(dataset_dir: Path) -> bool: Higgs dataset from UCI machine learning repository https://archive.ics.uci.edu/ml/datasets/HIGGS - TaskType:binclass - NumberOfFeatures:28 - NumberOfInstances:11M + Classification task. n_classes = 2. + higgs X train dataset (8799999, 28) + higgs y train dataset (8799999, 1) + higgs X test dataset (2200000, 28) + higgs y test dataset (2200000, 1) """ dataset_name = 'higgs' os.makedirs(dataset_dir, exist_ok=True) @@ -479,9 +488,11 @@ def higgs_one_m(dataset_dir: Path) -> bool: Only first 1.5M samples is taken - TaskType:binclass - NumberOfFeatures:28 - NumberOfInstances:1.5M + Classification task. n_classes = 2. + higgs1m X train dataset (1000000, 28) + higgs1m y train dataset (1000000, 1) + higgs1m X test dataset (500000, 28) + higgs1m y test dataset (500000, 1) """ dataset_name = 'higgs1m' os.makedirs(dataset_dir, exist_ok=True) diff --git a/datasets/loader_multiclass.py b/datasets/loader_multiclass.py index 0c2013a1f..34033a714 100644 --- a/datasets/loader_multiclass.py +++ b/datasets/loader_multiclass.py @@ -99,9 +99,11 @@ def covtype(dataset_dir: Path) -> bool: https://archive.ics.uci.edu/ml/datasets/covertype y contains 7 unique class labels from 1 to 7 inclusive. - TaskType:multiclass - NumberOfFeatures:54 - NumberOfInstances:581012 + Classification task. n_classes = 7. + covtype X train dataset (464809, 54) + covtype y train dataset (464809, 1) + covtype X test dataset (116203, 54) + covtype y test dataset (116203, 1) """ dataset_name = 'covtype' os.makedirs(dataset_dir, exist_ok=True) @@ -125,9 +127,11 @@ def letters(dataset_dir: Path) -> bool: """ http://archive.ics.uci.edu/ml/datasets/Letter+Recognition - TaskType:multiclass - NumberOfFeatures:16 - NumberOfInstances:20.000 + Classification task. n_classes = 26. + letters X train dataset (16000, 16) + letters y train dataset (16000, 1) + letters X test dataset (4000, 16) + letters y test dataset (4000, 1) """ dataset_name = 'letters' os.makedirs(dataset_dir, exist_ok=True) @@ -204,9 +208,11 @@ def msrank(dataset_dir: Path) -> bool: """ Dataset from szilard benchmarks: https://github.com/szilard/GBM-perf - TaskType:multiclass - NumberOfFeatures:137 - NumberOfInstances:1.2M + Classification task. n_classes = 5. + msrank X train dataset (958671, 137) + msrank y train dataset (958671, 1) + msrank X test dataset (241521, 137) + msrank y test dataset (241521, 1) """ dataset_name = 'msrank' os.makedirs(dataset_dir, exist_ok=True) @@ -264,7 +270,7 @@ def sensit(dataset_dir: Path) -> bool: Author: M. Duarte, Y. H. Hu Source: [original](http://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets) - Multiclass classification task + Classification task. n_classes = 3. sensit X train dataset (78822, 100) sensit y train dataset (78822, 1) sensit X test dataset (19706, 100) diff --git a/datasets/loader_regression.py b/datasets/loader_regression.py index 2f330f799..3a5553e61 100644 --- a/datasets/loader_regression.py +++ b/datasets/loader_regression.py @@ -32,9 +32,10 @@ def abalone(dataset_dir: Path) -> bool: """ https://archive.ics.uci.edu/ml/machine-learning-databases/abalone - TaskType:regression - NumberOfFeatures:8 - NumberOfInstances:4177 + abalone x train dataset (3341, 8) + abalone y train dataset (3341, 1) + abalone x test dataset (836, 8) + abalone y train dataset (836, 1) """ dataset_name = 'abalone' os.makedirs(dataset_dir, exist_ok=True) @@ -196,9 +197,10 @@ def year_prediction_msd(dataset_dir: Path) -> bool: YearPredictionMSD dataset from UCI repository https://archive.ics.uci.edu/ml/datasets/yearpredictionmsd - TaskType:regression - NumberOfFeatures:90 - NumberOfInstances:515345 + year_prediction_msd x train dataset (463715, 90) + year_prediction_msd y train dataset (463715, 1) + year_prediction_msd x test dataset (51630, 90) + year_prediction_msd y train dataset (51630, 1) """ dataset_name = 'year_prediction_msd' os.makedirs(dataset_dir, exist_ok=True) diff --git a/modelbuilders_bench/lgbm_mb.py b/modelbuilders_bench/lgbm_mb.py index 2b4c29616..d70f46772 100644 --- a/modelbuilders_bench/lgbm_mb.py +++ b/modelbuilders_bench/lgbm_mb.py @@ -141,11 +141,14 @@ test_metric_daal = metric_func(y_test, daal_pred.prediction) utils.print_output( - library='modelbuilders', algorithm=f'lightgbm_{task}_and_modelbuilder', + library='modelbuilders', + algorithm=f'lightgbm_{task}_and_modelbuilder', stages=['lgbm_train', 'lgbm_predict', 'daal4py_predict'], - params=params, functions=['lgbm_dataset', 'lgbm_dataset', 'lgbm_train', - 'lgbm_predict', 'lgbm_to_daal', 'daal_compute'], + params=params, + functions=['lgbm_dataset', 'lgbm_dataset', 'lgbm_train', + 'lgbm_predict', 'lgbm_to_daal', 'daal_compute'], times=[t_creat_train, t_train, t_creat_test, t_lgbm_pred, t_trans, t_daal_pred], - accuracy_type=metric_name, accuracies=[train_metric, test_metric_lgbm, - test_metric_daal], - data=[X_train, X_test, X_test]) + metric_type=metric_name, + metrics=[train_metric, test_metric_lgbm, test_metric_daal], + data=[X_train, X_test, X_test], +) diff --git a/modelbuilders_bench/mb_utils.py b/modelbuilders_bench/mb_utils.py index d66adaac8..2d659dc09 100644 --- a/modelbuilders_bench/mb_utils.py +++ b/modelbuilders_bench/mb_utils.py @@ -37,7 +37,7 @@ def get_accuracy(true_labels, prediction): def print_output(library, algorithm, stages, params, functions, - times, accuracy_type, accuracies, data): + times, metric_type, metrics, data): if params.output_format == 'json': output = [] output.append({ @@ -67,7 +67,7 @@ def print_output(library, algorithm, stages, params, functions, else: result.update({'matrix_creation_time': times[2 * i], 'prediction_time': times[2 * i + 1]}) - if accuracies[i] is not None: - result.update({f'{accuracy_type}': accuracies[i]}) + if metrics[i] is not None: + result.update({f'{metric_type}': metrics[i]}) output.append(result) print(json.dumps(output, indent=4)) diff --git a/modelbuilders_bench/xgb_mb.py b/modelbuilders_bench/xgb_mb.py index 92764142d..54bf4bd95 100644 --- a/modelbuilders_bench/xgb_mb.py +++ b/modelbuilders_bench/xgb_mb.py @@ -196,6 +196,6 @@ def predict(dmatrix): # type: ignore 'daal4py.get_gbt_model_from_xgboost', 'daal4py.compute'], times=[t_creat_train, fit_time, t_creat_test, predict_time, transform_time, predict_time_daal], - accuracy_type=metric_name, - accuracies=[None, train_metric, None, test_metric, None, test_metric_daal], + metric_type=metric_name, + metrics=[None, train_metric, None, test_metric, None, test_metric_daal], data=[X_train, X_train, X_test, X_test, X_test, X_test]) diff --git a/sklearn_bench/dbscan.py b/sklearn_bench/dbscan.py index 6765b2970..94a55bafa 100644 --- a/sklearn_bench/dbscan.py +++ b/sklearn_bench/dbscan.py @@ -21,7 +21,6 @@ def main(): from sklearn.cluster import DBSCAN - from sklearn.metrics.cluster import davies_bouldin_score # Load generated data X, _, _, _ = bench.load_data(params, add_dtype=True) @@ -40,11 +39,11 @@ def main(): labels = dbscan.labels_ params.n_clusters = len(set(labels)) - (1 if -1 in labels else 0) - acc = davies_bouldin_score(X, labels) + acc = bench.davies_bouldin_score(X, labels) bench.print_output(library='sklearn', algorithm='dbscan', stages=['training'], params=params, functions=['DBSCAN'], times=[time], - accuracies=[acc], accuracy_type='davies_bouldin_score', + metrics=[acc], metric_type='davies_bouldin_score', data=[X], alg_instance=dbscan) diff --git a/sklearn_bench/df_clsf.py b/sklearn_bench/df_clsf.py index bb69185a3..95709340c 100644 --- a/sklearn_bench/df_clsf.py +++ b/sklearn_bench/df_clsf.py @@ -18,7 +18,6 @@ import bench import numpy as np -from sklearn.metrics import accuracy_score def main(): @@ -43,18 +42,34 @@ def main(): fit_time, _ = bench.measure_function_time(clf.fit, X_train, y_train, params=params) y_pred = clf.predict(X_train) - train_acc = 100 * accuracy_score(y_pred, y_train) + y_proba = clf.predict_proba(X_train) + train_acc = bench.accuracy_score(y_train, y_pred) + train_log_loss = bench.log_loss(y_train, y_proba) + train_roc_auc = bench.roc_auc_score(y_train, y_proba) predict_time, y_pred = bench.measure_function_time( clf.predict, X_test, params=params) - test_acc = 100 * accuracy_score(y_pred, y_test) + y_proba = clf.predict_proba(X_test) + test_acc = bench.accuracy_score(y_test, y_pred) + test_log_loss = bench.log_loss(y_test, y_proba) + test_roc_auc = bench.roc_auc_score(y_test, y_proba) - bench.print_output(library='sklearn', algorithm='decision_forest_classification', - stages=['training', 'prediction'], params=params, - functions=['df_clsf.fit', 'df_clsf.predict'], - times=[fit_time, predict_time], accuracy_type='accuracy[%]', - accuracies=[train_acc, test_acc], data=[X_train, X_test], - alg_instance=clf) + bench.print_output( + library='sklearn', + algorithm='decision_forest_classification', + stages=['training', 'prediction'], + params=params, + functions=['df_clsf.fit', 'df_clsf.predict'], + times=[fit_time, predict_time], + metric_type=['accuracy', 'log_loss', 'roc_auc'], + metrics=[ + [train_acc, test_acc], + [train_log_loss, test_log_loss], + [train_roc_auc, test_roc_auc], + ], + data=[X_train, X_test], + alg_instance=clf, + ) if __name__ == "__main__": diff --git a/sklearn_bench/df_regr.py b/sklearn_bench/df_regr.py index 53d3c8afd..2d12c65f7 100644 --- a/sklearn_bench/df_regr.py +++ b/sklearn_bench/df_regr.py @@ -40,18 +40,26 @@ def main(): fit_time, _ = bench.measure_function_time(regr.fit, X_train, y_train, params=params) y_pred = regr.predict(X_train) - train_rmse = bench.rmse_score(y_pred, y_train) + train_rmse = bench.rmse_score(y_train, y_pred) + train_r2 = bench.r2_score(y_train, y_pred) predict_time, y_pred = bench.measure_function_time( regr.predict, X_test, params=params) - test_rmse = bench.rmse_score(y_pred, y_test) + test_rmse = bench.rmse_score(y_test, y_pred) + test_r2 = bench.r2_score(y_test, y_pred) - bench.print_output(library='sklearn', algorithm='decision_forest_regression', - stages=['training', 'prediction'], params=params, - functions=['df_regr.fit', 'df_regr.predict'], - times=[fit_time, predict_time], accuracy_type='rmse', - accuracies=[train_rmse, test_rmse], data=[X_train, X_test], - alg_instance=regr) + bench.print_output( + library='sklearn', + algorithm='decision_forest_regression', + stages=['training', 'prediction'], + params=params, + functions=['df_regr.fit', 'df_regr.predict'], + times=[fit_time, predict_time], + metric_type=['rmse', 'r2_score'], + metrics=[[train_rmse, test_rmse], [train_r2, test_r2]], + data=[X_train, X_test], + alg_instance=regr, + ) if __name__ == "__main__": diff --git a/sklearn_bench/distances.py b/sklearn_bench/distances.py index a1eeaa989..c708513d1 100644 --- a/sklearn_bench/distances.py +++ b/sklearn_bench/distances.py @@ -30,7 +30,7 @@ def main(): bench.print_output(library='sklearn', algorithm='distances', stages=['computation'], params=params, functions=[params.metric.capitalize()], - times=[time], accuracy_type=None, accuracies=[None], data=[X], + times=[time], metric_type=None, metrics=[None], data=[X], alg_params={'metric': params.metric}) diff --git a/sklearn_bench/elasticnet.py b/sklearn_bench/elasticnet.py index ac7f34050..764995e0b 100755 --- a/sklearn_bench/elasticnet.py +++ b/sklearn_bench/elasticnet.py @@ -33,19 +33,27 @@ def main(): fit_time, _ = bench.measure_function_time(regr.fit, X_train, y_train, params=params) # Time predict - predict_time, pred_train = bench.measure_function_time(regr.predict, - X_train, params=params) + predict_time, y_pred = bench.measure_function_time(regr.predict, + X_train, params=params) - train_rmse = bench.rmse_score(pred_train, y_train) - pred_test = regr.predict(X_test) - test_rmse = bench.rmse_score(pred_test, y_test) + train_rmse = bench.rmse_score(y_train, y_pred) + train_r2 = bench.r2_score(y_train, y_pred) + y_pred = regr.predict(X_test) + test_rmse = bench.rmse_score(y_test, y_pred) + test_r2 = bench.r2_score(y_test, y_pred) - bench.print_output(library='sklearn', algorithm='elastic-net', - stages=['training', 'prediction'], params=params, - functions=['ElasticNet.fit', 'ElasticNet.predict'], - times=[fit_time, predict_time], accuracy_type='rmse', - accuracies=[train_rmse, test_rmse], data=[X_train, X_train], - alg_instance=regr) + bench.print_output( + library='sklearn', + algorithm='elastic-net', + stages=['training', 'prediction'], + params=params, + functions=['ElasticNet.fit', 'ElasticNet.predict'], + times=[fit_time, predict_time], + metric_type=['rmse', 'r2_score'], + metrics=[[train_rmse, test_rmse], [train_r2, test_r2]], + data=[X_train, X_train], + alg_instance=regr, + ) if __name__ == "__main__": diff --git a/sklearn_bench/kmeans.py b/sklearn_bench/kmeans.py index 4df5ba03f..b522a0e92 100644 --- a/sklearn_bench/kmeans.py +++ b/sklearn_bench/kmeans.py @@ -48,7 +48,8 @@ def main(): def fit_kmeans(X, X_init): alg = KMeans(n_clusters=params.n_clusters, tol=params.tol, - max_iter=params.maxiter, init=X_init, n_init=1) + max_iter=params.maxiter, init=X_init, n_init=params.n_init, + algorithm=params.algorithm, random_state=params.random_state) alg.fit(X) return alg @@ -65,13 +66,22 @@ def fit_kmeans(X, X_init): acc_test = davies_bouldin_score(X_test, test_predict) - bench.print_output(library='sklearn', algorithm='kmeans', - stages=['training', 'prediction'], - params=params, functions=['KMeans.fit', 'KMeans.predict'], - times=[fit_time, predict_time], - accuracy_type='davies_bouldin_score', - accuracies=[acc_train, acc_test], data=[X_train, X_test], - alg_instance=kmeans) + bench.print_output( + library='sklearn', + algorithm='kmeans', + stages=['training', 'prediction'], + params=params, + functions=['KMeans.fit', 'KMeans.predict'], + times=[fit_time, predict_time], + metric_type=['davies_bouldin_score', 'inertia', 'iter'], + metrics=[ + [acc_train, acc_test], + [kmeans.inertia_, kmeans.inertia_], + [kmeans.n_iter_, kmeans.n_iter_] + ], + data=[X_train, X_test], + alg_instance=kmeans, + ) if __name__ == "__main__": @@ -83,5 +93,12 @@ def fit_kmeans(X, X_init): parser.add_argument('--maxiter', type=int, default=100, help='Maximum number of iterations') parser.add_argument('--n-clusters', type=int, help='Number of clusters') + parser.add_argument('--algorithm', type=str, default='full', + help='K-means algorithm to use') + parser.add_argument('--n_init', type=int, default=1, + help='Number of time the k-means algorithm ' + 'will be run with different centroid seeds') + parser.add_argument('--random_state', type=int, default=777, + help='Random state') params = bench.parse_args(parser) bench.run_with_context(params, main) diff --git a/sklearn_bench/knn_clsf.py b/sklearn_bench/knn_clsf.py index 6c2c28af8..ef581f537 100755 --- a/sklearn_bench/knn_clsf.py +++ b/sklearn_bench/knn_clsf.py @@ -18,7 +18,6 @@ import bench import numpy as np -from sklearn.metrics import accuracy_score def main(): @@ -40,33 +39,53 @@ def main(): knn_clsf.fit, X_train, y_train, params=params) if params.task == 'classification': y_pred = knn_clsf.predict(X_train) - train_acc = 100 * accuracy_score(y_pred, y_train) + y_proba = knn_clsf.predict_proba(X_train) + train_acc = bench.accuracy_score(y_train, y_pred) + train_log_loss = bench.log_loss(y_train, y_proba) + train_roc_auc = bench.roc_auc_score(y_train, y_proba) # Measure time and accuracy on prediction if params.task == 'classification': predict_time, yp = bench.measure_function_time(knn_clsf.predict, X_test, params=params) - test_acc = 100 * accuracy_score(yp, y_test) + y_proba = knn_clsf.predict_proba(X_test) + test_acc = bench.accuracy_score(y_test, yp) + test_log_loss = bench.log_loss(y_test, y_proba) + test_roc_auc = bench.roc_auc_score(y_test, y_proba) else: predict_time, _ = bench.measure_function_time(knn_clsf.kneighbors, X_test, params=params) if params.task == 'classification': - bench.print_output(library='sklearn', - algorithm=knn_clsf._fit_method + '_knn_classification', - stages=['training', 'prediction'], params=params, - functions=['knn_clsf.fit', 'knn_clsf.predict'], - times=[train_time, predict_time], - accuracies=[train_acc, test_acc], accuracy_type='accuracy[%]', - data=[X_train, X_test], alg_instance=knn_clsf) + bench.print_output( + library='sklearn', + algorithm=knn_clsf._fit_method + '_knn_classification', + stages=['training', 'prediction'], + params=params, + functions=['knn_clsf.fit', 'knn_clsf.predict'], + times=[train_time, predict_time], + metric_type=['accuracy', 'log_loss', 'roc_auc'], + metrics=[ + [train_acc, test_acc], + [train_log_loss, test_log_loss], + [train_roc_auc, test_roc_auc], + ], + data=[X_train, X_test], + alg_instance=knn_clsf, + ) else: - bench.print_output(library='sklearn', - algorithm=knn_clsf._fit_method + '_knn_search', - stages=['training', 'search'], params=params, - functions=['knn_clsf.fit', 'knn_clsf.kneighbors'], - times=[train_time, predict_time], - accuracies=[], accuracy_type=None, - data=[X_train, X_test], alg_instance=knn_clsf) + bench.print_output( + library='sklearn', + algorithm=knn_clsf._fit_method + '_knn_search', + stages=['training', 'search'], + params=params, + functions=['knn_clsf.fit', 'knn_clsf.kneighbors'], + times=[train_time, predict_time], + metric_type=None, + metrics=[], + data=[X_train, X_test], + alg_instance=knn_clsf, + ) if __name__ == "__main__": diff --git a/sklearn_bench/lasso.py b/sklearn_bench/lasso.py index 32fd0d591..d0e10cb7c 100755 --- a/sklearn_bench/lasso.py +++ b/sklearn_bench/lasso.py @@ -33,19 +33,27 @@ def main(): fit_time, _ = bench.measure_function_time(regr.fit, X_train, y_train, params=params) # Time predict - predict_time, pred_train = bench.measure_function_time( + predict_time, yp = bench.measure_function_time( regr.predict, X_train, params=params) - train_rmse = bench.rmse_score(pred_train, y_train) - pred_test = regr.predict(X_test) - test_rmse = bench.rmse_score(pred_test, y_test) + train_rmse = bench.rmse_score(y_train, yp) + train_r2 = bench.r2_score(y_train, yp) + yp = regr.predict(X_test) + test_rmse = bench.rmse_score(y_test, yp) + test_r2 = bench.r2_score(y_test, yp) - bench.print_output(library='sklearn', algorithm='lasso', - stages=['training', 'prediction'], params=params, - functions=['Lasso.fit', 'Lasso.predict'], - times=[fit_time, predict_time], accuracy_type='rmse', - accuracies=[train_rmse, test_rmse], data=[X_train, X_test], - alg_instance=regr) + bench.print_output( + library='sklearn', + algorithm='lasso', + stages=['training', 'prediction'], + params=params, + functions=['Lasso.fit', 'Lasso.predict'], + times=[fit_time, predict_time], + metric_type=['rmse', 'r2_score'], + metrics=[[train_rmse, test_rmse], [train_r2, test_r2]], + data=[X_train, X_test], + alg_instance=regr, + ) if __name__ == "__main__": diff --git a/sklearn_bench/linear.py b/sklearn_bench/linear.py index f926c6ce8..c7390efbe 100644 --- a/sklearn_bench/linear.py +++ b/sklearn_bench/linear.py @@ -15,7 +15,6 @@ # =============================================================================== import argparse - import bench @@ -36,16 +35,22 @@ def main(): # Time predict predict_time, yp = bench.measure_function_time(regr.predict, X_test, params=params) - test_rmse = bench.rmse_score(yp, y_test) + test_rmse = bench.rmse_score(y_test, yp) + test_r2 = bench.r2_score(y_test, yp) yp = regr.predict(X_train) - train_rmse = bench.rmse_score(yp, y_train) - - bench.print_output(library='sklearn', algorithm='linear_regression', - stages=['training', 'prediction'], - params=params, functions=['Linear.fit', 'Linear.predict'], - times=[fit_time, predict_time], accuracy_type='rmse', - accuracies=[train_rmse, test_rmse], data=[X_train, X_test], - alg_instance=regr) + train_rmse = bench.rmse_score(y_train, yp) + train_r2 = bench.r2_score(y_train, yp) + + bench.print_output( + library='sklearn', algorithm='linear_regression', + stages=['training', 'prediction'], + params=params, functions=['Linear.fit', 'Linear.predict'], + times=[fit_time, predict_time], + metric_type=['rmse', 'r2_score'], + metrics=[[train_rmse, test_rmse], [train_r2, test_r2]], + data=[X_train, X_test], + alg_instance=regr, + ) if __name__ == "__main__": diff --git a/sklearn_bench/log_reg.py b/sklearn_bench/log_reg.py index dc9d750bd..1053d3819 100644 --- a/sklearn_bench/log_reg.py +++ b/sklearn_bench/log_reg.py @@ -22,7 +22,6 @@ def main(): from sklearn.linear_model import LogisticRegression - from sklearn.metrics import accuracy_score # Load generated data X_train, X_test, y_train, y_test = bench.load_data(params) @@ -45,18 +44,34 @@ def main(): fit_time, _ = bench.measure_function_time(clf.fit, X_train, y_train, params=params) y_pred = clf.predict(X_train) - train_acc = 100 * accuracy_score(y_pred, y_train) + y_proba = clf.predict_proba(X_train) + train_acc = bench.accuracy_score(y_train, y_pred) + train_log_loss = bench.log_loss(y_train, y_proba) + train_roc_auc = bench.roc_auc_score(y_train, y_proba) predict_time, y_pred = bench.measure_function_time( clf.predict, X_test, params=params) - test_acc = 100 * accuracy_score(y_pred, y_test) + y_proba = clf.predict_proba(X_test) + test_acc = bench.accuracy_score(y_test, y_pred) + test_log_loss = bench.log_loss(y_test, y_proba) + test_roc_auc = bench.roc_auc_score(y_test, y_proba) - bench.print_output(library='sklearn', algorithm='logistic_regression', - stages=['training', 'prediction'], params=params, - functions=['LogReg.fit', 'LogReg.predict'], - times=[fit_time, predict_time], accuracy_type='accuracy[%]', - accuracies=[train_acc, test_acc], data=[X_train, X_test], - alg_instance=clf) + bench.print_output( + library='sklearn', + algorithm='logistic_regression', + stages=['training', 'prediction'], + params=params, + functions=['LogReg.fit', 'LogReg.predict'], + times=[fit_time, predict_time], + metric_type=['accuracy', 'log_loss', 'roc_auc'], + metrics=[ + [train_acc, test_acc], + [train_log_loss, test_log_loss], + [train_roc_auc, test_roc_auc], + ], + data=[X_train, X_test], + alg_instance=clf, + ) if __name__ == "__main__": diff --git a/sklearn_bench/nusvc.py b/sklearn_bench/nusvc.py index 52f5b6d3d..d98b184df 100644 --- a/sklearn_bench/nusvc.py +++ b/sklearn_bench/nusvc.py @@ -43,33 +43,45 @@ def main(): if params.probability: state_predict = 'predict_proba' - accuracy_type = 'log_loss' clf_predict = clf.predict_proba - - def metric_call(x, y): - return bench.log_loss(x, y) + y_proba_train = clf_predict(X_train) + y_proba_test = clf_predict(X_test) + train_log_loss = bench.log_loss(y_train, y_proba_train) + test_log_loss = bench.log_loss(y_test, y_proba_test) + train_roc_auc = bench.roc_auc_score(y_train, y_proba_train) + test_roc_auc = bench.roc_auc_score(y_test, y_proba_test) else: state_predict = 'prediction' - accuracy_type = 'accuracy[%]' clf_predict = clf.predict - - def metric_call(x, y): - return bench.accuracy_score(x, y) + train_log_loss = None + test_log_loss = None + train_roc_auc = None + test_roc_auc = None predict_train_time, y_pred = bench.measure_function_time( clf_predict, X_train, params=params) - train_acc = metric_call(y_train, y_pred) + train_acc = bench.accuracy_score(y_train, y_pred) _, y_pred = bench.measure_function_time( clf_predict, X_test, params=params) - test_acc = metric_call(y_test, y_pred) - - bench.print_output(library='sklearn', algorithm='nusvc', - stages=['training', state_predict], - params=params, functions=['NuSVC.fit', f'NuSVC.{state_predict}'], - times=[fit_time, predict_train_time], accuracy_type=accuracy_type, - accuracies=[train_acc, test_acc], data=[X_train, X_train], - alg_instance=clf) + test_acc = bench.accuracy_score(y_test, y_pred) + + bench.print_output( + library='sklearn', + algorithm='nusvc', + stages=['training', state_predict], + params=params, functions=['NuSVC.fit', f'NuSVC.{state_predict}'], + times=[fit_time, predict_train_time], + metric_type=['accuracy', 'log_loss', 'roc_auc', 'n_sv'], + metrics=[ + [train_acc, test_acc], + [train_log_loss, test_log_loss], + [train_roc_auc, test_roc_auc], + [int(clf.n_support_.sum()), int(clf.n_support_.sum())], + ], + data=[X_train, X_train], + alg_instance=clf, + ) if __name__ == "__main__": diff --git a/sklearn_bench/nusvr.py b/sklearn_bench/nusvr.py index 3d9a23780..d31b7d26e 100644 --- a/sklearn_bench/nusvr.py +++ b/sklearn_bench/nusvr.py @@ -44,17 +44,29 @@ def main(): predict_train_time, y_pred = bench.measure_function_time( regr.predict, X_train, params=params) train_rmse = bench.rmse_score(y_train, y_pred) + train_r2 = bench.r2_score(y_train, y_pred) _, y_pred = bench.measure_function_time( regr.predict, X_test, params=params) test_rmse = bench.rmse_score(y_test, y_pred) - - bench.print_output(library='sklearn', algorithm='nusvr', - stages=['training', 'prediction'], - params=params, functions=['NuSVR.fit', 'NuSVR.predict'], - times=[fit_time, predict_train_time], accuracy_type='rmse', - accuracies=[train_rmse, test_rmse], data=[X_train, X_train], - alg_instance=regr) + test_r2 = bench.r2_score(y_test, y_pred) + + bench.print_output( + library='sklearn', + algorithm='nusvr', + stages=['training', 'prediction'], + params=params, + functions=['NuSVR.fit', 'NuSVR.predict'], + times=[fit_time, predict_train_time], + metric_type=['rmse', 'r2_score', 'n_sv'], + metrics=[ + [train_rmse, test_rmse], + [train_r2, test_r2], + [int(regr.n_support_.sum()), int(regr.n_support_.sum())], + ], + data=[X_train, X_train], + alg_instance=regr, + ) if __name__ == "__main__": diff --git a/sklearn_bench/pca.py b/sklearn_bench/pca.py index b810603a8..356bfed09 100644 --- a/sklearn_bench/pca.py +++ b/sklearn_bench/pca.py @@ -40,12 +40,18 @@ def main(): transform_time, _ = bench.measure_function_time( pca.transform, X_train, params=params) - bench.print_output(library='sklearn', algorithm='pca', - stages=['training', 'transformation'], - params=params, functions=['PCA.fit', 'PCA.transform'], - times=[fit_time, transform_time], accuracy_type=None, - accuracies=[None, None], data=[X_train, X_test], - alg_instance=pca) + bench.print_output( + library='sklearn', + algorithm='pca', + stages=['training', 'transformation'], + params=params, + functions=['PCA.fit', 'PCA.transform'], + times=[fit_time, transform_time], + metric_type='noise_variance', + metrics=[pca.noise_variance_, pca.noise_variance_], + data=[X_train, X_test], + alg_instance=pca, + ) if __name__ == "__main__": diff --git a/sklearn_bench/ridge.py b/sklearn_bench/ridge.py index ee47a616d..3b8f138d2 100644 --- a/sklearn_bench/ridge.py +++ b/sklearn_bench/ridge.py @@ -36,16 +36,24 @@ def main(): # Time predict predict_time, yp = bench.measure_function_time(regr.predict, X_test, params=params) - test_rmse = bench.rmse_score(yp, y_test) + test_rmse = bench.rmse_score(y_test, yp) + test_r2 = bench.r2_score(y_test, yp) yp = regr.predict(X_train) - train_rmse = bench.rmse_score(yp, y_train) - - bench.print_output(library='sklearn', algorithm='ridge_regression', - stages=['training', 'prediction'], params=params, - functions=['Ridge.fit', 'Ridge.predict'], - times=[fit_time, predict_time], accuracy_type='rmse', - accuracies=[train_rmse, test_rmse], data=[X_train, X_test], - alg_instance=regr) + train_rmse = bench.rmse_score(y_train, yp) + train_r2 = bench.r2_score(y_train, yp) + + bench.print_output( + library='sklearn', + algorithm='ridge_regression', + stages=['training', 'prediction'], + params=params, + functions=['Ridge.fit', 'Ridge.predict'], + times=[fit_time, predict_time], + metric_type=['rmse', 'r2_score'], + metrics=[[train_rmse, test_rmse], [train_r2, test_r2]], + data=[X_train, X_test], + alg_instance=regr, + ) if __name__ == "__main__": diff --git a/sklearn_bench/svm.py b/sklearn_bench/svm.py index 9491ed136..6e17ea00a 100644 --- a/sklearn_bench/svm.py +++ b/sklearn_bench/svm.py @@ -43,33 +43,46 @@ def main(): if params.probability: state_predict = 'predict_proba' - accuracy_type = 'log_loss' clf_predict = clf.predict_proba - - def metric_call(x, y): - return bench.log_loss(x, y) + y_proba_train = clf_predict(X_train) + y_proba_test = clf_predict(X_test) + train_log_loss = bench.log_loss(y_train, y_proba_train) + test_log_loss = bench.log_loss(y_test, y_proba_test) + train_roc_auc = bench.roc_auc_score(y_train, y_proba_train) + test_roc_auc = bench.roc_auc_score(y_test, y_proba_test) else: state_predict = 'prediction' - accuracy_type = 'accuracy[%]' clf_predict = clf.predict - - def metric_call(x, y): - return bench.accuracy_score(x, y) + train_log_loss = None + test_log_loss = None + train_roc_auc = None + test_roc_auc = None predict_train_time, y_pred = bench.measure_function_time( clf_predict, X_train, params=params) - train_acc = metric_call(y_train, y_pred) + train_acc = bench.accuracy_score(y_train, y_pred) _, y_pred = bench.measure_function_time( clf_predict, X_test, params=params) - test_acc = metric_call(y_test, y_pred) - - bench.print_output(library='sklearn', algorithm='svc', - stages=['training', state_predict], - params=params, functions=['SVM.fit', f'SVM.{state_predict}'], - times=[fit_time, predict_train_time], accuracy_type=accuracy_type, - accuracies=[train_acc, test_acc], data=[X_train, X_train], - alg_instance=clf) + test_acc = bench.accuracy_score(y_test, y_pred) + + bench.print_output( + library='sklearn', + algorithm='svc', + stages=['training', state_predict], + params=params, + functions=['SVM.fit', f'SVM.{state_predict}'], + times=[fit_time, predict_train_time], + metric_type=['accuracy', 'log_loss', 'roc_auc', 'n_sv'], + metrics=[ + [train_acc, test_acc], + [train_log_loss, test_log_loss], + [train_roc_auc, test_roc_auc], + [int(clf.n_support_.sum()), int(clf.n_support_.sum())], + ], + data=[X_train, X_train], + alg_instance=clf, + ) if __name__ == "__main__": diff --git a/sklearn_bench/svr.py b/sklearn_bench/svr.py index 0a20c4638..a3447332b 100644 --- a/sklearn_bench/svr.py +++ b/sklearn_bench/svr.py @@ -44,17 +44,29 @@ def main(): predict_train_time, y_pred = bench.measure_function_time( regr.predict, X_train, params=params) train_rmse = bench.rmse_score(y_train, y_pred) + train_r2 = bench.r2_score(y_train, y_pred) _, y_pred = bench.measure_function_time( regr.predict, X_test, params=params) test_rmse = bench.rmse_score(y_test, y_pred) - - bench.print_output(library='sklearn', algorithm='svr', - stages=['training', 'prediction'], - params=params, functions=['SVR.fit', 'SVR.predict'], - times=[fit_time, predict_train_time], accuracy_type='rmse', - accuracies=[train_rmse, test_rmse], data=[X_train, X_train], - alg_instance=regr) + test_r2 = bench.r2_score(y_test, y_pred) + + bench.print_output( + library='sklearn', + algorithm='svr', + stages=['training', 'prediction'], + params=params, + functions=['SVR.fit', 'SVR.predict'], + times=[fit_time, predict_train_time], + metric_type=['rmse', 'r2_score', 'n_sv'], + metrics=[ + [train_rmse, test_rmse], + [train_r2, test_r2], + [int(regr.n_support_.sum()), int(regr.n_support_.sum())], + ], + data=[X_train, X_train], + alg_instance=regr, + ) if __name__ == "__main__": diff --git a/sklearn_bench/train_test_split.py b/sklearn_bench/train_test_split.py index aac1ec2e3..046719b48 100644 --- a/sklearn_bench/train_test_split.py +++ b/sklearn_bench/train_test_split.py @@ -47,8 +47,8 @@ def main(): bench.print_output(library='sklearn', algorithm='train_test_split', stages=['training'], params=params, - functions=['train_test_split'], times=[time], accuracies=[None], - accuracy_type=None, data=[X], alg_params=tts_params) + functions=['train_test_split'], times=[time], metrics=[None], + metric_type=None, data=[X], alg_params=tts_params) if __name__ == "__main__": diff --git a/xgboost_bench/gbt.py b/xgboost_bench/gbt.py index aa54a094d..8540f4f5a 100644 --- a/xgboost_bench/gbt.py +++ b/xgboost_bench/gbt.py @@ -176,6 +176,6 @@ def predict(dmatrix): # type: ignore bench.print_output(library='xgboost', algorithm=f'gradient_boosted_trees_{task}', stages=['training', 'prediction'], params=params, functions=['gbt.fit', 'gbt.predict'], - times=[fit_time, predict_time], accuracy_type=metric_name, - accuracies=[train_metric, test_metric], data=[X_train, X_test], + times=[fit_time, predict_time], metric_type=metric_name, + metrics=[train_metric, test_metric], data=[X_train, X_test], alg_instance=booster, alg_params=xgb_params)