在 sklearn cross_val_score 上评估多个分数

我正在尝试使用 sklearn 评估多个机器学习算法的几个指标（准确性、召回率、精度等等）。

对于我从这里的文档和源代码（我使用的是 sklearn 0.17）中了解到的，cross_val_score 函数每次执行只接收一个记分器。所以为了计算多个分数，我必须：

多次执行

实现我的（耗时且容易出错）记分器

我已经用这段代码执行了多次：

from sklearn.svm import SVC
from sklearn.naive_bayes import GaussianNB
from sklearn.tree import DecisionTreeClassifier
from sklearn.cross_validation import  cross_val_score
import time
from sklearn.datasets import  load_iris
iris = load_iris()
models = [GaussianNB(), DecisionTreeClassifier(), SVC()]
names = ["Naive Bayes", "Decision Tree", "SVM"]
for model, name in zip(models, names):
    print name
    start = time.time()
    for score in ["accuracy", "precision", "recall"]:
        print score,
        print " : ",
        print cross_val_score(model, iris.data, iris.target,scoring=score, cv=10).mean()
    print time.time() - start

我得到这个输出：

Naive Bayes
accuracy  :  0.953333333333
precision  :  0.962698412698
recall  :  0.953333333333
0.0383198261261
Decision Tree
accuracy  :  0.953333333333
precision  :  0.958888888889
recall  :  0.953333333333
0.0494720935822
SVM
accuracy  :  0.98
precision  :  0.983333333333
recall  :  0.98
0.063080072403

这没关系，但对于我自己的数据来说很慢。如何衡量所有分数？

自从撰写这篇文章以来，scikit-learn已经更新并使我的答案过时，请参阅下面的更干净的解决方案

您可以编写自己的评分函数来捕获所有三条信息，但是用于交叉验证的评分函数必须只返回scikit-learn中的一个数字（这可能是出于兼容性原因）。以下示例显示每个交叉验证切片的每个分数打印到控制台，返回的值只是三个指标的总和。如果要返回所有这些值，则必须对cross_val_score（cross_validation.py的第 1351 行）和_score（第 1601 行或同一文件）进行一些更改。

from sklearn.svm import SVC
from sklearn.naive_bayes import GaussianNB
from sklearn.tree import DecisionTreeClassifier
from sklearn.cross_validation import  cross_val_score
import time
from sklearn.datasets import  load_iris
from sklearn.metrics import accuracy_score, precision_score, recall_score
iris = load_iris()
models = [GaussianNB(), DecisionTreeClassifier(), SVC()]
names = ["Naive Bayes", "Decision Tree", "SVM"]
def getScores(estimator, x, y):
    yPred = estimator.predict(x)
    return (accuracy_score(y, yPred), 
            precision_score(y, yPred, pos_label=3, average='macro'), 
            recall_score(y, yPred, pos_label=3, average='macro'))
def my_scorer(estimator, x, y):
    a, p, r = getScores(estimator, x, y)
    print a, p, r
    return a+p+r
for model, name in zip(models, names):
    print name
    start = time.time()
    m = cross_val_score(model, iris.data, iris.target,scoring=my_scorer, cv=10).mean()
    print 'nSum:',m, 'nn'
    print 'time', time.time() - start, 'nn'

~~这给了：~~

Naive Bayes
0.933333333333 0.944444444444 0.933333333333
0.933333333333 0.944444444444 0.933333333333
1.0 1.0 1.0
0.933333333333 0.944444444444 0.933333333333
0.933333333333 0.944444444444 0.933333333333
0.933333333333 0.944444444444 0.933333333333
0.866666666667 0.904761904762 0.866666666667
1.0 1.0 1.0
1.0 1.0 1.0
1.0 1.0 1.0
Sum: 2.86936507937 

time 0.0249638557434 

Decision Tree
1.0 1.0 1.0
0.933333333333 0.944444444444 0.933333333333
1.0 1.0 1.0
0.933333333333 0.944444444444 0.933333333333
0.933333333333 0.944444444444 0.933333333333
0.866666666667 0.866666666667 0.866666666667
0.933333333333 0.944444444444 0.933333333333
0.933333333333 0.944444444444 0.933333333333
1.0 1.0 1.0
1.0 1.0 1.0
Sum: 2.86555555556 

time 0.0237860679626 

SVM
1.0 1.0 1.0
0.933333333333 0.944444444444 0.933333333333
1.0 1.0 1.0
1.0 1.0 1.0
1.0 1.0 1.0
0.933333333333 0.944444444444 0.933333333333
0.933333333333 0.944444444444 0.933333333333
1.0 1.0 1.0
1.0 1.0 1.0
1.0 1.0 1.0
Sum: 2.94333333333 

time 0.043044090271

从scikit-learn 0.19.0开始，解决方案变得更加容易

from sklearn.model_selection import cross_validate
from sklearn.datasets import  load_iris
from sklearn.svm import SVC
iris = load_iris()
clf = SVC()
scoring = {'acc': 'accuracy',
           'prec_macro': 'precision_macro',
           'rec_micro': 'recall_macro'}
scores = cross_validate(clf, iris.data, iris.target, scoring=scoring,
                         cv=5, return_train_score=True)
print(scores.keys())
print(scores['test_acc'])

这给了：

['test_acc', 'score_time', 'train_acc', 'fit_time', 'test_rec_micro', 'train_rec_micro', 'train_prec_macro', 'test_prec_macro']
[ 0.96666667  1.          0.96666667  0.96666667  1.        ]

我遇到了同样的问题，我创建了一个可以在cross_val_score中支持多个指标的模块。
为了使用此模块完成您想要的内容，您可以编写：

from multiscorer import MultiScorer
import numpy as np
from sklearn.metrics import accuracy_score, precision_score, recall_score          
from sklearn.model_selection import cross_val_score
from numpy import average
scorer = MultiScorer({
    'Accuracy'  : (accuracy_score , {}),
    'Precision' : (precision_score, {'pos_label': 3, 'average':'macro'}),
    'Recall'    : (recall_score   , {'pos_label': 3, 'average':'macro'})
})
for model, name in zip(models, names):
    print name
    start = time.time()
    _ = cross_val_score(model, iris.data, iris.target,scoring=scorer, cv=10) # Added assignment of the result to `_` in order to illustrate that the return value will not be used
    results = scorer.get_results()
    for metric_name in results.keys():
        average_score = np.average(results[metric_name])
        print('%s : %f' % (metric_name, average_score))
    print 'time', time.time() - start, 'nn'

您可以从 GitHub 查看并下载此模块。希望对您有所帮助。

from sklearn import model_selection
def error_metrics(model, train_data, train_targ, kfold):
    scoring = ["accuracy","roc_auc","neg_log_loss","r2",
             "neg_mean_squared_error","neg_mean_absolute_error"] 
    error_metrics = pd.DataFrame()
    error_metrics["model"] = model
    for scor in scoring:
        score = []
        for mod in model:
           
            result = model_selection.cross_val_score(estimator= mod, X=train_data, y=train_targ,cv=kfold,scoring=scor )
            score.append(result.mean())
            
        error_metrics[scor] =pd.Series(score)
        
    return error_metrics

2023 年 4 月更新，scikit-learn>= 0.19.0：

由于cross_val_score方法与以前的版本相比略有变化正如文件所说：

使用 cross_validate 对多个指标运行交叉验证，并返回列车分数、拟合时间和分数时间

from sklearn.datasets import load_iris
from sklearn.model_selection import cross_validate
from pprint import pprint
# Load the Iris dataset
iris = load_iris()
X = iris.data
y = iris.target
# Create models
models = [GaussianNB(), DecisionTreeClassifier(), SVC()]
names = ["Naive Bayes", "Decision Tree", "SVM"]

# Define custom scoring metrics
scoring = {
    'accuracy': make_scorer(accuracy_score),
    'precision': make_scorer(precision_score, average='weighted'),
    'recall': make_scorer(recall_score, average='weighted'),
    'f1_score': make_scorer(f1_score, average='macro')
}
for model, name in zip(models, names):
    # Perform 5-fold cross-validation with custom scoring metrics
    pprint(name)
    pprint(cross_validate(model, X, y, cv=5, scoring=scoring))

结果如下所示：

'Naive Bayes'
{'fit_time': array([0.0012691 , 0.00121498, 0.00065112, 0.000664  , 0.000741  ]),
 'score_time': array([0.00356674, 0.00290489, 0.00418973, 0.00282598, 0.00429797]),
 'test_accuracy': array([0.93333333, 0.96666667, 0.93333333, 0.93333333, 1.        ]),
 'test_f1_score': array([0.93333333, 0.96658312, 0.93265993, 0.93265993, 1.        ]),
 'test_precision': array([0.93333333, 0.96969697, 0.94444444, 0.94444444, 1.        ]),
 'test_recall': array([0.93333333, 0.96666667, 0.93333333, 0.93333333, 1.        ])}
'Decision Tree'
{'fit_time': array([0.0010047 , 0.00049806, 0.00131512, 0.00049615, 0.00048304]),
 'score_time': array([0.003232  , 0.00246   , 0.00605106, 0.00245786, 0.00233197]),
 'test_accuracy': array([0.96666667, 0.96666667, 0.9       , 0.96666667, 1.        ]),
 'test_f1_score': array([0.96658312, 0.96658312, 0.89974937, 0.96658312, 1.        ]),
 'test_precision': array([0.96969697, 0.96969697, 0.9023569 , 0.96969697, 1.        ]),
 'test_recall': array([0.96666667, 0.96666667, 0.9       , 0.96666667, 1.        ])}
'SVM'
{'fit_time': array([0.00082183, 0.00068903, 0.0019362 , 0.00088406, 0.00114012]),
 'score_time': array([0.00263715, 0.00342083, 0.00375986, 0.00331903, 0.00372481]),
 'test_accuracy': array([0.96666667, 0.96666667, 0.96666667, 0.93333333, 1.        ]),
 'test_f1_score': array([0.96658312, 0.96658312, 0.96658312, 0.93333333, 1.        ]),
 'test_precision': array([0.96969697, 0.96969697, 0.96969697, 0.93333333, 1.        ]),
 'test_recall': array([0.96666667, 0.96666667, 0.96666667, 0.93333333, 1.        ])}

相关内容

最新更新

热门标签：