scikit中的特征选择可以学习多个变量和数千个特征

Question

我正在尝试为逻辑回归分类器执行特征选择。最初有4个变量:姓名、位置、性别和标签=种族。这三个变量，即名字，会产生成千上万个"特征"，例如，名字"John Snow"会产生两个字母的子字符串，如"jo"，"oh"，"hn"……等。特征集经过DictVectorization。

我正在尝试遵循这个教程(http://scikit-learn.org/stable/auto_examples/feature_selection/plot_feature_selection.html)，但我不确定我是否做得对，因为教程使用了少量的功能，而我的矢量化后有数万个。并且 plt.show()显示一个空白的图形。

# coding=utf-8
import pandas as pd
from pandas import DataFrame, Series
import numpy as np
import re
import random
import time
from random import randint
import csv
import sys
reload(sys)
sys.setdefaultencoding('utf-8')
from sklearn import svm
from sklearn.metrics import classification_report
from sklearn.linear_model import LogisticRegression
from sklearn.svm import LinearSVC
from sklearn.tree import DecisionTreeClassifier
from sklearn.naive_bayes import MultinomialNB
from sklearn.feature_extraction import DictVectorizer
from sklearn.feature_selection import SelectPercentile, f_classif
from sklearn.metrics import confusion_matrix as sk_confusion_matrix
from sklearn.metrics import roc_curve, auc
import matplotlib.pyplot as plt
from sklearn.metrics import precision_recall_curve
# Assign X and y variables
X = df.raw_name.values
X2 = df.name.values
X3 = df.gender.values
X4 = df.location.values
y = df.ethnicity_scan.values
# Feature extraction functions
def feature_full_name(nameString):
    try:
        full_name = nameString
        if len(full_name) > 1: # not accept name with only 1 character
            return full_name
        else: return '?'
    except: return '?'
def feature_avg_wordLength(nameString):
    try:
        space = 0
        for i in nameString:
            if i == ' ':
                space += 1
        length = float(len(nameString) - space)
        name_entity = float(space + 1)
        avg = round(float(length/name_entity), 0)
        return avg
    except:
        return 0
def feature_name_entity(nameString2):
    space = 0
    try:
        for i in nameString2:
            if i == ' ':
                space += 1
        return space+1
    except: return 0
def feature_gender(genString):
    try:
        gender = genString
        if len(gender) >= 1:
            return gender
        else: return '?'
    except: return '?'
def feature_noNeighborLoc(locString):
    try:
        x = re.sub(r'^[^, ]*', '', locString) # remove everything before and include first ','
        y = x[2:] # remove subsequent ',' and ' '
        return y
    except: return '?'
def list_to_dict(substring_list):
    try:
        substring_dict = {}
        for i in substring_list:
            substring_dict['substring='+str(i)] = True
        return substring_dict
    except: return '?'
# Transform format of X variables, and spit out a numpy array for all features
my_dict13 = [{'name-entity': feature_name_entity(feature_full_name(i))} for i in X2]
my_dict14 = [{'avg-length': feature_avg_wordLength(feature_full_name(i))} for i in X]
my_dict15 = [{'gender': feature_full_name(i)} for i in X3]
my_dict16 = [{'location': feature_noNeighborLoc(feature_full_name(i))} for i in X4]
my_dict17 = [{'dummy1': 1} for i in X]
my_dict18 = [{'dummy2': random.randint(0,2)} for i in X]
all_dict = []
for i in range(0, len(my_dict)):
    temp_dict = dict(my_dict13[i].items() + my_dict14[i].items()
        + my_dict15[i].items() + my_dict16[i].items() + my_dict17[i].items() + my_dict18[i].items()
        )
    all_dict.append(temp_dict)
newX = dv.fit_transform(all_dict)
# Separate the training and testing data sets
half_cut = int(len(df)/2.0)*-1
X_train = newX[:half_cut]
X_test = newX[half_cut:]
y_train = y[:half_cut]
y_test = y[half_cut:]
# Fitting X and y into model, using training data
lr = LogisticRegression()
lr.fit(X_train, y_train)
dv = DictVectorizer()
# Feature selection
plt.figure(1)
plt.clf()
X_indices = np.arange(X_train.shape[-1])
selector = SelectPercentile(f_classif, percentile=10)
selector.fit(X_train, y_train)
scores = -np.log10(selector.pvalues_)
scores /= scores.max()
plt.bar(X_indices - .45, scores, width=.2,
    label=r'Univariate score ($-Log(p_{value})$)', color='g')
plt.show()

警告:

E:Program Files ExtraPython27libsite-packagessklearnfeature_selectionunivariate_selection.py:111: UserWarning: Features [[0 0 0 ..., 0 0 0]] are constant.

Answer 1

看起来您将数据分为训练集和测试集的方式不起作用:

# Separate the training and testing data sets
X_train = newX[:half_cut]
X_test = newX[half_cut:]

如果您已经使用了sklearn，那么使用内置的拆分例程会方便得多:

X_train, X_test, y_train, y_test = cross_validation.train_test_split(X, y, test_size=0.5, random_state=0)