Python机器学习库sklearn数据预处理,数据集构建,特征选择

from IPython.display import Image
%matplotlib inline
# Added version check for recent scikit-learn 0.18 checks
from distutils.version import LooseVersion as Version
from sklearn import __version__ as sklearn_version

处理缺省值

import pandas as pd
from io import StringIO

csv_data = '''A,B,C,D
1.0,2.0,3.0,4.0
5.0,6.0,,8.0
10.0,11.0,12.0,'''
print(type(csv_data))
# If you are using Python 2.7, you need to convert the string to unicode:
#csv_data = unicode(csv_data)

df = pd.read_csv(StringIO(csv_data))   
#StringIO的行为与file对象非常像,但它不是磁盘上文件,而是一个内存里的“文件”,可以将操作磁盘文件那样来操作StringIO
#StringIO在内存中读写str,要想把str写入StringIO:先创建一个StringIO,然后像文件一样写入即可
df
#type(df) #pandas.core.frame.DataFrame

df.isnull().sum()  #isnull判定每个位置是否是NaN,计算每一特征中:有几个样本在该特征上存在缺失
#Output:
#A    0
#B    0
#C    1
#D    1
#dtype: int64

1.)直接删除缺省值多的样本或者特征

df.dropna()   #删存在缺省值的样本

df.dropna(axis=1)   #删存在缺省值的特征

df.dropna(how='all')   #删所以特征都缺失的样本

df.dropna(thresh=4)   #删特征数少于参数指定个数的样本

df.dropna(subset=['C'])   #删在指定特征处有缺失的样本

2.)重新计算缺省值


from sklearn.preprocessing import Imputer
#http://scikit-learn.org/stable/modules/generated/sklearn.preprocessing.Imputer.html#sklearn.preprocessing.Imputer
imr = Imputer(missing_values='NaN', strategy='mean', axis=0)   #用NaN所在特征列的均值填充NaN
imr = imr.fit(df)
imputed_data = imr.transform(df.values)
imputed_data
#Output:
#array([[  1. ,   2. ,   3. ,   4. ],
#       [  5. ,   6. ,   7.5,   8. ],
#       [ 10. ,  11. ,  12. ,   6. ]])
df.values
#Output:
#array([[  1.,   2.,   3.,   4.],
#       [  5.,   6.,  nan,   8.],
#       [ 10.,  11.,  12.,  nan]])

处理类别型数据

import pandas as pd
df = pd.DataFrame([['green', 'M', 10.1, 'class1'],
                   ['red', 'L', 13.5, 'class2'],
                   ['blue', 'XL', 15.3, 'class1']])
df.columns = ['color', 'size', 'price', 'classlabel']
df

1.)序列特征映射

size_mapping = {'XL': 3,'L': 2,'M': 1}
df['size'] = df['size'].map(size_mapping)   #将size特征列的XL等字符替换成字典size_mapping映射对应的数字
df

inv_size_mapping = {v: k for k, v in size_mapping.items()}   
#字典推导式,items()是将字典中键值转化成元组对,整体以列表返回
#size_mapping.items():dict_items([('XL', 3), ('L', 2), ('M', 1)])
df['size'].map(inv_size_mapping)
#Output:
#0     M
#1     L
#2    XL
#Name: size, dtype: object

2.)类别编码

import numpy as np
class_mapping = {label: idx for idx, label in enumerate(np.unique(df['classlabel']))}   
#enumerate将列表参数映射成带序号的元组对
#class_mapping作用是找出出现一个新classlabel时对应的index
class_mapping
#Output:
#{'class1': 0, 'class2': 1}
df['classlabel'] = df['classlabel'].map(class_mapping)
df

inv_class_mapping = {v: k for k, v in class_mapping.items()}
df['classlabel'] = df['classlabel'].map(inv_class_mapping)
df


from sklearn.preprocessing import LabelEncoder
#http://scikit-learn.org/stable/modules/preprocessing_targets.html#preprocessing-targets
class_le = LabelEncoder()
y = class_le.fit_transform(df['classlabel'].values)
y
#Output:
#array([0, 1, 0], dtype=int64)
class_le.inverse_transform(y)
#Output:
#array(['class1', 'class2', 'class1'], dtype=object)

3.)对类别型的特征用one-hot编码


X = df[['color', 'size', 'price']].values
from sklearn.preprocessing import LabelEncoder
color_le = LabelEncoder()
X[:, 0] = color_le.fit_transform(X[:, 0])   #对color特征列进行顺序出现编码
X
#Output:
#array([[1, 1, 10.1],
#       [2, 2, 13.5],
#       [0, 3, 15.3]], dtype=object)
#检验color特征列编码序号原理,对应df结构看
from sklearn.preprocessing import LabelEncoder
color_le = LabelEncoder()
color_le.fit(df['color'])
#Output:
#array(['blue', 'green', 'red'], dtype=object)
color_le.classes_
from sklearn.preprocessing import OneHotEncoder
#http://scikit-learn.org/stable/modules/generated/sklearn.preprocessing.OneHotEncoder.html#sklearn.preprocessing.OneHotEncoder
ohe = OneHotEncoder(categorical_features=[0])   #categorical_features参数指定独热特征列
ohe.fit_transform(X).toarray()
#Output:
#array([[  0. ,   1. ,   0. ,   1. ,  10.1],
#       [  0. ,   0. ,   1. ,   2. ,  13.5],
#       [  1. ,   0. ,   0. ,   3. ,  15.3]])
pd.get_dummies(df[['price', 'color', 'size']])   #pandas内的独热方法get_dummies()

切分数据集(训练集与测试集)

df_wine = pd.read_csv('https://archive.ics.uci.edu/ml/machine-learning-databases/wine/wine.data',header=None)
df_wine.columns = ['Class label', 'Alcohol', 'Malic acid', 'Ash',
                   'Alcalinity of ash', 'Magnesium', 'Total phenols',
                   'Flavanoids', 'Nonflavanoid phenols', 'Proanthocyanins',
                   'Color intensity', 'Hue', 'OD280/OD315 of diluted wines',
                   'Proline']
print('Class labels', np.unique(df_wine['Class label']))
#df_wine.to_csv('wine.csv')
df_wine.head()
#Output:
#Class labels [1 2 3]

if Version(sklearn_version) < '0.18':
    from sklearn.cross_validation import train_test_split
else:
    from sklearn.model_selection import train_test_split
X, y = df_wine.iloc[:, 1:].values, df_wine.iloc[:, 0].values   #df_wine数据的第一列作为标签类别,其余列作为特征
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=0)   #全部数据的30%作为测试集

对连续值特征做幅度缩放(scaling)


from sklearn.preprocessing import MinMaxScaler
#http://scikit-learn.org/stable/modules/generated/sklearn.preprocessing.MinMaxScaler.html#sklearn.preprocessing.MinMaxScaler
#U(0,1)区间[0,1]上的均匀分布缩放,按单个特征列看
mms = MinMaxScaler()
X_train_norm = mms.fit_transform(X_train)
X_test_norm = mms.transform(X_test)
from sklearn.preprocessing import StandardScaler
#N(0,1)正态分布缩放
stdsc = StandardScaler()
X_train_std = stdsc.fit_transform(X_train)
X_test_std = stdsc.transform(X_test)

特征选择

1.) 通过L1正则化的截断型效应选择


from sklearn.linear_model import LogisticRegression
lr = LogisticRegression(penalty='l1', C=0.1)
lr.fit(X_train_std, y_train)
print('Training accuracy:', lr.score(X_train_std, y_train))
print('Test accuracy:', lr.score(X_test_std, y_test))
#Output:
#Training accuracy: 0.983870967742
#Test accuracy: 0.981481481481
lr.intercept_
#Output:
#array([-0.38380842, -0.15812178, -0.70041244])
lr.coef_
#Output:
#array([[ 0.28025703,  0.        ,  0.        , -0.02792998,  0.        ,
#         0.        ,  0.71008021,  0.        ,  0.        ,  0.        ,
#         0.        ,  0.        ,  1.23625518],
#       [-0.64387692, -0.06882766, -0.05718195,  0.        ,  0.        ,
#         0.        ,  0.        ,  0.        ,  0.        , -0.92717674,
#         0.05982319,  0.        , -0.37086571],
#       [ 0.        ,  0.06151692,  0.        ,  0.        ,  0.        ,
#         0.        , -0.63612079,  0.        ,  0.        ,  0.49812314,
#        -0.35823174, -0.57118053,  0.        ]])
import matplotlib.pyplot as plt
fig = plt.figure()
ax = plt.subplot(111)    
colors = ['blue', 'green', 'red', 'cyan', 'magenta', 'yellow', 'black',
          'pink', 'lightgreen', 'lightblue', 'gray', 'indigo', 'orange']
weights, params = [], []
for c in range(-4, 6):
    lr = LogisticRegression(penalty='l1', C=10**c, random_state=0)
    lr.fit(X_train_std, y_train)
    weights.append(lr.coef_[1])
    params.append(10**c)
weights = np.array(weights)   #10*13的数组
for column, color in zip(range(weights.shape[1]), colors):
    plt.plot(params, weights[:, column], label=df_wine.columns[column + 1], color=color)
#column+1去掉第一列label列
plt.axhline(0, color='black', linestyle='--', linewidth=1)
plt.xlim([10**(-5), 10**5])
plt.ylabel('weight coefficient')
plt.xlabel('C')
plt.xscale('log')
plt.legend(loc='upper left')
ax.legend(loc='upper center', bbox_to_anchor=(1.38, 1.03), ncol=1, fancybox=True)
# plt.savefig('./figures/l1_path.png', dpi=300)
plt.show()

2.) 遍历子特征集选择法

from sklearn.base import clone
from itertools import combinations
import numpy as np
from sklearn.metrics import accuracy_score
#http://scikit-learn.org/stable/modules/generated/sklearn.metrics.accuracy_score.html#sklearn.metrics.accuracy_score
if Version(sklearn_version) < '0.18':
    from sklearn.cross_validation import train_test_split
else:
    from sklearn.model_selection import train_test_split

class SBS():
    def __init__(self, estimator, k_features, scoring=accuracy_score, test_size=0.25, random_state=1):
        self.scoring = scoring
        self.estimator = clone(estimator)   #构造一个具有相同参数的估计器
        #http://scikit-learn.org/stable/modules/generated/sklearn.base.clone.html#sklearn.base.clone
        self.k_features = k_features
        self.test_size = test_size
        self.random_state = random_state

    def fit(self, X, y):
        X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=self.test_size, random_state=self.random_state)
        dim = X_train.shape[1]   #13
        self.indices_ = tuple(range(dim))   #(0,1,2,...,12)
        self.subsets_ = [self.indices_]
        score = self._calc_score(X_train, y_train, X_test, y_test, self.indices_)   #全特征组合得分
        self.scores_ = [score]

        while dim > self.k_features:   #至少要考虑到k_features个特征
            scores = []
            subsets = []
            for p in combinations(self.indices_, r=dim - 1):   #combinations求组合,不取全部13个特征:每次只取12个特征,11个特征...
                score = self._calc_score(X_train, y_train, X_test, y_test, p)
                scores.append(score)
                subsets.append(p)
            best = np.argmax(scores)   #argmax求分数最高的特征组合所在的索引
            self.indices_ = subsets[best]   #根据索引,取分数最高特征组合
            self.subsets_.append(self.indices_)   #同个数特征的各个组合中,得分最高的组合构成的列表
            dim -= 1
            self.scores_.append(scores[best])   #同个数特征的各个组合中,最优特征组合的得分构成的列表
        self.k_score_ = self.scores_[-1]   #每k个特征组合的最好得分,直接取scores_列表中最好加入的元素
        return self

    def transform(self, X):
        return X[:, self.indices_]

    def _calc_score(self, X_train, y_train, X_test, y_test, indices):
        self.estimator.fit(X_train[:, indices], y_train)
        y_pred = self.estimator.predict(X_test[:, indices])
        score = self.scoring(y_test, y_pred)   #用accuracy_score方法计算
        return score
#indices_是每个维度最优特征组合元组,subsets_是包含各个维度最优特征组合元组的列表,scores_是包含各个维度最优特征组合得分的列表
import matplotlib.pyplot as plt
from sklearn.neighbors import KNeighborsClassifier
knn = KNeighborsClassifier(n_neighbors=2)
# selecting features
sbs = SBS(knn, k_features=1)
sbs.fit(X_train_std, y_train)

# plotting performance of feature subsets
k_feat = [len(k) for k in sbs.subsets_]

plt.plot(k_feat, sbs.scores_, marker='o')
plt.ylim([0.7, 1.1])
plt.ylabel('Accuracy')
plt.xlabel('Number of features')
plt.grid()
plt.tight_layout()
# plt.savefig('./sbs.png', dpi=300)
plt.show()

k5 = list(sbs.subsets_[8])
print(df_wine.columns[1:][k5])   #去除第一个标签列计数
#Output:
#Index(['Alcohol', 'Malic acid', 'Alcalinity of ash', 'Hue', 'Proline'], dtype='object')
knn.fit(X_train_std, y_train)   #全部特征建模得分
print('Training accuracy:', knn.score(X_train_std, y_train))
print('Test accuracy:', knn.score(X_test_std, y_test))
#Output:
#Training accuracy: 0.983870967742
#Test accuracy: 0.944444444444
knn.fit(X_train_std[:, k5], y_train)   #只选k5组合包含的特征建模得分
print('Training accuracy:', knn.score(X_train_std[:, k5], y_train))
print('Test accuracy:', knn.score(X_test_std[:, k5], y_test))
#Output:
#Training accuracy: 0.959677419355
#Test accuracy: 0.962962962963

3.) 通过随机森林对特征重要性排序

from sklearn.ensemble import RandomForestClassifier
feat_labels = df_wine.columns[1:]   #特征列名
forest = RandomForestClassifier(n_estimators=2000, random_state=0, n_jobs=-1)  #2000棵树,并行工作数是运行服务器决定
forest.fit(X_train, y_train)
importances = forest.feature_importances_   #feature_importances_特征列重要性占比
indices = np.argsort(importances)[::-1]   #对参数从小到大排序的索引序号取逆,即最重要特征索引——>最不重要特征索引
for f in range(X_train.shape[1]):
    print("%2d) %-*s %f" % (f + 1, 30, feat_labels[indices[f]], importances[indices[f]]))   
#-*s表示左对齐字段feat_labels[indices[f]]宽为30
plt.title('Feature Importances')
plt.bar(range(X_train.shape[1]), importances[indices], color='lightblue', align='center')
plt.xticks(range(X_train.shape[1]), feat_labels[indices], rotation=90)
plt.xlim([-1, X_train.shape[1]])
plt.tight_layout()
#plt.savefig('./random_forest.png', dpi=300)
plt.show()
#Output:
# 1) Color intensity                0.183084
# 2) Proline                        0.163305
# 3) Flavanoids                     0.152030
# 4) OD280/OD315 of diluted wines   0.133887
# 5) Alcohol                        0.103847
# 6) Hue                            0.077784
# 7) Total phenols                  0.058778
# 8) Alcalinity of ash              0.031300
# 9) Malic acid                     0.024420
#10) Magnesium                      0.021949
#11) Proanthocyanins                0.021870
#12) Nonflavanoid phenols           0.014450
#13) Ash                            0.013294

if Version(sklearn_version) < '0.18':
    X_selected = forest.transform(X_train, threshold=0.15)
else:
    from sklearn.feature_selection import SelectFromModel
    sfm = SelectFromModel(forest, threshold=0.15, prefit=True)   
    #forest参数是建模完带特征重要性信息的估计器,threshold是特征筛选阈值
    X_selected = sfm.transform(X_train)   #transform对样本X_train进行了选定特征列过滤
X_selected.shape
#Output:
#(124, 3)
for f in range(X_selected.shape[1]):
    print("%2d) %-*s %f" % (f + 1, 30, feat_labels[indices[f]], importances[indices[f]]))
#Output:
# 1) Color intensity                0.183084
# 2) Proline                        0.163305
# 3) Flavanoids                     0.152030

版权声明:本文为cymy001原创文章,遵循 CC 4.0 BY-SA 版权协议,转载请附上原文出处链接和本声明。
本文链接:https://blog.csdn.net/cymy001/article/details/78546185

智能推荐

Rtthread学习笔记(十三)RT-Thread Studio开启硬件看门狗Watchdog

一、开启硬件看门狗Watchdog 1、配置RT-Thread Settings 2、开启stm32f1xx_hal_conf.h中的宏定义 3.使用RT接口函数初始化硬件看门狗...

TYVJ 4864 天天去哪吃 || 清北学堂金秋杯大奖赛

题目描述: 记录一下i这个值上次出现的位置在哪里,就是pre...

java反编译

jvm 把Boolean类型的值flag当做int类型处理。​​​ Foo.java: 由 class 文件生成 jasm 文件:java -jar asmtools.jar jdis Foo.class > Foo.jasm  修改jasm文件: 执行反编译: java -jar jd-gui-1.6.6.jar File 打开Foo.class文件:b修改为2 重新执行java...

【学习笔记】03-v-html的学习和示例

v-html的认识和使用 示例: 显示结果: 注意:v-html是有复制的...

Java实现在线考试系统(系统介绍)

1.和现在有的考试系统有以下几种优势: a.和现在有的系统比较起来,本系统有科目、章节、老师、学生、班级等信息的管理,还有批阅试卷查看已批阅试卷等。传统的考试系统划分并不细,业务功能简单。 b.和学校的考试系统还有外面的考试系统比较起来,本系统是B/S结构,学校的考试系统一般为C/S结构,性能方面不如B/S结构,并且C/S接口需要安装客户端,客户端压力很大,我的系统只需要电脑具有浏览器,在同一局域...

猜你喜欢

计算机视觉--多视几何初步尝试

基础矩阵的原理 K和K’分别是两个相机的参数矩阵。p和p’是X在平面π的坐标表示。所以可以得出 具体计算过程 代码: #!/usr/bin/env python coding: utf-8 from PIL import Image from numpy import * from pylab import * import numpy as np from imp ...

java初学者怎么学习才可以快速入门

java初学者怎么学习才可以快速入门 一、了解JAVA 我们要知道:Java是由Sun Microsystems公司于1995年5月推出的Java面向对象程序设计语言。 Java之父:詹姆斯·高斯林 1.1 java的三个体系 Java SE(Java Platform Standard Edition)。Java SE 以前称为 J2SE。它允许开发和部署在桌面、服务器、嵌入式环境...

字段属性之主键&增删改查&自增长&唯一键约束

字段属性之主键&自增长&唯一键约束 主键 主键:primary key 主要的键 一张表中只有一个字段可以使用对应的键,用来唯一的约束该字段里面的数据,不能重复,这种称之为主键 一张表只能最多一个主键 增加主键 SQL操作中有多种方式增加主键大体分为三种 1.在创建表的时候直接在字段之后跟primary key关键字(主键本身不允许为空) 优点:非常直接:缺点:只能使用一个字段作为...

linux下 基于libmad的socket多用户mp3音频在线播放服务器

在众多大神的帮助下,这个在线播放流媒体服务器终于完成啦。。。。 这个mp3流媒体服务器设计的思路是,服务器程序server用多线程实现和多个客户端的通信(这是必然的),然后发送给客户端当前的音频列表公客户端选择,之后根据k客户端的选择给多个客户端传输相应mp3文件的数据,同时,客户端进行实时地音频解码并播放。 关于libmad开源mp3音频解码库的使用,见上一篇博客吧。。。。 在服务器程序这一端,...

Nginx

Nginx Nginx简介: Nginx是一个高性能的http和反向代理服务器,特点是有内存少,并发能力强,事实上Nginx的并发能力确实在同类型网页服务器中表现较好, Nginx用作web服务器:Nginx可以作为静态页面的web服务器,同时还支持CGI语言,但不支持java,java程序只能通过Tomcat配合完成。Nginx专为性能优化而开发,性能是其最重要的考量,实现上非常注重效率,能经受...