背景:天津市津南區(qū)堅(jiān)持以綠色經(jīng)濟(jì)驅(qū)動(dòng)傳統(tǒng)經(jīng)濟(jì)向數(shù)字化經(jīng)濟(jì)轉(zhuǎn)型以實(shí)現(xiàn)津南高質(zhì)量發(fā)展，正依托物聯(lián)網(wǎng)、云計(jì)算、三網(wǎng)融合等新一代信息技術(shù)，加快全區(qū)智能工業(yè)全面發(fā)展，推動(dòng)產(chǎn)業(yè)轉(zhuǎn)型升級(jí)，培育新興產(chǎn)業(yè)，實(shí)現(xiàn)“津南制造”向“津南智造”轉(zhuǎn)變。
2019津南數(shù)字制造聚焦智能制造，以原料企業(yè)工藝優(yōu)化為課題，要求以異煙酸生產(chǎn)過程中的各參數(shù)，設(shè)計(jì)精確智能的優(yōu)秀算法，提升異煙酸的收率，助力企業(yè)實(shí)現(xiàn)轉(zhuǎn)型升級(jí)，提升行業(yè)競(jìng)爭(zhēng)力。大賽數(shù)據(jù)提供方天津漢德威藥業(yè)有限公司，為大賽提供真實(shí)生產(chǎn)數(shù)據(jù)，提供工藝專家的專業(yè)指導(dǎo)，從軟硬件環(huán)境諸多方面提供大賽支撐。
使用工具是Jupyter Notebook
項(xiàng)目目前提供了兩份數(shù)據(jù),分別是jinnan_round1_train_20181227(訓(xùn)練集,用來構(gòu)建模型)和jinnan_round1_testA_20181227(測(cè)試集,用來對(duì)模型準(zhǔn)確度進(jìn)行測(cè)試)

讀取并觀察數(shù)據(jù)

import os 
import re
import time
import datetime
import warnings
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import lightgbm as lgb
import xgboost as xgb
from scipy import sparse
from sklearn.svm import SVR
from sklearn.metrics import mean_squared_error
from sklearn.metrics import log_loss
from sklearn.ensemble import RandomForestRegressor
from sklearn.neighbors import KNeighborsClassifier
from sklearn.feature_selection import SelectKBest, f_classif, f_regression
from sklearn.linear_model import BayesianRidge,Lasso,LinearRegression 
from sklearn.model_selection import KFold, RepeatedKFold,train_test_split,cross_val_score
from sklearn.preprocessing import OneHotEncoder, LabelEncoder
warnings.filterwarnings("ignore")
pd.set_option('max_columns',1000) 
pd.set_option('max_row',300)


#顯示所有列
# pd.set_option('display.max_columns', None)
#顯示所有行
# pd.set_option('display.max_rows', None)
#設(shè)置value的顯示長(zhǎng)度為100，默認(rèn)為50
pd.set_option('max_colwidth',100)

train = pd.read_csv("data/jinnan_round1_train_20181227.csv",encoding="gb18030")
test = pd.read_csv("data/jinnan_round1_testA_20181227.csv",encoding="gb18030")
train.head()

# 查詢列值類別較少的特征
cols = list(train.columns)
# del cols[-1]
# del cols[0]
values_cols = []
name_cols = []
for i in cols:
    if len(train[i].unique()) <= 4:
        values_cols.append(train[i].unique())
        name_cols.append(i)
print(values_cols)
print(name_cols)

[array([300, 250, 200], dtype=int64), array([ nan, 125.]), array([405., 340., nan, 270.]), array([700, 590, 980, 470], dtype=int64), array([100, 101, 102, 103], dtype=int64), array([0.2 , 0.15, 0.12]), array([0.2, 0.1]), array([ 9. , 10. , 8. , 3.5]), array([ 5., 4., nan, 10.]), array([3.5 , 0.15, nan, 3.6 ]), array([3.5, nan, 3.6]), array([0.15, 0.03, 0.06, nan])]
['A1', 'A2', 'A3', 'A4', 'A10', 'A13', 'A18', 'A22', 'A23', 'B2', 'B3', 'B13']

# 查詢?nèi)笔食^90%的列(列中某一的值占有率達(dá)90%以上)
cols = []
good_cols = list(train.columns)
for i in train.columns:
    rate =train[i].value_counts(normalize=True, dropna=False).values[0]
    if rate > 0.9:
        cols.append(i)
        good_cols.remove(i)
print(cols)

# 將缺失率超過90%特征的刪除
for i in cols:
    if i in good_cols:
        good_cols.remove(i)
print(good_cols)

['A1', 'A2', 'A3', 'A4', 'A13', 'A18', 'A23', 'B2', 'B3', 'B13']
['樣本id', 'A5', 'A6', 'A7', 'A8', 'A9', 'A10', 'A11', 'A12', 'A14', 'A15', 'A16', 'A17', 'A19', 'A20', 'A21', 'A22', 'A24', 'A25', 'A26', 'A27', 'A28', 'B1', 'B4', 'B5', 'B6', 'B7', 'B8', 'B9', 'B10', 'B11', 'B12', 'B14', '收率']

# 將收率過低的異常值刪除

train_edit = train[train['收率']>=0.85]

train = train_edit[good_cols]
good_cols.remove('收率')
good_cols.remove('樣本id')
test = test[good_cols]
target_train = train['收率']
train = train[good_cols]
# 將訓(xùn)練集和測(cè)試集合并
data = pd.concat([train,test],axis=0,ignore_index=True)

time_cols = ['A5','A7','A9','A11',
             'A14','A16','A24',
             'A26','B5','B7']
time_d_cols = ['A20','A28','B4','B9','B10','B11']


# 查看時(shí)間列是否有異常值
def time_check(t):
    if pd.isnull(t):
        pass
    else:
        try:
            t,m,s=t.split(":")
        except:
            print(t)
            
            
def time_check_d(t):  
    if pd.isnull(t):
        pass
    else:
        try:
            sh,sm,eh,em=re.findall(r"\d+\.?\d*",t)
        except:
            print(t)
        
for i in time_cols:
    data_one = data[i].apply(time_check)
print('---------')
for i in time_d_cols:
    data_two = data[i].apply(time_check_d)

1900/1/21 0:00
1900/1/29 0:00
1900/1/21 0:00
1900/1/22 0:00
1900/1/9 7:00
700
1900/1/1 2:30
1900/1/12 0:00
1900/3/13 0:00

15:00-1600
19:-20:05

# 可以看出時(shí)間列類型分為上述兩種,固定時(shí)間和間隔時(shí)間
# 異常值諸如:1900/1/21 0:00 默認(rèn)時(shí)間為0,并將空值均賦為時(shí)間為0,上述700值認(rèn)為是7個(gè)小時(shí)
# 分別對(duì)時(shí)間列進(jìn)行處理,轉(zhuǎn)化為具體數(shù)值(小時(shí))

def time_edit_second(t):
    try:
        t,m,s=t.split(":")
    except:
        if t=='1990/1/9 7:00':
            return 7
        elif t=='1990/1/1 2:30':
            return (2*3600+30*60)/3600
        elif t=='700':
            return 7
        else:
            return 0 
    try:
        time = (int(t)*3600+int(m)*60+int(s))/3600
        return time
    except:
        return 0


for i in time_cols:
    data[i] = data[i].apply(time_edit_second)


def time_d_edit_second(t):
    try:
        sh,sm,eh,em = re.findall(r"\d+\.?\d*",t)
    except:
        if t=='15:00-1600' or t=='19:-20:05':
            return 1
        else:
            return 0
    try:    
        if int(sh)>int(eh):
            time_d = (int(eh)*3600+int(em)*60-int(sm)*60-int(sh)*3600)/3600 + 24
        else:
            time_d = (int(eh)*3600+int(em)*60-int(sm)*60-int(sh)*3600)/3600
        return time_d
    except:
        return 0
    
#     return time_d


for i in time_d_cols:
    data[i] = data[i].apply(time_d_edit_second)

# 觀察轉(zhuǎn)換后的時(shí)間值有無異常值
# 觀察修改后的數(shù)據(jù),講缺失值補(bǔ)足
# A8列的空值均用0代替,其他列值缺失1-3個(gè)值,可以用列的平均值進(jìn)行代替
data.info()

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 1540 entries, 0 to 1539
Data columns (total 32 columns):
A5 1540 non-null float64
A6 1540 non-null float64
A7 1540 non-null float64
A8 167 non-null float64
A9 1540 non-null float64
A10 1540 non-null int64
A11 1540 non-null float64
A12 1540 non-null float64
A14 1540 non-null float64
A15 1540 non-null float64
A16 1540 non-null float64
A17 1540 non-null float64
A19 1540 non-null int64
A20 1540 non-null float64
A21 1537 non-null float64
A22 1540 non-null float64
A24 1540 non-null float64
A25 1539 non-null object
A26 1540 non-null float64
A27 1539 non-null float64
A28 1540 non-null float64
B1 1529 non-null float64
B4 1540 non-null float64
B5 1540 non-null float64
B6 1540 non-null int64
B7 1540 non-null float64
B8 1539 non-null float64
B9 1540 non-null float64
B10 1540 non-null float64
B11 1540 non-null float64
B12 1539 non-null float64
B14 1540 non-null int64
dtypes: float64(27), int64(4), object(1)
memory usage: 385.1+ KB

data['A8'] = data['A8'].fillna(0)
list_edit = ['A21','A27','B1','B8','B12']
for i in list_edit:
    data[i] = data[i].fillna(data[i].mean())

# 報(bào)錯(cuò),因?yàn)锳25是object類型并且有一單元值為1900/3/10 0:00,無法轉(zhuǎn)換
data.loc[list(data['A25'].values).index('1900/3/10 0:00'),'A25']=0

data=pd.DataFrame(data,dtype=np.float)
data['A25'] = data['A25'].fillna(data['A25'].mean())

# 將清理后的數(shù)據(jù)進(jìn)行分配
train = data[0:train.shape[0]] # 取前1390行
test  = data[train.shape[0]:]  # 取1390行之后的所有行

# 對(duì)特征進(jìn)行分析,驗(yàn)證特征與目標(biāo)值的相關(guān)性
figsize = 15,8
figure = plt.subplots(figsize=figsize)
selector = SelectKBest(f_classif,k=15)
selector.fit(train,target)
score = -np.log10(selector.pvalues_)
plt.bar(range(len(train.columns)), score)
plt.xticks(range(len(train.columns)), train.columns)

plt.show()
# 對(duì)特征進(jìn)行篩選,去除 A6,A24,A28特征

# 觀察訓(xùn)練集和測(cè)試集的特征值分布情況
fig = plt.subplots(figsize=(30,20))
j = 1
for cols in data.columns:
    plt.subplot(4,8,j)
    sns.distplot(train[cols])
    sns.distplot(test[cols])
    j+=1

# 去除 A6,A24,A28特征
good_columns = list(train.columns)
remove_col = ['A6','A24','A28']
for i in remove_col:
    good_columns.remove(i)
train = train[good_columns]
test = test[good_columns]


x_train,x_test,y_train,y_test = train_test_split(train,target,
                                                 test_size = 0.3,random_state=0)
#封裝,并對(duì)結(jié)果進(jìn)行k折交叉驗(yàn)證
def kfold_score(alg):
    kf = KFold(n_splits=5,random_state=1,shuffle=False)
    alg.fit(x_train, y_train)
    score = cross_val_score(alg, x_train, y_train, cv=kf)
    print(score.mean())
    print(alg.score(x_test, y_test))
    y_pred = alg.predict(test)
    return y_pred

近日略忙! 后續(xù)繼續(xù)更新!