前言
邏輯回歸是統(tǒng)計(jì)學(xué)習(xí)中的經(jīng)典分類算法，如：可用于二分類
邏輯回歸有以下幾個(gè)特點(diǎn)：
優(yōu)點(diǎn)：計(jì)算代價(jià)不高，易于理解和實(shí)現(xiàn)
缺點(diǎn)：容易欠擬合，分類精度可能不高
適用數(shù)據(jù)類型：數(shù)值型和標(biāo)稱型數(shù)據(jù)
二項(xiàng)邏輯回歸模型的數(shù)學(xué)推導(dǎo)
設(shè){x, y}是輸入樣本，y = 1表示正類，y = 0表示負(fù)類。那么y = 1的概率和y = 0的概率可以表示為：
image
w是權(quán)值，b稱為偏置
模型參數(shù)估計(jì)
邏輯回歸常用的方法是極大似然估計(jì)，從而得到回歸模型。
2.PNG
這樣問(wèn)題就變成了對(duì)對(duì)數(shù)似然函數(shù)為目標(biāo)函數(shù)的最優(yōu)化問(wèn)題，常用的方法是梯度下降法以及擬牛頓法，本章采用梯度上升法和隨機(jī)梯度上升法來(lái)求該模型的最優(yōu)化問(wèn)題。
3.PNG
但是這樣我們無(wú)法直接求出L(w)的最大值對(duì)應(yīng)的w，那么就可以采用梯度上升法求解這個(gè)問(wèn)題。
4.PNG
沿著梯度的方向，每次移動(dòng)一個(gè)布長(zhǎng)，直到達(dá)到最大值。
公式exp(x) / (1 + exp(x))python代碼的實(shí)現(xiàn)：
def sigmod(inx):
return exp(inx) / (1 + exp(inx))
梯度上升法代碼的實(shí)現(xiàn)：
#梯度上升法
def grad_ascent(data_matin, class_label):
data_matrix = mat(data_matin) #將data_matin轉(zhuǎn)為100 * 3矩陣
label_matrix = mat(class_label).transpose() #將class_label轉(zhuǎn)為100 * 1的矩陣
m,n = shape(data_matrix) #m = 100, n = 100
alpha = 0.001
max_cycles = 500
weights = ones((n, 1)) #生成100 * 1權(quán)值的單位矩陣
for k in range(max_cycles):
h = sigmod(data_matrix * weights)
error = (label_matrix - h)
weights = weights + alpha * data_matrix.transpose() * error
return weights
隨機(jī)梯度上升法代碼的實(shí)現(xiàn)
#隨機(jī)梯度上升法
def stoc_grad_ascent0(data_matrix, class_label):
m,n = shape(data_matrix)
alpha = 0.01
weights = ones(n) #創(chuàng)建權(quán)值一維數(shù)組
#print(weights)
for i in range(m):
h = sigmod(sum(data_matrix[i] * weights))
error = class_label[i] - h
temp = []
for k in data_matrix[i]:
temp.append(alpha * error * k)
#print(temp)
weights = weights + temp
return weights
改進(jìn)型隨機(jī)梯度上升法代碼的實(shí)現(xiàn)：
#改進(jìn)型隨機(jī)梯度上升法
def stoc_grad_ascent1(data_matrix, class_label, num_iter = 150):
m,n = shape(data_matrix)
weights = ones(n)
data_index = range(m)
for j in range(num_iter):
for i in range(m):
alpha = 4 / (1 + j + i) + 0.01
rand_index = int(random.uniform(0, len(data_index)))
h = sigmod(sum(data_matrix[rand_index] * weights))
error = class_label[rand_index] - h
temp = []
for k in data_matrix[rand_index]:
temp.append(alpha * error * k)
weights = weights + temp
#del(data_index[rand_index])
return weights
完整python實(shí)現(xiàn)代碼如下：
from numpy import *
import matplotlib.pyplot as plt
import random
def load_data_set():
data_mat = []
label_mat = []
fr = open("test_set.txt")
for lines in fr.readlines(): #讀取每一行數(shù)據(jù)
line_arr = lines.strip().split() #將每一行分隔數(shù)據(jù)作為一個(gè)列表
data_mat.append([1.0, float(line_arr[0]), float(line_arr[1])]) #x0 = 1 x1 = line_arr[0] x2 = line_arr[1]
label_mat.append(int(line_arr[2])) #標(biāo)簽
return data_mat, label_mat #返回訓(xùn)練數(shù)據(jù)和標(biāo)簽
def sigmod(inx):
return exp(inx) / (1 + exp(inx))
#梯度上升法
def grad_ascent(data_matin, class_label):
data_matrix = mat(data_matin) #將data_matin轉(zhuǎn)為100 * 3矩陣
label_matrix = mat(class_label).transpose() #將class_label轉(zhuǎn)為100 * 1的矩陣
m,n = shape(data_matrix) #m = 100, n = 100
alpha = 0.001
max_cycles = 500
weights = ones((n, 1)) #生成100 * 1權(quán)值的單位矩陣
for k in range(max_cycles):
h = sigmod(data_matrix * weights)
error = (label_matrix - h)
weights = weights + alpha * data_matrix.transpose() * error
return weights
#隨機(jī)梯度上升法
def stoc_grad_ascent0(data_matrix, class_label):
m,n = shape(data_matrix)
alpha = 0.01
weights = ones(n) #創(chuàng)建權(quán)值一維數(shù)組
#print(weights)
for i in range(m):
h = sigmod(sum(data_matrix[i] * weights))
error = class_label[i] - h
temp = []
for k in data_matrix[i]:
temp.append(alpha * error * k)
#print(temp)
weights = weights + temp
return weights
#改進(jìn)型隨機(jī)梯度上升法
def stoc_grad_ascent1(data_matrix, class_label, num_iter = 150):
m,n = shape(data_matrix)
weights = ones(n)
data_index = range(m)
for j in range(num_iter):
for i in range(m):
alpha = 4 / (1 + j + i) + 0.01
rand_index = int(random.uniform(0, len(data_index)))
h = sigmod(sum(data_matrix[rand_index] * weights))
error = class_label[rand_index] - h
temp = []
for k in data_matrix[rand_index]:
temp.append(alpha * error * k)
weights = weights + temp
#del(data_index[rand_index])
return weights
def plot_best_fit(wei):
#weights = wei.getA()
weights = wei
data_mat, label_mat = load_data_set() #讀取原始數(shù)據(jù)
data_arr = array(data_mat)
n = shape(data_arr)[0]
xcord1 = []
xcord2 = []
ycord1 = []
ycord2 = []
for i in range(n):
if int(label_mat[i]) == 1:
xcord1.append(data_arr[i, 1])
ycord1.append(data_arr[i, 2])
else:
xcord2.append(data_arr[i, 1])
ycord2.append(data_arr[i, 2])
fig = plt.figure()
ax = fig.add_subplot(1, 1, 1)
ax.scatter(xcord1, ycord1, s = 30, c = "red", marker = "s")
ax.scatter(xcord2, ycord2, s = 30, c = "green")
x = arange(-5.0, 5.0, 0.1)
y = (-weights[0] - weights[1] * x) / weights[2]
ax.plot(x, y)
plt.xlabel("X1")
plt.ylabel("X2")
plt.show()
def main():
data_mat, label_mat = load_data_set()
#weights = grad_ascent(data_mat, label_mat)
weights = stoc_grad_ascent0(data_mat, label_mat)
#weights = stoc_grad_ascent1(data_mat, label_mat)
print(weights)
plot_best_fit(weights)
main()
有幾點(diǎn)需要注意：在畫圖函數(shù)中，若算法選擇梯度上升法則將weights = wei注釋，取消weights = wei.getA()的注釋。若算法選擇隨機(jī)梯度上升法和改進(jìn)型隨機(jī)梯度上升法，則將weights = wei.getA()注釋，取消weights = wei的注釋。
輸入數(shù)據(jù)：
-0.017612 14.053064 0
-1.395634 4.662541 1
-0.752157 6.538620 0
-1.322371 7.152853 0
0.423363 11.054677 0
0.406704 7.067335 1
0.667394 12.741452 0
-2.460150 6.866805 1
0.569411 9.548755 0
-0.026632 10.427743 0
0.850433 6.920334 1
1.347183 13.175500 0
1.176813 3.167020 1
-1.781871 9.097953 0
-0.566606 5.749003 1
0.931635 1.589505 1
-0.024205 6.151823 1
-0.036453 2.690988 1
-0.196949 0.444165 1
1.014459 5.754399 1
1.985298 3.230619 1
-1.693453 -0.557540 1
-0.576525 11.778922 0
-0.346811 -1.678730 1
-2.124484 2.672471 1
1.217916 9.597015 0
-0.733928 9.098687 0
-3.642001 -1.618087 1
0.315985 3.523953 1
1.416614 9.619232 0
-0.386323 3.989286 1
0.556921 8.294984 1
1.224863 11.587360 0
-1.347803 -2.406051 1
1.196604 4.951851 1
0.275221 9.543647 0
0.470575 9.332488 0
-1.889567 9.542662 0
-1.527893 12.150579 0
-1.185247 11.309318 0
-0.445678 3.297303 1
1.042222 6.105155 1
-0.618787 10.320986 0
1.152083 0.548467 1
0.828534 2.676045 1
-1.237728 10.549033 0
-0.683565 -2.166125 1
0.229456 5.921938 1
-0.959885 11.555336 0
0.492911 10.993324 0
0.184992 8.721488 0
-0.355715 10.325976 0
-0.397822 8.058397 0
0.824839 13.730343 0
1.507278 5.027866 1
0.099671 6.835839 1
-0.344008 10.717485 0
1.785928 7.718645 1
-0.918801 11.560217 0
-0.364009 4.747300 1
-0.841722 4.119083 1
0.490426 1.960539 1
-0.007194 9.075792 0
0.356107 12.447863 0
0.342578 12.281162 0
-0.810823 -1.466018 1
2.530777 6.476801 1
1.296683 11.607559 0
0.475487 12.040035 0
-0.783277 11.009725 0
0.074798 11.023650 0
-1.337472 0.468339 1
-0.102781 13.763651 0
-0.147324 2.874846 1
0.518389 9.887035 0
1.015399 7.571882 0
-1.658086 -0.027255 1
1.319944 2.171228 1
2.056216 5.019981 1
-0.851633 4.375691 1
-1.510047 6.061992 0
-1.076637 -3.181888 1
1.821096 10.283990 0
3.010150 8.401766 1
-1.099458 1.688274 1
-0.834872 -1.733869 1
-0.846637 3.849075 1
1.400102 12.628781 0
1.752842 5.468166 1
0.078557 0.059736 1
0.089392 -0.715300 1
1.825662 12.693808 0
0.197445 9.744638 0
0.126117 0.922311 1
-0.679797 1.220530 1
0.677983 2.556666 1
0.761349 10.693862 0
-2.168791 0.143632 1
1.388610 9.341997 0
0.317029 14.739025 0
實(shí)驗(yàn)結(jié)果如下所示：
梯度上升法：
5.PNG
隨機(jī)梯度上升法：
6.PNG
改進(jìn)型隨機(jī)梯度上升法：
7.PNG
由實(shí)驗(yàn)結(jié)果可知，改進(jìn)型隨機(jī)梯度上升法和梯度上升法的效果差不多，隨機(jī)梯度上升法的效果則差一些。
