一維數(shù)據(jù)集上的神經(jīng)網(wǎng)絡(luò)
# 1 引入包，創(chuàng)建會話
import tensorflow as tf
import numpy as np
sess = tf.Session()
# 2 初始化數(shù)據(jù)
data_size = 25
data_1d = np.random.normal(size=data_size)
x_input_1d = tf.placeholder(dtype=tf.float32, shape=[data_size])
# 3 定義卷積層
def conv_layer_1d(input_1d, my_filter):
# Make 1d input 4d
input_2d = tf.expand_dims(input_1d, 0)
input_3d = tf.expand_dims(input_2d, 0)
input_4d = tf.expand_dims(input_3d, 3)
# Perform convolution
convolution_output = tf.nn.conv2d(input_4d, filter=my_filter,
strides=[1,1,1,1], padding='VALID')
conv_output_1d = tf.squeeze(convolution_output)
return conv_output_1d
# Now drop extra dimensions
my_filter = tf.Variable(tf.random_normal(shape=[1,5,1,1]))
my_convolution_output = conv_layer_1d(x_input_1d, my_filter)
# 4 激勵函數(shù)
def activation(input_1d):
return tf.nn.relu(input_1d)
my_activation_output = activation(my_convolution_output)
# 池化
def max_pool(input_1d, width):
# First we make the 1d input into 4d.
input_2d = tf.expand_dims(input_1d, 0)
input_3d = tf.expand_dims(input_2d, 0)
input_4d = tf.expand_dims(input_3d, 3)
# Perform the max pool operation
pool_output = tf.nn.max_pool(input_4d, ksize=[1, 1, width, 1], strides=[1, 1, 1, 1],
padding='VALID')
pool_output_1d = tf.squeeze(pool_output)
return pool_output_1d
my_maxpool_output = max_pool(my_activation_output, width=5)
# 全連接層
def fully_connected(input_layer, num_outputs):
# Create weights
weight_shape = tf.squeeze(tf.stack([tf.shape(input_layer), [num_outputs]]))
weight = tf.random_normal(weight_shape, stddev=0.1)
bias = tf.random_normal(shape=[num_outputs])
# make input into 2d
input_layer_2d = tf.expand_dims(input_layer, 0)
# perform fully connected operations
full_output = tf.add(tf.matmul(input_layer_2d, weight), bias)
# Drop extra dimmensions
full_output_1d = tf.squeeze(full_output)
return full_output_1d
my_full_output = fully_connected(my_maxpool_output, 5)
# 初始化變量，運(yùn)行計(jì)算圖大陰每層輸出結(jié)果
init = tf.global_variables_initializer()
sess.run(init)
feed_dict = {x_input_1d:data_1d}
# Convolution Output
print("Input = array of length 25")
print("Convolution w/filter length = 5, stride size = 1, results in an array of legth 21:")
print(sess.run(my_convolution_output, feed_dict = feed_dict))
# Activation Output
print('\nInput = the above array of length 21')
print('Relu element wise returns the array of length 21:')
print(sess.run(my_activation_output, feed_dict=feed_dict))
# Maxpool output
print('\nInput = the above array of length 21')
print('MaxPool, window length = 5, stride size = 1, results in the array of length 17')
print(sess.run(my_maxpool_output, feed_dict=feed_dict))
# Fully Connected Output
print('Input = the above array of length 17')
print('Fully connected layer on all four rows with five outputs')
print(sess.run(my_full_output, feed_dict=feed_dict))
# 關(guān)閉會話
sess.close()
輸出結(jié)果如下：
Input = array of length 25
Convolution w/filter length = 5, stride size = 1, results in an array of legth 21:
[ 0.7306204 0.09220226 -0.8647339 -1.7677759 3.0679996 -0.42977548
-1.4834487 2.084762 -0.63769084 -1.6181873 0.8859257 0.94589835
-2.3447719 1.4659762 0.86647564 -0.5625909 0.02268941 1.3069543
-1.5059514 3.0157318 -2.7027912 ]
Input = the above array of length 21
Relu element wise returns the array of length 21:
[0.7306204 0.09220226 0. 0. 3.0679996 0.
0. 2.084762 0. 0. 0.8859257 0.94589835
0. 1.4659762 0.86647564 0. 0.02268941 1.3069543
0. 3.0157318 0. ]
Input = the above array of length 21
MaxPool, window length = 5, stride size = 1, results in the array of length 17
[3.0679996 3.0679996 3.0679996 3.0679996 3.0679996 2.084762
2.084762 2.084762 0.94589835 1.4659762 1.4659762 1.4659762
1.4659762 1.4659762 1.3069543 3.0157318 3.0157318 ]
Input = the above array of length 17
Fully connected layer on all four rows with five outputs
[ 1.8550391 -1.1319994 0.44229037 -1.3700286 -1.7920521 ]
卷積層
首先，卷積層輸入序列是25個元素的一維數(shù)組。卷積層的功能是相鄰5個元素與過濾器（長度為5的向量）內(nèi)積。因?yàn)橐苿硬介L為1，所以25個元素的序列中一共有21個相鄰為5的序列，最終輸出也是5。
激勵函數(shù)
將卷積成的輸出，21個元素的向量通過relu函數(shù)逐元素轉(zhuǎn)化。輸出仍是21個元素的向量。
池化層，最大值池化
取相鄰5個元素的最大值。輸入21個元素的序列，輸出17個元素的序列。
全連接層
上述17個元素通過全連接層，有5個輸出。
注意上述過程的輸出都做了維度的裁剪。但在每一步的過程中都是擴(kuò)充成4維張量操作的。
二維數(shù)據(jù)上的神經(jīng)網(wǎng)絡(luò)
# 1 引入包，創(chuàng)建會話
import tensorflow as tf
import numpy as np
sess = tf.Session()
# 2 創(chuàng)建數(shù)據(jù)和占位符
data_size = [10, 10]
data_2d = np.random.normal(size=data_size)
x_input_2d = tf.placeholder(dtype=tf.float32, shape=data_size)
# 3 卷積層：2x2過濾器
def conv_layer_2d(input_2d, my_filter):
# First, change 2d input to 4d
input_3d = tf.expand_dims(input_2d, 0)
input_4d = tf.expand_dims(input_3d, 3)
# Perform convolution
convolution_output = tf.nn.conv2d(input_4d, filter=my_filter, strides=[1,2,2,1], padding='VALID')
# Drop extra dimensions
conv_output_2d = tf.squeeze(convolution_output)
return conv_output_2d
my_filter = tf.Variable(tf.random_normal(shape=[2,2,1,1]))
my_convolution_output = conv_layer_2d(x_input_2d, my_filter)
# 4 激勵函數(shù)
def activation(input_2d):
return tf.nn.relu(input_2d)
my_activation_output = activation(my_convolution_output)
# 5 池化層
def max_pool(input_2d, width, height):
# Make 2d input into 4d
input_3d = tf.expand_dims(input_2d, 0)
input_4d = tf.expand_dims(input_3d, 3)
# Perform max pool
pool_output = tf.nn.max_pool(input_4d, ksize=[1, height, width, 1], strides=[1,1,1,1], padding='VALID')
# Drop extra dimensions
pool_output_2d = tf.squeeze(pool_output)
return pool_output_2d
my_maxpool_output = max_pool(my_activation_output, width=2, height=2)
# 6 全連接層
def fully_connected(input_layer, num_outputs):
# Flatten into 1d
flat_input = tf.reshape(input_layer, [-1])
# Create weights
weight_shape = tf.squeeze(tf.stack([tf.shape(flat_input), [num_outputs]]))
weight = tf.random_normal(weight_shape, stddev=0.1)
bias = tf.random_normal(shape=[num_outputs])
# Change into 2d
input_2d = tf.expand_dims(flat_input, 0)
# Perform fully connected operations
full_output = tf.add(tf.matmul(input_2d, weight), bias)
# Drop extra dimensions
full_output_2d = tf.squeeze(full_output)
return full_output_2d
my_full_output = fully_connected(my_maxpool_output, 5)
# 7 初始化變量
init = tf.initialize_all_variables()
sess.run(init)
feed_dict = {x_input_2d: data_2d}
# 8 打印每層輸出結(jié)果
# Convolution Output
print('Input = [10 x 10] array')
print('2x2 Convolution, stride size = [2x2], results in the [5x5] array:')
print(sess.run(my_convolution_output, feed_dict=feed_dict))
# Activation Output
print('\nInput = the above [5x5] array')
print('Relu element wise returns the [5x5] array:')
print(sess.run(my_activation_output, feed_dict=feed_dict))
# Max Pool Output
print('\nInput = the above [5x5] array')
print('[2x2] MaxPool, stride size = [1x1] results in the [4x4] array:')
print(sess.run(my_maxpool_output, feed_dict = feed_dict))
# Fully connected output
print('\nInput = the above [4x4] array')
print('Fully connected layer on all four rows with five outputs:')
print(sess.run(my_full_output, feed_dict=feed_dict))
輸出結(jié)果如下：
Input = [10 x 10] array
2x2 Convolution, stride size = [2x2], results in the [5x5] array:
[[ 0.80993664 -1.2700474 0.27375805 0.54493535 -1.0037322 ]
[-1.2054954 2.7807589 -0.9015032 -0.24516574 2.126141 ]
[ 0.19843565 -0.3517378 2.624067 -3.2827137 1.0169035 ]
[-1.3321284 -0.98290706 0.7477172 1.655221 1.5588429 ]
[ 1.2763401 0.88586557 -2.230918 -1.5759512 1.1120629 ]]
Input = the above [5x5] array
Relu element wise returns the [5x5] array:
[[0.80993664 0. 0.27375805 0.54493535 0. ]
[0. 2.7807589 0. 0. 2.126141 ]
[0.19843565 0. 2.624067 0. 1.0169035 ]
[0. 0. 0.7477172 1.655221 1.5588429 ]
[1.2763401 0.88586557 0. 0. 1.1120629 ]]
Input = the above [5x5] array
[2x2] MaxPool, stride size = [1x1] results in the [4x4] array:
[[2.7807589 2.7807589 0.54493535 2.126141 ]
[2.7807589 2.7807589 2.624067 2.126141 ]
[0.19843565 2.624067 2.624067 1.655221 ]
[1.2763401 0.88586557 1.655221 1.655221 ]]
Input = the above [4x4] array
Fully connected layer on all four rows with five outputs:
[ 0.7709798 -0.2126801 -0.7047844 0.89408153 -0.46939346]
TensorFlow 實(shí)現(xiàn)多層神經(jīng)網(wǎng)絡(luò)
# 1 引入包
import tensorflow as tf
import matplotlib.pyplot as plt
import requests
import numpy as np
import os
import csv
sess = tf.Session()
# 2 導(dǎo)入數(shù)據(jù)
# name of data file
birth_weight_file = 'birth_weight.csv'
birthdata_url = 'https://github.com/nfmcclure/tensorflow_cookbook/raw/master' \
'/01_Introduction/07_Working_with_Data_Sources/birthweight_data/birthweight.dat'
# Download data and create data file if file does not exist in current directory
if not os.path.exists(birth_weight_file):
birth_file = requests.get(birthdata_url)
birth_data = birth_file.text.split('\r\n')
birth_header = birth_data[0].split('\t')
birth_data = [[float(x) for x in y.split('\t') if len(x) >= 1]
for y in birth_data[1:] if len(y) >= 1]
with open(birth_weight_file, "w") as f:
writer = csv.writer(f)
writer.writerows([birth_header])
writer.writerows(birth_data)
# read birth weight data into memory
birth_data = []
with open(birth_weight_file, newline='') as csvfile:
csv_reader = csv.reader(csvfile)
birth_header = next(csv_reader)
for row in csv_reader:
if len(row)>0:
birth_data.append(row)
birth_data = [[float(x) for x in row] for row in birth_data]
# Extract y-target (birth weight)
y_vals = np.array([x[8] for x in birth_data])
# Filter for features of interest
cols_of_interest = ['AGE', 'LWT', 'RACE', 'SMOKE', 'PTL', 'HT', 'UI']
x_vals = np.array([[x[ix] for ix, feature in enumerate(birth_header) if feature in cols_of_interest]
for x in birth_data])
# 3 設(shè)置種子
seed = 4
tf.set_random_seed(seed)
np.random.seed(seed)
batch_size = 100
# 4 劃分訓(xùn)練集和測試集
train_indices = np.random.choice(len(x_vals), round(len(x_vals)*0.8), replace=False)
test_indices = np.array(list(set(range(len(x_vals))) - set(train_indices)))
x_vals_train = x_vals[train_indices]
x_vals_test = x_vals[test_indices]
y_vals_train = y_vals[train_indices]
y_vals_test = y_vals[test_indices]
def normalize_cols(m):
col_max = m.max(axis=0)
col_min = m.min(axis=0)
return (m - col_min) /(col_max - col_min)
x_vals_train = np.nan_to_num(normalize_cols(x_vals_train))
x_vals_test = np.nan_to_num(normalize_cols(x_vals_test))
# 5 定義一個設(shè)置變量和bias的函數(shù)
def init_weight(shape, st_dev):
weight = tf.Variable(tf.random_normal(shape, stddev=st_dev))
return weight
def init_bias(shape, st_dev):
bias = tf.Variable(tf.random_normal(shape, stddev=st_dev))
return bias
# 6 初始化占位符
x_data = tf.placeholder(shape=[None, 7], dtype=tf.float32)
y_target = tf.placeholder(shape=[None, 1], dtype=tf.float32)
# 7 創(chuàng)建全連接層函數(shù)，方便重復(fù)使用
def fully_connected(input_layer, weights, biases):
layer = tf.add(tf.matmul(input_layer, weights), biases)
return tf.nn.relu(layer)
# 8 創(chuàng)建算法模型
# Create second layer (25 hidden nodes)
weight_1 = init_weight(shape=[7, 25], st_dev=10.0)
bias_1 = init_weight(shape=[25], st_dev=10.0)
layer_1 = fully_connected(x_data, weight_1, bias_1)
# Create second layer (10 hidden nodes)
weight_2 = init_weight(shape=[25, 10], st_dev=10.0)
bias_2 = init_weight(shape=[10], st_dev=10.0)
layer_2 = fully_connected(layer_1, weight_2, bias_2)
# Create third layer (3 hidden nodes)
weight_3 = init_weight(shape=[10, 3], st_dev=10.0)
bias_3 = init_weight(shape=[3], st_dev=10.0)
layer_3 = fully_connected(layer_2, weight_3, bias_3)
# Create output layer (1 output value)
weight_4 = init_weight(shape=[3, 1], st_dev=10.0)
bias_4 = init_bias(shape=[1], st_dev=10.0)
final_output = fully_connected(layer_3, weight_4, bias_4)
# 9 L1損失函數(shù)
loss = tf.reduce_mean(tf.abs(y_target - final_output))
my_opt = tf.train.AdamOptimizer(0.05)
train_step = my_opt.minimize(loss)
init = tf.global_variables_initializer()
sess.run(init)
# 10 迭代200
# Initialize the loss vectors
loss_vec = []
test_loss = []
for i in range(200):
# Choose random indices for batch selection
rand_index = np.random.choice(len(x_vals_train), size=batch_size)
# Get random batch
rand_x = x_vals_train[rand_index]
rand_y = np.transpose([y_vals_train[rand_index]])
# Run the training step
sess.run(train_step, feed_dict={x_data: rand_x, y_target:rand_y})
# Get and store the train loss
temp_loss = sess.run(loss, feed_dict = {x_data:rand_x, y_target:rand_y})
loss_vec.append(temp_loss)
# get and store the test loss
test_temp_loss = sess.run(loss, feed_dict = {x_data:x_vals_test, y_target:np.transpose([y_vals_test])})
test_loss.append(test_temp_loss)
if (i+1)% 25 == 0:
print('Generation: ' + str(i+1)+'.Loss = ' + str(temp_loss))
# 12 繪圖
plt.plot(loss_vec, 'k-', label='Train Loss')
plt.plot(test_loss, 'r--', label='Test Loss')
plt.title('Loss per Generation')
plt.xlabel('Generation')
plt.ylabel('Loss')
plt.legend(loc='upper right')
plt.show()
# Model Accuracy
actuals = np.array([x[0] for x in birth_data])
test_actuals = actuals[test_indices]
train_actuals = actuals[train_indices]
test_preds = [x[0] for x in sess.run(final_output, feed_dict={x_data: x_vals_test})]
train_preds = [x[0] for x in sess.run(final_output, feed_dict={x_data: x_vals_train})]
test_preds = np.array([0.0 if x < 2500.0 else 1.0 for x in test_preds])
train_preds = np.array([0.0 if x < 2500.0 else 1.0 for x in train_preds])
# Print out accuracies
test_acc = np.mean([x == y for x, y in zip(test_preds, test_actuals)])
train_acc = np.mean([x == y for x, y in zip(train_preds, train_actuals)])
print('On predicting the category of low birthweight from regression output (<2500g):')
print('Test Accuracy: {}'.format(test_acc))
print('Train Accuracy: {}'.format(train_acc))
實(shí)現(xiàn)了一個含有三層隱藏層的全連接神經(jīng)網(wǎng)絡(luò)。
線性預(yù)測模型的優(yōu)化
# 1 導(dǎo)入必要的庫
import matplotlib.pyplot as plt
import numpy as np
import tensorflow as tf
import requests
import os
import csv
sess = tf.Session()
# 加載數(shù)據(jù)集，進(jìn)行數(shù)據(jù)抽取和歸一化
# name of data file
birth_weight_file = 'birth_weight.csv'
birthdata_url = 'https://github.com/nfmcclure/tensorflow_cookbook/raw/master' \
'/01_Introduction/07_Working_with_Data_Sources/birthweight_data/birthweight.dat'
# Download data and create data file if file does not exist in current directory
if not os.path.exists(birth_weight_file):
birth_file = requests.get(birthdata_url)
birth_data = birth_file.text.split('\r\n')
birth_header = birth_data[0].split('\t')
birth_data = [[float(x) for x in y.split('\t') if len(x) >= 1]
for y in birth_data[1:] if len(y) >= 1]
with open(birth_weight_file, "w") as f:
writer = csv.writer(f)
writer.writerows([birth_header])
writer.writerows(birth_data)
# read birth weight data into memory
birth_data = []
with open(birth_weight_file, newline='') as csvfile:
csv_reader = csv.reader(csvfile)
birth_header = next(csv_reader)
for row in csv_reader:
if len(row)>0:
birth_data.append(row)
birth_data = [[float(x) for x in row] for row in birth_data]
# Extract y-target (birth weight)
y_vals = np.array([x[0] for x in birth_data])
# Filter for features of interest
cols_of_interest = ['AGE', 'LWT', 'RACE', 'SMOKE', 'PTL', 'HT', 'UI']
x_vals = np.array([[x[ix] for ix, feature in enumerate(birth_header) if feature in cols_of_interest]
for x in birth_data])
# 4 劃分訓(xùn)練集和測試集
train_indices = np.random.choice(len(x_vals), round(len(x_vals)*0.8), replace=False)
test_indices = np.array(list(set(range(len(x_vals))) - set(train_indices)))
x_vals_train = x_vals[train_indices]
x_vals_test = x_vals[test_indices]
y_vals_train = y_vals[train_indices]
y_vals_test = y_vals[test_indices]
def normalize_cols(m):
col_max = m.max(axis=0)
col_min = m.min(axis=0)
return (m - col_min) /(col_max - col_min)
x_vals_train = np.nan_to_num(normalize_cols(x_vals_train))
x_vals_test = np.nan_to_num(normalize_cols(x_vals_test))
# 3 聲明批量大小和占位符
batch_size = 90
x_data = tf.placeholder(shape=[None, 7], dtype=tf.float32)
y_target = tf.placeholder(shape=[None, 1], dtype=tf.float32)
# 4 聲明函數(shù)來初始化變量和層
def init_variable(shape):
return tf.Variable(tf.random_normal(shape=shape))
# Create a logistic layer definition
def logistic(input_layer, multiplication_weight, bias_weight, activation=True):
linear_layer = tf.add(tf.matmul(input_layer, multiplication_weight), bias_weight)
if activation :
return tf.nn.sigmoid(linear_layer)
else :
return linear_layer
# 5 聲明神經(jīng)網(wǎng)絡(luò)的兩個隱藏層和輸出層
# First logistic layer (7 inputs to 14 hidden nodes)
A1 = init_variable(shape=[7, 14])
b1 = init_variable(shape=[14])
logistic_layer1 = logistic(x_data, A1, b1)
# Second logistic layer (14 inputs to 5 hidden nodes)
A2 = init_variable(shape=[14, 5])
b2 = init_variable(shape=[5])
logistic_layer2 = logistic(logistic_layer1, A2, b2)
# Final output layer (5 hidden nodes to 1 output)
A3 = init_variable(shape=[5, 1])
b3 = init_variable(shape=[1])
final_output = logistic(logistic_layer2, A3, b3, activation=False)
# 6 聲明損失函數(shù)和優(yōu)化方法
# Create loss function
loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(
labels = y_target, logits=final_output))
# Declare optimizer
my_opt = tf.train.AdamOptimizer(learning_rate = 0.002)
train_step = my_opt.minimize(loss)
# Initialize variables
init = tf.global_variables_initializer()
sess.run(init)
# 7 評估精度
prediction = tf.round(tf.nn.sigmoid(final_output))
predictions_correct = tf.cast(tf.equal(prediction, y_target), tf.float32)
accuracy = tf.reduce_mean(predictions_correct)
# 8 迭代訓(xùn)練模型
# Initialize loss and accuracy vectors
loss_vec = []
train_acc = []
test_acc = []
for i in range(1500):
# Select random indicies for batch selection
rand_index = np.random.choice(len(x_vals_train), size=batch_size)
# Select batch
rand_x = x_vals_train[rand_index]
rand_y = np.transpose([y_vals_train[rand_index]])
# Run training step
sess.run(train_step, feed_dict={x_data:rand_x, y_target:rand_y})
# Get training loss
temp_loss = sess.run(loss, feed_dict={x_data:rand_x, y_target:rand_y})
loss_vec.append(temp_loss)
# Get training accuracy
temp_acc_train = sess.run(accuracy, feed_dict = {x_data:x_vals_train, y_target:np.transpose([y_vals_train])})
train_acc.append(temp_acc_train)
# Get test accuracy
temp_acc_test = sess.run(accuracy, feed_dict = {x_data:x_vals_test, y_target:np.transpose([y_vals_test])})
test_acc.append(temp_acc_test)
if (i+1)%150==0:
print('Loss = ' + str(temp_loss))
# 9 繪圖
# Plot loss over time
plt.plot(loss_vec, 'k-')
plt.title('Cross Entropy Loss per Generation')
plt.xlabel('Generation')
plt.ylabel('Cross Entropy Loss')
plt.show()
# Plot train and test accuracy
plt.plot(train_acc, 'k-', label='Train Set Accuracy')
plt.plot(test_acc, 'r--', label='Test Set Accuracy')
plt.title('Train and Test Accuracy')
plt.xlabel('Generation')
plt.ylabel('Accuracy')
plt.legend(loc='lower right')
plt.show()
這一個仍然是全連接，只是只有兩層隱藏層，節(jié)點(diǎn)數(shù)也減少了。
