# network2.py

## network2.py

``````#!/usr/bin/python
# -*- coding: utf-8 -*-
"""network2.py
~~~~~~~~~~~~~~

An improved version of network.py, implementing the stochastic
gradient descent learning algorithm for a feedforward neural network.
Improvements include the addition of the cross-entropy cost function,
regularization, and better initialization of network weights.  Note
that I have focused on making the code simple, easily readable, and
easily modifiable.  It is not optimized, and omits many desirable
features.

"""

#### Libraries
# Standard library
import json
import random
import sys

# Third-party libraries
import numpy as np

#### Define the quadratic and cross-entropy cost functions

@staticmethod
def fn(a, y):
"""Return the cost associated with an output ``a`` and desired output ``y``."""
return 0.5*np.linalg.norm(a-y)**2

@staticmethod
def delta(z, a, y):
"""Return the error delta from the output layer."""
return (a-y) * sigmoid_prime(z)

class CrossEntropyCost(object):

@staticmethod
def fn(a, y):
"""Return the cost associated with an output ``a`` and desired output
``y``.  Note that np.nan_to_num is used to ensure numerical
stability.  In particular, if both ``a`` and ``y`` have a 1.0
in the same slot, then the expression (1-y)*np.log(1-a)
returns nan.  The np.nan_to_num ensures that that is converted
to the correct value (0.0).

"""
return np.sum(np.nan_to_num(-y*np.log(a)-(1-y)*np.log(1-a)))

@staticmethod
def delta(z, a, y):
"""Return the error delta from the output layer.  Note that the
parameter ``z`` is not used by the method.  It is included in
the method's parameters in order to make the interface
consistent with the delta method for other cost classes.

"""
return (a-y)

#********************************************************
class LogLikelihoodCost(object):

@staticmethod
def fn(a, y):
"""
C = -log(a[i])

a(10,1) y(10,1)
y = [0,0,1,0,0,0,0,0,0,0,0]
i = 2
C = -log a[2,0]
"""
i = np.argmax(y)
return -np.log(a[i,0])

@staticmethod
def delta(z, a, y):
"""
delta_j = aj-yj
"""
return (a-y)
#********************************************************

#### Main Network class
class Network(object):

def __init__(self, sizes, cost=CrossEntropyCost):
"""The list ``sizes`` contains the number of neurons in the respective
layers of the network.  For example, if the list was [2, 3, 1]
then it would be a three-layer network, with the first layer
containing 2 neurons, the second layer 3 neurons, and the
third layer 1 neuron.  The biases and weights for the network
are initialized randomly, using
``self.default_weight_initializer`` (see docstring for that
method).

"""
self.num_layers = len(sizes)
self.sizes = sizes
self.cost=cost
# init use_softmax with cost type
if self.cost == LogLikelihoodCost:
self.use_softmax = True
else:
self.use_softmax = False

# init weight initializer and feedforward method
if self.use_softmax:
self.default_weight_initializer = self.default_weight_initializer_with_softmax
self.feedforward = self.feedforward_with_softmax
else:
self.default_weight_initializer = self.default_weight_initializer_1
self.feedforward = self.feedforward_1
# init weights and biases
self.default_weight_initializer()
# at least an input and output layers
assert self.num_layers>=2

def default_weight_initializer_with_softmax(self):
# sigmoid neurons: w(0,1/sqrt(n_in)) b(0,1)
# softmax neurons: w = b = 0
# len(sizes)>=2
# (1) for sigmoid neuros
self.biases = [np.random.randn(y, 1) for y in self.sizes[1:-1]]
self.weights = [np.random.randn(y, x)/np.sqrt(x)
for x, y in zip(self.sizes[:-1], self.sizes[1:-1])]
#(2) for last somtmax neurons
x = self.sizes[-2]
y = self.sizes[-1]
last_b = np.zeros((y, 1))
last_w = np.zeros((y, x))

self.biases.append(last_b)
self.weights.append(last_w)

def default_weight_initializer_1(self):
"""Initialize each weight using a Gaussian distribution with mean 0
and standard deviation 1 over the square root of the number of
weights connecting to the same neuron.  Initialize the biases
using a Gaussian distribution with mean 0 and standard
deviation 1.

Note that the first layer is assumed to be an input layer, and
by convention we won't set any biases for those neurons, since
biases are only ever used in computing the outputs from later
layers.

"""
self.biases = [np.random.randn(y, 1) for y in self.sizes[1:]]
self.weights = [np.random.randn(y, x)/np.sqrt(x)
for x, y in zip(self.sizes[:-1], self.sizes[1:])]

def large_weight_initializer(self):
"""Initialize the weights using a Gaussian distribution with mean 0
and standard deviation 1.  Initialize the biases using a
Gaussian distribution with mean 0 and standard deviation 1.

Note that the first layer is assumed to be an input layer, and
by convention we won't set any biases for those neurons, since
biases are only ever used in computing the outputs from later
layers.

This weight and bias initializer uses the same approach as in
Chapter 1, and is included for purposes of comparison.  It
will usually be better to use the default weight initializer

"""
self.biases = [np.random.randn(y, 1) for y in self.sizes[1:]]
self.weights = [np.random.randn(y, x)
for x, y in zip(self.sizes[:-1], self.sizes[1:])]

def feedforward_with_softmax(self, a):
"""Return the output of the network if ``a`` is input."""
for b, w in zip(self.biases[:-1], self.weights[:-1]):
a = sigmoid(np.dot(w, a)+b)
# last layer
b,w = self.biases[-1],self.weights[-1]
last_z = np.dot(w, a)+b
last_a = softmax(last_z)
return last_a

def feedforward_1(self, a):
"""Return the output of the network if ``a`` is input."""
for b, w in zip(self.biases, self.weights):
a = sigmoid(np.dot(w, a)+b)
return a

def SGD(self, training_data, epochs, mini_batch_size, eta,
lmbda = 0.0,
evaluation_data=None,
monitor_evaluation_cost=False,
monitor_evaluation_accuracy=False,
monitor_training_cost=False,
monitor_training_accuracy=False):
"""Train the neural network using mini-batch stochastic gradient
descent.  The ``training_data`` is a list of tuples ``(x, y)``
representing the training inputs and the desired outputs.  The
other non-optional parameters are self-explanatory, as is the
regularization parameter ``lmbda``.  The method also accepts
``evaluation_data``, usually either the validation or test
data.  We can monitor the cost and accuracy on either the
evaluation data or the training data, by setting the
appropriate flags.  The method returns a tuple containing four
lists: the (per-epoch) costs on the evaluation data, the
accuracies on the evaluation data, the costs on the training
data, and the accuracies on the training data.  All values are
evaluated at the end of each training epoch.  So, for example,
if we train for 30 epochs, then the first element of the tuple
will be a 30-element list containing the cost on the
evaluation data at the end of each epoch. Note that the lists
are empty if the corresponding flag is not set.

"""
if evaluation_data: n_data = len(evaluation_data)
n = len(training_data)
num_batches = n/mini_batch_size
evaluation_cost, evaluation_accuracy = [], []
training_cost, training_accuracy = [], []
for j in xrange(epochs):
random.shuffle(training_data)
for k in xrange(0,num_batches):
mini_batch = training_data[k*mini_batch_size : (k+1)*mini_batch_size]
self.update_mini_batch(mini_batch, eta, lmbda, len(training_data))
print "Epoch %s training complete" % j
if monitor_training_cost:
cost = self.total_cost(training_data, lmbda)
training_cost.append(cost)
print "Cost on training data: {}".format(cost)
if monitor_training_accuracy:
accuracy = self.accuracy(training_data, convert=True)
training_accuracy.append(accuracy)
print "Accuracy on training data: {} / {}".format(
accuracy, n)
if monitor_evaluation_cost:
cost = self.total_cost(evaluation_data, lmbda, convert=True)
evaluation_cost.append(cost)
print "Cost on evaluation data: {}".format(cost)
if monitor_evaluation_accuracy:
accuracy = self.accuracy(evaluation_data)
evaluation_accuracy.append(accuracy)
print "Accuracy on evaluation data: {} / {}".format(
self.accuracy(evaluation_data), n_data)
print
return evaluation_cost, evaluation_accuracy, training_cost, training_accuracy

def calculate_sum_derivatives_of_mini_batch(self,mini_batch):
"""
计算m个样本的总梯度和。
利用反向传播计算每一个样本(x,y)对应的梯度。
"""
nabla_b = [np.zeros(b.shape) for b in self.biases]
nabla_w = [np.zeros(w.shape) for w in self.weights]
for x, y in mini_batch:
# 给定一个样本X,利用反向传播算法计算对应w,b的梯度
delta_nabla_b, delta_nabla_w = self.backprop(x, y)
# 对m个样本的梯度进行累计求和
nabla_b = [nb+dnb for nb, dnb in zip(nabla_b, delta_nabla_b)]
nabla_w = [nw+dnw for nw, dnw in zip(nabla_w, delta_nabla_w)]
return nabla_b,nabla_w

def update_mini_batch(self, mini_batch, eta, lmbda, n):
"""Update the network's weights and biases by applying gradient
descent using backpropagation to a single mini batch.  The
``mini_batch`` is a list of tuples ``(x, y)``, ``eta`` is the
learning rate, ``lmbda`` is the regularization parameter, and
``n`` is the total size of the training data set.

"""
m = len(mini_batch)
nabla_b,nabla_w = self.calculate_sum_derivatives_of_mini_batch(mini_batch)

#self.weights = [w-(eta/m)*nw for w, nw in zip(self.weights, nabla_w)]
#self.biases =  [b-(eta/m)*nb for b, nb in zip(self.biases,  nabla_b)]

# L2 regularization
weight_decay = 1-eta*(lmbda/n)
self.weights = [weight_decay*w-(eta/m)*nw
for w, nw in zip(self.weights, nabla_w)]
self.biases = [b-(eta/m)*nb
for b, nb in zip(self.biases, nabla_b)]

def backprop(self, x, y):
"""Return a tuple "(nabla_b, nabla_w)" representing the
gradient for the cost function C_x.  "nabla_b" and
"nabla_w" are layer-by-layer lists of numpy arrays, similar
to "self.biases" and "self.weights"."""
# 初始化nb,nw,结构和b,w一样
nabla_b = [np.zeros(b.shape) for b in self.biases]
nabla_w = [np.zeros(w.shape) for w in self.weights]

# feedforward
# 执行算法的feedforward阶段
#　(1)初始化x作为a_1
activation = x
activations = [x] # list to store all the activations, layer by layer
zs = [] # list to store all the z vectors, layer by layer
# (2)l=2,....L层，分别计算z_l,a_l并且保存下来。
#*************************************************************
if self.use_softmax:
for b, w in zip(self.biases[:-1], self.weights[:-1]):
z = np.dot(w, activation)+b
zs.append(z)
activation = sigmoid(z)
activations.append(activation)
#last layer
b,w = self.biases[-1],self.weights[-1]
last_z = np.dot(w, activation)+b
last_a = softmax(last_z)
zs.append(last_z)
activations.append(last_a)
#*************************************************************
else:
for b, w in zip(self.biases, self.weights):
z = np.dot(w, activation)+b
zs.append(z)
activation = sigmoid(z)
activations.append(activation)

#========================================================================
# 先计算所有的误差delta，最后计算所有层的梯度nb,nw，代码可读性更高一些
#========================================================================
# method2
# backward pass
# 执行算法的backward阶段
# (3)初始化第L层的误差,delta_L　= cost(z,a,y)
l = -1
#***************************************************
delta = self.cost.delta(zs[l],activations[l], y)
#***************************************************
deltas = [delta] # list to store all the errors,layer by layer
# (4)初始化l=L-1,....2层的误差,delta_l = np.dot(w_l+1^T,delta_l+1)* sigmoid_prime(z_l)
for i in xrange(2, self.num_layers):
l = -i #(-２代表L-1,-3代表L-2,-(L-1)代表2)
delta = np.dot(self.weights[l+1].transpose(), deltas[l+1]) * sigmoid_prime(zs[l])
deltas.insert(0,delta) # 确保误差的顺序，从后往前计算，所以需要insert在数组的最前面

#(5)l=L,L-1,....2层，计算所有的梯度向量nb,nw
for i in xrange(1, self.num_layers):
l = -i #(-1,-2,....-(L-1))
nabla_b[l] = deltas[l]
nabla_w[l] = np.dot(deltas[l], activations[l-1].transpose())

return (nabla_b, nabla_w)

def accuracy(self, data, convert=False):
"""Return the number of inputs in ``data`` for which the neural
network outputs the correct result. The neural network's
output is assumed to be the index of whichever neuron in the
final layer has the highest activation.

The flag ``convert`` should be set to False if the data set is
validation or test data (the usual case), and to True if the
data set is the training data. The need for this flag arises
due to differences in the way the results ``y`` are
represented in the different data sets.  In particular, it
flags whether we need to convert between the different
representations.  It may seem strange to use different
representations for the different data sets.  Why not use the
same representation for all three data sets?  It's done for
efficiency reasons -- the program usually evaluates the cost
on the training data and the accuracy on other data sets.
These are different types of computations, and using different
representations speeds things up.  More details on the
representations can be found in

"""
if convert: # train data
results = [(np.argmax(self.feedforward(x)), np.argmax(y))
for (x, y) in data]
else: # val/test data
results = [(np.argmax(self.feedforward(x)), y)
for (x, y) in data]
return sum(int(x == y) for (x, y) in results)

def total_cost(self, data, lmbda, convert=False):
"""Return the total cost for the data set ``data``.  The flag
``convert`` should be set to False if the data set is the
training data (the usual case), and to True if the data set is
the validation or test data.  See comments on the similar (but
reversed) convention for the ``accuracy`` method, above.
"""
cost = 0.0
n = len(data)
for x, y in data:
a = self.feedforward(x)
if convert: # val/test data
y = vectorized_result(y)
cost += self.cost.fn(a, y)/n
# L2 term = lmbda/(2n)*sum(w**2)
l2_term = 0.5*(lmbda/n)*sum(np.linalg.norm(w)**2 for w in self.weights)
cost += l2_term
return cost

def save(self, filename):
"""Save the neural network to the file ``filename``."""
data = {"sizes": self.sizes,
"weights": [w.tolist() for w in self.weights],
"biases": [b.tolist() for b in self.biases],
"cost": str(self.cost.__name__)}
f = open(filename, "w")
json.dump(data, f)
f.close()

"""Load a neural network from the file ``filename``.  Returns an
instance of Network.

"""
f = open(filename, "r")
f.close()
name = sys.modules[__name__]
cost = getattr(name, data["cost"])
net = Network(data["sizes"], cost=cost)
net.weights = [np.array(w) for w in data["weights"]]
net.biases = [np.array(b) for b in data["biases"]]
return net

#### Miscellaneous functions
def vectorized_result(j):
"""Return a 10-dimensional unit vector with a 1.0 in the j'th position
and zeroes elsewhere.  This is used to convert a digit (0...9)
into a corresponding desired output from the neural network.

"""
e = np.zeros((10, 1))
e[j] = 1.0
return e

def sigmoid(z):
"""The sigmoid function."""
return 1.0/(1.0+np.exp(-z))

def sigmoid_prime(z):
"""Derivative of the sigmoid function."""
return sigmoid(z)*(1-sigmoid(z))

def softmax(z):
#e^z/ sum(e^z)
ez = np.exp(z)
sum_ez = sum(ez)
return ez/sum_ez
``````
``````import mnist_loader
``````
``````net = load("./1.json")
net.accuracy(validation_data)
``````
``````9385
``````
``````net = load("./0.json")
net.accuracy(validation_data)
``````
``````8904
``````

## History

• 20180807: created.

Author: kezunlin
Reprint policy: All articles in this blog are used except for special statements CC BY 4.0 reprint polocy. If reproduced, please indicate source kezunlin !

TOC