深度学习入门：神经网络与反向传播

深度学习入门：神经网络与反向传播

神经网络基础

感知机（Perceptron）

基本单元：

import numpy as np

class Perceptron:
    def __init__(self, input_size):
        # 初始化权重和偏置
        self.weights = np.random.randn(input_size)
        self.bias = np.random.randn()
    
    def forward(self, x):
        # 加权求和 + 偏置
        z = np.dot(x, self.weights) + self.bias
        # 激活函数（阶跃函数）
        return 1 if z > 0 else 0
    
    def predict(self, X):
        return np.array([self.forward(x) for x in X])
​

多层感知机（MLP）

结构：

输入层 → 隐藏层 → 输出层
​

前向传播：

class NeuralNetwork:
    def __init__(self, layer_sizes):
        self.layer_sizes = layer_sizes
        self.weights = []
        self.biases = []
        
        # 初始化参数
        for i in range(len(layer_sizes) - 1):
            self.weights.append(np.random.randn(layer_sizes[i], layer_sizes[i + 1]) * 0.01)
            self.biases.append(np.zeros((1, layer_sizes[i + 1])))
    
    def sigmoid(self, z):
        return 1 / (1 + np.exp(-z))
    
    def sigmoid_derivative(self, z):
        s = self.sigmoid(z)
        return s * (1 - s)
    
    def forward(self, X):
        self.activations = [X]
        self.z_values = []
        
        for i in range(len(self.weights)):
            z = np.dot(self.activations[-1], self.weights[i]) + self.biases[i]
            self.z_values.append(z)
            
            if i == len(self.weights) - 1:
                # 输出层使用 sigmoid
                a = self.sigmoid(z)
            else:
                # 隐藏层使用 sigmoid 或 ReLU
                a = self.sigmoid(z)
            
            self.activations.append(a)
        
        return self.activations[-1]
​

激活函数

1. Sigmoid

def sigmoid(x):
    return 1 / (1 + np.exp(-x))

def sigmoid_derivative(x):
    s = sigmoid(x)
    return s * (1 - s)

# 特点
# - 输出范围 (0, 1)
# - 梯度消失问题
# - 输出不以零为中心
​

2. Tanh

def tanh(x):
    return np.tanh(x)

def tanh_derivative(x):
    return 1 - np.tanh(x) ** 2

# 特点
# - 输出范围 (-1, 1)
# - 零中心化
# - 仍有梯度消失问题
​

3. ReLU（推荐）

def relu(x):
    return np.maximum(0, x)

def relu_derivative(x):
    return (x > 0).astype(float)

# 特点
# - 计算简单
# - 缓解梯度消失
# - 稀疏激活
# - Dead ReLU 问题
​

4. Leaky ReLU

def leaky_relu(x, alpha=0.01):
    return np.maximum(alpha * x, x)

def leaky_relu_derivative(x, alpha=0.01):
    dx = np.ones_like(x)
    dx[x < 0] = alpha
    return dx

# 解决 Dead ReLU 问题
​

损失函数

1. 均方误差（MSE）

def mse_loss(y_true, y_pred):
    return np.mean((y_true - y_pred) ** 2)

def mse_loss_derivative(y_true, y_pred):
    return 2 * (y_pred - y_true) / y_true.size

# 适用于回归问题
​

2. 交叉熵损失

def binary_crossentropy(y_true, y_pred):
    # 避免log(0)
    y_pred = np.clip(y_pred, 1e-15, 1 - 1e-15)
    return -np.mean(y_true * np.log(y_pred) + (1 - y_true) * np.log(1 - y_pred))

# 适用于二分类
​

def categorical_crossentropy(y_true, y_pred):
    # 避免log(0)
    y_pred = np.clip(y_pred, 1e-15, 1 - 1e-15)
    return -np.sum(y_true * np.log(y_pred)) / y_true.shape[0]

# 适用于多分类
​

反向传播

数学推导

链式法则：

\frac{\partial L}{\partial w} = \frac{\partial L}{\partial y} \cdot \frac{\partial y}{\partial z} \cdot \frac{\partial z}{\partial w}

实现

class NeuralNetwork:
    # ... 前面的代码 ...
    
    def backward(self, X, y, learning_rate=0.01):
        m = X.shape[0]
        
        # 输出层误差
        delta = self.activations[-1] - y
        
        # 反向传播
        for i in reversed(range(len(self.weights))):
            # 计算梯度
            dW = np.dot(self.activations[i].T, delta) / m
            db = np.sum(delta, axis=0, keepdims=True) / m
            
            # 计算下一层的误差
            if i > 0:
                delta = np.dot(delta, self.weights[i].T) * self.sigmoid_derivative(self.z_values[i - 1])
            
            # 更新参数
            self.weights[i] -= learning_rate * dW
            self.biases[i] -= learning_rate * db
    
    def train(self, X, y, epochs=1000, learning_rate=0.01):
        for epoch in range(epochs):
            # 前向传播
            y_pred = self.forward(X)
            
            # 计算损失
            loss = mse_loss(y, y_pred)
            
            # 反向传播
            self.backward(X, y, learning_rate)
            
            if epoch % 100 == 0:
                print(f"Epoch {epoch}, Loss: {loss:.4f}")
​

优化算法

1. 随机梯度下降（SGD）

class SGD:
    def __init__(self, learning_rate=0.01):
        self.learning_rate = learning_rate
    
    def update(self, weights, gradients):
        for i in range(len(weights)):
            weights[i] -= self.learning_rate * gradients[i]
        return weights
​

2. 动量法（Momentum）

class Momentum:
    def __init__(self, learning_rate=0.01, momentum=0.9):
        self.learning_rate = learning_rate
        self.momentum = momentum
        self.velocity = None
    
    def update(self, weights, gradients):
        if self.velocity is None:
            self.velocity = [np.zeros_like(w) for w in weights]
        
        for i in range(len(weights)):
            self.velocity[i] = self.momentum * self.velocity[i] + self.learning_rate * gradients[i]
            weights[i] -= self.velocity[i]
        
        return weights
​

3. Adam（推荐）

class Adam:
    def __init__(self, learning_rate=0.001, beta1=0.9, beta2=0.999, epsilon=1e-8):
        self.learning_rate = learning_rate
        self.beta1 = beta1
        self.beta2 = beta2
        self.epsilon = epsilon
        self.m = None
        self.v = None
        self.t = 0
    
    def update(self, weights, gradients):
        if self.m is None:
            self.m = [np.zeros_like(w) for w in weights]
            self.v = [np.zeros_like(w) for w in weights]
        
        self.t += 1
        
        for i in range(len(weights)):
            # 更新一阶矩估计
            self.m[i] = self.beta1 * self.m[i] + (1 - self.beta1) * gradients[i]
            # 更新二阶矩估计
            self.v[i] = self.beta2 * self.v[i] + (1 - self.beta2) * (gradients[i] ** 2)
            
            # 偏差修正
            m_hat = self.m[i] / (1 - self.beta1 ** self.t)
            v_hat = self.v[i] / (1 - self.beta2 ** self.t)
            
            # 更新参数
            weights[i] -= self.learning_rate * m_hat / (np.sqrt(v_hat) + self.epsilon)
        
        return weights
​

正则化技术

1. L1/L2 正则化

def l2_regularization(weights, lambda_reg):
    reg_loss = 0
    for w in weights:
        reg_loss += np.sum(w ** 2)
    return lambda_reg * reg_loss / 2

# 在损失函数中添加
total_loss = mse_loss + l2_regularization(weights, lambda_reg=0.01)
​

2. Dropout

def dropout(x, keep_prob=0.5, training=True):
    if not training:
        return x
    
    mask = (np.random.rand(*x.shape) < keep_prob) / keep_prob
    return x * mask

# 在前向传播中使用
# z = dropout(z, keep_prob=0.8)
​

3. Batch Normalization

class BatchNormalization:
    def __init__(self, epsilon=1e-5, momentum=0.9):
        self.epsilon = epsilon
        self.momentum = momentum
        self.gamma = None
        self.beta = None
        self.running_mean = None
        self.running_var = None
    
    def forward(self, x, training=True):
        if self.gamma is None:
            N, D = x.shape
            self.gamma = np.ones(D)
            self.beta = np.zeros(D)
            self.running_mean = np.zeros(D)
            self.running_var = np.zeros(D)
        
        if training:
            mu = np.mean(x, axis=0)
            var = np.var(x, axis=0)
            
            # 更新运行统计
            self.running_mean = self.momentum * self.running_mean + (1 - self.momentum) * mu
            self.running_var = self.momentum * self.running_var + (1 - self.momentum) * var
            
            x_normalized = (x - mu) / np.sqrt(var + self.epsilon)
        else:
            x_normalized = (x - self.running_mean) / np.sqrt(self.running_var + self.epsilon)
        
        out = self.gamma * x_normalized + self.beta
        return out
​

实战示例

手写数字识别（MNIST）

import numpy as np
from sklearn.datasets import load_digits
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler

# 加载数据
digits = load_digits()
X = digits.data
y = digits.target

# One-hot 编码
y_onehot = np.zeros((y.size, 10))
y_onehot[np.arange(y.size), y] = 1

# 划分数据集
X_train, X_test, y_train, y_test = train_test_split(X, y_onehot, test_size=0.2)

# 标准化
scaler = StandardScaler()
X_train = scaler.fit_transform(X_train)
X_test = scaler.transform(X_test)

# 创建网络
nn = NeuralNetwork(layer_sizes=[64, 128, 64, 10])

# 训练
nn.train(X_train, y_train, epochs=1000, learning_rate=0.01)

# 测试
y_pred = nn.forward(X_test)
accuracy = np.mean(np.argmax(y_pred, axis=1) == np.argmax(y_test, axis=1))
print(f"Test Accuracy: {accuracy:.4f}")
​

常见问题

1. 梯度消失/爆炸

解决方案：

• 使用 ReLU 激活函数
• Batch Normalization
• 权重初始化（Xavier/He）

2. 过拟合

解决方案：

• 增加数据量
• Dropout
• L1/L2 正则化
• 早停

3. 训练不收敛

解决方案：

• 调整学习率
• 使用更好的优化器（Adam）
• 检查数据预处理
• 调整网络结构

总结

深度学习是神经网络的艺术：

1. 基础组件：神经元、层、激活函数
2. 前向传播：数据从输入流向输出
3. 反向传播：梯度从输出流向输入
4. 优化算法：SGD、Adam 等
5. 正则化：防止过拟合

掌握这些基础，你就可以构建各种深度学习模型！