梯度下降
%matplotlib inline import numpy as np import torch import time from torch import nn, optim import math import sys sys.path.append('/home/input') import d2lzh1981 as d2l
一维梯度下降
证明：沿梯度反方向移动自变量可以减小函数值
泰勒展开：
def f(x): return x**2 # Objective function def gradf(x): return 2 * x # Its derivative def gd(eta): x = 10 results = [x] for i in range(10): x -= eta * gradf(x) results.append(x) print('epoch 10, x:', x) return results res = gd(0.2) def show_trace(res): n = max(abs(min(res)), abs(max(res))) f_line = np.arange(-n, n, 0.01) d2l.set_figsize((3.5, 2.5)) d2l.plt.plot(f_line, [f(x) for x in f_line],'-') d2l.plt.plot(res, [f(x) for x in res],'-o') d2l.plt.xlabel('x') d2l.plt.ylabel('f(x)') show_trace(res)
学习率
show_trace(gd(0.05))
局部极小值
c = 0.15 * np.pi def f(x): return x * np.cos(c * x) def gradf(x): return np.cos(c * x) - c * x * np.sin(c * x) show_trace(gd(2))
多维梯度下降
def train_2d(trainer, steps=20): x1, x2 = -5, -2 results = [(x1, x2)] for i in range(steps): x1, x2 = trainer(x1, x2) results.append((x1, x2)) print('epoch %d, x1 %f, x2 %f' % (i + 1, x1, x2)) return results def show_trace_2d(f, results): d2l.plt.plot(*zip(*results), '-o', color='#ff7f0e') x1, x2 = np.meshgrid(np.arange(-5.5, 1.0, 0.1), np.arange(-3.0, 1.0, 0.1)) d2l.plt.contour(x1, x2, f(x1, x2), colors='#1f77b4') d2l.plt.xlabel('x1') d2l.plt.ylabel('x2') eta = 0.1 def f_2d(x1, x2): # 目标函数 return x1 ** 2 + 2 * x2 ** 2 def gd_2d(x1, x2): return (x1 - eta * 2 * x1, x2 - eta * 4 * x2) show_trace_2d(f_2d, train_2d(gd_2d))
自适应方法
牛顿法
c = 0.5 def f(x): return np.cosh(c * x) # Objective def gradf(x): return c * np.sinh(c * x) # Derivative def hessf(x): return c**2 * np.cosh(c * x) # Hessian # Hide learning rate for now def newton(eta=1): x = 10 results = [x] for i in range(10): x -= eta * gradf(x) / hessf(x) results.append(x) print('epoch 10, x:', x) return results show_trace(newton())
收敛性分析
随机梯度下降参数更新
def f(x1, x2): return x1 ** 2 + 2 * x2 ** 2 # Objective def gradf(x1, x2): return (2 * x1, 4 * x2) # Gradient def sgd(x1, x2): # Simulate noisy gradient global lr # Learning rate scheduler (g1, g2) = gradf(x1, x2) # Compute gradient (g1, g2) = (g1 + np.random.normal(0.1), g2 + np.random.normal(0.1)) eta_t = eta * lr() # Learning rate at time t return (x1 - eta_t * g1, x2 - eta_t * g2) # Update variables eta = 0.1 lr = (lambda: 1) # Constant learning rate show_trace_2d(f, train_2d(sgd, steps=50))
动态学习率
def exponential(): global ctr ctr += 1 return math.exp(-0.1 * ctr) ctr = 1 lr = exponential # Set up learning rate show_trace_2d(f, train_2d(sgd, steps=1000))
小批量随机梯度下降
读取数据
读取数据
def get_data_ch7(): # 本函数已保存在d2lzh_pytorch包中方便以后使用 data = np.genfromtxt('/home/kesci/input/airfoil4755/airfoil_self_noise.dat', delimiter='\t') data = (data - data.mean(axis=0)) / data.std(axis=0) # 标准化 return torch.tensor(data[:1500, :-1], dtype=torch.float32), \ torch.tensor(data[:1500, -1], dtype=torch.float32) # 前1500个样本(每个样本5个特征) features, labels = get_data_ch7() import pandas as pd df = pd.read_csv('/home/kesci/input/airfoil4755/airfoil_self_noise.dat', delimiter='\t', header=None) df.head(10)
从零开始实现
def sgd(params, states, hyperparams): for p in params: p.data -= hyperparams['lr'] * p.grad.data # 本函数已保存在d2lzh_pytorch包中方便以后使用 def train_ch7(optimizer_fn, states, hyperparams, features, labels, batch_size=10, num_epochs=2): # 初始化模型 net, loss = d2l.linreg, d2l.squared_loss w = torch.nn.Parameter(torch.tensor(np.random.normal(0, 0.01, size=(features.shape[1], 1)), dtype=torch.float32), requires_grad=True) b = torch.nn.Parameter(torch.zeros(1, dtype=torch.float32), requires_grad=True) def eval_loss(): return loss(net(features, w, b), labels).mean().item() ls = [eval_loss()] data_iter = torch.utils.data.DataLoader( torch.utils.data.TensorDataset(features, labels), batch_size, shuffle=True) for _ in range(num_epochs): start = time.time() for batch_i, (X, y) in enumerate(data_iter): l = loss(net(X, w, b), y).mean() # 使用平均损失 # 梯度清零 if w.grad is not None: w.grad.data.zero_() b.grad.data.zero_() l.backward() optimizer_fn([w, b], states, hyperparams) # 迭代模型参数 if (batch_i + 1) * batch_size % 100 == 0: ls.append(eval_loss()) # 每100个样本记录下当前训练误差 # 打印结果和作图 print('loss: %f, %f sec per epoch' % (ls[-1], time.time() - start)) d2l.set_figsize() d2l.plt.plot(np.linspace(0, num_epochs, len(ls)), ls) d2l.plt.xlabel('epoch') d2l.plt.ylabel('loss') def train_sgd(lr, batch_size, num_epochs=2): train_ch7(sgd, None, {'lr': lr}, features, labels, batch_size, num_epochs) train_sgd(1, 1500, 6)
简洁实现
# 例如: optimizer_fn=torch.optim.SGD, optimizer_hyperparams={"lr": 0.05} def train_pytorch_ch7(optimizer_fn, optimizer_hyperparams, features, labels, batch_size=10, num_epochs=2): # 初始化模型 net = nn.Sequential( nn.Linear(features.shape[-1], 1) ) loss = nn.MSELoss() optimizer = optimizer_fn(net.parameters(), **optimizer_hyperparams) def eval_loss(): return loss(net(features).view(-1), labels).item() / 2 ls = [eval_loss()] data_iter = torch.utils.data.DataLoader( torch.utils.data.TensorDataset(features, labels), batch_size, shuffle=True) for _ in range(num_epochs): start = time.time() for batch_i, (X, y) in enumerate(data_iter): # 除以2是为了和train_ch7保持一致, 因为squared_loss中除了2 l = loss(net(X).view(-1), y) / 2 optimizer.zero_grad() l.backward() optimizer.step() if (batch_i + 1) * batch_size % 100 == 0: ls.append(eval_loss()) # 打印结果和作图 print('loss: %f, %f sec per epoch' % (ls[-1], time.time() - start)) d2l.set_figsize() d2l.plt.plot(np.linspace(0, num_epochs, len(ls)), ls) d2l.plt.xlabel('epoch') d2l.plt.ylabel('loss') train_pytorch_ch7(optim.SGD, {"lr": 0.05}, features, labels, 10)
参考文献
[1]《动手深度学习》李沐
[2]伯禹教育课程
