随机梯度下降
随机梯度下降参数更新
对于有 
个样本对训练数据集，设 
是第 
个样本的损失函数, 则目标函数为:
其梯度为:
使用该梯度的一次更新的时间复杂度为 
随机梯度下降更新公式 
:
且有：
e.g.
def f(x1, x2): return x1 ** 2 + 2 * x2 ** 2 # Objective def gradf(x1, x2): return (2 * x1, 4 * x2) # Gradient def sgd(x1, x2): # Simulate noisy gradient global lr # Learning rate scheduler (g1, g2) = gradf(x1, x2) # Compute gradient (g1, g2) = (g1 + np.random.normal(0.1), g2 + np.random.normal(0.1)) eta_t = eta * lr() # Learning rate at time t return (x1 - eta_t * g1, x2 - eta_t * g2) # Update variables eta = 0.1 lr = (lambda: 1) # Constant learning rate show_trace_2d(f, train_2d(sgd, steps=50))
epoch 50, x1 -0.027566, x2 0.137605
动态学习率
def exponential(): global ctr ctr += 1 return math.exp(-0.1 * ctr) ctr = 1 lr = exponential # Set up learning rate show_trace_2d(f, train_2d(sgd, steps=1000))
epoch 1000, x1 -0.677947, x2 -0.089379
def polynomial(): global ctr ctr += 1 return (1 + 0.1 * ctr)**(-0.5) ctr = 1 lr = polynomial # Set up learning rate show_trace_2d(f, train_2d(sgd, steps=50))
epoch 50, x1 -0.095244, x2 -0.041674
小批量随机梯度下降
读取数据
def get_data_ch7(): # 本函数已保存在d2lzh_pytorch包中方便以后使用 data = np.genfromtxt('/home/kesci/input/airfoil4755/airfoil_self_noise.dat', delimiter='\t') data = (data - data.mean(axis=0)) / data.std(axis=0) # 标准化 return torch.tensor(data[:1500, :-1], dtype=torch.float32), \ torch.tensor(data[:1500, -1], dtype=torch.float32) # 前1500个样本(每个样本5个特征) features, labels = get_data_ch7() features.shape
torch.Size([1500, 5])
import pandas as pd df = pd.read_csv('/home/kesci/input/airfoil4755/airfoil_self_noise.dat', delimiter='\t', header=None) df.head(10)
从零开始实现
def sgd(params, states, hyperparams): for p in params: p.data -= hyperparams['lr'] * p.grad.data
# 本函数已保存在d2lzh_pytorch包中方便以后使用 def train_ch7(optimizer_fn, states, hyperparams, features, labels, batch_size=10, num_epochs=2): # 初始化模型 net, loss = d2l.linreg, d2l.squared_loss w = torch.nn.Parameter(torch.tensor(np.random.normal(0, 0.01, size=(features.shape[1], 1)), dtype=torch.float32), requires_grad=True) b = torch.nn.Parameter(torch.zeros(1, dtype=torch.float32), requires_grad=True) def eval_loss(): return loss(net(features, w, b), labels).mean().item() ls = [eval_loss()] data_iter = torch.utils.data.DataLoader( torch.utils.data.TensorDataset(features, labels), batch_size, shuffle=True) for _ in range(num_epochs): start = time.time() for batch_i, (X, y) in enumerate(data_iter): l = loss(net(X, w, b), y).mean() # 使用平均损失 # 梯度清零 if w.grad is not None: w.grad.data.zero_() b.grad.data.zero_() l.backward() optimizer_fn([w, b], states, hyperparams) # 迭代模型参数 if (batch_i + 1) * batch_size % 100 == 0: ls.append(eval_loss()) # 每100个样本记录下当前训练误差 # 打印结果和作图 print('loss: %f, %f sec per epoch' % (ls[-1], time.time() - start)) d2l.set_figsize() d2l.plt.plot(np.linspace(0, num_epochs, len(ls)), ls) d2l.plt.xlabel('epoch') d2l.plt.ylabel('loss')
def train_sgd(lr, batch_size, num_epochs=2): train_ch7(sgd, None, {'lr': lr}, features, labels, batch_size, num_epochs)
对比
train_sgd(1, 1500, 6)
loss: 0.244373, 0.009881 sec per epoch
train_sgd(0.005, 1)
loss: 0.245968, 0.463836 sec per epoch
train_sgd(0.05, 10)
loss: 0.243900, 0.065017 sec per epoch
简洁实现
# 本函数与原书不同的是这里第一个参数优化器函数而不是优化器的名字 # 例如: optimizer_fn=torch.optim.SGD, optimizer_hyperparams={"lr": 0.05} def train_pytorch_ch7(optimizer_fn, optimizer_hyperparams, features, labels, batch_size=10, num_epochs=2): # 初始化模型 net = nn.Sequential( nn.Linear(features.shape[-1], 1) ) loss = nn.MSELoss() optimizer = optimizer_fn(net.parameters(), **optimizer_hyperparams) def eval_loss(): return loss(net(features).view(-1), labels).item() / 2 ls = [eval_loss()] data_iter = torch.utils.data.DataLoader( torch.utils.data.TensorDataset(features, labels), batch_size, shuffle=True) for _ in range(num_epochs): start = time.time() for batch_i, (X, y) in enumerate(data_iter): # 除以2是为了和train_ch7保持一致, 因为squared_loss中除了2 l = loss(net(X).view(-1), y) / 2 optimizer.zero_grad() l.backward() optimizer.step() if (batch_i + 1) * batch_size % 100 == 0: ls.append(eval_loss()) # 打印结果和作图 print('loss: %f, %f sec per epoch' % (ls[-1], time.time() - start)) d2l.set_figsize() d2l.plt.plot(np.linspace(0, num_epochs, len(ls)), ls) d2l.plt.xlabel('epoch') d2l.plt.ylabel('loss')
train_pytorch_ch7(optim.SGD, {"lr": 0.05}, features, labels, 10)
loss: 0.243770, 0.047664 sec per epoch
