1 引言
Keras提供了四种衰减策略分别是ExponentialDecay(指数衰减)、 PiecewiseConstantDecay(分段常数衰减) 、 PolynomialDecay(多项式衰减)和InverseTimeDecay(逆时间衰减)。只要在Optimizer中指定衰减策略，一行代码就能实现，在以下方法一中详细介绍。
如果想要自定义学习率的衰减，有第二种方法，更加灵活，需要使用callbacks来实现动态、自定义学习率衰减策略，方法二中将详细介绍。
如果两种方法同时使用，默认优先使用第二种，第一种方法将被忽略。
2 实现
2.1 方法一
在Optimizer中指定衰减策略即可，实现简单。
（1）指数衰减
lr_scheduler = tf.keras.optimizers.schedules.ExponentialDecay(
initial_learning_rate=1e-2,
decay_steps=10000,
decay_rate=0.96)
optimizer = tf.keras.optimizers.SGD(learning_rate=lr_scheduler)
（2）分段衰减
[0~1000]steps，学习率为1.0,[10001～9000]steps，学习率为0.5，其他steps，学习率为0.1
step = tf.Variable(0, trainable=False)
boundaries = [1000, 10000]
values = [1.0, 0.5, 0.1]
learning_rate_fn = tf.keras.optimizers.schedules.PiecewiseConstantDecay(boundaries, values)
lr_scheduler = learning_rate_fn(step)
optimizer = tf.keras.optimizers.SGD(learning_rate=lr_scheduler)
（3）多项式衰减
在10000步中从0.1衰减到0.001，使用开根式( power=0.5)
start_lr = 0.1
end_lr = 0.001
decay_steps = 10000
lr_scheduler = tf.keras.optimizers.schedules.PolynomialDecay(
start_lr,
decay_steps,
end_lr,
power=0.5)
optimizer = tf.keras.optimizers.SGD(learning_rate=lr_scheduler)
（4）逆时间衰减
initial_lr = 0.1
decay_steps = 1.0
decay_rate = 0.5
lr_scheduler = keras.optimizers.schedules.InverseTimeDecay(
initial_lr, decay_steps, decay_rate)
optimizer = tf.keras.optimizers.SGD(learning_rate=lr_scheduler)
（5）完整实例
from sklearn import datasets
import numpy as np
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense
from tensorflow.keras.wrappers.scikit_learn import KerasClassifier
from tensorflow.keras.optimizers import SGD
from tensorflow.keras.callbacks import LearningRateScheduler,ModelCheckpoint
from math import pow,floor
dataset =datasets.load_iris()
X = dataset.data
Y = dataset.target
# 设定随机种子
seed =7
np.random.seed(seed)
def create_model(init = 'glorot_uniform'):
#构造模型
model = Sequential()
model.add(Dense(units=4,activation='relu',input_dim=4,kernel_initializer=init))
model.add(Dense(units=6,activation='relu',kernel_initializer=init))
model.add(Dense(units=3,activation='softmax',kernel_initializer=init))
lr_scheduler = tf.keras.optimizers.schedules.ExponentialDecay(
initial_learning_rate=1e-2,
decay_steps=10000,
decay_rate=0.96)
optimizer = SGD(lr=lr_scheduler,momentum=0.9,decay=0.0,nesterov=False)
model.compile(loss='categorical_crossentropy',optimizer=optimizer,metrics=['accuracy'])
if __name__=="__main__":
checkpoint = ModelCheckpoint(filepath="weight.h5",monitor='val_acc',berbose=1,save_best_only=True,mode='max')
model = KerasClassifier(build_fn = create_model,epochs=200,batch_size=5,verbose=1,callbacks=[checkpoint])
model.fit(X,Y)
2.2 方法二
可以自定义学习率的衰减，该方法灵活。
（1）自定义指数衰减
前100epoch学习率不变，之后的epoch指数衰减。在以下程序的注释中，共三个步骤。
from sklearn import datasets
import numpy as np
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense
from tensorflow.keras.wrappers.scikit_learn import KerasClassifier
from tensorflow.keras.optimizers import SGD
from tensorflow.keras.callbacks import LearningRateScheduler,ModelCheckpoint
from math import pow,floor
dataset =datasets.load_iris()
X = dataset.data
Y = dataset.target
seed =7
np.random.seed(seed)
# 第一步：自定义指数衰减策略
def step_decay(epoch):
init_lr = 0.1
drop=0.5
epochs_drop=10
if epoch<100:
return init_lr
else:
return init_lr*pow(drop,floor(1+epoch)/epochs_drop)
def create_model(init = 'glorot_uniform'):
model = Sequential()
model.add(Dense(units=4,activation='relu',input_dim=4,kernel_initializer=init))
model.add(Dense(units=6,activation='relu',kernel_initializer=init))
model.add(Dense(units=3,activation='softmax',kernel_initializer=init))
lr_scheduler = tf.keras.optimizers.schedules.ExponentialDecay(
initial_learning_rate=1e-2,
decay_steps=10000,
decay_rate=0.96)
optimizer = SGD(lr=lr_scheduler,momentum=0.9,decay=0.0,nesterov=False)
model.compile(loss='categorical_crossentropy',optimizer=optimizer,metrics=['accuracy'])
if __name__=="__main__":
checkpoint = ModelCheckpoint(filepath="weight.h5",monitor='val_acc',berbose=1,save_best_only=True,mode='max')
# 第二步：用LearningRateScheduler封装学习率衰减策略
lr_callback = LearningRateScheduler(step_decay)
# 第三步：加入callbacks
model = KerasClassifier(build_fn = create_model,epochs=200,batch_size=5,verbose=1,callbacks=[checkpoint,lr_callback])
model.fit(X,Y)
（2）动态修改学习率
ReduceLROnPlateau(monitor=‘val_acc’, mode=‘max’,min_delta=0.1,factor=0.2,patience=5, min_lr=0.001)
训练集连续patience个epochs的val_acc小于min_delta时，学习率将会乘以factor。mode可以选择max或者min，根据monitor的选择而灵活设定。min_lr是学习率的最低值。
from sklearn import datasets
import numpy as np
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense
from tensorflow.keras.wrappers.scikit_learn import KerasClassifier
from tensorflow.keras.optimizers import SGD
from tensorflow.keras.callbacks import LearningRateScheduler,ModelCheckpoint
from math import pow,floor
dataset =datasets.load_iris()
X = dataset.data
Y = dataset.target
seed =7
np.random.seed(seed)
def create_model(init = 'glorot_uniform'):
model = Sequential()
model.add(Dense(units=4,activation='relu',input_dim=4,kernel_initializer=init))
model.add(Dense(units=6,activation='relu',kernel_initializer=init))
model.add(Dense(units=3,activation='softmax',kernel_initializer=init))
lr_scheduler = tf.keras.optimizers.schedules.ExponentialDecay(
initial_learning_rate=1e-2,
decay_steps=10000,
decay_rate=0.96)
optimizer = SGD(lr=lr_scheduler,momentum=0.9,decay=0.0,nesterov=False)
model.compile(loss='categorical_crossentropy',optimizer=optimizer,metrics=['accuracy'])
if __name__=="__main__":
checkpoint = ModelCheckpoint(filepath="weight.h5",monitor='val_acc',berbose=1,save_best_only=True,mode='max')
# 第一步：ReduceLROnPlateau定义学习动态变化策略
reduce_lr_callback = ReduceLROnPlateau(monitor='val_acc', factor=0.2,patience=5, min_lr=0.001)
# 第二步：加入callbacks
model = KerasClassifier(build_fn = create_model,epochs=200,batch_size=5,verbose=1,callbacks=[checkpoint,reduce_lr_callback])
model.fit(X,Y)
