强化学习（RL）在机器人领域的应用

import gym
import numpy as np
import random
from collections import defaultdict
 
# 环境初始化
env = gym.make("CartPole-v1")
 
# Q-learning参数
alpha = 0.1  # 学习率
gamma = 0.99  # 折扣因子
epsilon = 1.0  # 初始探索率
epsilon_decay = 0.995  # 探索率衰减
epsilon_min = 0.01  # 最小探索率
num_episodes = 1000  # 总训练回合数
max_steps = 200  # 每回合最大步数
 
# 初始化Q表格
Q = defaultdict(lambda: np.zeros(env.action_space.n))
 
# 将观测值离散化
def discretize_state(state, bins):
    discrete_state = []
    for i in range(len(state)):
        discrete_state.append(np.digitize(state[i], bins[i]) - 1)
    return tuple(discrete_state)
 
# 创建观察值的离散桶
state_bins = [
    np.linspace(-4.8, 4.8, 10),
    np.linspace(-4, 4, 10),
    np.linspace(-0.418, 0.418, 10),
    np.linspace(-4, 4, 10)
]
 
def select_action(state, epsilon):
    if random.uniform(0, 1) < epsilon:
        return env.action_space.sample()  # 随机动作
    else:
        return np.argmax(Q[state])  # 贪婪动作
 
# 训练过程
for episode in range(num_episodes):
    state = discretize_state(env.reset(), state_bins)
    total_reward = 0
    
    for step in range(max_steps):
        action = select_action(state, epsilon)
        next_state_raw, reward, done, _ = env.step(action)
        next_state = discretize_state(next_state_raw, state_bins)
 
        best_next_action = np.argmax(Q[next_state])
        td_target = reward + gamma * Q[next_state][best_next_action]
        td_delta = td_target - Q[state][action]
        Q[state][action] += alpha * td_delta
 
        state = next_state
        total_reward += reward
 
        if done:
            break
 
    epsilon = max(epsilon_min, epsilon_decay * epsilon)
 
    print(f"Episode {episode + 1}: Total Reward = {total_reward}")
 
print("训练完成！")
 
# 测试训练后的智能体
state = discretize_state(env.reset(), state_bins)
for t in range(max_steps):
    env.render()
    action = select_action(state, epsilon=0.0)  # 使用贪婪策略
    next_state, reward, done, _ = env.step(action)
    state = discretize_state(next_state, state_bins)
    if done:
        break
 
env.close()