### Q-Learning算法解决迷宫问题的Python实现 以下是一个完整的Q-Learning算法解决迷宫问题的Python代码示例。该代码结合了环境定义、Q-table更新以及可视化展示的功能。 ```python import numpy as np import pandas as pd import time class MazeEnv: def __init__(self): self.actions = ['up', 'down', 'left', 'right'] self.n_actions = len(self.actions) self.state_space = [(i, j) for i in range(4) for j in range(4)] self.goal_state = (3, 3) self.obstacle_states = [(1, 1), (2, 2)] def reset(self): return (0, 0) def step(self, state, action): x, y = state if action == 'up' and x > 0: x -= 1 elif action == 'down' and x < 3: x += 1 elif action == 'left' and y > 0: y -= 1 elif action == 'right' and y < 3: y += 1 next_state = (x, y) if next_state in self.obstacle_states: reward = -10 done = False elif next_state == self.goal_state: reward = 10 done = True else: reward = -1 done = False return next_state, reward, done class QLearningAgent: def __init__(self, actions, learning_rate=0.1, discount_factor=0.9, epsilon=0.1): self.actions = actions self.lr = learning_rate self.gamma = discount_factor self.epsilon = epsilon self.q_table = pd.DataFrame(columns=self.actions, dtype=np.float64) def choose_action(self, state): self.check_state_exist(state) if np.random.uniform() < self.epsilon: action = np.random.choice(self.actions) else: state_actions = self.q_table.loc[state, :] action = np.random.choice(state_actions[state_actions == np.max(state_actions)].index) return action def learn(self, state, action, reward, next_state): self.check_state_exist(next_state) q_predict = self.q_table.loc[state, action] if next_state != (3, 3): q_target = reward + self.gamma * self.q_table.loc[next_state, :].max() else: q_target = reward self.q_table.loc[state, action] += self.lr * (q_target - q_predict) def check_state_exist(self, state): if state not in self.q_table.index: self.q_table = self.q_table.append( pd.Series([0] * len(self.actions), index=self.q_table.columns, name=state)) def update(): for episode in range(100): state = env.reset() while True: env.render(state) action = agent.choose_action(str(state)) next_state, reward, done = env.step(state, action) agent.learn(str(state), action, reward, str(next_state)) state = next_state if done: break print('Game Over') env.render(state) if __name__ == "__main__": env = MazeEnv() agent = QLearningAgent(env.actions) update() ``` ### 代码说明 1. **环境定义**：`MazeEnv`类定义了迷宫的状态空间、动作空间以及奖励机制。智能体通过与环境交互获得奖励和下一个状态[^2]。 2. **智能体实现**：`QLearningAgent`类实现了Q-Learning的核心逻辑，包括选择动作、学习Q值表等操作[^5]。 3. **训练过程**：`update`函数控制了整个训练流程，通过多次迭代让智能体逐步学会最优策略[^4]。 ### 迷宫环境的可视化 可以通过`matplotlib`库对迷宫进行可视化展示。例如，可以将迷宫表示为一个4x4网格图，并用不同颜色标记障碍物、起点和终点。 ```python import matplotlib.pyplot as plt import matplotlib.patches as patches def render_maze(ax, state, goal_state=(3, 3), obstacle_states=[(1, 1), (2, 2)]): ax.clear() ax.set_xlim(0, 4) ax.set_ylim(0, 4) ax.invert_yaxis() ax.set_xticks(range(5)) ax.set_yticks(range(5)) ax.grid(True) # Draw obstacles for obs in obstacle_states: rect = patches.Rectangle((obs[1], obs[0]), 1, 1, linewidth=1, edgecolor='black', facecolor='gray') ax.add_patch(rect) # Draw goal rect = patches.Rectangle((goal_state[1], goal_state[0]), 1, 1, linewidth=1, edgecolor='black', facecolor='green') ax.add_patch(rect) # Draw current state rect = patches.Rectangle((state[1], state[0]), 1, 1, linewidth=1, edgecolor='black', facecolor='blue') ax.add_patch(rect) fig, ax = plt.subplots() for i in range(100): state = env.reset() while True: render_maze(ax, state) plt.pause(0.5) action = agent.choose_action(str(state)) next_state, _, done = env.step(state, action) state = next_state if done: break plt.show() ``` ### 注意事项 - 在实际运行中，需要确保安装了`numpy`、`pandas`和`matplotlib`库[^1]。 - 可以根据需求调整迷宫的大小、障碍物位置以及奖励机制[^3]。