### Deep Q-Network (DQN) 的 Python 实现 #### 使用 OpenAI Gym 中 CartPole 环境的 DQN 训练示例 为了展示如何实现 DQN，下面提供了一段基于 PyTorch 和 OpenAI Gym 库的简化版代码。这段代码展示了创建一个简单的 DQN 类以及其主要功能。 ```python import torch import torch.nn as nn import gymnasium as gym import random from collections import deque import numpy as np class DQN(nn.Module): def __init__(self, state_size, action_size): super(DQN, self).__init__() self.fc = nn.Sequential( nn.Linear(state_size, 24), nn.ReLU(), nn.Linear(24, 24), nn.ReLU(), nn.Linear(24, action_size) ) def forward(self, x): return self.fc(x) class Agent: def __init__(self, state_space, action_space): self.state_space = state_space self.action_space = action_space self.memory = deque(maxlen=2000) self.model = DQN(state_space.shape[0], action_space.n).float() self.target_model = DQN(state_space.shape[0], action_space.n).float() self.optimizer = torch.optim.Adam(self.model.parameters(), lr=0.001) self.gamma = 0.95 self.epsilon = 1.0 self.epsilon_min = 0.01 self.epsilon_decay = 0.995 def remember(self, state, action, reward, next_state, done): self.memory.append((state, action, reward, next_state, done)) def act(self, state): if np.random.rand() <= self.epsilon: return random.randrange(self.action_space.n) with torch.no_grad(): act_values = self.model(torch.tensor(state)) return torch.argmax(act_values).item() def replay(self, batch_size): minibatch = random.sample(self.memory, batch_size) for state, action, reward, next_state, done in minibatch: target = reward if not done: with torch.no_grad(): target += self.gamma * \ torch.max(self.target_model(torch.tensor(next_state))) target_f = self.model(torch.tensor(state)).clone().detach() target_f[action] = target criterion = nn.MSELoss() loss = criterion(target_f.unsqueeze(0), self.model(torch.tensor(state)).unsqueeze(0)) self.optimizer.zero_grad() loss.backward() self.optimizer.step() if self.epsilon > self.epsilon_min: self.epsilon *= self.epsilon_decay def update_target_network(self): self.target_model.load_state_dict(self.model.state_dict()) env = gym.make('CartPole-v1') agent = Agent(env.observation_space, env.action_space) batch_size = 32 episodes = 1000 for e in range(episodes): state, _ = env.reset() state = np.reshape(state, [1, env.observation_space.shape[0]]) for time_t in range(500): action = agent.act(state) next_state, reward, terminated, truncated, info = env.step(action) next_state = np.reshape(next_state, [1, env.observation_space.shape[0]]) agent.remember(state, action, reward, next_state, terminated or truncated) state = next_state if len(agent.memory) > batch_size and time_t % 10 == 0: agent.replay(batch_size) if terminated or truncated: break if e % 10 == 0: agent.update_target_network() ``` 此代码片段实现了基本的 DQN 架构并应用于经典的 `CartPole` 游戏环境中[^4]。该架构包括两个重要的组件：一个是用于决策的动作价值函数估计器（即策略网络），另一个是为了提高稳定性而引入的目标网络。随着训练次数增加，代理学会更好地平衡杆子而不让它倒下。