### 使用PyTorch实现MADDPG算法 MADDPG (Multi-Agent Deep Deterministic Policy Gradient) 是一种用于处理多智能体环境下的强化学习方法。下面提供了一个基于PyTorch框架下实现该算法的核心部分。 #### 初始化配置与导入库 为了确保代码可以在GPU上高效执行，需按照如下方式设置： ```python import torch from torch import nn import numpy as np device = 'cuda' if torch.cuda.is_available() else 'cpu' print(f"Using {device} device") ``` #### 定义Actor-Critic网络结构 定义两个主要组件——策略网络（actor）和价值函数网络（critic），二者均继承自`nn.Module`类[^4]。 ```python class Actor(nn.Module): def __init__(self, input_dim, output_dim): super(Actor, self).__init__() self.network = nn.Sequential( nn.Linear(input_dim, 64), nn.ReLU(), nn.Linear(64, 128), nn.ReLU(), nn.Linear(128, output_dim), nn.Tanh() ) def forward(self, state): action = self.network(state) return action class Critic(nn.Module): def __init__(self, obs_dim, act_dim, n_agents): super(Critic, self).__init__() total_input_size = obs_dim * n_agents + act_dim * n_agents self.network = nn.Sequential( nn.Linear(total_input_size, 256), nn.ReLU(), nn.Linear(256, 256), nn.ReLU(), nn.Linear(256, 1) ) def forward(self, all_states, all_actions): concat_input = torch.cat([all_states.view(-1), all_actions.view(-1)], dim=-1).float().to(device) value = self.network(concat_input) return value ``` #### MADDPG Agent 类设计 封装上述两者的交互逻辑，并加入经验回放缓冲区等功能模块。 ```python from collections import deque import random BUFFER_SIZE = int(1e5) BATCH_SIZE = 64 class MADDPGAgent(): """Interacts with and learns from the environment.""" memory = deque(maxlen=BUFFER_SIZE) def __init__(self, agent_id, actor_local, critic_local, actor_target, critic_target, actor_optimizer, critic_optimizer, noise=None): self.agent_id = agent_id # Networks self.actor_local = actor_local.to(device) self.critic_local = critic_local.to(device) self.actor_target = actor_target.to(device) self.critic_target = critic_target.to(device) # Optimizers self.actor_optimizer = actor_optimizer self.critic_optimizer = critic_optimizer # Noise process self.noise = noise @staticmethod def step(states, actions, rewards, next_states, dones): experiences = (states, actions, rewards, next_states, dones) MADDPGAgent.memory.append(experiences) if len(MADDPGAgent.memory) > BATCH_SIZE: samples = random.sample(list(MADDPGAgent.memory), k=BATCH_SIZE) states_batch, actions_batch, rewards_batch, next_states_batch, dones_batch = zip(*samples) for i in range(len(samples)): maddpg_agent.learn((np.vstack(states_batch[i]), np.vstack(actions_batch[i]), rewards_batch[i], np.vstack(next_states_batch[i]), dones_batch[i])) ``` 此段代码展示了如何利用PyTorch来搭建一个多智能体系统的训练流程，其中包含了必要的模型初始化、数据预处理以及更新规则等内容[^1]。