### PyTorch Transformer 模型使用教程 #### 创建数据集并准备输入 为了使用 `torch.nn.Transformer` 进行训练，需要创建合适的数据集。对于时间序列预测任务而言，这通常意味着将连续的时间窗口作为输入，并尝试预测下一个时间步的结果。 ```python import torch from torch import nn, Tensor import math from typing import Tuple class TimeSeriesDataset(torch.utils.data.Dataset): def __init__(self, data: Tensor, seq_len: int = 10): self.seq_len = seq_len self.data = data def __len__(self) -> int: return len(self.data) - self.seq_len def __getitem__(self, idx: int) -> Tuple[Tensor, Tensor]: x = self.data[idx : (idx + self.seq_len)] y = self.data[(idx + self.seq_len)].unsqueeze(0) return x.float(), y.float() ``` #### 定义Transformer模型结构 定义一个基于PyTorch的变压器架构来适应特定的任务需求。这里展示了一个简化版本的例子[^2]： ```python class PositionalEncoding(nn.Module): def __init__(self, d_model: int, dropout: float = 0.1, max_len: int = 5000): super().__init__() position = torch.arange(max_len).unsqueeze(1) div_term = torch.exp(torch.arange(0, d_model, 2) * (-math.log(10000.0) / d_model)) pe = torch.zeros(max_len, 1, d_model) pe[:, 0, 0::2] = torch.sin(position * div_term) pe[:, 0, 1::2] = torch.cos(position * div_term) self.register_buffer('pe', pe) def forward(self, x: Tensor) -> Tensor: """ Args: x: Tensor, shape [seq_len, batch_size, embedding_dim] """ x = x + self.pe[:x.size(0)] return x class TransformerModel(nn.Module): def __init__(self, input_dim: int, nhead: int, num_encoder_layers: int, num_decoder_layers: int, dim_feedforward: int, output_dim: int): super().__init__() self.model_type = 'Transformer' self.pos_encoder = PositionalEncoding(input_dim) self.encoder_layer = nn.TransformerEncoderLayer(d_model=input_dim, nhead=nhead, dim_feedforward=dim_feedforward, batch_first=True) self.transformer_encoder = nn.TransformerEncoder(self.encoder_layer, num_layers=num_encoder_layers) self.decoder = nn.Linear(input_dim, output_dim) self.init_weights() def init_weights(self) -> None: initrange = 0.1 self.decoder.bias.data.zero_() self.decoder.weight.data.uniform_(-initrange, initrange) def forward(self, src: Tensor) -> Tensor: """ Arguments: src: Tensor, shape ``[batch_size, seq_len, input_dim]`` Returns: output Tensor of shape ``[batch_size, output_dim]`` """ src = self.pos_encoder(src) output = self.transformer_encoder(src) output = self.decoder(output.mean(dim=1)) return output ``` #### 训练过程 设置优化器、损失函数以及编写循环来进行参数更新。下面是一个简单的例子说明如何完成这部分工作: ```python device = torch.device("cuda" if torch.cuda.is_available() else "cpu") model = TransformerModel( input_dim=..., # 输入特征维度 nhead=8, # 头数 num_encoder_layers=6,# 编码层数量 num_decoder_layers=6,# 解码层数量（如果不需要解码层可以设为零） dim_feedforward=2048,# 前馈网络隐藏单元数量 output_dim=... # 输出类别数目或回归目标维数 ).to(device) criterion = nn.MSELoss().to(device) # 对于回归问题采用均方误差损失；如果是分类则可能要用交叉熵等其他形式 optimizer = torch.optim.Adam(model.parameters(), lr=0.001) for epoch in range(num_epochs): # 遍历整个数据集多次 running_loss = 0. for i, data in enumerate(train_loader, start=0): inputs, labels = data optimizer.zero_grad() outputs = model(inputs.to(device)) loss = criterion(outputs.squeeze(), labels.to(device)) loss.backward() optimizer.step() running_loss += loss.item() ```