### Transformer Encoder 实现代码 在 PyTorch 中实现 Transformer 的 Encoder 部分涉及多个模块的组合，包括多头自注意力机制、残差连接和前馈神经网络。以下是基于这些组件构建的一个完整的 Transformer Encoder 层的 Python 代码示例： ```python import torch import torch.nn as nn import math class PositionalEncoding(nn.Module): def __init__(self, d_model: int, dropout: float = 0.1, max_len: int = 5000): super(PositionalEncoding, self).__init__() pe = torch.zeros(max_len, d_model) position = torch.arange(0, max_len, dtype=torch.float).unsqueeze(1) div_term = torch.exp(torch.arange(0, d_model, 2).float() * (-math.log(10000.0) / d_model)) pe[:, 0::2] = torch.sin(position * div_term) pe[:, 1::2] = torch.cos(position * div_term) pe = pe.unsqueeze(0).transpose(0, 1) mask = self._generate_square_subsequent_mask(pe.size(0)).to(next(self.parameters()).device) self.register_buffer('pe', pe) def forward(self, x): x = x + self.pe[:x.size(0), :] return x def _generate_square_subsequent_mask(self, sz: int) -> torch.Tensor: """Generates an upper-triangular matrix of -inf, with zeros on diag.""" return torch.triu(torch.ones(sz, sz) * float('-inf'), diagonal=1) class TransformerEncoderLayer(nn.Module): def __init__(self, d_model: int, nhead: int, dim_feedforward: int = 2048, dropout: float = 0.1): super().__init__() self.self_attn = nn.MultiheadAttention(d_model, nhead, dropout=dropout) # Implementation of Feedforward model self.linear1 = nn.Linear(d_model, dim_feedforward) self.dropout = nn.Dropout(dropout) self.linear2 = nn.Linear(dim_feedforward, d_model) self.norm1 = nn.LayerNorm(d_model) self.norm2 = nn.LayerNorm(d_model) self.dropout1 = nn.Dropout(dropout) self.dropout2 = nn.Dropout(dropout) self.activation = nn.ReLU() def forward(self, src: torch.Tensor, src_mask: torch.Tensor = None, src_key_padding_mask: torch.Tensor = None) -> torch.Tensor: src2 = self.self_attn(src, src, src, attn_mask=src_mask, key_padding_mask=src_key_padding_mask)[0] src = src + self.dropout1(src2) src = self.norm1(src) src2 = self.linear2(self.dropout(self.activation(self.linear1(src)))) src = src + self.dropout2(src2) src = self.norm2(src) return src class TransformerModel(nn.Module): def __init__(self, ninp: int, nhead: int, nhid: int, nlayers: int, dropout: float = 0.5): super().__init__() self.model_type = 'Transformer' self.src_mask = None self.pos_encoder = PositionalEncoding(ninp, dropout) encoder_layers = TransformerEncoderLayer(ninp, nhead, nhid, dropout) self.transformer_encoder = nn.TransformerEncoder(encoder_layers, nlayers) self.ninp = ninp def _generate_square_subsequent_mask(self, sz: int) -> torch.Tensor: mask = (torch.triu(torch.ones(sz, sz)) == 1).transpose(0, 1) mask = mask.float().masked_fill(mask == 0, float('-inf')).masked_fill(mask == 1, float(0.0)) return mask def forward(self, src: torch.Tensor, has_mask=True): if has_mask: device = src.device if self.src_mask is None or self.src_mask.size(0) != len(src): mask = self._generate_square_subsequent_mask(len(src)).to(device) self.src_mask = mask else: self.src_mask = None src = self.pos_encoder(src) output = self.transformer_encoder(src, self.src_mask) return output ``` 上述代码展示了如何定义位置编码器 `PositionalEncoding` 和单个编码层 `TransformerEncoderLayer`，并最终组装成一个多层的变压器编码器模型 `TransformerModel`[^3]。 #### 关键点解释 - **位置编码**：为了赋予模型处理顺序信息的能力，引入了正弦波形式的位置嵌入。 - **Multi-head Attention**：允许模型在同一层次上关注不同子空间中的信息流。 - **Add & Norm**：每个子层周围都有一个残差连接，并跟随一层归一化操作。 - **Feed Forward Networks**：全连接前馈网络应用于每个位置独立地相同方式。