### 关于 Transformer 论文复现的方法 Transformer 是一种基于注意力机制的神经网络架构，最初由 Vaswani 等人在论文《Attention is All You Need》中提出[^3]。以下是关于如何通过 PyTorch 和 TensorFlow 来实现 Transformer 的具体方法。 #### 使用 PyTorch 实现 Transformer PyTorch 提供了一个官方的 `torch.nn.Transformer` 类来简化 Transformer 架构的构建过程。如果希望从头开始实现，则可以参考以下代码： ```python import torch import torch.nn as nn class PositionalEncoding(nn.Module): def __init__(self, d_model, max_len=5000): super(PositionalEncoding, self).__init__() pe = torch.zeros(max_len, d_model) position = torch.arange(0, max_len, dtype=torch.float).unsqueeze(1) div_term = torch.exp(torch.arange(0, d_model, 2).float() * (-torch.log(torch.tensor(10000.0)) / d_model)) pe[:, 0::2] = torch.sin(position * div_term) pe[:, 1::2] = torch.cos(position * div_term) pe = pe.unsqueeze(0).transpose(0, 1) self.register_buffer('pe', pe) def forward(self, x): x = x + self.pe[:x.size(0), :] return x class TransformerModel(nn.Module): def __init__(self, ntoken, ninp, nhead, nhid, nlayers, dropout=0.5): super(TransformerModel, self).__init__() from torch.nn import TransformerEncoder, TransformerEncoderLayer self.model_type = 'Transformer' self.src_mask = None self.pos_encoder = PositionalEncoding(ninp) encoder_layers = TransformerEncoderLayer(ninp, nhead, nhid, dropout) self.transformer_encoder = TransformerEncoder(encoder_layers, nlayers) self.encoder = nn.Embedding(ntoken, ninp) self.ninp = ninp self.decoder = nn.Linear(ninp, ntoken) self.init_weights() def _generate_square_subsequent_mask(self, sz): mask = (torch.triu(torch.ones(sz, sz)) == 1).transpose(0, 1) mask = mask.float().masked_fill(mask == 0, float('-inf')).masked_fill(mask == 1, float(0.0)) return mask def init_weights(self): initrange = 0.1 self.encoder.weight.data.uniform_(-initrange, initrange) self.decoder.bias.data.zero_() self.decoder.weight.data.uniform_(-initrange, initrange) def forward(self, src): if self.src_mask is None or self.src_mask.size(0) != len(src): device = src.device mask = self._generate_square_subsequent_mask(len(src)).to(device) self.src_mask = mask src = self.encoder(src) * math.sqrt(self.ninp) src = self.pos_encoder(src) output = self.transformer_encoder(src, self.src_mask) output = self.decoder(output) return output ``` 上述代码展示了如何手动创建一个完整的 Transformer 模型，并包含了位置编码的功能[^4]。 #### 使用 TensorFlow/Keras 实现 Transformer TensorFlow 同样提供了内置模块支持 Transformer 的快速搭建。下面是一个简单的例子： ```python from tensorflow.keras.layers import LayerNormalization, Embedding, Dense, MultiHeadAttention, Dropout from tensorflow.keras.models import Model import numpy as np def get_positional_encoding(seq_length, embed_dim): angle_rads = np.array([[pos / np.power(10000, 2*(i//2)/embed_dim) for i in range(embed_dim)] for pos in range(seq_length)]) angle_rads[:, 0::2] = np.sin(angle_rads[:, 0::2]) angle_rads[:, 1::2] = np.cos(angle_rads[:, 1::2]) pos_encoding = angle_rads[np.newaxis, ...] return tf.cast(pos_encoding, dtype=tf.float32) class TransformerBlock(Model): def __init__(self, embed_dim, num_heads, ff_dim, rate=0.1): super().__init__() self.att = MultiHeadAttention(num_heads=num_heads, key_dim=embed_dim) self.ffn = Sequential([ Dense(ff_dim, activation="relu"), Dense(embed_dim), ]) self.layernorm1 = LayerNormalization(epsilon=1e-6) self.layernorm2 = LayerNormalization(epsilon=1e-6) self.dropout1 = Dropout(rate) self.dropout2 = Dropout(rate) def call(self, inputs, training=False): attn_output = self.att(inputs, inputs) attn_output = self.dropout1(attn_output, training=training) out1 = self.layernorm1(inputs + attn_output) ffn_output = self.ffn(out1) ffn_output = self.dropout2(ffn_output, training=training) return self.layernorm2(out1 + ffn_output) class TokenAndPositionEmbedding(Model): def __init__(self, maxlen, vocab_size, embed_dim): super().__init__() self.token_emb = Embedding(input_dim=vocab_size, output_dim=embed_dim) self.pos_emb = Embedding(input_dim=maxlen, output_dim=embed_dim) def call(self, x): seq_len = tf.shape(x)[1] positions = tf.range(start=0, limit=seq_len, delta=1) positions = self.pos_emb(positions) x = self.token_emb(x) return x + positions ``` 这段代码定义了 Transformer 块以及嵌入层的位置编码功能[^5]。 #### GitHub 资源推荐 对于更详细的实现教程和开源项目，可以从以下几个 GitHub 链接获取帮助： - [PyTorch 官方 Transformer 教程](https://pytorch.org/tutorials/beginner/transformer_tutorial.html)[^6] - [Hugging Face Transformers 库](https://github.com/huggingface/transformers)，这是一个非常流行的库，用于加载预训练模型并微调它们。 - [Vision Transformer (ViT) PyTorch 实现](https://github.com/lucidrains/vit-pytorch)[^7] --- ###