### Transformer模型中多头注意力机制的主要目的 Transformer模型中的多头注意力机制是其核心组成部分之一，它的主要目的是增强模型捕捉输入序列中多种依赖关系的能力[^3]。通过在不同的表示子空间中并行地关注输入的不同部分，多头注意力机制能够更全面地提取序列信息。 具体来说，多头注意力机制允许模型在同一时间从多个角度分析数据，从而避免了自注意力机制可能存在的缺陷，即模型在对当前位置的信息进行编码时过度集中于自身位置，导致有效信息抓取能力下降的问题[^4]。通过引入多个注意力头，每个头可以独立学习输入的不同特征，最终将这些特征组合起来形成更丰富的表示。 以下是多头注意力机制的计算公式示例，展示如何通过线性变换生成键（Key）、值（Value）和查询（Query），并结合缩放点积注意力完成计算： ```python import torch import torch.nn as nn class MultiHeadAttention(nn.Module): def __init__(self, d_model, num_heads): super(MultiHeadAttention, self).__init__() self.d_model = d_model self.num_heads = num_heads assert d_model % num_heads == 0, "d_model must be divisible by num_heads" self.depth = d_model // num_heads self.wq = nn.Linear(d_model, d_model) self.wk = nn.Linear(d_model, d_model) self.wv = nn.Linear(d_model, d_model) self.dense = nn.Linear(d_model, d_model) def split_heads(self, x, batch_size): x = x.view(batch_size, -1, self.num_heads, self.depth) return x.permute(0, 2, 1, 3) def scaled_dot_product_attention(self, q, k, v): matmul_qk = torch.matmul(q, k.transpose(-2, -1)) dk = k.size()[-1] scaled_attention_logits = matmul_qk / torch.sqrt(torch.tensor(dk, dtype=torch.float32)) attention_weights = torch.softmax(scaled_attention_logits, dim=-1) output = torch.matmul(attention_weights, v) return output, attention_weights def forward(self, v, k, q): batch_size = q.size(0) q = self.wq(q) # (batch_size, seq_len, d_model) k = self.wk(k) # (batch_size, seq_len, d_model) v = self.wv(v) # (batch_size, seq_len, d_model) q = self.split_heads(q, batch_size) # (batch_size, num_heads, seq_len, depth) k = self.split_heads(k, batch_size) # (batch_size, num_heads, seq_len, depth) v = self.split_heads(v, batch_size) # (batch_size, num_heads, seq_len, depth) scaled_attention, attention_weights = self.scaled_dot_product_attention(q, k, v) scaled_attention = scaled_attention.permute(0, 2, 1, 3).contiguous() concat_attention = scaled_attention.view(batch_size, -1, self.d_model) output = self.dense(concat_attention) return output, attention_weights ```