### 基于 TensorFlow 1.x 实现 Transformer 模型 #### 导入必要的库 为了兼容旧版代码，在导入 `tensorflow` 时需指定使用 1.x 版本的行为模式[^1]。 ```python import tensorflow.compat.v1 as tf tf.disable_v2_behavior() ``` #### 定义超参数配置类 创建一个用于保存模型所需的各种设置项的类，以便后续调用更加方便简洁。 ```python class Config(object): """Model hyperparameters and configurations.""" def __init__(self, vocab_size=30522, hidden_size=768, num_hidden_layers=12, num_attention_heads=12, intermediate_size=3072, hidden_act="gelu", hidden_dropout_prob=0.1, attention_probs_dropout_prob=0.1, max_position_embeddings=512, type_vocab_size=2, initializer_range=0.02): self.vocab_size = vocab_size self.hidden_size = hidden_size self.num_hidden_layers = num_hidden_layers self.num_attention_heads = num_attention_heads self.intermediate_size = intermediate_size self.hidden_act = hidden_act self.hidden_dropout_prob = hidden_dropout_prob self.attention_probs_dropout_prob = attention_probs_dropout_prob self.max_position_embeddings = max_position_embeddings self.type_vocab_size = type_vocab_size self.initializer_range = initializer_range ``` #### 构建多头注意力机制层 这是Transformer架构的核心组件之一，负责捕捉输入序列中的不同位置间的关系[^2]。 ```python def multihead_attention(queries, keys, values, config, mask=None): Q = tf.layers.dense(queries, config.hidden_size * config.num_attention_heads, use_bias=False) K = tf.layers.dense(keys, config.hidden_size * config.num_attention_heads, use_bias=False) V = tf.layers.dense(values, config.hidden_size * config.num_attention_heads, use_bias=False) # Split heads Q_ = split_heads(Q, config.num_attention_heads) K_ = split_heads(K, config.num_attention_heads) V_ = split_heads(V, config.num_attention_heads) outputs = scaled_dot_product_attention(Q_, K_, V_, mask) outputs = combine_heads(outputs) outputs = tf.layers.dense(outputs, config.hidden_size) return outputs def split_heads(x, num_heads): batch_size, seq_len, depth = x.get_shape().as_list() assert depth % num_heads == 0 reshaped_x = tf.reshape(x, [-1, seq_len, num_heads, depth // num_heads]) transposed_x = tf.transpose(reshaped_x, [0, 2, 1, 3]) # (batch_size, num_heads, seq_len, depth_per_head) return transposed_x def combine_heads(x): _, num_heads, seq_len, depth_per_head = x.get_shape().as_list() transposed_x = tf.transpose(x, [0, 2, 1, 3]) # (batch_size, seq_len, num_heads, depth_per_head) concatenated_x = tf.reshape(transposed_x, [-1, seq_len, num_heads * depth_per_head]) return concatenated_x def scaled_dot_product_attention(q, k, v, mask=None): matmul_qk = tf.matmul(q, k, transpose_b=True) dk = tf.cast(tf.shape(k)[-1], dtype=tf.float32) logits = matmul_qk / tf.sqrt(dk) if mask is not None: paddings = tf.ones_like(logits) * (-2 ** 32 + 1) logits = tf.where(mask, logits, paddings) weights = tf.nn.softmax(logits, axis=-1) output = tf.matmul(weights, v) return output ``` #### 创建前馈神经网络模块 该部分用来处理来自自注意力层的信息并进一步提取特征。 ```python def feed_forward_network(inputs, config): dense_layer_1 = tf.layers.dense( inputs=inputs, units=config.intermediate_size, activation=get_activation(config.hidden_act), kernel_initializer=create_initializer(config.initializer_range)) dense_layer_2 = tf.layers.dense( inputs=dense_layer_1, units=config.hidden_size, kernel_initializer=create_initializer(config.initializer_range)) return dense_layer_2 def get_activation(activation_string): act_fn = getattr(tf.nn, activation_string.lower()) return act_fn def create_initializer(initializer_range=0.02): return tf.truncated_normal_initializer(stddev=initializer_range) ``` #### 组合各部件形成完整的编码器堆栈 通过重复上述定义好的两个主要功能块来构建整个Encoder结构。 ```python def transformer_encoder(input_tensor, config, attention_mask=None): prev_output = input_tensor all_layer_outputs = [] for layer_idx in range(config.num_hidden_layers): with tf.variable_scope(f"layer_{layer_idx}"): layer_input = prev_output with tf.variable_scope("attention/self"): attention_output = multihead_attention(layer_input, layer_input, layer_input, config, mask=attention_mask) with tf.variable_scope("ffn"): ffn_output = feed_forward_network(attention_output, config) prev_output = ffn_output all_layer_outputs.append(prev_output) final_outputs = all_layer_outputs[-1] return final_outputs ``` 此段代码展示了如何利用TensorFlow 1.x API 来实现一个基础版本的Transformer Encoder。对于Decoder以及其他细节优化，则可以根据具体应用场景做相应调整扩展。