以下是一个完整的 Python 实现的 Charformer 模型，适用于文本分类任务，包含数据预处理、模型定义、训练和评估流程。 ### 数据预处理 文本数据需转换为字符序列，并构建字符到索引的映射。训练和测试数据集应分别进行处理，并确保输入长度一致。 ```python import torch from torch.utils.data import Dataset, DataLoader class CharDataset(Dataset): def __init__(self, texts, labels, char_to_idx, max_len=128): self.texts = texts self.labels = labels self.char_to_idx = char_to_idx self.max_len = max_len def __len__(self): return len(self.texts) def __getitem__(self, idx): text = self.texts[idx] label = self.labels[idx] char_indices = [self.char_to_idx.get(c, 0) for c in text[:self.max_len]] char_indices += [0] * (self.max_len - len(char_indices)) return torch.tensor(char_indices), torch.tensor(label) # 示例数据 texts = ["hello world", "good morning", "bad day", "great evening"] labels = [1, 1, 0, 1] # 二分类任务 # 构建字符映射 chars = set("".join(texts)) char_to_idx = {c: i + 1 for i, c in enumerate(chars)} # 0作为填充符 char_to_idx['<PAD>'] = 0 # 创建数据集和数据加载器 train_dataset = CharDataset(texts, labels, char_to_idx) train_loader = DataLoader(train_dataset, batch_size=2, shuffle=True) ``` ### 模型定义 Charformer 模型由字符嵌入、位置编码和 Transformer 编码器组成。通过字符级别的嵌入，模型可以捕捉更细粒度的语义信息。 ```python import torch import torch.nn as nn class PositionalEncoding(nn.Module): def __init__(self, d_model, max_len=512): super(PositionalEncoding, self).__init__() pe = torch.zeros(max_len, d_model) position = torch.arange(0, max_len, dtype=torch.float).unsqueeze(1) div_term = torch.exp(torch.arange(0, d_model, 2).float() * (-torch.log(torch.tensor(10000.0)) / d_model)) pe[:, 0::2] = torch.sin(position * div_term) pe[:, 1::2] = torch.cos(position * div_term) pe = pe.unsqueeze(0) self.register_buffer('pe', pe) def forward(self, x): return x + self.pe[:, :x.size(1)] class Charformer(nn.Module): def __init__(self, vocab_size, d_model=256, nhead=8, num_layers=4, num_classes=2): super(Charformer, self).__init__() self.embedding = nn.Embedding(vocab_size, d_model) self.pos_encoder = PositionalEncoding(d_model) encoder_layer = nn.TransformerEncoderLayer(d_model, nhead) self.transformer_encoder = nn.TransformerEncoder(encoder_layer, num_layers) self.fc = nn.Linear(d_model, num_classes) def forward(self, x): x = self.embedding(x) x = self.pos_encoder(x) x = x.permute(1, 0, 2) x = self.transformer_encoder(x) x = x.mean(dim=0) return self.fc(x) ``` ### 模型训练 使用交叉熵损失函数和 Adam 优化器进行训练。训练过程中，模型会学习字符序列的表示，并通过 Transformer 编码器提取特征。 ```python import torch from torch.optim import Adam from torch.utils.data import DataLoader # 假设已准备好训练数据 train_loader = DataLoader(train_dataset, batch_size=2, shuffle=True) model = Charformer(vocab_size=len(char_to_idx), num_classes=2) criterion = nn.CrossEntropyLoss() optimizer = Adam(model.parameters(), lr=1e-3) device = torch.device("cuda" if torch.cuda.is_available() else "cpu") model.to(device) for epoch in range(5): model.train() total_loss = 0 for inputs, labels in train_loader: inputs, labels = inputs.to(device), labels.to(device) optimizer.zero_grad() outputs = model(inputs) loss = criterion(outputs, labels) loss.backward() optimizer.step() total_loss += loss.item() print(f"Epoch {epoch+1}, Loss: {total_loss/len(train_loader)}") ``` ### 模型评估 在模型训练完成后，通过准确率和召回率评估其性能。使用 `sklearn.metrics` 提供的 `accuracy_score` 和 `recall_score` 函数进行评估。 ```python from sklearn.metrics import accuracy_score, recall_score model.eval() all_preds = [] all_labels = [] with torch.no_grad(): for inputs, labels in train_loader: inputs = inputs.to(device) outputs = model(inputs) preds = torch.argmax(outputs, dim=1).cpu().numpy() all_preds.extend(preds) all_labels.extend(labels.numpy()) accuracy = accuracy_score(all_labels, all_preds) recall = recall_score(all_labels, all_preds, average='binary') print(f"Accuracy: {accuracy:.4f}, Recall: {recall:.4f}") ``` ###