### 构建基于深度学习的文本分类和情感分析系统的替代方案 以下是一个使用PyTorch构建LSTM模型进行文本分类和情感分析的示例。此外，还提供了BERT模型的实现方式，以展示如何在不使用TensorFlow的情况下完成任务。 --- #### 1. 数据预处理 使用Jieba和NLTK进行分词、去停用词等操作[^3]。 ```python import re import jieba from nltk.corpus import stopwords def preprocess_data(texts): stop_words = set(stopwords.words('english')) # 英文停用词 processed_texts = [] for text in texts: # 去除HTML标签、符号和空白字符 text = re.sub(r'<.*?>', '', text) # 移除HTML标签 text = re.sub(r'[^\w\s]', '', text) # 移除非字母数字字符 text = text.strip().lower() # 转为小写并去除首尾空格 # 分词（中文使用jieba，英文使用nltk） if any("\u4e00" <= char <= "\u9fff" for char in text): # 判断是否包含中文 words = jieba.lcut(text) else: words = text.split() # 去除停用词 words = [word for word in words if word not in stop_words] processed_texts.append(" ".join(words)) return processed_texts ``` --- #### 2. 特征提取 使用Word2Vec或BERT进行特征提取[^3]。 ```python from gensim.models import Word2Vec def build_word2vec_model(texts): tokenized_texts = [text.split() for text in texts] model = Word2Vec(sentences=tokenized_texts, vector_size=100, window=5, min_count=1, workers=4) return model ``` --- #### 3. LSTM模型训练（使用PyTorch） 使用PyTorch构建LSTM模型进行文本分类[^1]。 ```python import torch import torch.nn as nn import torch.optim as optim from torch.utils.data import DataLoader, Dataset class TextDataset(Dataset): def __init__(self, texts, labels, tokenizer, max_length=100): self.texts = texts self.labels = labels self.tokenizer = tokenizer self.max_length = max_length def __len__(self): return len(self.texts) def __getitem__(self, idx): text = self.texts[idx] label = self.labels[idx] tokens = self.tokenizer.encode(text)[:self.max_length] return torch.tensor(tokens), torch.tensor(label) class LSTMModel(nn.Module): def __init__(self, vocab_size, embedding_dim, hidden_dim, output_dim): super(LSTMModel, self).__init__() self.embedding = nn.Embedding(vocab_size, embedding_dim) self.lstm = nn.LSTM(embedding_dim, hidden_dim, batch_first=True) self.fc = nn.Linear(hidden_dim, output_dim) def forward(self, x): embedded = self.embedding(x) lstm_out, _ = self.lstm(embedded) out = self.fc(lstm_out[:, -1, :]) return out # 示例数据 texts = ["I love this product", "This is terrible"] labels = [1, 0] # Tokenizer tokenizer = build_word2vec_model(texts) # 数据集与加载器 dataset = TextDataset(texts, labels, tokenizer) dataloader = DataLoader(dataset, batch_size=2, shuffle=True) # 模型、损失函数与优化器 vocab_size = len(tokenizer.wv.key_to_index) + 1 model = LSTMModel(vocab_size=vocab_size, embedding_dim=100, hidden_dim=128, output_dim=1) criterion = nn.BCEWithLogitsLoss() optimizer = optim.Adam(model.parameters(), lr=0.001) # 训练 for epoch in range(5): for batch in dataloader: inputs, targets = batch optimizer.zero_grad() outputs = model(inputs) loss = criterion(outputs.squeeze(), targets.float()) loss.backward() optimizer.step() ``` --- #### 4. BERT模型训练（使用Hugging Face Transformers库） 使用Hugging Face的Transformers库加载预训练的BERT模型[^4]。 ```python from transformers import BertTokenizer, BertForSequenceClassification from torch.optim import AdamW # 加载预训练的BERT模型和分词器 tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') model = BertForSequenceClassification.from_pretrained('bert-base-uncased') # 示例数据 texts = ["I love this product", "This is terrible"] labels = [1, 0] # 编码输入 encoded_inputs = tokenizer(texts, padding=True, truncation=True, return_tensors='pt') # 损失函数与优化器 optimizer = AdamW(model.parameters(), lr=5e-5) # 训练 for epoch in range(3): optimizer.zero_grad() outputs = model(**encoded_inputs, labels=torch.tensor(labels)) loss = outputs.loss loss.backward() optimizer.step() ``` --- #### 5. 性能评估 计算准确率、F1-score和混淆矩阵[^5]。 ```python from sklearn.metrics import accuracy_score, f1_score, confusion_matrix def evaluate_model(model, dataloader): all_preds = [] all_labels = [] model.eval() with torch.no_grad(): for inputs, labels in dataloader: outputs = model(inputs) preds = (torch.sigmoid(outputs).squeeze() > 0.5).int() all_preds.extend(preds.numpy()) all_labels.extend(labels.numpy()) accuracy = accuracy_score(all_labels, all_preds) f1 = f1_score(all_labels, all_preds) cm = confusion_matrix(all_labels, all_preds) return accuracy, f1, cm accuracy, f1, cm = evaluate_model(model, dataloader) print(f"Accuracy: {accuracy}, F1-Score: {f1}") print("Confusion Matrix:\n", cm) ``` --- #### 6. 实时流数据处理 使用Apache Flink处理实时流数据。 ```python from pyflink.datastream import StreamExecutionEnvironment from pyflink.table import StreamTableEnvironment env = StreamExecutionEnvironment.get_execution_environment() t_env = StreamTableEnvironment.create(env) # 定义数据源和sink t_env.execute_sql(""" CREATE TABLE comments ( id BIGINT, content STRING ) WITH ( 'connector' = 'kafka', 'topic' = 'comments', 'properties.bootstrap.servers' = 'localhost:9092', 'format' = 'json' ) """) # 处理逻辑 t_env.execute_sql(""" INSERT INTO classified_comments SELECT id, classify(content) AS sentiment FROM comments """) ``` ---