以下是一个基于 Python 的代码示例，用于爬取 10000 条新浪新闻数据，进行去重、清洗、填补空缺值处理，存储到 D 盘并生成 5 个可视化图形。 ```python import scrapy from scrapy.crawler import CrawlerProcess import pandas as pd import matplotlib.pyplot as plt import seaborn as sns from scrapy.exceptions import CloseSpider # 定义新闻 Item class SinaNewsItem(scrapy.Item): title = scrapy.Field() content = scrapy.Field() publish_time = scrapy.Field() url = scrapy.Field() # 定义新浪新闻 Spider class SinaNewsSpider(scrapy.Spider): name = "sina_news" allowed_domains = ["news.sina.com.cn"] start_urls = ["https://news.sina.com.cn/"] count = 0 def parse(self, response): news_links = response.css('a::attr(href)').re(r'https://news.sina.com.cn/.*?\.shtml') for link in news_links: if self.count < 10000: yield response.follow(link, self.parse_news) next_pages = response.css('a.next-page::attr(href)').getall() for next_page in next_pages: if next_page: yield response.follow(next_page, self.parse) def parse_news(self, response): item = SinaNewsItem() item['title'] = response.css('h1.main-title::text').get() item['content'] = ''.join(response.css('div.article p::text').getall()) item['publish_time'] = response.css('span.date::text').get() item['url'] = response.url if item['title'] and item['content']: self.count += 1 yield item if self.count >= 10000: raise CloseSpider('Reached 10000 news items') # 爬取新闻数据 process = CrawlerProcess() process.crawl(SinaNewsSpider) process.start() # 读取爬取的数据 df = pd.read_json('sina_news.json') # 数据清洗 # 去重 df = df.drop_duplicates() # 填补空缺值 for col in df.columns: if df[col].dtype == 'object': df[col] = df[col].fillna('Unknown') # 存储数据到 D 盘 df.to_csv('D:/sina_news_cleaned.csv', index=False) # 数据可视化 # 1. 新闻标题长度分布 df['title_length'] = df['title'].str.len() plt.figure(figsize=(10, 6)) sns.histplot(df['title_length'], bins=20, kde=True) plt.title('Distribution of News Title Length') plt.xlabel('Title Length') plt.ylabel('Count') plt.savefig('D:/title_length_distribution.png') plt.show() # 2. 新闻发布时间分布（按年份） df['publish_year'] = pd.to_datetime(df['publish_time']).dt.year year_counts = df['publish_year'].value_counts() plt.figure(figsize=(10, 6)) sns.barplot(x=year_counts.index, y=year_counts.values) plt.title('News Distribution by Year') plt.xlabel('Year') plt.ylabel('Count') plt.savefig('D:/news_distribution_by_year.png') plt.show() # 3. 新闻内容长度分布 df['content_length'] = df['content'].str.len() plt.figure(figsize=(10, 6)) sns.histplot(df['content_length'], bins=20, kde=True) plt.title('Distribution of News Content Length') plt.xlabel('Content Length') plt.ylabel('Count') plt.savefig('D:/content_length_distribution.png') plt.show() # 4. 新闻标题长度与内容长度的关系 plt.figure(figsize=(10, 6)) sns.scatterplot(x=df['title_length'], y=df['content_length']) plt.title('Relationship between Title Length and Content Length') plt.xlabel('Title Length') plt.ylabel('Content Length') plt.savefig('D:/title_content_length_relationship.png') plt.show() # 5. 新闻发布时间分布（按月份） df['publish_month'] = pd.to_datetime(df['publish_time']).dt.month month_counts = df['publish_month'].value_counts() plt.figure(figsize=(10, 6)) sns.barplot(x=month_counts.index, y=month_counts.values) plt.title('News Distribution by Month') plt.xlabel('Month') plt.ylabel('Count') plt.savefig('D:/news_distribution_by_month.png') plt.show() ```