### 使用Scrapy框架爬取安居客房价信息 为了高效地从安居客网站获取房价数据，可以充分利用Scrapy这一强大的网络抓取框架。以下是具体实现方案及相关注意事项。 #### 环境准备 确保已安装Python 3.8及以上版本，并按照官方推荐的方式安装Scrapy框架[^1]。如果计划构建分布式爬虫系统，则还需额外配置`scrapy-redis`等相关组件及其依赖库[^2]。 ```bash pip install Scrapy ``` 对于更复杂的项目需求，例如需要支持分布式的队列管理或者去重过滤等功能时，可进一步扩展环境设置： ```bash pip install scrapy-redis==0.6.8 redis==3.0.1 pymysql sqlalchemy twisted ``` #### 创建Scrapy工程 初始化一个新的Scrapy项目结构以便于后续模块划分与维护工作开展顺利进行下去。 ```bash scrapy startproject anjuke_spider cd anjuke_spider/ ``` 接着定义目标模型类表示待采集的数据实体属性集合[^5]。 ```python # anjuke_spider/items.py import scrapy class AnjukeItem(scrapy.Item): title = scrapy.Field() # 房产标题 house_mode = scrapy.Field() # 户型描述 total_area = scrapy.Field() # 总面积 build_year = scrapy.Field() # 建筑年份 loc_detail = scrapy.Field() # 地址详情 total_price = scrapy.Field() # 成交总价 unit_price = scrapy.Field() # 单价水平 city_name = scrapy.Field() # 所属城市 region = scrapy.Field() # 行政区域名 ``` 随后编写具体的Spider子类负责实际业务逻辑处理过程[^4]。 ```python # anjuke_spider/spiders/anjuke_house.py import scrapy from ..items import AnjukeItem class AnjukeHouseSpider(scrapy.Spider): name = 'anjuke' allowed_domains = ['anjuke.com'] start_urls = [ 'https://www.anjuke.com/market/' ] custom_settings = { 'ITEM_PIPELINES': {'anjuke_spider.pipelines.AnjukePipeline': 300}, 'DOWNLOADER_MIDDLEWARES': { 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware': None, 'anjuke_spider.middlewares.RandomUserAgentMiddleware': 400} } def parse(self, response): houses = response.css('.list-item') for house in houses: item = AnjukeItem() item['title'] = house.css('a::attr(title)').get().strip(), item['house_mode'] = house.css('.details .room em::text').extract_first(default='N/A'), item['total_area'] = house.css('.details .size span:nth-child(2)::text').re(r'\d+\.?\d*')[0], item['build_year'] = house.css('.details .year::text').re(r'\d{4}')[0], item['loc_detail'] = house.css('.address a::text').getall()[0].strip(), item['total_price'] = float(house.css('.price-det strong::text').get()), item['unit_price'] = int(house.css('.unit-price::text').re(r'\d+')[0]), item['city_name'] = self.settings.get('CITY_NAME', default='Shanghai'), # 默认上海 item['region'] = house.css('.location a:last-of-type::text').get(), yield item next_page = response.css('a.next-page::attr(href)').get() if next_page is not None: yield response.follow(next_page, callback=self.parse) ``` 最后设计管道(Pipeline)完成持久化存储任务。 ```python # anjuke_spider/pipelines.py import pymysql.cursors class MySQLPipeline(object): def open_spider(self, spider): db_config = getattr(spider, 'db_config', {}) self.connection = pymysql.connect(**db_config) self.cursor = self.connection.cursor() def close_spider(self, spider): self.cursor.close() self.connection.close() def process_item(self, item, spider): sql_template = """ INSERT INTO anjuke ( title, house_mode, total_area, build_year, loc_detail, total_price, unit_price, city_name, region ) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s); """ try: values_tuple = tuple([item[key][0] if isinstance(item[key], list) else item[key] for key in ('title','house_mode','total_area', 'build_year','loc_detail','total_price', 'unit_price','city_name','region')]) self.cursor.execute(sql_template, values_tuple) self.connection.commit() except Exception as e: print(e) self.connection.rollback() return item ``` --- ###