### 完整的Python爬虫代码用于爬取链家网站数据 以下是一份可以直接运行的完整Python爬虫代码，旨在从链家网站抓取二手房信息。此代码综合了多个引用中的技术和建议[^1]，并进行了优化以适应实际需求。 #### 1. 环境准备 首先确保安装所需库： ```bash pip install requests beautifulsoup4 pandas pymongo mysql-connector-python fake-useragent ``` #### 2. Python爬虫代码 以下是完整的Python爬虫代码： ```python import requests from bs4 import BeautifulSoup import pandas as pd import time from fake_useragent import UserAgent from pymongo import MongoClient import mysql.connector # 配置请求头 ua = UserAgent() headers = {'User-Agent': ua.random} def fetch_page(url): """ 发送HTTP请求并返回页面内容 """ try: response = requests.get(url, headers=headers) if response.status_code == 200: return response.text else: print(f"Error fetching {url}. Status code: {response.status_code}") return None except Exception as e: print(f"Exception occurred while fetching page: {e}") return None def parse_html(html): """ 解析HTML页面，提取房屋信息 """ soup = BeautifulSoup(html, 'html.parser') listings = [] houses = soup.find_all('li', class_='clear LOGCLICKDATA') for house in houses: title = house.find('div', class_='title').get_text(strip=True) total_price = house.find('div', class_='totalPrice').get_text(strip=True) unit_price = house.find('div', class_='unitPrice').get_text(strip=True) if house.find('div', class_='unitPrice') else '' listing = { 'title': title, 'total_price': total_price, 'unit_price': unit_price } listings.append(listing) return listings def save_to_mongodb(data): """ 将数据保存到MongoDB """ client = MongoClient('mongodb://localhost:27017/') db = client['real_estate'] collection = db['listings'] result = collection.insert_many(data) print(f"{len(result.inserted_ids)} records inserted into MongoDB.") def save_to_mysql(data): """ 将数据保存到MySQL """ conn = mysql.connector.connect( host='localhost', user='root', password='your_password', database='real_estate' ) cursor = conn.cursor() create_table_query = ''' CREATE TABLE IF NOT EXISTS listings ( id INT AUTO_INCREMENT PRIMARY KEY, title VARCHAR(255), total_price VARCHAR(50), unit_price VARCHAR(50) ); ''' cursor.execute(create_table_query) insert_query = "INSERT INTO listings (title, total_price, unit_price) VALUES (%s, %s, %s)" values = [(item['title'], item['total_price'], item['unit_price']) for item in data] cursor.executemany(insert_query, values) conn.commit() print(f"{cursor.rowcount} records inserted into MySQL.") cursor.close() conn.close() def main(): base_url = 'https://sh.lianjia.com/ershoufang/pg{}' all_data = [] for i in range(1, 101): # 假设要爬取前100页 url = base_url.format(i) html = fetch_page(url) if not html: continue listings = parse_html(html) all_data.extend(listings) print(f"Page {i} processed successfully.") time.sleep(random.uniform(1, 3)) # 设置随机延时防反爬 if all_data: save_to_mongodb(all_data) save_to_mysql(all_data) if __name__ == '__main__': main() ``` --- ### 关键点解释 1. **请求头伪装**：使用`fake-useragent`库动态生成浏览器头部信息，降低被封禁的风险[^2]。 2. **分页处理**：通过循环拼接URL地址完成多页数据的抓取，并加入随机延迟时间防止触发反爬机制[^5]。 3. **数据解析**：借助`BeautifulSoup`解析HTML文档，精准定位目标字段并提取相关内容[^4]。 4. **数据存储**：提供了两种主流数据库（MongoDB 和 MySQL）作为数据持久化的选项[^3]。 ---