### 编写Python爬虫抓取当当网图书信息并存储至MySQL 为了实现这一目标，需要构建一个完整的流程来抓取当当网上的书籍详情，并将其保存到MySQL数据库中。此过程涉及多个方面的工作，包括但不限于发送HTTP请求、解析HTML文档、处理异常情况以及与数据库交互。 #### 准备工作 首先确保环境中已安装必要的库： ```bash pip install requests lxml pymysql dbutils ``` 接着定义用于连接MySQL数据库的配置参数[^4]： ```python import pymysql from DBUtils.PooledDB import PooledDB pool = PooledDB( creator=pymysql, maxconnections=6, host='localhost', user='root', password='password', database='books_db', charset='utf8' ) def get_conn(): return pool.connection() ``` 创建名为`dangdang_books`的数据表用来储存所获取的信息: ```sql CREATE TABLE IF NOT EXISTS dangdang_books ( id INT AUTO_INCREMENT PRIMARY KEY, title VARCHAR(255), author VARCHAR(255), price DECIMAL(10, 2), pub_date DATE, publisher VARCHAR(255), description TEXT ); ``` #### 抓取页面内容 利用`requests`库发起GET请求获得指定URL对应的响应体；再借助`lxml.html`模块解析返回的结果，从中抽取所需字段[^3]。 ```python import requests from lxml import html def fetch_page(url): headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64)' } response = requests.get(url=url, headers=headers) tree = html.fromstring(response.text) items = [] for item in tree.xpath('//ul[@class="bigimg"]/li'): try: book_info = {} # 提取书名 book_info['title'] = item.xpath('.//a/@title')[0].strip() # 获取作者名字 book_info['author'] = item.xpath('.//p[@class="search_book_author"]/span[1]/a/text()') if not book_info['author']: continue book_info['author'] = ''.join(book_info['author']).strip() # 取得价格 book_info['price'] = float(item.xpath('.//p[@class="price"]/span[1]/text()')[0][1:].replace(',', '')) # 发布时间 date_str = item.xpath('.//p[@class="search_book_author"]/span[2]/text()') if date_str and '-' in date_str[-1]: year, month, day = map(int, date_str[-1].split('-')) from datetime import date book_info['pub_date'] = date(year, month, day).isoformat() # 出版社名称 book_info['publisher'] = item.xpath('.//p[@class="search_book_author"]/span[3]/a/text()') if not book_info['publisher']: continue book_info['publisher'] = ''.join(book_info['publisher']).strip() # 描述文字 desc = item.xpath('.//p[@class="detail"]/text()') if desc: book_info['description'] = '\n'.join(desc).strip() else: book_info['description'] = '' items.append(book_info) except Exception as e: print(f'Error processing {url}: ', str(e)) pass return items ``` #### 数据入库操作 设计函数负责把单条或多条记录批量插入到之前建立好的表格里去。 ```python def save_to_mysql(data_list): conn = get_conn() cursor = conn.cursor() insert_sql = """ INSERT INTO dangdang_books(title, author, price, pub_date, publisher, description) VALUES (%s,%s,%s,%s,%s,%s) """ params = [(item["title"], item["author"], item["price"], item["pub_date"], item["publisher"], item["description"]) for item in data_list] try: cursor.executemany(insert_sql, params) conn.commit() finally: cursor.close() conn.close() ``` #### 主逻辑控制 最后编写主程序入口点，循环遍历各个分类下的商品列表页链接地址，调用上述方法完成整个任务链路闭环。 ```python if __name__ == '__main__': base_url = "http://bang.dangdang.com/books/fivestars/{}" pages = range(1, 26) # 假设有25页 all_items = [] for page_num in pages: url = base_url.format(page_num) books_on_this_page = fetch_page(url) all_items.extend(books_on_this_page) save_to_mysql(all_items) ```