在JupyterLab中运行代码时，通常会将代码分块（即分成多个单元格），以便逐步执行和调试。以下是将你提供的爬虫代码分块的建议，每个块都可以作为一个独立的Jupyter Notebook单元格。 ### 分块实现 #### 块 1: 导入必要的库 ```python from selenium import webdriver from selenium.webdriver.common.by import By from selenium.webdriver.support.ui import WebDriverWait from selenium.webdriver.support import expected_conditions as EC import time import random import csv ``` #### 块 2: 初始化浏览器函数 ```python def init_browser(): try: options = webdriver.ChromeOptions() options.add_argument("--start-maximized") driver = webdriver.Chrome(options=options) return driver except Exception as e: print(f"浏览器初始化失败: {e}") return None ``` #### 块 3: 等待登录函数 ```python def wait_for_login(driver): input("请手动完成登录后按回车键继续...") ``` #### 块 4: 爬取单页数据函数 ```python def crawl_page(driver, current_page): page_data = [] items = driver.find_elements(By.CSS_SELECTOR, ".gl-item") for item in items: title = item.find_element(By.CSS_SELECTOR, "div.p-name").text.strip() price = item.find_element(By.CSS_SELECTOR, "div.p-price i").text.strip() shop = item.find_element(By.CSS_SELECTOR, "div.p-shopnum a").get_attribute("title").strip() link = item.find_element(By.CSS_SELECTOR, "div.p-name a").get_attribute("href").strip() comments = item.find_element(By.CSS_SELECTOR, "div.p-commit strong").text.strip() # 促销标签可能不存在 promotion = "" try: promotion = item.find_element(By.CSS_SELECTOR, "div.p-icons").text.strip() except: pass page_data.append({ "标题": title, "价格": price, "评论数": comments, "店铺": shop, "促销标签": promotion, "链接": link }) print(f"第 {current_page} 页爬取成功，共 {len(page_data)} 条商品数据") return page_data ``` #### 块 5: 主程序逻辑 ```python if __name__ == "__main__": data_list = [] driver = init_browser() if not driver: print("浏览器初始化失败，无法继续爬取") exit(1) try: # 初始化页面 URL = "https://search.jd.com/Search?keyword=%E5%B9%B3%E6%9D%BF%E7%94%B5%E8%84%91&enc=utf-8&wq=%E5%B9%B3%E6%9D%BF%E7%94%B5%E8%84%91&pvid=1bd09c25786446a4b9a4262e1c795902" driver.get(URL) time.sleep(random.uniform(2, 4)) # 等待手动登录 wait_for_login(driver) current_page = 1 MAX_PAGES = 10 # 设置最大页数 while current_page <= MAX_PAGES: try: # 尝试加载当前页 page_url = f"{URL}&page={current_page * 2 - 1}" # 构造奇数页URL print(f"\n开始爬取第 {current_page} 页: {page_url}") driver.get(page_url) # 加载新页面 time.sleep(random.uniform(2, 4)) # 等待页面加载 # 动态检测当前页是否正确 current_page_element = driver.find_element(By.CSS_SELECTOR, ".p-skip .curr") actual_page = int(current_page_element.text.strip()) if actual_page != current_page: print(f"当前页不匹配，实际为第 {actual_page} 页，重新调整...") continue # 跳过此页 # 等待商品列表加载完成 WebDriverWait(driver, 10).until( EC.presence_of_all_elements_located((By.CSS_SELECTOR, ".gl-item")) ) # 爬取当前页数据 page_data = crawl_page(driver, current_page) data_list.extend(page_data) # 下一页 current_page += 1 except Exception as e: print(f"第 {current_page} 页爬取失败: {e}") break # 出现错误后停止爬取 # 保存数据到 CSV if data_list: fieldnames = ["标题", "价格", "评论数", "店铺", "促销标签", "链接"] OUTPUT_FILE = "jd_products.csv" with open(OUTPUT_FILE, 'w', newline='', encoding='utf-8-sig') as f: writer = csv.DictWriter(f, fieldnames=fieldnames) writer.writeheader() writer.writerows(data_list) print(f"\n数据已保存至 {OUTPUT_FILE}，共 {len(data_list)} 个商品") else: print("未获取到数据") except Exception as e: print(f"爬取过程出错: {e}") finally: # 不退出浏览器，保持界面打开 input("\n爬取已完成或出错，请按 Enter 键退出程序...") if driver: driver.quit() ``` --- ### 解释 1. **块 1**: 导入所有需要的库，确保代码运行环境完整。 2. **块 2**: 定义`init_browser`函数，用于初始化浏览器实例。 3. **块 3**: 定义`wait_for_login`函数，等待用户手动完成登录操作。 4. **块 4**: 定义`crawl_page`函数，负责从单页中提取商品信息。 5. **块 5**: 主程序逻辑，包括页面加载、异常处理、数据保存等功能。 通过这种方式分块，可以在JupyterLab中逐步测试和调试每一部分代码。 --- ###