### 使用Python编写网络爬虫抓取世纪佳缘相亲网站内容 为了实现这一目标，可以采用`requests`库来处理HTTP请求，并利用`BeautifulSoup`解析HTML文档。对于图片下载，则可以通过`urllib.request.urlretrieve()`函数完成。 #### 安装依赖包 首先安装所需的第三方库： ```bash pip install requests beautifulsoup4 lxml urllib3 ``` #### 导入必要的模块并设置基本配置 ```python import os from bs4 import BeautifulSoup as BSoup import requests import re import time from urllib.parse import urljoin, urlparse from urllib.request import urlretrieve def get_page(url): headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64)', 'Referer': 'https://www.jiayuan.com/' } try: response = requests.get(url=url, headers=headers) if response.status_code == 200: return response.text else: print(f"Failed to fetch page {url}") return None except Exception as e: print(e) return None base_url = "https://www.jiayuan.com" start_urls = ["https://www.jiayuan.com/search"] # 假设这是起始页面URL列表 visited_links = set() image_folder = './images' os.makedirs(image_folder, exist_ok=True) if __name__ == '__main__': while start_urls: current_url = start_urls.pop(0).strip() html_content = get_page(current_url) soup = BSoup(html_content, features="lxml") parse_and_save_images(soup=soup, base_url=base_url, image_folder=image_folder) find_next_pages(soup=soup, visited_links=visited_links, start_urls=start_urls, base_url=base_url) ``` #### 解析网页获取图片链接并保存到本地文件夹 定义两个辅助方法用于提取图片链接以及查找分页导航条中的下一页链接： ```python def extract_image_urls_from_html(soup, base_url): img_tags = soup.find_all('img') urls = [] for tag in img_tags: src = tag.get('src', '') full_src = urljoin(base_url, src.strip()) parsed_uri = urlparse(full_src) domain = '{uri.scheme}://{uri.netloc}/'.format(uri=parsed_uri) if not any(domain.endswith(ext) for ext in ['.js', '.css']): urls.append(full_src) return list(set(urls)) def download_image(img_url, folder_path='.'): filename = os.path.basename(img_url.split('?')[0]) filepath = os.path.join(folder_path, filename) try: urlretrieve(img_url, filepath) print(f'Downloaded: {filename}') except IOError as io_err: print(io_err) def save_images_to_disk(images_list, output_dir='./output'): for img_link in images_list: download_image(img_link, output_dir) ``` #### 查找更多待访问的页面链接 遍历当前页面上的所有超链接标签 `<a>` 并判断其 href 属性是否指向其他有效子页面；如果是的话就将其加入队列等待后续处理。 ```python def collect_more_links(soup, visited_set=set(), queue=list()): links = [] a_tags = soup.select('a[href]') for link_tag in a_tags: href_value = link_tag['href'].strip().lower() absolute_href = urljoin(base_url, href_value) if ( absolute_href.startswith(base_url.lower()) and absolute_href not in visited_set and all(not absolute_href.endswith(ext) for ext in ('.jpg', '.png')) ): links.append(absolute_href) visited_set.add(absolute_href) queue.insert(-1, absolute_href) return links ``` 以上代码片段展示了如何构建一个简单的基于广度优先搜索算法(BFS) 的Web Scraper 来收集来自指定起点 URL 的图像资源[^2]。