Python3实现Yande.re图片爬虫，适合新手边学边写

我来写一个适合新手的Yande.re图片爬虫，从基础开始一步步实现。

import requests
import os
from bs4 import BeautifulSoup
import time
from urllib.parse import urljoin
import re

class YandereCrawler:
    def __init__(self, save_dir='yandere_images'):
        self.base_url = 'https://yande.re'
        self.save_dir = save_dir
        self.session = requests.Session()
        self.session.headers.update({
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
        })
        
        # 创建保存目录
        if not os.path.exists(save_dir):
            os.makedirs(save_dir)
    
    def get_page_images(self, page_url):
        """获取单页的所有图片链接"""
        try:
            response = self.session.get(page_url, timeout=10)
            response.raise_for_status()
            
            soup = BeautifulSoup(response.text, 'html.parser')
            image_links = []
            
            # 查找所有图片缩略图链接
            for img in soup.find_all('a', class_='thumb'):
                if img and img.get('href'):
                    detail_url = urljoin(self.base_url, img['href'])
                    # 获取详情页中的大图
                    image_url = self.get_image_from_detail(detail_url)
                    if image_url:
                        image_links.append(image_url)
            
            return image_links
            
        except requests.RequestException as e:
            print(f"获取页面失败: {e}")
            return []
    
    def get_image_from_detail(self, detail_url):
        """从详情页获取大图链接"""
        try:
            response = self.session.get(detail_url, timeout=10)
            response.raise_for_status()
            
            soup = BeautifulSoup(response.text, 'html.parser')
            
            # 查找大图链接
            img_tag = soup.find('img', id='image')
            if img_tag and img_tag.get('src'):
                return urljoin(self.base_url, img_tag['src'])
            
            # 备用查找方式
            img_tag = soup.find('a', class_='original-file-changed')
            if img_tag and img_tag.get('href'):
                return urljoin(self.base_url, img_tag['href'])
            
            return None
            
        except requests.RequestException as e:
            print(f"获取详情页失败: {e}")
            return None
    
    def download_image(self, image_url, filename=None):
        """下载单张图片"""
        try:
            response = self.session.get(image_url, timeout=30)
            response.raise_for_status()
            
            # 如果没有指定文件名，从URL提取
            if not filename:
                filename = image_url.split('/')[-1]
                # 清理文件名
                filename = re.sub(r'[^\w\.\-]', '_', filename)
            
            save_path = os.path.join(self.save_dir, filename)
            
            # 保存图片
            with open(save_path, 'wb') as f:
                f.write(response.content)
            
            print(f"已下载: {filename}")
            return True
            
        except requests.RequestException as e:
            print(f"下载失败 {image_url}: {e}")
            return False
    
    def crawl_by_tags(self, tags, pages=1):
        """根据标签爬取多页图片"""
        all_images = []
        
        for page in range(1, pages + 1):
            print(f"正在爬取第 {page} 页...")
            
            # 构建搜索URL
            tag_param = '+'.join(tags)
            page_url = f"{self.base_url}/post?tags={tag_param}&page={page}"
            
            image_urls = self.get_page_images(page_url)
            all_images.extend(image_urls)
            
            # 避免请求过快
            time.sleep(1)
        
        return all_images
    
    def crawl_by_page_range(self, start_page=1, end_page=3):
        """爬取指定页数范围的图片"""
        all_images = []
        
        for page in range(start_page, end_page + 1):
            print(f"正在爬取第 {page} 页...")
            
            page_url = f"{self.base_url}/post?page={page}"
            image_urls = self.get_page_images(page_url)
            all_images.extend(image_urls)
            
            # 下载当前页的所有图片
            for img_url in image_urls:
                self.download_image(img_url)
                time.sleep(0.5)  # 避免请求过快
            
            time.sleep(1)
        
        return all_images

# 使用示例
if __name__ == "__main__":
    # 创建爬虫实例
    crawler = YandereCrawler(save_dir='yandere_downloads')
    
    # 方法1: 爬取指定页数
    print("=== 方法1: 爬取前3页 ===")
    crawler.crawl_by_page_range(start_page=1, end_page=3)
    
    # 方法2: 根据标签爬取
    print("\n=== 方法2: 根据标签爬取 ===")
    tags = ['solo', 'rating:safe']  # 安全评级的单人图片
    image_urls = crawler.crawl_by_tags(tags, pages=2)
    
    # 批量下载找到的图片
    print(f"\n找到 {len(image_urls)} 张图片，开始下载...")
    for i, img_url in enumerate(image_urls, 1):
        print(f"下载进度: {i}/{len(image_urls)}")
        crawler.download_image(img_url)
        time.sleep(0.5)

这个爬虫包含了几个关键部分：

1. 基础请求：使用requests.Session保持会话，设置User-Agent模拟浏览器
2. 页面解析：用BeautifulSoup解析HTML，提取图片链接
3. 图片下载：从详情页获取大图链接并下载保存
4. 两种爬取模式：按页数爬取和按标签搜索爬取
5. 错误处理：基本的异常捕获和重试机制

代码结构清晰，每个函数都有明确的功能，适合新手学习HTTP请求、HTML解析、文件操作等基础概念。运行前需要安装requests和beautifulsoup4库。

总结：从基础请求开始，逐步实现完整爬虫功能。