Python爬虫框架Scrapy中代理IP失效后如何自动更换IP

在Scrapy里处理代理IP失效自动切换，核心是用中间件配合重试机制。下面给个完整方案：

# middlewares.py
import random
from scrapy import signals
from scrapy.downloadermiddlewares.retry import RetryMiddleware
from scrapy.utils.response import response_status_message

class ProxyMiddleware:
    def __init__(self, proxy_list):
        self.proxy_list = proxy_list
        self.current_proxy = None
    
    @classmethod
    def from_crawler(cls, crawler):
        # 从配置文件读取代理列表
        proxy_list = crawler.settings.get('PROXY_LIST', [])
        return cls(proxy_list)
    
    def process_request(self, request, spider):
        if not request.meta.get('proxy') and self.proxy_list:
            proxy = random.choice(self.proxy_list)
            request.meta['proxy'] = proxy
            self.current_proxy = proxy

class RetryWithProxyMiddleware(RetryMiddleware):
    def __init__(self, settings):
        super().__init__(settings)
        self.proxy_list = settings.get('PROXY_LIST', [])
    
    def process_response(self, request, response, spider):
        # 检查是否需要重试
        if response.status in [407, 429, 500, 502, 503, 504]:
            reason = response_status_message(response.status)
            return self._retry(request, reason, spider) or response
        return response
    
    def _retry(self, request, reason, spider):
        # 移除当前失效的代理
        current_proxy = request.meta.get('proxy')
        if current_proxy in self.proxy_list:
            self.proxy_list.remove(current_proxy)
        
        # 选择新代理
        if self.proxy_list:
            new_proxy = random.choice(self.proxy_list)
            request.meta['proxy'] = new_proxy
            spider.logger.info(f'切换代理: {current_proxy} -> {new_proxy}')
        
        return super()._retry(request, reason, spider)

# settings.py
DOWNLOADER_MIDDLEWARES = {
    'your_project.middlewares.ProxyMiddleware': 543,
    'your_project.middlewares.RetryWithProxyMiddleware': 550,
    'scrapy.downloadermiddlewares.retry.RetryMiddleware': None,  # 禁用默认重试中间件
}

RETRY_TIMES = 3  # 重试次数
PROXY_LIST = [
    'http://proxy1:port',
    'http://proxy2:port',
    # ... 更多代理
]

# spiders/example.py
import scrapy

class ExampleSpider(scrapy.Spider):
    name = 'example'
    
    def start_requests(self):
        # 请求会自动使用代理中间件
        yield scrapy.Request('http://example.com', callback=self.parse)
    
    def parse(self, response):
        # 你的解析逻辑
        pass

这个方案的工作原理：

1. ProxyMiddleware负责给请求分配初始代理
2. RetryWithProxyMiddleware继承自Scrapy的重试中间件，当遇到代理错误或服务器错误时自动触发重试
3. 重试时会从代理池中移除失效代理并选择新代理
4. 通过RETRY_TIMES控制重试次数

关键点：

• 代理失效检测基于HTTP状态码（407代理认证、429限速、5xx服务器错误）
• 每次重试都会更换代理IP
• 代理池用完时会继续使用最后可用的代理

建议搭配代理池服务使用效果更好。