Python爬虫实战：如何应对某众点评网的花式反爬策略？

import requests
import time
import random
from fake_useragent import UserAgent
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC

class DianpingSpider:
    def __init__(self):
        self.ua = UserAgent()
        self.session = requests.Session()
        self.setup_headers()
        
    def setup_headers(self):
        """设置基础请求头"""
        self.session.headers.update({
            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
            'Accept-Language': 'zh-CN,zh;q=0.9',
            'Accept-Encoding': 'gzip, deflate, br',
            'Connection': 'keep-alive',
            'Upgrade-Insecure-Requests': '1',
            'Sec-Fetch-Dest': 'document',
            'Sec-Fetch-Mode': 'navigate',
            'Sec-Fetch-Site': 'none',
            'Cache-Control': 'max-age=0'
        })
    
    def get_dynamic_headers(self):
        """生成动态请求头"""
        return {
            'User-Agent': self.ua.random,
            'Referer': 'https://www.dianping.com/',
            'X-Requested-With': 'XMLHttpRequest' if random.random() > 0.5 else None
        }
    
    def rotate_proxy(self):
        """代理IP轮换（示例结构）"""
        # 实际使用时需要配置代理池
        proxies = {
            'http': 'http://your_proxy:port',
            'https': 'https://your_proxy:port'
        }
        return proxies if random.random() > 0.7 else None
    
    def human_like_delay(self):
        """模拟人类操作间隔"""
        time.sleep(random.uniform(1.5, 3.5))
    
    def bypass_webdriver_detection(self):
        """使用Selenium绕过WebDriver检测"""
        options = webdriver.ChromeOptions()
        options.add_argument('--disable-blink-features=AutomationControlled')
        options.add_experimental_option("excludeSwitches", ["enable-automation"])
        options.add_experimental_option('useAutomationExtension', False)
        
        driver = webdriver.Chrome(options=options)
        driver.execute_cdp_cmd('Page.addScriptToEvaluateOnNewDocument', {
            'source': '''
                Object.defineProperty(navigator, 'webdriver', {get: () => undefined})
                window.chrome = {runtime: {}}
            '''
        })
        return driver
    
    def handle_ajax_data(self, shop_id):
        """处理AJAX加载的数据"""
        url = f'https://www.dianping.com/ajax/json/shopDynamic/allReview'
        params = {
            'shopId': shop_id,
            'cityId': 1,
            'page': 1,
            '_': int(time.time() * 1000)
        }
        
        headers = self.get_dynamic_headers()
        response = self.session.get(url, params=params, headers=headers)
        
        if response.status_code == 200:
            return response.json()
        return None
    
    def parse_with_selenium(self, url):
        """使用Selenium解析页面"""
        driver = self.bypass_webdriver_detection()
        try:
            driver.get(url)
            WebDriverWait(driver, 10).until(
                EC.presence_of_element_located((By.CLASS_NAME, "review-list"))
            )
            
            # 模拟滚动加载
            for _ in range(3):
                driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
                time.sleep(random.uniform(2, 4))
            
            # 提取数据
            reviews = driver.find_elements(By.CLASS_NAME, "review-words")
            return [review.text for review in reviews[:5]]
        finally:
            driver.quit()
    
    def main_crawl(self, target_url):
        """主爬取函数"""
        print("开始爬取...")
        
        # 方法1: 直接请求（适合简单页面）
        headers = self.get_dynamic_headers()
        response = self.session.get(target_url, headers=headers)
        
        if response.status_code == 403:
            print("触发反爬，切换到Selenium方案")
            # 方法2: 使用Selenium
            data = self.parse_with_selenium(target_url)
            return data
        elif response.status_code == 200:
            # 解析响应内容
            # 这里添加具体的解析逻辑
            return "直接请求成功"
        
        self.human_like_delay()
        return None

# 使用示例
if __name__ == "__main__":
    spider = DianpingSpider()
    result = spider.main_crawl("https://www.dianping.com/shop/12345678")
    print(f"爬取结果: {result}")

核心思路就是多手段组合：随机请求头+动态间隔+Selenium反检测+代理备用。

只要能被用户看到，理论上就可以被爬，但是大规模的爬已经有很多方法控制，现在就是小规模爬取上面打游击战

BASE64 编码