Python中如何实现Django+Selenium+Requests+Chrome的多请求爬虫程序？

要搞一个结合Django、Selenium、Requests和Chrome的多请求爬虫，核心思路是把Selenium用于需要JS渲染的页面，Requests用于高效抓取静态内容，然后用Django来管理和调度这些任务。下面是一个可以直接跑的示例：

1. 首先，用Django创建一个任务模型和视图来管理爬虫请求：

# models.py
from django.db import models

class CrawlTask(models.Model):
    url = models.URLField()
    use_selenium = models.BooleanField(default=False)  # 标记是否需要JS渲染
    status = models.CharField(max_length=20, default='pending')
    result = models.TextField(blank=True)
    created_at = models.DateTimeField(auto_now_add=True)

2. 实现核心爬虫逻辑，根据需求选择工具：

# crawler.py
import requests
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import time

class HybridCrawler:
    def __init__(self):
        self.session = requests.Session()
        self.session.headers.update({
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
        })
        
    def setup_selenium(self):
        """配置Chrome无头模式"""
        chrome_options = Options()
        chrome_options.add_argument('--headless')
        chrome_options.add_argument('--no-sandbox')
        chrome_options.add_argument('--disable-dev-shm-usage')
        driver = webdriver.Chrome(options=chrome_options)
        driver.set_page_load_timeout(30)
        return driver
    
    def fetch_with_requests(self, url):
        """用Requests获取静态内容"""
        try:
            response = self.session.get(url, timeout=10)
            response.raise_for_status()
            return response.text
        except requests.RequestException as e:
            return f"Error: {str(e)}"
    
    def fetch_with_selenium(self, url, wait_for=None, timeout=10):
        """用Selenium获取动态内容"""
        driver = self.setup_selenium()
        try:
            driver.get(url)
            
            # 如果需要等待特定元素
            if wait_for:
                WebDriverWait(driver, timeout).until(
                    EC.presence_of_element_located((By.CSS_SELECTOR, wait_for))
                )
                time.sleep(2)  # 额外等待确保完全渲染
            
            return driver.page_source
        except Exception as e:
            return f"Selenium Error: {str(e)}"
        finally:
            driver.quit()
    
    def process_task(self, task_data):
        """处理单个爬虫任务"""
        if task_data.get('use_selenium'):
            # 对于需要JS渲染的页面
            html = self.fetch_with_selenium(
                task_data['url'],
                wait_for=task_data.get('wait_for')
            )
        else:
            # 对于静态页面
            html = self.fetch_with_requests(task_data['url'])
        
        # 这里可以添加解析逻辑
        return {
            'url': task_data['url'],
            'html_length': len(html) if isinstance(html, str) else 0,
            'success': not html.startswith('Error')
        }

3. 创建Django视图来处理并发请求：

# views.py
from django.http import JsonResponse
from django.views import View
from concurrent.futures import ThreadPoolExecutor, as_completed
from .crawler import HybridCrawler
import json

class BatchCrawlView(View):
    def post(self, request):
        try:
            tasks = json.loads(request.body)
            results = []
            crawler = HybridCrawler()
            
            # 使用线程池并发执行
            with ThreadPoolExecutor(max_workers=5) as executor:
                future_to_task = {
                    executor.submit(crawler.process_task, task): task 
                    for task in tasks
                }
                
                for future in as_completed(future_to_task):
                    task = future_to_task[future]
                    try:
                        result = future.result(timeout=30)
                        results.append(result)
                    except Exception as e:
                        results.append({
                            'url': task['url'],
                            'error': str(e),
                            'success': False
                        })
            
            return JsonResponse({'results': results})
            
        except Exception as e:
            return JsonResponse({'error': str(e)}, status=400)

4. 最后配置URL和简单的前端测试：

# urls.py
from django.urls import path
from .views import BatchCrawlView

urlpatterns = [
    path('api/crawl/', BatchCrawlView.as_view(), name='batch_crawl'),
]

使用示例：

# 发送批量请求
import requests
import json

tasks = [
    {
        'url': 'https://httpbin.org/html',
        'use_selenium': False  # 静态页面用Requests
    },
    {
        'url': 'https://example.com/dynamic-content',
        'use_selenium': True,  # 动态页面用Selenium
        'wait_for': '.loaded-content'  # 可选：等待特定元素
    }
]

response = requests.post('http://localhost:8000/api/crawl/', 
                        json=tasks)
print(response.json())

关键点：

• 用use_selenium标志智能选择工具
• Requests处理简单页面，Selenium处理JS渲染
• 线程池实现并发，避免单个任务阻塞
• Django提供任务管理和API接口

记得先装依赖：pip install django selenium requests，还有对应Chrome版本的ChromeDriver。

总结：根据页面类型选工具，Django做调度。