如何用Python爬虫爬取猫眼等网站上的电影信息？

import requests
from bs4 import BeautifulSoup
import pandas as pd
import time
import random

# 伪装浏览器头
headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
    'Accept-Language': 'zh-CN,zh;q=0.9',
    'Connection': 'keep-alive'
}

def get_movie_info(url):
    """获取单页电影信息"""
    try:
        response = requests.get(url, headers=headers, timeout=10)
        response.raise_for_status()
        response.encoding = 'utf-8'
        
        soup = BeautifulSoup(response.text, 'html.parser')
        movies = []
        
        # 猫眼电影列表项选择器（需要根据实际网站结构调整）
        items = soup.select('.movie-item')
        
        for item in items:
            try:
                # 电影名称
                name = item.select_one('.movie-title').text.strip()
                
                # 评分
                score_elem = item.select_one('.score')
                score = score_elem.text.strip() if score_elem else '暂无评分'
                
                # 上映时间
                release_elem = item.select_one('.release-time')
                release = release_elem.text.strip() if release_elem else '未知'
                
                # 演员（示例选择器，实际需调整）
                actors_elem = item.select_one('.actors')
                actors = actors_elem.text.strip() if actors_elem else '未知'
                
                movies.append({
                    '电影名称': name,
                    '评分': score,
                    '上映时间': release,
                    '主演': actors
                })
                
            except AttributeError as e:
                print(f"解析单个电影时出错: {e}")
                continue
                
        return movies
        
    except requests.RequestException as e:
        print(f"请求失败: {e}")
        return []

def crawl_maoyan(pages=5):
    """爬取多页电影数据"""
    base_url = "https://maoyan.com/films?offset={}"
    all_movies = []
    
    for page in range(pages):
        offset = page * 30  # 猫眼每页30部电影
        url = base_url.format(offset)
        print(f"正在爬取第{page+1}页: {url}")
        
        movies = get_movie_info(url)
        all_movies.extend(movies)
        
        # 随机延迟避免被封
        time.sleep(random.uniform(1, 3))
    
    return all_movies

def save_to_csv(movies, filename='maoyan_movies.csv'):
    """保存数据到CSV文件"""
    if movies:
        df = pd.DataFrame(movies)
        df.to_csv(filename, index=False, encoding='utf-8-sig')
        print(f"数据已保存到 {filename}, 共{len(movies)}条记录")
    else:
        print("没有获取到数据")

if __name__ == "__main__":
    # 爬取5页电影数据
    movies_data = crawl_maoyan(pages=5)
    
    # 保存数据
    save_to_csv(movies_data)
    
    # 显示前几条数据
    if movies_data:
        print("\n前5条电影信息:")
        for i, movie in enumerate(movies_data[:5], 1):
            print(f"{i}. {movie['电影名称']} - 评分: {movie['评分']}")

# 重要说明：
# 1. 实际使用时需要检查猫眼网站的实际HTML结构，调整CSS选择器
# 2. 可能需要处理动态加载的内容（考虑使用Selenium）
# 3. 遵守网站的robots.txt和爬虫协议
# 4. 添加适当的异常处理和日志记录

核心要点：用requests获取页面，BeautifulSoup解析HTML，注意伪装请求头和设置延迟。