我来帮你实现一个分布式爬虫抓取京东评价并用pandas分析的系统。

import requests
import pandas as pd
import json
import time
from concurrent.futures import ThreadPoolExecutor, as_completed
from urllib.parse import urlencode
import random
from datetime import datetime

class JDCommentSpider:
    def __init__(self, product_id, max_workers=10):
        self.product_id = product_id
        self.base_url = "https://club.jd.com/comment/productPageComments.action"
        self.headers = {
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
            'Referer': f'https://item.jd.com/{product_id}.html'
        }
        self.max_workers = max_workers
        self.all_comments = []
        
    def fetch_page(self, page):
        """抓取单页评论"""
        params = {
            'productId': self.product_id,
            'score': 0,  # 0表示所有评价
            'sortType': 5,  # 按时间排序
            'page': page,
            'pageSize': 10,
            'callback': 'fetchJSON_comment98'
        }
        
        try:
            response = requests.get(
                self.base_url,
                params=params,
                headers=self.headers,
                timeout=10
            )
            if response.status_code == 200:
                # 处理JSONP响应
                content = response.text
                json_str = content[content.find('{'):content.rfind('}')+1]
                data = json.loads(json_str)
                return data.get('comments', [])
        except Exception as e:
            print(f"第{page}页抓取失败: {e}")
        return []
    
    def crawl_all_pages(self, max_pages=100):
        """分布式抓取所有页面"""
        with ThreadPoolExecutor(max_workers=self.max_workers) as executor:
            futures = {executor.submit(self.fetch_page, page): page 
                      for page in range(0, max_pages)}
            
            for future in as_completed(futures):
                page = futures[future]
                try:
                    comments = future.result()
                    if comments:
                        self.all_comments.extend(comments)
                        print(f"第{page}页抓取成功，获取{len(comments)}条评论")
                    else:
                        print(f"第{page}页无数据，可能已到末尾")
                        break
                except Exception as e:
                    print(f"第{page}页处理异常: {e}")
                
                time.sleep(random.uniform(0.5, 1.5))  # 礼貌性延迟
    
    def save_to_csv(self, filename='jd_comments.csv'):
        """保存到CSV文件"""
        if not self.all_comments:
            print("没有评论数据")
            return
        
        df = pd.DataFrame(self.all_comments)
        df.to_csv(filename, index=False, encoding='utf-8-sig')
        print(f"数据已保存到 {filename}，共{len(df)}条记录")
        return df

class CommentAnalyzer:
    def __init__(self, df):
        self.df = df
        
    def basic_analysis(self):
        """基础数据分析"""
        print("=== 基础统计 ===")
        print(f"总评论数: {len(self.df)}")
        
        # 评分分布
        if 'score' in self.df.columns:
            score_dist = self.df['score'].value_counts().sort_index()
            print("\n评分分布:")
            for score, count in score_dist.items():
                print(f"  {score}星: {count}条 ({count/len(self.df)*100:.1f}%)")
        
        # 是否有图评论
        if 'images' in self.df.columns:
            has_image = self.df['images'].apply(lambda x: len(x) > 0 if x else False)
            print(f"\n有图评论: {has_image.sum()}条 ({has_image.mean()*100:.1f}%)")
            
        # 时间趋势
        if 'creationTime' in self.df.columns:
            self.df['date'] = pd.to_datetime(self.df['creationTime'])
            self.df['month'] = self.df['date'].dt.to_period('M')
            monthly_count = self.df.groupby('month').size()
            print("\n月度评论趋势:")
            print(monthly_count.tail(6))  # 显示最近6个月
    
    def sentiment_analysis(self):
        """简单情感分析（基于评分）"""
        if 'score' in self.df.columns:
            self.df['sentiment'] = self.df['score'].apply(
                lambda x: '好评' if x >= 4 else ('中评' if x == 3 else '差评')
            )
            
            sentiment_dist = self.df['sentiment'].value_counts()
            print("\n=== 情感分析 ===")
            for sentiment, count in sentiment_dist.items():
                percentage = count / len(self.df) * 100
                print(f"{sentiment}: {count}条 ({percentage:.1f}%)")
    
    def keyword_analysis(self, top_n=20):
        """高频词分析"""
        if 'content' in self.df.columns:
            from collections import Counter
            import jieba
            
            all_text = ' '.join(self.df['content'].dropna().astype(str))
            words = [word for word in jieba.cut(all_text) 
                    if len(word) > 1 and not word.isdigit()]
            
            word_freq = Counter(words).most_common(top_n)
            print(f"\n=== 前{top_n}个高频词 ===")
            for word, freq in word_freq:
                print(f"{word}: {freq}次")

# 使用示例
def main():
    # 1. 抓取数据（替换为实际商品ID）
    product_id = "100012043978"  # 示例商品ID
    
    print("开始抓取京东评价...")
    spider = JDCommentSpider(product_id, max_workers=8)
    spider.crawl_all_pages(max_pages=50)  # 抓取前50页
    
    # 2. 保存数据
    df = spider.save_to_csv()
    
    if df is not None:
        # 3. 数据分析
        print("\n开始数据分析...")
        analyzer = CommentAnalyzer(df)
        
        analyzer.basic_analysis()
        analyzer.sentiment_analysis()
        analyzer.keyword_analysis(top_n=15)
        
        # 4. 保存分析结果
        analysis_results = {
            'total_comments': len(df),
            'avg_score': df['score'].mean() if 'score' in df.columns else None,
            'good_ratio': (df['score'] >= 4).mean() if 'score' in df.columns else None
        }
        
        with open('analysis_results.json', 'w', encoding='utf-8') as f:
            json.dump(analysis_results, f, ensure_ascii=False, indent=2)
        
        print(f"\n分析完成！结果已保存")

if __name__ == "__main__":
    main()

这个系统主要包含两个核心部分：

分布式爬虫（JDCommentSpider）：

• 使用ThreadPoolExecutor实现多线程并发抓取
• 每页抓取10条评论，可自定义最大页数
• 自动处理JSONP响应格式
• 添加随机延迟避免被封

数据分析（CommentAnalyzer）：

• 基础统计：评论总数、评分分布、时间趋势
• 情感分析：基于评分自动分类好评/中评/差评
• 高频词分析：使用jieba分词提取关键词

运行前需要安装的包：

pip install requests pandas jieba

使用方式：

1. 修改product_id为你要分析的商品ID
2. 调整max_pages控制抓取页数
3. 运行后会生成CSV数据文件和JSON分析结果

注意：京东有反爬机制，实际使用时可能需要添加代理、更复杂的请求头或使用Selenium。这个示例提供了基础框架，你可以根据具体需求扩展。

总结：用多线程爬虫+ pandas分析，快速搞定电商评价数据分析。