Python 爬虫实践：如何构建简书用户画像

玩 python 多久了，看着还挺有意思的嘛

我理解你想用Python爬取简书数据来分析用户画像。这个需求需要先获取用户数据，然后进行分析。下面是一个完整的爬虫示例，包含数据采集和基础分析：

import requests
import json
import pandas as pd
from datetime import datetime
import time
import random

class JianShuSpider:
    def __init__(self):
        self.headers = {
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
            'Referer': 'https://www.jianshu.com/'
        }
        self.session = requests.Session()
    
    def get_user_info(self, user_id):
        """获取用户基本信息"""
        url = f'https://www.jianshu.com/u/{user_id}'
        try:
            response = self.session.get(url, headers=self.headers, timeout=10)
            # 这里需要解析HTML页面，实际项目中建议使用BeautifulSoup
            # 简书页面结构可能变化，以下为示例解析逻辑
            user_data = {
                'user_id': user_id,
                'nickname': self._extract_nickname(response.text),
                'followers': self._extract_followers(response.text),
                'following': self._extract_following(response.text),
                'articles': self._extract_article_count(response.text),
                'words': self._extract_total_words(response.text),
                'likes': self._extract_likes(response.text)
            }
            return user_data
        except Exception as e:
            print(f"获取用户{user_id}信息失败: {e}")
            return None
    
    def get_user_articles(self, user_id, limit=20):
        """获取用户文章列表"""
        articles = []
        page = 1
        
        while len(articles) < limit:
            url = f'https://www.jianshu.com/u/{user_id}?order_by=shared_at&page={page}'
            try:
                response = self.session.get(url, headers=self.headers)
                # 解析文章列表
                page_articles = self._parse_articles(response.text)
                if not page_articles:
                    break
                articles.extend(page_articles)
                page += 1
                time.sleep(random.uniform(1, 2))  # 礼貌性延迟
            except Exception as e:
                print(f"获取文章列表失败: {e}")
                break
        
        return articles[:limit]
    
    def _extract_nickname(self, html):
        """从HTML中提取昵称（示例方法）"""
        # 实际需要根据页面结构编写解析逻辑
        import re
        match = re.search(r'<title>(.*?) - 简书</title>', html)
        return match.group(1) if match else ''
    
    def _extract_followers(self, html):
        """提取粉丝数"""
        import re
        match = re.search(r'粉丝.*?(\d+)', html)
        return int(match.group(1)) if match else 0
    
    # 其他提取方法类似，需要根据实际页面结构编写
    
    def _parse_articles(self, html):
        """解析文章列表"""
        articles = []
        # 这里需要根据实际HTML结构编写解析代码
        # 示例：使用BeautifulSoup解析
        from bs4 import BeautifulSoup
        soup = BeautifulSoup(html, 'html.parser')
        
        # 假设文章在特定的div中
        article_elements = soup.find_all('div', class_='content')  # 类名需要实际查看
        
        for elem in article_elements:
            article = {
                'title': elem.find('a', class_='title').text if elem.find('a', class_='title') else '',
                'publish_time': elem.find('time')['datetime'] if elem.find('time') else '',
                'read_count': self._extract_number(elem.find('span', class_='read-count')),
                'like_count': self._extract_number(elem.find('span', class_='like-count')),
                'comment_count': self._extract_number(elem.find('span', class_='comment-count'))
            }
            articles.append(article)
        
        return articles
    
    def _extract_number(self, element):
        """从元素中提取数字"""
        if element and element.text:
            import re
            match = re.search(r'\d+', element.text)
            return int(match.group()) if match else 0
        return 0

class UserProfileAnalyzer:
    """用户画像分析器"""
    
    def __init__(self, user_data, articles_data):
        self.user_data = user_data
        self.articles_data = articles_data
    
    def generate_profile(self):
        """生成用户画像"""
        profile = {
            '基础信息': {
                '用户ID': self.user_data.get('user_id'),
                '昵称': self.user_data.get('nickname'),
                '粉丝数': self.user_data.get('followers'),
                '关注数': self.user_data.get('following')
            },
            '创作能力': {
                '文章总数': self.user_data.get('articles'),
                '总字数': self.user_data.get('words'),
                '平均阅读量': self._calculate_avg_reads(),
                '平均点赞数': self._calculate_avg_likes()
            },
            '活跃度': {
                '最近发文时间': self._get_latest_article_time(),
                '发文频率': self._calculate_post_frequency(),
                '互动率': self._calculate_engagement_rate()
            },
            '内容偏好': {
                '热门标签': self._extract_top_tags(),
                '平均文章长度': self._calculate_avg_article_length()
            }
        }
        return profile
    
    def _calculate_avg_reads(self):
        """计算平均阅读量"""
        if not self.articles_data:
            return 0
        reads = [article.get('read_count', 0) for article in self.articles_data]
        return sum(reads) / len(reads)
    
    def _calculate_avg_likes(self):
        """计算平均点赞数"""
        if not self.articles_data:
            return 0
        likes = [article.get('like_count', 0) for article in self.articles_data]
        return sum(likes) / len(likes)
    
    def _get_latest_article_time(self):
        """获取最新文章时间"""
        if not self.articles_data:
            return None
        times = [article.get('publish_time') for article in self.articles_data if article.get('publish_time')]
        return max(times) if times else None
    
    def _calculate_post_frequency(self):
        """计算发文频率"""
        if len(self.articles_data) < 2:
            return '数据不足'
        
        # 计算时间间隔
        times = [datetime.fromisoformat(t.replace('Z', '+00:00')) 
                for t in [a.get('publish_time') for a in self.articles_data if a.get('publish_time')]]
        
        if len(times) < 2:
            return '数据不足'
        
        time_diff = (max(times) - min(times)).days
        if time_diff == 0:
            return '高频发文'
        
        frequency = len(times) / time_diff
        if frequency > 1:
            return f'每日{frequency:.1f}篇'
        elif frequency > 0.1:
            return f'每{1/frequency:.0f}天1篇'
        else:
            return '低频发文'
    
    def _calculate_engagement_rate(self):
        """计算互动率（点赞/阅读）"""
        avg_reads = self._calculate_avg_reads()
        avg_likes = self._calculate_avg_likes()
        
        if avg_reads == 0:
            return 0
        return (avg_likes / avg_reads) * 100
    
    def _extract_top_tags(self):
        """提取热门标签（需要文章内容分析）"""
        # 这里需要实际的文章内容来分析标签
        # 示例返回
        return ['Python', '爬虫', '数据分析']
    
    def _calculate_avg_article_length(self):
        """计算平均文章长度"""
        # 需要获取文章正文内容
        return '需要正文内容分析'

# 使用示例
def main():
    # 初始化爬虫
    spider = JianShuSpider()
    
    # 获取用户数据（示例用户ID）
    user_id = "example_user_id"
    
    print("正在爬取用户数据...")
    user_info = spider.get_user_info(user_id)
    
    if user_info:
        print("正在爬取文章数据...")
        articles = spider.get_user_articles(user_id, limit=10)
        
        # 分析用户画像
        analyzer = UserProfileAnalyzer(user_info, articles)
        profile = analyzer.generate_profile()
        
        # 输出结果
        print("\n=== 用户画像分析结果 ===")
        for category, data in profile.items():
            print(f"\n{category}:")
            for key, value in data.items():
                print(f"  {key}: {value}")
        
        # 保存数据
        with open(f'user_profile_{user_id}.json', 'w', encoding='utf-8') as f:
            json.dump(profile, f, ensure_ascii=False, indent=2)
        print(f"\n数据已保存到 user_profile_{user_id}.json")
    else:
        print("无法获取用户数据")

if __name__ == "__main__":
    main()

这个爬虫框架包含了：