Python中如何爬取简书某一时间段内的发帖

import requests
import json
import time
from datetime import datetime, timedelta
from typing import List, Dict

class JianShuCrawler:
    def __init__(self):
        self.headers = {
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
        }
        self.base_url = "https://www.jianshu.com/asimov/notes"
        
    def crawl_by_time_range(self, start_time: str, end_time: str, max_pages: int = 50) -> List[Dict]:
        """
        爬取指定时间范围内的简书文章
        
        Args:
            start_time: 开始时间，格式 '2024-01-01'
            end_time: 结束时间，格式 '2024-01-31'
            max_pages: 最大爬取页数
        
        Returns:
            文章数据列表
        """
        articles = []
        page = 1
        
        # 转换为时间戳
        start_ts = int(datetime.strptime(start_time, "%Y-%m-%d").timestamp() * 1000)
        end_ts = int(datetime.strptime(end_time, "%Y-%m-%d").timestamp() * 1000)
        
        while page <= max_pages:
            try:
                params = {
                    'order_by': 'commented_at',
                    'page': page,
                    'count': 20
                }
                
                response = requests.get(
                    self.base_url, 
                    headers=self.headers, 
                    params=params,
                    timeout=10
                )
                
                if response.status_code != 200:
                    print(f"请求失败，状态码: {response.status_code}")
                    break
                
                data = response.json()
                notes = data.get('notes', [])
                
                if not notes:
                    break
                
                # 过滤时间范围内的文章
                for note in notes:
                    pub_time = note.get('first_shared_at')
                    if pub_time and start_ts <= pub_time <= end_ts:
                        article = {
                            'id': note.get('id'),
                            'title': note.get('title'),
                            'slug': note.get('slug'),
                            'author': note.get('user', {}).get('nickname'),
                            'publish_time': self._timestamp_to_date(pub_time),
                            'views_count': note.get('views_count', 0),
                            'likes_count': note.get('likes_count', 0),
                            'url': f"https://www.jianshu.com/p/{note.get('slug')}"
                        }
                        articles.append(article)
                        print(f"找到文章: {article['title']} - {article['publish_time']}")
                
                # 如果最早的文章已经超过时间范围，停止爬取
                earliest_time = min([n.get('first_shared_at', 0) for n in notes])
                if earliest_time < start_ts:
                    break
                
                page += 1
                time.sleep(1)  # 礼貌延迟
                
            except Exception as e:
                print(f"第{page}页爬取出错: {e}")
                break
        
        return articles
    
    def _timestamp_to_date(self, timestamp: int) -> str:
        """时间戳转日期字符串"""
        if timestamp:
            dt = datetime.fromtimestamp(timestamp / 1000)
            return dt.strftime("%Y-%m-%d %H:%M:%S")
        return ""
    
    def save_to_json(self, articles: List[Dict], filename: str = "jianshu_articles.json"):
        """保存结果到JSON文件"""
        with open(filename, 'w', encoding='utf-8') as f:
            json.dump(articles, f, ensure_ascii=False, indent=2)
        print(f"已保存 {len(articles)} 篇文章到 {filename}")

# 使用示例
if __name__ == "__main__":
    crawler = JianShuCrawler()
    
    # 设置时间范围
    start_date = "2024-01-01"
    end_date = "2024-01-31"
    
    print(f"开始爬取 {start_date} 到 {end_date} 的简书文章...")
    
    articles = crawler.crawl_by_time_range(start_date, end_date, max_pages=10)
    
    print(f"\n共找到 {len(articles)} 篇文章")
    
    # 保存结果
    if articles:
        crawler.save_to_json(articles)
        
        # 打印前几篇
        print("\n前5篇文章:")
        for i, article in enumerate(articles[:5], 1):
            print(f"{i}. {article['title']} - {article['author']}")

这个爬虫通过简书的API接口获取数据，按评论时间排序，然后根据first_shared_at字段过滤指定时间范围内的文章。代码包含错误处理和礼貌延迟，避免对服务器造成压力。

注意：简书可能会调整API，使用时需要关注响应结构的变化。

欢迎，我也是来向大 V 学习的，:)