Python中如何存储和获取随机100个网页的状态？

import requests
import random
import time
from concurrent.futures import ThreadPoolExecutor, as_completed
import json

def check_url_status(url):
    """检查单个URL的状态"""
    try:
        response = requests.get(url, timeout=5)
        return {
            'url': url,
            'status_code': response.status_code,
            'success': response.status_code == 200,
            'timestamp': time.time()
        }
    except requests.RequestException as e:
        return {
            'url': url,
            'status_code': None,
            'success': False,
            'error': str(e),
            'timestamp': time.time()
        }

def generate_random_urls(count=100):
    """生成随机URL列表"""
    domains = [
        'https://www.google.com',
        'https://www.github.com',
        'https://www.stackoverflow.com',
        'https://www.python.org',
        'https://www.wikipedia.org',
        'https://www.reddit.com',
        'https://www.amazon.com',
        'https://www.microsoft.com'
    ]
    
    urls = []
    for _ in range(count):
        domain = random.choice(domains)
        # 添加随机路径增加URL多样性
        path = ''.join(random.choices('abcdefghijklmnopqrstuvwxyz', k=random.randint(3, 10)))
        urls.append(f"{domain}/{path}")
    
    return urls

def store_results(results, filename='url_status.json'):
    """存储结果到JSON文件"""
    with open(filename, 'w') as f:
        json.dump(results, f, indent=2)

def load_results(filename='url_status.json'):
    """从JSON文件加载结果"""
    try:
        with open(filename, 'r') as f:
            return json.load(f)
    except FileNotFoundError:
        return []

def main():
    # 1. 生成100个随机URL
    urls = generate_random_urls(100)
    print(f"已生成 {len(urls)} 个随机URL")
    
    # 2. 并发检查URL状态
    results = []
    with ThreadPoolExecutor(max_workers=10) as executor:
        future_to_url = {executor.submit(check_url_status, url): url for url in urls}
        
        for future in as_completed(future_to_url):
            result = future.result()
            results.append(result)
            print(f"Checked: {result['url']} - Status: {result.get('status_code', 'Error')}")
    
    # 3. 存储结果
    store_results(results)
    print(f"\n结果已保存到 url_status.json")
    
    # 4. 统计信息
    successful = sum(1 for r in results if r['success'])
    print(f"成功访问: {successful}/{len(results)}")
    
    # 5. 示例：如何获取特定状态的结果
    print("\n=== 状态码200的URL ===")
    for result in results:
        if result['success']:
            print(f"✓ {result['url']}")

if __name__ == "__main__":
    main()

这个方案做了几件事：

1. 生成随机URL：从预设域名列表随机组合生成100个URL
2. 并发检查：使用线程池同时检查多个URL，提高效率
3. 结果存储：将结果保存为JSON格式，包含URL、状态码、时间戳等信息
4. 错误处理：处理网络请求异常，确保程序健壮性
5. 数据统计：提供基本的成功/失败统计

关键点说明：

• 使用requests库进行HTTP请求
• ThreadPoolExecutor实现并发检查
• JSON格式便于后续分析和处理
• 包含时间戳便于追踪检查时间

使用方式：

# 运行检查
python check_urls.py

# 加载之前的结果
results = load_results()
for item in results:
    print(f"{item['url']}: {item['status_code']}")

扩展建议： 可以改用数据库存储以便长期追踪。