求推荐一个Python的多线程HTTP下载库
写工具的时候需要多线程下载数据。
计划直接 subprocess 调用 axel,但是大部分发行版并没有预装 axel。
所以比较麻烦。
但不知道有没有 Python 现成的轮子,求各位大神指导一下。
谢谢
求推荐一个Python的多线程HTTP下载库
9 回复
发行版没有预装 axel
写个脚本装个 axel 不就行了?
既然要用 axel 下载
想必是连网状态
对于Python的多线程HTTP下载,我首推 aiohttp 配合 asyncio。虽然它基于异步IO,但通过 asyncio.gather() 可以轻松实现并发下载,效率远超传统多线程。下面是一个完整示例:
import aiohttp
import asyncio
import os
async def download_file(session, url, save_path):
try:
async with session.get(url) as response:
if response.status == 200:
with open(save_path, 'wb') as f:
while True:
chunk = await response.content.read(8192)
if not chunk:
break
f.write(chunk)
print(f"下载完成: {save_path}")
else:
print(f"下载失败 {url}: HTTP {response.status}")
except Exception as e:
print(f"下载出错 {url}: {e}")
async def download_multiple(urls, save_dir):
if not os.path.exists(save_dir):
os.makedirs(save_dir)
async with aiohttp.ClientSession() as session:
tasks = []
for i, url in enumerate(urls):
filename = url.split('/')[-1] or f"file_{i}"
save_path = os.path.join(save_dir, filename)
task = download_file(session, url, save_path)
tasks.append(task)
await asyncio.gather(*tasks)
# 使用示例
if __name__ == "__main__":
urls = [
"https://example.com/file1.zip",
"https://example.com/file2.zip",
"https://example.com/file3.zip"
]
asyncio.run(download_multiple(urls, "./downloads"))
如果你坚持要用传统多线程,可以用 concurrent.futures:
import requests
from concurrent.futures import ThreadPoolExecutor
import os
def download_worker(url, save_path):
try:
r = requests.get(url, stream=True)
with open(save_path, 'wb') as f:
for chunk in r.iter_content(chunk_size=8192):
f.write(chunk)
print(f"下载完成: {save_path}")
except Exception as e:
print(f"下载失败 {url}: {e}")
def download_with_threads(urls, save_dir, max_workers=5):
if not os.path.exists(save_dir):
os.makedirs(save_dir)
with ThreadPoolExecutor(max_workers=max_workers) as executor:
futures = []
for i, url in enumerate(urls):
filename = url.split('/')[-1] or f"file_{i}"
save_path = os.path.join(save_dir, filename)
future = executor.submit(download_worker, url, save_path)
futures.append(future)
# 等待所有任务完成
for future in futures:
future.result()
# 使用示例
urls = ["https://example.com/file1.zip", "https://example.com/file2.zip"]
download_with_threads(urls, "./downloads")
用 aiohttp 吧,现在异步才是王道。
aiohttp,异步
python3.6 + asyncio + uvloop + aiohttp
接下来你只需要担心你的带宽了!
逃 : )
不一定有权限啊
可以装在家目录
aria2c 单个可执行文件
可能 lz 想要的是类似 requests 那样一条命令下载的,ayncio 还要手动分隔数据段再请求。
支持使用 aria2c (
grequests ?异步版 requests

