Python中如何实现Tumblr爬虫
好几个月前写的了，写的比较挫。
并没有写成爬取一个博客的所有内容，本来是用来网站的，如果要爬所有内容，会让用户等待太久。
# -*- coding=utf-8 -*-
from threading import Thread
import Queue
import requests
import re
import os
import sys
import time
api_url=‘http://%s.tumblr.com/api/read?&num=50&start=’
UQueue=Queue.Queue()
def getpost(uid,queue):
url=‘http://%s.tumblr.com/api/read?&num=50’%uid
page=requests.get(url).content
total=re.findall(’<posts start=“0” total="(.?)">’,page)[0]
total=int(total)
a=[i50 for i in range(1000) if i*50-total<0]
ul=api_url%uid
for i in a:
queue.put(ul+str(i))
extractpicre = re.compile(r’(?<=<photo-url max-width=“1280”>).+?(?=</photo-url>)’,flags=re.S) #search for url of maxium size of a picture, which starts with ‘<photo-url max-width=“1280”>’ and ends with ‘</photo-url>’
extractvideore=re.compile(’/tumblr_(.*?)" type=“video/mp4”’)
video_links = []
pic_links = []
vhead = ‘https://vt.tumblr.com/tumblr_%s.mp4’
class Consumer(Thread):
def __init__(self, l_queue):
super(Consumer,self).__init__()
self.queue = l_queue
def run(self):
session = requests.Session()
while 1:
link = self.queue.get()
print 'start parse post: ' + link
try:
content = session.get(link).content
videos = extractvideore.findall(content)
video_links.extend([vhead % v for v in videos])
pic_links.extend(extractpicre.findall(content))
except:
print 'url: %s parse failed\n' % link
if self.queue.empty():
break
def main():
task=[]
for i in range(min(10,UQueue.qsize())):
t=Consumer(UQueue)
task.append(t)
for t in task:
t.start()
for t in task:
t.join
while 1:
for t in task:
if t.is_alive():
continue
else:
task.remove(t)
if len(task)==0:
break
def write():
videos=[i.replace(’/480’,’’) for i in video_links]
pictures=pic_links
with open(‘pictures.txt’,‘w’) as f:
for i in pictures:
f.write(’%s\n’%i)
with open(‘videos.txt’,‘w’) as f:
for i in videos:
f.write(’%s\n’%i)
if name==‘main’:
#name=sys.argv[1]
#name=name.strip()
name=‘mzcyx2011’
getpost(name,UQueue)
main()
write()
Python中如何实现Tumblr爬虫
要写一个Tumblr爬虫，最直接的方法是使用他们的官方API。不过，如果你想直接爬取公开页面，用requests和BeautifulSoup也行，但得注意反爬和页面结构变化。
这里给你一个基于API的示例，这是最稳定和推荐的方式。首先，你需要去Tumblr申请一个OAuth应用，拿到API Key（也叫Consumer Key）。
import requests
import json
# 替换成你自己的API Key
API_KEY = 'YOUR_API_KEY_HERE'
BLOG_NAME = 'staff' # 要爬取的博客名，例如官方博客 'staff'
# 构建API请求URL来获取基础信息
url = f'https://api.tumblr.com/v2/blog/{BLOG_NAME}.tumblr.com/posts?api_key={API_KEY}'
try:
response = requests.get(url)
response.raise_for_status() # 检查请求是否成功
data = response.json()
# 解析返回的JSON数据
posts = data.get('response', {}).get('posts', [])
for post in posts:
# 这里根据帖子类型处理不同内容
post_id = post.get('id')
post_type = post.get('type')
summary = post.get('summary', 'No summary')
print(f"Post ID: {post_id}")
print(f"Type: {post_type}")
print(f"Summary: {summary[:100]}...") # 打印前100个字符
print("-" * 40)
except requests.exceptions.RequestException as e:
print(f"请求出错: {e}")
except json.JSONDecodeError:
print("解析JSON响应失败")
这个脚本会获取指定博客的最新帖子列表。Tumblr API返回的是JSON格式，结构清晰。post['type']字段能告诉你帖子是文本（text）、图片（photo）、引用（quote）还是其他类型，你需要根据这个写不同的处理逻辑。
如果你想爬取更多帖子，API响应里有个_links字段包含下一页的链接，可以用来做分页请求。记得在请求间加点延时，别把人家服务器搞崩了。
总结：用官方API最省心。
下载到一半会这样
Traceback (most recent call last):
File “turmla.py”, line 150, in <module>
for square in tqdm(pool.imap_unordered(download_base_dir, urls), total=len(urls)):
File “/home/leetom/.pyenv/versions/2.7.10/lib/python2.7/site-packages/tqdm/_tqdm.py”, line 713, in iter
for obj in iterable:
File “/home/leetom/.pyenv/versions/2.7.10/lib/python2.7/multiprocessing/pool.py”, line 668, in next
raise value
Exception: Unexpected response.
with open(‘pictures.txt’,‘r’) as fobj:
for eachline in fobj:
pngurl=eachline.strip()
filename=’.//getpic//test-{0}.jpg’.format(i)
print ‘[-]parsing:{0}’.format(filename)
urllib.urlretrieve(pngurl,filename)
i+=1
