Python新手求助：如何编写一个简单的爬虫程序？
最新 http://adr.meizitu.net/wp-json/wp/v2/posts?page=1&per_page=20 图片详情 http://adr.meizitu.net/wp-json/wp/v2/i?id=152201
Python新手求助：如何编写一个简单的爬虫程序？
import requests
from bs4 import BeautifulSoup
def simple_crawler(url):
try:
# 发送HTTP请求
headers = {'User-Agent': 'Mozilla/5.0'}
response = requests.get(url, headers=headers, timeout=5)
response.raise_for_status() # 检查请求是否成功
# 解析HTML内容
soup = BeautifulSoup(response.text, 'html.parser')
# 提取所有链接
links = []
for link in soup.find_all('a'):
href = link.get('href')
if href and href.startswith('http'):
links.append(href)
# 提取文本内容（示例：获取所有段落文本）
paragraphs = [p.get_text().strip() for p in soup.find_all('p')]
return {
'title': soup.title.string if soup.title else '无标题',
'links': links[:10], # 只显示前10个链接
'paragraphs': paragraphs[:5] # 只显示前5个段落
}
except requests.exceptions.RequestException as e:
print(f"请求出错: {e}")
return None
# 使用示例
if __name__ == "__main__":
result = simple_crawler("https://www.example.com")
if result:
print(f"页面标题: {result['title']}")
print(f"\n前10个链接:")
for link in result['links']:
print(f" - {link}")
print(f"\n前5个段落:")
for para in result['paragraphs']:
print(f" - {para[:100]}...") # 只显示前100字符
代码说明：
- 用
requests库发送HTTP请求，添加User-Agent头模拟浏览器访问 BeautifulSoup解析HTML，find_all('a')找所有链接，find_all('p')找段落- 异常处理确保程序健壮性
- 返回字典包含标题、链接和文本内容
运行准备：
pip install requests beautifulsoup4
一句话建议： 先从这个基础爬虫开始，理解网页结构后再学习更复杂的功能。
贴个 csdn 的。。我记得之前学异步协程时候拿来测试过，直接复制下来就能跑。。。（ https://blog.csdn.net/ns2250225/article/details/78448955 ）
好奇这是个网站吗？网站首页是个啥？ http://adr.meizitu.net/是 403 https://www.meizitu.net/也打不开，只有 https://i2.meizitu.net/能打开，但是是个朦胧月色？ What ？？？
<br><?php<br>$jsons = file_get_contents('<a target="_blank" href="http://adr.meizitu.net/wp-json/wp/v2/posts?page=1&per_page=20');" rel="nofollow noopener">http://adr.meizitu.net/wp-json/wp/v2/posts?page=1&per_page=20');</a><br>$json_arr = json_decode($jsons, true);<br>foreach ($json_arr as $value) {<br> file_put_contents($value['id'].'.jpg', file_get_contents($value['thumb_src']));<br>}<br>
改了一下你的
$json=Invoke-WebRequest “http://adr.meizitu.net/wp-json/wp/v2/posts?page=1&per_page=20” -Method Get -UseBasicParsing| ConvertFrom-Json
$wc = New-Object System.Net.WebClient
foreach ($i in $json) {
$output = split-path -Leaf $i.thumb_src
$wc.DownloadFile($i.thumb_src,$output)
}
import requests
import json
url = ‘https://adr.meizitu.net/wp-json/wp/v2/posts?page={}&per_page={}‘
per_page = 100
page = 52
print(’##### spider start #####’)
while True:
page += 1
json_data = requests.get(url.format(page, per_page))
data = json.loads(json_data.text)
if not isinstance(data, list):
print(’##### spider end #####’)
break
for item in data:
thumb_src = item[‘thumb_src’]
title = item[‘title’] + ‘.jpg’
print(’[+] downloading {} …’.format(title))
meizi = requests.get(thumb_src)
if meizi.status_code == 200:
with open(title, ‘wb’) as f:
f.write(meizi.content)
[{“id”:181372,“title”:“辣妹温心怡情趣内衣 SM 诱惑 丰胸美臀身材热辣销魂”,“img_num”:46,“thumb_src”:“https://i2.meizitu.net/2019/04/23a25.jpg”,“thumb_src_min”:“https://i2.meizitu.net/thumbs/2019/05/181372_23a25_236.jpg”},{“id”:180743,“title”:“大胸女神恩一雪白玉兔诱人呈现 手捧巨乳再掀性感狂潮”…
要啥爬虫, 一行命令的事情
curl -s http://adr.meizitu.net/wp-json/wp/v2/posts?page=1&per_page=20 | jq -r ‘.[] | .thumb_src’ | xargs -IX curl -s -O X
import json
import os
import requests
if name == ‘main’:
flag = True
page = 1
while flag:
print(“page:” + str(page))
list_html = requests.get(“http://adr.meizitu.net/wp-json/wp/v2/posts?page=” + str(page) + “&per_page=20”)
content = list_html.content
json_loads = json.loads(content)
if isinstance(json_loads, list):
print(json_loads)
for list_item in json_loads:
id_ = list_item[‘id’]
title = list_item[‘title’]
print(title)
detail_html = requests.get(“http://adr.meizitu.net/wp-json/wp/v2/i?id=” + str(id_))
detail_json = json.loads(detail_html.content)
print(detail_json)
str_content_ = detail_json[‘content’]
content__split = str_content_.split(’,’)
print(‘downloading ’ + str(content__split))
for detail_item in content__split:
print(detail_item)
rfind = detail_item.rfind(’/’)
file_name = detail_item[rfind + 1:len(detail_item)]
folder_path = “./images/” + title + ‘/’
if not os.path.exists(folder_path):
os.makedirs(folder_path)
requests_get = requests.get(detail_item)
with open(folder_path + file_name, “wb”) as f:
f.write(requests_get.content)
else:
code_ = json_loads[‘code’]
if code_ != ‘rest_post_invalid_page_number’:
print(code_)
else:
print(code_ + " exit")
flag = False
page += 1
我怀疑你在开车。。而且我有证据。。\u8fa3\u59b9\u6e29\u5fc3\u6021\u60c5\u8da3\u5185\u8863SM\u8bf1\u60d1\u4e30\u80f8\u7f8e\u81c0\u8eab\u6750\u70ed\u8fa3\u9500\u9b42
