demo

地址在这里 zhihu_fun Github，求 star !!!!

。。。嗯，其实也可以爬其他方面的东西，但是目前只支持抓图片，其实这不是我的本意，我是想抓技术方面的东西的，但是我同事想要看这个，所以我就先抓这些图片了。。。Python 菜鸟一枚，代码写的很烂，还望各位大神多多指教

为什么要用 Selenium ？

新版知乎部分页面用 React 重写了，必须得加载 JS ，很多加载啥的都需要点击，所以我就用到了 Selenium

为什么不用知乎 API ？

单纯的想写一个爬虫而已

Python中如何使用selenium编写知乎关键词爬虫并抓取钓鱼贴图片

gougou168 1楼

一百次点击，无人回复，干的漂亮。

vueper 2楼

要写一个知乎关键词爬虫抓钓鱼贴图片，用Selenium得先装好驱动。核心思路是模拟登录、搜索关键词、过滤钓鱼贴、提取图片链接。下面给个完整示例：

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import time
import requests
import os

class ZhihuImageCrawler:
    def __init__(self):
        self.driver = webdriver.Chrome()  # 需要提前下载chromedriver
        self.wait = WebDriverWait(self.driver, 10)
        
    def login(self, username, password):
        """模拟登录知乎"""
        self.driver.get("https://www.zhihu.com/signin")
        
        # 点击密码登录
        self.wait.until(EC.element_to_be_clickable(
            (By.CSS_SELECTOR, ".SignFlow-tab")
        )).click()
        
        # 输入账号密码
        self.driver.find_element(By.NAME, "username").send_keys(username)
        self.driver.find_element(By.NAME, "password").send_keys(password)
        
        # 点击登录
        self.driver.find_element(
            By.CSS_SELECTOR, ".Button.SignFlow-submitButton"
        ).click()
        
        time.sleep(3)  # 等待登录完成
        
    def search_keyword(self, keyword):
        """搜索关键词"""
        search_input = self.wait.until(EC.presence_of_element_located(
            (By.CSS_SELECTOR, ".SearchBar-input input")
        ))
        search_input.clear()
        search_input.send_keys(keyword)
        search_input.send_keys(Keys.RETURN)
        
        # 切换到"回答"标签页
        self.wait.until(EC.element_to_be_clickable(
            (By.XPATH, "//div[text()='回答']")
        )).click()
        
        time.sleep(2)
        
    def extract_fishing_posts(self, max_scroll=5):
        """提取钓鱼贴内容，通过特定关键词识别"""
        fishing_keywords = ['钓鱼', '骗局', '诈骗', '上当']
        image_urls = []
        
        for _ in range(max_scroll):
            # 获取当前页面的所有回答卡片
            cards = self.driver.find_elements(By.CSS_SELECTOR, ".List-item")
            
            for card in cards:
                try:
                    content = card.text
                    # 判断是否为钓鱼贴
                    if any(keyword in content for keyword in fishing_keywords):
                        # 提取图片
                        images = card.find_elements(By.TAG_NAME, "img")
                        for img in images:
                            src = img.get_attribute("src")
                            if src and "http" in src:
                                image_urls.append(src)
                except:
                    continue
            
            # 滚动加载更多
            self.driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
            time.sleep(2)
            
        return list(set(image_urls))  # 去重
    
    def download_images(self, urls, save_dir="zhihu_images"):
        """下载图片到本地"""
        if not os.path.exists(save_dir):
            os.makedirs(save_dir)
            
        for i, url in enumerate(urls):
            try:
                response = requests.get(url, timeout=10)
                if response.status_code == 200:
                    # 提取图片格式
                    ext = url.split('.')[-1].split('?')[0]
                    if len(ext) > 4:
                        ext = 'jpg'
                    
                    filename = f"{save_dir}/image_{i}.{ext}"
                    with open(filename, 'wb') as f:
                        f.write(response.content)
                    print(f"已下载: {filename}")
            except Exception as e:
                print(f"下载失败 {url}: {e}")
    
    def run(self, keyword, username=None, password=None):
        """主运行函数"""
        try:
            if username and password:
                self.login(username, password)
            
            self.search_keyword(keyword)
            image_urls = self.extract_fishing_posts(max_scroll=3)
            
            print(f"找到 {len(image_urls)} 张图片")
            if image_urls:
                self.download_images(image_urls)
                
        finally:
            self.driver.quit()

# 使用示例
if __name__ == "__main__":
    crawler = ZhihuImageCrawler()
    
    # 如果需要登录就传账号密码
    crawler.run(
        keyword="网络诈骗",
        # username="your_username",
        # password="your_password"
    )

几个关键点：