Rust的robots.txt解析库robotstxt使用指南

概述

robotstxt是一个用于解析和生成robots.txt文件的Rust库，帮助开发者处理网站爬虫规则。该库支持解析现有的robots.txt文件，也能生成符合标准的规则文件。

安装方法

在Cargo.toml中添加依赖：

[dependencies]
robotstxt = "0.10"

基本用法

解析robots.txt文件

use robotstxt::DefaultMatcher;

fn main() {
    let contents = r#"
        User-agent: *
        Disallow: /private/
        Allow: /public/
        Crawl-delay: 5
    "#;

    let matcher = DefaultMatcher::default();
    let result = matcher.one_agent_allowed_by_robots(contents, "MyBot", "/public/about.html");
    
    println!("允许访问: {}", result); // 输出: true
}

检查特定路径的访问权限

use robotstxt::DefaultMatcher;

fn check_access(robots_txt: &str, user_agent: &str, path: &str) -> bool {
    let matcher = DefaultMatcher::default();
    matcher.one_agent_allowed_by_robots(robots_txt, user_agent, path)
}

// 使用示例
let robots_content = "
User-agent: Googlebot
Disallow: /private/

User-agent: *
Disallow: /admin/
";

println!("Googlebot访问/admin/: {}", 
    check_access(robots_content, "Googlebot", "/admin/")); // false
println!("OtherBot访问/admin/: {}", 
    check_access(robots_content, "OtherBot", "/admin/")); // false

生成robots.txt文件

use robotstxt::RobotsBuilder;

fn generate_robots_txt() -> String {
    RobotsBuilder::new()
        .add_user_agent("*")
        .disallow("/private/")
        .allow("/public/")
        .crawl_delay(10)
        .add_user_agent("Googlebot")
        .disallow("/temp/")
        .finish()
        .to_string()
}

println!("{}", generate_robots_txt());

高级功能

处理多个User-agent

use robotstxt::DefaultMatcher;

fn check_multiple_agents() {
    let robots_txt = "
        User-agent: Bot1
        Disallow: /dir1/
        
        User-agent: Bot2
        Allow: /dir1/
        Disallow: /dir2/
    ";

    let matcher = DefaultMatcher::default();
    
    // 检查Bot1对/dir1/page的访问
    let result1 = matcher.one_agent_allowed_by_robots(robots_txt, "Bot1", "/dir1/page.html");
    println!("Bot1访问/dir1/page.html: {}", result1); // false
    
    // 检查Bot2对/dir1/page的访问
    let result2 = matcher.one_agent_allowed_by_robots(robots_txt, "Bot2", "/dir1/page.html");
    println!("Bot2访问/dir1/page.html: {}", result2); // true
}

处理爬虫延迟设置

use robotstxt::{DefaultMatcher, RobotsParseHandler};

fn get_crawl_delay(robots_txt: &str, user_agent: &str) -> Option<u64> {
    let mut handler = RobotsParseHandler::new();
    handler.parse(robots_txt.as_bytes());
    
    for group in handler.get_groups() {
        if group.applies_to(user_agent) {
            return group.get_crawl_delay();
        }
    }
    None
}

let delay = get_crawl_delay(
    "User-agent: *\nCrawl-delay: 5",
    "MyBot"
);
println!("爬虫延迟: {:?}秒", delay); // Some(5)

错误处理

use robotstxt::DefaultMatcher;

fn safe_check(robots_txt: &str, user_agent: &str, path: &str) -> Result<bool, &'static str> {
    if robots_txt.is_empty() {
        return Err("空的robots.txt内容");
    }
    
    let matcher = DefaultMatcher::default();
    Ok(matcher.one_agent_allowed_by_robots(robots_txt, user_agent, path))
}

实际应用示例

use robotstxt::DefaultMatcher;
use reqwest::blocking::get;

fn check_website_permission(url: &str, user_agent: &str, path: &str) -> bool {
    let robots_url = format!("{}/robots.txt", url.trim_end_matches('/'));
    
    match get(&robots_url) {
        Ok(response) => {
            if let Ok(content) = response.text() {
                let matcher = DefaultMatcher::default();
                matcher.one_agent_allowed_by_robots(&content, user_agent, path)
            } else {
                true // 如果无法读取内容，默认允许访问
            }
        }
        Err(_) => true // 如果无法获取robots.txt，默认允许访问
    }
}

完整示例demo

// 完整示例：构建一个robots.txt检查工具
use robotstxt::{DefaultMatcher, RobotsBuilder};
use std::io::{self, Write};

fn main() {
    // 示例1：生成robots.txt文件
    println!("=== 生成robots.txt示例 ===");
    let robots_content = generate_robots_example();
    println!("生成的robots.txt内容:\n{}", robots_content);
    
    // 示例2：检查访问权限
    println!("\n=== 访问权限检查示例 ===");
    check_access_permissions(&robots_content);
    
    // 示例3：交互式检查工具
    println!("\n=== 交互式检查工具 ===");
    interactive_check_tool(&robots_content);
}

// 生成示例robots.txt
fn generate_robots_example() -> String {
    RobotsBuilder::new()
        .add_user_agent("*")
        .disallow("/admin/")
        .disallow("/private/")
        .allow("/public/")
        .crawl_delay(5)
        .add_user_agent("Googlebot")
        .allow("/news/")
        .disallow("/temp/")
        .add_user_agent("Bingbot")
        .disallow("/search/")
        .finish()
        .to_string()
}

// 检查不同用户代理的访问权限
fn check_access_permissions(robots_txt: &str) {
    let matcher = DefaultMatcher::default();
    
    let test_cases = vec![
        ("Googlebot", "/news/article1", "Googlebot访问新闻文章"),
        ("Googlebot", "/admin/dashboard", "Googlebot访问管理面板"),
        ("Bingbot", "/search/results", "Bingbot访问搜索结果"),
        ("MyCrawler", "/public/docs", "普通爬虫访问公开文档"),
        ("MyCrawler", "/private/data", "普通爬虫访问私有数据"),
    ];
    
    for (ua, path, description) in test_cases {
        let allowed = matcher.one_agent_allowed_by_robots(robots_txt, ua, path);
        println!("{}: {}", description, if allowed { "允许" } else { "拒绝" });
    }
}

// 交互式检查工具
fn interactive_check_tool(robots_txt: &str) {
    let matcher = DefaultMatcher::default();
    
    loop {
        print!("请输入用户代理名称 (输入 'quit' 退出): ");
        io::stdout().flush().unwrap();
        
        let mut user_agent = String::new();
        io::stdin().read_line(&mut user_agent).unwrap();
        let user_agent = user_agent.trim();
        
        if user_agent.eq_ignore_ascii_case("quit") {
            break;
        }
        
        print!("请输入要检查的路径: ");
        io::stdout().flush().unwrap();
        
        let mut path = String::new();
        io::stdin().read_line(&mut path).unwrap();
        let path = path.trim();
        
        let allowed = matcher.one_agent_allowed_by_robots(robots_txt, user_agent, path);
        println!("用户代理 '{}' 访问路径 '{}': {}",
            user_agent, path, 
            if allowed { "允许" } else { "拒绝" }
        );
        println!();
    }
}

这个库提供了完整的robots.txt处理功能，适合在爬虫项目或web应用中集成机器人协议检查功能。