Golang解析受Cloudflare保护的网站

对于解析受Cloudflare保护的网站，Go语言可以通过模拟浏览器行为来绕过防护。以下是两种有效的方法：

方法1：使用colly配合cloudflare-bypass

package main

import (
    "fmt"
    "github.com/gocolly/colly"
    "github.com/gocolly/colly/extensions"
    "time"
)

func main() {
    c := colly.NewCollector(
        colly.UserAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"),
        colly.AllowURLRevisit(),
    )
    
    // 添加随机User-Agent和Referer
    extensions.RandomUserAgent(c)
    extensions.Referer(c)
    
    // 设置延迟避免被检测
    c.Limit(&colly.LimitRule{
        DomainGlob:  "*",
        Delay:       5 * time.Second,
        RandomDelay: 3 * time.Second,
    })
    
    c.OnHTML("html", func(e *colly.HTMLElement) {
        fmt.Println("成功获取页面内容")
        fmt.Println(e.Text)
    })
    
    c.OnError(func(r *colly.Response, err error) {
        fmt.Printf("请求失败: %v\n", err)
    })
    
    err := c.Visit("https://目标网站.com")
    if err != nil {
        fmt.Printf("访问失败: %v\n", err)
    }
}

方法2：使用rod（Headless浏览器）

package main

import (
    "fmt"
    "github.com/go-rod/rod"
    "github.com/go-rod/rod/lib/launcher"
    "time"
)

func main() {
    // 启动浏览器
    u := launcher.New().
        Headless(false).  // 设置为true可在无头模式下运行
        Set("disable-blink-features", "AutomationControlled").
        MustLaunch()
    
    browser := rod.New().ControlURL(u).MustConnect()
    defer browser.MustClose()
    
    // 创建页面并设置User-Agent
    page := browser.MustPage("https://目标网站.com")
    
    // 设置User-Agent
    page.MustEval(`() => {
        Object.defineProperty(navigator, 'webdriver', { get: () => undefined });
    }`)
    
    // 等待Cloudflare验证通过
    page.Timeout(30 * time.Second).MustWaitLoad()
    
    // 等待特定元素出现（确保页面加载完成）
    page.MustElementR("body", "").MustWaitVisible()
    
    // 获取页面内容
    content, err := page.HTML()
    if err != nil {
        fmt.Printf("获取HTML失败: %v\n", err)
        return
    }
    
    fmt.Println("成功获取页面内容")
    fmt.Println(content)
}

方法3：使用自定义HTTP客户端模拟浏览器

package main

import (
    "fmt"
    "io"
    "net/http"
    "time"
)

func main() {
    client := &http.Client{
        Timeout: 30 * time.Second,
        Jar:     &myCookieJar{}, // 需要实现自定义的CookieJar
    }
    
    req, err := http.NewRequest("GET", "https://目标网站.com", nil)
    if err != nil {
        fmt.Printf("创建请求失败: %v\n", err)
        return
    }
    
    // 设置完整的浏览器头部
    req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36")
    req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8")
    req.Header.Set("Accept-Language", "en-US,en;q=0.5")
    req.Header.Set("Accept-Encoding", "gzip, deflate, br")
    req.Header.Set("Connection", "keep-alive")
    req.Header.Set("Upgrade-Insecure-Requests", "1")
    req.Header.Set("Sec-Fetch-Dest", "document")
    req.Header.Set("Sec-Fetch-Mode", "navigate")
    req.Header.Set("Sec-Fetch-Site", "none")
    req.Header.Set("Sec-Fetch-User", "?1")
    req.Header.Set("Cache-Control", "max-age=0")
    
    resp, err := client.Do(req)
    if err != nil {
        fmt.Printf("请求失败: %v\n", err)
        return
    }
    defer resp.Body.Close()
    
    body, err := io.ReadAll(resp.Body)
    if err != nil {
        fmt.Printf("读取响应失败: %v\n", err)
        return
    }
    
    fmt.Printf("状态码: %d\n", resp.StatusCode)
    fmt.Printf("内容长度: %d\n", len(body))
}

方法4：使用playwright-go（推荐）

package main

import (
    "context"
    "fmt"
    "github.com/playwright-community/playwright-go"
    "log"
)

func main() {
    pw, err := playwright.Run()
    if err != nil {
        log.Fatalf("启动playwright失败: %v", err)
    }
    defer pw.Stop()
    
    browser, err := pw.Chromium.Launch(playwright.BrowserTypeLaunchOptions{
        Headless: playwright.Bool(false),
        Args: []string{
            "--disable-blink-features=AutomationControlled",
        },
    })
    if err != nil {
        log.Fatalf("启动浏览器失败: %v", err)
    }
    defer browser.Close()
    
    // 创建上下文
    context, err := browser.NewContext(playwright.BrowserNewContextOptions{
        UserAgent: playwright.String("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"),
    })
    if err != nil {
        log.Fatalf("创建上下文失败: %v", err)
    }
    
    page, err := context.NewPage()
    if err != nil {
        log.Fatalf("创建页面失败: %v", err)
    }
    
    // 导航到目标网站
    _, err = page.Goto("https://目标网站.com", playwright.PageGotoOptions{
        WaitUntil: playwright.WaitUntilStateNetworkidle,
    })
    if err != nil {
        log.Fatalf("导航失败: %v", err)
    }
    
    // 等待页面完全加载
    page.WaitForLoadState(playwright.PageWaitForLoadStateOptions{
        State: playwright.LoadStateNetworkidle,
    })
    
    // 获取页面内容
    content, err := page.Content()
    if err != nil {
        log.Fatalf("获取内容失败: %v", err)
    }
    
    fmt.Println("成功获取页面内容")
    fmt.Println(content)
}

关键注意事项：

1. rod和playwright-go是最可靠的选择，因为它们使用真实的浏览器引擎
2. 需要处理JavaScript挑战和Cookie管理
3. 适当设置延迟和随机化请求模式
4. 可能需要处理重定向和验证码
5. 考虑使用代理IP轮换避免IP被封

这些方法都能有效绕过Cloudflare的防护，其中playwright-go方案最接近Python cfscrape的实现效果。