使用goreadability提取网页内容摘要

goreadability是一个Go语言实现的网页内容提取库，类似于Python的readability-lxml，它可以从网页中提取主要内容并去除无关的广告、导航等元素。

安装goreadability

go get github.com/RadhiFadlillah/go-readability

基本使用方法

package main

import (
	"fmt"
	"log"
	"net/http"
	"time"

	"github.com/RadhiFadlillah/go-readability"
)

func main() {
	// 目标URL
	url := "https://example.com/article"

	// 创建HTTP客户端
	client := &http.Client{Timeout: 30 * time.Second}

	// 获取网页内容
	resp, err := client.Get(url)
	if err != nil {
		log.Fatalf("Failed to fetch URL: %v", err)
	}
	defer resp.Body.Close()

	// 解析网页内容
	article, err := readability.FromReader(resp.Body, url)
	if err != nil {
		log.Fatalf("Failed to parse article: %v", err)
	}

	// 输出提取的内容
	fmt.Println("标题:", article.Title)
	fmt.Println("作者:", article.Byline)
	fmt.Println("内容长度:", len(article.TextContent))
	fmt.Println("主要内容:")
	fmt.Println(article.TextContent)
}

高级功能

1. 自定义解析选项

// 创建自定义解析器
parser := readability.Parser{
	MaxContentLength: 100000,  // 最大内容长度
	MinTextLength:    100,     // 最小文本长度
	RemoveUnlikely:   true,    // 移除不太可能是内容的元素
}

// 使用自定义解析器
article, err := parser.Parse(resp.Body, url)

2. 处理重定向

client := &http.Client{
	Timeout: 30 * time.Second,
	CheckRedirect: func(req *http.Request, via []*http.Request) error {
		// 记录重定向
		fmt.Println("Redirected to:", req.URL)
		return nil
	},
}

3. 错误处理增强

article, err := readability.FromReader(resp.Body, url)
if err != nil {
	switch err {
	case readability.ErrParseFailed:
		fmt.Println("解析失败，可能是非文章页面")
	case readability.ErrNoContent:
		fmt.Println("没有找到有效内容")
	default:
		fmt.Println("未知错误:", err)
	}
	return
}

实际应用示例

下面是一个完整的HTTP服务示例，提供网页内容摘要提取API：

package main

import (
	"encoding/json"
	"log"
	"net/http"
	"time"

	"github.com/RadhiFadlillah/go-readability"
)

type SummaryResponse struct {
	URL      string `json:"url"`
	Title    string `json:"title"`
	Author   string `json:"author"`
	Content  string `json:"content"`
	Length   int    `json:"length"`
	Excerpt  string `json:"excerpt"`
	SiteName string `json:"site_name"`
}

func handleSummary(w http.ResponseWriter, r *http.Request) {
	url := r.URL.Query().Get("url")
	if url == "" {
		http.Error(w, "URL parameter is required", http.StatusBadRequest)
		return
	}

	// 获取网页内容
	client := &http.Client{Timeout: 30 * time.Second}
	resp, err := client.Get(url)
	if err != nil {
		http.Error(w, "Failed to fetch URL", http.StatusBadGateway)
		return
	}
	defer resp.Body.Close()

	// 解析内容
	article, err := readability.FromReader(resp.Body, url)
	if err != nil {
		http.Error(w, "Failed to extract content", http.StatusInternalServerError)
		return
	}

	// 准备响应
	response := SummaryResponse{
		URL:      url,
		Title:    article.Title,
		Author:   article.Byline,
		Content:  article.TextContent,
		Length:   len(article.TextContent),
		Excerpt:  article.Excerpt,
		SiteName: article.SiteName,
	}

	// 返回JSON响应
	w.Header().Set("Content-Type", "application/json")
	json.NewEncoder(w).Encode(response)
}

func main() {
	http.HandleFunc("/summary", handleSummary)
	log.Println("Starting server on :8080")
	log.Fatal(http.ListenAndServe(":8080", nil))
}

性能优化建议

1. 缓存结果：对频繁请求的URL进行缓存
2. 并发控制：使用worker pool限制并发解析数量
3. 超时设置：为HTTP请求和解析过程设置合理的超时
4. 用户代理：设置合理的User-Agent减少被屏蔽的概率

// 带缓存的解析函数
var cache = make(map[string]*readability.Article)

func getCachedArticle(url string) (*readability.Article, error) {
	if article, ok := cache[url]; ok {
		return article, nil
	}

	// 获取并解析内容
	client := &http.Client{
		Timeout: 30 * time.Second,
		Transport: &http.Transport{
			DisableKeepAlives: true,
		},
	}
	
	req, _ := http.NewRequest("GET", url, nil)
	req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; MyBot/1.0)")
	
	resp, err := client.Do(req)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	article, err := readability.FromReader(resp.Body, url)
	if err != nil {
		return nil, err
	}

	// 缓存结果
	cache[url] = article
	return article, nil
}

goreadability是一个简单但功能强大的库，适合需要从网页中提取主要内容的Go应用程序。通过合理配置和错误处理，可以构建出稳定可靠的网页内容摘要服务。