Golang中各种配置的规范和最佳实践

在Go中处理配置的惯用方式是使用结构体来封装配置选项，并通过函数选项模式（Functional Options Pattern）提供灵活的配置方式。对于你的爬虫客户端，可以这样重构：

package client

import (
	"bufio"
	"fmt"
	"log"
	"math/rand"
	"os"
	"time"

	"github.com/gocolly/colly"
)

type Config struct {
	MaxDepth      int
	RandomDelay   time.Duration
	Parallelism   int
	AllowedDomains []string
	DomainGlob     []string
	UserAgentFile  string
}

type Option func(*Config)

func WithMaxDepth(depth int) Option {
	return func(c *Config) {
		c.MaxDepth = depth
	}
}

func WithRateLimit(delay time.Duration, parallelism int) Option {
	return func(c *Config) {
		c.RandomDelay = delay
		c.Parallelism = parallelism
	}
}

func WithAllowedDomains(domains ...string) Option {
	return func(c *Config) {
		c.AllowedDomains = domains
	}
}

func WithDomainGlob(globs ...string) Option {
	return func(c *Config) {
		c.DomainGlob = globs
	}
}

func WithUserAgentFile(file string) Option {
	return func(c *Config) {
		c.UserAgentFile = file
	}
}

func NewCollector(opts ...Option) *colly.Collector {
	rand.Seed(time.Now().UnixNano())

	// 默认配置
	config := &Config{
		MaxDepth:     2,
		RandomDelay:  10 * time.Second,
		Parallelism:  2,
		UserAgentFile: "client/agents.txt",
	}

	// 应用选项
	for _, opt := range opts {
		opt(config)
	}

	// 读取User-Agent
	userAgents := readUserAgents(config.UserAgentFile)

	// 创建Collector
	collectorOpts := []colly.CollectorOption{
		colly.MaxDepth(config.MaxDepth),
	}

	if len(config.AllowedDomains) > 0 {
		collectorOpts = append(collectorOpts, colly.AllowedDomains(config.AllowedDomains...))
	}

	if len(config.DomainGlob) > 0 {
		collectorOpts = append(collectorOpts, colly.DomainGlob(config.DomainGlob...))
	}

	c := colly.NewCollector(collectorOpts...)

	c.Limit(&colly.LimitRule{
		RandomDelay: config.RandomDelay,
		Parallelism: config.Parallelism,
	})

	c.OnRequest(func(r *colly.Request) {
		if len(userAgents) > 0 {
			r.Headers.Set("User-Agent", userAgents[rand.Intn(len(userAgents))])
		}
	})

	c.OnError(func(_ *colly.Response, err error) {
		log.Printf("Request error: %v", err)
	})

	c.OnScraped(func(r *colly.Response) {
		log.Printf("Done %s", r.Request.URL)
	})

	return c
}

func readUserAgents(filename string) []string {
	file, err := os.Open(filename)
	if err != nil {
		log.Printf("Warning: cannot open user-agent file: %v", err)
		return []string{}
	}
	defer file.Close()

	var agents []string
	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		agents = append(agents, scanner.Text())
	}

	if err := scanner.Err(); err != nil {
		log.Printf("Warning: error reading user-agent file: %v", err)
	}

	return agents
}

使用示例：

package main

import (
	"time"
	
	"yourproject/client"
)

func main() {
	// 使用默认配置
	defaultCollector := client.NewCollector()
	
	// 自定义配置
	customCollector := client.NewCollector(
		client.WithMaxDepth(3),
		client.WithRateLimit(5*time.Second, 4),
		client.WithAllowedDomains("example.com", "test.com"),
		client.WithDomainGlob("*.example.com"),
		client.WithUserAgentFile("custom_agents.txt"),
	)
	
	// 使用collector进行爬取
	_ = defaultCollector
	_ = customCollector
}

对于跨项目的通用配置，可以创建独立的配置包：

package config

import "time"

type CrawlerConfig struct {
	MaxDepth      int
	RandomDelay   time.Duration
	Parallelism   int
	AllowedDomains []string
	DomainGlob     []string
	UserAgentFile  string
	Timeout        time.Duration
	CacheDir       string
	Debug          bool
}

func NewDefaultConfig() *CrawlerConfig {
	return &CrawlerConfig{
		MaxDepth:      2,
		RandomDelay:   10 * time.Second,
		Parallelism:   2,
		UserAgentFile: "agents.txt",
		Timeout:       30 * time.Second,
		CacheDir:      ".cache",
		Debug:         false,
	}
}

然后在客户端包中使用：

package client

import (
	"yourproject/config"
	// ... 其他导入
)

func NewCollectorFromConfig(cfg *config.CrawlerConfig) *colly.Collector {
	// 基于配置创建collector
	// ...
}

这种模式提供了类型安全的配置方式，良好的可测试性，并且保持了API的简洁性。