Golang中HTTP客户端的并发性能影响

在Golang中，HTTP客户端的并发性能主要受net/http包中默认的DefaultTransport配置影响。案例2性能下降的核心原因是连接复用限制。

问题分析

http.DefaultClient使用http.DefaultTransport，其关键配置如下：

MaxIdleConns:          100,
MaxIdleConnsPerHost:   2,  // 关键限制
MaxConnsPerHost:       0,
IdleConnTimeout:       90 * time.Second,

根本原因

1. 连接池竞争：案例2中每个请求都创建goroutine调用服务B，导致大量并发HTTP请求
2. MaxIdleConnsPerHost=2：每个主机最多只能保持2个空闲连接
3. 连接建立开销：超过限制后需要频繁创建新TCP连接

解决方案

方案1：优化Transport配置

var customTransport = &http.Transport{
    MaxIdleConns:        100,
    MaxIdleConnsPerHost: 50,  // 提高每个主机的空闲连接数
    MaxConnsPerHost:     100, // 限制每个主机的总连接数
    IdleConnTimeout:     90 * time.Second,
    TLSHandshakeTimeout: 10 * time.Second,
}

var customClient = &http.Client{
    Transport: customTransport,
    Timeout:   30 * time.Second,
}

func httpGetB() {
    resp, err := customClient.Get("http://service-b:8080/")
    if err != nil {
        log.Printf("Error: %v", err)
        return
    }
    defer resp.Body.Close()
}

方案2：使用连接池和工作池

type WorkerPool struct {
    workers int
    tasks   chan func()
}

func NewWorkerPool(workers int) *WorkerPool {
    wp := &WorkerPool{
        workers: workers,
        tasks:   make(chan func(), 1000),
    }
    for i := 0; i < workers; i++ {
        go wp.worker()
    }
    return wp
}

func (wp *WorkerPool) worker() {
    for task := range wp.tasks {
        task()
    }
}

func (wp *WorkerPool) Submit(task func()) {
    select {
    case wp.tasks <- task:
    default:
        log.Println("Worker pool full, dropping task")
    }
}

// 全局工作池
var pool = NewWorkerPool(100)

func withGoroutineOptimized(w http.ResponseWriter, r *http.Request) {
    httpGetA()
    
    w.Write([]byte(""))
    
    pool.Submit(func() {
        httpGetB()
    })
}

方案3：批量处理异步请求

type BatchProcessor struct {
    batchSize int
    buffer    chan *http.Request
    client    *http.Client
}

func NewBatchProcessor(batchSize int) *BatchProcessor {
    bp := &BatchProcessor{
        batchSize: batchSize,
        buffer:    make(chan *http.Request, 10000),
        client:    customClient,
    }
    go bp.process()
    return bp
}

func (bp *BatchProcessor) process() {
    var batch []*http.Request
    ticker := time.NewTicker(100 * time.Millisecond)
    
    for {
        select {
        case req := <-bp.buffer:
            batch = append(batch, req)
            if len(batch) >= bp.batchSize {
                bp.flush(batch)
                batch = nil
            }
        case <-ticker.C:
            if len(batch) > 0 {
                bp.flush(batch)
                batch = nil
            }
        }
    }
}

func (bp *BatchProcessor) flush(batch []*http.Request) {
    var wg sync.WaitGroup
    for _, req := range batch {
        wg.Add(1)
        go func(r *http.Request) {
            defer wg.Done()
            bp.client.Do(r)
        }(req)
    }
    wg.Wait()
}

性能对比测试

func benchmarkTest() {
    // 测试不同配置的性能
    tests := []struct {
        name      string
        transport *http.Transport
    }{
        {
            name: "Default",
            transport: &http.Transport{
                MaxIdleConnsPerHost: 2,
            },
        },
        {
            name: "Optimized",
            transport: &http.Transport{
                MaxIdleConns:        100,
                MaxIdleConnsPerHost: 50,
                MaxConnsPerHost:     100,
            },
        },
    }
    
    for _, test := range tests {
        client := &http.Client{
            Transport: test.transport,
            Timeout:   30 * time.Second,
        }
        
        start := time.Now()
        var wg sync.WaitGroup
        for i := 0; i < 1000; i++ {
            wg.Add(1)
            go func() {
                defer wg.Done()
                resp, _ := client.Get("http://service-b:8080/")
                if resp != nil {
                    resp.Body.Close()
                }
            }()
        }
        wg.Wait()
        fmt.Printf("%s: %v\n", test.name, time.Since(start))
    }
}

监控指标

func monitorConnections() {
    go func() {
        for {
            time.Sleep(5 * time.Second)
            stats := customTransport.(*http.Transport)
            fmt.Printf("Idle connections: %d\n", len(stats.IdleConn))
        }
    }()
}

关键点：

1. 默认的MaxIdleConnsPerHost=2是主要瓶颈
2. 大量goroutine创建HTTP请求会导致连接竞争
3. 优化Transport配置可以显著提升并发性能
4. 使用工作池可以控制并发度，避免资源耗尽

建议根据实际负载调整MaxIdleConnsPerHost和MaxConnsPerHost的值，并通过监控确认最佳配置。