Golang中如何实现高效低成本的去重消息ID生成器

在Golang中实现高效低成本的去重ID生成器，可以考虑以下几种方案：

1. 基于时间戳的ID生成器

package main

import (
    "fmt"
    "sync/atomic"
    "time"
)

type TimestampIDGenerator struct {
    lastTimestamp int64
    sequence      int64
}

func NewTimestampIDGenerator() *TimestampIDGenerator {
    return &TimestampIDGenerator{
        lastTimestamp: time.Now().UnixNano(),
        sequence:      0,
    }
}

func (g *TimestampIDGenerator) Generate() string {
    now := time.Now().UnixNano()
    seq := atomic.AddInt64(&g.sequence, 1)
    
    // 如果时间戳相同，使用序列号区分
    if now == atomic.LoadInt64(&g.lastTimestamp) {
        return fmt.Sprintf("%d-%d", now, seq)
    }
    
    atomic.StoreInt64(&g.lastTimestamp, now)
    atomic.StoreInt64(&g.sequence, 0)
    return fmt.Sprintf("%d", now)
}

2. 基于Snowflake算法的ID生成器

package main

import (
    "fmt"
    "sync"
    "time"
)

const (
    workerIDBits     = 5
    sequenceBits     = 12
    maxWorkerID      = -1 ^ (-1 << workerIDBits)
    maxSequence      = -1 ^ (-1 << sequenceBits)
    timeShift        = workerIDBits + sequenceBits
    workerShift      = sequenceBits
    epoch            = 1609459200000 // 2021-01-01 00:00:00 UTC
)

type SnowflakeIDGenerator struct {
    mu        sync.Mutex
    workerID  int64
    sequence  int64
    lastStamp int64
}

func NewSnowflakeIDGenerator(workerID int64) *SnowflakeIDGenerator {
    if workerID > maxWorkerID || workerID < 0 {
        panic(fmt.Sprintf("worker ID must be between 0 and %d", maxWorkerID))
    }
    return &SnowflakeIDGenerator{
        workerID: workerID,
    }
}

func (g *SnowflakeIDGenerator) Generate() string {
    g.mu.Lock()
    defer g.mu.Unlock()

    now := time.Now().UnixMilli()
    
    if now == g.lastStamp {
        g.sequence = (g.sequence + 1) & maxSequence
        if g.sequence == 0 {
            for now <= g.lastStamp {
                now = time.Now().UnixMilli()
            }
        }
    } else {
        g.sequence = 0
    }

    g.lastStamp = now
    
    id := ((now - epoch) << timeShift) |
        (g.workerID << workerShift) |
        g.sequence
    
    return fmt.Sprintf("%d", id)
}

3. 基于消息内容哈希的ID生成器

package main

import (
    "crypto/sha256"
    "encoding/hex"
    "fmt"
    "hash/fnv"
)

// 使用FNV哈希（更快但可能冲突）
func GenerateDedupIDByHash(message string) string {
    h := fnv.New64a()
    h.Write([]byte(message))
    return fmt.Sprintf("%x", h.Sum64())
}

// 使用SHA256哈希（更安全但稍慢）
func GenerateDedupIDBySHA256(message string) string {
    hash := sha256.Sum256([]byte(message))
    return hex.EncodeToString(hash[:16]) // 取前16字节
}

// 结合时间戳和内容哈希
func GenerateHybridDedupID(message string) string {
    timestamp := time.Now().UnixNano()
    h := fnv.New64a()
    h.Write([]byte(message))
    contentHash := h.Sum64()
    return fmt.Sprintf("%d-%x", timestamp, contentHash)
}

4. 使用递增序列的ID生成器

package main

import (
    "fmt"
    "sync/atomic"
)

type SequentialIDGenerator struct {
    prefix string
    counter uint64
}

func NewSequentialIDGenerator(prefix string) *SequentialIDGenerator {
    return &SequentialIDGenerator{
        prefix: prefix,
        counter: 0,
    }
}

func (g *SequentialIDGenerator) Generate() string {
    id := atomic.AddUint64(&g.counter, 1)
    return fmt.Sprintf("%s-%d", g.prefix, id)
}

// 带时间窗口的序列生成器
func (g *SequentialIDGenerator) GenerateWithTimestamp() string {
    timestamp := time.Now().Unix()
    id := atomic.AddUint64(&g.counter, 1)
    return fmt.Sprintf("%d-%s-%d", timestamp, g.prefix, id)
}

5. 实际使用示例

package main

import (
    "fmt"
    "time"
)

func main() {
    // 示例1: 使用时间戳生成器
    tsGen := NewTimestampIDGenerator()
    fmt.Println("Timestamp ID:", tsGen.Generate())
    
    // 示例2: 使用Snowflake生成器
    snowGen := NewSnowflakeIDGenerator(1)
    fmt.Println("Snowflake ID:", snowGen.Generate())
    
    // 示例3: 基于消息内容生成去重ID
    message := "192.168.1.100 - - [28/Oct/2021:10:27:10 -0500] GET /index.html HTTP/1.1 200 3395"
    dedupID := GenerateHybridDedupID(message)
    fmt.Println("Hybrid Dedup ID:", dedupID)
    
    // 示例4: 序列生成器
    seqGen := NewSequentialIDGenerator("msg")
    fmt.Println("Sequential ID:", seqGen.Generate())
}

性能对比

package main

import (
    "testing"
    "github.com/google/uuid"
)

func BenchmarkUUID(b *testing.B) {
    for i := 0; i < b.N; i++ {
        _ = uuid.New().String()
    }
}

func BenchmarkTimestampID(b *testing.B) {
    gen := NewTimestampIDGenerator()
    b.ResetTimer()
    for i := 0; i < b.N; i++ {
        _ = gen.Generate()
    }
}

func BenchmarkSnowflakeID(b *testing.B) {
    gen := NewSnowflakeIDGenerator(1)
    b.ResetTimer()
    for i := 0; i < b.N; i++ {
        _ = gen.Generate()
    }
}

这些方案都比UUID生成更高效：

• 时间戳方案：简单快速，适合单机场景
• Snowflake方案：分布式友好，保证全局唯一
• 哈希方案：适合基于消息内容去重
• 序列方案：最高性能，适合高吞吐场景

选择哪种方案取决于你的具体需求：是否需要分布式支持、是否需要基于内容去重、以及对性能的要求程度。