使用Golang实现Unicode文本分段处理

Unicode分段处理是根据Unicode标准附件#29(Unicode Standard Annex #29)定义的规则将文本分割为有意义的单元(如字素、单词、句子等)。在Go中，我们可以使用标准库和第三方库来实现这些功能。

1. 使用标准库实现基本分段

Go的标准库unicode和unicode/utf8提供了一些基础功能：

package main

import (
	"fmt"
	"unicode"
	"unicode/utf8"
)

// 按Unicode字素簇分割文本
func SegmentGraphemes(text string) []string {
	var segments []string
	var currentCluster []rune

	for _, r := range text {
		if len(currentCluster) == 0 {
			currentCluster = append(currentCluster, r)
			continue
		}

		// 检查是否属于同一字素簇
		if isSameGraphemeCluster(currentCluster[len(currentCluster)-1], r) {
			currentCluster = append(currentCluster, r)
		} else {
			segments = append(segments, string(currentCluster))
			currentCluster = []rune{r}
		}
	}

	if len(currentCluster) > 0 {
		segments = append(segments, string(currentCluster))
	}

	return segments
}

func isSameGraphemeCluster(prev, curr rune) bool {
	// 简单实现 - 实际UAX#29规则更复杂
	return unicode.IsMark(curr)
}

func main() {
	text := "Hello, 世界! 👋🏽"
	segments := SegmentGraphemes(text)
	for i, seg := range segments {
		fmt.Printf("Segment %d: %q (length %d)\n", i, seg, utf8.RuneCountInString(seg))
	}
}

2. 使用第三方库

更完整的实现可以使用第三方库如github.com/rivo/uniseg，它完全实现了UAX#29标准：

package main

import (
	"fmt"
	"github.com/rivo/uniseg"
)

func main() {
	text := "Hello, 世界! 👋🏽"

	// 字素簇分割
	fmt.Println("Grapheme clusters:")
	gr := uniseg.NewGraphemes(text)
	for gr.Next() {
		fmt.Printf("%q ", gr.Str())
	}
	fmt.Println()

	// 单词分割
	fmt.Println("\nWord boundaries:")
	state := -1
	var words []string
	var currentWord []rune
	for i, r := range text {
		if i == 0 {
			state, _ = uniseg.WordBreak(r)
		}
		newState, boundary := uniseg.WordBreak(r)
		if boundary {
			if len(currentWord) > 0 {
				words = append(words, string(currentWord))
				currentWord = nil
			}
		}
		currentWord = append(currentWord, r)
		state = newState
	}
	if len(currentWord) > 0 {
		words = append(words, string(currentWord))
	}
	for i, word := range words {
		fmt.Printf("Word %d: %q\n", i, word)
	}

	// 句子分割
	fmt.Println("\nSentence boundaries:")
	state = -1
	var sentences []string
	var currentSentence []rune
	for i, r := range text {
		if i == 0 {
			state, _ = uniseg.SentenceBreak(r)
		}
		newState, boundary := uniseg.SentenceBreak(r)
		if boundary {
			if len(currentSentence) > 0 {
				sentences = append(sentences, string(currentSentence))
				currentSentence = nil
			}
		}
		currentSentence = append(currentSentence, r)
		state = newState
	}
	if len(currentSentence) > 0 {
		sentences = append(sentences, string(currentSentence))
	}
	for i, sentence := range sentences {
		fmt.Printf("Sentence %d: %q\n", i, sentence)
	}
}

3. 实现自定义分段插件库

下面是一个简单的插件式分段库设计：

package segment

import (
	"unicode"
)

// Segmenter 定义分段接口
type Segmenter interface {
	Segment(text string) []string
}

// GraphemeSegmenter 实现字素簇分割
type GraphemeSegmenter struct{}

func (gs *GraphemeSegmenter) Segment(text string) []string {
	var segments []string
	var current []rune

	for _, r := range text {
		if len(current) == 0 {
			current = append(current, r)
			continue
		}

		last := current[len(current)-1]
		if isSameGrapheme(last, r) {
			current = append(current, r)
		} else {
			segments = append(segments, string(current))
			current = []rune{r}
		}
	}

	if len(current) > 0 {
		segments = append(segments, string(current))
	}

	return segments
}

func isSameGrapheme(prev, curr rune) bool {
	// 简化的字素簇判断逻辑
	if unicode.IsMark(curr) {
		return true
	}
	// 处理emoji变体选择器
	if prev == 0xFE0F && curr == 0xFE0F {
		return true
	}
	return false
}

// WordSegmenter 实现单词分割
type WordSegmenter struct{}

func (ws *WordSegmenter) Segment(text string) []string {
	var words []string
	var current []rune
	var prevIsLetter bool

	for _, r := range text {
		isLetter := unicode.IsLetter(r)
		if isLetter {
			if !prevIsLetter && len(current) > 0 {
				words = append(words, string(current))
				current = nil
			}
			current = append(current, r)
		} else {
			if len(current) > 0 {
				words = append(words, string(current))
				current = nil
			}
			if !unicode.IsSpace(r) {
				current = append(current, r)
			}
		}
		prevIsLetter = isLetter
	}

	if len(current) > 0 {
		words = append(words, string(current))
	}

	return words
}

// NewSegmenter 创建分段器
func NewSegmenter(segmentType string) Segmenter {
	switch segmentType {
	case "grapheme":
		return &GraphemeSegmenter{}
	case "word":
		return &WordSegmenter{}
	default:
		return &GraphemeSegmenter{}
	}
}

使用示例：

package main

import (
	"fmt"
	"yourmodule/segment"
)

func main() {
	text := "Hello, 世界! 👋🏽"

	// 使用字素分段器
	graphemeSegmenter := segment.NewSegmenter("grapheme")
	graphemes := graphemeSegmenter.Segment(text)
	fmt.Println("Graphemes:", graphemes)

	// 使用单词分段器
	wordSegmenter := segment.NewSegmenter("word")
	words := wordSegmenter.Segment(text)
	fmt.Println("Words:", words)
}

4. 性能考虑

处理大量文本时，需要注意：

1. 避免频繁的内存分配
2. 使用[]rune而不是频繁的字符串操作
3. 考虑并行处理大文本

func ParallelSegment(text string, segmenter Segmenter, workers int) []string {
	// 将文本分割为块
	chunks := splitText(text, workers)
	
	results := make(chan []string, workers)
	for _, chunk := range chunks {
		go func(chunk string) {
			results <- segmenter.Segment(chunk)
		}(chunk)
	}
	
	// 合并结果
	var allSegments []string
	for i := 0; i < workers; i++ {
		allSegments = append(allSegments, <-results...)
	}
	
	return allSegments
}

总结

Go语言提供了良好的Unicode支持，结合第三方库可以完全实现UAX#29标准。对于生产环境，推荐使用成熟的第三方库如uniseg，它们经过了充分测试并完整实现了标准。自定义实现时，需要注意处理各种边界情况和组合字符。