使用Go高效读写Parquet文件 - parquet-go库详解

Parquet是一种高效的列式存储格式，特别适合大数据处理场景。在Go生态中，parquet-go是一个高性能的Parquet文件读写库。

安装parquet-go

go get github.com/xitongsys/parquet-go
go get github.com/xitongsys/parquet-go-source

基本使用示例

1. 写入Parquet文件

package main

import (
	"log"
	"os"
	"time"

	"github.com/xitongsys/parquet-go-source/local"
	"github.com/xitongsys/parquet-go/parquet"
	"github.com/xitongsys/parquet-go/writer"
)

type Student struct {
	Name    string  `parquet:"name=name, type=UTF8"`
	Age     int32   `parquet:"name=age, type=INT32"`
	Id      int64   `parquet:"name=id, type=INT64"`
	Weight  float32 `parquet:"name=weight, type=FLOAT"`
	Sex     bool    `parquet:"name=sex, type=BOOLEAN"`
	Day     int32   `parquet:"name=day, type=DATE"`
	Ignored int32   // 没有parquet标签的字段会被忽略
}

func main() {
	// 创建本地文件写入器
	fw, err := local.NewLocalFileWriter("output.parquet")
	if err != nil {
		log.Println("Can't create local file", err)
		return
	}
	defer fw.Close()

	// 创建Parquet writer
	pw, err := writer.NewParquetWriter(fw, new(Student), 4)
	if err != nil {
		log.Println("Can't create parquet writer", err)
		return
	}

	// 设置Parquet文件属性
	pw.RowGroupSize = 128 * 1024 * 1024 // 128M
	pw.CompressionType = parquet.CompressionCodec_SNAPPY

	// 写入数据
	students := []Student{
		{Name: "张三", Age: 23, Id: 1, Weight: 60.5, Sex: true, Day: int32(time.Now().Unix() / 3600 / 24)},
		{Name: "李四", Age: 24, Id: 2, Weight: 70.5, Sex: false, Day: int32(time.Now().Unix()/3600/24) + 1},
		{Name: "王五", Age: 25, Id: 3, Weight: 80.5, Sex: true, Day: int32(time.Now().Unix()/3600/24) + 2},
	}

	for _, student := range students {
		if err = pw.Write(student); err != nil {
			log.Println("Write error", err)
		}
	}

	// 停止写入并关闭
	if err = pw.WriteStop(); err != nil {
		log.Println("WriteStop error", err)
		return
	}

	log.Println("Write Finished")
}

2. 读取Parquet文件

package main

import (
	"log"

	"github.com/xitongsys/parquet-go-source/local"
	"github.com/xitongsys/parquet-go/reader"
)

type Student struct {
	Name   string  `parquet:"name=name, type=UTF8"`
	Age    int32   `parquet:"name=age, type=INT32"`
	Id     int64   `parquet:"name=id, type=INT64"`
	Weight float32 `parquet:"name=weight, type=FLOAT"`
	Sex    bool    `parquet:"name=sex, type=BOOLEAN"`
	Day    int32   `parquet:"name=day, type=DATE"`
}

func main() {
	// 打开Parquet文件
	fr, err := local.NewLocalFileReader("output.parquet")
	if err != nil {
		log.Println("Can't open file", err)
		return
	}
	defer fr.Close()

	// 创建Parquet reader
	pr, err := reader.NewParquetReader(fr, new(Student), 4)
	if err != nil {
		log.Println("Can't create parquet reader", err)
		return
	}
	defer pr.ReadStop()

	// 获取行数
	num := int(pr.GetNumRows())
	log.Printf("Total rows: %d\n", num)

	// 读取数据
	students := make([]Student, num)
	if err = pr.Read(&students); err != nil {
		log.Println("Read error", err)
		return
	}

	// 打印数据
	for i, student := range students {
		log.Printf("Row %d: %+v\n", i, student)
	}
}

高级特性

1. 自定义Schema

// 定义Schema
schema := `message test {
    required binary name (UTF8);
    required int32 age;
    required int64 id;
    required float weight;
    required boolean sex;
    required int32 day (DATE);
}`

// 使用Schema创建writer
pw, err := writer.NewParquetWriterFromWriter(fw, schema, 4)

2. 性能优化

// 增加并行度
pr, err := reader.NewParquetReader(fr, new(Student), 8) // 使用8个goroutine

// 预分配内存
students := make([]Student, 0, num) // 预分配足够容量

3. 读取部分列

// 只读取name和age列
pr.ReadPartial(&students, "name", "age")

最佳实践

1. 批量写入：单次写入多条记录比多次写入单条记录更高效
2. 合理设置RowGroup大小：通常128MB-1GB之间
3. 选择合适的压缩算法：SNAPPY在CPU和压缩率间有较好平衡
4. 列裁剪：读取时只选择需要的列
5. 类型匹配：确保Go结构体类型与Parquet类型匹配

与其他库比较

parquet-go相比其他Go Parquet库(如parquet-go、goparquet)的优势：

• 性能更高
• 支持更完整的Parquet特性
• 更活跃的维护
• 更好的文档和示例

希望这个指南能帮助你在Go中高效地处理Parquet文件！