Golang Go语言中 flate.NewWriter 和 os.(*File).readdir 内存占用奇高

func (a *Agent) readFileEvents(writer *io.PipeWriter) {
defer writer.Close()
w := bufio.NewWriter(writer)
defer w.Flush()
ioBuff := make([]byte, 131072)
log := a.logger.WithField(“component”, “readFileEvents”)
var buff bytes.Buffer
vw, err := bsonrw.NewBSONValueWriter(&buff)
if err != nil {
a.logger.WithFields(logrus.Fields{
“error”: err.Error(),
}).Error(“Failed to create bson value writer”)
return
}
encoder, err := bson.NewEncoder(vw)
if err != nil {
a.logger.WithFields(logrus.Fields{
“error”: err.Error(),
}).Error(“Failed to create bson encoder”)
return
}

var gzBuff bytes.Buffer
zw, _ := flate.NewWriter(&gzBuff, 3)
handleEvent := func(event *fsnotify.Event) {
    a.logger.Debug("handle event")
    gzBuff.Reset()
    zw.Reset(&gzBuff)
    var entry *EventInfo
    a.metrics.eventsCounter.WithLabelValues(event.Op.String()).Inc()
    log.WithFields(logrus.Fields{
        "filename": event.Name,
        "type":     event.Op.String(),
    }).Debug("event")

    defer buff.Reset()

    fileInfo, err := os.Stat(event.Name)
    if err != nil {
        log.WithFields(logrus.Fields{
            "filename": event.Name,
            "error":    err.Error(),
        }).Debug("Failed to get file info")
        return
    }
    if fileInfo.Mode()&fs.ModeSymlink == fs.ModeSymlink {
        return
    }

    result := a.fileFilter(event.Name)
    if !result {
        return
    }
    file, err := os.Open(event.Name)
    if err != nil {
        log.WithFields(logrus.Fields{
            "error":    err.Error(),
            "filename": event.Name,
        }).Error("Failed to open file")
        return
    }
    defer file.Close()

    entry = eventInfoPool.Get().(*EventInfo)
    defer eventInfoPool.Put(entry)
    entry.ClientId = a.uuid
    entry.FilePath = event.Name

    if _, err = io.CopyBuffer(zw, file, ioBuff); err != nil {
        a.logger.WithFields(logrus.Fields{
            "error":    err.Error(),
            "filename": event.Name,
        }).Error("Failed to read file")
    }
    if err := zw.Close(); err != nil {
        log.WithFields(logrus.Fields{
            "error": err.Error(),
        }).Error("Failed to compress file")
        return
    }
    entry.FileContent = gzBuff.Bytes()

    err = encoder.Encode(entry)
    if err != nil {
        a.logger.WithFields(logrus.Fields{
            "error":    err.Error(),
            "filepath": entry.FilePath,
        }).Error("Failed to serialize object")
        return
    }

    if _, err = w.Write(buff.Bytes()); err != nil {
        a.logger.WithFields(logrus.Fields{
            "error": err.Error(),
        }).Error("Failed to transfer data")
        return
    }

    a.metrics.eventsSentCounter.WithLabelValues(event.Op.String()).Inc()
    a.metrics.fileCounter.Inc()
}
for {
    select {
    case event, ok := <-a.watcherChan:
        if !ok {
            return
        }
        handleEvent(&event)
    case <-a.ctx.Done():
        return
    }
}
}

使用 pprof 发现 flate.NewWriter 方法消耗了大量内存。 这张图是 inuse_space：

这张图是 alloc_space：

这张是 ReadDir：

顺便吐槽下 os.(*File).ReadDir 内存占用也很离谱。 有没有大佬提供下优化思路 qaq