Golang上传~10MB文件时出现"The connection was reset"错误如何避免将文件读取到内存中

这个错误通常是由于服务器端读取超时或请求体大小限制导致的。以下是解决方案：

package main

import (
    "fmt"
    "io"
    "net/http"
    "os"
    "time"
)

func main() {
    http.HandleFunc("/upload", uploadHandler)
    
    server := &http.Server{
        Addr: ":8080",
        // 增加读取超时时间
        ReadTimeout: 30 * time.Minute,
        // 增加写入超时时间
        WriteTimeout: 30 * time.Minute,
        // 禁用请求体大小限制
        MaxHeaderBytes: 1 << 20, // 1MB
    }
    
    // 创建上传目录
    os.MkdirAll("uploads", 0755)
    
    fmt.Println("Server starting on :8080")
    server.ListenAndServe()
}

func uploadHandler(w http.ResponseWriter, r *http.Request) {
    // 设置内存缓冲区大小（可选，用于小文件）
    // 这里设置为0表示直接写入磁盘，不经过内存缓冲区
    err := r.ParseMultipartForm(0)
    if err != nil {
        http.Error(w, "Failed to parse multipart form", http.StatusBadRequest)
        return
    }
    
    // 获取文件
    file, header, err := r.FormFile("file")
    if err != nil {
        http.Error(w, "Failed to get file from form", http.StatusBadRequest)
        return
    }
    defer file.Close()
    
    // 创建目标文件
    dst, err := os.Create("uploads/" + header.Filename)
    if err != nil {
        http.Error(w, "Failed to create destination file", http.StatusInternalServerError)
        return
    }
    defer dst.Close()
    
    // 使用io.Copy流式传输，避免将整个文件读入内存
    _, err = io.Copy(dst, file)
    if err != nil {
        http.Error(w, "Failed to save file", http.StatusInternalServerError)
        return
    }
    
    w.WriteHeader(http.StatusOK)
    fmt.Fprintf(w, "File uploaded successfully: %s", header.Filename)
}

或者使用更细粒度的控制：

package main

import (
    "fmt"
    "io"
    "net/http"
    "os"
    "time"
)

func main() {
    http.HandleFunc("/upload", streamingUploadHandler)
    
    server := &http.Server{
        Addr: ":8080",
        ReadTimeout: 30 * time.Minute,
        WriteTimeout: 30 * time.Minute,
    }
    
    os.MkdirAll("uploads", 0755)
    fmt.Println("Server starting on :8080")
    server.ListenAndServe()
}

func streamingUploadHandler(w http.ResponseWriter, r *http.Request) {
    // 手动解析multipart/form-data
    reader, err := r.MultipartReader()
    if err != nil {
        http.Error(w, "Failed to create multipart reader", http.StatusBadRequest)
        return
    }
    
    // 遍历multipart的各个部分
    for {
        part, err := reader.NextPart()
        if err == io.EOF {
            break
        }
        if err != nil {
            http.Error(w, "Error reading multipart section", http.StatusBadRequest)
            return
        }
        
        // 检查是否是文件字段
        if part.FileName() != "" {
            // 创建目标文件
            dst, err := os.Create("uploads/" + part.FileName())
            if err != nil {
                http.Error(w, "Failed to create file", http.StatusInternalServerError)
                return
            }
            defer dst.Close()
            
            // 流式复制文件内容
            _, err = io.Copy(dst, part)
            if err != nil {
                http.Error(w, "Failed to save file", http.StatusInternalServerError)
                return
            }
            
            part.Close()
            w.WriteHeader(http.StatusOK)
            fmt.Fprintf(w, "File uploaded successfully: %s", part.FileName())
            return
        }
        
        part.Close()
    }
    
    http.Error(w, "No file found in request", http.StatusBadRequest)
}

对于生产环境，建议添加以下配置：

package main

import (
    "fmt"
    "io"
    "net/http"
    "os"
    "time"
)

func main() {
    // 使用自定义的handler来处理大文件上传
    http.Handle("/upload", &UploadHandler{
        MaxUploadSize: 100 * 1024 * 1024, // 100MB
        UploadDir:     "uploads",
    })
    
    server := &http.Server{
        Addr:         ":8080",
        ReadTimeout:  60 * time.Minute,
        WriteTimeout: 60 * time.Minute,
        IdleTimeout:  120 * time.Second,
    }
    
    os.MkdirAll("uploads", 0755)
    fmt.Println("Server starting on :8080")
    server.ListenAndServe()
}

type UploadHandler struct {
    MaxUploadSize int64
    UploadDir     string
}

func (h *UploadHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
    if r.Method != "POST" {
        http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
        return
    }
    
    // 限制请求体大小
    r.Body = http.MaxBytesReader(w, r.Body, h.MaxUploadSize)
    
    // 使用MultipartReader进行流式处理
    reader, err := r.MultipartReader()
    if err != nil {
        http.Error(w, "Failed to create multipart reader", http.StatusBadRequest)
        return
    }
    
    for {
        part, err := reader.NextPart()
        if err == io.EOF {
            break
        }
        if err != nil {
            http.Error(w, "Error reading part", http.StatusBadRequest)
            return
        }
        
        if part.FileName() != "" {
            filePath := h.UploadDir + "/" + part.FileName()
            dst, err := os.Create(filePath)
            if err != nil {
                http.Error(w, "Failed to create file", http.StatusInternalServerError)
                return
            }
            defer dst.Close()
            
            // 使用32KB缓冲区进行流式复制
            buf := make([]byte, 32*1024)
            _, err = io.CopyBuffer(dst, part, buf)
            if err != nil {
                http.Error(w, "Failed to save file", http.StatusInternalServerError)
                return
            }
            
            part.Close()
            w.WriteHeader(http.StatusOK)
            fmt.Fprintf(w, "File uploaded successfully: %s", part.FileName())
            return
        }
        
        part.Close()
    }
    
    http.Error(w, "No file found", http.StatusBadRequest)
}

关键点：

1. 增加服务器超时时间（ReadTimeout 和 WriteTimeout）
2. 使用 r.ParseMultipartForm(0) 或 r.MultipartReader() 避免内存缓冲
3. 使用 io.Copy 进行流式文件传输
4. 对于超大文件，考虑使用分块上传或断点续传方案