Golang实现S3存储操作及s3-proxy使用指南

S3基本操作实现

以下是使用Golang实现S3存储的GET/PUT/DELETE操作的示例代码：

package main

import (
	"context"
	"fmt"
	"log"
	"os"

	"github.com/aws/aws-sdk-go-v2/aws"
	"github.com/aws/aws-sdk-go-v2/config"
	"github.com/aws/aws-sdk-go-v2/service/s3"
)

// S3Client 封装S3操作
type S3Client struct {
	client *s3.Client
	bucket string
}

// NewS3Client 创建S3客户端
func NewS3Client(endpoint, region, bucket, accessKey, secretKey string) (*S3Client, error) {
	customResolver := aws.EndpointResolverWithOptionsFunc(func(service, region string, options ...interface{}) (aws.Endpoint, error) {
		if endpoint != "" {
			return aws.Endpoint{
				URL:           endpoint,
				SigningRegion: region,
			}, nil
		}
		return aws.Endpoint{}, &aws.EndpointNotFoundError{}
	})

	cfg, err := config.LoadDefaultConfig(context.TODO(),
		config.WithRegion(region),
		config.WithEndpointResolverWithOptions(customResolver),
		config.WithCredentialsProvider(aws.CredentialsProviderFunc(func(ctx context.Context) (aws.Credentials, error) {
			return aws.Credentials{
				AccessKeyID:     accessKey,
				SecretAccessKey: secretKey,
			}, nil
		})),
		),
	)
	if err != nil {
		return nil, fmt.Errorf("failed to load AWS config: %v", err)
	}

	client := s3.NewFromConfig(cfg)
	return &S3Client{client: client, bucket: bucket}, nil
}

// PutObject 上传文件到S3
func (s *S3Client) PutObject(key string, filePath string) error {
	file, err := os.Open(filePath)
	if err != nil {
		return fmt.Errorf("failed to open file: %v", err)
	}
	defer file.Close()

	_, err = s.client.PutObject(context.TODO(), &s3.PutObjectInput{
		Bucket: aws.String(s.bucket),
		Key:    aws.String(key),
		Body:   file,
	})
	if err != nil {
		return fmt.Errorf("failed to put object: %v", err)
	}

	return nil
}

// GetObject 从S3下载文件
func (s *S3Client) GetObject(key string, filePath string) error {
	result, err := s.client.GetObject(context.TODO(), &s3.GetObjectInput{
		Bucket: aws.String(s.bucket),
		Key:    aws.String(key),
	})
	if err != nil {
		return fmt.Errorf("failed to get object: %v", err)
	}
	defer result.Body.Close()

	file, err := os.Create(filePath)
	if err != nil {
		return fmt.Errorf("failed to create file: %v", err)
	}
	defer file.Close()

	_, err = io.Copy(file, result.Body)
	if err != nil {
		return fmt.Errorf("failed to write file: %v", err)
	}

	return nil
}

// DeleteObject 从S3删除文件
func (s *S3Client) DeleteObject(key string) error {
	_, err := s.client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
		Bucket: aws.String(s.bucket),
		Key:    aws.String(key),
	})
	if err != nil {
		return fmt.Errorf("failed to delete object: %v", err)
	}

	return nil
}

func main() {
	// 初始化S3客户端
	client, err := NewS3Client(
		"https://s3.example.com", // S3端点
		"us-east-1",             // 区域
		"my-bucket",             // 存储桶名称
		"ACCESS_KEY",            // 访问密钥
		"SECRET_KEY",            // 秘密密钥
	)
	if err != nil {
		log.Fatal(err)
	}

	// 上传文件
	err = client.PutObject("test.txt", "./test.txt")
	if err != nil {
		log.Fatal(err)
	}
	fmt.Println("File uploaded successfully")

	// 下载文件
	err = client.GetObject("test.txt", "./downloaded.txt")
	if err != nil {
		log.Fatal(err)
	}
	fmt.Println("File downloaded successfully")

	// 删除文件
	err = client.DeleteObject("test.txt")
	if err != nil {
		log.Fatal(err)
	}
	fmt.Println("File deleted successfully")
}

s3-proxy使用指南

s3-proxy是一个开源的S3代理服务器，支持OpenID和Basic认证。以下是使用s3-proxy的步骤：

1. 安装s3-proxy

# 使用Docker安装
docker pull ghcr.io/oxyno-zeta/s3-proxy:latest

# 或者从源码编译
git clone https://github.com/oxyno-zeta/s3-proxy.git
cd s3-proxy
make build

2. 配置s3-proxy

创建配置文件config.yaml:

# 监听配置
listen:
  address: "0.0.0.0"
  port: 8080

# S3目标配置
target:
  bucket: "my-bucket"
  endpoint: "https://s3.example.com"
  region: "us-east-1"
  credentials:
    accessKey: "ACCESS_KEY"
    secretKey: "SECRET_KEY"

# 认证配置
auth:
  basic:
    enabled: true
    users:
      - username: "admin"
        password: "$2a$10$N9qo8uLOickgx2ZMRZoMy.Mrq4H9zLMHbJY3ZQ5yQ6eM3BvV7DfG." # bcrypt加密的密码
  oidc:
    enabled: true
    issuerURL: "https://accounts.google.com"
    clientID: "your-client-id"
    clientSecret: "your-client-secret"
    redirectURL: "http://localhost:8080/oauth2/callback"
    scopes: ["openid", "email", "profile"]

# 模板配置
templates:
  targetList:
    enabled: true
    path: "/"
  notFound:
    enabled: true
    path: "/404"

3. 运行s3-proxy

# 使用Docker运行
docker run -d \
  -p 8080:8080 \
  -v $(pwd)/config.yaml:/etc/s3-proxy/config.yaml \
  ghcr.io/oxyno-zeta/s3-proxy:latest

# 或者直接运行二进制
./s3-proxy --config ./config.yaml

4. 使用认证

Basic认证

curl -u admin:password http://localhost:8080/file.txt

OpenID认证

1. 访问 http://localhost:8080
2. 将被重定向到OpenID提供商登录页面
3. 登录后将被重定向回s3-proxy

5. 高级配置

• 缓存控制: 可以配置缓存头
• CORS: 支持跨域资源共享配置
• HTTPS: 支持TLS配置
• 路径重写: 支持URL路径重写规则

# 高级配置示例
server:
  tls:
    enabled: true
    cert: "/path/to/cert.pem"
    key: "/path/to/key.pem"
  cors:
    enabled: true
    allowedOrigins: ["*"]
    allowedMethods: ["GET", "POST", "PUT", "DELETE"]
    allowedHeaders: ["*"]

cache:
  enabled: true
  defaultMaxAge: 3600

总结

本文介绍了如何使用Golang实现S3的基本操作以及如何使用s3-proxy作为S3的代理服务器。s3-proxy提供了灵活的认证方式和丰富的配置选项，可以方便地集成到现有系统中。