Skip to content

AWS S3兼容文档

前言

介绍

本文档主要是针对osca兼容AWS S3协议后,与S3的一些异同

支持的SDK

  • Java SDK
  • GO SDK
  • Python SDK
  • PHP SDK

支持的接口

Bucket

  • List Buckets
  • Get Bucket Acl
  • Delete Bucket
  • Head Bucket
  • List Objects

Object

  • Put Object
  • Get Object
  • Copy Object
  • Delete Object
  • Delete Multiple Objects
  • Head Object
  • PutObjectTaggingHandler
  • GetObjectTaggingHandler
  • DeleteObjectTaggingHandler

Multipart Upload

  • Initiate Multipart Upload
  • Upload Part
  • List Parts
  • List Multipart Uploads
  • Complete Multipart Upload
  • Abort Multipart Upload

约束说明

分项详细说明
加密暂不支持
Acl支持get
Grant暂不支持
Lifecycle暂不支持
多版本不支持
Region默认为us-east-1
StorageClassSTANDARD
Authorization支持v4,v2,目前支持在header加入签名信息、支持url签名
MFA授权暂不支持

Bucket接口

List Buckets

兼容说明

接口和S3兼容

代码示例

GO:

go
	conf := &aws.Config{
		Region: aws.String("us-east-1"),

		Endpoint:         aws.String(endpoint),
		Credentials:      credentials.NewStaticCredentials(AK, SK, ""),
		S3ForcePathStyle: aws.Bool(true),
	}
	sess := session.Must(session.NewSessionWithOptions(session.Options{Config: *conf}))
	svc := s3.New(sess)
	//查询所有桶
	result, err := svc.ListBuckets(nil)
	if err != nil {
		fmt.Printf("err: %v\n", err)
	}
	fmt.Printf("result: %v\n", result)

Get Bucket Acl

兼容说明

因为OSCA-OSS的权限不能赋给其他的用户,所以返回的Body中只有一条记录

代码示例

GO:

go
conf := &aws.Config{
		Region: aws.String("us-east-1"),

		Endpoint:         aws.String(endpoint),
		Credentials:      credentials.NewStaticCredentials(AK, SK, ""),
		S3ForcePathStyle: aws.Bool(true),
	}
	sess := session.Must(session.NewSessionWithOptions(session.Options{Config: *conf}))
	svc := s3.New(sess)

	//查询桶中对象
	bucketname := "BucketName"

	params1 := &s3.GetBucketAclInput{
		Bucket: aws.String(bucketname),
	}
	resp, err := svc.GetBucketAcl(params1)
	if err != nil {
		fmt.Printf("err: %v\n", err)
		return
	}
	fmt.Printf("resp: %v\n", resp)

Delete Bucket

兼容说明

接口和S3兼容

代码示例

GO:

go
conf := &aws.Config{
		Region: aws.String("us-east-1"),

		Endpoint:         aws.String(endpoint),
		Credentials:      credentials.NewStaticCredentials(AK, SK, ""),
		S3ForcePathStyle: aws.Bool(true),
	}
	sess := session.Must(session.NewSessionWithOptions(session.Options{Config: *conf}))
	svc := s3.New(sess)

	//查询桶中对象
	bucketname := "BucketName"

	params1 := &s3.DeleteBucketInput{
		Bucket: aws.String(bucketname),
	}
	resp, err := svc.DeleteBucket(params1)
	if err != nil {
		fmt.Printf("err: %v\n", err)
		return
	}
	fmt.Printf("resp: %v\n", resp)

Head Bucket

兼容说明

接口和S3兼容

代码示例

GO:

go
conf := &aws.Config{
		Region: aws.String("us-east-1"),

		Endpoint:         aws.String(endpoint),
		Credentials:      credentials.NewStaticCredentials(AK, SK, ""),
		S3ForcePathStyle: aws.Bool(true),
	}
	sess := session.Must(session.NewSessionWithOptions(session.Options{Config: *conf}))
	svc := s3.New(sess)

	//查询桶中对象
	bucketname := "BucketName"

	params1 := &s3.HeadBucketInput{
		Bucket: aws.String(bucketname),
	}
	resp, err := svc.HeadBucket(params1)
	if err != nil {
		fmt.Printf("err: %v\n", err)
		return
	}
	fmt.Printf("resp: %v\n", resp)

Object接口

Put Object

兼容说明

  • x-amz-website-redirect-location 不支持

代码示例

GO:

go
conf := &aws.Config{
		Region: aws.String("us-east-1"),

		Endpoint:         aws.String(endpoint),
		Credentials:      credentials.NewStaticCredentials(AK, SK, ""),
		S3ForcePathStyle: aws.Bool(true),
	}
	sess := session.Must(session.NewSessionWithOptions(session.Options{Config: *conf}))
	svc := s3.New(sess)

	keyName := "S3test.txt"
	BucketName := "BucketName"

	params := &s3.PutObjectInput{
		Bucket:          aws.String(BucketName),                 // bucket名称
		Key:             aws.String(keyName),                    // object key
		Body:            bytes.NewReader([]byte("TEST UPLOAD")), //要上传的内容
		ContentType:     aws.String("application/ocet-stream"),  //设置content-type
		Tagging:         aws.String("key=value"),
		ContentEncoding: aws.String("gzip"),
	}
	resp, err := svc.PutObject(params)
	if err != nil {
		fmt.Printf("err: %v\n", err)

	}
	fmt.Printf("resp: %v\n", resp)

Get Object

兼容说明

接口和S3兼容

代码示例

GO:

go
conf := &aws.Config{
		Region: aws.String("us-east-1"),

		Endpoint:         aws.String(endpoint),
		Credentials:      credentials.NewStaticCredentials(AK, SK, ""),
		S3ForcePathStyle: aws.Bool(true),
	}
	sess := session.Must(session.NewSessionWithOptions(session.Options{Config: *conf}))
	svc := s3.New(sess)

	keyName := "S3test.txt"
	BucketName := "BucketName"

	params := &s3.GetObjectInput{
		Bucket: aws.String(BucketName), // bucket名称
		Key:    aws.String(keyName),    // object key
	}
	resp, err := svc.GetObject(params)
	if err != nil {
		fmt.Printf("err: %v\n", err)

	}
	fmt.Printf("resp: %v\n", resp)

Copy Object

兼容说明

接口和S3兼容

代码示例

GO:

go
conf := &aws.Config{
		Region: aws.String("us-east-1"),
		Endpoint:         aws.String(endpoint),
		Credentials:      credentials.NewStaticCredentials(AK, SK, ""),
		S3ForcePathStyle: aws.Bool(true),
	}
	sess := session.Must(session.NewSessionWithOptions(session.Options{Config: *conf}))
	svc := s3.New(sess)

	keyName := "S3test.txt"
	BucketName := "BucketName"
    DestBucketName:=""
    DestkeyName:=""
	params := &s3.CopyObjectInput{
		Bucket:     aws.String(DestBucketName), // bucket名称
		Key:        aws.String(DestkeyName),    // object key
		CopySource: aws.String(BucketName+"/"+keyName),
	}
	resp, err := svc.CopyObject(params)
	if err != nil {
		fmt.Printf("err: %v\n", err)

	}
	fmt.Printf("resp: %v\n", resp)

Delete Object

兼容说明

接口和S3兼容

代码示例

GO:

go
conf := &aws.Config{
		Region: aws.String("us-east-1"),

		Endpoint:         aws.String(endpoint),
		Credentials:      credentials.NewStaticCredentials(AK, SK, ""),
		S3ForcePathStyle: aws.Bool(true),
	}
	sess := session.Must(session.NewSessionWithOptions(session.Options{Config: *conf}))
	svc := s3.New(sess)

	keyName := "S3test.txt"
	BucketName := "BucketName"

	params := &s3.DeleteObjectInput{
		Bucket: aws.String(BucketName), // bucket名称
		Key:    aws.String(keyName),    // object key
	}
	resp, err := svc.DeleteObject(params)
	if err != nil {
		fmt.Printf("err: %v\n", err)

	}
	fmt.Printf("resp: %v\n", resp)

Delete Multiple Objects

兼容说明

接口和S3兼容

代码示例

GO:

go
conf := &aws.Config{
		Region: aws.String("us-east-1"),

		Endpoint:         aws.String(endpoint),
		Credentials:      credentials.NewStaticCredentials(AK, SK, ""),
		S3ForcePathStyle: aws.Bool(true),
	}
	sess := session.Must(session.NewSessionWithOptions(session.Options{Config: *conf}))
	svc := s3.New(sess)

	objects := []*s3.ObjectIdentifier{
		{
			Key: aws.String("config.ini"),
		},
		{
			Key: aws.String("confxx.ini"),
		},
		// 添加更多要删除的对象
	}
	BucketName := "BucketName"

	params := &s3.DeleteObjectsInput{
		Bucket: aws.String(BucketName), // bucket名称
		Delete: &s3.Delete{
			Objects: objects,
			Quiet:   aws.Bool(false), // 设置为 true 可以静默删除,不返回被删除对象的信息
			
		},
	}
	resp, err := svc.DeleteObjects(params)
	if err != nil {
		fmt.Printf("err: %v\n", err)

	}
	fmt.Printf("resp: %v\n", resp)
}

Head Object

兼容说明

接口和S3兼容

代码示例

GO:

go
conf := &aws.Config{
		Region: aws.String("us-east-1"),

		Endpoint:         aws.String(endpoint),
		Credentials:      credentials.NewStaticCredentials(AK, SK, ""),
		S3ForcePathStyle: aws.Bool(true),
	}
	sess := session.Must(session.NewSessionWithOptions(session.Options{Config: *conf}))
	svc := s3.New(sess)

	keyName := "S3test.txt"
	BucketName := "BucketName"

	params := &s3.HeadObjectInput{
		Bucket: aws.String(BucketName), // bucket名称
		Key:    aws.String(keyName),    // object key
	}
	resp, err := svc.HeadObject(params)
	if err != nil {
		fmt.Printf("err: %v\n", err)

	}
	fmt.Printf("resp: %v\n", resp)

List Objects

兼容说明

接口和S3兼容

代码示例

GO:

go
conf := &aws.Config{
		Region: aws.String("us-east-1"),

		Endpoint:         aws.String(endpoint),
		Credentials:      credentials.NewStaticCredentials(AK, SK, ""),
		S3ForcePathStyle: aws.Bool(true),
	}
	sess := session.Must(session.NewSessionWithOptions(session.Options{Config: *conf}))
	svc := s3.New(sess)

	BucketName := "BucketName"

	params := &s3.ListObjectsInput{
		Bucket: aws.String(BucketName), // bucket名称
	}
	resp, err := svc.ListObjects(params)
	if err != nil {
		fmt.Printf("err: %v\n", err)

	}
	fmt.Printf("resp: %v\n", resp)

GetObjectTagging

兼容说明

接口和S3兼容

代码示例

GO:

go
conf := &aws.Config{
		Region: aws.String("us-east-1"),

		Endpoint:         aws.String(endpoint),
		Credentials:      credentials.NewStaticCredentials(AK, SK, ""),
		S3ForcePathStyle: aws.Bool(true),
	}
	sess := session.Must(session.NewSessionWithOptions(session.Options{Config: *conf}))
	svc := s3.New(sess)
    keyName := "S3test.txt"
	BucketName := "BucketName"

	params := &s3.GetObjectTaggingInput{
		Bucket: aws.String(BucketName), // bucket名称
        Key:    aws.String(keyName),    // object key
	}
	resp, err := svc.GetObjectTagging(params)
	if err != nil {
		fmt.Printf("err: %v\n", err)

	}
	fmt.Printf("resp: %v\n", resp)

PutObjectTagging

兼容说明

接口和S3兼容

代码示例

GO:

go
conf := &aws.Config{
		Region: aws.String("us-east-1"),

		Endpoint:         aws.String(endpoint),
		Credentials:      credentials.NewStaticCredentials(AK, SK, ""),
		S3ForcePathStyle: aws.Bool(true),
	}
	sess := session.Must(session.NewSessionWithOptions(session.Options{Config: *conf}))
	svc := s3.New(sess)
    keyName := "S3test.txt"
	BucketName := "BucketName"

	params := &s3.PutObjectTaggingInput{
		Bucket: aws.String(BucketName), // bucket名称
		Key:    aws.String(keyName),    // object key
		Tagging: &s3.Tagging{
			TagSet: []*s3.Tag{
				{
					Key:   aws.String("aa"), // 键1
					Value: aws.String("bb"), // 值1
				},
				{
					Key:   aws.String("cc"), // 键2
					Value: aws.String("dd"), // 值2
				},
			},
		},
	}
	resp, err := svc.PutObjectTagging(params)
	if err != nil {
		fmt.Printf("err: %v\n", err)

	}
	fmt.Printf("resp: %v\n", resp)

DeleteObjectTagging

兼容说明

接口和S3兼容

代码示例

GO:

go
conf := &aws.Config{
		Region: aws.String("us-east-1"),

		Endpoint:         aws.String(endpoint),
		Credentials:      credentials.NewStaticCredentials(AK, SK, ""),
		S3ForcePathStyle: aws.Bool(true),
	}
	sess := session.Must(session.NewSessionWithOptions(session.Options{Config: *conf}))
	svc := s3.New(sess)
    keyName := "S3test.txt"
	BucketName := "BucketName"

    params := &s3.DeleteObjectTaggingInput{
    Bucket: aws.String(BucketName), // bucket名称
    Key:    aws.String(keyName),    // object key
	}
	resp, err := svc.DeleteObjectTagging(params)
	if err != nil {
		fmt.Printf("err: %v\n", err)

	}
	fmt.Printf("resp: %v\n", resp)

Multipart Upload接口

Initiate Multipart Upload

兼容说明

x-amz-website-redirect-location,不支持

代码示例

GO:

go
conf := &aws.Config{
		Region: aws.String("us-east-1"),

		Endpoint:         aws.String(endpoint),
		Credentials:      credentials.NewStaticCredentials(AK, SK, ""),
		S3ForcePathStyle: aws.Bool(true),
	}
	sess := session.Must(session.NewSessionWithOptions(session.Options{Config: *conf}))
	svc := s3.New(sess)

	BucketName := "BucketName"
    keyName="S3test.txt"
	params := &s3.CreateMultipartUploadInput{
			Bucket:      aws.String(BucketName),                // bucket名称
		Key:         aws.String(keyName),                   // object key
		ContentType: aws.String("application/ocet-stream"), //设置content-type
	}
	resp, err := svc.CreateMultipartUpload(params)
	if err != nil {
		fmt.Printf("err: %v\n", err)

	}
	fmt.Printf("resp: %v\n", resp)

Upload Part

兼容说明

接口和S3兼容

代码示例

GO:

go
conf := &aws.Config{
		Region: aws.String("us-east-1"),

		Endpoint:         aws.String(endpoint),
		Credentials:      credentials.NewStaticCredentials(AK, SK, ""),
		S3ForcePathStyle: aws.Bool(true),
	}
	sess := session.Must(session.NewSessionWithOptions(session.Options{Config: *conf}))
	svc := s3.New(sess)

	upID := ""
	file, err := os.Open("S3test.txt")
	if err != nil {
		log.Fatal(err)
	}
	defer file.Close()

	const partSize = int64(5 * 1024 * 1024) // 每个块的大小,单位为字节
	size, _ := file.Seek(0, io.SeekEnd)
	file.Seek(0, io.SeekStart)

	var parts []*s3.CompletedPart
	for i := int64(1); i <= (size+partSize-1)/partSize; i++ {
		buf := make([]byte, partSize)
		n, err := file.Read(buf)
		if err != nil && err != io.EOF {
			log.Fatal(err)
		}
		keyName := "S3test.txt"
		BucketName := "BucketName"
		params := &s3.UploadPartInput{
			Body:          bytes.NewReader(buf[:n]),
			Bucket:        aws.String(BucketName),
			Key:           aws.String(keyName),
			PartNumber:    aws.Int64(i),
			UploadId:      aws.String(upID),
			ContentLength: aws.Int64(int64(n)),
		}
		resp, err := svc.UploadPart(params)
		if err != nil {
			fmt.Printf("err: %v\n", err)

		}
		parts = append(parts, &s3.CompletedPart{ETag: resp.ETag, PartNumber: aws.Int64(i)})
		fmt.Printf("resp: %v\n", resp)
	}

List Parts

兼容说明

因为OSCA-OSS没有生命周期的概念,所以响应中不会加入lifecycle相关的响应头

代码示例

GO:

go
conf := &aws.Config{
		Region: aws.String("us-east-1"),

		Endpoint:         aws.String(endpoint),
		Credentials:      credentials.NewStaticCredentials(AK, SK, ""),
		S3ForcePathStyle: aws.Bool(true),
	}
	sess := session.Must(session.NewSessionWithOptions(session.Options{Config: *conf}))
	svc := s3.New(sess)

	BucketName := "BucketName"
	upID := ""
    keyName="S3test.txt"
	params := &s3.ListPartsInput{
		Bucket:   aws.String(BucketName), // bucket名称
		Key:      aws.String(keyName),    // object key
		UploadId: aws.String(uploadId),       //由初始化获取到得uploadid
	}
	resp, err := svc.ListParts(params)
	if err != nil {
		fmt.Printf("err: %v\n", err)

	}
	fmt.Printf("resp: %v\n", resp)

List Multipart Uploads

兼容说明

  • request只支持max-uploads,key-marker参数

代码示例

GO:

go
conf := &aws.Config{
		Region: aws.String("us-east-1"),

		Endpoint:         aws.String(endpoint),
		Credentials:      credentials.NewStaticCredentials(AK, SK, ""),
		S3ForcePathStyle: aws.Bool(true),
	}
	sess := session.Must(session.NewSessionWithOptions(session.Options{Config: *conf}))
	svc := s3.New(sess)

	BucketName := "BucketName"

	params := &s3.ListMultipartUploadsInput{
		Bucket: aws.String(BucketName), // bucket名称
	}
	resp, err := svc.ListMultipartUploads(params)
	if err != nil {
		fmt.Printf("err: %v\n", err)

	}
	fmt.Printf("resp: %v\n", resp)

Complete Multipart Upload

兼容说明

除了版本控制和加密,OSCA-OSS完全兼容S3

代码示例

GO:

go
conf := &aws.Config{
		Region: aws.String("us-east-1"),

		Endpoint:         aws.String(endpoint),
		Credentials:      credentials.NewStaticCredentials(AK, SK, ""),
		S3ForcePathStyle: aws.Bool(true),
	}
	sess := session.Must(session.NewSessionWithOptions(session.Options{Config: *conf}))
	svc := s3.New(sess)
    keyName := "S3test3.txt"
	BucketName := "BucketName"
	upID := ""

    params := &s3.CompleteMultipartUploadInput{
		Bucket:   aws.String(BucketName),
		Key:      aws.String(keyName),
		UploadId: aws.String(upID),
		MultipartUpload: &s3.CompletedMultipartUpload{
			Parts: parts,
		},
	}
	resp, err := svc.CompleteMultipartUpload(params)
	if err != nil {
		fmt.Printf("err: %v\n", err)

	}
	fmt.Printf("resp: %v\n", resp)

Abort Multipart Upload

兼容说明

接口和S3兼容

代码示例

GO:

go
conf := &aws.Config{
		Region: aws.String("us-east-1"),

		Endpoint:         aws.String(endpoint),
		Credentials:      credentials.NewStaticCredentials(AK, SK, ""),
		S3ForcePathStyle: aws.Bool(true),
	}
	sess := session.Must(session.NewSessionWithOptions(session.Options{Config: *conf}))
	svc := s3.New(sess)
    keyName := "S3test3.txt"
	BucketName := "BucketName"
	upID := ""

    params := &s3.AbortMultipartUploadInput{
		Bucket:   aws.String(BucketName),
		Key:      aws.String(keyName),
		UploadId: aws.String(upID),
	}
	resp, err := svc.AbortMultipartUpload(params)
	if err != nil {
		fmt.Printf("err: %v\n", err)

	}
	fmt.Printf("resp: %v\n", resp)

完整的分块上传示例

go
func main() {
    bucketName:=""
    objectName:=""
    filePath:=""
	getObjectDonwloadUrl(getS3Client(), bucketName, objectName,filePath)

}
func getS3Client() *s3.S3 {
	creds := credentials.NewStaticCredentials(ak, sk, "")
	config := &aws.Config{
		Region:           aws.String(region),
		Endpoint:         aws.String(endpoint),
		S3ForcePathStyle: aws.Bool(true),
		Credentials:      creds,
		DisableSSL:       aws.Bool(false),
	}
	sess := session.Must(session.NewSession(config))
	return s3.New(sess)
}
func MultipartUpload(s3Client *s3.S3, bucketName, objectName, filePath string) {
	uploadId, err := InitMultipartUpload(s3Client, bucketName, objectName)
	if err != nil {
		fmt.Println("InitMultipartUpload : " + err.Error())
		return
	}
	file, err := os.Open(filePath)
	if err != nil {
		fmt.Println("open file : " + err.Error())
		return
	}
	partSize := 2 * 1024 * 1024
	var partNum int64 = 0
	buffer := make([]byte, partSize)
	var parts []*s3.CompletedPart
	for {
		partNum++
		readLen, err := file.Read(buffer)
		if err != nil || readLen == 0 {
			break
		}
		uploadPartResp, err := UploadPart(s3Client, bucketName, objectName, uploadId, partNum, bytes.NewReader(buffer[0:readLen]))
		if err != nil {
			fmt.Println("UploadPart : " + err.Error())
			break
		}
		part := s3.CompletedPart{ETag: uploadPartResp.ETag, PartNumber: aws.Int64(partNum)}
		fmt.Println("partETage : ", part.ETag, " , partNum : ", part.PartNumber)
		parts = append(parts, &part)
	}

	comp := s3.CompleteMultipartUploadInput{Bucket: aws.String(bucketName), Key: aws.String(objectName), UploadId: aws.String(uploadId),
		MultipartUpload: &s3.CompletedMultipartUpload{Parts: parts}}
	_, err = s3Client.CompleteMultipartUpload(&comp)
	if err != nil {
		fmt.Println("CompleteMultipartUpload : " + err.Error())
		_, err := s3Client.AbortMultipartUpload(&s3.AbortMultipartUploadInput{Bucket: aws.String(bucketName),
			Key: aws.String(objectName), UploadId: aws.String(uploadId)})
		if err != nil {
			fmt.Println("AbortMultipartUpload failed ")
			return
		}
		return
	} else {
		fmt.Println("CompleteMultipartUpload Ok")
	}
}
func InitMultipartUpload(s3Client *s3.S3, bucketName, objectName string) (uploadId string, err error) {
	resp, err := s3Client.CreateMultipartUpload(&s3.CreateMultipartUploadInput{Bucket: aws.String(bucketName),
		Key: aws.String(objectName)})
	if err != nil {
		fmt.Println(err.Error())
		return
	} else {
		uploadId = *resp.UploadId
		fmt.Println("uploadId : ", uploadId)
		return
	}
}
func UploadPart(s3Client *s3.S3, bucketName, objectName string, uploadId string, partNum int64, buffer *bytes.Reader) (resp *s3.UploadPartOutput, err error) {

	params := &s3.UploadPartInput{
		Body:          buffer,
		Bucket:        aws.String(bucketName),
		Key:           aws.String(objectName),
		PartNumber:    aws.Int64(partNum),
		UploadId:      aws.String(uploadId),
		ContentLength: aws.Int64(buffer.Size()),
	}

	res, err := s3Client.UploadPart(params)
	if err != nil {
		fmt.Printf("err: %v\n", err)
	}
	fmt.Printf("UploadPart res: %v\n", res)
	return res, err
}