主题切换
AWS S3兼容文档
前言
介绍
本文档主要是针对osca兼容AWS S3协议后,与S3的一些异同
支持的SDK
- Java SDK
- GO SDK
- Python SDK
- PHP SDK
支持的接口
Bucket
- List Buckets
- Get Bucket Acl
- Delete Bucket
- Head Bucket
- List Objects
Object
- Put Object
- Get Object
- Copy Object
- Delete Object
- Delete Multiple Objects
- Head Object
- PutObjectTaggingHandler
- GetObjectTaggingHandler
- DeleteObjectTaggingHandler
Multipart Upload
- Initiate Multipart Upload
- Upload Part
- List Parts
- List Multipart Uploads
- Complete Multipart Upload
- Abort Multipart Upload
约束说明
分项 | 详细说明 |
---|---|
加密 | 暂不支持 |
Acl | 支持get |
Grant | 暂不支持 |
Lifecycle | 暂不支持 |
多版本 | 不支持 |
Region | 默认为us-east-1 |
StorageClass | STANDARD |
Authorization | 支持v4,v2,目前支持在header加入签名信息、支持url签名 |
MFA授权 | 暂不支持 |
Bucket接口
List Buckets
兼容说明
接口和S3兼容
代码示例
GO:
go
conf := &aws.Config{
Region: aws.String("us-east-1"),
Endpoint: aws.String(endpoint),
Credentials: credentials.NewStaticCredentials(AK, SK, ""),
S3ForcePathStyle: aws.Bool(true),
}
sess := session.Must(session.NewSessionWithOptions(session.Options{Config: *conf}))
svc := s3.New(sess)
//查询所有桶
result, err := svc.ListBuckets(nil)
if err != nil {
fmt.Printf("err: %v\n", err)
}
fmt.Printf("result: %v\n", result)
Get Bucket Acl
兼容说明
因为OSCA-OSS的权限不能赋给其他的用户,所以返回的Body中只有一条记录
代码示例
GO:
go
conf := &aws.Config{
Region: aws.String("us-east-1"),
Endpoint: aws.String(endpoint),
Credentials: credentials.NewStaticCredentials(AK, SK, ""),
S3ForcePathStyle: aws.Bool(true),
}
sess := session.Must(session.NewSessionWithOptions(session.Options{Config: *conf}))
svc := s3.New(sess)
//查询桶中对象
bucketname := "BucketName"
params1 := &s3.GetBucketAclInput{
Bucket: aws.String(bucketname),
}
resp, err := svc.GetBucketAcl(params1)
if err != nil {
fmt.Printf("err: %v\n", err)
return
}
fmt.Printf("resp: %v\n", resp)
Delete Bucket
兼容说明
接口和S3兼容
代码示例
GO:
go
conf := &aws.Config{
Region: aws.String("us-east-1"),
Endpoint: aws.String(endpoint),
Credentials: credentials.NewStaticCredentials(AK, SK, ""),
S3ForcePathStyle: aws.Bool(true),
}
sess := session.Must(session.NewSessionWithOptions(session.Options{Config: *conf}))
svc := s3.New(sess)
//查询桶中对象
bucketname := "BucketName"
params1 := &s3.DeleteBucketInput{
Bucket: aws.String(bucketname),
}
resp, err := svc.DeleteBucket(params1)
if err != nil {
fmt.Printf("err: %v\n", err)
return
}
fmt.Printf("resp: %v\n", resp)
Head Bucket
兼容说明
接口和S3兼容
代码示例
GO:
go
conf := &aws.Config{
Region: aws.String("us-east-1"),
Endpoint: aws.String(endpoint),
Credentials: credentials.NewStaticCredentials(AK, SK, ""),
S3ForcePathStyle: aws.Bool(true),
}
sess := session.Must(session.NewSessionWithOptions(session.Options{Config: *conf}))
svc := s3.New(sess)
//查询桶中对象
bucketname := "BucketName"
params1 := &s3.HeadBucketInput{
Bucket: aws.String(bucketname),
}
resp, err := svc.HeadBucket(params1)
if err != nil {
fmt.Printf("err: %v\n", err)
return
}
fmt.Printf("resp: %v\n", resp)
Object接口
Put Object
兼容说明
- x-amz-website-redirect-location 不支持
代码示例
GO:
go
conf := &aws.Config{
Region: aws.String("us-east-1"),
Endpoint: aws.String(endpoint),
Credentials: credentials.NewStaticCredentials(AK, SK, ""),
S3ForcePathStyle: aws.Bool(true),
}
sess := session.Must(session.NewSessionWithOptions(session.Options{Config: *conf}))
svc := s3.New(sess)
keyName := "S3test.txt"
BucketName := "BucketName"
params := &s3.PutObjectInput{
Bucket: aws.String(BucketName), // bucket名称
Key: aws.String(keyName), // object key
Body: bytes.NewReader([]byte("TEST UPLOAD")), //要上传的内容
ContentType: aws.String("application/ocet-stream"), //设置content-type
Tagging: aws.String("key=value"),
ContentEncoding: aws.String("gzip"),
}
resp, err := svc.PutObject(params)
if err != nil {
fmt.Printf("err: %v\n", err)
}
fmt.Printf("resp: %v\n", resp)
Get Object
兼容说明
接口和S3兼容
代码示例
GO:
go
conf := &aws.Config{
Region: aws.String("us-east-1"),
Endpoint: aws.String(endpoint),
Credentials: credentials.NewStaticCredentials(AK, SK, ""),
S3ForcePathStyle: aws.Bool(true),
}
sess := session.Must(session.NewSessionWithOptions(session.Options{Config: *conf}))
svc := s3.New(sess)
keyName := "S3test.txt"
BucketName := "BucketName"
params := &s3.GetObjectInput{
Bucket: aws.String(BucketName), // bucket名称
Key: aws.String(keyName), // object key
}
resp, err := svc.GetObject(params)
if err != nil {
fmt.Printf("err: %v\n", err)
}
fmt.Printf("resp: %v\n", resp)
Copy Object
兼容说明
接口和S3兼容
代码示例
GO:
go
conf := &aws.Config{
Region: aws.String("us-east-1"),
Endpoint: aws.String(endpoint),
Credentials: credentials.NewStaticCredentials(AK, SK, ""),
S3ForcePathStyle: aws.Bool(true),
}
sess := session.Must(session.NewSessionWithOptions(session.Options{Config: *conf}))
svc := s3.New(sess)
keyName := "S3test.txt"
BucketName := "BucketName"
DestBucketName:=""
DestkeyName:=""
params := &s3.CopyObjectInput{
Bucket: aws.String(DestBucketName), // bucket名称
Key: aws.String(DestkeyName), // object key
CopySource: aws.String(BucketName+"/"+keyName),
}
resp, err := svc.CopyObject(params)
if err != nil {
fmt.Printf("err: %v\n", err)
}
fmt.Printf("resp: %v\n", resp)
Delete Object
兼容说明
接口和S3兼容
代码示例
GO:
go
conf := &aws.Config{
Region: aws.String("us-east-1"),
Endpoint: aws.String(endpoint),
Credentials: credentials.NewStaticCredentials(AK, SK, ""),
S3ForcePathStyle: aws.Bool(true),
}
sess := session.Must(session.NewSessionWithOptions(session.Options{Config: *conf}))
svc := s3.New(sess)
keyName := "S3test.txt"
BucketName := "BucketName"
params := &s3.DeleteObjectInput{
Bucket: aws.String(BucketName), // bucket名称
Key: aws.String(keyName), // object key
}
resp, err := svc.DeleteObject(params)
if err != nil {
fmt.Printf("err: %v\n", err)
}
fmt.Printf("resp: %v\n", resp)
Delete Multiple Objects
兼容说明
接口和S3兼容
代码示例
GO:
go
conf := &aws.Config{
Region: aws.String("us-east-1"),
Endpoint: aws.String(endpoint),
Credentials: credentials.NewStaticCredentials(AK, SK, ""),
S3ForcePathStyle: aws.Bool(true),
}
sess := session.Must(session.NewSessionWithOptions(session.Options{Config: *conf}))
svc := s3.New(sess)
objects := []*s3.ObjectIdentifier{
{
Key: aws.String("config.ini"),
},
{
Key: aws.String("confxx.ini"),
},
// 添加更多要删除的对象
}
BucketName := "BucketName"
params := &s3.DeleteObjectsInput{
Bucket: aws.String(BucketName), // bucket名称
Delete: &s3.Delete{
Objects: objects,
Quiet: aws.Bool(false), // 设置为 true 可以静默删除,不返回被删除对象的信息
},
}
resp, err := svc.DeleteObjects(params)
if err != nil {
fmt.Printf("err: %v\n", err)
}
fmt.Printf("resp: %v\n", resp)
}
Head Object
兼容说明
接口和S3兼容
代码示例
GO:
go
conf := &aws.Config{
Region: aws.String("us-east-1"),
Endpoint: aws.String(endpoint),
Credentials: credentials.NewStaticCredentials(AK, SK, ""),
S3ForcePathStyle: aws.Bool(true),
}
sess := session.Must(session.NewSessionWithOptions(session.Options{Config: *conf}))
svc := s3.New(sess)
keyName := "S3test.txt"
BucketName := "BucketName"
params := &s3.HeadObjectInput{
Bucket: aws.String(BucketName), // bucket名称
Key: aws.String(keyName), // object key
}
resp, err := svc.HeadObject(params)
if err != nil {
fmt.Printf("err: %v\n", err)
}
fmt.Printf("resp: %v\n", resp)
List Objects
兼容说明
接口和S3兼容
代码示例
GO:
go
conf := &aws.Config{
Region: aws.String("us-east-1"),
Endpoint: aws.String(endpoint),
Credentials: credentials.NewStaticCredentials(AK, SK, ""),
S3ForcePathStyle: aws.Bool(true),
}
sess := session.Must(session.NewSessionWithOptions(session.Options{Config: *conf}))
svc := s3.New(sess)
BucketName := "BucketName"
params := &s3.ListObjectsInput{
Bucket: aws.String(BucketName), // bucket名称
}
resp, err := svc.ListObjects(params)
if err != nil {
fmt.Printf("err: %v\n", err)
}
fmt.Printf("resp: %v\n", resp)
GetObjectTagging
兼容说明
接口和S3兼容
代码示例
GO:
go
conf := &aws.Config{
Region: aws.String("us-east-1"),
Endpoint: aws.String(endpoint),
Credentials: credentials.NewStaticCredentials(AK, SK, ""),
S3ForcePathStyle: aws.Bool(true),
}
sess := session.Must(session.NewSessionWithOptions(session.Options{Config: *conf}))
svc := s3.New(sess)
keyName := "S3test.txt"
BucketName := "BucketName"
params := &s3.GetObjectTaggingInput{
Bucket: aws.String(BucketName), // bucket名称
Key: aws.String(keyName), // object key
}
resp, err := svc.GetObjectTagging(params)
if err != nil {
fmt.Printf("err: %v\n", err)
}
fmt.Printf("resp: %v\n", resp)
PutObjectTagging
兼容说明
接口和S3兼容
代码示例
GO:
go
conf := &aws.Config{
Region: aws.String("us-east-1"),
Endpoint: aws.String(endpoint),
Credentials: credentials.NewStaticCredentials(AK, SK, ""),
S3ForcePathStyle: aws.Bool(true),
}
sess := session.Must(session.NewSessionWithOptions(session.Options{Config: *conf}))
svc := s3.New(sess)
keyName := "S3test.txt"
BucketName := "BucketName"
params := &s3.PutObjectTaggingInput{
Bucket: aws.String(BucketName), // bucket名称
Key: aws.String(keyName), // object key
Tagging: &s3.Tagging{
TagSet: []*s3.Tag{
{
Key: aws.String("aa"), // 键1
Value: aws.String("bb"), // 值1
},
{
Key: aws.String("cc"), // 键2
Value: aws.String("dd"), // 值2
},
},
},
}
resp, err := svc.PutObjectTagging(params)
if err != nil {
fmt.Printf("err: %v\n", err)
}
fmt.Printf("resp: %v\n", resp)
DeleteObjectTagging
兼容说明
接口和S3兼容
代码示例
GO:
go
conf := &aws.Config{
Region: aws.String("us-east-1"),
Endpoint: aws.String(endpoint),
Credentials: credentials.NewStaticCredentials(AK, SK, ""),
S3ForcePathStyle: aws.Bool(true),
}
sess := session.Must(session.NewSessionWithOptions(session.Options{Config: *conf}))
svc := s3.New(sess)
keyName := "S3test.txt"
BucketName := "BucketName"
params := &s3.DeleteObjectTaggingInput{
Bucket: aws.String(BucketName), // bucket名称
Key: aws.String(keyName), // object key
}
resp, err := svc.DeleteObjectTagging(params)
if err != nil {
fmt.Printf("err: %v\n", err)
}
fmt.Printf("resp: %v\n", resp)
Multipart Upload接口
Initiate Multipart Upload
兼容说明
x-amz-website-redirect-location,不支持
代码示例
GO:
go
conf := &aws.Config{
Region: aws.String("us-east-1"),
Endpoint: aws.String(endpoint),
Credentials: credentials.NewStaticCredentials(AK, SK, ""),
S3ForcePathStyle: aws.Bool(true),
}
sess := session.Must(session.NewSessionWithOptions(session.Options{Config: *conf}))
svc := s3.New(sess)
BucketName := "BucketName"
keyName="S3test.txt"
params := &s3.CreateMultipartUploadInput{
Bucket: aws.String(BucketName), // bucket名称
Key: aws.String(keyName), // object key
ContentType: aws.String("application/ocet-stream"), //设置content-type
}
resp, err := svc.CreateMultipartUpload(params)
if err != nil {
fmt.Printf("err: %v\n", err)
}
fmt.Printf("resp: %v\n", resp)
Upload Part
兼容说明
接口和S3兼容
代码示例
GO:
go
conf := &aws.Config{
Region: aws.String("us-east-1"),
Endpoint: aws.String(endpoint),
Credentials: credentials.NewStaticCredentials(AK, SK, ""),
S3ForcePathStyle: aws.Bool(true),
}
sess := session.Must(session.NewSessionWithOptions(session.Options{Config: *conf}))
svc := s3.New(sess)
upID := ""
file, err := os.Open("S3test.txt")
if err != nil {
log.Fatal(err)
}
defer file.Close()
const partSize = int64(5 * 1024 * 1024) // 每个块的大小,单位为字节
size, _ := file.Seek(0, io.SeekEnd)
file.Seek(0, io.SeekStart)
var parts []*s3.CompletedPart
for i := int64(1); i <= (size+partSize-1)/partSize; i++ {
buf := make([]byte, partSize)
n, err := file.Read(buf)
if err != nil && err != io.EOF {
log.Fatal(err)
}
keyName := "S3test.txt"
BucketName := "BucketName"
params := &s3.UploadPartInput{
Body: bytes.NewReader(buf[:n]),
Bucket: aws.String(BucketName),
Key: aws.String(keyName),
PartNumber: aws.Int64(i),
UploadId: aws.String(upID),
ContentLength: aws.Int64(int64(n)),
}
resp, err := svc.UploadPart(params)
if err != nil {
fmt.Printf("err: %v\n", err)
}
parts = append(parts, &s3.CompletedPart{ETag: resp.ETag, PartNumber: aws.Int64(i)})
fmt.Printf("resp: %v\n", resp)
}
List Parts
兼容说明
因为OSCA-OSS没有生命周期的概念,所以响应中不会加入lifecycle相关的响应头
代码示例
GO:
go
conf := &aws.Config{
Region: aws.String("us-east-1"),
Endpoint: aws.String(endpoint),
Credentials: credentials.NewStaticCredentials(AK, SK, ""),
S3ForcePathStyle: aws.Bool(true),
}
sess := session.Must(session.NewSessionWithOptions(session.Options{Config: *conf}))
svc := s3.New(sess)
BucketName := "BucketName"
upID := ""
keyName="S3test.txt"
params := &s3.ListPartsInput{
Bucket: aws.String(BucketName), // bucket名称
Key: aws.String(keyName), // object key
UploadId: aws.String(uploadId), //由初始化获取到得uploadid
}
resp, err := svc.ListParts(params)
if err != nil {
fmt.Printf("err: %v\n", err)
}
fmt.Printf("resp: %v\n", resp)
List Multipart Uploads
兼容说明
- request只支持max-uploads,key-marker参数
代码示例
GO:
go
conf := &aws.Config{
Region: aws.String("us-east-1"),
Endpoint: aws.String(endpoint),
Credentials: credentials.NewStaticCredentials(AK, SK, ""),
S3ForcePathStyle: aws.Bool(true),
}
sess := session.Must(session.NewSessionWithOptions(session.Options{Config: *conf}))
svc := s3.New(sess)
BucketName := "BucketName"
params := &s3.ListMultipartUploadsInput{
Bucket: aws.String(BucketName), // bucket名称
}
resp, err := svc.ListMultipartUploads(params)
if err != nil {
fmt.Printf("err: %v\n", err)
}
fmt.Printf("resp: %v\n", resp)
Complete Multipart Upload
兼容说明
除了版本控制和加密,OSCA-OSS完全兼容S3
代码示例
GO:
go
conf := &aws.Config{
Region: aws.String("us-east-1"),
Endpoint: aws.String(endpoint),
Credentials: credentials.NewStaticCredentials(AK, SK, ""),
S3ForcePathStyle: aws.Bool(true),
}
sess := session.Must(session.NewSessionWithOptions(session.Options{Config: *conf}))
svc := s3.New(sess)
keyName := "S3test3.txt"
BucketName := "BucketName"
upID := ""
params := &s3.CompleteMultipartUploadInput{
Bucket: aws.String(BucketName),
Key: aws.String(keyName),
UploadId: aws.String(upID),
MultipartUpload: &s3.CompletedMultipartUpload{
Parts: parts,
},
}
resp, err := svc.CompleteMultipartUpload(params)
if err != nil {
fmt.Printf("err: %v\n", err)
}
fmt.Printf("resp: %v\n", resp)
Abort Multipart Upload
兼容说明
接口和S3兼容
代码示例
GO:
go
conf := &aws.Config{
Region: aws.String("us-east-1"),
Endpoint: aws.String(endpoint),
Credentials: credentials.NewStaticCredentials(AK, SK, ""),
S3ForcePathStyle: aws.Bool(true),
}
sess := session.Must(session.NewSessionWithOptions(session.Options{Config: *conf}))
svc := s3.New(sess)
keyName := "S3test3.txt"
BucketName := "BucketName"
upID := ""
params := &s3.AbortMultipartUploadInput{
Bucket: aws.String(BucketName),
Key: aws.String(keyName),
UploadId: aws.String(upID),
}
resp, err := svc.AbortMultipartUpload(params)
if err != nil {
fmt.Printf("err: %v\n", err)
}
fmt.Printf("resp: %v\n", resp)
完整的分块上传示例
go
func main() {
bucketName:=""
objectName:=""
filePath:=""
getObjectDonwloadUrl(getS3Client(), bucketName, objectName,filePath)
}
func getS3Client() *s3.S3 {
creds := credentials.NewStaticCredentials(ak, sk, "")
config := &aws.Config{
Region: aws.String(region),
Endpoint: aws.String(endpoint),
S3ForcePathStyle: aws.Bool(true),
Credentials: creds,
DisableSSL: aws.Bool(false),
}
sess := session.Must(session.NewSession(config))
return s3.New(sess)
}
func MultipartUpload(s3Client *s3.S3, bucketName, objectName, filePath string) {
uploadId, err := InitMultipartUpload(s3Client, bucketName, objectName)
if err != nil {
fmt.Println("InitMultipartUpload : " + err.Error())
return
}
file, err := os.Open(filePath)
if err != nil {
fmt.Println("open file : " + err.Error())
return
}
partSize := 2 * 1024 * 1024
var partNum int64 = 0
buffer := make([]byte, partSize)
var parts []*s3.CompletedPart
for {
partNum++
readLen, err := file.Read(buffer)
if err != nil || readLen == 0 {
break
}
uploadPartResp, err := UploadPart(s3Client, bucketName, objectName, uploadId, partNum, bytes.NewReader(buffer[0:readLen]))
if err != nil {
fmt.Println("UploadPart : " + err.Error())
break
}
part := s3.CompletedPart{ETag: uploadPartResp.ETag, PartNumber: aws.Int64(partNum)}
fmt.Println("partETage : ", part.ETag, " , partNum : ", part.PartNumber)
parts = append(parts, &part)
}
comp := s3.CompleteMultipartUploadInput{Bucket: aws.String(bucketName), Key: aws.String(objectName), UploadId: aws.String(uploadId),
MultipartUpload: &s3.CompletedMultipartUpload{Parts: parts}}
_, err = s3Client.CompleteMultipartUpload(&comp)
if err != nil {
fmt.Println("CompleteMultipartUpload : " + err.Error())
_, err := s3Client.AbortMultipartUpload(&s3.AbortMultipartUploadInput{Bucket: aws.String(bucketName),
Key: aws.String(objectName), UploadId: aws.String(uploadId)})
if err != nil {
fmt.Println("AbortMultipartUpload failed ")
return
}
return
} else {
fmt.Println("CompleteMultipartUpload Ok")
}
}
func InitMultipartUpload(s3Client *s3.S3, bucketName, objectName string) (uploadId string, err error) {
resp, err := s3Client.CreateMultipartUpload(&s3.CreateMultipartUploadInput{Bucket: aws.String(bucketName),
Key: aws.String(objectName)})
if err != nil {
fmt.Println(err.Error())
return
} else {
uploadId = *resp.UploadId
fmt.Println("uploadId : ", uploadId)
return
}
}
func UploadPart(s3Client *s3.S3, bucketName, objectName string, uploadId string, partNum int64, buffer *bytes.Reader) (resp *s3.UploadPartOutput, err error) {
params := &s3.UploadPartInput{
Body: buffer,
Bucket: aws.String(bucketName),
Key: aws.String(objectName),
PartNumber: aws.Int64(partNum),
UploadId: aws.String(uploadId),
ContentLength: aws.Int64(buffer.Size()),
}
res, err := s3Client.UploadPart(params)
if err != nil {
fmt.Printf("err: %v\n", err)
}
fmt.Printf("UploadPart res: %v\n", res)
return res, err
}