Merge pull request #164 from agin719/cos-v4-dev

Cos v4 dev

auth.go

@@ -64,12 +64,6 @@ var NeedSignHeaders = map[string]bool{
 	"x-cos-object-type":              true,
 }
 
-var ciParameters = map[string]bool{
-	"imagemogr2/": true,
-	"watermark/":  true,
-	"imageview2/": true,
-}
-
 // 非线程安全，只能在进程初始化（而不是Client初始化）时做设置
 func SetNeedSignHeaders(key string, val bool) {
 	NeedSignHeaders[key] = val
@@ -88,7 +82,7 @@ func safeURLEncode(s string) string {
 type valuesSignMap map[string][]string
 
 func (vs valuesSignMap) Add(key, value string) {
-	key = strings.ToLower(key)
+	key = strings.ToLower(safeURLEncode(key))
 	vs[key] = append(vs[key], value)
 }
 
@@ -106,7 +100,7 @@ func (vs valuesSignMap) Encode() string {
 		for _, val := range items {
 			pairs = append(
 				pairs,
-				fmt.Sprintf("%s=%s", safeURLEncode(k), safeURLEncode(val)))
+				fmt.Sprintf("%s=%s", k, safeURLEncode(val)))
 		}
 	}
 	return strings.Join(pairs, "&")
@@ -232,12 +226,9 @@ func genFormatString(method string, uri url.URL, formatParameters, formatHeaders
 func genFormatParameters(parameters url.Values) (formatParameters string, signedParameterList []string) {
 	ps := valuesSignMap{}
 	for key, values := range parameters {
-		key = strings.ToLower(key)
 		for _, value := range values {
-			if !isCIParameter(key) {
-				ps.Add(key, value)
-				signedParameterList = append(signedParameterList, key)
-			}
+			ps.Add(key, value)
+			signedParameterList = append(signedParameterList, strings.ToLower(safeURLEncode(key)))
 		}
 	}
 	//formatParameters = strings.ToLower(ps.Encode())
@@ -250,11 +241,10 @@ func genFormatParameters(parameters url.Values) (formatParameters string, signed
 func genFormatHeaders(headers http.Header) (formatHeaders string, signedHeaderList []string) {
 	hs := valuesSignMap{}
 	for key, values := range headers {
-		key = strings.ToLower(key)
-		for _, value := range values {
-			if isSignHeader(key) {
+		if isSignHeader(strings.ToLower(key)) {
+		    for _, value := range values {
 				hs.Add(key, value)
-				signedHeaderList = append(signedHeaderList, key)
+				signedHeaderList = append(signedHeaderList, strings.ToLower(safeURLEncode(key)))
 			}
 		}
 	}
@@ -277,15 +267,6 @@ func calHMACDigest(key, msg, signMethod string) []byte {
 	return h.Sum(nil)
 }
 
-func isCIParameter(key string) bool {
-	for k, v := range ciParameters {
-		if strings.HasPrefix(key, k) && v {
-			return true
-		}
-	}
-	return false
-}
-
 func isSignHeader(key string) bool {
 	for k, v := range NeedSignHeaders {
 		if key == k && v {

bucket.go

@@ -111,6 +111,17 @@ func (s *BucketService) Head(ctx context.Context) (*Response, error) {
 	return resp, err
 }
 
+func (s *BucketService) IsExist(ctx context.Context) (bool, error) {
+	_, err := s.Head(ctx)
+	if err == nil {
+		return true, nil
+	}
+	if IsNotFoundError(err) {
+		return false, nil
+	}
+	return false, err
+}
+
 // Bucket is the meta info of Bucket
 type Bucket struct {
 	Name         string

bucket_test.go

@@ -151,6 +151,24 @@ func TestBucketService_Head(t *testing.T) {
 	}
 }
 
+func TestBucketService_IsExist(t *testing.T) {
+	setup()
+	defer teardown()
+
+	mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
+		testMethod(t, r, http.MethodHead)
+		w.WriteHeader(http.StatusNotFound)
+	})
+
+	isExisted, err := client.Bucket.IsExist(context.Background())
+	if err != nil {
+		t.Fatalf("Bucket.Head returned error: %v", err)
+	}
+	if isExisted != false {
+		t.Errorf("bucket IsExist failed")
+	}
+}
+
 func TestBucketService_GetObjectVersions(t *testing.T) {
 	setup()
 	defer teardown()

ci.go

@@ -829,9 +829,9 @@ func (s *CIService) PutFromFile(ctx context.Context, name string, filePath strin
 func (s *CIService) Get(ctx context.Context, name string, operation string, opt *ObjectGetOptions, id ...string) (*Response, error) {
 	var u string
 	if len(id) == 1 {
-		u = fmt.Sprintf("/%s?versionId=%s&%s", encodeURIComponent(name), id[0], operation)
+		u = fmt.Sprintf("/%s?versionId=%s&%s", encodeURIComponent(name), id[0], encodeURIComponent(operation))
 	} else if len(id) == 0 {
-		u = fmt.Sprintf("/%s?%s", encodeURIComponent(name), operation)
+		u = fmt.Sprintf("/%s?%s", encodeURIComponent(name), encodeURIComponent(operation))
 	} else {
 		return nil, errors.New("wrong params")
 	}

cos.go

@@ -23,7 +23,7 @@ import (
 
 const (
 	// Version current go sdk version
-	Version               = "0.7.32"
+	Version               = "0.7.33"
 	userAgent             = "cos-go-sdk-v5/" + Version
 	contentTypeXML        = "application/xml"
 	defaultServiceBaseURL = "http://service.cos.myqcloud.com"

example/bucket/getObjectVersion.go

@@ -50,15 +50,26 @@ func main() {
 		},
 	})
 
-	opt := &cos.BucketGetObjectVersionsOptions{
-		Delimiter: "/",
-		MaxKeys:   1,
+	keyMarker := ""
+	versionIdMarker := ""
+	isTruncated := true
+	opt := &cos.BucketGetObjectVersionsOptions{}
+	for isTruncated {
+		opt.KeyMarker = keyMarker
+		opt.VersionIdMarker = versionIdMarker
+		v, _, err := c.Bucket.GetObjectVersions(context.Background(), opt)
+		if err != nil {
+			log_status(err)
+			break
+		}
+		for _, vc := range v.Version {
+			fmt.Printf("Version: %v, %v, %v, %v\n", vc.Key, vc.Size, vc.VersionId, vc.IsLatest)
+		}
+		for _, dc := range v.DeleteMarker {
+			fmt.Printf("DeleteMarker: %v, %v, %v\n", dc.Key, dc.VersionId, dc.IsLatest)
+		}
+		keyMarker = v.NextKeyMarker
+		versionIdMarker = v.NextVersionIdMarker
+		isTruncated = v.IsTruncated
 	}
-	v, _, err := c.Bucket.GetObjectVersions(context.Background(), opt)
-	log_status(err)
-
-	for _, c := range v.Version {
-		fmt.Printf("%v, %v, %v\n", c.Key, c.Size, c.IsLatest)
-	}
-
 }

example/bucket/is_exist.go

@@ -0,0 +1,41 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"net/url"
+	"os"
+
+	"net/http"
+
+	"github.com/tencentyun/cos-go-sdk-v5"
+	"github.com/tencentyun/cos-go-sdk-v5/debug"
+)
+
+func main() {
+	u, _ := url.Parse("https://test-1259654469.cos.ap-guangzhou.myqcloud.com")
+	b := &cos.BaseURL{
+		BucketURL: u,
+	}
+	c := cos.NewClient(b, &http.Client{
+		Transport: &cos.AuthorizationTransport{
+			SecretID:  os.Getenv("COS_SECRETID"),
+			SecretKey: os.Getenv("COS_SECRETKEY"),
+			Transport: &debug.DebugRequestTransport{
+				RequestHeader:  true,
+				RequestBody:    true,
+				ResponseHeader: true,
+				ResponseBody:   true,
+			},
+		},
+	})
+
+	ok, err := c.Bucket.IsExist(context.Background())
+	if err == nil && ok {
+		fmt.Printf("bucket exists\n")
+	} else if err != nil {
+		fmt.Printf("head bucket failed: %v\n", err)
+	} else {
+		fmt.Printf("bucket does not exist\n")
+	}
+}

example/object/append.go

@@ -34,15 +34,20 @@ func log_status(err error) {
 }
 
 func main() {
+	// 存储桶名称，由bucketname-appid 组成，appid必须填入，可以在COS控制台查看存储桶名称。 https://console.cloud.tencent.com/cos5/bucket
+	// 替换为用户的 region，存储桶region可以在COS控制台“存储桶概览”查看 https://console.cloud.tencent.com/ ，关于地域的详情见 https://cloud.tencent.com/document/product/436/6224 。
 	u, _ := url.Parse("http://test-1259654469.cos.ap-guangzhou.myqcloud.com")
 	b := &cos.BaseURL{BucketURL: u}
 	c := cos.NewClient(b, &http.Client{
 		Transport: &cos.AuthorizationTransport{
-			SecretID:  os.Getenv("COS_SECRETID"),
+			// 通过环境变量获取密钥
+			// 环境变量 COS_SECRETID 表示用户的 SecretId，登录访问管理控制台查看密钥，https://console.cloud.tencent.com/cam/capi
+			SecretID: os.Getenv("COS_SECRETID"),
+			// 环境变量 COS_SECRETKEY 表示用户的 SecretKey，登录访问管理控制台查看密钥，https://console.cloud.tencent.com/cam/capi
 			SecretKey: os.Getenv("COS_SECRETKEY"),
+			// Debug 模式，把对应 请求头部、请求内容、响应头部、响应内容 输出到标准输出
 			Transport: &debug.DebugRequestTransport{
-				RequestHeader: true,
-				// Notice when put a large file and set need the request body, might happend out of memory error.
+				RequestHeader:  true,
 				RequestBody:    false,
 				ResponseHeader: true,
 				ResponseBody:   false,

example/object/batchGet.go

@@ -6,7 +6,8 @@ import (
 	"net/http"
 	"net/url"
 	"os"
-	"path/filepath"
+	"path"
+	"strings"
 	"sync"
 
 	"github.com/tencentyun/cos-go-sdk-v5"
@@ -32,11 +33,11 @@ func log_status(err error) {
 	}
 }
 
-func upload(wg *sync.WaitGroup, c *cos.Client, keysCh <-chan string) {
+func download(wg *sync.WaitGroup, c *cos.Client, keysCh <-chan []string) {
 	defer wg.Done()
-	for key := range keysCh {
-		// 下载文件到当前目录
-		_, filename := filepath.Split(key)
+	for keys := range keysCh {
+		key := keys[0]
+		filename := keys[1]
 		_, err := c.Object.GetToFile(context.Background(), key, filename, nil)
 		if err != nil {
 			log_status(err)
@@ -51,24 +52,52 @@ func main() {
 			SecretID:  os.Getenv("COS_SECRETID"),
 			SecretKey: os.Getenv("COS_SECRETKEY"),
 			Transport: &debug.DebugRequestTransport{
-				RequestHeader: true,
+				RequestHeader: false,
 				// Notice when put a large file and set need the request body, might happend out of memory error.
 				RequestBody:    false,
-				ResponseHeader: true,
+				ResponseHeader: false,
 				ResponseBody:   false,
 			},
 		},
 	})
-	keysCh := make(chan string, 2)
-	keys := []string{"test/test1", "test/test2", "test/test3"}
+	// 多线程执行
+	keysCh := make(chan []string, 3)
 	var wg sync.WaitGroup
-	threadpool := 2
+	threadpool := 3
 	for i := 0; i < threadpool; i++ {
 		wg.Add(1)
-		go upload(&wg, c, keysCh)
+		go download(&wg, c, keysCh)
 	}
-	for _, key := range keys {
-		keysCh <- key
+	isTruncated := true
+	prefix := "dir" // 下载 dir 目录下所有文件
+	marker := ""
+	localDir := "./local/"
+	for isTruncated {
+		opt := &cos.BucketGetOptions{
+			Prefix:       prefix,
+			Marker:       marker,
+			EncodingType: "url", // url编码
+		}
+		// 列出目录
+		v, _, err := c.Bucket.Get(context.Background(), opt)
+		if err != nil {
+			fmt.Println(err)
+			break
+		}
+		for _, c := range v.Contents {
+			key, _ := cos.DecodeURIComponent(c.Key) //EncodingType: "url"，先对 key 进行 url decode
+			localfile := localDir + key
+			if _, err := os.Stat(path.Dir(localfile)); err != nil && os.IsNotExist(err) {
+				os.MkdirAll(path.Dir(localfile), os.ModePerm)
+			}
+			// 目录不需要下载
+			if strings.HasSuffix(localfile, "/") {
+				continue
+			}
+			keysCh <- []string{key, localfile}
+		}
+		marker = v.NextMarker
+		isTruncated = v.IsTruncated
 	}
 	close(keysCh)
 	wg.Wait()

example/object/copy.go

@@ -36,12 +36,18 @@ func log_status(err error) {
 }
 
 func main() {
+	// 存储桶名称，由bucketname-appid 组成，appid必须填入，可以在COS控制台查看存储桶名称。 https://console.cloud.tencent.com/cos5/bucket
+	// 替换为用户的 region，存储桶region可以在COS控制台“存储桶概览”查看 https://console.cloud.tencent.com/ ，关于地域的详情见 https://cloud.tencent.com/document/product/436/6224 。
 	u, _ := url.Parse("https://test-1259654469.cos.ap-guangzhou.myqcloud.com")
 	b := &cos.BaseURL{BucketURL: u}
 	c := cos.NewClient(b, &http.Client{
 		Transport: &cos.AuthorizationTransport{
+			// 通过环境变量获取密钥
+			// 环境变量 COS_SECRETID 表示用户的 SecretId，登录访问管理控制台查看密钥，https://console.cloud.tencent.com/cam/capi
 			SecretID:  os.Getenv("COS_SECRETID"),
+			// 环境变量 COS_SECRETKEY 表示用户的 SecretKey，登录访问管理控制台查看密钥，https://console.cloud.tencent.com/cam/capi
 			SecretKey: os.Getenv("COS_SECRETKEY"),
+			// Debug 模式，把对应 请求头部、请求内容、响应头部、响应内容 输出到标准输出
 			Transport: &debug.DebugRequestTransport{
 				RequestHeader:  true,
 				RequestBody:    true,

example/object/download.go

@@ -31,12 +31,18 @@ func log_status(err error) {
 }
 
 func main() {
+	// 存储桶名称，由bucketname-appid 组成，appid必须填入，可以在COS控制台查看存储桶名称。 https://console.cloud.tencent.com/cos5/bucket
+	// 替换为用户的 region，存储桶region可以在COS控制台“存储桶概览”查看 https://console.cloud.tencent.com/ ，关于地域的详情见 https://cloud.tencent.com/document/product/436/6224 。
 	u, _ := url.Parse("https://test-1259654469.cos.ap-guangzhou.myqcloud.com")
 	b := &cos.BaseURL{BucketURL: u}
 	c := cos.NewClient(b, &http.Client{
 		Transport: &cos.AuthorizationTransport{
+			// 通过环境变量获取密钥
+			// 环境变量 COS_SECRETID 表示用户的 SecretId，登录访问管理控制台查看密钥，https://console.cloud.tencent.com/cam/capi
 			SecretID:  os.Getenv("COS_SECRETID"),
+			// 环境变量 COS_SECRETKEY 表示用户的 SecretKey，登录访问管理控制台查看密钥，https://console.cloud.tencent.com/cam/capi
 			SecretKey: os.Getenv("COS_SECRETKEY"),
+			// Debug 模式，把对应 请求头部、请求内容、响应头部、响应内容 输出到标准输出
 			Transport: &debug.DebugRequestTransport{
 				RequestHeader:  false,
 				RequestBody:    false,

example/object/get.go

@@ -33,12 +33,18 @@ func log_status(err error) {
 }
 
 func main() {
+	// 存储桶名称，由bucketname-appid 组成，appid必须填入，可以在COS控制台查看存储桶名称。 https://console.cloud.tencent.com/cos5/bucket
+	// 替换为用户的 region，存储桶region可以在COS控制台“存储桶概览”查看 https://console.cloud.tencent.com/ ，关于地域的详情见 https://cloud.tencent.com/document/product/436/6224 。
 	u, _ := url.Parse("https://test-1259654469.cos.ap-guangzhou.myqcloud.com")
 	b := &cos.BaseURL{BucketURL: u}
 	c := cos.NewClient(b, &http.Client{
 		Transport: &cos.AuthorizationTransport{
-			SecretID:  os.Getenv("COS_SECRETID"),
+			// 通过环境变量获取密钥
+			// 环境变量 COS_SECRETID 表示用户的 SecretId，登录访问管理控制台查看密钥，https://console.cloud.tencent.com/cam/capi
+			SecretID: os.Getenv("COS_SECRETID"),
+			// 环境变量 COS_SECRETKEY 表示用户的 SecretKey，登录访问管理控制台查看密钥，https://console.cloud.tencent.com/cam/capi
 			SecretKey: os.Getenv("COS_SECRETKEY"),
+			// Debug 模式，把对应 请求头部、请求内容、响应头部、响应内容 输出到标准输出
 			Transport: &debug.DebugRequestTransport{
 				RequestHeader:  true,
 				RequestBody:    true,