From 126a1b7f4111dd51ec02d73568d8e864239679f2 Mon Sep 17 00:00:00 2001 From: Anis Elleuch Date: Sun, 17 Jul 2016 18:31:27 +0100 Subject: [PATCH] Add ListObjectsV2 API (#447) --- api-list.go | 175 +++++++++++++++++++++++++++++++++++++- api-s3-datatypes.go | 29 +++++++ api_functional_v2_test.go | 12 +++ api_functional_v4_test.go | 12 +++ 4 files changed, 226 insertions(+), 2 deletions(-) diff --git a/api-list.go b/api-list.go index ed1a4a6c0b..60c03e3a8e 100644 --- a/api-list.go +++ b/api-list.go @@ -53,6 +53,179 @@ func (c Client) ListBuckets() ([]BucketInfo, error) { return listAllMyBucketsResult.Buckets.Bucket, nil } +/// Bucket Read Operations. + +// ListObjectsV2 lists all objects matching the objectPrefix from +// the specified bucket. If recursion is enabled it would list +// all subdirectories and all its contents. +// +// Your input parameters are just bucketName, objectPrefix, recursive +// and a done channel for pro-actively closing the internal go +// routine. If you enable recursive as 'true' this function will +// return back all the objects in a given bucket name and object +// prefix. +// +// api := client.New(....) +// // Create a done channel. +// doneCh := make(chan struct{}) +// defer close(doneCh) +// // Recurively list all objects in 'mytestbucket' +// recursive := true +// for message := range api.ListObjectsV2("mytestbucket", "starthere", recursive, doneCh) { +// fmt.Println(message) +// } +// +func (c Client) ListObjectsV2(bucketName, objectPrefix string, recursive bool, doneCh <-chan struct{}) <-chan ObjectInfo { + // Allocate new list objects channel. + objectStatCh := make(chan ObjectInfo, 1) + // Default listing is delimited at "/" + delimiter := "/" + if recursive { + // If recursive we do not delimit. + delimiter = "" + } + // Validate bucket name. + if err := isValidBucketName(bucketName); err != nil { + defer close(objectStatCh) + objectStatCh <- ObjectInfo{ + Err: err, + } + return objectStatCh + } + // Validate incoming object prefix. + if err := isValidObjectPrefix(objectPrefix); err != nil { + defer close(objectStatCh) + objectStatCh <- ObjectInfo{ + Err: err, + } + return objectStatCh + } + + // Initiate list objects goroutine here. + go func(objectStatCh chan<- ObjectInfo) { + defer close(objectStatCh) + // Save continuationToken for next request. + var continuationToken string + for { + // Get list of objects a maximum of 1000 per request. + result, err := c.listObjectsV2Query(bucketName, objectPrefix, continuationToken, delimiter, 1000) + if err != nil { + objectStatCh <- ObjectInfo{ + Err: err, + } + return + } + + // If contents are available loop through and send over channel. + for _, object := range result.Contents { + // Save the marker. + select { + // Send object content. + case objectStatCh <- object: + // If receives done from the caller, return here. + case <-doneCh: + return + } + } + + // Send all common prefixes if any. + // NOTE: prefixes are only present if the request is delimited. + for _, obj := range result.CommonPrefixes { + object := ObjectInfo{} + object.Key = obj.Prefix + object.Size = 0 + select { + // Send object prefixes. + case objectStatCh <- object: + // If receives done from the caller, return here. + case <-doneCh: + return + } + } + + // If continuation token present, save it for next request. + if result.NextContinuationToken != "" { + continuationToken = result.NextContinuationToken + } + + // Listing ends result is not truncated, return right here. + if !result.IsTruncated { + return + } + } + }(objectStatCh) + return objectStatCh +} + +// listObjectsV2Query - (List Objects V2) - List some or all (up to 1000) of the objects in a bucket. +// +// You can use the request parameters as selection criteria to return a subset of the objects in a bucket. +// request parameters :- +// --------- +// ?continuation-token - Specifies the key to start with when listing objects in a bucket. +// ?delimiter - A delimiter is a character you use to group keys. +// ?prefix - Limits the response to keys that begin with the specified prefix. +// ?max-keys - Sets the maximum number of keys returned in the response body. +func (c Client) listObjectsV2Query(bucketName, objectPrefix, continuationToken, delimiter string, maxkeys int) (listBucketV2Result, error) { + // Validate bucket name. + if err := isValidBucketName(bucketName); err != nil { + return listBucketV2Result{}, err + } + // Validate object prefix. + if err := isValidObjectPrefix(objectPrefix); err != nil { + return listBucketV2Result{}, err + } + // Get resources properly escaped and lined up before + // using them in http request. + urlValues := make(url.Values) + + // Always set list-type in ListObjects V2 + urlValues.Set("list-type", "2") + + // Set object prefix. + if objectPrefix != "" { + urlValues.Set("prefix", objectPrefix) + } + // Set continuation token + if continuationToken != "" { + urlValues.Set("continuation-token", continuationToken) + } + // Set delimiter. + if delimiter != "" { + urlValues.Set("delimiter", delimiter) + } + + // maxkeys should default to 1000 or less. + if maxkeys == 0 || maxkeys > 1000 { + maxkeys = 1000 + } + // Set max keys. + urlValues.Set("max-keys", fmt.Sprintf("%d", maxkeys)) + + // Execute GET on bucket to list objects. + resp, err := c.executeMethod("GET", requestMetadata{ + bucketName: bucketName, + queryValues: urlValues, + }) + defer closeResponse(resp) + if err != nil { + return listBucketV2Result{}, err + } + if resp != nil { + if resp.StatusCode != http.StatusOK { + return listBucketV2Result{}, httpRespToErrorResponse(resp, bucketName, "") + } + } + + // Decode listBuckets XML. + listBucketResult := listBucketV2Result{} + err = xmlDecoder(resp.Body, &listBucketResult) + if err != nil { + return listBucketResult, err + } + return listBucketResult, nil +} + // ListObjects - (List Objects) - List some objects or all recursively. // // ListObjects lists all objects matching the objectPrefix from @@ -158,8 +331,6 @@ func (c Client) ListObjects(bucketName, objectPrefix string, recursive bool, don return objectStatCh } -/// Bucket Read Operations. - // listObjects - (List Objects) - List some or all (up to 1000) of the objects in a bucket. // // You can use the request parameters as selection criteria to return a subset of the objects in a bucket. diff --git a/api-s3-datatypes.go b/api-s3-datatypes.go index ca81e302d2..a07bbcf06f 100644 --- a/api-s3-datatypes.go +++ b/api-s3-datatypes.go @@ -41,6 +41,35 @@ type commonPrefix struct { Prefix string } +// listBucketResult container for listObjects V2 response. +type listBucketV2Result struct { + // A response can contain CommonPrefixes only if you have + // specified a delimiter. + CommonPrefixes []commonPrefix + // Metadata about each object returned. + Contents []ObjectInfo + Delimiter string + + // Encoding type used to encode object keys in the response. + EncodingType string + + // A flag that indicates whether or not ListObjects returned all of the results + // that satisfied the search criteria. + IsTruncated bool + MaxKeys int64 + Name string + + // Hold the token that will be sent in the next request to fetch the next group of keys + NextContinuationToken string + + ContinuationToken string + Prefix string + + // FetchOwner and StartAfter are currently not used + FetchOwner string + StartAfter string +} + // listBucketResult container for listObjects response. type listBucketResult struct { // A response can contain CommonPrefixes only if you have diff --git a/api_functional_v2_test.go b/api_functional_v2_test.go index ff4a1bfc55..718d44485b 100644 --- a/api_functional_v2_test.go +++ b/api_functional_v2_test.go @@ -1144,6 +1144,18 @@ func TestFunctionalV2(t *testing.T) { t.Fatal("Error: object " + objectName + " not found.") } + objFound = false + isRecursive = true // Recursive is true. + for obj := range c.ListObjects(bucketName, objectName, isRecursive, doneCh) { + if obj.Key == objectName { + objFound = true + break + } + } + if !objFound { + t.Fatal("Error: object " + objectName + " not found.") + } + incompObjNotFound := true for objIncompl := range c.ListIncompleteUploads(bucketName, objectName, isRecursive, doneCh) { if objIncompl.Key != "" { diff --git a/api_functional_v4_test.go b/api_functional_v4_test.go index 168833362d..25fc50d03c 100644 --- a/api_functional_v4_test.go +++ b/api_functional_v4_test.go @@ -1215,6 +1215,18 @@ func TestFunctional(t *testing.T) { t.Fatal("Error: object " + objectName + " not found.") } + objFound = false + isRecursive = true // Recursive is true. + for obj := range c.ListObjectsV2(bucketName, objectName, isRecursive, doneCh) { + if obj.Key == objectName { + objFound = true + break + } + } + if !objFound { + t.Fatal("Error: object " + objectName + " not found.") + } + incompObjNotFound := true for objIncompl := range c.ListIncompleteUploads(bucketName, objectName, isRecursive, doneCh) { if objIncompl.Key != "" {