update README and fix minio pagination performance (#9)

.github/scripts/plugin-manifest.yaml

@@ -1,6 +1,6 @@
 name: ndc-storage
 version: "${CLI_VERSION}"
-shortDescription: "CLI plugin for Hasura Prometheus data connector"
+shortDescription: "CLI plugin for Hasura Storage data connector"
 homepage: https://github.com/hasura/ndc-storage
 hidden: true
 platforms:

README.md

@@ -19,6 +19,22 @@ This connector is built using the [Go Data Connector SDK](https://github.com/has
 
 (\*): Support Amazon S3 Compatible Cloud Storage providers. The connector uses [MinIO Go Client SDK](https://github.com/minio/minio-go) behind the scenes.
 
+### Supported Features
+
+Below, you'll find a matrix of all supported features for the Storage connector:
+
+| Feature                | Supported | Notes |
+| ---------------------- | --------- | ----- |
+| List Buckets           | ✅        |       |
+| Create Bucket          | ✅        |       |
+| Update Bucket          | ✅        |       |
+| Delete Bucket          | ✅        |       |
+| List Objects           | ✅        |       |
+| Upload Object          | ✅        |       |
+| Download Object        | ✅        |       |
+| Delete Object          | ✅        |       |
+| Generate Presigned-URL | ✅        |       |
+
 ## Get Started
 
 Follow the [Quick Start Guide](https://hasura.io/docs/3.0/getting-started/overview/) in Hasura DDN docs. At the `Connect to data` step, choose the `hasura/storage` data connector from the dropdown and follow the interactive prompts to set the required environment variables.

connector/storage/minio/object.go

@@ -21,17 +21,12 @@ import (
 )
 
 // ListObjects list objects in a bucket.
-func (mc *Client) ListObjects(ctx context.Context, bucketName string, opts *common.ListStorageObjectsOptions, predicate func(string) bool) (*common.StorageObjectListResults, error) { //nolint:funlen
+func (mc *Client) ListObjects(ctx context.Context, bucketName string, opts *common.ListStorageObjectsOptions, predicate func(string) bool) (*common.StorageObjectListResults, error) {
 	ctx, span := mc.startOtelSpan(ctx, "ListObjects", bucketName)
 	defer span.End()
 
 	logger := connector.GetLogger(ctx)
 	maxResults := opts.MaxResults
-	// Do not set the limit if the post-predicate function exists.
-	// Results will be filtered and paginated by the client.
-	if predicate != nil {
-		opts.MaxResults = 0
-	}
 
 	objChan := mc.client.ListObjects(ctx, bucketName, mc.validateListObjectsOptions(span, opts))
 	minioObjects := []minio.ObjectInfo{}

connector/storage/minio/utils.go

@@ -229,7 +229,6 @@ func (mc *Client) validateListObjectsOptions(span trace.Span, opts *common.ListS
 		WithMetadata: opts.Include.Metadata,
 		Prefix:       opts.Prefix,
 		Recursive:    !opts.Hierarchy,
-		MaxKeys:      opts.MaxResults,
 		StartAfter:   opts.StartAfter,
 	}
 }

docs/objects.md

@@ -17,6 +17,9 @@ query PresignedUploadUrl {
 }
 ```
 
+> [!NOTE]
+> If the endpoint host is a private DNS the presigned-URL cannot be accessed. In this case, you must configure the `publicHost` in the client settings.
+
 ### Direct Upload
 
 The object data must be encoded as a base64 string.
@@ -64,6 +67,9 @@ query GetSignedDownloadURL {
 }
 ```
 
+> [!NOTE]
+> If the endpoint host is a private DNS the presigned-URL cannot be accessed. In this case, you must configure the `publicHost` in the client settings.
+
 ### Direct Download
 
 The response is a base64-encode string. The client must decode the string to get the raw content.
@@ -118,10 +124,10 @@ query DownloadObjectText {
 You can use either `clientId`, `bucket`, `prefix`, or `where` boolean expression to filter object results. The `where` argument is mainly used for permissions. The filter expression is evaluated twice, before and after fetching the results. Cloud storage APIs usually support filtering by the name prefix only. Other operators (`_contains`, `_icontains`) are filtered from fetched results by pure logic.
 
 ```graphql
-query ListObjects {
+query RelayListObjects {
   storageObjectConnections(
     prefix: "hello"
-    where: { object: { _contains: "world" } }
+    where: { name: { _contains: "world" } }
   ) {
     edges {
       cursor
@@ -135,12 +141,27 @@ query ListObjects {
 }
 ```
 
+In `storageObjects` query, the `prefix` argument doesn't exist. You should use the `_starts_with` operator in the `where` predicate instead.
+
+```graphql
+query ListObjects {
+  storageObjects(
+    where: { name: { _starts_with: "hello", _contains: "world" } }
+  ) {
+    clientId
+    bucket
+    name
+    # ...
+  }
+}
+```
+
 #### Pagination
 
 Relay style suits object listing because most cloud storage services only support cursor-based pagination. The object name is used as the cursor ID.
 
 ```graphql
-query ListObjectConnections {
+query RelayListObjects {
   storageObjectConnections(after: "hello.txt", first: 3) {
     pageInfo {
       hasNextPage
@@ -182,7 +203,15 @@ query ListObjectConnections {
 }
 ```
 
-Or you can use the `storageObjects` collection. The response structure is simpler but some argument isn't functional such as sorting.
+The `storageObjects` collection doesn't return pagination information. You need to get the `name` in the last object to paginate by the `after` cursor.
+
+> [!NOTE]
+> **Why do `storageObjects` and `storageObjectConnections` operations exist?**
+> 
+> The `storageObjects` collection provides a simpler response structure that PromptQL can query easily. The `storageObjectConnections` function returns a better cursor-based pagination response on but the schema is complicated for PromptQL to understand.
+
+> [!NOTE]
+> Sorting isn't supported.
 
 ```graphql
 query ListObjects {