diff --git a/.env b/.env index c4125ee..2e724b4 100644 --- a/.env +++ b/.env @@ -13,4 +13,9 @@ APP_STORAGE_WRITE_URL="http://local.hasura.dev:8080" AZURE_STORAGE_ENDPOINT=http://local.hasura.dev:10000 AZURE_STORAGE_DEFAULT_BUCKET=azure-test AZURE_STORAGE_ACCOUNT_NAME=local -AZURE_STORAGE_ACCOUNT_KEY=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw== \ No newline at end of file +AZURE_STORAGE_ACCOUNT_KEY=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw== + +GOOGLE_STORAGE_DEFAULT_BUCKET=gcp-bucket +GOOGLE_STORAGE_ENDPOINT=http://gcp-storage-emulator:4443/storage/v1/ +GOOGLE_STORAGE_PUBLIC_HOST=http://localhost:10010 +GOOGLE_PROJECT_ID=test-local-project diff --git a/compose.yaml b/compose.yaml index 75ae592..dbdfd59 100644 --- a/compose.yaml +++ b/compose.yaml @@ -8,6 +8,7 @@ services: - 8080:8080 volumes: - ./tests/configuration:/etc/connector:ro + - ./tests/certs/service_account.json:/service_account.json extra_hosts: - local.hasura.dev=host-gateway environment: @@ -26,6 +27,11 @@ services: AZURE_STORAGE_DEFAULT_BUCKET: $AZURE_STORAGE_DEFAULT_BUCKET AZURE_STORAGE_ACCOUNT_NAME: $AZURE_STORAGE_ACCOUNT_NAME AZURE_STORAGE_ACCOUNT_KEY: $AZURE_STORAGE_ACCOUNT_KEY + GOOGLE_STORAGE_DEFAULT_BUCKET: $GOOGLE_STORAGE_DEFAULT_BUCKET + GOOGLE_STORAGE_ENDPOINT: $GOOGLE_STORAGE_ENDPOINT + GOOGLE_STORAGE_PUBLIC_HOST: $GOOGLE_STORAGE_PUBLIC_HOST + GOOGLE_PROJECT_ID: $GOOGLE_PROJECT_ID + GOOGLE_STORAGE_CREDENTIALS_FILE: /service_account.json HASURA_LOG_LEVEL: debug OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4317 OTEL_METRICS_EXPORTER: prometheus @@ -68,6 +74,21 @@ services: environment: AZURITE_ACCOUNTS: "${AZURE_STORAGE_ACCOUNT_NAME}:${AZURE_STORAGE_ACCOUNT_KEY}" + # https://github.com/fsouza/fake-gcs-server + gcp-storage-emulator: + image: fsouza/fake-gcs-server:1.52.1 + command: + [ + "-scheme", + "http", + "-public-host", + "http://localhost:10010", + "-log-level", + "debug", + ] + ports: + - "10010:4443" + volumes: minio_data: s3_data: diff --git a/connector/connector.go b/connector/connector.go index cd8aaf2..3c77d15 100644 --- a/connector/connector.go +++ b/connector/connector.go @@ -11,6 +11,7 @@ import ( "github.com/hasura/ndc-sdk-go/connector" "github.com/hasura/ndc-sdk-go/schema" "github.com/hasura/ndc-sdk-go/utils" + "github.com/hasura/ndc-storage/configuration/version" "github.com/hasura/ndc-storage/connector/functions" "github.com/hasura/ndc-storage/connector/storage" "github.com/hasura/ndc-storage/connector/types" @@ -72,7 +73,7 @@ func (c *Connector) ParseConfiguration(ctx context.Context, configurationDir str func (c *Connector) TryInitState(ctx context.Context, configuration *types.Configuration, metrics *connector.TelemetryState) (*types.State, error) { logger := connector.GetLogger(ctx) - manager, err := storage.NewManager(ctx, configuration.Clients, logger) + manager, err := storage.NewManager(ctx, configuration.Clients, logger, version.BuildVersion) if err != nil { return nil, err } diff --git a/connector/connector_test.go b/connector/connector_test.go index 031a60f..c08e3f5 100644 --- a/connector/connector_test.go +++ b/connector/connector_test.go @@ -9,7 +9,6 @@ import ( ) func TestConnector(t *testing.T) { - azureBlobEndpoint := "http://local.hasura.dev:10000" azureAccountName := "local" azureAccountKey := "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==" @@ -27,6 +26,10 @@ func TestConnector(t *testing.T) { t.Setenv("AZURE_STORAGE_ACCOUNT_NAME", azureAccountName) t.Setenv("AZURE_STORAGE_ACCOUNT_KEY", azureAccountKey) t.Setenv("AZURE_STORAGE_CONNECTION_STRING", fmt.Sprintf("DefaultEndpointsProtocol=http;AccountName=%s;AccountKey=%s;BlobEndpoint=%s", azureAccountName, azureAccountKey, azureBlobEndpoint)) + t.Setenv("GOOGLE_STORAGE_DEFAULT_BUCKET", "gcp-bucket") + t.Setenv("GOOGLE_PROJECT_ID", "test-local-project") + t.Setenv("GOOGLE_STORAGE_ENDPOINT", "http://localhost:10010/storage/v1/") + t.Setenv("GOOGLE_STORAGE_CREDENTIALS_FILE", "../tests/certs/service_account.json") for _, dir := range []string{"01-setup", "02-get", "03-cleanup"} { ndctest.TestConnector(t, &Connector{}, ndctest.TestConnectorOptions{ diff --git a/connector/functions/bucket.go b/connector/functions/bucket.go index c719e95..b02c2fa 100644 --- a/connector/functions/bucket.go +++ b/connector/functions/bucket.go @@ -3,6 +3,7 @@ package functions import ( "context" + "github.com/hasura/ndc-sdk-go/schema" "github.com/hasura/ndc-sdk-go/utils" "github.com/hasura/ndc-storage/connector/functions/internal" "github.com/hasura/ndc-storage/connector/storage/common" @@ -19,147 +20,88 @@ func ProcedureCreateStorageBucket(ctx context.Context, state *types.State, args } // FunctionStorageBuckets list all buckets. -func FunctionStorageBuckets(ctx context.Context, state *types.State, args *common.ListStorageBucketArguments) ([]common.StorageBucketInfo, error) { - request := internal.ObjectPredicate{} - - if err := request.EvalSelection(utils.CommandSelectionFieldFromContext(ctx)); err != nil { - return nil, err - } - - return state.Storage.ListBuckets(ctx, args.ClientID, common.BucketOptions{ - IncludeTags: request.Include.Tags, - NumThreads: state.Concurrency.Query, - }) -} - -// FunctionStorageBucketExists checks if a bucket exists. -func FunctionStorageBucketExists(ctx context.Context, state *types.State, args *common.StorageBucketArguments) (bool, error) { - return state.Storage.BucketExists(ctx, args) -} - -// ProcedureRemoveStorageBucket removes a bucket, bucket should be empty to be successfully removed. -func ProcedureRemoveStorageBucket(ctx context.Context, state *types.State, args *common.StorageBucketArguments) (bool, error) { - if err := state.Storage.RemoveBucket(ctx, args); err != nil { - return false, err +func FunctionStorageBuckets(ctx context.Context, state *types.State, args *common.ListStorageBucketArguments) (common.StorageBucketListResults, error) { + if args.MaxResults <= 0 { + return common.StorageBucketListResults{}, schema.UnprocessableContentError("maxResults must be larger than 0", nil) } - return true, nil -} - -// ProcedureSetStorageBucketTags sets tags to a bucket. -func ProcedureSetStorageBucketTags(ctx context.Context, state *types.State, args *common.SetStorageBucketTaggingArguments) (bool, error) { - if err := state.Storage.SetBucketTagging(ctx, args); err != nil { - return false, err + request, err := internal.EvalObjectPredicate(common.StorageBucketArguments{}, "", args.Where, types.QueryVariablesFromContext(ctx)) + if err != nil { + return common.StorageBucketListResults{}, err } - return true, nil -} - -// FunctionStorageBucketPolicy gets access permissions on a bucket or a prefix. -func FunctionStorageBucketPolicy(ctx context.Context, state *types.State, args *common.StorageBucketArguments) (string, error) { - return state.Storage.GetBucketPolicy(ctx, args) -} - -// FunctionStorageBucketNotification gets notification configuration on a bucket. -func FunctionStorageBucketNotification(ctx context.Context, state *types.State, args *common.StorageBucketArguments) (*common.NotificationConfig, error) { - return state.Storage.GetBucketNotification(ctx, args) -} - -// ProcedureSetStorageBucketNotification sets a new notification configuration on a bucket. -func ProcedureSetStorageBucketNotification(ctx context.Context, state *types.State, args *common.SetBucketNotificationArguments) (bool, error) { - if err := state.Storage.SetBucketNotification(ctx, args); err != nil { - return false, err + if !request.IsValid { + return common.StorageBucketListResults{ + Buckets: []common.StorageBucket{}, + }, nil } - return true, nil -} - -// ProcedureSetStorageBucketLifecycle sets lifecycle on bucket or an object prefix. -func ProcedureSetStorageBucketLifecycle(ctx context.Context, state *types.State, args *common.SetStorageBucketLifecycleArguments) (bool, error) { - err := state.Storage.SetBucketLifecycle(ctx, args) - if err != nil { - return false, err + if err := request.EvalSelection(utils.CommandSelectionFieldFromContext(ctx)); err != nil { + return common.StorageBucketListResults{}, err } - return true, nil -} - -// FunctionStorageBucketLifecycle gets lifecycle on a bucket or a prefix. -func FunctionStorageBucketLifecycle(ctx context.Context, state *types.State, args *common.StorageBucketArguments) (*common.BucketLifecycleConfiguration, error) { - return state.Storage.GetBucketLifecycle(ctx, args) -} - -// ProcedureSetStorageBucketEncryption sets default encryption configuration on a bucket. -func ProcedureSetStorageBucketEncryption(ctx context.Context, state *types.State, args *common.SetStorageBucketEncryptionArguments) (bool, error) { - err := state.Storage.SetBucketEncryption(ctx, args) - if err != nil { - return false, err + predicate := request.BucketPredicate.CheckPostPredicate + if !request.BucketPredicate.HasPostPredicate() { + predicate = nil } - return true, nil -} - -// FunctionStorageBucketEncryption gets default encryption configuration set on a bucket. -func FunctionStorageBucketEncryption(ctx context.Context, state *types.State, args *common.StorageBucketArguments) (*common.ServerSideEncryptionConfiguration, error) { - return state.Storage.GetBucketEncryption(ctx, args) -} - -// ProcedureSetObjectLockConfig sets object lock configuration in given bucket. mode, validity and unit are either all set or all nil. -func ProcedureSetStorageObjectLockConfig(ctx context.Context, state *types.State, args *common.SetStorageObjectLockArguments) (bool, error) { - err := state.Storage.SetObjectLockConfig(ctx, args) + result, err := state.Storage.ListBuckets(ctx, request.ClientID, &common.ListStorageBucketsOptions{ + Prefix: request.BucketPredicate.GetPrefix(), + MaxResults: args.MaxResults, + StartAfter: args.StartAfter, + Include: common.BucketIncludeOptions{ + Tags: request.Include.Tags, + Versioning: request.Include.Versions, + Lifecycle: request.Include.Lifecycle, + Encryption: request.Include.Encryption, + ObjectLock: request.Include.ObjectLock, + }, + NumThreads: state.Concurrency.Query, + }, predicate) if err != nil { - return false, err + return common.StorageBucketListResults{}, err } - return true, nil + return *result, nil } -// FunctionStorageObjectLockConfig gets object lock configuration of given bucket. -func FunctionStorageObjectLockConfig(ctx context.Context, state *types.State, args *common.StorageBucketArguments) (*common.StorageObjectLockConfig, error) { - return state.Storage.GetObjectLockConfig(ctx, args) -} +// FunctionStorageBucket gets a bucket by name. +func FunctionStorageBucket(ctx context.Context, state *types.State, args *common.StorageBucketArguments) (*common.StorageBucket, error) { + request := internal.PredicateEvaluator{} -// ProcedureEnableStorageBucketVersioning enables bucket versioning support. -func ProcedureEnableStorageBucketVersioning(ctx context.Context, state *types.State, args *common.StorageBucketArguments) (bool, error) { - if err := state.Storage.EnableVersioning(ctx, args); err != nil { - return false, err - } - - return true, nil -} - -// ProcedureSuspendStorageBucketVersioning disables bucket versioning support. -func ProcedureSuspendStorageBucketVersioning(ctx context.Context, state *types.State, args *common.StorageBucketArguments) (bool, error) { - if err := state.Storage.SuspendVersioning(ctx, args); err != nil { - return false, err + if err := request.EvalSelection(utils.CommandSelectionFieldFromContext(ctx)); err != nil { + return nil, err } - return true, nil + return state.Storage.GetBucket(ctx, args, common.BucketOptions{ + Include: common.BucketIncludeOptions{ + Tags: request.Include.Tags, + Versioning: request.Include.Versions, + Lifecycle: request.Include.Lifecycle, + Encryption: request.Include.Encryption, + ObjectLock: request.Include.ObjectLock, + }, + NumThreads: state.Concurrency.Query, + }) } -// FunctionStorageBucketVersioning gets versioning configuration set on a bucket. -func FunctionStorageBucketVersioning(ctx context.Context, state *types.State, args *common.StorageBucketArguments) (*common.StorageBucketVersioningConfiguration, error) { - return state.Storage.GetBucketVersioning(ctx, args) +// FunctionStorageBucketExists checks if a bucket exists. +func FunctionStorageBucketExists(ctx context.Context, state *types.State, args *common.StorageBucketArguments) (bool, error) { + return state.Storage.BucketExists(ctx, args) } -// ProcedureSetStorageBucketReplication sets replication configuration on a bucket. Role can be obtained by first defining the replication target on MinIO -// to associate the source and destination buckets for replication with the replication endpoint. -func ProcedureSetStorageBucketReplication(ctx context.Context, state *types.State, args *common.SetStorageBucketReplicationArguments) (bool, error) { - if err := state.Storage.SetBucketReplication(ctx, args); err != nil { +// ProcedureRemoveStorageBucket removes a bucket, bucket should be empty to be successfully removed. +func ProcedureRemoveStorageBucket(ctx context.Context, state *types.State, args *common.StorageBucketArguments) (bool, error) { + if err := state.Storage.RemoveBucket(ctx, args); err != nil { return false, err } return true, nil } -// FunctionGetBucketReplication gets current replication config on a bucket. -func FunctionStorageBucketReplication(ctx context.Context, state *types.State, args *common.StorageBucketArguments) (*common.StorageReplicationConfig, error) { - return state.Storage.GetBucketReplication(ctx, args) -} - -// RemoveBucketReplication removes replication configuration on a bucket. -func ProcedureRemoveStorageBucketReplication(ctx context.Context, state *types.State, args *common.StorageBucketArguments) (bool, error) { - if err := state.Storage.RemoveBucketReplication(ctx, args); err != nil { +// ProcedureUpdateStorageBucket updates the bucket's configuration. +func ProcedureUpdateStorageBucket(ctx context.Context, state *types.State, args *common.UpdateBucketArguments) (bool, error) { + if err := state.Storage.UpdateBucket(ctx, args); err != nil { return false, err } diff --git a/connector/functions/internal/predicate.go b/connector/functions/internal/predicate.go index c2dc2f4..013518b 100644 --- a/connector/functions/internal/predicate.go +++ b/connector/functions/internal/predicate.go @@ -10,29 +10,34 @@ import ( "github.com/hasura/ndc-storage/connector/storage/common" ) -// ObjectPredicate the structured predicate result which is evaluated from the raw expression. -type ObjectPredicate struct { - common.StorageBucketArguments - - IsValid bool - Include common.StorageObjectIncludeOptions - Prefix string - - variables map[string]any - objectNamePrePredicate *StringComparisonOperator - objectNamePostPredicates []StringComparisonOperator +// PredicateEvaluator the structured predicate result which is evaluated from the raw expression. +type PredicateEvaluator struct { + ClientID *common.StorageClientID + IsValid bool + Include common.StorageObjectIncludeOptions + + variables map[string]any + BucketPredicate StringFilterPredicate + ObjectNamePredicate StringFilterPredicate } // EvalObjectPredicate evaluates the predicate condition of the query request. -func EvalObjectPredicate(bucketInfo common.StorageBucketArguments, objectName string, predicate schema.Expression, variables map[string]any) (*ObjectPredicate, error) { - result := &ObjectPredicate{ - StorageBucketArguments: bucketInfo, - Include: common.StorageObjectIncludeOptions{}, - variables: variables, +func EvalObjectPredicate(bucketInfo common.StorageBucketArguments, objectName string, predicate schema.Expression, variables map[string]any) (*PredicateEvaluator, error) { + result := &PredicateEvaluator{ + ClientID: bucketInfo.ClientID, + Include: common.StorageObjectIncludeOptions{}, + variables: variables, + } + + if bucketInfo.Bucket != "" { + result.BucketPredicate.Pre = &StringComparisonOperator{ + Value: bucketInfo.Bucket, + Operator: OperatorEqual, + } } if objectName != "" { - result.objectNamePrePredicate = &StringComparisonOperator{ + result.ObjectNamePredicate.Pre = &StringComparisonOperator{ Value: normalizeObjectName(objectName), Operator: OperatorEqual, } @@ -49,21 +54,22 @@ func EvalObjectPredicate(bucketInfo common.StorageBucketArguments, objectName st } } - if result.objectNamePrePredicate != nil { - result.Prefix = result.objectNamePrePredicate.Value - } - result.IsValid = true return result, nil } -// HasPostPredicate checks if the request has post-predicate expressions -func (cor ObjectPredicate) HasPostPredicate() bool { - return len(cor.objectNamePostPredicates) > 0 +// GetBucketArguments get bucket arguments information +func (pe PredicateEvaluator) GetBucketArguments() common.StorageBucketArguments { + result := common.StorageBucketArguments{ + ClientID: pe.ClientID, + Bucket: pe.BucketPredicate.GetPrefix(), + } + + return result } -func (cor *ObjectPredicate) EvalSelection(selection schema.NestedField) error { +func (pe *PredicateEvaluator) EvalSelection(selection schema.NestedField) error { //nolint:gocognit if len(selection) == 0 { return nil } @@ -75,7 +81,7 @@ func (cor *ObjectPredicate) EvalSelection(selection schema.NestedField) error { switch expr := exprT.(type) { case *schema.NestedArray: - return cor.EvalSelection(expr.Fields) + return pe.EvalSelection(expr.Fields) case *schema.NestedObject: if objectsField, ok := expr.Fields["objects"]; ok { objectsColumn, err := objectsField.AsColumn() @@ -83,42 +89,62 @@ func (cor *ObjectPredicate) EvalSelection(selection schema.NestedField) error { return err } - return cor.EvalSelection(objectsColumn.Fields) + return pe.EvalSelection(objectsColumn.Fields) } - if _, metadataExists := expr.Fields["metadata"]; metadataExists { - cor.Include.Metadata = true - } else if _, metadataExists := expr.Fields["userMetadata"]; metadataExists { - cor.Include.Metadata = true + for _, key := range []string{"metadata", "rawMetadata"} { + if _, ok := expr.Fields[key]; ok { + pe.Include.Metadata = true + + break + } } for _, key := range checksumColumnNames { if _, ok := expr.Fields[key]; ok { - cor.Include.Checksum = true + pe.Include.Checksum = true break } } - if _, metadataExists := expr.Fields["userTags"]; metadataExists { - cor.Include.Tags = true - } else if _, metadataExists := expr.Fields["tags"]; metadataExists { - cor.Include.Tags = true + if _, ok := expr.Fields["tags"]; ok { + pe.Include.Tags = true } - if _, versionExists := expr.Fields["versionId"]; versionExists { - cor.Include.Versions = true + if _, ok := expr.Fields["copy"]; ok { + pe.Include.Copy = true + } + + for _, key := range []string{"versionId", "versioning"} { + if _, ok := expr.Fields[key]; ok { + pe.Include.Versions = true + + break + } } if _, legalHoldExists := expr.Fields["legalHold"]; legalHoldExists { - cor.Include.LegalHold = true + pe.Include.LegalHold = true + } + + if _, ok := expr.Fields["lifecycle"]; ok { + pe.Include.Lifecycle = true + } + + if _, ok := expr.Fields["encryption"]; ok { + pe.Include.Encryption = true + } + + if _, ok := expr.Fields["objectLock"]; ok { + pe.Include.ObjectLock = true } } return nil } -func (cor *ObjectPredicate) evalQueryPredicate(expression schema.Expression) (bool, error) { +func (pe *PredicateEvaluator) evalQueryPredicate(expression schema.Expression) (bool, error) { exprT, err := expression.InterfaceT() if err != nil { return false, err @@ -127,7 +153,7 @@ func (cor *ObjectPredicate) evalQueryPredicate(expression schema.Expression) (bo switch expr := exprT.(type) { case *schema.ExpressionAnd: for _, nestedExpr := range expr.Expressions { - ok, err := cor.evalQueryPredicate(nestedExpr) + ok, err := pe.evalQueryPredicate(nestedExpr) if err != nil { return false, err } @@ -143,7 +169,7 @@ func (cor *ObjectPredicate) evalQueryPredicate(expression schema.Expression) (bo return false, fmt.Errorf("%s: unsupported comparison target `%s`", expr.Column.Name, expr.Column.Type) } - isNull, err := cor.evalIsNullBoolExp(expr) + isNull, err := pe.evalIsNullBoolExp(expr) if err != nil { return false, err } @@ -154,25 +180,25 @@ func (cor *ObjectPredicate) evalQueryPredicate(expression schema.Expression) (bo switch expr.Column.Name { case StorageObjectColumnClientID: - return cor.evalPredicateClientID(expr) + return pe.evalPredicateClientID(expr) case StorageObjectColumnBucket: - return cor.evalPredicateBucket(expr) + return pe.evalStringFilter(&pe.BucketPredicate, expr) case StorageObjectColumnObject: - return cor.evalObjectName(expr) + return pe.evalStringFilter(&pe.ObjectNamePredicate, expr) default: - return false, errors.New("unsupport predicate on column " + expr.Column.Name) + return false, errors.New("unsupported predicate on column " + expr.Column.Name) } default: return false, fmt.Errorf("unsupported expression: %+v", expression) } } -func (cor *ObjectPredicate) evalIsNullBoolExp(expr *schema.ExpressionBinaryComparisonOperator) (*bool, error) { +func (pe *PredicateEvaluator) evalIsNullBoolExp(expr *schema.ExpressionBinaryComparisonOperator) (*bool, error) { if expr.Operator != OperatorIsNull { return nil, nil } - boolValue, err := getComparisonValueBoolean(expr.Value, cor.variables) + boolValue, err := getComparisonValueBoolean(expr.Value, pe.variables) if err != nil { return nil, fmt.Errorf("%s: %w", expr.Column.Name, err) } @@ -180,10 +206,10 @@ func (cor *ObjectPredicate) evalIsNullBoolExp(expr *schema.ExpressionBinaryCompa return boolValue, nil } -func (cor *ObjectPredicate) evalPredicateClientID(expr *schema.ExpressionBinaryComparisonOperator) (bool, error) { +func (pe *PredicateEvaluator) evalPredicateClientID(expr *schema.ExpressionBinaryComparisonOperator) (bool, error) { switch expr.Operator { case OperatorEqual: - value, err := getComparisonValueString(expr.Value, cor.variables) + value, err := getComparisonValueString(expr.Value, pe.variables) if err != nil { return false, fmt.Errorf("clientId: %w", err) } @@ -192,49 +218,25 @@ func (cor *ObjectPredicate) evalPredicateClientID(expr *schema.ExpressionBinaryC return true, nil } - if cor.ClientID == nil || *cor.ClientID == "" { + if pe.ClientID == nil || *pe.ClientID == "" { clientID := common.StorageClientID(*value) - cor.ClientID = &clientID + pe.ClientID = &clientID return true, nil } - return string(*cor.ClientID) == *value, nil + return string(*pe.ClientID) == *value, nil default: return false, fmt.Errorf("unsupported operator `%s` for clientId", expr.Operator) } } -func (cor *ObjectPredicate) evalPredicateBucket(expr *schema.ExpressionBinaryComparisonOperator) (bool, error) { - switch expr.Operator { - case OperatorEqual: - value, err := getComparisonValueString(expr.Value, cor.variables) - if err != nil { - return false, fmt.Errorf("bucket: %w", err) - } - - if value == nil { - return true, nil - } - - if cor.Bucket == "" { - cor.Bucket = *value - - return true, nil - } - - return cor.Bucket == *value, nil - default: - return false, fmt.Errorf("unsupported operator `%s` for bucket", expr.Operator) - } -} - -func (cor *ObjectPredicate) evalObjectName(expr *schema.ExpressionBinaryComparisonOperator) (bool, error) { //nolint:gocognit,cyclop +func (pe *PredicateEvaluator) evalStringFilter(predicate *StringFilterPredicate, expr *schema.ExpressionBinaryComparisonOperator) (bool, error) { //nolint:gocognit,cyclop if !slices.Contains([]string{OperatorStartsWith, OperatorEqual}, expr.Operator) { - return false, fmt.Errorf("unsupported operator `%s` for object name", expr.Operator) + return false, fmt.Errorf("unsupported operator `%s` for string filter expression", expr.Operator) } - value, err := getComparisonValueString(expr.Value, cor.variables) + value, err := getComparisonValueString(expr.Value, pe.variables) if err != nil { return false, fmt.Errorf("bucket: %w", err) } @@ -245,14 +247,14 @@ func (cor *ObjectPredicate) evalObjectName(expr *schema.ExpressionBinaryComparis valueStr := normalizeObjectName(*value) - if cor.objectNamePrePredicate == nil { + if predicate.Pre == nil { if expr.Operator == OperatorStartsWith || expr.Operator == OperatorEqual { - cor.objectNamePrePredicate = &StringComparisonOperator{ + predicate.Pre = &StringComparisonOperator{ Value: valueStr, Operator: expr.Operator, } } else { - cor.objectNamePostPredicates = append(cor.objectNamePostPredicates, StringComparisonOperator{ + predicate.Post = append(predicate.Post, StringComparisonOperator{ Value: valueStr, Operator: expr.Operator, }) @@ -263,81 +265,92 @@ func (cor *ObjectPredicate) evalObjectName(expr *schema.ExpressionBinaryComparis switch expr.Operator { case OperatorStartsWith: - switch cor.objectNamePrePredicate.Operator { + switch predicate.Pre.Operator { case OperatorStartsWith: - if len(cor.objectNamePrePredicate.Value) >= len(valueStr) { - return strings.HasPrefix(cor.objectNamePrePredicate.Value, valueStr), nil + if len(predicate.Pre.Value) >= len(valueStr) { + return strings.HasPrefix(predicate.Pre.Value, valueStr), nil } - if !strings.HasPrefix(valueStr, cor.objectNamePrePredicate.Value) { + if !strings.HasPrefix(valueStr, predicate.Pre.Value) { return false, nil } - cor.objectNamePrePredicate.Value = valueStr + predicate.Pre.Value = valueStr case OperatorEqual: - return strings.HasPrefix(cor.objectNamePrePredicate.Value, valueStr), nil + return strings.HasPrefix(predicate.Pre.Value, valueStr), nil } case OperatorEqual: - switch cor.objectNamePrePredicate.Operator { + switch predicate.Pre.Operator { case OperatorStartsWith: - if !strings.HasPrefix(cor.objectNamePrePredicate.Value, valueStr) { + if !strings.HasPrefix(predicate.Pre.Value, valueStr) { return false, nil } - cor.objectNamePrePredicate = &StringComparisonOperator{ + predicate.Pre = &StringComparisonOperator{ Value: valueStr, Operator: OperatorEqual, } case OperatorEqual: - return cor.objectNamePrePredicate.Value == valueStr, nil + return predicate.Pre.Value == valueStr, nil } case OperatorContains: - switch cor.objectNamePrePredicate.Operator { + switch predicate.Pre.Operator { case OperatorStartsWith: - if strings.Contains(cor.objectNamePrePredicate.Value, valueStr) { + if strings.Contains(predicate.Pre.Value, valueStr) { return true, nil } - cor.objectNamePostPredicates = append(cor.objectNamePostPredicates, StringComparisonOperator{ + predicate.Post = append(predicate.Post, StringComparisonOperator{ Value: valueStr, Operator: expr.Operator, }) case OperatorEqual: - return strings.Contains(cor.objectNamePrePredicate.Value, valueStr), nil + return strings.Contains(predicate.Pre.Value, valueStr), nil } case OperatorInsensitiveContains: - switch cor.objectNamePrePredicate.Operator { + switch predicate.Pre.Operator { case OperatorStartsWith: - if strings.Contains(strings.ToLower(cor.objectNamePrePredicate.Value), strings.ToLower(valueStr)) { + if strings.Contains(strings.ToLower(predicate.Pre.Value), strings.ToLower(valueStr)) { return true, nil } - cor.objectNamePostPredicates = append(cor.objectNamePostPredicates, StringComparisonOperator{ + predicate.Post = append(predicate.Post, StringComparisonOperator{ Value: valueStr, Operator: expr.Operator, }) case OperatorEqual: - return strings.Contains(strings.ToLower(cor.objectNamePrePredicate.Value), strings.ToLower(valueStr)), nil + return strings.Contains(strings.ToLower(predicate.Pre.Value), strings.ToLower(valueStr)), nil } } return true, nil } -// CheckPostObjectPredicate the predicate function to filter the object with post conditions -func (cor ObjectPredicate) CheckPostObjectPredicate(input common.StorageObject) bool { - if len(cor.objectNamePostPredicates) == 0 { - return true +// StringFilterPredicate the structured predicate result which is evaluated from the raw expression. +type StringFilterPredicate struct { + Pre *StringComparisonOperator + Post []StringComparisonOperator +} + +// HasPostPredicate checks if the request has post-predicate expressions +func (sfp StringFilterPredicate) GetPrefix() string { + if sfp.Pre != nil { + return sfp.Pre.Value } - return cor.CheckPostObjectNamePredicate(input.Name) + return "" +} + +// HasPostPredicate checks if the request has post-predicate expressions +func (sfp StringFilterPredicate) HasPostPredicate() bool { + return len(sfp.Post) > 0 } // CheckPostObjectPredicate the predicate function to filter the object with post conditions -func (cor ObjectPredicate) CheckPostObjectNamePredicate(name string) bool { - for _, pred := range cor.objectNamePostPredicates { - if (pred.Operator == OperatorContains && !strings.Contains(name, pred.Value)) || - (pred.Operator == OperatorInsensitiveContains && !strings.Contains(strings.ToLower(name), strings.ToLower(pred.Value))) { +func (sfp StringFilterPredicate) CheckPostPredicate(input string) bool { + for _, pred := range sfp.Post { + if (pred.Operator == OperatorContains && !strings.Contains(input, pred.Value)) || + (pred.Operator == OperatorInsensitiveContains && !strings.Contains(strings.ToLower(input), strings.ToLower(pred.Value))) { return false } } diff --git a/connector/functions/internal/types.go b/connector/functions/internal/types.go index a75d269..cb9d7de 100644 --- a/connector/functions/internal/types.go +++ b/connector/functions/internal/types.go @@ -22,8 +22,8 @@ const ( const ( ScalarStorageClientID = "StorageClientID" - ScalarBucketName = "BucketName" - ScalarObjectPath = "ObjectPath" + ScalarBucketName = "StorageBucketName" + ScalarStringFilter = "StorageStringFilter" ) var checksumColumnNames = []string{"checksumCrc32", "checksumCrc32C", "checksumCrc64Nvme", "checksumSha1", "checksumSha256"} @@ -39,7 +39,17 @@ func GetConnectorSchema(clientIDs []string) *schema.SchemaResponse { return &schema.SchemaResponse{ Collections: []schema.CollectionInfo{}, ObjectTypes: schema.SchemaResponseObjectTypes{ - "StorageObjectSimple": schema.ObjectType{ + "StorageBucketFilter": schema.ObjectType{ + Fields: schema.ObjectTypeFields{ + StorageObjectColumnClientID: schema.ObjectField{ + Type: schema.NewNamedType(ScalarStorageClientID).Encode(), + }, + StorageObjectColumnBucket: schema.ObjectField{ + Type: schema.NewNamedType(ScalarStringFilter).Encode(), + }, + }, + }, + "StorageObjectFilter": schema.ObjectType{ Fields: schema.ObjectTypeFields{ StorageObjectColumnClientID: schema.ObjectField{ Type: schema.NewNamedType(ScalarStorageClientID).Encode(), @@ -48,7 +58,7 @@ func GetConnectorSchema(clientIDs []string) *schema.SchemaResponse { Type: schema.NewNamedType(ScalarBucketName).Encode(), }, StorageObjectColumnObject: schema.ObjectField{ - Type: schema.NewNamedType(ScalarObjectPath).Encode(), + Type: schema.NewNamedType(ScalarStringFilter).Encode(), }, }, }, @@ -61,13 +71,13 @@ func GetConnectorSchema(clientIDs []string) *schema.SchemaResponse { }, Representation: schema.NewTypeRepresentationString().Encode(), }, - ScalarObjectPath: schema.ScalarType{ + ScalarStringFilter: schema.ScalarType{ AggregateFunctions: schema.ScalarTypeAggregateFunctions{}, ComparisonOperators: map[string]schema.ComparisonOperatorDefinition{ OperatorEqual: schema.NewComparisonOperatorEqual().Encode(), - OperatorStartsWith: schema.NewComparisonOperatorCustom(schema.NewNamedType(ScalarObjectPath)).Encode(), - OperatorContains: schema.NewComparisonOperatorCustom(schema.NewNamedType(ScalarObjectPath)).Encode(), - OperatorInsensitiveContains: schema.NewComparisonOperatorCustom(schema.NewNamedType(ScalarObjectPath)).Encode(), + OperatorStartsWith: schema.NewComparisonOperatorCustom(schema.NewNamedType(ScalarStringFilter)).Encode(), + OperatorContains: schema.NewComparisonOperatorCustom(schema.NewNamedType(ScalarStringFilter)).Encode(), + OperatorInsensitiveContains: schema.NewComparisonOperatorCustom(schema.NewNamedType(ScalarStringFilter)).Encode(), }, Representation: schema.NewTypeRepresentationString().Encode(), }, diff --git a/connector/functions/object.go b/connector/functions/object.go index 69d0d2b..af1ebca 100644 --- a/connector/functions/object.go +++ b/connector/functions/object.go @@ -14,11 +14,7 @@ import ( // FunctionStorageObjects lists objects in a bucket. func FunctionStorageObjects(ctx context.Context, state *types.State, args *common.ListStorageObjectsArguments) (common.StorageObjectListResults, error) { - if args.MaxResults != nil && *args.MaxResults <= 0 { - return common.StorageObjectListResults{}, schema.UnprocessableContentError("maxResults must be larger than 0", nil) - } - - request, err := internal.EvalObjectPredicate(common.StorageBucketArguments{}, "", args.Where, types.QueryVariablesFromContext(ctx)) + request, options, err := evalStorageObjectsArguments(ctx, state, args) if err != nil { return common.StorageObjectListResults{}, err } @@ -29,32 +25,40 @@ func FunctionStorageObjects(ctx context.Context, state *types.State, args *commo }, nil } - if err := request.EvalSelection(utils.CommandSelectionFieldFromContext(ctx)); err != nil { - return common.StorageObjectListResults{}, err + predicate := request.ObjectNamePredicate.CheckPostPredicate + + if !request.ObjectNamePredicate.HasPostPredicate() { + predicate = nil } - options := &common.ListStorageObjectsOptions{ - Prefix: request.Prefix, - Recursive: args.Recursive, - Include: request.Include, - NumThreads: state.Concurrency.Query, + objects, err := state.Storage.ListObjects(ctx, request.GetBucketArguments(), options, predicate) + if err != nil { + return common.StorageObjectListResults{}, err } - if args.MaxResults != nil { - options.MaxResults = *args.MaxResults + return *objects, nil +} + +// FunctionStorageDeletedObjects list deleted objects in a bucket. +func FunctionStorageDeletedObjects(ctx context.Context, state *types.State, args *common.ListStorageObjectsArguments) (common.StorageObjectListResults, error) { + request, options, err := evalStorageObjectsArguments(ctx, state, args) + if err != nil { + return common.StorageObjectListResults{}, err } - if args.StartAfter != nil { - options.StartAfter = *args.StartAfter + if !request.IsValid { + return common.StorageObjectListResults{ + Objects: []common.StorageObject{}, + }, nil } - predicate := request.CheckPostObjectNamePredicate + predicate := request.ObjectNamePredicate.CheckPostPredicate - if !request.HasPostPredicate() { + if !request.ObjectNamePredicate.HasPostPredicate() { predicate = nil } - objects, err := state.Storage.ListObjects(ctx, request.StorageBucketArguments, options, predicate) + objects, err := state.Storage.ListDeletedObjects(ctx, request.GetBucketArguments(), options, predicate) if err != nil { return common.StorageObjectListResults{}, err } @@ -80,7 +84,7 @@ func FunctionStorageObject(ctx context.Context, state *types.State, args *common opts := args.GetStorageObjectOptions opts.Include = request.Include - return state.Storage.StatObject(ctx, request.StorageBucketArguments, request.Prefix, opts) + return state.Storage.StatObject(ctx, request.GetBucketArguments(), request.ObjectNamePredicate.GetPrefix(), opts) } // FunctionDownloadStorageObject returns a stream of the object data. Most of the common errors occur when reading the stream. @@ -129,7 +133,7 @@ func downloadStorageObject(ctx context.Context, state *types.State, args *common return nil, nil } - return state.Storage.GetObject(ctx, request.StorageBucketArguments, request.Prefix, args.GetStorageObjectOptions) + return state.Storage.GetObject(ctx, request.GetBucketArguments(), request.ObjectNamePredicate.GetPrefix(), args.GetStorageObjectOptions) } // FunctionStoragePresignedDownloadUrl generates a presigned URL for HTTP GET operations. @@ -146,7 +150,7 @@ func FunctionStoragePresignedDownloadUrl(ctx context.Context, state *types.State return nil, nil } - return state.Storage.PresignedGetObject(ctx, request.StorageBucketArguments, request.Prefix, args.PresignedGetStorageObjectOptions) + return state.Storage.PresignedGetObject(ctx, request.GetBucketArguments(), request.ObjectNamePredicate.GetPrefix(), args.PresignedGetStorageObjectOptions) } // FunctionStoragePresignedUploadUrl generates a presigned URL for HTTP PUT operations. @@ -163,10 +167,210 @@ func FunctionStoragePresignedUploadUrl(ctx context.Context, state *types.State, return nil, nil } - return state.Storage.PresignedPutObject(ctx, request.StorageBucketArguments, request.Prefix, args.Expiry) + return state.Storage.PresignedPutObject(ctx, request.GetBucketArguments(), request.ObjectNamePredicate.GetPrefix(), args.Expiry) } // FunctionStorageIncompleteUploads list partially uploaded objects in a bucket. func FunctionStorageIncompleteUploads(ctx context.Context, state *types.State, args *common.ListIncompleteUploadsArguments) ([]common.StorageObjectMultipartInfo, error) { return state.Storage.ListIncompleteUploads(ctx, args.StorageBucketArguments, args.ListIncompleteUploadsOptions) } + +// PutStorageObjectArguments represents input arguments of the PutObject method. +type PutStorageObjectArguments struct { + common.StorageBucketArguments + + Object string `json:"object"` + Options common.PutStorageObjectOptions `json:"options,omitempty"` + Where schema.Expression `json:"where" ndc:"predicate=StorageObjectFilter"` +} + +// PutStorageObjectArguments represents input arguments of the PutObject method. +type PutStorageObjectBase64Arguments struct { + PutStorageObjectArguments + + Data scalar.Bytes `json:"data"` +} + +// ProcedureUploadStorageObject uploads object that are less than 128MiB in a single PUT operation. For objects that are greater than 128MiB in size, +// PutObject seamlessly uploads the object as parts of 128MiB or more depending on the actual file size. The max upload size for an object is 5TB. +func ProcedureUploadStorageObject(ctx context.Context, state *types.State, args *PutStorageObjectBase64Arguments) (common.StorageUploadInfo, error) { + return uploadStorageObject(ctx, state, &args.PutStorageObjectArguments, args.Data.Bytes()) +} + +func uploadStorageObject(ctx context.Context, state *types.State, args *PutStorageObjectArguments, data []byte) (common.StorageUploadInfo, error) { + request, err := internal.EvalObjectPredicate(args.StorageBucketArguments, args.Object, args.Where, types.QueryVariablesFromContext(ctx)) + if err != nil { + return common.StorageUploadInfo{}, err + } + + if !request.IsValid { + return common.StorageUploadInfo{}, schema.ForbiddenError("permission denied", nil) + } + + result, err := state.Storage.PutObject(ctx, request.GetBucketArguments(), request.ObjectNamePredicate.GetPrefix(), &args.Options, data) + if err != nil { + return common.StorageUploadInfo{}, err + } + + return *result, nil +} + +// PutStorageObjectTextArguments represents input arguments of the PutStorageObjectText method. +type PutStorageObjectTextArguments struct { + PutStorageObjectArguments + + Data string `json:"data"` +} + +// ProcedureUploadStorageObjectText uploads object in plain text to the storage server. The file content is not encoded to base64 so the input size is smaller than 30%. +func ProcedureUploadStorageObjectText(ctx context.Context, state *types.State, args *PutStorageObjectTextArguments) (common.StorageUploadInfo, error) { + return uploadStorageObject(ctx, state, &args.PutStorageObjectArguments, []byte(args.Data)) +} + +// ProcedureCopyStorageObject creates or replaces an object through server-side copying of an existing object. +// It supports conditional copying, copying a part of an object and server-side encryption of destination and decryption of source. +// To copy multiple source objects into a single destination object see the ComposeObject API. +func ProcedureCopyStorageObject(ctx context.Context, state *types.State, args *common.CopyStorageObjectArguments) (common.StorageUploadInfo, error) { + result, err := state.Storage.CopyObject(ctx, args) + if err != nil { + return common.StorageUploadInfo{}, err + } + + return *result, nil +} + +// ProcedureComposeStorageObject creates an object by concatenating a list of source objects using server-side copying. +func ProcedureComposeStorageObject(ctx context.Context, state *types.State, args *common.ComposeStorageObjectArguments) (common.StorageUploadInfo, error) { + result, err := state.Storage.ComposeObject(ctx, args) + if err != nil { + return common.StorageUploadInfo{}, err + } + + return *result, nil +} + +// ProcedureUpdateStorageObject updates the object's configuration. +func ProcedureUpdateStorageObject(ctx context.Context, state *types.State, args *common.UpdateStorageObjectArguments) (bool, error) { + request, err := internal.EvalObjectPredicate(args.StorageBucketArguments, args.Object, args.Where, types.QueryVariablesFromContext(ctx)) + if err != nil { + return false, err + } + + if !request.IsValid { + return false, errPermissionDenied + } + + if err := state.Storage.UpdateObject(ctx, request.GetBucketArguments(), request.ObjectNamePredicate.GetPrefix(), args.UpdateStorageObjectOptions); err != nil { + return false, err + } + + return true, nil +} + +// ProcedureRemoveStorageObject removes an object with some specified options. +func ProcedureRemoveStorageObject(ctx context.Context, state *types.State, args *common.RemoveStorageObjectArguments) (bool, error) { + request, err := internal.EvalObjectPredicate(args.StorageBucketArguments, args.Object, args.Where, types.QueryVariablesFromContext(ctx)) + if err != nil { + return false, err + } + + if !request.IsValid { + return false, errPermissionDenied + } + + if err := state.Storage.RemoveObject(ctx, request.GetBucketArguments(), request.ObjectNamePredicate.GetPrefix(), args.RemoveStorageObjectOptions); err != nil { + return false, err + } + + return true, nil +} + +// ProcedureRemoveStorageObjects remove a list of objects obtained from an input channel. The call sends a delete request to the server up to 1000 objects at a time. +// The errors observed are sent over the error channel. +func ProcedureRemoveStorageObjects(ctx context.Context, state *types.State, args *common.RemoveStorageObjectsArguments) ([]common.RemoveStorageObjectError, error) { + request, err := internal.EvalObjectPredicate(args.StorageBucketArguments, "", args.Where, types.QueryVariablesFromContext(ctx)) + if err != nil { + return nil, err + } + + if !request.IsValid { + return nil, nil + } + + predicate := request.ObjectNamePredicate.CheckPostPredicate + if !request.ObjectNamePredicate.HasPostPredicate() { + predicate = nil + } + + return state.Storage.RemoveObjects(ctx, request.GetBucketArguments(), &common.RemoveStorageObjectsOptions{ + ListStorageObjectsOptions: common.ListStorageObjectsOptions{ + Prefix: request.ObjectNamePredicate.GetPrefix(), + Recursive: args.Recursive, + StartAfter: args.StartAfter, + }, + GovernanceBypass: args.GovernanceBypass, + }, predicate) +} + +// ProcedureRemoveIncompleteStorageUpload removes a partially uploaded object. +func ProcedureRemoveIncompleteStorageUpload(ctx context.Context, state *types.State, args *common.RemoveIncompleteUploadArguments) (bool, error) { + if err := state.Storage.RemoveIncompleteUpload(ctx, args); err != nil { + return false, err + } + + return true, nil +} + +// ProcedureRestoreStorageObject restore a soft-deleted object. +func ProcedureRestoreStorageObject(ctx context.Context, state *types.State, args *common.RestoreStorageObjectArguments) (bool, error) { + request, err := internal.EvalObjectPredicate(args.StorageBucketArguments, args.Object, args.Where, types.QueryVariablesFromContext(ctx)) + if err != nil { + return false, err + } + + if !request.IsValid { + return false, errPermissionDenied + } + + if err := state.Storage.RestoreObject(ctx, request.GetBucketArguments(), request.ObjectNamePredicate.GetPrefix()); err != nil { + return false, err + } + + return true, nil +} + +func evalStorageObjectsArguments(ctx context.Context, state *types.State, args *common.ListStorageObjectsArguments) (*internal.PredicateEvaluator, *common.ListStorageObjectsOptions, error) { + if args.MaxResults != nil && *args.MaxResults <= 0 { + return nil, nil, schema.UnprocessableContentError("maxResults must be larger than 0", nil) + } + + request, err := internal.EvalObjectPredicate(common.StorageBucketArguments{}, "", args.Where, types.QueryVariablesFromContext(ctx)) + if err != nil { + return nil, nil, err + } + + if !request.IsValid { + return request, nil, nil + } + + if err := request.EvalSelection(utils.CommandSelectionFieldFromContext(ctx)); err != nil { + return nil, nil, err + } + + options := &common.ListStorageObjectsOptions{ + Prefix: request.ObjectNamePredicate.GetPrefix(), + Recursive: args.Recursive, + Include: request.Include, + NumThreads: state.Concurrency.Query, + } + + if args.MaxResults != nil { + options.MaxResults = *args.MaxResults + } + + if args.StartAfter != nil { + options.StartAfter = *args.StartAfter + } + + return request, options, nil +} diff --git a/connector/functions/object_cud.go b/connector/functions/object_cud.go deleted file mode 100644 index cbd6d19..0000000 --- a/connector/functions/object_cud.go +++ /dev/null @@ -1,193 +0,0 @@ -package functions - -import ( - "context" - - "github.com/hasura/ndc-sdk-go/scalar" - "github.com/hasura/ndc-sdk-go/schema" - "github.com/hasura/ndc-storage/connector/functions/internal" - "github.com/hasura/ndc-storage/connector/storage/common" - "github.com/hasura/ndc-storage/connector/types" -) - -// PutStorageObjectArguments represents input arguments of the PutObject method. -type PutStorageObjectArguments struct { - common.StorageBucketArguments - - Object string `json:"object"` - Options common.PutStorageObjectOptions `json:"options,omitempty"` - Where schema.Expression `json:"where" ndc:"predicate=StorageObjectSimple"` -} - -// PutStorageObjectArguments represents input arguments of the PutObject method. -type PutStorageObjectBase64Arguments struct { - PutStorageObjectArguments - - Data scalar.Bytes `json:"data"` -} - -// ProcedureUploadStorageObject uploads object that are less than 128MiB in a single PUT operation. For objects that are greater than 128MiB in size, -// PutObject seamlessly uploads the object as parts of 128MiB or more depending on the actual file size. The max upload size for an object is 5TB. -func ProcedureUploadStorageObject(ctx context.Context, state *types.State, args *PutStorageObjectBase64Arguments) (common.StorageUploadInfo, error) { - return uploadStorageObject(ctx, state, &args.PutStorageObjectArguments, args.Data.Bytes()) -} - -func uploadStorageObject(ctx context.Context, state *types.State, args *PutStorageObjectArguments, data []byte) (common.StorageUploadInfo, error) { - request, err := internal.EvalObjectPredicate(args.StorageBucketArguments, args.Object, args.Where, types.QueryVariablesFromContext(ctx)) - if err != nil { - return common.StorageUploadInfo{}, err - } - - if !request.IsValid { - return common.StorageUploadInfo{}, schema.ForbiddenError("permission dennied", nil) - } - - result, err := state.Storage.PutObject(ctx, request.StorageBucketArguments, request.Prefix, &args.Options, data) - if err != nil { - return common.StorageUploadInfo{}, err - } - - return *result, nil -} - -// PutStorageObjectTextArguments represents input arguments of the PutStorageObjectText method. -type PutStorageObjectTextArguments struct { - PutStorageObjectArguments - - Data string `json:"data"` -} - -// ProcedureUploadStorageObjectText uploads object in plain text to the storage server. The file content is not encoded to base64 so the input size is smaller than 30%. -func ProcedureUploadStorageObjectText(ctx context.Context, state *types.State, args *PutStorageObjectTextArguments) (common.StorageUploadInfo, error) { - return uploadStorageObject(ctx, state, &args.PutStorageObjectArguments, []byte(args.Data)) -} - -// ProcedureCopyStorageObject creates or replaces an object through server-side copying of an existing object. -// It supports conditional copying, copying a part of an object and server-side encryption of destination and decryption of source. -// To copy multiple source objects into a single destination object see the ComposeObject API. -func ProcedureCopyStorageObject(ctx context.Context, state *types.State, args *common.CopyStorageObjectArguments) (common.StorageUploadInfo, error) { - result, err := state.Storage.CopyObject(ctx, args) - if err != nil { - return common.StorageUploadInfo{}, err - } - - return *result, nil -} - -// ProcedureComposeStorageObject creates an object by concatenating a list of source objects using server-side copying. -func ProcedureComposeStorageObject(ctx context.Context, state *types.State, args *common.ComposeStorageObjectArguments) (common.StorageUploadInfo, error) { - result, err := state.Storage.ComposeObject(ctx, args) - if err != nil { - return common.StorageUploadInfo{}, err - } - - return *result, nil -} - -// ProcedureSetStorageObjectTags sets new object Tags to the given object, replaces/overwrites any existing tags. -func ProcedureSetStorageObjectTags(ctx context.Context, state *types.State, args *common.SetStorageObjectTagsArguments) (bool, error) { - request, err := internal.EvalObjectPredicate(args.StorageBucketArguments, args.Object, args.Where, types.QueryVariablesFromContext(ctx)) - if err != nil { - return false, err - } - - if !request.IsValid { - return false, errPermissionDenied - } - - if err := state.Storage.SetObjectTags(ctx, request.StorageBucketArguments, request.Prefix, args.SetStorageObjectTagsOptions); err != nil { - return false, err - } - - return true, nil -} - -// ProcedureRemoveStorageObject removes an object with some specified options. -func ProcedureRemoveStorageObject(ctx context.Context, state *types.State, args *common.RemoveStorageObjectArguments) (bool, error) { - request, err := internal.EvalObjectPredicate(args.StorageBucketArguments, args.Object, args.Where, types.QueryVariablesFromContext(ctx)) - if err != nil { - return false, err - } - - if !request.IsValid { - return false, errPermissionDenied - } - - if err := state.Storage.RemoveObject(ctx, request.StorageBucketArguments, request.Prefix, args.RemoveStorageObjectOptions); err != nil { - return false, err - } - - return true, nil -} - -// ProcedureRemoveStorageObjects remove a list of objects obtained from an input channel. The call sends a delete request to the server up to 1000 objects at a time. -// The errors observed are sent over the error channel. -func ProcedureRemoveStorageObjects(ctx context.Context, state *types.State, args *common.RemoveStorageObjectsArguments) ([]common.RemoveStorageObjectError, error) { - request, err := internal.EvalObjectPredicate(args.StorageBucketArguments, "", args.Where, types.QueryVariablesFromContext(ctx)) - if err != nil { - return nil, err - } - - if !request.IsValid { - return nil, nil - } - - predicate := request.CheckPostObjectNamePredicate - if !request.HasPostPredicate() { - predicate = nil - } - - return state.Storage.RemoveObjects(ctx, request.StorageBucketArguments, &common.RemoveStorageObjectsOptions{ - ListStorageObjectsOptions: common.ListStorageObjectsOptions{ - Prefix: request.Prefix, - Recursive: args.Recursive, - StartAfter: args.StartAfter, - }, - GovernanceBypass: args.GovernanceBypass, - }, predicate) -} - -// ProcedureSetStorageObjectRetention applies object retention lock onto an object. -func ProcedureSetStorageObjectRetention(ctx context.Context, state *types.State, args *common.SetStorageObjectRetentionArguments) (bool, error) { - request, err := internal.EvalObjectPredicate(args.StorageBucketArguments, args.Object, args.Where, types.QueryVariablesFromContext(ctx)) - if err != nil { - return false, err - } - - if !request.IsValid { - return false, errPermissionDenied - } - - if err := state.Storage.SetObjectRetention(ctx, request.StorageBucketArguments, request.Prefix, args.SetStorageObjectRetentionOptions); err != nil { - return false, err - } - - return true, nil -} - -// ProcedureSetStorageObjectLegalHold applies legal-hold onto an object. -func ProcedureSetStorageObjectLegalHold(ctx context.Context, state *types.State, args *common.SetStorageObjectLegalHoldArguments) (bool, error) { - request, err := internal.EvalObjectPredicate(args.StorageBucketArguments, args.Object, args.Where, types.QueryVariablesFromContext(ctx)) - if err != nil { - return false, err - } - - if !request.IsValid { - return false, errPermissionDenied - } - - if err := state.Storage.SetObjectLegalHold(ctx, request.StorageBucketArguments, request.Prefix, args.SetStorageObjectLegalHoldOptions); err != nil { - return false, err - } - - return true, nil -} - -// ProcedureRemoveIncompleteStorageUpload removes a partially uploaded object. -func ProcedureRemoveIncompleteStorageUpload(ctx context.Context, state *types.State, args *common.RemoveIncompleteUploadArguments) (bool, error) { - if err := state.Storage.RemoveIncompleteUpload(ctx, args); err != nil { - return false, err - } - - return true, nil -} diff --git a/connector/functions/types.generated.go b/connector/functions/types.generated.go index 7305177..c4836d5 100644 --- a/connector/functions/types.generated.go +++ b/connector/functions/types.generated.go @@ -98,7 +98,7 @@ func (dch DataConnectorHandler) execQuery(ctx context.Context, state *types.Stat }) return FunctionDownloadStorageObjectText(ctx, state, &args) - case "storageBucketEncryption": + case "storageBucket": selection, err := queryFields.AsObject() if err != nil { @@ -117,7 +117,7 @@ func (dch DataConnectorHandler) execQuery(ctx context.Context, state *types.Stat connector_addSpanEvent(span, logger, "execute_function", map[string]any{ "arguments": args, }) - rawResult, err := FunctionStorageBucketEncryption(ctx, state, &args) + rawResult, err := FunctionStorageBucket(ctx, state, &args) if err != nil { return nil, err @@ -153,99 +153,7 @@ func (dch DataConnectorHandler) execQuery(ctx context.Context, state *types.Stat }) return FunctionStorageBucketExists(ctx, state, &args) - case "storageBucketLifecycle": - - selection, err := queryFields.AsObject() - if err != nil { - return nil, schema.UnprocessableContentError("the selection field type must be object", map[string]any{ - "cause": err.Error(), - }) - } - var args common.StorageBucketArguments - parseErr := args.FromValue(rawArgs) - if parseErr != nil { - return nil, schema.UnprocessableContentError("failed to resolve arguments", map[string]any{ - "cause": parseErr.Error(), - }) - } - - connector_addSpanEvent(span, logger, "execute_function", map[string]any{ - "arguments": args, - }) - rawResult, err := FunctionStorageBucketLifecycle(ctx, state, &args) - - if err != nil { - return nil, err - } - - if rawResult == nil { - return nil, nil - } - connector_addSpanEvent(span, logger, "evaluate_response_selection", map[string]any{ - "raw_result": rawResult, - }) - result, err := utils.EvalNestedColumnObject(selection, rawResult) - if err != nil { - return nil, err - } - return result, nil - - case "storageBucketNotification": - - selection, err := queryFields.AsObject() - if err != nil { - return nil, schema.UnprocessableContentError("the selection field type must be object", map[string]any{ - "cause": err.Error(), - }) - } - var args common.StorageBucketArguments - parseErr := args.FromValue(rawArgs) - if parseErr != nil { - return nil, schema.UnprocessableContentError("failed to resolve arguments", map[string]any{ - "cause": parseErr.Error(), - }) - } - - connector_addSpanEvent(span, logger, "execute_function", map[string]any{ - "arguments": args, - }) - rawResult, err := FunctionStorageBucketNotification(ctx, state, &args) - - if err != nil { - return nil, err - } - - if rawResult == nil { - return nil, nil - } - connector_addSpanEvent(span, logger, "evaluate_response_selection", map[string]any{ - "raw_result": rawResult, - }) - result, err := utils.EvalNestedColumnObject(selection, rawResult) - if err != nil { - return nil, err - } - return result, nil - - case "storageBucketPolicy": - - if len(queryFields) > 0 { - return nil, schema.UnprocessableContentError("cannot evaluate selection fields for scalar", nil) - } - var args common.StorageBucketArguments - parseErr := args.FromValue(rawArgs) - if parseErr != nil { - return nil, schema.UnprocessableContentError("failed to resolve arguments", map[string]any{ - "cause": parseErr.Error(), - }) - } - - connector_addSpanEvent(span, logger, "execute_function", map[string]any{ - "arguments": args, - }) - return FunctionStorageBucketPolicy(ctx, state, &args) - - case "storageBucketReplication": + case "storageBuckets": selection, err := queryFields.AsObject() if err != nil { @@ -253,7 +161,7 @@ func (dch DataConnectorHandler) execQuery(ctx context.Context, state *types.Stat "cause": err.Error(), }) } - var args common.StorageBucketArguments + var args common.ListStorageBucketArguments parseErr := args.FromValue(rawArgs) if parseErr != nil { return nil, schema.UnprocessableContentError("failed to resolve arguments", map[string]any{ @@ -264,15 +172,12 @@ func (dch DataConnectorHandler) execQuery(ctx context.Context, state *types.Stat connector_addSpanEvent(span, logger, "execute_function", map[string]any{ "arguments": args, }) - rawResult, err := FunctionStorageBucketReplication(ctx, state, &args) + rawResult, err := FunctionStorageBuckets(ctx, state, &args) if err != nil { return nil, err } - if rawResult == nil { - return nil, nil - } connector_addSpanEvent(span, logger, "evaluate_response_selection", map[string]any{ "raw_result": rawResult, }) @@ -282,7 +187,7 @@ func (dch DataConnectorHandler) execQuery(ctx context.Context, state *types.Stat } return result, nil - case "storageBucketVersioning": + case "storageDeletedObjects": selection, err := queryFields.AsObject() if err != nil { @@ -290,7 +195,7 @@ func (dch DataConnectorHandler) execQuery(ctx context.Context, state *types.Stat "cause": err.Error(), }) } - var args common.StorageBucketArguments + var args common.ListStorageObjectsArguments parseErr := args.FromValue(rawArgs) if parseErr != nil { return nil, schema.UnprocessableContentError("failed to resolve arguments", map[string]any{ @@ -301,15 +206,12 @@ func (dch DataConnectorHandler) execQuery(ctx context.Context, state *types.Stat connector_addSpanEvent(span, logger, "execute_function", map[string]any{ "arguments": args, }) - rawResult, err := FunctionStorageBucketVersioning(ctx, state, &args) + rawResult, err := FunctionStorageDeletedObjects(ctx, state, &args) if err != nil { return nil, err } - if rawResult == nil { - return nil, nil - } connector_addSpanEvent(span, logger, "evaluate_response_selection", map[string]any{ "raw_result": rawResult, }) @@ -319,40 +221,6 @@ func (dch DataConnectorHandler) execQuery(ctx context.Context, state *types.Stat } return result, nil - case "storageBuckets": - - selection, err := queryFields.AsArray() - if err != nil { - return nil, schema.UnprocessableContentError("the selection field type must be array", map[string]any{ - "cause": err.Error(), - }) - } - var args common.ListStorageBucketArguments - parseErr := args.FromValue(rawArgs) - if parseErr != nil { - return nil, schema.UnprocessableContentError("failed to resolve arguments", map[string]any{ - "cause": parseErr.Error(), - }) - } - - connector_addSpanEvent(span, logger, "execute_function", map[string]any{ - "arguments": args, - }) - rawResult, err := FunctionStorageBuckets(ctx, state, &args) - - if err != nil { - return nil, err - } - - connector_addSpanEvent(span, logger, "evaluate_response_selection", map[string]any{ - "raw_result": rawResult, - }) - result, err := utils.EvalNestedColumnArrayIntoSlice(selection, rawResult) - if err != nil { - return nil, err - } - return result, nil - case "storageIncompleteUploads": selection, err := queryFields.AsArray() @@ -424,43 +292,6 @@ func (dch DataConnectorHandler) execQuery(ctx context.Context, state *types.Stat } return result, nil - case "storageObjectLockConfig": - - selection, err := queryFields.AsObject() - if err != nil { - return nil, schema.UnprocessableContentError("the selection field type must be object", map[string]any{ - "cause": err.Error(), - }) - } - var args common.StorageBucketArguments - parseErr := args.FromValue(rawArgs) - if parseErr != nil { - return nil, schema.UnprocessableContentError("failed to resolve arguments", map[string]any{ - "cause": parseErr.Error(), - }) - } - - connector_addSpanEvent(span, logger, "execute_function", map[string]any{ - "arguments": args, - }) - rawResult, err := FunctionStorageObjectLockConfig(ctx, state, &args) - - if err != nil { - return nil, err - } - - if rawResult == nil { - return nil, nil - } - connector_addSpanEvent(span, logger, "evaluate_response_selection", map[string]any{ - "raw_result": rawResult, - }) - result, err := utils.EvalNestedColumnObject(selection, rawResult) - if err != nil { - return nil, err - } - return result, nil - case "storageObjects": selection, err := queryFields.AsObject() @@ -574,7 +405,7 @@ func (dch DataConnectorHandler) execQuery(ctx context.Context, state *types.Stat } } -var enumValues_FunctionName = []string{"downloadStorageObject", "downloadStorageObjectText", "storageBucketEncryption", "storageBucketExists", "storageBucketLifecycle", "storageBucketNotification", "storageBucketPolicy", "storageBucketReplication", "storageBucketVersioning", "storageBuckets", "storageIncompleteUploads", "storageObject", "storageObjectLockConfig", "storageObjects", "storagePresignedDownloadUrl", "storagePresignedUploadUrl"} +var enumValues_FunctionName = []string{"downloadStorageObject", "downloadStorageObjectText", "storageBucket", "storageBucketExists", "storageBuckets", "storageDeletedObjects", "storageIncompleteUploads", "storageObject", "storageObjects", "storagePresignedDownloadUrl", "storagePresignedUploadUrl"} // MutationExists check if the mutation name exists func (dch DataConnectorHandler) MutationExists(name string) bool { @@ -670,25 +501,6 @@ func (dch DataConnectorHandler) Mutation(ctx context.Context, state *types.State } return schema.NewProcedureResult(result).Encode(), nil - case "enableStorageBucketVersioning": - - if len(operation.Fields) > 0 { - return nil, schema.UnprocessableContentError("cannot evaluate selection fields for scalar", nil) - } - var args common.StorageBucketArguments - if err := json.Unmarshal(operation.Arguments, &args); err != nil { - return nil, schema.UnprocessableContentError("failed to decode arguments", map[string]any{ - "cause": err.Error(), - }) - } - span.AddEvent("execute_procedure") - result, err := ProcedureEnableStorageBucketVersioning(ctx, state, &args) - - if err != nil { - return nil, err - } - return schema.NewProcedureResult(result).Encode(), nil - case "removeIncompleteStorageUpload": if len(operation.Fields) > 0 { @@ -727,25 +539,6 @@ func (dch DataConnectorHandler) Mutation(ctx context.Context, state *types.State } return schema.NewProcedureResult(result).Encode(), nil - case "removeStorageBucketReplication": - - if len(operation.Fields) > 0 { - return nil, schema.UnprocessableContentError("cannot evaluate selection fields for scalar", nil) - } - var args common.StorageBucketArguments - if err := json.Unmarshal(operation.Arguments, &args); err != nil { - return nil, schema.UnprocessableContentError("failed to decode arguments", map[string]any{ - "cause": err.Error(), - }) - } - span.AddEvent("execute_procedure") - result, err := ProcedureRemoveStorageBucketReplication(ctx, state, &args) - - if err != nil { - return nil, err - } - return schema.NewProcedureResult(result).Encode(), nil - case "removeStorageObject": if len(operation.Fields) > 0 { @@ -796,190 +589,57 @@ func (dch DataConnectorHandler) Mutation(ctx context.Context, state *types.State } return schema.NewProcedureResult(result).Encode(), nil - case "setStorageBucketEncryption": - - if len(operation.Fields) > 0 { - return nil, schema.UnprocessableContentError("cannot evaluate selection fields for scalar", nil) - } - var args common.SetStorageBucketEncryptionArguments - if err := json.Unmarshal(operation.Arguments, &args); err != nil { - return nil, schema.UnprocessableContentError("failed to decode arguments", map[string]any{ - "cause": err.Error(), - }) - } - span.AddEvent("execute_procedure") - result, err := ProcedureSetStorageBucketEncryption(ctx, state, &args) - - if err != nil { - return nil, err - } - return schema.NewProcedureResult(result).Encode(), nil - - case "setStorageBucketLifecycle": + case "restoreStorageObject": if len(operation.Fields) > 0 { return nil, schema.UnprocessableContentError("cannot evaluate selection fields for scalar", nil) } - var args common.SetStorageBucketLifecycleArguments + var args common.RestoreStorageObjectArguments if err := json.Unmarshal(operation.Arguments, &args); err != nil { return nil, schema.UnprocessableContentError("failed to decode arguments", map[string]any{ "cause": err.Error(), }) } span.AddEvent("execute_procedure") - result, err := ProcedureSetStorageBucketLifecycle(ctx, state, &args) + result, err := ProcedureRestoreStorageObject(ctx, state, &args) if err != nil { return nil, err } return schema.NewProcedureResult(result).Encode(), nil - case "setStorageBucketNotification": + case "updateStorageBucket": if len(operation.Fields) > 0 { return nil, schema.UnprocessableContentError("cannot evaluate selection fields for scalar", nil) } - var args common.SetBucketNotificationArguments + var args common.UpdateBucketArguments if err := json.Unmarshal(operation.Arguments, &args); err != nil { return nil, schema.UnprocessableContentError("failed to decode arguments", map[string]any{ "cause": err.Error(), }) } span.AddEvent("execute_procedure") - result, err := ProcedureSetStorageBucketNotification(ctx, state, &args) + result, err := ProcedureUpdateStorageBucket(ctx, state, &args) if err != nil { return nil, err } return schema.NewProcedureResult(result).Encode(), nil - case "setStorageBucketReplication": + case "updateStorageObject": if len(operation.Fields) > 0 { return nil, schema.UnprocessableContentError("cannot evaluate selection fields for scalar", nil) } - var args common.SetStorageBucketReplicationArguments - if err := json.Unmarshal(operation.Arguments, &args); err != nil { - return nil, schema.UnprocessableContentError("failed to decode arguments", map[string]any{ - "cause": err.Error(), - }) - } - span.AddEvent("execute_procedure") - result, err := ProcedureSetStorageBucketReplication(ctx, state, &args) - - if err != nil { - return nil, err - } - return schema.NewProcedureResult(result).Encode(), nil - - case "setStorageBucketTags": - - if len(operation.Fields) > 0 { - return nil, schema.UnprocessableContentError("cannot evaluate selection fields for scalar", nil) - } - var args common.SetStorageBucketTaggingArguments - if err := json.Unmarshal(operation.Arguments, &args); err != nil { - return nil, schema.UnprocessableContentError("failed to decode arguments", map[string]any{ - "cause": err.Error(), - }) - } - span.AddEvent("execute_procedure") - result, err := ProcedureSetStorageBucketTags(ctx, state, &args) - - if err != nil { - return nil, err - } - return schema.NewProcedureResult(result).Encode(), nil - - case "setStorageObjectLegalHold": - - if len(operation.Fields) > 0 { - return nil, schema.UnprocessableContentError("cannot evaluate selection fields for scalar", nil) - } - var args common.SetStorageObjectLegalHoldArguments - if err := json.Unmarshal(operation.Arguments, &args); err != nil { - return nil, schema.UnprocessableContentError("failed to decode arguments", map[string]any{ - "cause": err.Error(), - }) - } - span.AddEvent("execute_procedure") - result, err := ProcedureSetStorageObjectLegalHold(ctx, state, &args) - - if err != nil { - return nil, err - } - return schema.NewProcedureResult(result).Encode(), nil - - case "setStorageObjectLockConfig": - - if len(operation.Fields) > 0 { - return nil, schema.UnprocessableContentError("cannot evaluate selection fields for scalar", nil) - } - var args common.SetStorageObjectLockArguments - if err := json.Unmarshal(operation.Arguments, &args); err != nil { - return nil, schema.UnprocessableContentError("failed to decode arguments", map[string]any{ - "cause": err.Error(), - }) - } - span.AddEvent("execute_procedure") - result, err := ProcedureSetStorageObjectLockConfig(ctx, state, &args) - - if err != nil { - return nil, err - } - return schema.NewProcedureResult(result).Encode(), nil - - case "setStorageObjectRetention": - - if len(operation.Fields) > 0 { - return nil, schema.UnprocessableContentError("cannot evaluate selection fields for scalar", nil) - } - var args common.SetStorageObjectRetentionArguments - if err := json.Unmarshal(operation.Arguments, &args); err != nil { - return nil, schema.UnprocessableContentError("failed to decode arguments", map[string]any{ - "cause": err.Error(), - }) - } - span.AddEvent("execute_procedure") - result, err := ProcedureSetStorageObjectRetention(ctx, state, &args) - - if err != nil { - return nil, err - } - return schema.NewProcedureResult(result).Encode(), nil - - case "setStorageObjectTags": - - if len(operation.Fields) > 0 { - return nil, schema.UnprocessableContentError("cannot evaluate selection fields for scalar", nil) - } - var args common.SetStorageObjectTagsArguments - if err := json.Unmarshal(operation.Arguments, &args); err != nil { - return nil, schema.UnprocessableContentError("failed to decode arguments", map[string]any{ - "cause": err.Error(), - }) - } - span.AddEvent("execute_procedure") - result, err := ProcedureSetStorageObjectTags(ctx, state, &args) - - if err != nil { - return nil, err - } - return schema.NewProcedureResult(result).Encode(), nil - - case "suspendStorageBucketVersioning": - - if len(operation.Fields) > 0 { - return nil, schema.UnprocessableContentError("cannot evaluate selection fields for scalar", nil) - } - var args common.StorageBucketArguments + var args common.UpdateStorageObjectArguments if err := json.Unmarshal(operation.Arguments, &args); err != nil { return nil, schema.UnprocessableContentError("failed to decode arguments", map[string]any{ "cause": err.Error(), }) } span.AddEvent("execute_procedure") - result, err := ProcedureSuspendStorageBucketVersioning(ctx, state, &args) + result, err := ProcedureUpdateStorageObject(ctx, state, &args) if err != nil { return nil, err @@ -1053,7 +713,7 @@ func (dch DataConnectorHandler) Mutation(ctx context.Context, state *types.State } } -var enumValues_ProcedureName = []string{"composeStorageObject", "copyStorageObject", "createStorageBucket", "enableStorageBucketVersioning", "removeIncompleteStorageUpload", "removeStorageBucket", "removeStorageBucketReplication", "removeStorageObject", "removeStorageObjects", "setStorageBucketEncryption", "setStorageBucketLifecycle", "setStorageBucketNotification", "setStorageBucketReplication", "setStorageBucketTags", "setStorageObjectLegalHold", "setStorageObjectLockConfig", "setStorageObjectRetention", "setStorageObjectTags", "suspendStorageBucketVersioning", "uploadStorageObject", "uploadStorageObjectText"} +var enumValues_ProcedureName = []string{"composeStorageObject", "copyStorageObject", "createStorageBucket", "removeIncompleteStorageUpload", "removeStorageBucket", "removeStorageObject", "removeStorageObjects", "restoreStorageObject", "updateStorageBucket", "updateStorageObject", "uploadStorageObject", "uploadStorageObjectText"} func connector_addSpanEvent(span trace.Span, logger *slog.Logger, name string, data map[string]any, options ...trace.EventOption) { logger.Debug(name, slog.Any("data", data)) diff --git a/connector/schema.generated.go b/connector/schema.generated.go index c43e70f..87b39fd 100644 --- a/connector/schema.generated.go +++ b/connector/schema.generated.go @@ -14,81 +14,73 @@ func GetConnectorSchema() *schema.SchemaResponse { return &schema.SchemaResponse{ Collections: []schema.CollectionInfo{}, ObjectTypes: schema.SchemaResponseObjectTypes{ - "AbortIncompleteMultipartUpload": schema.ObjectType{ - Description: toPtr("structure, not supported yet on MinIO"), + "BucketAutoclass": schema.ObjectType{ Fields: schema.ObjectTypeFields{ - "daysAfterInitiation": schema.ObjectField{ - Type: schema.NewNullableType(schema.NewNamedType("Int32")).Encode(), + "enabled": schema.ObjectField{ + Type: schema.NewNamedType("Boolean").Encode(), }, - }, - }, - "BucketLifecycleConfiguration": schema.ObjectType{ - Description: toPtr("is a collection of lifecycle Rule objects."), - Fields: schema.ObjectTypeFields{ - "rules": schema.ObjectField{ - Type: schema.NewArrayType(schema.NewNamedType("BucketLifecycleRule")).Encode(), + "terminalStorageClass": schema.ObjectField{ + Type: schema.NewNamedType("String").Encode(), + }, + "terminalStorageClassUpdateTime": schema.ObjectField{ + Type: schema.NewNamedType("TimestampTZ").Encode(), + }, + "toggleTime": schema.ObjectField{ + Type: schema.NewNamedType("TimestampTZ").Encode(), }, }, }, - "BucketLifecycleRule": schema.ObjectType{ - Description: toPtr("represents a single rule in lifecycle configuration"), + "BucketCors": schema.ObjectType{ + Description: toPtr("is the bucket's Cross-Origin Resource Sharing (CORS) configuration."), Fields: schema.ObjectTypeFields{ - "abortIncompleteMultipartUpload": schema.ObjectField{ - Type: schema.NewNullableType(schema.NewNamedType("AbortIncompleteMultipartUpload")).Encode(), + "maxAge": schema.ObjectField{ + Type: schema.NewNamedType("Duration").Encode(), }, - "allVersionsExpiration": schema.ObjectField{ - Type: schema.NewNullableType(schema.NewNamedType("LifecycleAllVersionsExpiration")).Encode(), - }, - "delMarkerExpiration": schema.ObjectField{ - Type: schema.NewNullableType(schema.NewNamedType("LifecycleDelMarkerExpiration")).Encode(), - }, - "expiration": schema.ObjectField{ - Type: schema.NewNullableType(schema.NewNamedType("LifecycleExpiration")).Encode(), - }, - "filter": schema.ObjectField{ - Type: schema.NewNullableType(schema.NewNamedType("LifecycleFilter")).Encode(), - }, - "id": schema.ObjectField{ - Type: schema.NewNamedType("String").Encode(), - }, - "noncurrentVersionExpiration": schema.ObjectField{ - Type: schema.NewNullableType(schema.NewNamedType("LifecycleNoncurrentVersionExpiration")).Encode(), - }, - "noncurrentVersionTransition": schema.ObjectField{ - Type: schema.NewNullableType(schema.NewNamedType("LifecycleNoncurrentVersionTransition")).Encode(), + "methods": schema.ObjectField{ + Type: schema.NewArrayType(schema.NewNamedType("String")).Encode(), }, - "prefix": schema.ObjectField{ - Type: schema.NewNullableType(schema.NewNamedType("String")).Encode(), + "origins": schema.ObjectField{ + Type: schema.NewArrayType(schema.NewNamedType("String")).Encode(), }, - "status": schema.ObjectField{ - Type: schema.NewNullableType(schema.NewNamedType("String")).Encode(), + "responseHeaders": schema.ObjectField{ + Type: schema.NewArrayType(schema.NewNamedType("String")).Encode(), }, - "transition": schema.ObjectField{ - Type: schema.NewNullableType(schema.NewNamedType("LifecycleTransition")).Encode(), + }, + }, + "BucketHierarchicalNamespace": schema.ObjectType{ + Fields: schema.ObjectTypeFields{ + "enabled": schema.ObjectField{ + Type: schema.NewNamedType("Boolean").Encode(), }, }, }, - "DeleteMarkerReplication": schema.ObjectType{ - Description: toPtr("whether delete markers are replicated - https://docs.aws.amazon.com/AmazonS3/latest/dev/replication-add-config.html"), + "BucketLogging": schema.ObjectType{ + Description: toPtr("holds the bucket's logging configuration, which defines the destination bucket and optional name prefix for the current bucket's logs."), Fields: schema.ObjectTypeFields{ - "status": schema.ObjectField{ - Type: schema.NewNamedType("StorageReplicationRuleStatus").Encode(), + "logBucket": schema.ObjectField{ + Type: schema.NewNamedType("String").Encode(), + }, + "logObjectPrefix": schema.ObjectField{ + Type: schema.NewNamedType("String").Encode(), }, }, }, - "DeleteReplication": schema.ObjectType{ - Description: toPtr("whether versioned deletes are replicated. This is a MinIO specific extension"), + "BucketWebsite": schema.ObjectType{ + Description: toPtr("holds the bucket's website configuration, controlling how the service behaves when accessing bucket contents as a web site. See https://cloud.google.com/storage/docs/static-website for more information."), Fields: schema.ObjectTypeFields{ - "status": schema.ObjectField{ - Type: schema.NewNamedType("StorageReplicationRuleStatus").Encode(), + "mainPageSuffix": schema.ObjectField{ + Type: schema.NewNamedType("String").Encode(), + }, + "notFoundPage": schema.ObjectField{ + Type: schema.NewNullableType(schema.NewNamedType("String")).Encode(), }, }, }, - "ExistingObjectReplication": schema.ObjectType{ - Description: toPtr("whether existing object replication is enabled"), + "CustomPlacementConfig": schema.ObjectType{ + Description: toPtr("holds the bucket's custom placement configuration for Custom Dual Regions. See https://cloud.google.com/storage/docs/locations#location-dr for more information."), Fields: schema.ObjectTypeFields{ - "status": schema.ObjectField{ - Type: schema.NewNamedType("StorageReplicationRuleStatus").Encode(), + "DataLocations": schema.ObjectField{ + Type: schema.NewArrayType(schema.NewNamedType("String")).Encode(), }, }, }, @@ -109,106 +101,86 @@ func GetConnectorSchema() *schema.SchemaResponse { }, }, }, - "LifecycleAllVersionsExpiration": schema.ObjectType{ - Description: toPtr("represents AllVersionsExpiration actions element in an ILM policy"), + "ListIncompleteUploadsOptions": schema.ObjectType{ + Description: toPtr("the input arguments of the ListIncompleteUploads method."), Fields: schema.ObjectTypeFields{ - "days": schema.ObjectField{ - Type: schema.NewNullableType(schema.NewNamedType("Int32")).Encode(), + "prefix": schema.ObjectField{ + Type: schema.NewNamedType("String").Encode(), }, - "deleteMarker": schema.ObjectField{ + "recursive": schema.ObjectField{ Type: schema.NewNullableType(schema.NewNamedType("Boolean")).Encode(), }, }, }, - "LifecycleDelMarkerExpiration": schema.ObjectType{ - Description: toPtr("represents DelMarkerExpiration actions element in an ILM policy"), - Fields: schema.ObjectTypeFields{ - "days": schema.ObjectField{ - Type: schema.NewNullableType(schema.NewNamedType("Int32")).Encode(), - }, - }, - }, - "LifecycleExpiration": schema.ObjectType{ - Description: toPtr("expiration details of lifecycle configuration"), + "ListStorageObjectsOptions": schema.ObjectType{ + Description: toPtr("holds all options of a list object request."), Fields: schema.ObjectTypeFields{ - "date": schema.ObjectField{ - Type: schema.NewNullableType(schema.NewNamedType("Date")).Encode(), + "maxResults": schema.ObjectField{ + Type: schema.NewNamedType("Int32").Encode(), }, - "days": schema.ObjectField{ - Type: schema.NewNullableType(schema.NewNamedType("Int32")).Encode(), + "prefix": schema.ObjectField{ + Type: schema.NewNamedType("String").Encode(), }, - "expiredObjectAllVersions": schema.ObjectField{ - Type: schema.NewNullableType(schema.NewNamedType("Boolean")).Encode(), + "recursive": schema.ObjectField{ + Type: schema.NewNamedType("Boolean").Encode(), }, - "expiredObjectDeleteMarker": schema.ObjectField{ - Type: schema.NewNullableType(schema.NewNamedType("Boolean")).Encode(), + "startAfter": schema.ObjectField{ + Type: schema.NewNamedType("String").Encode(), }, }, }, - "LifecycleFilter": schema.ObjectType{ - Description: toPtr("will be used in selecting rule(s) for lifecycle configuration"), + "MakeStorageBucketOptions": schema.ObjectType{ + Description: toPtr("holds all options to tweak bucket creation."), Fields: schema.ObjectTypeFields{ - "and": schema.ObjectField{ - Type: schema.NewNullableType(schema.NewNamedType("LifecycleFilterAnd")).Encode(), - }, - "objectSizeGreaterThan": schema.ObjectField{ - Type: schema.NewNullableType(schema.NewNamedType("Int64")).Encode(), + "name": schema.ObjectField{ + Type: schema.NewNamedType("String").Encode(), }, - "objectSizeLessThan": schema.ObjectField{ - Type: schema.NewNullableType(schema.NewNamedType("Int64")).Encode(), + "objectLock": schema.ObjectField{ + Type: schema.NewNullableType(schema.NewNamedType("Boolean")).Encode(), }, - "prefix": schema.ObjectField{ + "region": schema.ObjectField{ Type: schema.NewNullableType(schema.NewNamedType("String")).Encode(), }, - "tag": schema.ObjectField{ - Type: schema.NewNullableType(schema.NewNamedType("StorageTag")).Encode(), + "tags": schema.ObjectField{ + Type: schema.NewNullableType(schema.NewNamedType("JSON")).Encode(), }, }, }, - "LifecycleFilterAnd": schema.ObjectType{ - Description: toPtr("the And Rule for LifecycleTag, to be used in LifecycleRuleFilter"), + "ObjectAbortIncompleteMultipartUpload": schema.ObjectType{ Fields: schema.ObjectTypeFields{ - "objectSizeGreaterThan": schema.ObjectField{ - Type: schema.NewNullableType(schema.NewNamedType("Int64")).Encode(), - }, - "objectSizeLessThan": schema.ObjectField{ - Type: schema.NewNullableType(schema.NewNamedType("Int64")).Encode(), - }, - "prefix": schema.ObjectField{ - Type: schema.NewNullableType(schema.NewNamedType("String")).Encode(), - }, - "tags": schema.ObjectField{ - Type: schema.NewNullableType(schema.NewArrayType(schema.NewNamedType("StorageTag"))).Encode(), + "daysAfterInitiation": schema.ObjectField{ + Type: schema.NewNullableType(schema.NewNamedType("Int32")).Encode(), }, }, }, - "LifecycleNoncurrentVersionExpiration": schema.ObjectType{ - Description: toPtr("- Specifies when noncurrent object versions expire. Upon expiration, server permanently deletes the noncurrent object versions. Set this lifecycle configuration action on a bucket that has versioning enabled (or suspended) to request server delete noncurrent object versions at a specific period in the object's lifetime."), + "ObjectLifecycleAllVersionsExpiration": schema.ObjectType{ + Description: toPtr("represents AllVersionsExpiration actions element in an ILM policy"), Fields: schema.ObjectTypeFields{ - "newerNoncurrentVersions": schema.ObjectField{ + "days": schema.ObjectField{ Type: schema.NewNullableType(schema.NewNamedType("Int32")).Encode(), }, - "noncurrentDays": schema.ObjectField{ - Type: schema.NewNullableType(schema.NewNamedType("Int32")).Encode(), + "deleteMarker": schema.ObjectField{ + Type: schema.NewNullableType(schema.NewNamedType("Boolean")).Encode(), }, }, }, - "LifecycleNoncurrentVersionTransition": schema.ObjectType{ - Description: toPtr("sets this action to request server to transition noncurrent object versions to different set storage classes at a specific period in the object's lifetime."), + "ObjectLifecycleConfiguration": schema.ObjectType{ + Description: toPtr("is a collection of lifecycle Rule objects."), Fields: schema.ObjectTypeFields{ - "newerNoncurrentVersions": schema.ObjectField{ - Type: schema.NewNullableType(schema.NewNamedType("Int32")).Encode(), + "rules": schema.ObjectField{ + Type: schema.NewArrayType(schema.NewNamedType("ObjectLifecycleRule")).Encode(), }, - "noncurrentDays": schema.ObjectField{ + }, + }, + "ObjectLifecycleDelMarkerExpiration": schema.ObjectType{ + Fields: schema.ObjectTypeFields{ + "days": schema.ObjectField{ Type: schema.NewNullableType(schema.NewNamedType("Int32")).Encode(), }, - "storageClass": schema.ObjectField{ - Type: schema.NewNullableType(schema.NewNamedType("String")).Encode(), - }, }, }, - "LifecycleTransition": schema.ObjectType{ - Description: toPtr("transition details of lifecycle configuration"), + "ObjectLifecycleExpiration": schema.ObjectType{ + Description: toPtr("expiration details of lifecycle configuration"), Fields: schema.ObjectTypeFields{ "date": schema.ObjectField{ Type: schema.NewNullableType(schema.NewNamedType("Date")).Encode(), @@ -216,172 +188,112 @@ func GetConnectorSchema() *schema.SchemaResponse { "days": schema.ObjectField{ Type: schema.NewNullableType(schema.NewNamedType("Int32")).Encode(), }, - "storageClass": schema.ObjectField{ - Type: schema.NewNullableType(schema.NewNamedType("String")).Encode(), - }, - }, - }, - "ListIncompleteUploadsOptions": schema.ObjectType{ - Description: toPtr("the input arguments of the ListIncompleteUploads method."), - Fields: schema.ObjectTypeFields{ - "prefix": schema.ObjectField{ - Type: schema.NewNamedType("String").Encode(), + "expiredObjectAllVersions": schema.ObjectField{ + Type: schema.NewNullableType(schema.NewNamedType("Boolean")).Encode(), }, - "recursive": schema.ObjectField{ + "expiredObjectDeleteMarker": schema.ObjectField{ Type: schema.NewNullableType(schema.NewNamedType("Boolean")).Encode(), }, }, }, - "ListStorageObjectsOptions": schema.ObjectType{ - Description: toPtr("holds all options of a list object request."), + "ObjectLifecycleFilter": schema.ObjectType{ + Description: toPtr("will be used in selecting rule(s) for lifecycle configuration"), Fields: schema.ObjectTypeFields{ - "maxResults": schema.ObjectField{ - Type: schema.NewNamedType("Int32").Encode(), - }, - "prefix": schema.ObjectField{ - Type: schema.NewNamedType("String").Encode(), - }, - "recursive": schema.ObjectField{ - Type: schema.NewNamedType("Boolean").Encode(), + "matchesPrefix": schema.ObjectField{ + Type: schema.NewNullableType(schema.NewArrayType(schema.NewNamedType("String"))).Encode(), }, - "startAfter": schema.ObjectField{ - Type: schema.NewNamedType("String").Encode(), + "matchesStorageClasses": schema.ObjectField{ + Type: schema.NewNullableType(schema.NewArrayType(schema.NewNamedType("String"))).Encode(), }, - }, - }, - "MakeStorageBucketOptions": schema.ObjectType{ - Description: toPtr("holds all options to tweak bucket creation."), - Fields: schema.ObjectTypeFields{ - "name": schema.ObjectField{ - Type: schema.NewNamedType("String").Encode(), + "matchesSuffix": schema.ObjectField{ + Type: schema.NewNullableType(schema.NewArrayType(schema.NewNamedType("String"))).Encode(), }, - "objectLocking": schema.ObjectField{ - Type: schema.NewNullableType(schema.NewNamedType("Boolean")).Encode(), + "objectSizeGreaterThan": schema.ObjectField{ + Type: schema.NewNullableType(schema.NewNamedType("Int64")).Encode(), }, - "region": schema.ObjectField{ - Type: schema.NewNullableType(schema.NewNamedType("String")).Encode(), + "objectSizeLessThan": schema.ObjectField{ + Type: schema.NewNullableType(schema.NewNamedType("Int64")).Encode(), }, "tags": schema.ObjectField{ Type: schema.NewNullableType(schema.NewNamedType("JSON")).Encode(), }, }, }, - "NotificationCommonConfig": schema.ObjectType{ - Description: toPtr("- represents one single notification configuration such as topic, queue or lambda configuration."), + "ObjectLifecycleNoncurrentVersionExpiration": schema.ObjectType{ + Description: toPtr("- Specifies when noncurrent object versions expire. Upon expiration, server permanently deletes the noncurrent object versions. Set this lifecycle configuration action on a bucket that has versioning enabled (or suspended) to request server delete noncurrent object versions at a specific period in the object's lifetime."), Fields: schema.ObjectTypeFields{ - "arn": schema.ObjectField{ - Type: schema.NewNullableType(schema.NewNamedType("String")).Encode(), - }, - "event": schema.ObjectField{ - Type: schema.NewArrayType(schema.NewNamedType("String")).Encode(), - }, - "filter": schema.ObjectField{ - Type: schema.NewNullableType(schema.NewNamedType("NotificationFilter")).Encode(), + "newerNoncurrentVersions": schema.ObjectField{ + Type: schema.NewNullableType(schema.NewNamedType("Int32")).Encode(), }, - "id": schema.ObjectField{ - Type: schema.NewNullableType(schema.NewNamedType("String")).Encode(), + "noncurrentDays": schema.ObjectField{ + Type: schema.NewNullableType(schema.NewNamedType("Int32")).Encode(), }, }, }, - "NotificationConfig": schema.ObjectType{ - Description: toPtr("the struct that represents a notification configration object."), + "ObjectLifecycleNoncurrentVersionTransition": schema.ObjectType{ + Description: toPtr("sets this action to request server to transition noncurrent object versions to different set storage classes at a specific period in the object's lifetime."), Fields: schema.ObjectTypeFields{ - "cloudFunctionConfigurations": schema.ObjectField{ - Type: schema.NewArrayType(schema.NewNamedType("NotificationLambdaConfig")).Encode(), - }, - "queueConfigurations": schema.ObjectField{ - Type: schema.NewArrayType(schema.NewNamedType("NotificationQueueConfig")).Encode(), + "newerNoncurrentVersions": schema.ObjectField{ + Type: schema.NewNullableType(schema.NewNamedType("Int32")).Encode(), }, - "topicConfigurations": schema.ObjectField{ - Type: schema.NewArrayType(schema.NewNamedType("NotificationTopicConfig")).Encode(), + "noncurrentDays": schema.ObjectField{ + Type: schema.NewNullableType(schema.NewNamedType("Int32")).Encode(), }, - }, - }, - "NotificationFilter": schema.ObjectType{ - Description: toPtr("- a tag in the notification xml structure which carries suffix/prefix filters"), - Fields: schema.ObjectTypeFields{ - "s3Key": schema.ObjectField{ - Type: schema.NewNullableType(schema.NewNamedType("NotificationS3Key")).Encode(), + "storageClass": schema.ObjectField{ + Type: schema.NewNullableType(schema.NewNamedType("String")).Encode(), }, }, }, - "NotificationFilterRule": schema.ObjectType{ - Description: toPtr("child of S3Key, a tag in the notification xml which carries suffix/prefix filters"), + "ObjectLifecycleRule": schema.ObjectType{ + Description: toPtr("represents a single rule in lifecycle configuration"), Fields: schema.ObjectTypeFields{ - "name": schema.ObjectField{ - Type: schema.NewNamedType("String").Encode(), + "abortIncompleteMultipartUpload": schema.ObjectField{ + Type: schema.NewNullableType(schema.NewNamedType("ObjectAbortIncompleteMultipartUpload")).Encode(), }, - "value": schema.ObjectField{ - Type: schema.NewNamedType("String").Encode(), + "allVersionsExpiration": schema.ObjectField{ + Type: schema.NewNullableType(schema.NewNamedType("ObjectLifecycleAllVersionsExpiration")).Encode(), }, - }, - }, - "NotificationLambdaConfig": schema.ObjectType{ - Description: toPtr("carries one single cloudfunction notification configuration"), - Fields: schema.ObjectTypeFields{ - "arn": schema.ObjectField{ - Type: schema.NewNullableType(schema.NewNamedType("String")).Encode(), + "delMarkerExpiration": schema.ObjectField{ + Type: schema.NewNullableType(schema.NewNamedType("ObjectLifecycleDelMarkerExpiration")).Encode(), }, - "cloudFunction": schema.ObjectField{ - Type: schema.NewNamedType("String").Encode(), + "enabled": schema.ObjectField{ + Type: schema.NewNullableType(schema.NewNamedType("Boolean")).Encode(), }, - "event": schema.ObjectField{ - Type: schema.NewArrayType(schema.NewNamedType("String")).Encode(), + "expiration": schema.ObjectField{ + Type: schema.NewNullableType(schema.NewNamedType("ObjectLifecycleExpiration")).Encode(), }, "filter": schema.ObjectField{ - Type: schema.NewNullableType(schema.NewNamedType("NotificationFilter")).Encode(), + Type: schema.NewNullableType(schema.NewArrayType(schema.NewNamedType("ObjectLifecycleFilter"))).Encode(), }, "id": schema.ObjectField{ Type: schema.NewNullableType(schema.NewNamedType("String")).Encode(), }, - }, - }, - "NotificationQueueConfig": schema.ObjectType{ - Description: toPtr("carries one single queue notification configuration"), - Fields: schema.ObjectTypeFields{ - "arn": schema.ObjectField{ - Type: schema.NewNullableType(schema.NewNamedType("String")).Encode(), - }, - "event": schema.ObjectField{ - Type: schema.NewArrayType(schema.NewNamedType("String")).Encode(), + "noncurrentVersionExpiration": schema.ObjectField{ + Type: schema.NewNullableType(schema.NewNamedType("ObjectLifecycleNoncurrentVersionExpiration")).Encode(), }, - "filter": schema.ObjectField{ - Type: schema.NewNullableType(schema.NewNamedType("NotificationFilter")).Encode(), + "noncurrentVersionTransition": schema.ObjectField{ + Type: schema.NewNullableType(schema.NewNamedType("ObjectLifecycleNoncurrentVersionTransition")).Encode(), }, - "id": schema.ObjectField{ + "prefix": schema.ObjectField{ Type: schema.NewNullableType(schema.NewNamedType("String")).Encode(), }, - "queue": schema.ObjectField{ - Type: schema.NewNamedType("String").Encode(), - }, - }, - }, - "NotificationS3Key": schema.ObjectType{ - Description: toPtr("child of Filter, a tag in the notification xml which carries suffix/prefix filters"), - Fields: schema.ObjectTypeFields{ - "filterRule": schema.ObjectField{ - Type: schema.NewNullableType(schema.NewArrayType(schema.NewNamedType("NotificationFilterRule"))).Encode(), + "transition": schema.ObjectField{ + Type: schema.NewNullableType(schema.NewNamedType("ObjectLifecycleTransition")).Encode(), }, }, }, - "NotificationTopicConfig": schema.ObjectType{ - Description: toPtr("carries one single topic notification configuration"), + "ObjectLifecycleTransition": schema.ObjectType{ + Description: toPtr("transition details of lifecycle configuration"), Fields: schema.ObjectTypeFields{ - "arn": schema.ObjectField{ - Type: schema.NewNullableType(schema.NewNamedType("String")).Encode(), - }, - "event": schema.ObjectField{ - Type: schema.NewArrayType(schema.NewNamedType("String")).Encode(), + "date": schema.ObjectField{ + Type: schema.NewNullableType(schema.NewNamedType("Date")).Encode(), }, - "filter": schema.ObjectField{ - Type: schema.NewNullableType(schema.NewNamedType("NotificationFilter")).Encode(), + "days": schema.ObjectField{ + Type: schema.NewNullableType(schema.NewNamedType("Int32")).Encode(), }, - "id": schema.ObjectField{ + "storageClass": schema.ObjectField{ Type: schema.NewNullableType(schema.NewNamedType("String")).Encode(), }, - "topic": schema.ObjectField{ - Type: schema.NewNamedType("String").Encode(), - }, }, }, "PresignedGetStorageObjectOptions": schema.ObjectType{ @@ -422,7 +334,7 @@ func GetConnectorSchema() *schema.SchemaResponse { Type: schema.NewNullableType(schema.NewNamedType("PutStorageObjectOptions")).Encode(), }, "where": schema.ObjectField{ - Type: schema.NewNullableType(schema.NewPredicateType("StorageObjectSimple")).Encode(), + Type: schema.NewNullableType(schema.NewPredicateType("StorageObjectFilter")).Encode(), }, }, }, @@ -465,8 +377,8 @@ func GetConnectorSchema() *schema.SchemaResponse { "legalHold": schema.ObjectField{ Type: schema.NewNullableType(schema.NewNamedType("Boolean")).Encode(), }, - "mode": schema.ObjectField{ - Type: schema.NewNullableType(schema.NewNamedType("StorageRetentionMode")).Encode(), + "metadata": schema.ObjectField{ + Type: schema.NewNullableType(schema.NewNamedType("JSON")).Encode(), }, "numThreads": schema.ObjectField{ Type: schema.NewNullableType(schema.NewNamedType("Int32")).Encode(), @@ -474,8 +386,8 @@ func GetConnectorSchema() *schema.SchemaResponse { "partSize": schema.ObjectField{ Type: schema.NewNullableType(schema.NewNamedType("Int64")).Encode(), }, - "retainUntilDate": schema.ObjectField{ - Type: schema.NewNullableType(schema.NewNamedType("TimestampTZ")).Encode(), + "retention": schema.ObjectField{ + Type: schema.NewNullableType(schema.NewNamedType("PutStorageObjectRetentionOptions")).Encode(), }, "sendContentMd5": schema.ObjectField{ Type: schema.NewNullableType(schema.NewNamedType("Boolean")).Encode(), @@ -483,10 +395,7 @@ func GetConnectorSchema() *schema.SchemaResponse { "storageClass": schema.ObjectField{ Type: schema.NewNullableType(schema.NewNamedType("String")).Encode(), }, - "userMetadata": schema.ObjectField{ - Type: schema.NewNullableType(schema.NewNamedType("JSON")).Encode(), - }, - "userTags": schema.ObjectField{ + "tags": schema.ObjectField{ Type: schema.NewNullableType(schema.NewNamedType("JSON")).Encode(), }, "websiteRedirectLocation": schema.ObjectField{ @@ -494,6 +403,20 @@ func GetConnectorSchema() *schema.SchemaResponse { }, }, }, + "PutStorageObjectRetentionOptions": schema.ObjectType{ + Description: toPtr("represent options of object retention configuration."), + Fields: schema.ObjectTypeFields{ + "governanceBypass": schema.ObjectField{ + Type: schema.NewNullableType(schema.NewNamedType("Boolean")).Encode(), + }, + "mode": schema.ObjectField{ + Type: schema.NewNamedType("StorageRetentionMode").Encode(), + }, + "retainUntilDate": schema.ObjectField{ + Type: schema.NewNamedType("TimestampTZ").Encode(), + }, + }, + }, "RemoveStorageObjectError": schema.ObjectType{ Description: toPtr("the container of Multi Delete S3 API error."), Fields: schema.ObjectTypeFields{ @@ -517,6 +440,9 @@ func GetConnectorSchema() *schema.SchemaResponse { "governanceBypass": schema.ObjectField{ Type: schema.NewNullableType(schema.NewNamedType("Boolean")).Encode(), }, + "softDelete": schema.ObjectField{ + Type: schema.NewNullableType(schema.NewNamedType("Boolean")).Encode(), + }, "versionId": schema.ObjectField{ Type: schema.NewNullableType(schema.NewNamedType("String")).Encode(), }, @@ -542,37 +468,13 @@ func GetConnectorSchema() *schema.SchemaResponse { }, }, }, - "ReplicaModifications": schema.ObjectType{ - Description: toPtr("specifies if replica modification sync is enabled"), - Fields: schema.ObjectTypeFields{ - "status": schema.ObjectField{ - Type: schema.NewNamedType("StorageReplicationRuleStatus").Encode(), - }, - }, - }, "ServerSideEncryptionConfiguration": schema.ObjectType{ Description: toPtr("is the default encryption configuration structure."), Fields: schema.ObjectTypeFields{ - "rules": schema.ObjectField{ - Type: schema.NewArrayType(schema.NewNamedType("ServerSideEncryptionRule")).Encode(), - }, - }, - }, - "ServerSideEncryptionRule": schema.ObjectType{ - Description: toPtr("rule layer encapsulates default encryption configuration"), - Fields: schema.ObjectTypeFields{ - "apply": schema.ObjectField{ - Type: schema.NewNamedType("StorageApplySSEByDefault").Encode(), - }, - }, - }, - "SetStorageObjectLegalHoldOptions": schema.ObjectType{ - Description: toPtr("represents options specified by user for PutObjectLegalHold call."), - Fields: schema.ObjectTypeFields{ - "status": schema.ObjectField{ - Type: schema.NewNullableType(schema.NewNamedType("Boolean")).Encode(), + "kmsMasterKeyId": schema.ObjectField{ + Type: schema.NewNullableType(schema.NewNamedType("String")).Encode(), }, - "versionId": schema.ObjectField{ + "sseAlgorithm": schema.ObjectField{ Type: schema.NewNullableType(schema.NewNamedType("String")).Encode(), }, }, @@ -603,39 +505,77 @@ func GetConnectorSchema() *schema.SchemaResponse { "retainUntilDate": schema.ObjectField{ Type: schema.NewNullableType(schema.NewNamedType("TimestampTZ")).Encode(), }, - "versionId": schema.ObjectField{ - Type: schema.NewNullableType(schema.NewNamedType("String")).Encode(), - }, }, }, - "SetStorageObjectTagsOptions": schema.ObjectType{ - Description: toPtr("holds an object version id to update tag(s) of a specific object version."), + "StorageBucket": schema.ObjectType{ + Description: toPtr("the container for bucket metadata."), Fields: schema.ObjectTypeFields{ - "tags": schema.ObjectField{ - Type: schema.NewNamedType("JSON").Encode(), + "autoclass": schema.ObjectField{ + Type: schema.NewNullableType(schema.NewNamedType("BucketAutoclass")).Encode(), }, - "versionId": schema.ObjectField{ + "cors": schema.ObjectField{ + Type: schema.NewNullableType(schema.NewArrayType(schema.NewNamedType("BucketCors"))).Encode(), + }, + "creationTime": schema.ObjectField{ + Type: schema.NewNullableType(schema.NewNamedType("TimestampTZ")).Encode(), + }, + "customPlacementConfig": schema.ObjectField{ + Type: schema.NewNullableType(schema.NewNamedType("CustomPlacementConfig")).Encode(), + }, + "defaultEventBasedHold": schema.ObjectField{ + Type: schema.NewNullableType(schema.NewNamedType("Boolean")).Encode(), + }, + "encryption": schema.ObjectField{ + Type: schema.NewNullableType(schema.NewNamedType("ServerSideEncryptionConfiguration")).Encode(), + }, + "etag": schema.ObjectField{ Type: schema.NewNullableType(schema.NewNamedType("String")).Encode(), }, - }, - }, - "SourceSelectionCriteria": schema.ObjectType{ - Description: toPtr("specifies additional source selection criteria in ReplicationConfiguration."), - Fields: schema.ObjectTypeFields{ - "replicaModifications": schema.ObjectField{ - Type: schema.NewNullableType(schema.NewNamedType("ReplicaModifications")).Encode(), + "hierarchicalNamespace": schema.ObjectField{ + Type: schema.NewNullableType(schema.NewNamedType("BucketHierarchicalNamespace")).Encode(), }, - }, - }, - "StorageApplySSEByDefault": schema.ObjectType{ - Description: toPtr("defines default encryption configuration, KMS or SSE. To activate KMS, SSEAlgoritm needs to be set to `aws:kms“. Minio currently does not support Kms."), - Fields: schema.ObjectTypeFields{ - "kmsMasterKeyId": schema.ObjectField{ + "lastModified": schema.ObjectField{ + Type: schema.NewNullableType(schema.NewNamedType("TimestampTZ")).Encode(), + }, + "lifecycle": schema.ObjectField{ + Type: schema.NewNullableType(schema.NewNamedType("ObjectLifecycleConfiguration")).Encode(), + }, + "locationType": schema.ObjectField{ Type: schema.NewNullableType(schema.NewNamedType("String")).Encode(), }, - "sseAlgorithm": schema.ObjectField{ + "logging": schema.ObjectField{ + Type: schema.NewNullableType(schema.NewNamedType("BucketLogging")).Encode(), + }, + "name": schema.ObjectField{ Type: schema.NewNamedType("String").Encode(), }, + "objectLock": schema.ObjectField{ + Type: schema.NewNullableType(schema.NewNamedType("StorageObjectLockConfig")).Encode(), + }, + "region": schema.ObjectField{ + Type: schema.NewNullableType(schema.NewNamedType("String")).Encode(), + }, + "requesterPays": schema.ObjectField{ + Type: schema.NewNullableType(schema.NewNamedType("Boolean")).Encode(), + }, + "rpo": schema.ObjectField{ + Type: schema.NewNullableType(schema.NewNamedType("GoogleStorageRPO")).Encode(), + }, + "softDeletePolicy": schema.ObjectField{ + Type: schema.NewNullableType(schema.NewNamedType("StorageObjectSoftDeletePolicy")).Encode(), + }, + "storageClass": schema.ObjectField{ + Type: schema.NewNullableType(schema.NewNamedType("String")).Encode(), + }, + "tags": schema.ObjectField{ + Type: schema.NewNullableType(schema.NewNamedType("JSON")).Encode(), + }, + "versioning": schema.ObjectField{ + Type: schema.NewNullableType(schema.NewNamedType("StorageBucketVersioningConfiguration")).Encode(), + }, + "website": schema.ObjectField{ + Type: schema.NewNullableType(schema.NewNamedType("BucketWebsite")).Encode(), + }, }, }, "StorageBucketArguments": schema.ObjectType{ @@ -649,23 +589,23 @@ func GetConnectorSchema() *schema.SchemaResponse { }, }, }, - "StorageBucketInfo": schema.ObjectType{ - Description: toPtr("container for bucket metadata."), + "StorageBucketListResults": schema.ObjectType{ + Description: toPtr("hold the paginated results of the storage bucket list."), Fields: schema.ObjectTypeFields{ - "creationDate": schema.ObjectField{ - Type: schema.NewNamedType("TimestampTZ").Encode(), + "buckets": schema.ObjectField{ + Type: schema.NewArrayType(schema.NewNamedType("StorageBucket")).Encode(), }, - "name": schema.ObjectField{ - Type: schema.NewNamedType("String").Encode(), - }, - "tags": schema.ObjectField{ - Type: schema.NewNullableType(schema.NewNamedType("JSON")).Encode(), + "pageInfo": schema.ObjectField{ + Type: schema.NewNamedType("StoragePaginationInfo").Encode(), }, }, }, "StorageBucketVersioningConfiguration": schema.ObjectType{ Description: toPtr("is the versioning configuration structure"), Fields: schema.ObjectTypeFields{ + "enabled": schema.ObjectField{ + Type: schema.NewNamedType("Boolean").Encode(), + }, "excludeFolders": schema.ObjectField{ Type: schema.NewNullableType(schema.NewNamedType("Boolean")).Encode(), }, @@ -675,9 +615,6 @@ func GetConnectorSchema() *schema.SchemaResponse { "mfaDelete": schema.ObjectField{ Type: schema.NewNullableType(schema.NewNamedType("String")).Encode(), }, - "status": schema.ObjectField{ - Type: schema.NewNullableType(schema.NewNamedType("String")).Encode(), - }, }, }, "StorageCopyDestOptions": schema.ObjectType{ @@ -689,28 +626,22 @@ func GetConnectorSchema() *schema.SchemaResponse { "legalHold": schema.ObjectField{ Type: schema.NewNullableType(schema.NewNamedType("Boolean")).Encode(), }, + "metadata": schema.ObjectField{ + Type: schema.NewNullableType(schema.NewNamedType("JSON")).Encode(), + }, "mode": schema.ObjectField{ Type: schema.NewNullableType(schema.NewNamedType("StorageRetentionMode")).Encode(), }, "object": schema.ObjectField{ Type: schema.NewNamedType("String").Encode(), }, - "replaceMetadata": schema.ObjectField{ - Type: schema.NewNullableType(schema.NewNamedType("Boolean")).Encode(), - }, - "replaceTags": schema.ObjectField{ - Type: schema.NewNullableType(schema.NewNamedType("Boolean")).Encode(), - }, "retainUntilDate": schema.ObjectField{ Type: schema.NewNullableType(schema.NewNamedType("TimestampTZ")).Encode(), }, "size": schema.ObjectField{ Type: schema.NewNullableType(schema.NewNamedType("Int64")).Encode(), }, - "userMetadata": schema.ObjectField{ - Type: schema.NewNullableType(schema.NewNamedType("JSON")).Encode(), - }, - "userTags": schema.ObjectField{ + "tags": schema.ObjectField{ Type: schema.NewNullableType(schema.NewNamedType("JSON")).Encode(), }, }, @@ -785,7 +716,7 @@ func GetConnectorSchema() *schema.SchemaResponse { Type: schema.NewNullableType(schema.NewNamedType("Boolean")).Encode(), }, "acl": schema.ObjectField{ - Type: schema.NewNullableType(schema.NewNamedType("String")).Encode(), + Type: schema.NewNullableType(schema.NewNamedType("JSON")).Encode(), }, "archiveStatus": schema.ObjectField{ Type: schema.NewNullableType(schema.NewNamedType("String")).Encode(), @@ -835,23 +766,8 @@ func GetConnectorSchema() *schema.SchemaResponse { "contentType": schema.ObjectField{ Type: schema.NewNullableType(schema.NewNamedType("String")).Encode(), }, - "copyCompletionTime": schema.ObjectField{ - Type: schema.NewNullableType(schema.NewNamedType("TimestampTZ")).Encode(), - }, - "copyId": schema.ObjectField{ - Type: schema.NewNullableType(schema.NewNamedType("String")).Encode(), - }, - "copyProgress": schema.ObjectField{ - Type: schema.NewNullableType(schema.NewNamedType("String")).Encode(), - }, - "copySource": schema.ObjectField{ - Type: schema.NewNullableType(schema.NewNamedType("String")).Encode(), - }, - "copyStatus": schema.ObjectField{ - Type: schema.NewNullableType(schema.NewNamedType("String")).Encode(), - }, - "copyStatusDescription": schema.ObjectField{ - Type: schema.NewNullableType(schema.NewNamedType("String")).Encode(), + "copy": schema.ObjectField{ + Type: schema.NewNullableType(schema.NewNamedType("StorageObjectCopyInfo")).Encode(), }, "creationTime": schema.ObjectField{ Type: schema.NewNullableType(schema.NewNamedType("TimestampTZ")).Encode(), @@ -868,9 +784,6 @@ func GetConnectorSchema() *schema.SchemaResponse { "destinationSnapshot": schema.ObjectField{ Type: schema.NewNullableType(schema.NewNamedType("String")).Encode(), }, - "encryptionScope": schema.ObjectField{ - Type: schema.NewNullableType(schema.NewNamedType("String")).Encode(), - }, "etag": schema.ObjectField{ Type: schema.NewNullableType(schema.NewNamedType("String")).Encode(), }, @@ -889,18 +802,15 @@ func GetConnectorSchema() *schema.SchemaResponse { "group": schema.ObjectField{ Type: schema.NewNullableType(schema.NewNamedType("String")).Encode(), }, - "immutabilityPolicyMode": schema.ObjectField{ - Type: schema.NewNullableType(schema.NewNamedType("String")).Encode(), - }, - "immutabilityPolicyUntilDate": schema.ObjectField{ - Type: schema.NewNullableType(schema.NewNamedType("TimestampTZ")).Encode(), - }, "incrementalCopy": schema.ObjectField{ Type: schema.NewNullableType(schema.NewNamedType("Boolean")).Encode(), }, "isLatest": schema.ObjectField{ Type: schema.NewNullableType(schema.NewNamedType("Boolean")).Encode(), }, + "kmsKeyName": schema.ObjectField{ + Type: schema.NewNullableType(schema.NewNamedType("String")).Encode(), + }, "lastAccessTime": schema.ObjectField{ Type: schema.NewNullableType(schema.NewNamedType("TimestampTZ")).Encode(), }, @@ -919,6 +829,9 @@ func GetConnectorSchema() *schema.SchemaResponse { "legalHold": schema.ObjectField{ Type: schema.NewNullableType(schema.NewNamedType("Boolean")).Encode(), }, + "mediaLink": schema.ObjectField{ + Type: schema.NewNullableType(schema.NewNamedType("String")).Encode(), + }, "metadata": schema.ObjectField{ Type: schema.NewNullableType(schema.NewNamedType("JSON")).Encode(), }, @@ -931,6 +844,9 @@ func GetConnectorSchema() *schema.SchemaResponse { "permissions": schema.ObjectField{ Type: schema.NewNullableType(schema.NewNamedType("String")).Encode(), }, + "rawMetadata": schema.ObjectField{ + Type: schema.NewNullableType(schema.NewNamedType("JSON")).Encode(), + }, "rehydratePriority": schema.ObjectField{ Type: schema.NewNullableType(schema.NewNamedType("String")).Encode(), }, @@ -949,6 +865,12 @@ func GetConnectorSchema() *schema.SchemaResponse { "restore": schema.ObjectField{ Type: schema.NewNullableType(schema.NewNamedType("StorageRestoreInfo")).Encode(), }, + "retentionMode": schema.ObjectField{ + Type: schema.NewNullableType(schema.NewNamedType("String")).Encode(), + }, + "retentionUntilDate": schema.ObjectField{ + Type: schema.NewNullableType(schema.NewNamedType("TimestampTZ")).Encode(), + }, "sealed": schema.ObjectField{ Type: schema.NewNullableType(schema.NewNamedType("Boolean")).Encode(), }, @@ -961,13 +883,10 @@ func GetConnectorSchema() *schema.SchemaResponse { "storageClass": schema.ObjectField{ Type: schema.NewNullableType(schema.NewNamedType("String")).Encode(), }, - "userMetadata": schema.ObjectField{ - Type: schema.NewNullableType(schema.NewNamedType("JSON")).Encode(), - }, - "userTagCount": schema.ObjectField{ + "tagCount": schema.ObjectField{ Type: schema.NewNullableType(schema.NewNamedType("Int32")).Encode(), }, - "userTags": schema.ObjectField{ + "tags": schema.ObjectField{ Type: schema.NewNullableType(schema.NewNamedType("JSON")).Encode(), }, "versionId": schema.ObjectField{ @@ -995,6 +914,29 @@ func GetConnectorSchema() *schema.SchemaResponse { }, }, }, + "StorageObjectCopyInfo": schema.ObjectType{ + Description: toPtr("holds the copy information if the object was copied from another object."), + Fields: schema.ObjectTypeFields{ + "completionTime": schema.ObjectField{ + Type: schema.NewNullableType(schema.NewNamedType("TimestampTZ")).Encode(), + }, + "id": schema.ObjectField{ + Type: schema.NewNamedType("String").Encode(), + }, + "progress": schema.ObjectField{ + Type: schema.NewNullableType(schema.NewNamedType("String")).Encode(), + }, + "source": schema.ObjectField{ + Type: schema.NewNullableType(schema.NewNamedType("String")).Encode(), + }, + "status": schema.ObjectField{ + Type: schema.NewNullableType(schema.NewNamedType("String")).Encode(), + }, + "statusDescription": schema.ObjectField{ + Type: schema.NewNullableType(schema.NewNamedType("String")).Encode(), + }, + }, + }, "StorageObjectListResults": schema.ObjectType{ Description: toPtr("hold the paginated results of the storage object list."), Fields: schema.ObjectTypeFields{ @@ -1002,18 +944,19 @@ func GetConnectorSchema() *schema.SchemaResponse { Type: schema.NewArrayType(schema.NewNamedType("StorageObject")).Encode(), }, "pageInfo": schema.ObjectField{ - Type: schema.NewNamedType("StorageObjectPaginationInfo").Encode(), + Type: schema.NewNamedType("StoragePaginationInfo").Encode(), }, }, }, "StorageObjectLockConfig": schema.ObjectType{ + Description: toPtr("represents the object lock configuration in given bucket"), Fields: schema.ObjectTypeFields{ + "enabled": schema.ObjectField{ + Type: schema.NewNamedType("Boolean").Encode(), + }, "mode": schema.ObjectField{ Type: schema.NewNullableType(schema.NewNamedType("StorageRetentionMode")).Encode(), }, - "objectLock": schema.ObjectField{ - Type: schema.NewNamedType("String").Encode(), - }, "unit": schema.ObjectField{ Type: schema.NewNullableType(schema.NewNamedType("StorageRetentionValidityUnit")).Encode(), }, @@ -1042,17 +985,14 @@ func GetConnectorSchema() *schema.SchemaResponse { }, }, }, - "StorageObjectPaginationInfo": schema.ObjectType{ - Description: toPtr("holds the pagination information."), + "StorageObjectSoftDeletePolicy": schema.ObjectType{ + Description: toPtr("contains the bucket's soft delete policy, which defines the period of time that soft-deleted objects will be retained, and cannot be permanently deleted."), Fields: schema.ObjectTypeFields{ - "cursor": schema.ObjectField{ - Type: schema.NewNullableType(schema.NewNamedType("String")).Encode(), - }, - "hasNextPage": schema.ObjectField{ - Type: schema.NewNamedType("Boolean").Encode(), + "effectiveTime": schema.ObjectField{ + Type: schema.NewNamedType("TimestampTZ").Encode(), }, - "nextCursor": schema.ObjectField{ - Type: schema.NewNullableType(schema.NewNamedType("String")).Encode(), + "retentionDuration": schema.ObjectField{ + Type: schema.NewNamedType("Duration").Encode(), }, }, }, @@ -1062,86 +1002,19 @@ func GetConnectorSchema() *schema.SchemaResponse { "id": schema.ObjectField{ Type: schema.NewNullableType(schema.NewNamedType("String")).Encode(), }, - "name": schema.ObjectField{ - Type: schema.NewNullableType(schema.NewNamedType("String")).Encode(), - }, - }, - }, - "StorageReplicationConfig": schema.ObjectType{ - Description: toPtr("replication configuration specified in https://docs.aws.amazon.com/AmazonS3/latest/dev/replication-add-config.html"), - Fields: schema.ObjectTypeFields{ - "role": schema.ObjectField{ - Type: schema.NewNullableType(schema.NewNamedType("String")).Encode(), - }, - "rules": schema.ObjectField{ - Type: schema.NewArrayType(schema.NewNamedType("StorageReplicationRule")).Encode(), - }, - }, - }, - "StorageReplicationDestination": schema.ObjectType{ - Fields: schema.ObjectTypeFields{ - "bucket": schema.ObjectField{ - Type: schema.NewNamedType("String").Encode(), - }, - "storageClass": schema.ObjectField{ - Type: schema.NewNullableType(schema.NewNamedType("String")).Encode(), - }, - }, - }, - "StorageReplicationFilter": schema.ObjectType{ - Description: toPtr("a filter for a replication configuration Rule."), - Fields: schema.ObjectTypeFields{ - "and": schema.ObjectField{ - Type: schema.NewNullableType(schema.NewNamedType("StorageReplicationFilterAnd")).Encode(), - }, - "rrefix": schema.ObjectField{ - Type: schema.NewNullableType(schema.NewNamedType("String")).Encode(), - }, - "tag": schema.ObjectField{ - Type: schema.NewNullableType(schema.NewNamedType("StorageTag")).Encode(), - }, - }, - }, - "StorageReplicationFilterAnd": schema.ObjectType{ - Description: toPtr("- a tag to combine a prefix and multiple tags for replication configuration rule."), - Fields: schema.ObjectTypeFields{ - "rrefix": schema.ObjectField{ - Type: schema.NewNullableType(schema.NewNamedType("String")).Encode(), - }, - "tag": schema.ObjectField{ - Type: schema.NewNullableType(schema.NewArrayType(schema.NewNamedType("StorageTag"))).Encode(), + "name": schema.ObjectField{ + Type: schema.NewNullableType(schema.NewNamedType("String")).Encode(), }, }, }, - "StorageReplicationRule": schema.ObjectType{ - Description: toPtr("a rule for replication configuration."), + "StoragePaginationInfo": schema.ObjectType{ + Description: toPtr("holds the pagination information."), Fields: schema.ObjectTypeFields{ - "deleteMarkerReplication": schema.ObjectField{ - Type: schema.NewNullableType(schema.NewNamedType("DeleteMarkerReplication")).Encode(), - }, - "deleteReplication": schema.ObjectField{ - Type: schema.NewNullableType(schema.NewNamedType("DeleteReplication")).Encode(), - }, - "destination": schema.ObjectField{ - Type: schema.NewNullableType(schema.NewNamedType("StorageReplicationDestination")).Encode(), - }, - "existingObjectReplication": schema.ObjectField{ - Type: schema.NewNullableType(schema.NewNamedType("ExistingObjectReplication")).Encode(), - }, - "filter": schema.ObjectField{ - Type: schema.NewNamedType("StorageReplicationFilter").Encode(), - }, - "id": schema.ObjectField{ + "cursor": schema.ObjectField{ Type: schema.NewNullableType(schema.NewNamedType("String")).Encode(), }, - "priority": schema.ObjectField{ - Type: schema.NewNamedType("Int32").Encode(), - }, - "sourceSelectionCriteria": schema.ObjectField{ - Type: schema.NewNullableType(schema.NewNamedType("SourceSelectionCriteria")).Encode(), - }, - "status": schema.ObjectField{ - Type: schema.NewNamedType("StorageReplicationRuleStatus").Encode(), + "hasNextPage": schema.ObjectField{ + Type: schema.NewNamedType("Boolean").Encode(), }, }, }, @@ -1156,17 +1029,6 @@ func GetConnectorSchema() *schema.SchemaResponse { }, }, }, - "StorageTag": schema.ObjectType{ - Description: toPtr("structure key/value pair representing an object tag to apply configuration"), - Fields: schema.ObjectTypeFields{ - "key": schema.ObjectField{ - Type: schema.NewNullableType(schema.NewNamedType("String")).Encode(), - }, - "value": schema.ObjectField{ - Type: schema.NewNullableType(schema.NewNamedType("String")).Encode(), - }, - }, - }, "StorageUploadInfo": schema.ObjectType{ Description: toPtr("represents the information of the uploaded object."), Fields: schema.ObjectTypeFields{ @@ -1220,6 +1082,46 @@ func GetConnectorSchema() *schema.SchemaResponse { }, }, }, + "UpdateStorageBucketOptions": schema.ObjectType{ + Description: toPtr("hold update options for the bucket."), + Fields: schema.ObjectTypeFields{ + "encryption": schema.ObjectField{ + Type: schema.NewNullableType(schema.NewNamedType("ServerSideEncryptionConfiguration")).Encode(), + }, + "lifecycle": schema.ObjectField{ + Type: schema.NewNullableType(schema.NewNamedType("ObjectLifecycleConfiguration")).Encode(), + }, + "objectLock": schema.ObjectField{ + Type: schema.NewNullableType(schema.NewNamedType("SetStorageObjectLockConfig")).Encode(), + }, + "tags": schema.ObjectField{ + Type: schema.NewNullableType(schema.NewNamedType("JSON")).Encode(), + }, + "versioningEnabled": schema.ObjectField{ + Type: schema.NewNullableType(schema.NewNamedType("Boolean")).Encode(), + }, + }, + }, + "UpdateStorageObjectOptions": schema.ObjectType{ + Description: toPtr("represents options specified by user for updating object."), + Fields: schema.ObjectTypeFields{ + "legalHold": schema.ObjectField{ + Type: schema.NewNullableType(schema.NewNamedType("Boolean")).Encode(), + }, + "metadata": schema.ObjectField{ + Type: schema.NewNullableType(schema.NewNamedType("JSON")).Encode(), + }, + "retention": schema.ObjectField{ + Type: schema.NewNullableType(schema.NewNamedType("SetStorageObjectRetentionOptions")).Encode(), + }, + "tags": schema.ObjectField{ + Type: schema.NewNullableType(schema.NewNamedType("JSON")).Encode(), + }, + "versionId": schema.ObjectField{ + Type: schema.NewNullableType(schema.NewNamedType("String")).Encode(), + }, + }, + }, }, Functions: []schema.FunctionInfo{ { @@ -1249,7 +1151,7 @@ func GetConnectorSchema() *schema.SchemaResponse { Type: schema.NewNullableType(schema.NewNamedType("String")).Encode(), }, "where": { - Type: schema.NewNullableType(schema.NewPredicateType("StorageObjectSimple")).Encode(), + Type: schema.NewNullableType(schema.NewPredicateType("StorageObjectFilter")).Encode(), }, }, }, @@ -1280,14 +1182,14 @@ func GetConnectorSchema() *schema.SchemaResponse { Type: schema.NewNullableType(schema.NewNamedType("String")).Encode(), }, "where": { - Type: schema.NewNullableType(schema.NewPredicateType("StorageObjectSimple")).Encode(), + Type: schema.NewNullableType(schema.NewPredicateType("StorageObjectFilter")).Encode(), }, }, }, { - Name: "storageBucketEncryption", - Description: toPtr("gets default encryption configuration set on a bucket."), - ResultType: schema.NewNullableType(schema.NewNamedType("ServerSideEncryptionConfiguration")).Encode(), + Name: "storageBucket", + Description: toPtr("gets a bucket by name."), + ResultType: schema.NewNullableType(schema.NewNamedType("StorageBucket")).Encode(), Arguments: map[string]schema.ArgumentInfo{ "bucket": { Type: schema.NewNullableType(schema.NewNamedType("String")).Encode(), @@ -1311,76 +1213,37 @@ func GetConnectorSchema() *schema.SchemaResponse { }, }, { - Name: "storageBucketLifecycle", - Description: toPtr("gets lifecycle on a bucket or a prefix."), - ResultType: schema.NewNullableType(schema.NewNamedType("BucketLifecycleConfiguration")).Encode(), - Arguments: map[string]schema.ArgumentInfo{ - "bucket": { - Type: schema.NewNullableType(schema.NewNamedType("String")).Encode(), - }, - "clientId": { - Type: schema.NewNullableType(schema.NewNamedType("StorageClientID")).Encode(), - }, - }, - }, - { - Name: "storageBucketNotification", - Description: toPtr("gets notification configuration on a bucket."), - ResultType: schema.NewNullableType(schema.NewNamedType("NotificationConfig")).Encode(), + Name: "storageBuckets", + Description: toPtr("list all buckets."), + ResultType: schema.NewNamedType("StorageBucketListResults").Encode(), Arguments: map[string]schema.ArgumentInfo{ - "bucket": { - Type: schema.NewNullableType(schema.NewNamedType("String")).Encode(), - }, - "clientId": { - Type: schema.NewNullableType(schema.NewNamedType("StorageClientID")).Encode(), + "maxResults": { + Type: schema.NewNullableType(schema.NewNamedType("Int32")).Encode(), }, - }, - }, - { - Name: "storageBucketPolicy", - Description: toPtr("gets access permissions on a bucket or a prefix."), - ResultType: schema.NewNamedType("String").Encode(), - Arguments: map[string]schema.ArgumentInfo{ - "bucket": { + "startAfter": { Type: schema.NewNullableType(schema.NewNamedType("String")).Encode(), }, - "clientId": { - Type: schema.NewNullableType(schema.NewNamedType("StorageClientID")).Encode(), + "where": { + Type: schema.NewNullableType(schema.NewPredicateType("StorageBucketFilter")).Encode(), }, }, }, { - Name: "storageBucketReplication", - ResultType: schema.NewNullableType(schema.NewNamedType("StorageReplicationConfig")).Encode(), + Name: "storageDeletedObjects", + Description: toPtr("list deleted objects in a bucket."), + ResultType: schema.NewNamedType("StorageObjectListResults").Encode(), Arguments: map[string]schema.ArgumentInfo{ - "bucket": { - Type: schema.NewNullableType(schema.NewNamedType("String")).Encode(), + "maxResults": { + Type: schema.NewNullableType(schema.NewNamedType("Int32")).Encode(), }, - "clientId": { - Type: schema.NewNullableType(schema.NewNamedType("StorageClientID")).Encode(), + "recursive": { + Type: schema.NewNullableType(schema.NewNamedType("Boolean")).Encode(), }, - }, - }, - { - Name: "storageBucketVersioning", - Description: toPtr("gets versioning configuration set on a bucket."), - ResultType: schema.NewNullableType(schema.NewNamedType("StorageBucketVersioningConfiguration")).Encode(), - Arguments: map[string]schema.ArgumentInfo{ - "bucket": { + "startAfter": { Type: schema.NewNullableType(schema.NewNamedType("String")).Encode(), }, - "clientId": { - Type: schema.NewNullableType(schema.NewNamedType("StorageClientID")).Encode(), - }, - }, - }, - { - Name: "storageBuckets", - Description: toPtr("list all buckets."), - ResultType: schema.NewArrayType(schema.NewNamedType("StorageBucketInfo")).Encode(), - Arguments: map[string]schema.ArgumentInfo{ - "clientId": { - Type: schema.NewNullableType(schema.NewNamedType("StorageClientID")).Encode(), + "where": { + Type: schema.NewNullableType(schema.NewPredicateType("StorageObjectFilter")).Encode(), }, }, }, @@ -1430,20 +1293,7 @@ func GetConnectorSchema() *schema.SchemaResponse { Type: schema.NewNullableType(schema.NewNamedType("String")).Encode(), }, "where": { - Type: schema.NewNullableType(schema.NewPredicateType("StorageObjectSimple")).Encode(), - }, - }, - }, - { - Name: "storageObjectLockConfig", - Description: toPtr("gets object lock configuration of given bucket."), - ResultType: schema.NewNullableType(schema.NewNamedType("StorageObjectLockConfig")).Encode(), - Arguments: map[string]schema.ArgumentInfo{ - "bucket": { - Type: schema.NewNullableType(schema.NewNamedType("String")).Encode(), - }, - "clientId": { - Type: schema.NewNullableType(schema.NewNamedType("StorageClientID")).Encode(), + Type: schema.NewNullableType(schema.NewPredicateType("StorageObjectFilter")).Encode(), }, }, }, @@ -1462,7 +1312,7 @@ func GetConnectorSchema() *schema.SchemaResponse { Type: schema.NewNullableType(schema.NewNamedType("String")).Encode(), }, "where": { - Type: schema.NewNullableType(schema.NewPredicateType("StorageObjectSimple")).Encode(), + Type: schema.NewNullableType(schema.NewPredicateType("StorageObjectFilter")).Encode(), }, }, }, @@ -1487,7 +1337,7 @@ func GetConnectorSchema() *schema.SchemaResponse { Type: schema.NewNullableType(schema.NewNamedType("JSON")).Encode(), }, "where": { - Type: schema.NewNullableType(schema.NewPredicateType("StorageObjectSimple")).Encode(), + Type: schema.NewNullableType(schema.NewPredicateType("StorageObjectFilter")).Encode(), }, }, }, @@ -1509,7 +1359,7 @@ func GetConnectorSchema() *schema.SchemaResponse { Type: schema.NewNamedType("String").Encode(), }, "where": { - Type: schema.NewNullableType(schema.NewPredicateType("StorageObjectSimple")).Encode(), + Type: schema.NewNullableType(schema.NewPredicateType("StorageObjectFilter")).Encode(), }, }, }, @@ -1558,7 +1408,7 @@ func GetConnectorSchema() *schema.SchemaResponse { "name": { Type: schema.NewNamedType("String").Encode(), }, - "objectLocking": { + "objectLock": { Type: schema.NewNullableType(schema.NewNamedType("Boolean")).Encode(), }, "region": { @@ -1569,19 +1419,6 @@ func GetConnectorSchema() *schema.SchemaResponse { }, }, }, - { - Name: "enableStorageBucketVersioning", - Description: toPtr("enables bucket versioning support."), - ResultType: schema.NewNamedType("Boolean").Encode(), - Arguments: map[string]schema.ArgumentInfo{ - "bucket": { - Type: schema.NewNullableType(schema.NewNamedType("String")).Encode(), - }, - "clientId": { - Type: schema.NewNullableType(schema.NewNamedType("StorageClientID")).Encode(), - }, - }, - }, { Name: "removeIncompleteStorageUpload", Description: toPtr("removes a partially uploaded object."), @@ -1611,18 +1448,6 @@ func GetConnectorSchema() *schema.SchemaResponse { }, }, }, - { - Name: "removeStorageBucketReplication", - ResultType: schema.NewNamedType("Boolean").Encode(), - Arguments: map[string]schema.ArgumentInfo{ - "bucket": { - Type: schema.NewNullableType(schema.NewNamedType("String")).Encode(), - }, - "clientId": { - Type: schema.NewNullableType(schema.NewNamedType("StorageClientID")).Encode(), - }, - }, - }, { Name: "removeStorageObject", Description: toPtr("removes an object with some specified options."), @@ -1643,11 +1468,14 @@ func GetConnectorSchema() *schema.SchemaResponse { "object": { Type: schema.NewNamedType("String").Encode(), }, + "softDelete": { + Type: schema.NewNullableType(schema.NewNamedType("Boolean")).Encode(), + }, "versionId": { Type: schema.NewNullableType(schema.NewNamedType("String")).Encode(), }, "where": { - Type: schema.NewNullableType(schema.NewPredicateType("StorageObjectSimple")).Encode(), + Type: schema.NewNullableType(schema.NewPredicateType("StorageObjectFilter")).Encode(), }, }, }, @@ -1678,45 +1506,13 @@ func GetConnectorSchema() *schema.SchemaResponse { Type: schema.NewNamedType("String").Encode(), }, "where": { - Type: schema.NewNullableType(schema.NewPredicateType("StorageObjectSimple")).Encode(), - }, - }, - }, - { - Name: "setStorageBucketEncryption", - Description: toPtr("sets default encryption configuration on a bucket."), - ResultType: schema.NewNamedType("Boolean").Encode(), - Arguments: map[string]schema.ArgumentInfo{ - "bucket": { - Type: schema.NewNullableType(schema.NewNamedType("String")).Encode(), - }, - "clientId": { - Type: schema.NewNullableType(schema.NewNamedType("StorageClientID")).Encode(), - }, - "rules": { - Type: schema.NewArrayType(schema.NewNamedType("ServerSideEncryptionRule")).Encode(), - }, - }, - }, - { - Name: "setStorageBucketLifecycle", - Description: toPtr("sets lifecycle on bucket or an object prefix."), - ResultType: schema.NewNamedType("Boolean").Encode(), - Arguments: map[string]schema.ArgumentInfo{ - "bucket": { - Type: schema.NewNullableType(schema.NewNamedType("String")).Encode(), - }, - "clientId": { - Type: schema.NewNullableType(schema.NewNamedType("StorageClientID")).Encode(), - }, - "rules": { - Type: schema.NewArrayType(schema.NewNamedType("BucketLifecycleRule")).Encode(), + Type: schema.NewNullableType(schema.NewPredicateType("StorageObjectFilter")).Encode(), }, }, }, { - Name: "setStorageBucketNotification", - Description: toPtr("sets a new notification configuration on a bucket."), + Name: "restoreStorageObject", + Description: toPtr("restore a soft-deleted object."), ResultType: schema.NewNamedType("Boolean").Encode(), Arguments: map[string]schema.ArgumentInfo{ "bucket": { @@ -1725,20 +1521,17 @@ func GetConnectorSchema() *schema.SchemaResponse { "clientId": { Type: schema.NewNullableType(schema.NewNamedType("StorageClientID")).Encode(), }, - "cloudFunctionConfigurations": { - Type: schema.NewArrayType(schema.NewNamedType("NotificationLambdaConfig")).Encode(), - }, - "queueConfigurations": { - Type: schema.NewArrayType(schema.NewNamedType("NotificationQueueConfig")).Encode(), + "object": { + Type: schema.NewNamedType("String").Encode(), }, - "topicConfigurations": { - Type: schema.NewArrayType(schema.NewNamedType("NotificationTopicConfig")).Encode(), + "where": { + Type: schema.NewNullableType(schema.NewPredicateType("StorageObjectFilter")).Encode(), }, }, }, { - Name: "setStorageBucketReplication", - Description: toPtr("sets replication configuration on a bucket. Role can be obtained by first defining the replication target on MinIO to associate the source and destination buckets for replication with the replication endpoint."), + Name: "updateStorageBucket", + Description: toPtr("updates the bucket's configuration."), ResultType: schema.NewNamedType("Boolean").Encode(), Arguments: map[string]schema.ArgumentInfo{ "bucket": { @@ -1747,79 +1540,26 @@ func GetConnectorSchema() *schema.SchemaResponse { "clientId": { Type: schema.NewNullableType(schema.NewNamedType("StorageClientID")).Encode(), }, - "role": { - Type: schema.NewNullableType(schema.NewNamedType("String")).Encode(), - }, - "rules": { - Type: schema.NewArrayType(schema.NewNamedType("StorageReplicationRule")).Encode(), + "encryption": { + Type: schema.NewNullableType(schema.NewNamedType("ServerSideEncryptionConfiguration")).Encode(), }, - }, - }, - { - Name: "setStorageBucketTags", - Description: toPtr("sets tags to a bucket."), - ResultType: schema.NewNamedType("Boolean").Encode(), - Arguments: map[string]schema.ArgumentInfo{ - "bucket": { - Type: schema.NewNullableType(schema.NewNamedType("String")).Encode(), + "lifecycle": { + Type: schema.NewNullableType(schema.NewNamedType("ObjectLifecycleConfiguration")).Encode(), }, - "clientId": { - Type: schema.NewNullableType(schema.NewNamedType("StorageClientID")).Encode(), + "objectLock": { + Type: schema.NewNullableType(schema.NewNamedType("SetStorageObjectLockConfig")).Encode(), }, "tags": { - Type: schema.NewNamedType("JSON").Encode(), - }, - }, - }, - { - Name: "setStorageObjectLegalHold", - Description: toPtr("applies legal-hold onto an object."), - ResultType: schema.NewNamedType("Boolean").Encode(), - Arguments: map[string]schema.ArgumentInfo{ - "bucket": { - Type: schema.NewNullableType(schema.NewNamedType("String")).Encode(), - }, - "clientId": { - Type: schema.NewNullableType(schema.NewNamedType("StorageClientID")).Encode(), - }, - "object": { - Type: schema.NewNamedType("String").Encode(), + Type: schema.NewNullableType(schema.NewNamedType("JSON")).Encode(), }, - "status": { + "versioningEnabled": { Type: schema.NewNullableType(schema.NewNamedType("Boolean")).Encode(), }, - "versionId": { - Type: schema.NewNullableType(schema.NewNamedType("String")).Encode(), - }, - "where": { - Type: schema.NewNullableType(schema.NewPredicateType("StorageObjectSimple")).Encode(), - }, - }, - }, - { - Name: "setStorageObjectLockConfig", - ResultType: schema.NewNamedType("Boolean").Encode(), - Arguments: map[string]schema.ArgumentInfo{ - "bucket": { - Type: schema.NewNullableType(schema.NewNamedType("String")).Encode(), - }, - "clientId": { - Type: schema.NewNullableType(schema.NewNamedType("StorageClientID")).Encode(), - }, - "mode": { - Type: schema.NewNullableType(schema.NewNamedType("StorageRetentionMode")).Encode(), - }, - "unit": { - Type: schema.NewNullableType(schema.NewNamedType("StorageRetentionValidityUnit")).Encode(), - }, - "validity": { - Type: schema.NewNullableType(schema.NewNamedType("Int32")).Encode(), - }, }, }, { - Name: "setStorageObjectRetention", - Description: toPtr("applies object retention lock onto an object."), + Name: "updateStorageObject", + Description: toPtr("updates the object's configuration."), ResultType: schema.NewNamedType("Boolean").Encode(), Arguments: map[string]schema.ArgumentInfo{ "bucket": { @@ -1828,61 +1568,26 @@ func GetConnectorSchema() *schema.SchemaResponse { "clientId": { Type: schema.NewNullableType(schema.NewNamedType("StorageClientID")).Encode(), }, - "governanceBypass": { + "legalHold": { Type: schema.NewNullableType(schema.NewNamedType("Boolean")).Encode(), }, - "mode": { - Type: schema.NewNullableType(schema.NewNamedType("StorageRetentionMode")).Encode(), + "metadata": { + Type: schema.NewNullableType(schema.NewNamedType("JSON")).Encode(), }, "object": { Type: schema.NewNamedType("String").Encode(), }, - "retainUntilDate": { - Type: schema.NewNullableType(schema.NewNamedType("TimestampTZ")).Encode(), - }, - "versionId": { - Type: schema.NewNullableType(schema.NewNamedType("String")).Encode(), - }, - "where": { - Type: schema.NewNullableType(schema.NewPredicateType("StorageObjectSimple")).Encode(), - }, - }, - }, - { - Name: "setStorageObjectTags", - Description: toPtr("sets new object Tags to the given object, replaces/overwrites any existing tags."), - ResultType: schema.NewNamedType("Boolean").Encode(), - Arguments: map[string]schema.ArgumentInfo{ - "bucket": { - Type: schema.NewNullableType(schema.NewNamedType("String")).Encode(), - }, - "clientId": { - Type: schema.NewNullableType(schema.NewNamedType("StorageClientID")).Encode(), - }, - "object": { - Type: schema.NewNamedType("String").Encode(), + "retention": { + Type: schema.NewNullableType(schema.NewNamedType("SetStorageObjectRetentionOptions")).Encode(), }, "tags": { - Type: schema.NewNamedType("JSON").Encode(), + Type: schema.NewNullableType(schema.NewNamedType("JSON")).Encode(), }, "versionId": { Type: schema.NewNullableType(schema.NewNamedType("String")).Encode(), }, "where": { - Type: schema.NewNullableType(schema.NewPredicateType("StorageObjectSimple")).Encode(), - }, - }, - }, - { - Name: "suspendStorageBucketVersioning", - Description: toPtr("disables bucket versioning support."), - ResultType: schema.NewNamedType("Boolean").Encode(), - Arguments: map[string]schema.ArgumentInfo{ - "bucket": { - Type: schema.NewNullableType(schema.NewNamedType("String")).Encode(), - }, - "clientId": { - Type: schema.NewNullableType(schema.NewNamedType("StorageClientID")).Encode(), + Type: schema.NewNullableType(schema.NewPredicateType("StorageObjectFilter")).Encode(), }, }, }, @@ -1907,7 +1612,7 @@ func GetConnectorSchema() *schema.SchemaResponse { Type: schema.NewNullableType(schema.NewNamedType("PutStorageObjectOptions")).Encode(), }, "where": { - Type: schema.NewNullableType(schema.NewPredicateType("StorageObjectSimple")).Encode(), + Type: schema.NewNullableType(schema.NewPredicateType("StorageObjectFilter")).Encode(), }, }, }, @@ -1932,7 +1637,7 @@ func GetConnectorSchema() *schema.SchemaResponse { Type: schema.NewNullableType(schema.NewNamedType("PutStorageObjectOptions")).Encode(), }, "where": { - Type: schema.NewNullableType(schema.NewPredicateType("StorageObjectSimple")).Encode(), + Type: schema.NewNullableType(schema.NewPredicateType("StorageObjectFilter")).Encode(), }, }, }, @@ -1963,6 +1668,11 @@ func GetConnectorSchema() *schema.SchemaResponse { ComparisonOperators: map[string]schema.ComparisonOperatorDefinition{}, Representation: schema.NewTypeRepresentationJSON().Encode(), }, + "GoogleStorageRPO": schema.ScalarType{ + AggregateFunctions: schema.ScalarTypeAggregateFunctions{}, + ComparisonOperators: map[string]schema.ComparisonOperatorDefinition{}, + Representation: schema.NewTypeRepresentationEnum([]string{"DEFAULT", "ASYNC_TURBO"}).Encode(), + }, "Int32": schema.ScalarType{ AggregateFunctions: schema.ScalarTypeAggregateFunctions{}, ComparisonOperators: map[string]schema.ComparisonOperatorDefinition{}, @@ -1988,15 +1698,10 @@ func GetConnectorSchema() *schema.SchemaResponse { ComparisonOperators: map[string]schema.ComparisonOperatorDefinition{}, Representation: schema.NewTypeRepresentationEnum([]string{"COMPLETED", "PENDING", "FAILED", "REPLICA"}).Encode(), }, - "StorageReplicationRuleStatus": schema.ScalarType{ - AggregateFunctions: schema.ScalarTypeAggregateFunctions{}, - ComparisonOperators: map[string]schema.ComparisonOperatorDefinition{}, - Representation: schema.NewTypeRepresentationEnum([]string{"Enabled", "Disabled"}).Encode(), - }, "StorageRetentionMode": schema.ScalarType{ AggregateFunctions: schema.ScalarTypeAggregateFunctions{}, ComparisonOperators: map[string]schema.ComparisonOperatorDefinition{}, - Representation: schema.NewTypeRepresentationEnum([]string{"Locked", "Unlocked"}).Encode(), + Representation: schema.NewTypeRepresentationEnum([]string{"Locked", "Unlocked", "Mutable", "Delete"}).Encode(), }, "StorageRetentionValidityUnit": schema.ScalarType{ AggregateFunctions: schema.ScalarTypeAggregateFunctions{}, diff --git a/connector/storage/azblob/bucket.go b/connector/storage/azblob/bucket.go index e8c7547..5ff926b 100644 --- a/connector/storage/azblob/bucket.go +++ b/connector/storage/azblob/bucket.go @@ -13,6 +13,7 @@ import ( "github.com/hasura/ndc-storage/connector/storage/common" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/trace" ) // MakeBucket creates a new bucket. @@ -40,18 +41,35 @@ func (c *Client) MakeBucket(ctx context.Context, args *common.MakeStorageBucketO } // ListBuckets lists all buckets. -func (c *Client) ListBuckets(ctx context.Context, options common.BucketOptions) ([]common.StorageBucketInfo, error) { +func (c *Client) ListBuckets(ctx context.Context, options *common.ListStorageBucketsOptions, predicate func(string) bool) (*common.StorageBucketListResults, error) { //nolint:gocognit,cyclop,funlen ctx, span := c.startOtelSpan(ctx, "ListBuckets", "") defer span.End() - pager := c.client.NewListContainersPager(&azblob.ListContainersOptions{ + opts := &azblob.ListContainersOptions{ Include: azblob.ListContainersInclude{ - Metadata: options.IncludeTags, + Metadata: options.Include.Tags, Deleted: false, }, - }) + } + + if options.Prefix != "" { + opts.Prefix = &options.Prefix + } + + maxResults := int32(options.MaxResults) + if options.MaxResults > 0 && predicate == nil { + opts.MaxResults = &maxResults + } + + if options.StartAfter != "" { + opts.Marker = &options.StartAfter + } + + pager := c.client.NewListContainersPager(opts) - var results []common.StorageBucketInfo + var count int32 + var results []common.StorageBucket + pageInfo := common.StoragePaginationInfo{} for pager.More() { resp, err := pager.NextPage(ctx) @@ -63,7 +81,11 @@ func (c *Client) ListBuckets(ctx context.Context, options common.BucketOptions) } for _, container := range resp.ContainerItems { - result := common.StorageBucketInfo{} + if container.Name == nil || (predicate != nil && !predicate(*container.Name)) { + continue + } + + result := common.StorageBucket{} if container.Name != nil { result.Name = *container.Name @@ -79,17 +101,61 @@ func (c *Client) ListBuckets(ctx context.Context, options common.BucketOptions) } } - if container.Properties != nil && container.Properties.LastModified != nil { - result.CreationDate = *container.Properties.LastModified + if container.Properties != nil { + result.LastModified = container.Properties.LastModified + + if container.Properties.IsImmutableStorageWithVersioningEnabled != nil { + result.Versioning = &common.StorageBucketVersioningConfiguration{ + Enabled: *container.Properties.IsImmutableStorageWithVersioningEnabled, + } + } + + if container.Properties.DefaultEncryptionScope != nil { + result.Encryption = &common.ServerSideEncryptionConfiguration{ + KmsMasterKeyID: *container.Properties.DefaultEncryptionScope, + } + } + + if container.Properties.RemainingRetentionDays != nil && container.Properties.HasImmutabilityPolicy != nil { + mode := common.StorageRetentionModeLocked + unit := common.StorageRetentionValidityUnitDays + days := uint(*container.Properties.RemainingRetentionDays) + + result.ObjectLock = &common.StorageObjectLockConfig{ + Enabled: true, + SetStorageObjectLockConfig: common.SetStorageObjectLockConfig{ + Mode: &mode, + Unit: &unit, + Validity: &days, + }, + } + } } results = append(results, result) + count++ + + if maxResults > 0 && count >= maxResults { + if pager.More() { + pageInfo.HasNextPage = true + pageInfo.Cursor = resp.NextMarker + } + + if resp.Marker != nil && *resp.Marker != "" { + pageInfo.Cursor = resp.Marker + } + + break + } } } span.SetAttributes(attribute.Int("storage.bucket_count", len(results))) - return results, nil + return &common.StorageBucketListResults{ + Buckets: results, + PageInfo: pageInfo, + }, nil } // BucketExists checks if a bucket exists. @@ -111,7 +177,7 @@ func (c *Client) BucketExists(ctx context.Context, bucketName string) (bool, err } // GetBucket gets a bucket by name. -func (c *Client) GetBucket(ctx context.Context, bucketName string, options common.BucketOptions) (*common.StorageBucketInfo, error) { +func (c *Client) GetBucket(ctx context.Context, bucketName string, options common.BucketOptions) (*common.StorageBucket, error) { ctx, span := c.startOtelSpan(ctx, "GetBucket", bucketName) defer span.End() @@ -126,11 +192,11 @@ func (c *Client) GetBucket(ctx context.Context, bucketName string, options commo return result, nil } -func (c *Client) getBucket(ctx context.Context, bucketName string, options common.BucketOptions) (*common.StorageBucketInfo, error) { +func (c *Client) getBucket(ctx context.Context, bucketName string, options common.BucketOptions) (*common.StorageBucket, error) { pager := c.client.NewListContainersPager(&service.ListContainersOptions{ Prefix: &bucketName, Include: service.ListContainersInclude{ - Metadata: options.IncludeTags, + Metadata: options.Include.Tags, Deleted: false, System: false, }, @@ -147,7 +213,7 @@ func (c *Client) getBucket(ctx context.Context, bucketName string, options commo continue } - result := common.StorageBucketInfo{} + result := common.StorageBucket{} if container.Name != nil { result.Name = *container.Name @@ -164,7 +230,7 @@ func (c *Client) getBucket(ctx context.Context, bucketName string, options commo } if container.Properties != nil && container.Properties.LastModified != nil { - result.CreationDate = *container.Properties.LastModified + result.LastModified = container.Properties.LastModified } return &result, nil @@ -174,6 +240,20 @@ func (c *Client) getBucket(ctx context.Context, bucketName string, options commo return nil, nil } +// UpdateBucket updates configurations for the bucket. +func (c *Client) UpdateBucket(ctx context.Context, bucketName string, opts common.UpdateStorageBucketOptions) error { + ctx, span := c.startOtelSpanWithKind(ctx, trace.SpanKindInternal, "UpdateBucket", bucketName) + defer span.End() + + if opts.Tags != nil { + if err := c.SetBucketTagging(ctx, bucketName, opts.Tags); err != nil { + return err + } + } + + return nil +} + // RemoveBucket removes a bucket, bucket should be empty to be successfully removed. func (c *Client) RemoveBucket(ctx context.Context, bucketName string) error { ctx, span := c.startOtelSpan(ctx, "RemoveBucket", bucketName) @@ -212,10 +292,6 @@ func (c *Client) SetBucketTagging(ctx context.Context, bucketName string, bucket inputTags = map[string]*string{} for key, value := range bucketTags { - if value == "" { - continue - } - span.SetAttributes(attribute.String("storage.bucket_tag"+key, value)) inputTags[key] = &value } @@ -259,74 +335,3 @@ func (c *Client) GetBucketPolicy(ctx context.Context, bucketName string) (string return result, nil } - -// GetBucketNotification gets notification configuration on a bucket. -func (c *Client) GetBucketNotification(ctx context.Context, bucketName string) (*common.NotificationConfig, error) { - return nil, errNotSupported -} - -// SetBucketNotification sets a new bucket notification on a bucket. -func (c *Client) SetBucketNotification(ctx context.Context, bucketName string, config common.NotificationConfig) error { - return errNotSupported -} - -// RemoveAllBucketNotification removes all configured bucket notifications on a bucket. -func (c *Client) RemoveAllBucketNotification(ctx context.Context, bucketName string) error { - return errNotSupported -} - -// GetBucketVersioning gets the versioning configuration set on a bucket. -func (c *Client) GetBucketVersioning(ctx context.Context, bucketName string) (*common.StorageBucketVersioningConfiguration, error) { - return nil, errNotSupported -} - -// SetBucketReplication sets replication configuration on a bucket. Role can be obtained by first defining the replication target -// to associate the source and destination buckets for replication with the replication endpoint. -func (c *Client) SetBucketReplication(ctx context.Context, bucketName string, cfg common.StorageReplicationConfig) error { - return errNotSupported -} - -// Get current replication config on a bucket. -func (c *Client) GetBucketReplication(ctx context.Context, bucketName string) (*common.StorageReplicationConfig, error) { - return nil, errNotSupported -} - -// RemoveBucketReplication removes replication configuration on a bucket. -func (c *Client) RemoveBucketReplication(ctx context.Context, bucketName string) error { - return errNotSupported -} - -// EnableVersioning enables bucket versioning support. -func (c *Client) EnableVersioning(ctx context.Context, bucketName string) error { - return errNotSupported -} - -// SuspendVersioning disables bucket versioning support. -func (c *Client) SuspendVersioning(ctx context.Context, bucketName string) error { - return errNotSupported -} - -// SetBucketLifecycle sets lifecycle on bucket or an object prefix. -func (c *Client) SetBucketLifecycle(ctx context.Context, bucketName string, config common.BucketLifecycleConfiguration) error { - return errNotSupported -} - -// GetBucketLifecycle gets lifecycle on a bucket or a prefix. -func (c *Client) GetBucketLifecycle(ctx context.Context, bucketName string) (*common.BucketLifecycleConfiguration, error) { - return nil, errNotSupported -} - -// SetBucketEncryption sets default encryption configuration on a bucket. -func (c *Client) SetBucketEncryption(ctx context.Context, bucketName string, input common.ServerSideEncryptionConfiguration) error { - return errNotSupported -} - -// GetBucketEncryption gets default encryption configuration set on a bucket. -func (c *Client) GetBucketEncryption(ctx context.Context, bucketName string) (*common.ServerSideEncryptionConfiguration, error) { - return nil, errNotSupported -} - -// RemoveBucketEncryption remove default encryption configuration set on a bucket. -func (c *Client) RemoveBucketEncryption(ctx context.Context, bucketName string) error { - return errNotSupported -} diff --git a/connector/storage/azblob/client.go b/connector/storage/azblob/client.go index 5336056..6beef5f 100644 --- a/connector/storage/azblob/client.go +++ b/connector/storage/azblob/client.go @@ -5,14 +5,16 @@ import ( "log/slog" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" + "github.com/hasura/ndc-sdk-go/utils" "github.com/hasura/ndc-storage/connector/storage/common" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" ) -// Client prepresents a Minio client wrapper. +// Client represents a Minio client wrapper. type Client struct { - client *azblob.Client + client *azblob.Client + isDebug bool } var _ common.StorageClient = &Client{} @@ -25,13 +27,21 @@ func New(ctx context.Context, cfg *ClientConfig, logger *slog.Logger) (*Client, } return &Client{ - client: client, + client: client, + isDebug: utils.IsDebug(logger), }, nil } func (c *Client) startOtelSpan(ctx context.Context, name string, bucketName string) (context.Context, trace.Span) { spanKind := trace.SpanKindClient + if c.isDebug { + spanKind = trace.SpanKindInternal + } + + return c.startOtelSpanWithKind(ctx, spanKind, name, bucketName) +} +func (c *Client) startOtelSpanWithKind(ctx context.Context, spanKind trace.SpanKind, name string, bucketName string) (context.Context, trace.Span) { ctx, span := tracer.Start(ctx, name, trace.WithSpanKind(spanKind)) span.SetAttributes( common.NewDBSystemAttribute(), diff --git a/connector/storage/azblob/config.go b/connector/storage/azblob/config.go index e5c4595..e90fbc2 100644 --- a/connector/storage/azblob/config.go +++ b/connector/storage/azblob/config.go @@ -121,7 +121,7 @@ func (at AuthType) Validate() error { return err } -// AuthCredentials represent the authentication credentials infomartion. +// AuthCredentials represent the authentication credentials information. type AuthCredentials struct { // The authentication type Type AuthType `json:"type" mapstructure:"type" yaml:"type"` @@ -160,6 +160,8 @@ type AuthCredentials struct { // Enable multitenant authentication. The credential may request tokens from in addition to the tenant specified by AZURE_TENANT_ID. // Set this value to "*" to enable the credential to request a token from any tenant. AdditionallyAllowedTenants []string `json:"additionallyAllowedTenants,omitempty" mapstructure:"additionallyAllowedTenants" yaml:"additionallyAllowedTenants,omitempty"` + // Audience to use when requesting tokens for Azure Active Directory authentication. + Audience *utils.EnvString `json:"audience,omitempty" mapstructure:"audience" yaml:"audience,omitempty"` } // JSONSchema is used to generate a custom jsonschema. @@ -245,6 +247,10 @@ func (ac AuthCredentials) JSONSchema() *jsonschema.Schema { //nolint:funlen Description: "the path of a file containing a Kubernetes service account token", Ref: envStringRefName, }) + entraProps.Set("audience", &jsonschema.Schema{ + Description: "Audience to use when requesting tokens for Azure Active Directory authentication", + Type: envStringRefName, + }) entraProps.Set("disableInstanceDiscovery", &jsonschema.Schema{ Type: "boolean", @@ -326,6 +332,15 @@ func (ac AuthCredentials) toAzureBlobClient(endpoint string, options *azblob.Cli return nil, err } + if ac.Audience != nil { + audience, err := ac.Audience.GetOrDefault("") + if err != nil { + return nil, fmt.Errorf("audience: %w", err) + } + + options.Audience = audience + } + return azblob.NewClient(serviceURL, cred, options) case AuthTypeConnectionString: if ac.ConnectionString == nil { diff --git a/connector/storage/azblob/object.go b/connector/storage/azblob/object.go index 8799393..9fc05aa 100644 --- a/connector/storage/azblob/object.go +++ b/connector/storage/azblob/object.go @@ -20,6 +20,7 @@ import ( "github.com/hasura/ndc-storage/connector/storage/common" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/trace" ) // ListObjects list objects in a bucket. @@ -34,11 +35,11 @@ func (c *Client) ListObjects(ctx context.Context, bucketName string, opts *commo Tags: opts.Include.Tags, Copy: opts.Include.Copy, Snapshots: opts.Include.Snapshots, - Deleted: opts.Include.Deleted, LegalHold: opts.Include.LegalHold, - ImmutabilityPolicy: opts.Include.ImmutabilityPolicy, - DeletedWithVersions: opts.Include.DeletedWithVersions, + ImmutabilityPolicy: opts.Include.Retention, Permissions: opts.Include.Permissions, + Deleted: false, + DeletedWithVersions: false, UncommittedBlobs: false, }, } @@ -59,7 +60,7 @@ func (c *Client) ListObjects(ctx context.Context, bucketName string, opts *commo var count int32 objects := make([]common.StorageObject, 0) pager := c.client.NewListBlobsFlatPager(bucketName, options) - pageInfo := common.StorageObjectPaginationInfo{} + pageInfo := common.StoragePaginationInfo{} for pager.More() { resp, err := pager.NextPage(ctx) @@ -82,11 +83,7 @@ func (c *Client) ListObjects(ctx context.Context, bucketName string, opts *commo if maxResults > 0 && count >= maxResults { if pager.More() { pageInfo.HasNextPage = true - pageInfo.NextCursor = resp.NextMarker - } - - if resp.Marker != nil && *resp.Marker != "" { - pageInfo.Cursor = resp.Marker + pageInfo.Cursor = resp.NextMarker } break @@ -156,6 +153,80 @@ func (c *Client) ListIncompleteUploads(ctx context.Context, bucketName string, a return objects, nil } +// ListDeletedObjects list soft-deleted objects in a bucket. +func (c *Client) ListDeletedObjects(ctx context.Context, bucketName string, opts *common.ListStorageObjectsOptions, predicate func(string) bool) (*common.StorageObjectListResults, error) { + ctx, span := c.startOtelSpan(ctx, "ListDeletedObjects", bucketName) + defer span.End() + + options := &container.ListBlobsFlatOptions{ + Include: container.ListBlobsInclude{ + Versions: opts.Include.Versions, + Metadata: opts.Include.Metadata, + Tags: opts.Include.Tags, + Copy: opts.Include.Copy, + Snapshots: opts.Include.Snapshots, + LegalHold: opts.Include.LegalHold, + ImmutabilityPolicy: opts.Include.Retention, + Permissions: opts.Include.Permissions, + Deleted: true, + DeletedWithVersions: true, + UncommittedBlobs: false, + }, + } + + if opts.Prefix != "" { + options.Prefix = &opts.Prefix + } + + maxResults := int32(opts.MaxResults) + + if opts.StartAfter != "" { + options.Marker = &opts.StartAfter + } + + var count int32 + objects := make([]common.StorageObject, 0) + pager := c.client.NewListBlobsFlatPager(bucketName, options) + pageInfo := common.StoragePaginationInfo{} + + for pager.More() { + resp, err := pager.NextPage(ctx) + if err != nil { + span.SetStatus(codes.Error, err.Error()) + span.RecordError(err) + + return nil, serializeErrorResponse(err) + } + + for _, item := range resp.Segment.BlobItems { + if item.Name == nil || item.Deleted == nil || !*item.Deleted || (predicate != nil && !predicate(*item.Name)) { + continue + } + + objects = append(objects, serializeObjectInfo(item)) + count++ + } + + if maxResults > 0 && count >= maxResults { + if pager.More() { + pageInfo.HasNextPage = true + pageInfo.Cursor = resp.NextMarker + } + + break + } + } + + span.SetAttributes(attribute.Int("storage.object_count", int(count))) + + results := &common.StorageObjectListResults{ + Objects: objects, + PageInfo: pageInfo, + } + + return results, nil +} + // RemoveIncompleteUpload removes a partially uploaded object. func (c *Client) RemoveIncompleteUpload(ctx context.Context, bucketName string, objectName string) error { return c.removeObject(ctx, "RemoveIncompleteUpload", bucketName, objectName, common.RemoveStorageObjectOptions{}) @@ -192,7 +263,7 @@ func (c *Client) PutObject(ctx context.Context, bucketName string, objectName st uploadOptions := &azblob.UploadStreamOptions{ HTTPHeaders: &blob.HTTPHeaders{}, - Tags: opts.UserTags, + Tags: opts.Tags, Metadata: map[string]*string{}, Concurrency: int(opts.NumThreads), BlockSize: int64(opts.PartSize), @@ -207,7 +278,7 @@ func (c *Client) PutObject(ctx context.Context, bucketName string, objectName st uploadOptions.AccessTier = &accessTier } - for key, value := range opts.UserMetadata { + for key, value := range opts.Metadata { if value != "" { uploadOptions.Metadata[key] = &value } @@ -252,6 +323,19 @@ func (c *Client) PutObject(ctx context.Context, bucketName string, objectName st return nil, serializeErrorResponse(err) } + if opts.Retention != nil && opts.Retention.Mode == common.StorageRetentionModeLocked { + err := c.SetObjectRetention(ctx, bucketName, objectName, "", common.SetStorageObjectRetentionOptions{ + Mode: &opts.Retention.Mode, + RetainUntilDate: &opts.Retention.RetainUntilDate, + }) + if err != nil { + span.SetStatus(codes.Error, err.Error()) + span.RecordError(err) + + return nil, err + } + } + result := serializeUploadObjectInfo(resp) result.Name = objectName @@ -276,12 +360,12 @@ func (c *Client) CopyObject(ctx context.Context, dest common.StorageCopyDestOpti blobClient := c.client.ServiceClient().NewContainerClient(dest.Bucket).NewBlobClient(dest.Object) options := &blob.CopyFromURLOptions{ - BlobTags: dest.UserTags, + BlobTags: dest.Tags, Metadata: make(map[string]*string), LegalHold: dest.LegalHold, } - for key, value := range dest.UserMetadata { + for key, value := range dest.Metadata { if value != "" { options.Metadata[key] = &value } @@ -456,8 +540,60 @@ func (c *Client) RemoveObjects(ctx context.Context, bucketName string, opts *com return errs } +// UpdateObject updates object configurations. +func (c *Client) UpdateObject(ctx context.Context, bucketName string, objectName string, opts common.UpdateStorageObjectOptions) error { + ctx, span := c.startOtelSpanWithKind(ctx, trace.SpanKindInternal, "UpdateObject", bucketName) + defer span.End() + + span.SetAttributes(attribute.String("storage.key", objectName)) + + if opts.VersionID != "" { + span.SetAttributes(attribute.String("storage.options.version", opts.VersionID)) + } + + if opts.LegalHold != nil { + if err := c.SetObjectLegalHold(ctx, bucketName, objectName, opts.VersionID, *opts.LegalHold); err != nil { + return err + } + } + + if opts.Tags != nil { + if err := c.SetObjectTags(ctx, bucketName, objectName, opts.VersionID, opts.Tags); err != nil { + return err + } + } + + if opts.Retention != nil { + if err := c.SetObjectRetention(ctx, bucketName, objectName, opts.VersionID, *opts.Retention); err != nil { + return err + } + } + + return nil +} + +// RestoreObject restores an object with some specified options. +func (c *Client) RestoreObject(ctx context.Context, bucketName string, objectName string) error { + ctx, span := c.startOtelSpan(ctx, "RestoreObject", bucketName) + defer span.End() + + span.SetAttributes(attribute.String("storage.key", objectName)) + + blobClient := c.client.ServiceClient().NewContainerClient(bucketName).NewBlobClient(objectName) + + _, err := blobClient.Undelete(ctx, nil) + if err != nil { + span.SetStatus(codes.Error, err.Error()) + span.RecordError(err) + + return serializeErrorResponse(err) + } + + return nil +} + // SetObjectRetention applies object retention lock onto an object. -func (c *Client) SetObjectRetention(ctx context.Context, bucketName string, objectName string, opts common.SetStorageObjectRetentionOptions) error { +func (c *Client) SetObjectRetention(ctx context.Context, bucketName string, objectName, versionID string, opts common.SetStorageObjectRetentionOptions) error { ctx, span := c.startOtelSpan(ctx, "SetObjectRetention", bucketName) defer span.End() @@ -465,8 +601,8 @@ func (c *Client) SetObjectRetention(ctx context.Context, bucketName string, obje attribute.String("storage.key", objectName), ) - if opts.VersionID != "" { - span.SetAttributes(attribute.String("storage.options.version", opts.VersionID)) + if versionID != "" { + span.SetAttributes(attribute.String("storage.options.version", versionID)) } client := c.client.ServiceClient().NewContainerClient(bucketName).NewBlobClient(objectName) @@ -498,11 +634,10 @@ func (c *Client) SetObjectRetention(ctx context.Context, bucketName string, obje } // SetObjectLegalHold applies legal-hold onto an object. -func (c *Client) SetObjectLegalHold(ctx context.Context, bucketName string, objectName string, options common.SetStorageObjectLegalHoldOptions) error { +func (c *Client) SetObjectLegalHold(ctx context.Context, bucketName string, objectName, versionID string, status bool) error { ctx, span := c.startOtelSpan(ctx, "SetObjectTags", bucketName) defer span.End() - status := options.Status != nil && *options.Status span.SetAttributes( attribute.String("storage.key", objectName), attribute.Bool("storage.options.status", status), @@ -522,22 +657,22 @@ func (c *Client) SetObjectLegalHold(ctx context.Context, bucketName string, obje } // PutObjectTagging sets new object Tags to the given object, replaces/overwrites any existing tags. -func (c *Client) SetObjectTags(ctx context.Context, bucketName string, objectName string, options common.SetStorageObjectTagsOptions) error { +func (c *Client) SetObjectTags(ctx context.Context, bucketName string, objectName, versionID string, tags map[string]string) error { ctx, span := c.startOtelSpan(ctx, "SetObjectTags", bucketName) defer span.End() opts := &blob.SetTagsOptions{ - VersionID: &options.VersionID, + VersionID: &versionID, } - if options.VersionID != "" { - span.SetAttributes(attribute.String("storage.options.version", options.VersionID)) - opts.VersionID = &options.VersionID + if versionID != "" { + span.SetAttributes(attribute.String("storage.options.version", versionID)) + opts.VersionID = &versionID } client := c.client.ServiceClient().NewContainerClient(bucketName).NewBlobClient(objectName) - _, err := client.SetTags(ctx, options.Tags, opts) + _, err := client.SetTags(ctx, tags, opts) if err != nil { span.SetStatus(codes.Error, err.Error()) span.RecordError(err) @@ -593,13 +728,3 @@ func (c *Client) presignedObject(ctx context.Context, method, bucketName, object return result, nil } - -// Set object lock configuration in given bucket. mode, validity and unit are either all set or all nil. -func (c *Client) SetObjectLockConfig(ctx context.Context, bucketname string, opts common.SetStorageObjectLockConfig) error { - return errNotSupported -} - -// Get object lock configuration of given bucket. -func (c *Client) GetObjectLockConfig(ctx context.Context, bucketName string) (*common.StorageObjectLockConfig, error) { - return nil, errNotSupported -} diff --git a/connector/storage/azblob/utils.go b/connector/storage/azblob/utils.go index 39e99ea..6e920a9 100644 --- a/connector/storage/azblob/utils.go +++ b/connector/storage/azblob/utils.go @@ -17,13 +17,12 @@ var tracer = connector.NewTracer("connector/storage/azblob") var errNotSupported = schema.NotSupportedError("Azure Blob Storage doesn't support this method", nil) -func serializeObjectInfo(item *container.BlobItem) common.StorageObject { //nolint:funlen +func serializeObjectInfo(item *container.BlobItem) common.StorageObject { //nolint:funlen,cyclop object := common.StorageObject{ - Metadata: make(map[string]string), - UserMetadata: make(map[string]string), - IsLatest: item.IsCurrentVersion, - Deleted: item.Deleted, - VersionID: item.VersionID, + Metadata: make(map[string]string), + IsLatest: item.IsCurrentVersion, + Deleted: item.Deleted, + VersionID: item.VersionID, } if item.Name != nil { @@ -31,14 +30,14 @@ func serializeObjectInfo(item *container.BlobItem) common.StorageObject { //noli } if item.BlobTags != nil && len(item.BlobTags.BlobTagSet) > 0 { - object.UserTags = make(map[string]string) + object.Tags = make(map[string]string) for _, bt := range item.BlobTags.BlobTagSet { if bt.Key == nil || bt.Value == nil { continue } - object.UserTags[*bt.Key] = *bt.Value + object.Tags[*bt.Key] = *bt.Value } } @@ -79,7 +78,9 @@ func serializeObjectInfo(item *container.BlobItem) common.StorageObject { //noli } if item.Properties.TagCount != nil { - object.UserTagCount = int(*item.Properties.TagCount) + object.TagCount = int(*item.Properties.TagCount) + } else { + object.TagCount = len(object.Tags) } if item.Properties.ExpiresOn != nil { @@ -103,21 +104,27 @@ func serializeObjectInfo(item *container.BlobItem) common.StorageObject { //noli object.ArchiveStatus = (*string)(item.Properties.ArchiveStatus) object.BlobSequenceNumber = item.Properties.BlobSequenceNumber object.BlobType = (*string)(item.Properties.BlobType) - object.CopyCompletionTime = item.Properties.CopyCompletionTime - object.CopyID = item.Properties.CopyID - object.CopyProgress = item.Properties.CopyProgress - object.CopySource = item.Properties.CopySource - object.CopyStatus = (*string)(item.Properties.CopyStatus) - object.CopyStatusDescription = item.Properties.CopyStatusDescription + + if item.Properties.CopyID != nil { + object.Copy = &common.StorageObjectCopyInfo{ + CompletionTime: item.Properties.CopyCompletionTime, + ID: *item.Properties.CopyID, + Progress: item.Properties.CopyProgress, + Source: item.Properties.CopySource, + Status: (*string)(item.Properties.CopyStatus), + StatusDescription: item.Properties.CopyStatusDescription, + } + } + object.CreationTime = item.Properties.CreationTime object.DeletedTime = item.Properties.DeletedTime object.CustomerProvidedKeySHA256 = item.Properties.CustomerProvidedKeySHA256 object.DestinationSnapshot = item.Properties.DestinationSnapshot object.ServerEncrypted = item.Properties.ServerEncrypted - object.EncryptionScope = item.Properties.EncryptionScope + object.KMSKeyName = item.Properties.EncryptionScope object.Group = item.Properties.Group - object.ImmutabilityPolicyUntilDate = item.Properties.ImmutabilityPolicyExpiresOn - object.ImmutabilityPolicyMode = (*string)(item.Properties.ImmutabilityPolicyMode) + object.RetentionUntilDate = item.Properties.ImmutabilityPolicyExpiresOn + object.RetentionMode = (*string)(item.Properties.ImmutabilityPolicyMode) object.IncrementalCopy = item.Properties.IncrementalCopy object.IsSealed = item.Properties.IsSealed object.LastAccessTime = item.Properties.LastAccessedOn diff --git a/connector/storage/bucket.go b/connector/storage/bucket.go index 618ae75..714b9be 100644 --- a/connector/storage/bucket.go +++ b/connector/storage/bucket.go @@ -7,46 +7,6 @@ import ( "github.com/hasura/ndc-storage/connector/storage/common" ) -// GetBucketPolicy gets access permissions on a bucket or a prefix. -func (m *Manager) GetBucketPolicy(ctx context.Context, args *common.StorageBucketArguments) (string, error) { - client, bucketName, err := m.GetClientAndBucket(args.ClientID, args.Bucket) - if err != nil { - return "", err - } - - return client.GetBucketPolicy(ctx, bucketName) -} - -// GetBucketNotification gets notification configuration on a bucket. -func (m *Manager) GetBucketNotification(ctx context.Context, args *common.StorageBucketArguments) (*common.NotificationConfig, error) { - client, bucketName, err := m.GetClientAndBucket(args.ClientID, args.Bucket) - if err != nil { - return nil, err - } - - return client.GetBucketNotification(ctx, bucketName) -} - -// SetBucketNotification sets a new bucket notification on a bucket. -func (m *Manager) SetBucketNotification(ctx context.Context, args *common.SetBucketNotificationArguments) error { - client, bucketName, err := m.GetClientAndBucket(args.ClientID, args.Bucket) - if err != nil { - return err - } - - return client.SetBucketNotification(ctx, bucketName, args.NotificationConfig) -} - -// Remove all configured bucket notifications on a bucket. -func (m *Manager) RemoveAllBucketNotification(ctx context.Context, args *common.StorageBucketArguments) error { - client, bucketName, err := m.GetClientAndBucket(args.ClientID, args.Bucket) - if err != nil { - return err - } - - return client.RemoveAllBucketNotification(ctx, bucketName) -} - // MakeBucket creates a new bucket. func (m *Manager) MakeBucket(ctx context.Context, clientID *common.StorageClientID, args *common.MakeStorageBucketOptions) error { client, bucketName, err := m.GetClientAndBucket(clientID, args.Name) @@ -59,173 +19,56 @@ func (m *Manager) MakeBucket(ctx context.Context, clientID *common.StorageClient return client.MakeBucket(ctx, args) } -// ListBuckets list all buckets. -func (m *Manager) ListBuckets(ctx context.Context, clientID *common.StorageClientID, options common.BucketOptions) ([]common.StorageBucketInfo, error) { - client, ok := m.GetClient(clientID) - if !ok { - return nil, schema.InternalServerError("client not found", nil) - } - - return client.ListBuckets(ctx, options) -} - -// BucketExists checks if a bucket exists. -func (m *Manager) BucketExists(ctx context.Context, args *common.StorageBucketArguments) (bool, error) { - client, bucketName, err := m.GetClientAndBucket(args.ClientID, args.Bucket) - if err != nil { - return false, err - } - - return client.BucketExists(ctx, bucketName) -} - -// RemoveBucket removes a bucket, bucket should be empty to be successfully removed. -func (m *Manager) RemoveBucket(ctx context.Context, args *common.StorageBucketArguments) error { - client, bucketName, err := m.GetClientAndBucket(args.ClientID, args.Bucket) - if err != nil { - return err - } - - return client.RemoveBucket(ctx, bucketName) -} - -// SetBucketTagging sets tags to a bucket. -func (m *Manager) SetBucketTagging(ctx context.Context, args *common.SetStorageBucketTaggingArguments) error { - client, bucketName, err := m.GetClientAndBucket(args.ClientID, args.Bucket) - if err != nil { - return err - } - - return client.SetBucketTagging(ctx, bucketName, args.Tags) -} - -// SetBucketLifecycle sets lifecycle on bucket or an object prefix. -func (m *Manager) SetBucketLifecycle(ctx context.Context, args *common.SetStorageBucketLifecycleArguments) error { - client, bucketName, err := m.GetClientAndBucket(args.ClientID, args.Bucket) - if err != nil { - return err - } - - return client.SetBucketLifecycle(ctx, bucketName, args.BucketLifecycleConfiguration) -} - -// GetBucketLifecycle gets lifecycle on a bucket or a prefix. -func (m *Manager) GetBucketLifecycle(ctx context.Context, args *common.StorageBucketArguments) (*common.BucketLifecycleConfiguration, error) { - client, bucketName, err := m.GetClientAndBucket(args.ClientID, args.Bucket) - if err != nil { - return nil, err - } - - return client.GetBucketLifecycle(ctx, bucketName) -} - -// SetBucketEncryption sets default encryption configuration on a bucket. -func (m *Manager) SetBucketEncryption(ctx context.Context, args *common.SetStorageBucketEncryptionArguments) error { - client, bucketName, err := m.GetClientAndBucket(args.ClientID, args.Bucket) - if err != nil { - return err - } - - return client.SetBucketEncryption(ctx, bucketName, args.ServerSideEncryptionConfiguration) -} - -// GetBucketEncryption gets default encryption configuration set on a bucket. -func (m *Manager) GetBucketEncryption(ctx context.Context, args *common.StorageBucketArguments) (*common.ServerSideEncryptionConfiguration, error) { - client, bucketName, err := m.GetClientAndBucket(args.ClientID, args.Bucket) - if err != nil { - return nil, err - } - - return client.GetBucketEncryption(ctx, bucketName) -} - -// RemoveBucketEncryption removes default encryption configuration set on a bucket. -func (m *Manager) RemoveBucketEncryption(ctx context.Context, args *common.StorageBucketArguments) error { - client, bucketName, err := m.GetClientAndBucket(args.ClientID, args.Bucket) - if err != nil { - return err - } - - return client.RemoveBucketEncryption(ctx, bucketName) -} - -// SetObjectLockConfig sets object lock configuration in given bucket. mode, validity and unit are either all set or all nil. -func (m *Manager) SetObjectLockConfig(ctx context.Context, args *common.SetStorageObjectLockArguments) error { - client, bucketName, err := m.GetClientAndBucket(args.ClientID, args.Bucket) - if err != nil { - return err - } - - return client.SetObjectLockConfig(ctx, bucketName, args.SetStorageObjectLockConfig) -} - -// GetObjectLockConfig gets object lock configuration of given bucket. -func (m *Manager) GetObjectLockConfig(ctx context.Context, args *common.StorageBucketArguments) (*common.StorageObjectLockConfig, error) { - client, bucketName, err := m.GetClientAndBucket(args.ClientID, args.Bucket) - if err != nil { - return nil, err +// UpdateBucket updates configurations for the bucket. +func (m *Manager) UpdateBucket(ctx context.Context, args *common.UpdateBucketArguments) error { + if args.UpdateStorageBucketOptions.IsEmpty() { + return nil } - return client.GetObjectLockConfig(ctx, bucketName) -} - -// EnableVersioning enables bucket versioning support. -func (m *Manager) EnableVersioning(ctx context.Context, args *common.StorageBucketArguments) error { client, bucketName, err := m.GetClientAndBucket(args.ClientID, args.Bucket) if err != nil { return err } - return client.EnableVersioning(ctx, bucketName) + return client.UpdateBucket(ctx, bucketName, args.UpdateStorageBucketOptions) } -// SuspendVersioning disables bucket versioning support. -func (m *Manager) SuspendVersioning(ctx context.Context, args *common.StorageBucketArguments) error { - client, bucketName, err := m.GetClientAndBucket(args.ClientID, args.Bucket) - if err != nil { - return err +// ListBuckets list all buckets. +func (m *Manager) ListBuckets(ctx context.Context, clientID *common.StorageClientID, options *common.ListStorageBucketsOptions, predicate func(string) bool) (*common.StorageBucketListResults, error) { + client, ok := m.GetClient(clientID) + if !ok { + return nil, schema.InternalServerError("client not found", nil) } - return client.SuspendVersioning(ctx, bucketName) + return client.ListBuckets(ctx, options, predicate) } -// GetBucketVersioning gets versioning configuration set on a bucket. -func (m *Manager) GetBucketVersioning(ctx context.Context, args *common.StorageBucketArguments) (*common.StorageBucketVersioningConfiguration, error) { - client, bucketName, err := m.GetClientAndBucket(args.ClientID, args.Bucket) +// GetBucket gets bucket by name. +func (m *Manager) GetBucket(ctx context.Context, bucketInfo *common.StorageBucketArguments, options common.BucketOptions) (*common.StorageBucket, error) { + client, bucketName, err := m.GetClientAndBucket(bucketInfo.ClientID, bucketInfo.Bucket) if err != nil { return nil, err } - return client.GetBucketVersioning(ctx, bucketName) -} - -// SetBucketReplication sets replication configuration on a bucket. Role can be obtained by first defining the replication target on MinIO -// to associate the source and destination buckets for replication with the replication endpoint. -func (m *Manager) SetBucketReplication(ctx context.Context, args *common.SetStorageBucketReplicationArguments) error { - client, bucketName, err := m.GetClientAndBucket(args.ClientID, args.Bucket) - if err != nil { - return err - } - - return client.SetBucketReplication(ctx, bucketName, args.StorageReplicationConfig) + return client.GetBucket(ctx, bucketName, options) } -// GetBucketReplication gets current replication config on a bucket. -func (m *Manager) GetBucketReplication(ctx context.Context, args *common.StorageBucketArguments) (*common.StorageReplicationConfig, error) { +// BucketExists checks if a bucket exists. +func (m *Manager) BucketExists(ctx context.Context, args *common.StorageBucketArguments) (bool, error) { client, bucketName, err := m.GetClientAndBucket(args.ClientID, args.Bucket) if err != nil { - return nil, err + return false, err } - return client.GetBucketReplication(ctx, bucketName) + return client.BucketExists(ctx, bucketName) } -// RemoveBucketReplication removes replication configuration on a bucket. -func (m *Manager) RemoveBucketReplication(ctx context.Context, args *common.StorageBucketArguments) error { +// RemoveBucket removes a bucket, bucket should be empty to be successfully removed. +func (m *Manager) RemoveBucket(ctx context.Context, args *common.StorageBucketArguments) error { client, bucketName, err := m.GetClientAndBucket(args.ClientID, args.Bucket) if err != nil { return err } - return client.RemoveBucketReplication(ctx, bucketName) + return client.RemoveBucket(ctx, bucketName) } diff --git a/connector/storage/common/arguments.go b/connector/storage/common/arguments.go index 677fba7..feb63e6 100644 --- a/connector/storage/common/arguments.go +++ b/connector/storage/common/arguments.go @@ -9,8 +9,18 @@ import ( // ListStorageBucketArguments represent the input arguments for the ListBuckets methods. type ListStorageBucketArguments struct { - // The storage client ID. - ClientID *StorageClientID `json:"clientId,omitempty"` + // The maximum number of objects requested per batch. + MaxResults int `json:"maxResults,omitempty"` + // StartAfter start listing lexically at this object onwards. + StartAfter string `json:"startAfter,omitempty"` + Where schema.Expression `json:"where" ndc:"predicate=StorageBucketFilter"` +} + +// StorageBucketArguments represent the common input arguments for bucket-related methods. +type GetStorageBucketArguments struct { + StorageBucketArguments + + Where schema.Expression `json:"where" ndc:"predicate=StorageBucketFilter"` } // StorageBucketArguments represent the common input arguments for bucket-related methods. @@ -51,18 +61,11 @@ type MakeStorageBucketOptions struct { // Bucket location Region string `json:"region,omitempty"` // Enable object locking - ObjectLocking bool `json:"objectLocking,omitempty"` + ObjectLock bool `json:"objectLock,omitempty"` // Optional tags Tags map[string]string `json:"tags,omitempty"` } -// SetBucketTaggingArguments represent the input arguments for the SetBucketTagging method. -type SetStorageBucketTaggingArguments struct { - StorageBucketArguments - - Tags map[string]string `json:"tags"` -} - // ListIncompleteUploadsArguments the input arguments of the ListIncompleteUploads method. type ListIncompleteUploadsArguments struct { StorageBucketArguments @@ -88,7 +91,7 @@ type PresignedGetStorageObjectArguments struct { PresignedGetStorageObjectOptions Object string `json:"object"` - Where schema.Expression `json:"where" ndc:"predicate=StorageObjectSimple"` + Where schema.Expression `json:"where" ndc:"predicate=StorageObjectFilter"` } // PresignedGetStorageObjectOptions represent the options for the PresignedGetObject method. @@ -103,7 +106,7 @@ type PresignedPutStorageObjectArguments struct { Object string `json:"object"` Expiry *scalar.Duration `json:"expiry"` - Where schema.Expression `json:"where" ndc:"predicate=StorageObjectSimple"` + Where schema.Expression `json:"where" ndc:"predicate=StorageObjectFilter"` } // ListStorageObjectsArguments holds all arguments of a list object request. @@ -117,7 +120,7 @@ type ListStorageObjectsArguments struct { // StartAfter start listing lexically at this object onwards. StartAfter *string `json:"startAfter,omitempty"` - Where schema.Expression `json:"where" ndc:"predicate=StorageObjectSimple"` + Where schema.Expression `json:"where" ndc:"predicate=StorageObjectFilter"` } // StorageObjectIncludeOptions hold options to be included for the object information. @@ -133,13 +136,14 @@ type StorageObjectIncludeOptions struct { // Include objects metadata in the listing Metadata bool - Copy bool - Snapshots bool - Deleted bool - LegalHold bool - ImmutabilityPolicy bool - DeletedWithVersions bool - Permissions bool + Copy bool + Snapshots bool + LegalHold bool + Retention bool + Permissions bool + Lifecycle bool + Encryption bool + ObjectLock bool } // ListStorageObjectsOptions holds all options of a list object request. @@ -165,7 +169,7 @@ type GetStorageObjectArguments struct { GetStorageObjectOptions Object string `json:"object"` - Where schema.Expression `json:"where" ndc:"predicate=StorageObjectSimple"` + Where schema.Expression `json:"where" ndc:"predicate=StorageObjectFilter"` } // GetStorageObjectOptions are used to specify additional headers or options during GET requests. @@ -198,19 +202,10 @@ type StorageCopyDestOptions struct { // if no user-metadata is provided, it is copied from source // (when there is only once source object in the compose // request) - UserMetadata map[string]string `json:"userMetadata,omitempty"` - // UserMetadata is only set to destination if ReplaceMetadata is true - // other value is UserMetadata is ignored and we preserve src.UserMetadata - // NOTE: if you set this value to true and now metadata is present - // in UserMetadata your destination object will not have any metadata - // set. - ReplaceMetadata bool `json:"replaceMetadata,omitempty"` - - // `userTags` is the user defined object tags to be set on destination. - // This will be set only if the `replaceTags` field is set to true. - // Otherwise this field is ignored - UserTags map[string]string `json:"userTags,omitempty"` - ReplaceTags bool `json:"replaceTags,omitempty"` + Metadata map[string]string `json:"metadata,omitempty"` + + // `tags` is the user defined object tags to be set on destination. + Tags map[string]string `json:"tags,omitempty"` // Specifies whether you want to apply a Legal Hold to the copied object. LegalHold *bool `json:"legalHold"` @@ -247,31 +242,49 @@ type RemoveStorageObjectArguments struct { RemoveStorageObjectOptions Object string `json:"object"` - Where schema.Expression `json:"where" ndc:"predicate=StorageObjectSimple"` + Where schema.Expression `json:"where" ndc:"predicate=StorageObjectFilter"` } // RemoveStorageObjectOptions represents options specified by user for RemoveObject call. type RemoveStorageObjectOptions struct { + SoftDelete bool `json:"softDelete,omitempty"` ForceDelete bool `json:"forceDelete,omitempty"` GovernanceBypass bool `json:"governanceBypass,omitempty"` VersionID string `json:"versionId,omitempty"` } -// SetStorageObjectRetentionArguments represents options specified by user for PutObject call. -type SetStorageObjectRetentionArguments struct { +// UpdateStorageObjectArguments represents options specified by user for updating object. +type UpdateStorageObjectArguments struct { StorageBucketArguments - SetStorageObjectRetentionOptions + UpdateStorageObjectOptions Object string `json:"object"` - Where schema.Expression `json:"where" ndc:"predicate=StorageObjectSimple"` + Where schema.Expression `json:"where" ndc:"predicate=StorageObjectFilter"` +} + +// UpdateStorageObjectOptions represents options specified by user for updating object. +type UpdateStorageObjectOptions struct { + VersionID string `json:"versionId,omitempty"` + Retention *SetStorageObjectRetentionOptions `json:"retention"` + LegalHold *bool `json:"legalHold"` + Metadata map[string]string `json:"metadata,omitempty"` + Tags map[string]string `json:"tags,omitempty"` +} + +// IsEmpty checks if all elements in the option object is null. +func (ubo UpdateStorageObjectOptions) IsEmpty() bool { + return ubo.VersionID == "" && + ubo.Tags == nil && + ubo.Retention == nil && + ubo.LegalHold == nil && + ubo.Metadata == nil } // SetStorageObjectRetentionOptions represents options specified by user for PutObject call. type SetStorageObjectRetentionOptions struct { - GovernanceBypass bool `json:"governanceBypass,omitempty"` Mode *StorageRetentionMode `json:"mode"` + GovernanceBypass bool `json:"governanceBypass,omitempty"` RetainUntilDate *time.Time `json:"retainUntilDate,omitempty"` - VersionID string `json:"versionId,omitempty"` } // RemoveStorageObjectsArguments represents arguments specified by user for RemoveObjects call. @@ -279,7 +292,7 @@ type RemoveStorageObjectsArguments struct { StorageBucketArguments RemoveStorageObjectsOptions - Where schema.Expression `json:"where" ndc:"predicate=StorageObjectSimple"` + Where schema.Expression `json:"where" ndc:"predicate=StorageObjectFilter"` } // RemoveStorageObjectsOptions represents options specified by user for RemoveObjects call. @@ -289,33 +302,24 @@ type RemoveStorageObjectsOptions struct { GovernanceBypass bool `json:"governanceBypass,omitempty"` } -// SetStorageObjectLegalHoldArguments represents options specified by user for PutObjectLegalHold call. -type SetStorageObjectLegalHoldArguments struct { - StorageBucketArguments - SetStorageObjectLegalHoldOptions - - Object string `json:"object"` - Where schema.Expression `json:"where" ndc:"predicate=StorageObjectSimple"` -} - -// SetStorageObjectLegalHoldOptions represents options specified by user for PutObjectLegalHold call. -type SetStorageObjectLegalHoldOptions struct { - VersionID string `json:"versionId,omitempty"` - Status *bool `json:"status"` +// PutStorageObjectRetentionOptions represent options of object retention configuration. +type PutStorageObjectRetentionOptions struct { + Mode StorageRetentionMode `json:"mode"` + RetainUntilDate time.Time `json:"retainUntilDate"` + GovernanceBypass bool `json:"governanceBypass,omitempty"` } // PutStorageObjectOptions represents options specified by user for PutObject call. type PutStorageObjectOptions struct { - UserMetadata map[string]string `json:"userMetadata,omitempty"` - UserTags map[string]string `json:"userTags,omitempty"` - ContentType string `json:"contentType,omitempty"` - ContentEncoding string `json:"contentEncoding,omitempty"` - ContentDisposition string `json:"contentDisposition,omitempty"` - ContentLanguage string `json:"contentLanguage,omitempty"` - CacheControl string `json:"cacheControl,omitempty"` - Expires *time.Time `json:"expires,omitempty"` - Mode *StorageRetentionMode `json:"mode,omitempty"` - RetainUntilDate *time.Time `json:"retainUntilDate,omitempty"` + Metadata map[string]string `json:"metadata,omitempty"` + Tags map[string]string `json:"tags,omitempty"` + ContentType string `json:"contentType,omitempty"` + ContentEncoding string `json:"contentEncoding,omitempty"` + ContentDisposition string `json:"contentDisposition,omitempty"` + ContentLanguage string `json:"contentLanguage,omitempty"` + CacheControl string `json:"cacheControl,omitempty"` + Expires *time.Time `json:"expires,omitempty"` + Retention *PutStorageObjectRetentionOptions `json:"retention,omitempty"` // ServerSideEncryption *ServerSideEncryptionMethod `json:"serverSideEncryption,omitempty"` NumThreads uint `json:"numThreads,omitempty"` StorageClass string `json:"storageClass,omitempty"` @@ -344,53 +348,40 @@ type PutStorageObjectOptions struct { ConcurrentStreamParts bool `json:"concurrentStreamParts,omitempty"` } -// SetStorageObjectTagsArguments holds an object version id to update tag(s) of a specific object version. -type SetStorageObjectTagsArguments struct { - StorageBucketArguments - SetStorageObjectTagsOptions - - Object string `json:"object"` - Where schema.Expression `json:"where" ndc:"predicate=StorageObjectSimple"` -} - -// SetStorageObjectTagsOptions holds an object version id to update tag(s) of a specific object version. -type SetStorageObjectTagsOptions struct { - Tags map[string]string `json:"tags"` - VersionID string `json:"versionId,omitempty"` -} - -// SetBucketNotificationArguments represents input arguments for the SetBucketNotification method. -type SetBucketNotificationArguments struct { - StorageBucketArguments - NotificationConfig +// PresignedURLResponse holds the presigned URL and expiry information. +type PresignedURLResponse struct { + URL string `json:"url"` + ExpiredAt string `json:"expiredAt"` } -// SetStorageBucketLifecycleArguments represents input arguments for the SetBucketLifecycle method. -type SetStorageBucketLifecycleArguments struct { +// UpdateBucketArguments hold update options for the bucket. +type UpdateBucketArguments struct { StorageBucketArguments - BucketLifecycleConfiguration + UpdateStorageBucketOptions } -// SetStorageBucketEncryptionArguments represents input arguments for the SetBucketEncryption method. -type SetStorageBucketEncryptionArguments struct { - StorageBucketArguments - ServerSideEncryptionConfiguration +// UpdateStorageBucketOptions hold update options for the bucket. +type UpdateStorageBucketOptions struct { + Tags map[string]string `json:"tags,omitempty"` + VersioningEnabled *bool `json:"versioningEnabled"` + Lifecycle *ObjectLifecycleConfiguration `json:"lifecycle"` + Encryption *ServerSideEncryptionConfiguration `json:"encryption"` + ObjectLock *SetStorageObjectLockConfig `json:"objectLock"` } -// SetStorageObjectLockArguments represents input arguments for the SetStorageObjectLock method. -type SetStorageObjectLockArguments struct { - StorageBucketArguments - SetStorageObjectLockConfig +// IsEmpty checks if all elements in the option object is null. +func (ubo UpdateStorageBucketOptions) IsEmpty() bool { + return ubo.VersioningEnabled == nil && + ubo.Tags == nil && + ubo.Lifecycle == nil && + ubo.Encryption == nil && + ubo.ObjectLock == nil } -// SetStorageBucketReplicationArguments storage bucket replication arguments. -type SetStorageBucketReplicationArguments struct { +// RestoreStorageObjectArguments represent arguments specified by user for RestoreObject call. +type RestoreStorageObjectArguments struct { StorageBucketArguments - StorageReplicationConfig -} -// PresignedURLResponse holds the presigned URL and expiry information. -type PresignedURLResponse struct { - URL string `json:"url"` - ExpiredAt string `json:"expiredAt"` + Object string `json:"object"` + Where schema.Expression `json:"where" ndc:"predicate=StorageObjectFilter"` } diff --git a/connector/storage/common/config.go b/connector/storage/common/config.go index a09eeeb..d0df994 100644 --- a/connector/storage/common/config.go +++ b/connector/storage/common/config.go @@ -114,10 +114,6 @@ func (bcc BaseClientConfig) GetJSONSchema(providerTypes []any) *jsonschema.Schem Description: "Endpoint of the storage server. Required for other S3 compatible services such as MinIO, Cloudflare R2, DigitalOcean Spaces, etc...", Ref: envStringRef, }) - properties.Set("publicHost", &jsonschema.Schema{ - Description: "The public host to be used for presigned URL generation", - Ref: envStringRef, - }) properties.Set("maxRetries", &jsonschema.Schema{ Description: "Maximum number of retry times", Type: "integer", diff --git a/connector/storage/common/storage.go b/connector/storage/common/storage.go index dbbe20d..f57357a 100644 --- a/connector/storage/common/storage.go +++ b/connector/storage/common/storage.go @@ -13,21 +13,21 @@ type StorageClient interface { //nolint:interfacebloat // MakeBucket creates a new bucket. MakeBucket(ctx context.Context, options *MakeStorageBucketOptions) error // ListBuckets list all buckets. - ListBuckets(ctx context.Context, options BucketOptions) ([]StorageBucketInfo, error) + ListBuckets(ctx context.Context, options *ListStorageBucketsOptions, predicate func(string) bool) (*StorageBucketListResults, error) // GetBucket gets a bucket by name. - GetBucket(ctx context.Context, name string, options BucketOptions) (*StorageBucketInfo, error) + GetBucket(ctx context.Context, name string, options BucketOptions) (*StorageBucket, error) // BucketExists checks if a bucket exists. BucketExists(ctx context.Context, bucketName string) (bool, error) // RemoveBucket removes a bucket, bucket should be empty to be successfully removed. RemoveBucket(ctx context.Context, bucketName string) error - // SetBucketTagging sets tags to a bucket. - SetBucketTagging(ctx context.Context, bucketName string, bucketTags map[string]string) error - // GetBucketPolicy gets access permissions on a bucket or a prefix. - GetBucketPolicy(ctx context.Context, bucketName string) (string, error) - // ListObjects lists objects in a bucket. + // UpdateBucket updates configurations for the bucket. + UpdateBucket(ctx context.Context, bucketName string, opts UpdateStorageBucketOptions) error + // ListObjects list objects in a bucket. ListObjects(ctx context.Context, bucketName string, opts *ListStorageObjectsOptions, predicate func(string) bool) (*StorageObjectListResults, error) // ListIncompleteUploads list partially uploaded objects in a bucket. ListIncompleteUploads(ctx context.Context, bucketName string, args ListIncompleteUploadsOptions) ([]StorageObjectMultipartInfo, error) + // ListDeletedObjects list deleted objects in a bucket. + ListDeletedObjects(ctx context.Context, bucketName string, opts *ListStorageObjectsOptions, predicate func(string) bool) (*StorageObjectListResults, error) // GetObject returns a stream of the object data. Most of the common errors occur when reading the stream. GetObject(ctx context.Context, bucketName string, objectName string, opts GetStorageObjectOptions) (io.ReadCloser, error) // PutObject uploads objects that are less than 128MiB in a single PUT operation. For objects that are greater than 128MiB in size, @@ -43,15 +43,13 @@ type StorageClient interface { //nolint:interfacebloat StatObject(ctx context.Context, bucketName string, objectName string, opts GetStorageObjectOptions) (*StorageObject, error) // RemoveObject removes an object with some specified options RemoveObject(ctx context.Context, bucketName string, objectName string, opts RemoveStorageObjectOptions) error - // SetObjectRetention applies object retention lock onto an object. - SetObjectRetention(ctx context.Context, bucketName string, objectName string, opts SetStorageObjectRetentionOptions) error // RemoveObjects remove a list of objects obtained from an input channel. The call sends a delete request to the server up to 1000 objects at a time. // The errors observed are sent over the error channel. RemoveObjects(ctx context.Context, bucketName string, opts *RemoveStorageObjectsOptions, predicate func(string) bool) []RemoveStorageObjectError - // PutObjectLegalHold applies legal-hold onto an object. - SetObjectLegalHold(ctx context.Context, bucketName string, objectName string, opts SetStorageObjectLegalHoldOptions) error - // SetObjectTags sets new object Tags to the given object, replaces/overwrites any existing tags. - SetObjectTags(ctx context.Context, bucketName string, objectName string, options SetStorageObjectTagsOptions) error + // UpdateObject updates object configurations. + UpdateObject(ctx context.Context, bucketName string, objectName string, opts UpdateStorageObjectOptions) error + // RestoreObject restores a soft-deleted object. + RestoreObject(ctx context.Context, bucketName string, objectName string) error // RemoveIncompleteUpload removes a partially uploaded object. RemoveIncompleteUpload(ctx context.Context, bucketName string, objectName string) error // PresignedGetObject generates a presigned URL for HTTP GET operations. Browsers/Mobile clients may point to this URL to directly download objects even if the bucket is private. @@ -63,55 +61,239 @@ type StorageClient interface { //nolint:interfacebloat // This presigned URL can have an associated expiration time in seconds after which it is no longer operational. // The default expiry is set to 7 days. PresignedPutObject(ctx context.Context, bucketName string, objectName string, expiry time.Duration) (string, error) - // GetBucketNotification gets notification configuration on a bucket. - GetBucketNotification(ctx context.Context, bucketName string) (*NotificationConfig, error) - // Set a new bucket notification on a bucket. - SetBucketNotification(ctx context.Context, bucketName string, config NotificationConfig) error - // Remove all configured bucket notifications on a bucket. - RemoveAllBucketNotification(ctx context.Context, bucketName string) error - // SetBucketLifecycle sets lifecycle on bucket or an object prefix. - SetBucketLifecycle(ctx context.Context, bucketname string, config BucketLifecycleConfiguration) error - // GetBucketLifecycle gets lifecycle on a bucket or a prefix. - GetBucketLifecycle(ctx context.Context, bucketName string) (*BucketLifecycleConfiguration, error) - // SetBucketEncryption sets default encryption configuration on a bucket. - SetBucketEncryption(ctx context.Context, bucketname string, config ServerSideEncryptionConfiguration) error - // GetBucketEncryption gets default encryption configuration set on a bucket. - GetBucketEncryption(ctx context.Context, bucketName string) (*ServerSideEncryptionConfiguration, error) - // RemoveBucketEncryption removes default encryption configuration set on a bucket. - RemoveBucketEncryption(ctx context.Context, bucketName string) error - // SetObjectLockConfig sets object lock configuration in given bucket. mode, validity and unit are either all set or all nil. - SetObjectLockConfig(ctx context.Context, bucketName string, config SetStorageObjectLockConfig) error - // GetObjectLockConfig gets object lock configuration of given bucket. - GetObjectLockConfig(ctx context.Context, bucketName string) (*StorageObjectLockConfig, error) - // EnableVersioning enables bucket versioning support. - EnableVersioning(ctx context.Context, bucketName string) error - // SuspendVersioning disables bucket versioning support. - SuspendVersioning(ctx context.Context, bucketName string) error - // GetBucketVersioning gets versioning configuration set on a bucket. - GetBucketVersioning(ctx context.Context, bucketName string) (*StorageBucketVersioningConfiguration, error) - // SetBucketReplication sets replication configuration on a bucket. Role can be obtained by first defining the replication target on MinIO - // to associate the source and destination buckets for replication with the replication endpoint. - SetBucketReplication(ctx context.Context, bucketname string, cfg StorageReplicationConfig) error - // GetBucketReplication gets current replication config on a bucket. - GetBucketReplication(ctx context.Context, bucketName string) (*StorageReplicationConfig, error) - // RemoveBucketReplication removes replication configuration on a bucket. - RemoveBucketReplication(ctx context.Context, bucketName string) error +} + +// ListStorageBucketsOptions holds all options of a list bucket request. +type ListStorageBucketsOptions struct { + // Only list objects with the prefix + Prefix string `json:"prefix"` + // The maximum number of objects requested per + // batch, advanced use-case not useful for most + // applications + MaxResults int `json:"maxResults"` + // StartAfter start listing lexically at this object onwards. + StartAfter string `json:"startAfter"` + // Options to be included for the object information. + Include BucketIncludeOptions `json:"-"` + NumThreads int `json:"-"` +} + +// BucketIncludeOptions contain include options for getting bucket information. +type BucketIncludeOptions struct { + Tags bool + Versioning bool + Lifecycle bool + Encryption bool + ObjectLock bool +} + +// IsEmpty checks if all include options are false +func (bio BucketIncludeOptions) IsEmpty() bool { + return !bio.Tags && !bio.Versioning && !bio.Lifecycle && !bio.Encryption && !bio.ObjectLock } // BucketOptions hold options to get bucket information. type BucketOptions struct { - IncludeTags bool `json:"includeTags,omitempty"` - NumThreads int `json:"-"` + Prefix string `json:"prefix"` + NumThreads int + Include BucketIncludeOptions } -// StorageBucketInfo container for bucket metadata. -type StorageBucketInfo struct { +// StorageBucketListResults hold the paginated results of the storage bucket list. +type StorageBucketListResults struct { + Buckets []StorageBucket `json:"buckets"` + PageInfo StoragePaginationInfo `json:"pageInfo"` +} + +// StorageBucket the container for bucket metadata. +type StorageBucket struct { // The name of the bucket. Name string `json:"name"` - // Date the bucket was created. - CreationDate time.Time `json:"creationDate"` // Bucket tags or metadata. Tags map[string]string `json:"tags,omitempty"` + // The versioning configuration + Versioning *StorageBucketVersioningConfiguration `json:"versioning"` + // The versioning configuration + Lifecycle *ObjectLifecycleConfiguration `json:"lifecycle"` + // The server-side encryption configuration. + Encryption *ServerSideEncryptionConfiguration `json:"encryption"` + + // Retention policy enforces a minimum retention time for all objects + // contained in the bucket. A RetentionPolicy of nil implies the bucket + // has no minimum data retention. + ObjectLock *StorageObjectLockConfig `json:"objectLock"` + + // The location of the bucket. + Region *string `json:"region"` + + // The bucket's custom placement configuration that holds a list of + // regional locations for custom dual regions. + CustomPlacementConfig *CustomPlacementConfig `json:"customPlacementConfig"` + + // DefaultEventBasedHold is the default value for event-based hold on newly created objects in this bucket. It defaults to false. + DefaultEventBasedHold *bool `json:"defaultEventBasedHold"` + + // StorageClass is the default storage class of the bucket. This defines + // how objects in the bucket are stored and determines the SLA and the cost of storage. + StorageClass *string `json:"storageClass"` + + // Date time the bucket was created. + CreationTime *time.Time `json:"creationTime"` + // Date time the bucket was created. + LastModified *time.Time `json:"lastModified"` + + // RequesterPays reports whether the bucket is a Requester Pays bucket. + // Clients performing operations on Requester Pays buckets must provide + // a user project (see BucketHandle.UserProject), which will be billed + // for the operations. + RequesterPays *bool `json:"requesterPays"` + + // The bucket's Cross-Origin Resource Sharing (CORS) configuration. + CORS []BucketCors `json:"cors,omitempty"` + + // The logging configuration. + Logging *BucketLogging `json:"logging"` + + // The website configuration. + Website *BucketWebsite `json:"website,omitempty"` + + // Etag is the HTTP/1.1 Entity tag for the bucket. + // This field is read-only. + Etag *string `json:"etag"` + + // LocationType describes how data is stored and replicated. + // Typical values are "multi-region", "region" and "dual-region". + LocationType *string `json:"locationType"` + + // RPO configures the Recovery Point Objective (RPO) policy of the bucket. + // Set to RPOAsyncTurbo to turn on Turbo Replication for a bucket. + // See https://cloud.google.com/storage/docs/managing-turbo-replication for + // more information. + RPO *GoogleStorageRPO `json:"rpo"` + + // Autoclass holds the bucket's autoclass configuration. If enabled, + // allows for the automatic selection of the best storage class + // based on object access patterns. + Autoclass *BucketAutoclass `json:"autoclass"` + + // SoftDeletePolicy contains the bucket's soft delete policy, which defines + // the period of time that soft-deleted objects will be retained, and cannot + // be permanently deleted. + SoftDeletePolicy *StorageObjectSoftDeletePolicy `json:"softDeletePolicy"` + + // HierarchicalNamespace contains the bucket's hierarchical namespace + // configuration. Hierarchical namespace enabled buckets can contain + // [cloud.google.com/go/storage/control/apiv2/controlpb.Folder] resources. + // It cannot be modified after bucket creation time. + // UniformBucketLevelAccess must also be enabled on the bucket. + HierarchicalNamespace *BucketHierarchicalNamespace `json:"hierarchicalNamespace"` +} + +// HierarchicalNamespace contains the bucket's hierarchical namespace +// configuration. Hierarchical namespace enabled buckets can contain +// [cloud.google.com/go/storage/control/apiv2/controlpb.Folder] resources. +type BucketHierarchicalNamespace struct { + // Enabled indicates whether hierarchical namespace features are enabled on + // the bucket. This can only be set at bucket creation time currently. + Enabled bool `json:"enabled"` +} + +// BucketLogging holds the bucket's logging configuration, which defines the +// destination bucket and optional name prefix for the current bucket's logs. +type BucketLogging struct { + // The destination bucket where the current bucket's logs + // should be placed. + LogBucket string `json:"logBucket"` + + // A prefix for log object names. + LogObjectPrefix string `json:"logObjectPrefix"` +} + +// GoogleStorageRPO (Recovery Point Objective) configures the turbo replication feature. See +// https://cloud.google.com/storage/docs/managing-turbo-replication for more information. +// @enum DEFAULT,ASYNC_TURBO +type GoogleStorageRPO string + +// BucketCors is the bucket's Cross-Origin Resource Sharing (CORS) configuration. +type BucketCors struct { + // MaxAge is the value to return in the Access-Control-Max-Age + // header used in preflight responses. + MaxAge scalar.Duration `json:"maxAge"` + + // Methods is the list of HTTP methods on which to include CORS response + // headers, (GET, OPTIONS, POST, etc) Note: "*" is permitted in the list + // of methods, and means "any method". + Methods []string `json:"methods"` + + // Origins is the list of Origins eligible to receive CORS response + // headers. Note: "*" is permitted in the list of origins, and means + // "any Origin". + Origins []string `json:"origins"` + + // ResponseHeaders is the list of HTTP headers other than the simple + // response headers to give permission for the user-agent to share + // across domains. + ResponseHeaders []string `json:"responseHeaders"` +} + +// BucketWebsite holds the bucket's website configuration, controlling how the +// service behaves when accessing bucket contents as a web site. See +// https://cloud.google.com/storage/docs/static-website for more information. +type BucketWebsite struct { + // If the requested object path is missing, the service will ensure the path has + // a trailing '/', append this suffix, and attempt to retrieve the resulting + // object. This allows the creation of index.html objects to represent directory + // pages. + MainPageSuffix string `json:"mainPageSuffix"` + + // If the requested object path is missing, and any mainPageSuffix object is + // missing, if applicable, the service will return the named object from this + // bucket as the content for a 404 Not Found result. + NotFoundPage *string `json:"notFoundPage,omitempty"` +} + +// CustomPlacementConfig holds the bucket's custom placement +// configuration for Custom Dual Regions. See +// https://cloud.google.com/storage/docs/locations#location-dr for more information. +type CustomPlacementConfig struct { + // The list of regional locations in which data is placed. + // Custom Dual Regions require exactly 2 regional locations. + DataLocations []string +} + +// Autoclass holds the bucket's autoclass configuration. If enabled, +// allows for the automatic selection of the best storage class +// based on object access patterns. See +// https://cloud.google.com/storage/docs/using-autoclass for more information. +type BucketAutoclass struct { + // Enabled specifies whether the autoclass feature is enabled + // on the bucket. + Enabled bool `json:"enabled"` + // ToggleTime is the time from which Autoclass was last toggled. + // If Autoclass is enabled when the bucket is created, the ToggleTime + // is set to the bucket creation time. This field is read-only. + ToggleTime time.Time `json:"toggleTime"` + // TerminalStorageClass: The storage class that objects in the bucket + // eventually transition to if they are not read for a certain length of + // time. Valid values are NEARLINE and ARCHIVE. + // To modify TerminalStorageClass, Enabled must be set to true. + TerminalStorageClass string `json:"terminalStorageClass"` + // TerminalStorageClassUpdateTime represents the time of the most recent + // update to "TerminalStorageClass". + TerminalStorageClassUpdateTime time.Time `json:"terminalStorageClassUpdateTime"` +} + +// StorageObjectSoftDeletePolicy contains the bucket's soft delete policy, which defines the +// period of time that soft-deleted objects will be retained, and cannot be +// permanently deleted. +type StorageObjectSoftDeletePolicy struct { + // EffectiveTime indicates the time from which the policy, or one with a + // greater retention, was effective. This field is read-only. + EffectiveTime time.Time `json:"effectiveTime"` + + // RetentionDuration is the amount of time that soft-deleted objects in the + // bucket will be retained and cannot be permanently deleted. + RetentionDuration scalar.Duration `json:"retentionDuration"` } // StorageOwner name. @@ -161,19 +343,17 @@ type StorageObject struct { Expires *time.Time `json:"expires"` // The date and time at which the object is no longer able to be cached. // Collection of additional metadata on the object. - // eg: x-amz-meta-*, content-encoding etc. + // In MinIO and S3, x-amz-meta-* headers stripped "x-amz-meta-" prefix containing the first value. Metadata map[string]string `json:"metadata,omitempty"` - // x-amz-meta-* headers stripped "x-amz-meta-" prefix containing the first value. - // Only returned by MinIO servers. - UserMetadata map[string]string `json:"userMetadata,omitempty"` + // Raw metadata headers, eg: x-amz-meta-*, content-encoding etc... Only returned by MinIO servers. + RawMetadata map[string]string `json:"rawMetadata,omitempty"` - // x-amz-tagging values in their k/v values. - // Only returned by MinIO servers. - UserTags map[string]string `json:"userTags,omitempty"` + // User tags + Tags map[string]string `json:"tags,omitempty"` - // x-amz-tagging-count value - UserTagCount int `json:"userTagCount,omitempty"` + // The total count value of tags + TagCount int `json:"tagCount,omitempty"` // Owner name. Owner *StorageOwner `json:"owner"` @@ -204,57 +384,61 @@ type StorageObject struct { StorageObjectChecksum // Azure Blob Store properties - ACL *string `json:"acl"` - AccessTierChangeTime *time.Time `json:"accessTierChangeTime"` - AccessTierInferred *bool `json:"accessTierInferred"` - ArchiveStatus *string `json:"archiveStatus"` - BlobSequenceNumber *int64 `json:"blobSequenceNumber"` - BlobType *string `json:"blobType"` - ContentMD5 *string `json:"contentMd5"` - CopyCompletionTime *time.Time `json:"copyCompletionTime"` - CopyID *string `json:"copyId"` - CopyProgress *string `json:"copyProgress"` - CopySource *string `json:"copySource"` - CopyStatus *string `json:"copyStatus"` - CopyStatusDescription *string `json:"copyStatusDescription"` - CreationTime *time.Time `json:"creationTime"` - DeletedTime *time.Time `json:"deletedTime"` - CustomerProvidedKeySHA256 *string `json:"customerProvidedKeySha256"` - DestinationSnapshot *string `json:"destinationSnapshot"` - + ACL any `json:"acl,omitempty"` + AccessTierChangeTime *time.Time `json:"accessTierChangeTime"` + AccessTierInferred *bool `json:"accessTierInferred"` + ArchiveStatus *string `json:"archiveStatus"` + BlobSequenceNumber *int64 `json:"blobSequenceNumber"` + BlobType *string `json:"blobType"` + ContentMD5 *string `json:"contentMd5"` + Copy *StorageObjectCopyInfo `json:"copy"` + CreationTime *time.Time `json:"creationTime"` + DeletedTime *time.Time `json:"deletedTime"` + CustomerProvidedKeySHA256 *string `json:"customerProvidedKeySha256"` + DestinationSnapshot *string `json:"destinationSnapshot"` + MediaLink *string `json:"mediaLink"` // The name of the encryption scope under which the blob is encrypted. - EncryptionScope *string `json:"encryptionScope"` - Group *string `json:"group"` - ImmutabilityPolicyUntilDate *time.Time `json:"immutabilityPolicyUntilDate"` - ImmutabilityPolicyMode *string `json:"immutabilityPolicyMode"` - IncrementalCopy *bool `json:"incrementalCopy"` - IsSealed *bool `json:"sealed"` - LastAccessTime *time.Time `json:"lastAccessTime"` - LeaseDuration *string `json:"leaseDuration"` - LeaseState *string `json:"leaseState"` - LeaseStatus *string `json:"leaseStatus"` - LegalHold *bool `json:"legalHold"` - Permissions *string `json:"permissions"` + KMSKeyName *string `json:"kmsKeyName"` + ServerEncrypted *bool `json:"serverEncrypted"` + Group *string `json:"group"` + RetentionUntilDate *time.Time `json:"retentionUntilDate"` + RetentionMode *string `json:"retentionMode"` + IncrementalCopy *bool `json:"incrementalCopy"` + IsSealed *bool `json:"sealed"` + LastAccessTime *time.Time `json:"lastAccessTime"` + LeaseDuration *string `json:"leaseDuration"` + LeaseState *string `json:"leaseState"` + LeaseStatus *string `json:"leaseStatus"` + LegalHold *bool `json:"legalHold"` + Permissions *string `json:"permissions"` // If an object is in rehydrate pending state then this header is returned with priority of rehydrate. Valid values are High // and Standard. RehydratePriority *string `json:"rehydratePriority"` RemainingRetentionDays *int32 `json:"remainingRetentionDays"` ResourceType *string `json:"resourceType"` - ServerEncrypted *bool `json:"serverEncrypted"` } -// StorageObjectPaginationInfo holds the pagination information. -type StorageObjectPaginationInfo struct { +// StorageObjectCopyInfo holds the copy information if the object was copied from another object. +type StorageObjectCopyInfo struct { + ID string `json:"id"` + CompletionTime *time.Time `json:"completionTime"` + Progress *string `json:"progress"` + Source *string `json:"source"` + Status *string `json:"status"` + StatusDescription *string `json:"statusDescription"` +} + +// StoragePaginationInfo holds the pagination information. +type StoragePaginationInfo struct { HasNextPage bool `json:"hasNextPage"` Cursor *string `json:"cursor"` - NextCursor *string `json:"nextCursor"` } // StorageObjectListResults hold the paginated results of the storage object list. type StorageObjectListResults struct { - Objects []StorageObject `json:"objects"` - PageInfo StorageObjectPaginationInfo `json:"pageInfo"` + Objects []StorageObject `json:"objects"` + PageInfo StoragePaginationInfo `json:"pageInfo"` } // StorageObjectReplicationStatus represents the x-amz-replication-status value enum. @@ -318,7 +502,7 @@ type StorageObjectMultipartInfo struct { // type ServerSideEncryptionMethod string // StorageRetentionMode the object retention mode. -// @enum Locked,Unlocked +// @enum Locked,Unlocked,Mutable,Delete type StorageRetentionMode string // RemoveStorageObjectError the container of Multi Delete S3 API error. @@ -359,7 +543,7 @@ type NotificationLambdaConfig struct { Lambda string `json:"cloudFunction"` } -// NotificationConfig the struct that represents a notification configration object. +// NotificationConfig the struct that represents a notification configuration object. type NotificationConfig struct { LambdaConfigs []NotificationLambdaConfig `json:"cloudFunctionConfigurations"` TopicConfigs []NotificationTopicConfig `json:"topicConfigurations"` @@ -384,33 +568,33 @@ type NotificationS3Key struct { FilterRules []NotificationFilterRule `json:"filterRule,omitempty"` } -// BucketLifecycleRule represents a single rule in lifecycle configuration -type BucketLifecycleRule struct { - AbortIncompleteMultipartUpload *AbortIncompleteMultipartUpload `json:"abortIncompleteMultipartUpload"` - Expiration *LifecycleExpiration `json:"expiration,omitempty"` - DelMarkerExpiration *LifecycleDelMarkerExpiration `json:"delMarkerExpiration,omitempty"` - AllVersionsExpiration *LifecycleAllVersionsExpiration `json:"allVersionsExpiration,omitempty"` - ID string `json:"id"` - RuleFilter *LifecycleFilter `json:"filter,omitempty"` - NoncurrentVersionExpiration *LifecycleNoncurrentVersionExpiration `json:"noncurrentVersionExpiration,omitempty"` - NoncurrentVersionTransition *LifecycleNoncurrentVersionTransition `json:"noncurrentVersionTransition,omitempty"` - Prefix *string `json:"prefix,omitempty"` - Status *string `json:"status"` - Transition *LifecycleTransition `json:"transition,omitempty"` +// ObjectLifecycleRule represents a single rule in lifecycle configuration +type ObjectLifecycleRule struct { + ID string `json:"id,omitempty"` + Enabled bool `json:"enabled,omitempty"` + AbortIncompleteMultipartUpload *ObjectAbortIncompleteMultipartUpload `json:"abortIncompleteMultipartUpload"` + Expiration *ObjectLifecycleExpiration `json:"expiration,omitempty"` + DelMarkerExpiration *ObjectLifecycleDelMarkerExpiration `json:"delMarkerExpiration,omitempty"` + AllVersionsExpiration *ObjectLifecycleAllVersionsExpiration `json:"allVersionsExpiration,omitempty"` + RuleFilter []ObjectLifecycleFilter `json:"filter,omitempty"` + NoncurrentVersionExpiration *ObjectLifecycleNoncurrentVersionExpiration `json:"noncurrentVersionExpiration,omitempty"` + NoncurrentVersionTransition *ObjectLifecycleNoncurrentVersionTransition `json:"noncurrentVersionTransition,omitempty"` + Prefix *string `json:"prefix,omitempty"` + Transition *ObjectLifecycleTransition `json:"transition,omitempty"` } -// BucketLifecycleConfiguration is a collection of lifecycle Rule objects. -type BucketLifecycleConfiguration struct { - Rules []BucketLifecycleRule `json:"rules"` +// ObjectLifecycleConfiguration is a collection of lifecycle Rule objects. +type ObjectLifecycleConfiguration struct { + Rules []ObjectLifecycleRule `json:"rules"` } // AbortIncompleteMultipartUpload structure, not supported yet on MinIO -type AbortIncompleteMultipartUpload struct { +type ObjectAbortIncompleteMultipartUpload struct { DaysAfterInitiation *int `json:"daysAfterInitiation"` } -// LifecycleExpiration expiration details of lifecycle configuration -type LifecycleExpiration struct { +// ObjectLifecycleExpiration expiration details of lifecycle configuration +type ObjectLifecycleExpiration struct { Date *scalar.Date `json:"date,omitempty"` Days *int `json:"days,omitempty"` DeleteMarker *bool `json:"expiredObjectDeleteMarker,omitempty"` @@ -418,91 +602,83 @@ type LifecycleExpiration struct { } // IsEmpty checks if all properties of the object are empty. -func (fe LifecycleExpiration) IsEmpty() bool { +func (fe ObjectLifecycleExpiration) IsEmpty() bool { return fe.DeleteAll == nil && fe.Date == nil && fe.Days == nil && fe.DeleteMarker == nil } -// LifecycleTransition transition details of lifecycle configuration -type LifecycleTransition struct { +// ObjectLifecycleTransition transition details of lifecycle configuration +type ObjectLifecycleTransition struct { Date *scalar.Date `json:"date"` StorageClass *string `json:"storageClass"` Days *int `json:"days"` } // IsEmpty checks if all properties of the object are empty. -func (fe LifecycleTransition) IsEmpty() bool { +func (fe ObjectLifecycleTransition) IsEmpty() bool { return fe.StorageClass == nil && fe.Date == nil && fe.Days == nil } // LifecycleDelMarkerExpiration represents DelMarkerExpiration actions element in an ILM policy -type LifecycleDelMarkerExpiration struct { +type ObjectLifecycleDelMarkerExpiration struct { Days *int `json:"days"` } -// LifecycleAllVersionsExpiration represents AllVersionsExpiration actions element in an ILM policy -type LifecycleAllVersionsExpiration struct { +// ObjectLifecycleAllVersionsExpiration represents AllVersionsExpiration actions element in an ILM policy +type ObjectLifecycleAllVersionsExpiration struct { Days *int `json:"days"` DeleteMarker *bool `json:"deleteMarker"` } -// LifecycleFilter will be used in selecting rule(s) for lifecycle configuration -type LifecycleFilter struct { - And *LifecycleFilterAnd `json:"and,omitempty"` - Prefix *string `json:"prefix,omitempty"` - Tag *StorageTag `json:"tag,omitempty"` - ObjectSizeLessThan *int64 `json:"objectSizeLessThan,omitempty"` - ObjectSizeGreaterThan *int64 `json:"objectSizeGreaterThan,omitempty"` -} +// ObjectLifecycleFilter will be used in selecting rule(s) for lifecycle configuration +type ObjectLifecycleFilter struct { + // MatchesPrefix is the condition matching an object if any of the + // matches_prefix strings are an exact prefix of the object's name. + MatchesPrefix []string `json:"matchesPrefix,omitempty"` -// LifecycleFilterAnd the And Rule for LifecycleTag, to be used in LifecycleRuleFilter -type LifecycleFilterAnd struct { - Prefix *string `json:"prefix,omitempty"` - Tags []StorageTag `json:"tags,omitempty"` - ObjectSizeLessThan *int64 `json:"objectSizeLessThan,omitempty"` - ObjectSizeGreaterThan *int64 `json:"objectSizeGreaterThan,omitempty"` -} + // MatchesStorageClasses is the condition matching the object's storage + // class. + // + // Values include "STANDARD", "NEARLINE", "COLDLINE" and "ARCHIVE". + MatchesStorageClasses []string `json:"matchesStorageClasses,omitempty"` + + // MatchesSuffix is the condition matching an object if any of the + // matches_suffix strings are an exact suffix of the object's name. + MatchesSuffix []string `json:"matchesSuffix,omitempty"` -// StorageTag structure key/value pair representing an object tag to apply configuration -type StorageTag struct { - Key *string `json:"key,omitempty"` - Value *string `json:"value,omitempty"` + // Tags structure key/value pair representing an object tag to apply configuration + Tags map[string]string `json:"tags,omitempty"` + ObjectSizeLessThan *int64 `json:"objectSizeLessThan,omitempty"` + ObjectSizeGreaterThan *int64 `json:"objectSizeGreaterThan,omitempty"` } -// LifecycleNoncurrentVersionExpiration - Specifies when noncurrent object versions expire. +// ObjectLifecycleNoncurrentVersionExpiration - Specifies when noncurrent object versions expire. // Upon expiration, server permanently deletes the noncurrent object versions. // Set this lifecycle configuration action on a bucket that has versioning enabled // (or suspended) to request server delete noncurrent object versions at a // specific period in the object's lifetime. -type LifecycleNoncurrentVersionExpiration struct { +type ObjectLifecycleNoncurrentVersionExpiration struct { NoncurrentDays *int `json:"noncurrentDays,omitempty"` NewerNoncurrentVersions *int `json:"newerNoncurrentVersions,omitempty"` } -// LifecycleNoncurrentVersionTransition sets this action to request server to +// ObjectLifecycleNoncurrentVersionTransition sets this action to request server to // transition noncurrent object versions to different set storage classes // at a specific period in the object's lifetime. -type LifecycleNoncurrentVersionTransition struct { +type ObjectLifecycleNoncurrentVersionTransition struct { StorageClass *string `json:"storageClass,omitempty"` NoncurrentDays *int `json:"noncurrentDays"` NewerNoncurrentVersions *int `json:"newerNoncurrentVersions,omitempty"` } -// StorageApplySSEByDefault defines default encryption configuration, KMS or SSE. To activate -// KMS, SSEAlgoritm needs to be set to `aws:kms“. -// Minio currently does not support Kms. -type StorageApplySSEByDefault struct { - KmsMasterKeyID *string `json:"kmsMasterKeyId,omitempty"` - SSEAlgorithm string `json:"sseAlgorithm"` -} - -// ServerSideEncryptionRule rule layer encapsulates default encryption configuration -type ServerSideEncryptionRule struct { - Apply StorageApplySSEByDefault `json:"apply"` -} - // ServerSideEncryptionConfiguration is the default encryption configuration structure. type ServerSideEncryptionConfiguration struct { - Rules []ServerSideEncryptionRule `json:"rules"` + KmsMasterKeyID string `json:"kmsMasterKeyId,omitempty"` + SSEAlgorithm string `json:"sseAlgorithm,omitempty"` +} + +// IsEmpty checks if the configuration is empty. +func (ssec ServerSideEncryptionConfiguration) IsEmpty() bool { + return ssec.KmsMasterKeyID == "" && ssec.SSEAlgorithm == "" } // SetStorageObjectLockConfig represents the object lock configuration options in given bucket @@ -512,11 +688,11 @@ type SetStorageObjectLockConfig struct { Unit *StorageRetentionValidityUnit `json:"unit"` } -// SetStorageObjectLockConfig represents the object lock configuration in given bucket +// StorageObjectLockConfig represents the object lock configuration in given bucket type StorageObjectLockConfig struct { SetStorageObjectLockConfig - ObjectLock string `json:"objectLock"` + Enabled bool `json:"enabled"` } // StorageRetentionValidityUnit retention validity unit. @@ -525,7 +701,7 @@ type StorageRetentionValidityUnit string // StorageBucketVersioningConfiguration is the versioning configuration structure type StorageBucketVersioningConfiguration struct { - Status *string `json:"status"` + Enabled bool `json:"enabled"` MFADelete *string `json:"mfaDelete"` // MinIO extension - allows selective, prefix-level versioning exclusion. // Requires versioning to be enabled @@ -591,13 +767,13 @@ type SourceSelectionCriteria struct { // StorageReplicationFilter a filter for a replication configuration Rule. type StorageReplicationFilter struct { - Prefix *string `json:"rrefix,omitempty"` + Prefix *string `json:"prefix,omitempty"` And *StorageReplicationFilterAnd `json:"and,omitempty"` - Tag *StorageTag `json:"tag,omitempty"` + Tag map[string]string `json:"tag,omitempty"` } // StorageReplicationFilterAnd - a tag to combine a prefix and multiple tags for replication configuration rule. type StorageReplicationFilterAnd struct { - Prefix *string `json:"rrefix,omitempty"` - Tags []StorageTag `json:"tag,omitempty"` + Prefix *string `json:"prefix,omitempty"` + Tags map[string]string `json:"tag,omitempty"` } diff --git a/connector/storage/common/telemetry.go b/connector/storage/common/telemetry.go index fd7bd21..debd81d 100644 --- a/connector/storage/common/telemetry.go +++ b/connector/storage/common/telemetry.go @@ -55,18 +55,14 @@ func SetObjectInfoSpanAttributes(span trace.Span, object *StorageObject) { span.SetAttributes(attribute.String("storage.object.version", *object.VersionID)) } - if object.UserTagCount > 0 { - span.SetAttributes(attribute.Int("storage.object.user_tag_count", object.UserTagCount)) + if object.TagCount > 0 { + span.SetAttributes(attribute.Int("storage.object.tags_count", object.TagCount)) } if len(object.Metadata) > 0 { span.SetAttributes(attribute.Int("storage.object.metadata_count", len(object.Metadata))) } - if len(object.UserMetadata) > 0 { - span.SetAttributes(attribute.Int("storage.object.user_metadata_count", len(object.UserMetadata))) - } - if !object.Expires.IsZero() { span.SetAttributes(attribute.String("storage.object.expires", object.Expires.Format(time.RFC3339))) } diff --git a/connector/storage/common/transport.go b/connector/storage/common/transport.go index 6b40e8e..dc2d734 100644 --- a/connector/storage/common/transport.go +++ b/connector/storage/common/transport.go @@ -17,7 +17,7 @@ import ( "go.opentelemetry.io/otel/trace" ) -var tranportTracer = connector.NewTracer("connector/storage/common/transport") +var transportTracer = connector.NewTracer("connector/storage/common/transport") // NewTransport creates a new HTTP transporter for the storage client. func NewTransport(logger *slog.Logger, port int, secure bool) http.RoundTripper { @@ -46,7 +46,7 @@ type debugRoundTripper struct { } func (mrt debugRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { - ctx, span := tranportTracer.Start(req.Context(), fmt.Sprintf("%s %s", req.Method, req.URL.Path), trace.WithSpanKind(trace.SpanKindClient)) + ctx, span := transportTracer.Start(req.Context(), fmt.Sprintf("%s %s", req.Method, req.URL.Path), trace.WithSpanKind(trace.SpanKindClient)) defer span.End() span.SetAttributes( diff --git a/connector/storage/common/types.generated.go b/connector/storage/common/types.generated.go index 4a783b9..a6758aa 100644 --- a/connector/storage/common/types.generated.go +++ b/connector/storage/common/types.generated.go @@ -85,7 +85,15 @@ func (j *ListIncompleteUploadsOptions) FromValue(input map[string]any) error { // FromValue decodes values from map func (j *ListStorageBucketArguments) FromValue(input map[string]any) error { var err error - j.ClientID, err = utils.DecodeNullableObjectValue[StorageClientID](input, "clientId") + j.MaxResults, err = utils.GetIntDefault[int](input, "maxResults") + if err != nil { + return err + } + j.StartAfter, err = utils.GetStringDefault(input, "startAfter") + if err != nil { + return err + } + j.Where, err = utils.DecodeObjectValueDefault[schema.Expression](input, "where") if err != nil { return err } @@ -187,79 +195,57 @@ func (j *StorageBucketArguments) FromValue(input map[string]any) error { } // ToMap encodes the struct to a value map -func (j AbortIncompleteMultipartUpload) ToMap() map[string]any { +func (j BucketAutoclass) ToMap() map[string]any { r := make(map[string]any) - r["daysAfterInitiation"] = j.DaysAfterInitiation + r["enabled"] = j.Enabled + r["terminalStorageClass"] = j.TerminalStorageClass + r["terminalStorageClassUpdateTime"] = j.TerminalStorageClassUpdateTime + r["toggleTime"] = j.ToggleTime return r } // ToMap encodes the struct to a value map -func (j BucketLifecycleConfiguration) ToMap() map[string]any { +func (j BucketCors) ToMap() map[string]any { r := make(map[string]any) - j_Rules := make([]any, len(j.Rules)) - for i, j_Rules_v := range j.Rules { - j_Rules[i] = j_Rules_v - } - r["rules"] = j_Rules + r["maxAge"] = j.MaxAge + r["methods"] = j.Methods + r["origins"] = j.Origins + r["responseHeaders"] = j.ResponseHeaders return r } // ToMap encodes the struct to a value map -func (j BucketLifecycleRule) ToMap() map[string]any { +func (j BucketHierarchicalNamespace) ToMap() map[string]any { r := make(map[string]any) - if j.AbortIncompleteMultipartUpload != nil { - r["abortIncompleteMultipartUpload"] = (*j.AbortIncompleteMultipartUpload) - } - if j.AllVersionsExpiration != nil { - r["allVersionsExpiration"] = (*j.AllVersionsExpiration) - } - if j.DelMarkerExpiration != nil { - r["delMarkerExpiration"] = (*j.DelMarkerExpiration) - } - if j.Expiration != nil { - r["expiration"] = (*j.Expiration) - } - if j.RuleFilter != nil { - r["filter"] = (*j.RuleFilter) - } - r["id"] = j.ID - if j.NoncurrentVersionExpiration != nil { - r["noncurrentVersionExpiration"] = (*j.NoncurrentVersionExpiration) - } - if j.NoncurrentVersionTransition != nil { - r["noncurrentVersionTransition"] = (*j.NoncurrentVersionTransition) - } - r["prefix"] = j.Prefix - r["status"] = j.Status - if j.Transition != nil { - r["transition"] = (*j.Transition) - } + r["enabled"] = j.Enabled return r } // ToMap encodes the struct to a value map -func (j DeleteMarkerReplication) ToMap() map[string]any { +func (j BucketLogging) ToMap() map[string]any { r := make(map[string]any) - r["status"] = j.Status + r["logBucket"] = j.LogBucket + r["logObjectPrefix"] = j.LogObjectPrefix return r } // ToMap encodes the struct to a value map -func (j DeleteReplication) ToMap() map[string]any { +func (j BucketWebsite) ToMap() map[string]any { r := make(map[string]any) - r["status"] = j.Status + r["mainPageSuffix"] = j.MainPageSuffix + r["notFoundPage"] = j.NotFoundPage return r } // ToMap encodes the struct to a value map -func (j ExistingObjectReplication) ToMap() map[string]any { +func (j CustomPlacementConfig) ToMap() map[string]any { r := make(map[string]any) - r["status"] = j.Status + r["DataLocations"] = j.DataLocations return r } @@ -276,213 +262,158 @@ func (j GetStorageObjectOptions) ToMap() map[string]any { } // ToMap encodes the struct to a value map -func (j LifecycleAllVersionsExpiration) ToMap() map[string]any { +func (j ListIncompleteUploadsOptions) ToMap() map[string]any { r := make(map[string]any) - r["days"] = j.Days - r["deleteMarker"] = j.DeleteMarker + r["prefix"] = j.Prefix + r["recursive"] = j.Recursive return r } // ToMap encodes the struct to a value map -func (j LifecycleDelMarkerExpiration) ToMap() map[string]any { +func (j ListStorageObjectsOptions) ToMap() map[string]any { r := make(map[string]any) - r["days"] = j.Days + r["maxResults"] = j.MaxResults + r["prefix"] = j.Prefix + r["recursive"] = j.Recursive + r["startAfter"] = j.StartAfter return r } // ToMap encodes the struct to a value map -func (j LifecycleExpiration) ToMap() map[string]any { +func (j MakeStorageBucketOptions) ToMap() map[string]any { r := make(map[string]any) - r["date"] = j.Date - r["days"] = j.Days - r["expiredObjectAllVersions"] = j.DeleteAll - r["expiredObjectDeleteMarker"] = j.DeleteMarker + r["name"] = j.Name + r["objectLock"] = j.ObjectLock + r["region"] = j.Region + r["tags"] = j.Tags return r } // ToMap encodes the struct to a value map -func (j LifecycleFilter) ToMap() map[string]any { +func (j ObjectAbortIncompleteMultipartUpload) ToMap() map[string]any { r := make(map[string]any) - if j.And != nil { - r["and"] = (*j.And) - } - r["objectSizeGreaterThan"] = j.ObjectSizeGreaterThan - r["objectSizeLessThan"] = j.ObjectSizeLessThan - r["prefix"] = j.Prefix - if j.Tag != nil { - r["tag"] = (*j.Tag) - } + r["daysAfterInitiation"] = j.DaysAfterInitiation return r } // ToMap encodes the struct to a value map -func (j LifecycleFilterAnd) ToMap() map[string]any { +func (j ObjectLifecycleAllVersionsExpiration) ToMap() map[string]any { r := make(map[string]any) - r["objectSizeGreaterThan"] = j.ObjectSizeGreaterThan - r["objectSizeLessThan"] = j.ObjectSizeLessThan - r["prefix"] = j.Prefix - j_Tags := make([]any, len(j.Tags)) - for i, j_Tags_v := range j.Tags { - j_Tags[i] = j_Tags_v - } - r["tags"] = j_Tags + r["days"] = j.Days + r["deleteMarker"] = j.DeleteMarker return r } // ToMap encodes the struct to a value map -func (j LifecycleNoncurrentVersionExpiration) ToMap() map[string]any { +func (j ObjectLifecycleConfiguration) ToMap() map[string]any { r := make(map[string]any) - r["newerNoncurrentVersions"] = j.NewerNoncurrentVersions - r["noncurrentDays"] = j.NoncurrentDays + j_Rules := make([]any, len(j.Rules)) + for i, j_Rules_v := range j.Rules { + j_Rules[i] = j_Rules_v + } + r["rules"] = j_Rules return r } // ToMap encodes the struct to a value map -func (j LifecycleNoncurrentVersionTransition) ToMap() map[string]any { +func (j ObjectLifecycleDelMarkerExpiration) ToMap() map[string]any { r := make(map[string]any) - r["newerNoncurrentVersions"] = j.NewerNoncurrentVersions - r["noncurrentDays"] = j.NoncurrentDays - r["storageClass"] = j.StorageClass + r["days"] = j.Days return r } // ToMap encodes the struct to a value map -func (j LifecycleTransition) ToMap() map[string]any { +func (j ObjectLifecycleExpiration) ToMap() map[string]any { r := make(map[string]any) r["date"] = j.Date r["days"] = j.Days - r["storageClass"] = j.StorageClass + r["expiredObjectAllVersions"] = j.DeleteAll + r["expiredObjectDeleteMarker"] = j.DeleteMarker return r } // ToMap encodes the struct to a value map -func (j ListIncompleteUploadsOptions) ToMap() map[string]any { +func (j ObjectLifecycleFilter) ToMap() map[string]any { r := make(map[string]any) - r["prefix"] = j.Prefix - r["recursive"] = j.Recursive + r["matchesPrefix"] = j.MatchesPrefix + r["matchesStorageClasses"] = j.MatchesStorageClasses + r["matchesSuffix"] = j.MatchesSuffix + r["objectSizeGreaterThan"] = j.ObjectSizeGreaterThan + r["objectSizeLessThan"] = j.ObjectSizeLessThan + r["tags"] = j.Tags return r } // ToMap encodes the struct to a value map -func (j ListStorageObjectsOptions) ToMap() map[string]any { +func (j ObjectLifecycleNoncurrentVersionExpiration) ToMap() map[string]any { r := make(map[string]any) - r["maxResults"] = j.MaxResults - r["prefix"] = j.Prefix - r["recursive"] = j.Recursive - r["startAfter"] = j.StartAfter + r["newerNoncurrentVersions"] = j.NewerNoncurrentVersions + r["noncurrentDays"] = j.NoncurrentDays return r } // ToMap encodes the struct to a value map -func (j MakeStorageBucketOptions) ToMap() map[string]any { +func (j ObjectLifecycleNoncurrentVersionTransition) ToMap() map[string]any { r := make(map[string]any) - r["name"] = j.Name - r["objectLocking"] = j.ObjectLocking - r["region"] = j.Region - r["tags"] = j.Tags + r["newerNoncurrentVersions"] = j.NewerNoncurrentVersions + r["noncurrentDays"] = j.NoncurrentDays + r["storageClass"] = j.StorageClass return r } // ToMap encodes the struct to a value map -func (j NotificationCommonConfig) ToMap() map[string]any { +func (j ObjectLifecycleRule) ToMap() map[string]any { r := make(map[string]any) - r["arn"] = j.Arn - r["event"] = j.Events - if j.Filter != nil { - r["filter"] = (*j.Filter) + if j.AbortIncompleteMultipartUpload != nil { + r["abortIncompleteMultipartUpload"] = (*j.AbortIncompleteMultipartUpload) } - r["id"] = j.ID - - return r -} - -// ToMap encodes the struct to a value map -func (j NotificationConfig) ToMap() map[string]any { - r := make(map[string]any) - j_LambdaConfigs := make([]any, len(j.LambdaConfigs)) - for i, j_LambdaConfigs_v := range j.LambdaConfigs { - j_LambdaConfigs[i] = j_LambdaConfigs_v + if j.AllVersionsExpiration != nil { + r["allVersionsExpiration"] = (*j.AllVersionsExpiration) } - r["cloudFunctionConfigurations"] = j_LambdaConfigs - j_QueueConfigs := make([]any, len(j.QueueConfigs)) - for i, j_QueueConfigs_v := range j.QueueConfigs { - j_QueueConfigs[i] = j_QueueConfigs_v + if j.DelMarkerExpiration != nil { + r["delMarkerExpiration"] = (*j.DelMarkerExpiration) } - r["queueConfigurations"] = j_QueueConfigs - j_TopicConfigs := make([]any, len(j.TopicConfigs)) - for i, j_TopicConfigs_v := range j.TopicConfigs { - j_TopicConfigs[i] = j_TopicConfigs_v + r["enabled"] = j.Enabled + if j.Expiration != nil { + r["expiration"] = (*j.Expiration) } - r["topicConfigurations"] = j_TopicConfigs - - return r -} - -// ToMap encodes the struct to a value map -func (j NotificationFilter) ToMap() map[string]any { - r := make(map[string]any) - if j.S3Key != nil { - r["s3Key"] = (*j.S3Key) + j_RuleFilter := make([]any, len(j.RuleFilter)) + for i, j_RuleFilter_v := range j.RuleFilter { + j_RuleFilter[i] = j_RuleFilter_v } - - return r -} - -// ToMap encodes the struct to a value map -func (j NotificationFilterRule) ToMap() map[string]any { - r := make(map[string]any) - r["name"] = j.Name - r["value"] = j.Value - - return r -} - -// ToMap encodes the struct to a value map -func (j NotificationLambdaConfig) ToMap() map[string]any { - r := make(map[string]any) - r = utils.MergeMap(r, j.NotificationCommonConfig.ToMap()) - r["cloudFunction"] = j.Lambda - - return r -} - -// ToMap encodes the struct to a value map -func (j NotificationQueueConfig) ToMap() map[string]any { - r := make(map[string]any) - r = utils.MergeMap(r, j.NotificationCommonConfig.ToMap()) - r["queue"] = j.Queue - - return r -} - -// ToMap encodes the struct to a value map -func (j NotificationS3Key) ToMap() map[string]any { - r := make(map[string]any) - j_FilterRules := make([]any, len(j.FilterRules)) - for i, j_FilterRules_v := range j.FilterRules { - j_FilterRules[i] = j_FilterRules_v + r["filter"] = j_RuleFilter + r["id"] = j.ID + if j.NoncurrentVersionExpiration != nil { + r["noncurrentVersionExpiration"] = (*j.NoncurrentVersionExpiration) + } + if j.NoncurrentVersionTransition != nil { + r["noncurrentVersionTransition"] = (*j.NoncurrentVersionTransition) + } + r["prefix"] = j.Prefix + if j.Transition != nil { + r["transition"] = (*j.Transition) } - r["filterRule"] = j_FilterRules return r } // ToMap encodes the struct to a value map -func (j NotificationTopicConfig) ToMap() map[string]any { +func (j ObjectLifecycleTransition) ToMap() map[string]any { r := make(map[string]any) - r = utils.MergeMap(r, j.NotificationCommonConfig.ToMap()) - r["topic"] = j.Topic + r["date"] = j.Date + r["days"] = j.Days + r["storageClass"] = j.StorageClass return r } @@ -520,19 +451,30 @@ func (j PutStorageObjectOptions) ToMap() map[string]any { r["disableMultipart"] = j.DisableMultipart r["expires"] = j.Expires r["legalHold"] = j.LegalHold - r["mode"] = j.Mode + r["metadata"] = j.Metadata r["numThreads"] = j.NumThreads r["partSize"] = j.PartSize - r["retainUntilDate"] = j.RetainUntilDate + if j.Retention != nil { + r["retention"] = (*j.Retention) + } r["sendContentMd5"] = j.SendContentMd5 r["storageClass"] = j.StorageClass - r["userMetadata"] = j.UserMetadata - r["userTags"] = j.UserTags + r["tags"] = j.Tags r["websiteRedirectLocation"] = j.WebsiteRedirectLocation return r } +// ToMap encodes the struct to a value map +func (j PutStorageObjectRetentionOptions) ToMap() map[string]any { + r := make(map[string]any) + r["governanceBypass"] = j.GovernanceBypass + r["mode"] = j.Mode + r["retainUntilDate"] = j.RetainUntilDate + + return r +} + // ToMap encodes the struct to a value map func (j RemoveStorageObjectError) ToMap() map[string]any { r := make(map[string]any) @@ -548,6 +490,7 @@ func (j RemoveStorageObjectOptions) ToMap() map[string]any { r := make(map[string]any) r["forceDelete"] = j.ForceDelete r["governanceBypass"] = j.GovernanceBypass + r["softDelete"] = j.SoftDelete r["versionId"] = j.VersionID return r @@ -562,39 +505,11 @@ func (j RemoveStorageObjectsOptions) ToMap() map[string]any { return r } -// ToMap encodes the struct to a value map -func (j ReplicaModifications) ToMap() map[string]any { - r := make(map[string]any) - r["status"] = j.Status - - return r -} - // ToMap encodes the struct to a value map func (j ServerSideEncryptionConfiguration) ToMap() map[string]any { r := make(map[string]any) - j_Rules := make([]any, len(j.Rules)) - for i, j_Rules_v := range j.Rules { - j_Rules[i] = j_Rules_v - } - r["rules"] = j_Rules - - return r -} - -// ToMap encodes the struct to a value map -func (j ServerSideEncryptionRule) ToMap() map[string]any { - r := make(map[string]any) - r["apply"] = j.Apply - - return r -} - -// ToMap encodes the struct to a value map -func (j SetStorageObjectLegalHoldOptions) ToMap() map[string]any { - r := make(map[string]any) - r["status"] = j.Status - r["versionId"] = j.VersionID + r["kmsMasterKeyId"] = j.KmsMasterKeyID + r["sseAlgorithm"] = j.SSEAlgorithm return r } @@ -615,35 +530,59 @@ func (j SetStorageObjectRetentionOptions) ToMap() map[string]any { r["governanceBypass"] = j.GovernanceBypass r["mode"] = j.Mode r["retainUntilDate"] = j.RetainUntilDate - r["versionId"] = j.VersionID return r } // ToMap encodes the struct to a value map -func (j SetStorageObjectTagsOptions) ToMap() map[string]any { +func (j StorageBucket) ToMap() map[string]any { r := make(map[string]any) + if j.Autoclass != nil { + r["autoclass"] = (*j.Autoclass) + } + j_CORS := make([]any, len(j.CORS)) + for i, j_CORS_v := range j.CORS { + j_CORS[i] = j_CORS_v + } + r["cors"] = j_CORS + r["creationTime"] = j.CreationTime + if j.CustomPlacementConfig != nil { + r["customPlacementConfig"] = (*j.CustomPlacementConfig) + } + r["defaultEventBasedHold"] = j.DefaultEventBasedHold + if j.Encryption != nil { + r["encryption"] = (*j.Encryption) + } + r["etag"] = j.Etag + if j.HierarchicalNamespace != nil { + r["hierarchicalNamespace"] = (*j.HierarchicalNamespace) + } + r["lastModified"] = j.LastModified + if j.Lifecycle != nil { + r["lifecycle"] = (*j.Lifecycle) + } + r["locationType"] = j.LocationType + if j.Logging != nil { + r["logging"] = (*j.Logging) + } + r["name"] = j.Name + if j.ObjectLock != nil { + r["objectLock"] = (*j.ObjectLock) + } + r["region"] = j.Region + r["requesterPays"] = j.RequesterPays + r["rpo"] = j.RPO + if j.SoftDeletePolicy != nil { + r["softDeletePolicy"] = (*j.SoftDeletePolicy) + } + r["storageClass"] = j.StorageClass r["tags"] = j.Tags - r["versionId"] = j.VersionID - - return r -} - -// ToMap encodes the struct to a value map -func (j SourceSelectionCriteria) ToMap() map[string]any { - r := make(map[string]any) - if j.ReplicaModifications != nil { - r["replicaModifications"] = (*j.ReplicaModifications) + if j.Versioning != nil { + r["versioning"] = (*j.Versioning) + } + if j.Website != nil { + r["website"] = (*j.Website) } - - return r -} - -// ToMap encodes the struct to a value map -func (j StorageApplySSEByDefault) ToMap() map[string]any { - r := make(map[string]any) - r["kmsMasterKeyId"] = j.KmsMasterKeyID - r["sseAlgorithm"] = j.SSEAlgorithm return r } @@ -658,11 +597,14 @@ func (j StorageBucketArguments) ToMap() map[string]any { } // ToMap encodes the struct to a value map -func (j StorageBucketInfo) ToMap() map[string]any { +func (j StorageBucketListResults) ToMap() map[string]any { r := make(map[string]any) - r["creationDate"] = j.CreationDate - r["name"] = j.Name - r["tags"] = j.Tags + j_Buckets := make([]any, len(j.Buckets)) + for i, j_Buckets_v := range j.Buckets { + j_Buckets[i] = j_Buckets_v + } + r["buckets"] = j_Buckets + r["pageInfo"] = j.PageInfo return r } @@ -670,10 +612,10 @@ func (j StorageBucketInfo) ToMap() map[string]any { // ToMap encodes the struct to a value map func (j StorageBucketVersioningConfiguration) ToMap() map[string]any { r := make(map[string]any) + r["enabled"] = j.Enabled r["excludeFolders"] = j.ExcludeFolders r["excludedPrefixes"] = j.ExcludedPrefixes r["mfaDelete"] = j.MFADelete - r["status"] = j.Status return r } @@ -683,14 +625,12 @@ func (j StorageCopyDestOptions) ToMap() map[string]any { r := make(map[string]any) r["bucket"] = j.Bucket r["legalHold"] = j.LegalHold + r["metadata"] = j.Metadata r["mode"] = j.Mode r["object"] = j.Object - r["replaceMetadata"] = j.ReplaceMetadata - r["replaceTags"] = j.ReplaceTags r["retainUntilDate"] = j.RetainUntilDate r["size"] = j.Size - r["userMetadata"] = j.UserMetadata - r["userTags"] = j.UserTags + r["tags"] = j.Tags return r } @@ -751,18 +691,14 @@ func (j StorageObject) ToMap() map[string]any { r["contentLanguage"] = j.ContentLanguage r["contentMd5"] = j.ContentMD5 r["contentType"] = j.ContentType - r["copyCompletionTime"] = j.CopyCompletionTime - r["copyId"] = j.CopyID - r["copyProgress"] = j.CopyProgress - r["copySource"] = j.CopySource - r["copyStatus"] = j.CopyStatus - r["copyStatusDescription"] = j.CopyStatusDescription + if j.Copy != nil { + r["copy"] = (*j.Copy) + } r["creationTime"] = j.CreationTime r["customerProvidedKeySha256"] = j.CustomerProvidedKeySHA256 r["deleted"] = j.Deleted r["deletedTime"] = j.DeletedTime r["destinationSnapshot"] = j.DestinationSnapshot - r["encryptionScope"] = j.EncryptionScope r["etag"] = j.ETag r["expiration"] = j.Expiration r["expirationRuleId"] = j.ExpirationRuleID @@ -773,22 +709,23 @@ func (j StorageObject) ToMap() map[string]any { } r["grant"] = j_Grant r["group"] = j.Group - r["immutabilityPolicyMode"] = j.ImmutabilityPolicyMode - r["immutabilityPolicyUntilDate"] = j.ImmutabilityPolicyUntilDate r["incrementalCopy"] = j.IncrementalCopy r["isLatest"] = j.IsLatest + r["kmsKeyName"] = j.KMSKeyName r["lastAccessTime"] = j.LastAccessTime r["lastModified"] = j.LastModified r["leaseDuration"] = j.LeaseDuration r["leaseState"] = j.LeaseState r["leaseStatus"] = j.LeaseStatus r["legalHold"] = j.LegalHold + r["mediaLink"] = j.MediaLink r["metadata"] = j.Metadata r["name"] = j.Name if j.Owner != nil { r["owner"] = (*j.Owner) } r["permissions"] = j.Permissions + r["rawMetadata"] = j.RawMetadata r["rehydratePriority"] = j.RehydratePriority r["remainingRetentionDays"] = j.RemainingRetentionDays r["replicationReady"] = j.ReplicationReady @@ -797,13 +734,14 @@ func (j StorageObject) ToMap() map[string]any { if j.Restore != nil { r["restore"] = (*j.Restore) } + r["retentionMode"] = j.RetentionMode + r["retentionUntilDate"] = j.RetentionUntilDate r["sealed"] = j.IsSealed r["serverEncrypted"] = j.ServerEncrypted r["size"] = j.Size r["storageClass"] = j.StorageClass - r["userMetadata"] = j.UserMetadata - r["userTagCount"] = j.UserTagCount - r["userTags"] = j.UserTags + r["tagCount"] = j.TagCount + r["tags"] = j.Tags r["versionId"] = j.VersionID return r @@ -821,6 +759,19 @@ func (j StorageObjectChecksum) ToMap() map[string]any { return r } +// ToMap encodes the struct to a value map +func (j StorageObjectCopyInfo) ToMap() map[string]any { + r := make(map[string]any) + r["completionTime"] = j.CompletionTime + r["id"] = j.ID + r["progress"] = j.Progress + r["source"] = j.Source + r["status"] = j.Status + r["statusDescription"] = j.StatusDescription + + return r +} + // ToMap encodes the struct to a value map func (j StorageObjectListResults) ToMap() map[string]any { r := make(map[string]any) @@ -838,7 +789,7 @@ func (j StorageObjectListResults) ToMap() map[string]any { func (j StorageObjectLockConfig) ToMap() map[string]any { r := make(map[string]any) r = utils.MergeMap(r, j.SetStorageObjectLockConfig.ToMap()) - r["objectLock"] = j.ObjectLock + r["enabled"] = j.Enabled return r } @@ -856,11 +807,10 @@ func (j StorageObjectMultipartInfo) ToMap() map[string]any { } // ToMap encodes the struct to a value map -func (j StorageObjectPaginationInfo) ToMap() map[string]any { +func (j StorageObjectSoftDeletePolicy) ToMap() map[string]any { r := make(map[string]any) - r["cursor"] = j.Cursor - r["hasNextPage"] = j.HasNextPage - r["nextCursor"] = j.NextCursor + r["effectiveTime"] = j.EffectiveTime + r["retentionDuration"] = j.RetentionDuration return r } @@ -875,76 +825,10 @@ func (j StorageOwner) ToMap() map[string]any { } // ToMap encodes the struct to a value map -func (j StorageReplicationConfig) ToMap() map[string]any { - r := make(map[string]any) - r["role"] = j.Role - j_Rules := make([]any, len(j.Rules)) - for i, j_Rules_v := range j.Rules { - j_Rules[i] = j_Rules_v - } - r["rules"] = j_Rules - - return r -} - -// ToMap encodes the struct to a value map -func (j StorageReplicationDestination) ToMap() map[string]any { - r := make(map[string]any) - r["bucket"] = j.Bucket - r["storageClass"] = j.StorageClass - - return r -} - -// ToMap encodes the struct to a value map -func (j StorageReplicationFilter) ToMap() map[string]any { - r := make(map[string]any) - if j.And != nil { - r["and"] = (*j.And) - } - r["rrefix"] = j.Prefix - if j.Tag != nil { - r["tag"] = (*j.Tag) - } - - return r -} - -// ToMap encodes the struct to a value map -func (j StorageReplicationFilterAnd) ToMap() map[string]any { - r := make(map[string]any) - r["rrefix"] = j.Prefix - j_Tags := make([]any, len(j.Tags)) - for i, j_Tags_v := range j.Tags { - j_Tags[i] = j_Tags_v - } - r["tag"] = j_Tags - - return r -} - -// ToMap encodes the struct to a value map -func (j StorageReplicationRule) ToMap() map[string]any { +func (j StoragePaginationInfo) ToMap() map[string]any { r := make(map[string]any) - if j.DeleteMarkerReplication != nil { - r["deleteMarkerReplication"] = (*j.DeleteMarkerReplication) - } - if j.DeleteReplication != nil { - r["deleteReplication"] = (*j.DeleteReplication) - } - if j.Destination != nil { - r["destination"] = (*j.Destination) - } - if j.ExistingObjectReplication != nil { - r["existingObjectReplication"] = (*j.ExistingObjectReplication) - } - r["filter"] = j.Filter - r["id"] = j.ID - r["priority"] = j.Priority - if j.SourceSelectionCriteria != nil { - r["sourceSelectionCriteria"] = (*j.SourceSelectionCriteria) - } - r["status"] = j.Status + r["cursor"] = j.Cursor + r["hasNextPage"] = j.HasNextPage return r } @@ -958,15 +842,6 @@ func (j StorageRestoreInfo) ToMap() map[string]any { return r } -// ToMap encodes the struct to a value map -func (j StorageTag) ToMap() map[string]any { - r := make(map[string]any) - r["key"] = j.Key - r["value"] = j.Value - - return r -} - // ToMap encodes the struct to a value map func (j StorageUploadInfo) ToMap() map[string]any { r := make(map[string]any) @@ -986,6 +861,38 @@ func (j StorageUploadInfo) ToMap() map[string]any { return r } +// ToMap encodes the struct to a value map +func (j UpdateStorageBucketOptions) ToMap() map[string]any { + r := make(map[string]any) + if j.Encryption != nil { + r["encryption"] = (*j.Encryption) + } + if j.Lifecycle != nil { + r["lifecycle"] = (*j.Lifecycle) + } + if j.ObjectLock != nil { + r["objectLock"] = (*j.ObjectLock) + } + r["tags"] = j.Tags + r["versioningEnabled"] = j.VersioningEnabled + + return r +} + +// ToMap encodes the struct to a value map +func (j UpdateStorageObjectOptions) ToMap() map[string]any { + r := make(map[string]any) + r["legalHold"] = j.LegalHold + r["metadata"] = j.Metadata + if j.Retention != nil { + r["retention"] = (*j.Retention) + } + r["tags"] = j.Tags + r["versionId"] = j.VersionID + + return r +} + // ScalarName get the schema name of the scalar func (j ChecksumType) ScalarName() string { return "ChecksumType" @@ -1054,47 +961,40 @@ func (s *ChecksumType) FromValue(value any) error { } // ScalarName get the schema name of the scalar -func (j StorageClientID) ScalarName() string { - return "StorageClientID" -} - -// ScalarName get the schema name of the scalar -func (j StorageObjectReplicationStatus) ScalarName() string { - return "StorageObjectReplicationStatus" +func (j GoogleStorageRPO) ScalarName() string { + return "GoogleStorageRPO" } const ( - StorageObjectReplicationStatusCompleted StorageObjectReplicationStatus = "COMPLETED" - StorageObjectReplicationStatusPending StorageObjectReplicationStatus = "PENDING" - StorageObjectReplicationStatusFailed StorageObjectReplicationStatus = "FAILED" - StorageObjectReplicationStatusReplica StorageObjectReplicationStatus = "REPLICA" + GoogleStorageRpoDefault GoogleStorageRPO = "DEFAULT" + GoogleStorageRpoAsyncTurbo GoogleStorageRPO = "ASYNC_TURBO" ) -var enumValues_StorageObjectReplicationStatus = []StorageObjectReplicationStatus{StorageObjectReplicationStatusCompleted, StorageObjectReplicationStatusPending, StorageObjectReplicationStatusFailed, StorageObjectReplicationStatusReplica} +var enumValues_GoogleStorageRpo = []GoogleStorageRPO{GoogleStorageRpoDefault, GoogleStorageRpoAsyncTurbo} -// ParseStorageObjectReplicationStatus parses a StorageObjectReplicationStatus enum from string -func ParseStorageObjectReplicationStatus(input string) (StorageObjectReplicationStatus, error) { - result := StorageObjectReplicationStatus(input) - if !slices.Contains(enumValues_StorageObjectReplicationStatus, result) { - return StorageObjectReplicationStatus(""), errors.New("failed to parse StorageObjectReplicationStatus, expect one of [COMPLETED, PENDING, FAILED, REPLICA]") +// ParseGoogleStorageRpo parses a GoogleStorageRPO enum from string +func ParseGoogleStorageRpo(input string) (GoogleStorageRPO, error) { + result := GoogleStorageRPO(input) + if !slices.Contains(enumValues_GoogleStorageRpo, result) { + return GoogleStorageRPO(""), errors.New("failed to parse GoogleStorageRPO, expect one of [DEFAULT, ASYNC_TURBO]") } return result, nil } // IsValid checks if the value is invalid -func (j StorageObjectReplicationStatus) IsValid() bool { - return slices.Contains(enumValues_StorageObjectReplicationStatus, j) +func (j GoogleStorageRPO) IsValid() bool { + return slices.Contains(enumValues_GoogleStorageRpo, j) } // UnmarshalJSON implements json.Unmarshaler. -func (j *StorageObjectReplicationStatus) UnmarshalJSON(b []byte) error { +func (j *GoogleStorageRPO) UnmarshalJSON(b []byte) error { var rawValue string if err := json.Unmarshal(b, &rawValue); err != nil { return err } - value, err := ParseStorageObjectReplicationStatus(rawValue) + value, err := ParseGoogleStorageRpo(rawValue) if err != nil { return err } @@ -1104,7 +1004,7 @@ func (j *StorageObjectReplicationStatus) UnmarshalJSON(b []byte) error { } // FromValue decodes the scalar from an unknown value -func (s *StorageObjectReplicationStatus) FromValue(value any) error { +func (s *GoogleStorageRPO) FromValue(value any) error { valueStr, err := utils.DecodeNullableString(value) if err != nil { return err @@ -1112,7 +1012,7 @@ func (s *StorageObjectReplicationStatus) FromValue(value any) error { if valueStr == nil { return nil } - result, err := ParseStorageObjectReplicationStatus(*valueStr) + result, err := ParseGoogleStorageRpo(*valueStr) if err != nil { return err } @@ -1122,40 +1022,47 @@ func (s *StorageObjectReplicationStatus) FromValue(value any) error { } // ScalarName get the schema name of the scalar -func (j StorageReplicationRuleStatus) ScalarName() string { - return "StorageReplicationRuleStatus" +func (j StorageClientID) ScalarName() string { + return "StorageClientID" +} + +// ScalarName get the schema name of the scalar +func (j StorageObjectReplicationStatus) ScalarName() string { + return "StorageObjectReplicationStatus" } const ( - StorageReplicationRuleStatusEnabled StorageReplicationRuleStatus = "Enabled" - StorageReplicationRuleStatusDisabled StorageReplicationRuleStatus = "Disabled" + StorageObjectReplicationStatusCompleted StorageObjectReplicationStatus = "COMPLETED" + StorageObjectReplicationStatusPending StorageObjectReplicationStatus = "PENDING" + StorageObjectReplicationStatusFailed StorageObjectReplicationStatus = "FAILED" + StorageObjectReplicationStatusReplica StorageObjectReplicationStatus = "REPLICA" ) -var enumValues_StorageReplicationRuleStatus = []StorageReplicationRuleStatus{StorageReplicationRuleStatusEnabled, StorageReplicationRuleStatusDisabled} +var enumValues_StorageObjectReplicationStatus = []StorageObjectReplicationStatus{StorageObjectReplicationStatusCompleted, StorageObjectReplicationStatusPending, StorageObjectReplicationStatusFailed, StorageObjectReplicationStatusReplica} -// ParseStorageReplicationRuleStatus parses a StorageReplicationRuleStatus enum from string -func ParseStorageReplicationRuleStatus(input string) (StorageReplicationRuleStatus, error) { - result := StorageReplicationRuleStatus(input) - if !slices.Contains(enumValues_StorageReplicationRuleStatus, result) { - return StorageReplicationRuleStatus(""), errors.New("failed to parse StorageReplicationRuleStatus, expect one of [Enabled, Disabled]") +// ParseStorageObjectReplicationStatus parses a StorageObjectReplicationStatus enum from string +func ParseStorageObjectReplicationStatus(input string) (StorageObjectReplicationStatus, error) { + result := StorageObjectReplicationStatus(input) + if !slices.Contains(enumValues_StorageObjectReplicationStatus, result) { + return StorageObjectReplicationStatus(""), errors.New("failed to parse StorageObjectReplicationStatus, expect one of [COMPLETED, PENDING, FAILED, REPLICA]") } return result, nil } // IsValid checks if the value is invalid -func (j StorageReplicationRuleStatus) IsValid() bool { - return slices.Contains(enumValues_StorageReplicationRuleStatus, j) +func (j StorageObjectReplicationStatus) IsValid() bool { + return slices.Contains(enumValues_StorageObjectReplicationStatus, j) } // UnmarshalJSON implements json.Unmarshaler. -func (j *StorageReplicationRuleStatus) UnmarshalJSON(b []byte) error { +func (j *StorageObjectReplicationStatus) UnmarshalJSON(b []byte) error { var rawValue string if err := json.Unmarshal(b, &rawValue); err != nil { return err } - value, err := ParseStorageReplicationRuleStatus(rawValue) + value, err := ParseStorageObjectReplicationStatus(rawValue) if err != nil { return err } @@ -1165,7 +1072,7 @@ func (j *StorageReplicationRuleStatus) UnmarshalJSON(b []byte) error { } // FromValue decodes the scalar from an unknown value -func (s *StorageReplicationRuleStatus) FromValue(value any) error { +func (s *StorageObjectReplicationStatus) FromValue(value any) error { valueStr, err := utils.DecodeNullableString(value) if err != nil { return err @@ -1173,7 +1080,7 @@ func (s *StorageReplicationRuleStatus) FromValue(value any) error { if valueStr == nil { return nil } - result, err := ParseStorageReplicationRuleStatus(*valueStr) + result, err := ParseStorageObjectReplicationStatus(*valueStr) if err != nil { return err } @@ -1190,15 +1097,17 @@ func (j StorageRetentionMode) ScalarName() string { const ( StorageRetentionModeLocked StorageRetentionMode = "Locked" StorageRetentionModeUnlocked StorageRetentionMode = "Unlocked" + StorageRetentionModeMutable StorageRetentionMode = "Mutable" + StorageRetentionModeDelete StorageRetentionMode = "Delete" ) -var enumValues_StorageRetentionMode = []StorageRetentionMode{StorageRetentionModeLocked, StorageRetentionModeUnlocked} +var enumValues_StorageRetentionMode = []StorageRetentionMode{StorageRetentionModeLocked, StorageRetentionModeUnlocked, StorageRetentionModeMutable, StorageRetentionModeDelete} // ParseStorageRetentionMode parses a StorageRetentionMode enum from string func ParseStorageRetentionMode(input string) (StorageRetentionMode, error) { result := StorageRetentionMode(input) if !slices.Contains(enumValues_StorageRetentionMode, result) { - return StorageRetentionMode(""), errors.New("failed to parse StorageRetentionMode, expect one of [Locked, Unlocked]") + return StorageRetentionMode(""), errors.New("failed to parse StorageRetentionMode, expect one of [Locked, Unlocked, Mutable, Delete]") } return result, nil diff --git a/connector/storage/common/types.go b/connector/storage/common/types.go index 374c1ed..2e28943 100644 --- a/connector/storage/common/types.go +++ b/connector/storage/common/types.go @@ -24,7 +24,7 @@ type StorageProviderType string const ( S3 StorageProviderType = "s3" - GoogleStorage StorageProviderType = "gs" + GoogleStorage StorageProviderType = "gcs" AzureBlobStore StorageProviderType = "azblob" ) diff --git a/connector/storage/config.go b/connector/storage/config.go index a3b0305..ce34417 100644 --- a/connector/storage/config.go +++ b/connector/storage/config.go @@ -12,6 +12,7 @@ import ( "github.com/hasura/ndc-sdk-go/schema" "github.com/hasura/ndc-storage/connector/storage/azblob" "github.com/hasura/ndc-storage/connector/storage/common" + "github.com/hasura/ndc-storage/connector/storage/gcs" "github.com/hasura/ndc-storage/connector/storage/minio" "github.com/invopop/jsonschema" ) @@ -95,7 +96,7 @@ func (cc ClientConfig) Validate() error { return errors.New("unsupported storage client: " + string(baseConfig.Type)) } -func (cc ClientConfig) toStorageClient(ctx context.Context, logger *slog.Logger) (*common.BaseClientConfig, common.StorageClient, error) { +func (cc ClientConfig) toStorageClient(ctx context.Context, logger *slog.Logger, version string) (*common.BaseClientConfig, common.StorageClient, error) { if len(cc) == 0 { return nil, nil, errConfigEmpty } @@ -111,7 +112,7 @@ func (cc ClientConfig) toStorageClient(ctx context.Context, logger *slog.Logger) } switch baseConfig.Type { - case common.S3, common.GoogleStorage: + case common.S3: minioConfig := minio.ClientConfig{ BaseClientConfig: baseConfig, } @@ -122,6 +123,18 @@ func (cc ClientConfig) toStorageClient(ctx context.Context, logger *slog.Logger) client, err := minio.New(ctx, baseConfig.Type, &minioConfig, logger) + return &baseConfig, client, err + case common.GoogleStorage: + gcsConfig := gcs.ClientConfig{ + BaseClientConfig: baseConfig, + } + + if err := mapstructure.Decode(cc, &gcsConfig.OtherConfig); err != nil { + return nil, nil, err + } + + client, err := gcs.New(ctx, &gcsConfig, logger, version) + return &baseConfig, client, err case common.AzureBlobStore: azConfig := azblob.ClientConfig{ @@ -145,6 +158,7 @@ func (cc ClientConfig) JSONSchema() *jsonschema.Schema { OneOf: []*jsonschema.Schema{ minio.ClientConfig{}.JSONSchema(), azblob.ClientConfig{}.JSONSchema(), + gcs.ClientConfig{}.JSONSchema(), }, } } diff --git a/connector/storage/gcs/bucket.go b/connector/storage/gcs/bucket.go new file mode 100644 index 0000000..29f346d --- /dev/null +++ b/connector/storage/gcs/bucket.go @@ -0,0 +1,210 @@ +package gcs + +import ( + "context" + "errors" + + "cloud.google.com/go/storage" + "github.com/hasura/ndc-sdk-go/schema" + "github.com/hasura/ndc-storage/connector/storage/common" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + "google.golang.org/api/iterator" +) + +// MakeBucket creates a new bucket. +func (c *Client) MakeBucket(ctx context.Context, args *common.MakeStorageBucketOptions) error { + ctx, span := c.startOtelSpan(ctx, "MakeBucket", args.Name) + defer span.End() + + attrs := &storage.BucketAttrs{ + Location: args.Region, + Labels: args.Tags, + Name: args.Name, + } + + handle := c.client.Bucket(args.Name) + + if args.ObjectLock { + handle = handle.SetObjectRetention(true) + } + + err := handle.Create(ctx, c.projectID, attrs) + if err != nil { + span.SetStatus(codes.Error, err.Error()) + span.RecordError(err) + + return serializeErrorResponse(err) + } + + return nil +} + +// ListBuckets lists all buckets. +func (c *Client) ListBuckets(ctx context.Context, options *common.ListStorageBucketsOptions, predicate func(string) bool) (*common.StorageBucketListResults, error) { + ctx, span := c.startOtelSpan(ctx, "ListBuckets", "") + defer span.End() + + pager := c.client.Buckets(ctx, c.projectID) + pager.Prefix = options.Prefix + + var count int + var results []common.StorageBucket + maxResults := options.MaxResults + pageInfo := common.StoragePaginationInfo{} + + for { + bucket, err := pager.Next() + if err != nil { + if errors.Is(err, iterator.Done) { + break + } + + span.SetStatus(codes.Error, err.Error()) + span.RecordError(err) + + return nil, serializeErrorResponse(err) + } + + var cursor *string + pi := pager.PageInfo() + + if pi.Token != "" { + cursor = &pi.Token + } + + if predicate == nil || predicate(bucket.Name) { + result := serializeBucketInfo(bucket) + results = append(results, result) + count++ + } + + if maxResults > 0 && count >= maxResults { + pageInfo.HasNextPage = pi.Remaining() > 0 + pageInfo.Cursor = cursor + + break + } + } + + span.SetAttributes(attribute.Int("storage.bucket_count", len(results))) + + return &common.StorageBucketListResults{ + Buckets: results, + PageInfo: pageInfo, + }, nil +} + +// GetBucket gets a bucket by name. +func (c *Client) GetBucket(ctx context.Context, name string, options common.BucketOptions) (*common.StorageBucket, error) { + ctx, span := c.startOtelSpan(ctx, "GetBucket", name) + defer span.End() + + bucketInfo, err := c.client.Bucket(name).Attrs(ctx) + if err != nil { + span.SetStatus(codes.Error, err.Error()) + span.RecordError(err) + + return nil, serializeErrorResponse(err) + } + + result := serializeBucketInfo(bucketInfo) + + return &result, nil +} + +// BucketExists checks if a bucket exists. +func (c *Client) BucketExists(ctx context.Context, bucketName string) (bool, error) { + ctx, span := c.startOtelSpan(ctx, "BucketExists", bucketName) + defer span.End() + + result, err := c.client.Bucket(bucketName).Attrs(ctx) + if err != nil { + span.SetStatus(codes.Error, err.Error()) + span.RecordError(err) + + return false, serializeErrorResponse(err) + } + + existed := result != nil + span.SetAttributes(attribute.Bool("storage.bucket_exist", existed)) + + return existed, nil +} + +// UpdateBucket updates configurations for the bucket. +func (c *Client) UpdateBucket(ctx context.Context, bucketName string, opts common.UpdateStorageBucketOptions) error { + ctx, span := c.startOtelSpan(ctx, "UpdateBucket", bucketName) + defer span.End() + + attrs, err := c.client.Bucket(bucketName).Attrs(ctx) + if err != nil { + span.SetStatus(codes.Error, err.Error()) + span.RecordError(err) + + return serializeErrorResponse(err) + } + + if attrs == nil { + return schema.UnprocessableContentError("bucket does not exist", nil) + } + + inputAttrs := storage.BucketAttrsToUpdate{} + + if opts.Tags != nil { + for key, value := range opts.Tags { + span.SetAttributes(attribute.String("storage.bucket_tag"+key, value)) + } + + for key := range attrs.Labels { + if _, ok := opts.Tags[key]; !ok { + inputAttrs.DeleteLabel(key) + } + } + + for key, value := range opts.Tags { + inputAttrs.SetLabel(key, value) + } + } + + if opts.VersioningEnabled != nil { + inputAttrs.VersioningEnabled = *opts.VersioningEnabled + } + + if opts.Encryption != nil { + inputAttrs.Encryption = &storage.BucketEncryption{ + DefaultKMSKeyName: opts.Encryption.KmsMasterKeyID, + } + } + + if opts.Lifecycle != nil { + lc := validateLifecycleConfiguration(*opts.Lifecycle) + inputAttrs.Lifecycle = lc + } + + _, err = c.client.Bucket(bucketName).Update(ctx, inputAttrs) + if err != nil { + span.SetStatus(codes.Error, err.Error()) + span.RecordError(err) + + return serializeErrorResponse(err) + } + + return nil +} + +// RemoveBucket removes a bucket, bucket should be empty to be successfully removed. +func (c *Client) RemoveBucket(ctx context.Context, bucketName string) error { + ctx, span := c.startOtelSpan(ctx, "RemoveBucket", bucketName) + defer span.End() + + err := c.client.Bucket(bucketName).Delete(ctx) + if err != nil { + span.SetStatus(codes.Error, err.Error()) + span.RecordError(err) + + return serializeErrorResponse(err) + } + + return nil +} diff --git a/connector/storage/gcs/client.go b/connector/storage/gcs/client.go new file mode 100644 index 0000000..3b1a6bc --- /dev/null +++ b/connector/storage/gcs/client.go @@ -0,0 +1,86 @@ +package gcs + +import ( + "context" + "fmt" + "log/slog" + "net/url" + + "cloud.google.com/go/storage" + "github.com/hasura/ndc-sdk-go/connector" + "github.com/hasura/ndc-sdk-go/utils" + "github.com/hasura/ndc-storage/connector/storage/common" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" +) + +var tracer = connector.NewTracer("connector/storage/gcs") + +// Client represents a Minio client wrapper. +type Client struct { + publicHost *url.URL + client *storage.Client + projectID string + isDebug bool +} + +var _ common.StorageClient = &Client{} + +// New creates a new Minio client. +func New(ctx context.Context, config *ClientConfig, logger *slog.Logger, version string) (*Client, error) { + publicHost, err := config.ValidatePublicHost() + if err != nil { + return nil, err + } + + projectID, err := config.ProjectID.GetOrDefault("") + if err != nil { + return nil, fmt.Errorf("projectId: %w", err) + } + + if projectID == "" && config.Authentication.Type == AuthTypeCredentials { + return nil, errRequireProjectID + } + + opts, err := config.toClientOptions(ctx, logger, version) + if err != nil { + return nil, err + } + + mc := &Client{ + publicHost: publicHost, + projectID: projectID, + isDebug: utils.IsDebug(logger), + } + + if config.UseGRPC { + mc.client, err = storage.NewGRPCClient(ctx, opts...) + } else { + mc.client, err = storage.NewClient(ctx, opts...) + } + + if err != nil { + return nil, fmt.Errorf("failed to initialize the Google Cloud Storage client: %w", err) + } + + return mc, nil +} + +func (c *Client) startOtelSpan(ctx context.Context, name string, bucketName string) (context.Context, trace.Span) { + spanKind := trace.SpanKindClient + if c.isDebug { + spanKind = trace.SpanKindInternal + } + + ctx, span := tracer.Start(ctx, name, trace.WithSpanKind(spanKind)) + span.SetAttributes( + common.NewDBSystemAttribute(), + attribute.String("rpc.system", "gcs"), + ) + + if bucketName != "" { + span.SetAttributes(attribute.String("storage.bucket", bucketName)) + } + + return ctx, span +} diff --git a/connector/storage/gcs/config.go b/connector/storage/gcs/config.go new file mode 100644 index 0000000..e03f0d9 --- /dev/null +++ b/connector/storage/gcs/config.go @@ -0,0 +1,249 @@ +package gcs + +import ( + "context" + "errors" + "fmt" + "log/slog" + "net/http" + "net/url" + "slices" + "strings" + + "github.com/hasura/ndc-sdk-go/utils" + "github.com/hasura/ndc-storage/connector/storage/common" + "github.com/invopop/jsonschema" + "google.golang.org/api/option" + ghttp "google.golang.org/api/transport/http" +) + +var ( + errRequireCredentials = errors.New("require either credential JSON or file") + errRequireProjectID = errors.New("projectId is required") +) + +// ClientConfig represent the raw configuration of a MinIO client. +type ClientConfig struct { + common.BaseClientConfig `yaml:",inline"` + OtherConfig `yaml:",inline"` +} + +// JSONSchema is used to generate a custom jsonschema. +func (cc ClientConfig) JSONSchema() *jsonschema.Schema { + envStringRef := "#/$defs/EnvString" + + result := cc.BaseClientConfig.GetJSONSchema([]any{common.GoogleStorage}) + result.Required = append(result.Required, "authentication", "projectId") + result.Properties.Set("authentication", cc.Authentication.JSONSchema()) + + result.Properties.Set("publicHost", &jsonschema.Schema{ + Description: "The public host to be used for presigned URL generation", + Ref: envStringRef, + }) + result.Properties.Set("projectId", &jsonschema.Schema{ + Description: "Project ID of the Google Cloud account", + Ref: envStringRef, + }) + + return result +} + +// OtherConfig holds MinIO-specific configurations +type OtherConfig struct { + // Project ID of the Google Cloud account. + ProjectID utils.EnvString `json:"projectId" mapstructure:"projectId" yaml:"projectId"` + // The public host to be used for presigned URL generation. + PublicHost *utils.EnvString `json:"publicHost,omitempty" mapstructure:"publicHost" yaml:"publicHost,omitempty"` + UseGRPC bool `json:"useGrpc,omitempty" mapstructure:"useGrpc" yaml:"useGrpc,omitempty"` + // GRPCConnPoolSize enable the connection pool for gRPC connections that requests will be balanced. + GRPCConnPoolSize int `json:"grpcConnPoolSize,omitempty" mapstructure:"grpcConnPoolSize" yaml:"grpcConnPoolSize,omitempty"` + // Authentication credentials. + Authentication AuthCredentials `json:"authentication" mapstructure:"authentication" yaml:"authentication"` +} + +func (cc ClientConfig) toClientOptions(ctx context.Context, logger *slog.Logger, version string) ([]option.ClientOption, error) { + opts := []option.ClientOption{ + option.WithLogger(logger), + option.WithUserAgent(fmt.Sprintf("hasura/ndc-storage (%s)", version)), + } + + cred, err := cc.Authentication.toCredentials() + if err != nil { + return nil, err + } + + opts = append(opts, cred) + + endpointURL, port, secure, err := cc.BaseClientConfig.ValidateEndpoint() + if err != nil { + return nil, err + } + + if endpointURL != nil { + opts = append(opts, option.WithEndpoint(endpointURL.String())) + } + + if cc.UseGRPC { + if cc.GRPCConnPoolSize > 0 { + opts = append(opts, option.WithGRPCConnectionPool(cc.GRPCConnPoolSize)) + } + } else if utils.IsDebug(logger) { + httpTransport, err := ghttp.NewTransport(ctx, common.NewTransport(logger, port, secure), opts...) + if err != nil { + return nil, err + } + + httpClient := &http.Client{Transport: httpTransport} + opts = append(opts, option.WithHTTPClient(httpClient)) + } + + return opts, nil +} + +// ValidatePublicHost validates the public host setting. +func (cc ClientConfig) ValidatePublicHost() (*url.URL, error) { + if cc.PublicHost == nil { + return nil, nil + } + + publicHost, err := cc.PublicHost.GetOrDefault("") + if err != nil { + return nil, fmt.Errorf("publicHost: %w", err) + } + + if strings.HasPrefix(publicHost, "http") { + result, err := url.Parse(publicHost) + if err != nil { + return nil, fmt.Errorf("publicHost: %w", err) + } + + return result, nil + } + + return &url.URL{ + Host: publicHost, + }, nil +} + +// AuthType represents the authentication type enum. +type AuthType string + +const ( + AuthTypeCredentials AuthType = "credentials" + AuthTypeAnonymous AuthType = "anonymous" +) + +var enumValues_AuthType = []AuthType{ + AuthTypeCredentials, AuthTypeAnonymous, +} + +// ParseAuthType parses the AuthType from string. +func ParseAuthType(input string) (AuthType, error) { + result := AuthType(input) + if !slices.Contains(enumValues_AuthType, result) { + return "", fmt.Errorf("invalid AuthType, expected one of %v, got: %s", enumValues_AuthType, input) + } + + return result, nil +} + +// Validate checks if the provider type is valid. +func (at AuthType) Validate() error { + _, err := ParseAuthType(string(at)) + + return err +} + +// AuthCredentials represent the authentication credentials information. +type AuthCredentials struct { + // The authentication type + Type AuthType `json:"type" mapstructure:"type" yaml:"type"` + // The given service account or refresh token JSON credentials in JSON string format. + Credentials *utils.EnvString `json:"credentials,omitempty" mapstructure:"credentials" yaml:"credentials,omitempty"` + // The given service account or refresh token JSON credentials file. + CredentialsFile *utils.EnvString `json:"credentialsFile,omitempty" mapstructure:"credentialsFile" yaml:"credentialsFile,omitempty"` +} + +// JSONSchema is used to generate a custom jsonschema. +func (ac AuthCredentials) JSONSchema() *jsonschema.Schema { + envStringRefName := "#/$defs/EnvString" + + credProps := jsonschema.NewProperties() + credProps.Set("type", &jsonschema.Schema{ + Type: "string", + Description: "Authorize with a service account or refresh token JSON credentials", + Enum: []any{AuthTypeCredentials}, + }) + credProps.Set("credentials", &jsonschema.Schema{ + Description: "The given service account or refresh token JSON credentials in JSON string format", + Ref: envStringRefName, + }) + credProps.Set("credentialsFile", &jsonschema.Schema{ + Description: "The given service account or refresh token JSON credentials file", + Ref: envStringRefName, + }) + + anonymousProps := jsonschema.NewProperties() + anonymousProps.Set("type", &jsonschema.Schema{ + Type: "string", + Enum: []any{AuthTypeAnonymous}, + }) + + return &jsonschema.Schema{ + OneOf: []*jsonschema.Schema{ + { + Type: "object", + Properties: credProps, + Required: []string{"type"}, + OneOf: []*jsonschema.Schema{ + {Required: []string{"credentials"}}, + {Required: []string{"credentialsFile"}}, + }, + }, + { + Type: "object", + Properties: anonymousProps, + Required: []string{"type"}, + }, + }, + } +} + +func (ac AuthCredentials) toCredentials() (option.ClientOption, error) { + switch ac.Type { + case AuthTypeAnonymous: + return option.WithoutAuthentication(), nil + case AuthTypeCredentials: + return ac.parseServiceAccount() + default: + return nil, fmt.Errorf("unsupported auth type %s", ac.Type) + } +} + +func (ac AuthCredentials) parseServiceAccount() (option.ClientOption, error) { + if ac.Credentials == nil && ac.CredentialsFile == nil { + return nil, errRequireCredentials + } + + if ac.Credentials != nil { + strCred, err := ac.Credentials.GetOrDefault("") + if err != nil { + return nil, err + } + + if strCred != "" { + return option.WithCredentialsJSON([]byte(strCred)), nil + } + } + + credPath, err := ac.CredentialsFile.GetOrDefault("") + if err != nil { + return nil, err + } + + if credPath == "" { + return nil, errRequireCredentials + } + + return option.WithCredentialsFile(credPath), nil +} diff --git a/connector/storage/gcs/object.go b/connector/storage/gcs/object.go new file mode 100644 index 0000000..dd4d8fa --- /dev/null +++ b/connector/storage/gcs/object.go @@ -0,0 +1,548 @@ +package gcs + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "log/slog" + "net/http" + "net/url" + "path/filepath" + "strconv" + "time" + + "cloud.google.com/go/storage" + "github.com/hasura/ndc-sdk-go/scalar" + "github.com/hasura/ndc-sdk-go/schema" + "github.com/hasura/ndc-storage/connector/storage/common" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + "golang.org/x/sync/errgroup" + "google.golang.org/api/iterator" +) + +// ListObjects list objects in a bucket. +func (c *Client) ListObjects(ctx context.Context, bucketName string, opts *common.ListStorageObjectsOptions, predicate func(string) bool) (*common.StorageObjectListResults, error) { + ctx, span := c.startOtelSpan(ctx, "ListObjects", bucketName) + defer span.End() + + var count int + maxResults := opts.MaxResults + objects := make([]common.StorageObject, 0) + q := c.validateListObjectsOptions(span, opts, false) + pager := c.client.Bucket(bucketName).Objects(ctx, q) + pageInfo := common.StoragePaginationInfo{} + + for { + object, err := pager.Next() + if err != nil { + if errors.Is(err, iterator.Done) { + break + } + + span.SetStatus(codes.Error, err.Error()) + span.RecordError(err) + + return nil, serializeErrorResponse(err) + } + + var cursor *string + pi := pager.PageInfo() + + if pi.Token != "" { + cursor = &pi.Token + } + + if predicate == nil || predicate(object.Name) { + result := serializeObjectInfo(object) + objects = append(objects, result) + count++ + } + + if maxResults > 0 && count >= maxResults { + pageInfo.HasNextPage = pi.Remaining() > 0 + pageInfo.Cursor = cursor + + break + } + } + + span.SetAttributes(attribute.Int("storage.object_count", count)) + + results := &common.StorageObjectListResults{ + Objects: objects, + PageInfo: pageInfo, + } + + return results, nil +} + +// ListIncompleteUploads list partially uploaded objects in a bucket. +func (c *Client) ListIncompleteUploads(ctx context.Context, bucketName string, args common.ListIncompleteUploadsOptions) ([]common.StorageObjectMultipartInfo, error) { + return []common.StorageObjectMultipartInfo{}, nil +} + +// RemoveIncompleteUpload removes a partially uploaded object. +func (c *Client) RemoveIncompleteUpload(ctx context.Context, bucketName string, objectName string) error { + return nil +} + +// ListDeletedObjects list deleted objects in a bucket. +func (c *Client) ListDeletedObjects(ctx context.Context, bucketName string, opts *common.ListStorageObjectsOptions, predicate func(string) bool) (*common.StorageObjectListResults, error) { + ctx, span := c.startOtelSpan(ctx, "ListDeletedObjects", bucketName) + defer span.End() + + var count int + maxResults := opts.MaxResults + objects := make([]common.StorageObject, 0) + q := c.validateListObjectsOptions(span, opts, true) + pager := c.client.Bucket(bucketName).Objects(ctx, q) + pageInfo := common.StoragePaginationInfo{} + + for { + object, err := pager.Next() + if err != nil { + if errors.Is(err, iterator.Done) { + break + } + + span.SetStatus(codes.Error, err.Error()) + span.RecordError(err) + + return nil, serializeErrorResponse(err) + } + + var cursor *string + pi := pager.PageInfo() + + if pi.Token != "" { + cursor = &pi.Token + } + + if (object.Deleted.IsZero() && object.SoftDeleteTime.IsZero()) || (predicate != nil && !predicate(object.Name)) { + pageInfo.Cursor = cursor + + continue + } + + result := serializeObjectInfo(object) + objects = append(objects, result) + count++ + + if maxResults > 0 && count >= maxResults { + pageInfo.HasNextPage = pi.Remaining() > 0 + pageInfo.Cursor = cursor + + break + } + } + + span.SetAttributes(attribute.Int("storage.object_count", count)) + + results := &common.StorageObjectListResults{ + Objects: objects, + PageInfo: pageInfo, + } + + return results, nil +} + +// GetObject returns a stream of the object data. Most of the common errors occur when reading the stream. +func (c *Client) GetObject(ctx context.Context, bucketName, objectName string, opts common.GetStorageObjectOptions) (io.ReadCloser, error) { + ctx, span := c.startOtelSpan(ctx, "GetObject", bucketName) + defer span.End() + + span.SetAttributes(attribute.String("storage.key", objectName)) + + object, err := c.client.Bucket(bucketName).Object(objectName).NewReader(ctx) + if err != nil { + span.SetStatus(codes.Error, err.Error()) + span.RecordError(err) + + return nil, serializeErrorResponse(err) + } + + return object, nil +} + +// PutObject uploads objects that are less than 128MiB in a single PUT operation. For objects that are greater than 128MiB in size, +// PutObject seamlessly uploads the object as parts of 128MiB or more depending on the actual file size. The max upload size for an object is 5TB. +func (c *Client) PutObject(ctx context.Context, bucketName string, objectName string, opts *common.PutStorageObjectOptions, reader io.Reader, objectSize int64) (*common.StorageUploadInfo, error) { + ctx, span := c.startOtelSpan(ctx, "PutObject", bucketName) + defer span.End() + + span.SetAttributes( + attribute.String("storage.key", objectName), + attribute.Int64("http.response.body.size", objectSize), + ) + + // estimate the chunk size. If the object size < 16MiB, + // the chunk size will be rounded up to the nearest multiple of 256K + chunkSize := 16 * 1024 * 1024 + if objectSize < int64(chunkSize) { + chunkSize = (int(objectSize/size256K) + 1) * size256K + } + + w := c.client.Bucket(bucketName).Object(objectName).NewWriter(ctx) + w.ChunkSize = chunkSize + w.Metadata = opts.Metadata + w.CacheControl = opts.CacheControl + w.ContentDisposition = opts.ContentDisposition + w.ContentEncoding = opts.ContentEncoding + w.ContentLanguage = opts.ContentLanguage + w.ContentType = opts.ContentType + w.TemporaryHold = opts.LegalHold != nil && *opts.LegalHold + w.StorageClass = opts.StorageClass + + if opts.Retention != nil { + retention := &storage.ObjectRetention{ + Mode: string(opts.Retention.Mode), + RetainUntil: opts.Retention.RetainUntilDate, + } + + w.Retention = retention + } + + _, err := io.Copy(w, reader) + if err != nil { + span.SetStatus(codes.Error, err.Error()) + span.RecordError(err) + + return nil, serializeErrorResponse(err) + } + + if err := w.Close(); err != nil { + span.SetStatus(codes.Error, "failed to close the file stream") + span.RecordError(err) + + return nil, serializeErrorResponse(err) + } + + result := serializeUploadObjectInfo(w) + common.SetUploadInfoAttributes(span, &result) + + return &result, nil +} + +// CopyObject creates or replaces an object through server-side copying of an existing object. +// It supports conditional copying, copying a part of an object and server-side encryption of destination and decryption of source. +// To copy multiple source objects into a single destination object see the ComposeObject API. +func (c *Client) CopyObject(ctx context.Context, dest common.StorageCopyDestOptions, src common.StorageCopySrcOptions) (*common.StorageUploadInfo, error) { + ctx, span := c.startOtelSpan(ctx, "CopyObject", dest.Bucket) + defer span.End() + + span.SetAttributes( + attribute.String("storage.key", dest.Object), + attribute.String("storage.copy_source", src.Object), + ) + + srcHandle := c.client.Bucket(src.Bucket).Object(src.Object) + copier := c.client.Bucket(dest.Bucket).Object(dest.Object).CopierFrom(srcHandle) + + object, err := copier.Run(ctx) + if err != nil { + span.SetStatus(codes.Error, err.Error()) + span.RecordError(err) + + return nil, serializeErrorResponse(err) + } + + result := serializeUploadObjectInfo(&storage.Writer{ + ObjectAttrs: *object, + }) + common.SetUploadInfoAttributes(span, &result) + + return &result, nil +} + +// ComposeObject creates an object by concatenating a list of source objects using server-side copying. +func (c *Client) ComposeObject(ctx context.Context, dest common.StorageCopyDestOptions, sources []common.StorageCopySrcOptions) (*common.StorageUploadInfo, error) { + return nil, errNotSupported +} + +// StatObject fetches metadata of an object. +func (c *Client) StatObject(ctx context.Context, bucketName, objectName string, opts common.GetStorageObjectOptions) (*common.StorageObject, error) { + ctx, span := c.startOtelSpan(ctx, "StatObject", bucketName) + defer span.End() + + span.SetAttributes(attribute.String("storage.key", objectName)) + + object, err := c.client.Bucket(bucketName).Object(objectName).Attrs(ctx) + if err != nil { + if errors.Is(err, storage.ErrObjectNotExist) { + return nil, nil + } + + span.SetStatus(codes.Error, err.Error()) + span.RecordError(err) + + return nil, serializeErrorResponse(err) + } + + result := serializeObjectInfo(object) + + return &result, nil +} + +// RemoveObject removes an object with some specified options. +func (c *Client) RemoveObject(ctx context.Context, bucketName string, objectName string, opts common.RemoveStorageObjectOptions) error { + ctx, span := c.startOtelSpan(ctx, "RemoveObject", bucketName) + defer span.End() + + span.SetAttributes( + attribute.String("storage.key", objectName), + attribute.Bool("storage.options.force_delete", opts.ForceDelete), + attribute.Bool("storage.options.governance_bypass", opts.GovernanceBypass), + ) + + if opts.VersionID != "" { + span.SetAttributes(attribute.String("storage.options.version", opts.VersionID)) + } + + objectClient := c.client.Bucket(bucketName).Object(objectName) + + if opts.SoftDelete { + objectClient = objectClient.SoftDeleted() + } + + err := objectClient.SoftDeleted().Delete(ctx) + if err != nil { + if errors.Is(err, storage.ErrObjectNotExist) { + return nil + } + + span.SetStatus(codes.Error, err.Error()) + span.RecordError(err) + + return serializeErrorResponse(err) + } + + return nil +} + +// RemoveObjects removes a list of objects obtained from an input channel. The call sends a delete request to the server up to 1000 objects at a time. +// The errors observed are sent over the error channel. +func (c *Client) RemoveObjects(ctx context.Context, bucketName string, opts *common.RemoveStorageObjectsOptions, predicate func(string) bool) []common.RemoveStorageObjectError { + ctx, span := c.startOtelSpan(ctx, "RemoveObjects", bucketName) + defer span.End() + + q := c.validateListObjectsOptions(span, &opts.ListStorageObjectsOptions, false) + pager := c.client.Bucket(bucketName).Objects(ctx, q) + objects := []*storage.ObjectAttrs{} + + for { + object, err := pager.Next() + if err != nil { + if errors.Is(err, iterator.Done) { + break + } + + span.SetStatus(codes.Error, err.Error()) + span.RecordError(err) + + return []common.RemoveStorageObjectError{ + { + Error: err, + }, + } + } + + objects = append(objects, object) + } + + span.SetAttributes(attribute.Int("storage.object_count", len(objects))) + + if len(objects) == 0 { + return nil + } + + errs := make([]common.RemoveStorageObjectError, 0) + + if opts.NumThreads <= 1 { + for _, item := range objects { + err := c.client.Bucket(bucketName).Object(item.Name).Delete(ctx) + if err != nil { + errs = append(errs, common.RemoveStorageObjectError{ + ObjectName: item.Name, + VersionID: strconv.Itoa(int(item.Generation)), + Error: err, + }) + } + } + } else { + eg := errgroup.Group{} + eg.SetLimit(opts.NumThreads) + + removeFunc := func(name string) { + eg.Go(func() error { + err := c.client.Bucket(bucketName).Object(name).Delete(ctx) + if err != nil { + errs = append(errs, common.RemoveStorageObjectError{ + ObjectName: name, + Error: err, + }) + } + + return nil + }) + } + + for _, item := range objects { + removeFunc(item.Name) + } + + if err := eg.Wait(); err != nil { + return []common.RemoveStorageObjectError{ + { + Error: err, + }, + } + } + } + + if len(errs) > 0 { + bs, err := json.Marshal(errs) + if err != nil { + slog.Error(err.Error()) + } + + span.SetAttributes(attribute.String("errors", string(bs))) + span.SetStatus(codes.Error, "failed to remove objects") + } + + return errs +} + +// UpdateObject updates object configurations. +func (c *Client) UpdateObject(ctx context.Context, bucketName string, objectName string, opts common.UpdateStorageObjectOptions) error { + ctx, span := c.startOtelSpan(ctx, "UpdateObject", bucketName) + defer span.End() + + span.SetAttributes(attribute.String("storage.key", objectName)) + + handle := c.client.Bucket(bucketName).Object(objectName) + + if opts.VersionID != "" { + span.SetAttributes(attribute.String("storage.options.version", opts.VersionID)) + + gen, err := strconv.ParseInt(opts.VersionID, 10, 64) + if err != nil { + return schema.UnprocessableContentError(fmt.Sprintf("invalid generation version: %s", err), nil) + } + + handle = handle.Generation(gen) + } + + updateAttrs := storage.ObjectAttrsToUpdate{} + + if opts.LegalHold != nil { + updateAttrs.TemporaryHold = *opts.LegalHold + } + + if opts.Retention != nil && opts.Retention.Mode != nil { + updateAttrs.Retention = &storage.ObjectRetention{ + Mode: string(*opts.Retention.Mode), + } + + if opts.Retention.RetainUntilDate != nil { + updateAttrs.Retention.RetainUntil = *opts.Retention.RetainUntilDate + } + } + + if opts.Metadata != nil { + updateAttrs.Metadata = opts.Metadata + } + + _, err := handle.Update(ctx, updateAttrs) + if err != nil { + span.SetStatus(codes.Error, err.Error()) + span.RecordError(err) + + return serializeErrorResponse(err) + } + + return nil +} + +// RestoreObject restores a soft-deleted object. +func (c *Client) RestoreObject(ctx context.Context, bucketName string, objectName string) error { + ctx, span := c.startOtelSpan(ctx, "RestoreObject", bucketName) + defer span.End() + + span.SetAttributes(attribute.String("storage.key", objectName)) + + blobClient := c.client.Bucket(bucketName).Object(objectName) + + _, err := blobClient.Restore(ctx, &storage.RestoreOptions{}) + if err != nil { + span.SetStatus(codes.Error, err.Error()) + span.RecordError(err) + + return serializeErrorResponse(err) + } + + return nil +} + +// PresignedGetObject generates a presigned URL for HTTP GET operations. Browsers/Mobile clients may point to this URL to directly download objects even if the bucket is private. +// This presigned URL can have an associated expiration time in seconds after which it is no longer operational. +// The maximum expiry is 604800 seconds (i.e. 7 days) and minimum is 1 second. +func (c *Client) PresignedGetObject(ctx context.Context, bucketName string, objectName string, opts common.PresignedGetStorageObjectOptions) (string, error) { + return c.presignObject(ctx, http.MethodGet, bucketName, objectName, opts) +} + +func (c *Client) presignObject(ctx context.Context, method string, bucketName string, objectName string, opts common.PresignedGetStorageObjectOptions) (string, error) { + _, span := c.startOtelSpan(ctx, method+" PresignedObject", bucketName) + defer span.End() + + reqParams := url.Values{} + + for key, params := range opts.RequestParams { + for _, param := range params { + reqParams.Add(key, param) + } + } + + span.SetAttributes( + attribute.String("storage.key", objectName), + attribute.String("url.query", reqParams.Encode()), + ) + + if opts.Expiry != nil { + span.SetAttributes(attribute.String("storage.expiry", opts.Expiry.String())) + } + + fileName := filepath.Base(objectName) + // Set request Parameters: for content-disposition. + reqParams.Set("response-content-disposition", fmt.Sprintf(`attachment; filename="%s"`, fileName)) + + options := &storage.SignedURLOptions{ + Method: method, + Expires: time.Now().Add(opts.Expiry.Duration), + QueryParameters: reqParams, + } + + if c.publicHost != nil { + options.Hostname = c.publicHost.Host + } + + result, err := c.client.Bucket(bucketName).SignedURL(objectName, options) + if err != nil { + span.SetStatus(codes.Error, err.Error()) + span.RecordError(err) + + return "", serializeErrorResponse(err) + } + + return result, nil +} + +// PresignedPutObject generates a presigned URL for HTTP PUT operations. Browsers/Mobile clients may point to this URL to upload objects directly to a bucket even if it is private. +// This presigned URL can have an associated expiration time in seconds after which it is no longer operational. The default expiry is set to 7 days. +func (c *Client) PresignedPutObject(ctx context.Context, bucketName string, objectName string, expiry time.Duration) (string, error) { + return c.presignObject(ctx, http.MethodPut, bucketName, objectName, common.PresignedGetStorageObjectOptions{ + Expiry: &scalar.Duration{Duration: expiry}, + }) +} diff --git a/connector/storage/gcs/types.go b/connector/storage/gcs/types.go new file mode 100644 index 0000000..af8adaa --- /dev/null +++ b/connector/storage/gcs/types.go @@ -0,0 +1,39 @@ +package gcs + +import "cloud.google.com/go/storage" + +// ACLRule represents a grant for a role to an entity (user, group or team) for a +// Google Cloud Storage object or bucket. +type ACLRule struct { + Entity storage.ACLEntity `json:"entity,omitempty"` + EntityID string `json:"entityId,omitempty"` + Role storage.ACLRole `json:"role,omitempty"` + Domain string `json:"domain,omitempty"` + Email string `json:"email,omitempty"` + ProjectTeam *ProjectTeam `json:"projectTeam,omitempty"` +} + +// ProjectTeam is the project team associated with the entity, if any. +type ProjectTeam struct { + ProjectNumber string `json:"projectNumber,omitempty"` + Team string `json:"team,omitempty"` +} + +func makeACLRule(acl storage.ACLRule) ACLRule { + rule := ACLRule{ + Entity: acl.Entity, + EntityID: acl.EntityID, + Role: acl.Role, + Domain: acl.Domain, + Email: acl.Email, + } + + if acl.ProjectTeam != nil { + rule.ProjectTeam = &ProjectTeam{ + ProjectNumber: acl.ProjectTeam.ProjectNumber, + Team: acl.ProjectTeam.Team, + } + } + + return rule +} diff --git a/connector/storage/gcs/utils.go b/connector/storage/gcs/utils.go new file mode 100644 index 0000000..1e5b220 --- /dev/null +++ b/connector/storage/gcs/utils.go @@ -0,0 +1,466 @@ +package gcs + +import ( + "encoding/base64" + "errors" + "math" + "strconv" + "time" + + "cloud.google.com/go/storage" + "github.com/hasura/ndc-sdk-go/scalar" + "github.com/hasura/ndc-sdk-go/schema" + "github.com/hasura/ndc-storage/connector/storage/common" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" + "google.golang.org/api/googleapi" +) + +const ( + size256K = 256 * 1024 +) + +var errNotSupported = schema.NotSupportedError("Google Cloud Storage doesn't support this method", nil) + +func serializeBucketInfo(bucket *storage.BucketAttrs) common.StorageBucket { + result := common.StorageBucket{ + Name: bucket.Name, + Tags: bucket.Labels, + CORS: make([]common.BucketCors, len(bucket.CORS)), + CreationTime: &bucket.Created, + LastModified: &bucket.Updated, + DefaultEventBasedHold: &bucket.DefaultEventBasedHold, + RequesterPays: &bucket.RequesterPays, + StorageClass: &bucket.StorageClass, + ObjectLock: serializeRetentionPolicy(bucket.RetentionPolicy), + Lifecycle: serializeLifecycleConfiguration(bucket.Lifecycle), + Versioning: &common.StorageBucketVersioningConfiguration{ + Enabled: bucket.VersioningEnabled, + }, + } + + if bucket.Location != "" { + result.Region = &bucket.Location + } + + if bucket.LocationType != "" { + result.LocationType = &bucket.LocationType + } + + if bucket.Etag != "" { + result.Etag = &bucket.Etag + } + + if bucket.Autoclass != nil { + result.Autoclass = &common.BucketAutoclass{ + Enabled: bucket.Autoclass.Enabled, + ToggleTime: bucket.Autoclass.ToggleTime, + TerminalStorageClass: bucket.Autoclass.TerminalStorageClass, + TerminalStorageClassUpdateTime: bucket.Autoclass.TerminalStorageClassUpdateTime, + } + } + + for i, cors := range bucket.CORS { + result.CORS[i] = common.BucketCors{ + MaxAge: scalar.NewDuration(cors.MaxAge), + Methods: cors.Methods, + Origins: cors.Origins, + ResponseHeaders: cors.ResponseHeaders, + } + } + + if bucket.CustomPlacementConfig != nil { + result.CustomPlacementConfig = &common.CustomPlacementConfig{ + DataLocations: bucket.CustomPlacementConfig.DataLocations, + } + } + + if bucket.HierarchicalNamespace != nil { + result.HierarchicalNamespace = &common.BucketHierarchicalNamespace{ + Enabled: bucket.HierarchicalNamespace.Enabled, + } + } + + if bucket.Logging != nil { + result.Logging = &common.BucketLogging{ + LogBucket: bucket.Logging.LogBucket, + LogObjectPrefix: bucket.Logging.LogObjectPrefix, + } + } + + if bucket.RPO != storage.RPOUnknown { + rpo := bucket.RPO.String() + result.RPO = (*common.GoogleStorageRPO)(&rpo) + } + + if bucket.SoftDeletePolicy != nil { + result.SoftDeletePolicy = &common.StorageObjectSoftDeletePolicy{ + EffectiveTime: bucket.SoftDeletePolicy.EffectiveTime, + RetentionDuration: scalar.NewDuration(bucket.SoftDeletePolicy.RetentionDuration), + } + } + + if bucket.Website != nil { + result.Website = &common.BucketWebsite{ + MainPageSuffix: bucket.Website.MainPageSuffix, + NotFoundPage: &bucket.Website.NotFoundPage, + } + + if bucket.Website.NotFoundPage != "" { + result.Website.NotFoundPage = &bucket.Website.NotFoundPage + } + } + + if bucket.Encryption != nil && bucket.Encryption.DefaultKMSKeyName != "" { + result.Encryption = &common.ServerSideEncryptionConfiguration{ + KmsMasterKeyID: bucket.Encryption.DefaultKMSKeyName, + } + } + + return result +} + +func serializeRetentionPolicy(retentionPolicy *storage.RetentionPolicy) *common.StorageObjectLockConfig { + if retentionPolicy == nil { + return nil + } + + unit := common.StorageRetentionValidityUnitDays + validity := uint(math.Ceil(retentionPolicy.RetentionPeriod.Hours())) + mode := common.StorageRetentionModeUnlocked + + if retentionPolicy.IsLocked { + mode = common.StorageRetentionModeLocked + } + + return &common.StorageObjectLockConfig{ + Enabled: true, + SetStorageObjectLockConfig: common.SetStorageObjectLockConfig{ + Mode: &mode, + Validity: &validity, + Unit: &unit, + }, + } +} + +func serializeObjectInfo(obj *storage.ObjectAttrs) common.StorageObject { + object := common.StorageObject{ + Bucket: obj.Bucket, + Name: obj.Name, + CreationTime: &obj.Created, + LastModified: obj.Updated, + Size: &obj.Size, + Metadata: obj.Metadata, + StorageClass: &obj.StorageClass, + LegalHold: &obj.TemporaryHold, + } + + if obj.Etag != "" { + object.ETag = &obj.Etag + } + + if obj.ContentType != "" { + object.ContentType = &obj.ContentType + } + + if obj.CacheControl != "" { + object.CacheControl = &obj.CacheControl + } + + if obj.ContentDisposition != "" { + object.ContentDisposition = &obj.ContentDisposition + } + + if obj.ContentEncoding != "" { + object.ContentEncoding = &obj.ContentEncoding + } + + if obj.ContentLanguage != "" { + object.ContentLanguage = &obj.ContentLanguage + } + + if obj.CustomerKeySHA256 != "" { + object.CustomerProvidedKeySHA256 = &obj.CustomerKeySHA256 + } + + if obj.KMSKeyName != "" { + object.KMSKeyName = &obj.KMSKeyName + } + + if obj.MediaLink != "" { + object.MediaLink = &obj.MediaLink + } + + if obj.Owner != "" { + object.Owner = &common.StorageOwner{ + DisplayName: &obj.Owner, + } + } + + if obj.Retention != nil { + if obj.Retention.Mode != "" { + object.RetentionMode = &obj.Retention.Mode + } + + if !obj.Retention.RetainUntil.IsZero() { + object.RetentionUntilDate = &obj.Retention.RetainUntil + } + } + + if !obj.RetentionExpirationTime.IsZero() { + object.Expiration = &obj.RetentionExpirationTime + } + + if !obj.Deleted.IsZero() { + deleted := true + object.Deleted = &deleted + object.DeletedTime = &obj.Deleted + } + + if obj.Generation > 0 { + versionID := strconv.Itoa(int(obj.Generation)) + object.VersionID = &versionID + } + + if len(obj.MD5) > 0 { + contentMd5 := base64.StdEncoding.EncodeToString(obj.MD5) + object.ContentMD5 = &contentMd5 + } + + aclRules := make([]ACLRule, len(obj.ACL)) + + for i, acl := range obj.ACL { + aclRules[i] = makeACLRule(acl) + } + + object.ACL = aclRules + + return object +} + +func (c *Client) validateListObjectsOptions(span trace.Span, opts *common.ListStorageObjectsOptions, includeDeleted bool) *storage.Query { + span.SetAttributes( + attribute.Bool("storage.options.recursive", opts.Recursive), + attribute.Bool("storage.options.with_deleted", includeDeleted), + attribute.Bool("storage.options.with_versions", opts.Include.Versions), + ) + + if opts.Prefix != "" { + span.SetAttributes(attribute.String("storage.options.prefix", opts.Prefix)) + } + + if opts.StartAfter != "" { + span.SetAttributes(attribute.String("storage.options.start_after", opts.StartAfter)) + } + + if opts.MaxResults > 0 { + span.SetAttributes(attribute.Int("storage.options.max_results", opts.MaxResults)) + } + + return &storage.Query{ + Versions: opts.Include.Versions, + Prefix: opts.Prefix, + StartOffset: opts.StartAfter, + SoftDeleted: includeDeleted, + } +} + +func serializeUploadObjectInfo(obj *storage.Writer) common.StorageUploadInfo { + object := common.StorageUploadInfo{ + Bucket: obj.Bucket, + Name: obj.Name, + } + + if obj.Etag != "" { + object.ETag = &obj.Etag + } + + if obj.Size > 0 { + object.Size = &obj.Size + } + + if !obj.Updated.IsZero() { + object.LastModified = &obj.Updated + } else if !obj.Created.IsZero() { + object.LastModified = &obj.Created + } + + if !obj.RetentionExpirationTime.IsZero() { + object.Expiration = &obj.RetentionExpirationTime + } + + versionID := strconv.Itoa(int(obj.Generation)) + object.VersionID = &versionID + + if len(obj.MD5) > 0 { + contentMd5 := base64.StdEncoding.EncodeToString(obj.MD5) + object.ContentMD5 = &contentMd5 + } + + return object +} + +func validateLifecycleRule(rule common.ObjectLifecycleRule) storage.LifecycleRule { + r := storage.LifecycleRule{} + + for _, filter := range rule.RuleFilter { + r.Condition.MatchesPrefix = append(r.Condition.MatchesPrefix, filter.MatchesPrefix...) + r.Condition.MatchesSuffix = append(r.Condition.MatchesSuffix, filter.MatchesSuffix...) + r.Condition.MatchesStorageClasses = append(r.Condition.MatchesStorageClasses, filter.MatchesStorageClasses...) + } + + if rule.NoncurrentVersionExpiration != nil { + if rule.NoncurrentVersionExpiration.NewerNoncurrentVersions != nil { + r.Condition.NumNewerVersions = int64(*rule.NoncurrentVersionExpiration.NewerNoncurrentVersions) + } + + if rule.NoncurrentVersionExpiration.NoncurrentDays != nil { + r.Condition.DaysSinceNoncurrentTime = int64(*rule.NoncurrentVersionExpiration.NoncurrentDays) + } + + r.Action.Type = storage.DeleteAction + } + + if rule.NoncurrentVersionTransition != nil { + if rule.NoncurrentVersionTransition.NewerNoncurrentVersions != nil { + r.Condition.NumNewerVersions = int64(*rule.NoncurrentVersionTransition.NewerNoncurrentVersions) + } + + if rule.NoncurrentVersionTransition.NoncurrentDays != nil { + r.Condition.DaysSinceNoncurrentTime = int64(*rule.NoncurrentVersionTransition.NoncurrentDays) + } + + if rule.NoncurrentVersionTransition.StorageClass != nil { + r.Action.StorageClass = *rule.NoncurrentVersionTransition.StorageClass + } + + r.Action.Type = storage.SetStorageClassAction + } + + if rule.Expiration != nil && rule.Expiration.Days != nil { + r.Condition.AgeInDays = int64(*rule.Expiration.Days) + r.Condition.AllObjects = *rule.Expiration.Days == 0 + + r.Action.Type = storage.DeleteAction + + return r + } + + if rule.Transition != nil || rule.Transition.StorageClass != nil { + if rule.Transition.Days != nil { + r.Condition.AgeInDays = int64(*rule.Transition.Days) + } else if rule.Transition.Date != nil { + r.Condition.AgeInDays = int64(time.Since(rule.Expiration.Date.Time).Hours() / 24) + } + + r.Condition.AllObjects = r.Condition.AgeInDays == 0 + r.Action.StorageClass = *rule.Transition.StorageClass + r.Action.Type = storage.SetStorageClassAction + + return r + } + + if rule.AbortIncompleteMultipartUpload != nil && rule.AbortIncompleteMultipartUpload.DaysAfterInitiation != nil { + r.Action.Type = storage.AbortIncompleteMPUAction + r.Condition.AgeInDays = int64(*rule.AbortIncompleteMultipartUpload.DaysAfterInitiation) + r.Condition.AllObjects = r.Condition.AgeInDays == 0 + + return r + } + + if rule.AllVersionsExpiration != nil && rule.AllVersionsExpiration.Days != nil { + r.Condition.AgeInDays = int64(*rule.AllVersionsExpiration.Days) + r.Condition.AllObjects = r.Condition.AgeInDays == 0 + r.Action.Type = storage.DeleteAction + + return r + } + + return r +} + +func validateLifecycleConfiguration(input common.ObjectLifecycleConfiguration) *storage.Lifecycle { + result := &storage.Lifecycle{ + Rules: make([]storage.LifecycleRule, len(input.Rules)), + } + + for i, rule := range input.Rules { + if !rule.Enabled { + continue + } + + r := validateLifecycleRule(rule) + result.Rules[i] = r + } + + return result +} + +func serializeLifecycleConfiguration(input storage.Lifecycle) *common.ObjectLifecycleConfiguration { + result := &common.ObjectLifecycleConfiguration{ + Rules: make([]common.ObjectLifecycleRule, len(input.Rules)), + } + + for i, rule := range input.Rules { + r := serializeLifecycleRule(rule) + result.Rules[i] = r + } + + return result +} + +func serializeLifecycleRule(rule storage.LifecycleRule) common.ObjectLifecycleRule { + r := common.ObjectLifecycleRule{} + + if len(rule.Condition.MatchesPrefix) > 0 || len(rule.Condition.MatchesSuffix) > 0 || len(rule.Condition.MatchesStorageClasses) > 0 { + r.RuleFilter = []common.ObjectLifecycleFilter{ + { + MatchesPrefix: rule.Condition.MatchesPrefix, + MatchesSuffix: rule.Condition.MatchesSuffix, + MatchesStorageClasses: rule.Condition.MatchesStorageClasses, + }, + } + } + + ageInDays := int(rule.Condition.AgeInDays) + + switch rule.Action.Type { + case storage.SetStorageClassAction: + r.Transition = &common.ObjectLifecycleTransition{ + Days: &ageInDays, + StorageClass: &rule.Action.StorageClass, + } + case storage.DeleteAction: + r.Expiration = &common.ObjectLifecycleExpiration{ + Days: &ageInDays, + } + case storage.AbortIncompleteMPUAction: + r.AbortIncompleteMultipartUpload = &common.ObjectAbortIncompleteMultipartUpload{ + DaysAfterInitiation: &ageInDays, + } + } + + return r +} + +func evalGoogleErrorResponse(err *googleapi.Error) *schema.ConnectorError { + details := map[string]any{ + "statusCode": err.Code, + "details": err.Details, + } + + if err.Code >= 500 { + return schema.NewConnectorError(err.Code, err.Message, details) + } + + return schema.UnprocessableContentError(err.Message, details) +} + +func serializeErrorResponse(err error) *schema.ConnectorError { + var e *googleapi.Error + if ok := errors.As(err, &e); ok { + return evalGoogleErrorResponse(e) + } + + return schema.UnprocessableContentError(err.Error(), nil) +} diff --git a/connector/storage/manager.go b/connector/storage/manager.go index ae2d80f..165e6bd 100644 --- a/connector/storage/manager.go +++ b/connector/storage/manager.go @@ -18,8 +18,8 @@ type Manager struct { clients []Client } -// NewManager creates a storage client manager instace. -func NewManager(ctx context.Context, configs []ClientConfig, logger *slog.Logger) (*Manager, error) { +// NewManager creates a storage client manager instance. +func NewManager(ctx context.Context, configs []ClientConfig, logger *slog.Logger, version string) (*Manager, error) { if len(configs) == 0 { return nil, errors.New("failed to initialize storage clients: config is empty") } @@ -29,7 +29,7 @@ func NewManager(ctx context.Context, configs []ClientConfig, logger *slog.Logger } for i, config := range configs { - baseConfig, client, err := config.toStorageClient(ctx, logger) + baseConfig, client, err := config.toStorageClient(ctx, logger, version) if err != nil { return nil, fmt.Errorf("failed to initialize storage client %d: %w", i, err) } diff --git a/connector/storage/minio/bucket.go b/connector/storage/minio/bucket.go index 95c6dfc..388fc8a 100644 --- a/connector/storage/minio/bucket.go +++ b/connector/storage/minio/bucket.go @@ -4,6 +4,7 @@ import ( "context" "errors" "net/http" + "strings" "github.com/hasura/ndc-sdk-go/schema" "github.com/hasura/ndc-storage/connector/storage/common" @@ -12,6 +13,7 @@ import ( "github.com/minio/minio-go/v7/pkg/tags" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/trace" "golang.org/x/sync/errgroup" ) @@ -22,7 +24,7 @@ func (mc *Client) MakeBucket(ctx context.Context, args *common.MakeStorageBucket err := mc.client.MakeBucket(ctx, args.Name, minio.MakeBucketOptions{ Region: args.Region, - ObjectLocking: args.ObjectLocking, + ObjectLocking: args.ObjectLock, }) if err != nil { span.SetStatus(codes.Error, err.Error()) @@ -44,7 +46,7 @@ func (mc *Client) MakeBucket(ctx context.Context, args *common.MakeStorageBucket } // ListBuckets lists all buckets. -func (mc *Client) ListBuckets(ctx context.Context, options common.BucketOptions) ([]common.StorageBucketInfo, error) { +func (mc *Client) ListBuckets(ctx context.Context, options *common.ListStorageBucketsOptions, predicate func(string) bool) (*common.StorageBucketListResults, error) { ctx, span := mc.startOtelSpan(ctx, "ListBuckets", "") defer span.End() @@ -56,12 +58,37 @@ func (mc *Client) ListBuckets(ctx context.Context, options common.BucketOptions) return nil, serializeErrorResponse(err) } + var filteredBuckets []minio.BucketInfo + + if options.Prefix == "" && predicate == nil { + filteredBuckets = bucketInfos + } else { + for _, info := range bucketInfos { + if (options.Prefix != "" && !strings.HasPrefix(info.Name, options.Prefix)) || + (predicate != nil && !predicate(info.Name)) { + continue + } + + filteredBuckets = append(filteredBuckets, info) + } + } + span.SetAttributes(attribute.Int("storage.bucket_count", len(bucketInfos))) - results := make([]common.StorageBucketInfo, len(bucketInfos)) + + if len(bucketInfos) == 0 { + return &common.StorageBucketListResults{ + Buckets: []common.StorageBucket{}, + }, nil + } + + results := make([]common.StorageBucket, len(filteredBuckets)) if options.NumThreads <= 1 { - for i, item := range bucketInfos { - bucket, err := mc.populateBucket(ctx, item, options) + for i, item := range filteredBuckets { + bucket, err := mc.populateBucket(ctx, item, common.BucketOptions{ + NumThreads: options.NumThreads, + Include: options.Include, + }) if err != nil { span.SetStatus(codes.Error, err.Error()) span.RecordError(err) @@ -72,7 +99,9 @@ func (mc *Client) ListBuckets(ctx context.Context, options common.BucketOptions) results[i] = bucket } - return results, nil + return &common.StorageBucketListResults{ + Buckets: results, + }, nil } eg := errgroup.Group{} @@ -80,7 +109,10 @@ func (mc *Client) ListBuckets(ctx context.Context, options common.BucketOptions) populateFunc := func(item minio.BucketInfo, index int) { eg.Go(func() error { - bucket, err := mc.populateBucket(ctx, item, options) + bucket, err := mc.populateBucket(ctx, item, common.BucketOptions{ + NumThreads: options.NumThreads, + Include: options.Include, + }) if err != nil { return err } @@ -91,7 +123,7 @@ func (mc *Client) ListBuckets(ctx context.Context, options common.BucketOptions) }) } - for i, item := range bucketInfos { + for i, item := range filteredBuckets { populateFunc(item, i) } @@ -102,11 +134,13 @@ func (mc *Client) ListBuckets(ctx context.Context, options common.BucketOptions) return nil, err } - return results, nil + return &common.StorageBucketListResults{ + Buckets: results, + }, nil } // GetBucket gets a bucket by name. -func (mc *Client) GetBucket(ctx context.Context, name string, options common.BucketOptions) (*common.StorageBucketInfo, error) { +func (mc *Client) GetBucket(ctx context.Context, name string, options common.BucketOptions) (*common.StorageBucket, error) { ctx, span := mc.startOtelSpan(ctx, "GetBucket", "") defer span.End() @@ -171,6 +205,50 @@ func (mc *Client) RemoveBucket(ctx context.Context, bucketName string) error { return nil } +// UpdateBucket updates configurations for the bucket. +func (mc *Client) UpdateBucket(ctx context.Context, bucketName string, opts common.UpdateStorageBucketOptions) error { + ctx, span := mc.startOtelSpanWithKind(ctx, trace.SpanKindInternal, "UpdateBucket", bucketName) + defer span.End() + + if opts.Tags != nil { + if err := mc.SetBucketTagging(ctx, bucketName, opts.Tags); err != nil { + return err + } + } + + if opts.VersioningEnabled != nil { + if *opts.VersioningEnabled { + if err := mc.EnableVersioning(ctx, bucketName); err != nil { + return err + } + } else if err := mc.SuspendVersioning(ctx, bucketName); err != nil { + return err + } + } + + if opts.Lifecycle != nil { + if err := mc.SetBucketLifecycle(ctx, bucketName, *opts.Lifecycle); err != nil { + return err + } + } + + if opts.ObjectLock != nil { + if err := mc.SetObjectLockConfig(ctx, bucketName, *opts.ObjectLock); err != nil { + return err + } + } + + if opts.Encryption == nil { + return nil + } + + if opts.Encryption.IsEmpty() { + return mc.RemoveBucketEncryption(ctx, bucketName) + } + + return mc.SetBucketEncryption(ctx, bucketName, *opts.Encryption) +} + // GetBucketTagging gets tags of a bucket. func (mc *Client) GetBucketTagging(ctx context.Context, bucketName string) (map[string]string, error) { ctx, span := mc.startOtelSpan(ctx, "GetBucketTagging", bucketName) @@ -337,14 +415,11 @@ func (mc *Client) GetBucketVersioning(ctx context.Context, bucketName string) (* } result := &common.StorageBucketVersioningConfiguration{ + Enabled: rawResult.Enabled(), ExcludedPrefixes: make([]string, len(rawResult.ExcludedPrefixes)), ExcludeFolders: &rawResult.ExcludeFolders, } - if rawResult.Status != "" { - result.Status = &rawResult.Status - } - if rawResult.MFADelete != "" { result.MFADelete = &rawResult.MFADelete } @@ -439,6 +514,59 @@ func (mc *Client) RemoveBucketReplication(ctx context.Context, bucketName string return nil } +// SetBucketEncryption sets default encryption configuration on a bucket. +func (mc *Client) SetBucketEncryption(ctx context.Context, bucketName string, input common.ServerSideEncryptionConfiguration) error { + ctx, span := mc.startOtelSpan(ctx, "SetBucketEncryption", bucketName) + defer span.End() + + err := mc.client.SetBucketEncryption(ctx, bucketName, validateBucketEncryptionConfiguration(input)) + if err != nil { + span.SetStatus(codes.Error, err.Error()) + span.RecordError(err) + + return serializeErrorResponse(err) + } + + return nil +} + +// GetBucketEncryption gets default encryption configuration set on a bucket. +func (mc *Client) GetBucketEncryption(ctx context.Context, bucketName string) (*common.ServerSideEncryptionConfiguration, error) { + ctx, span := mc.startOtelSpan(ctx, "GetBucketEncryption", bucketName) + defer span.End() + + rawResult, err := mc.client.GetBucketEncryption(ctx, bucketName) + if err != nil { + respError := evalNotFoundError(err, "ServerSideEncryptionConfigurationNotFoundError") + if respError == nil { + return nil, nil + } + + span.SetStatus(codes.Error, err.Error()) + span.RecordError(err) + + return nil, respError + } + + return serializeBucketEncryptionConfiguration(rawResult), nil +} + +// RemoveBucketEncryption remove default encryption configuration set on a bucket. +func (mc *Client) RemoveBucketEncryption(ctx context.Context, bucketName string) error { + ctx, span := mc.startOtelSpan(ctx, "RemoveBucketEncryption", bucketName) + defer span.End() + + err := mc.client.RemoveBucketEncryption(ctx, bucketName) + if err != nil { + span.SetStatus(codes.Error, err.Error()) + span.RecordError(err) + + return serializeErrorResponse(err) + } + + return nil +} + func validateBucketReplicationConfig(input common.StorageReplicationConfig) replication.Config { result := replication.Config{ Rules: make([]replication.Rule, len(input.Rules)), @@ -502,13 +630,12 @@ func validateBucketReplicationFilter(input common.StorageReplicationFilter) repl result.Prefix = *input.Prefix } - if input.Tag != nil { - if input.Tag.Key != nil { - result.Tag.Key = *input.Tag.Key - } + if len(input.Tag) > 0 { + for key, value := range input.Tag { + result.Tag.Key = key + result.Tag.Value = value - if input.Tag.Value != nil { - result.Tag.Value = *input.Tag.Value + break } } @@ -517,19 +644,13 @@ func validateBucketReplicationFilter(input common.StorageReplicationFilter) repl result.And.Prefix = *input.Prefix } - result.And.Tags = make([]replication.Tag, len(input.And.Tags)) - - for i, tag := range input.And.Tags { - t := replication.Tag{} - if tag.Key != nil { - t.Key = *tag.Key + for key, value := range input.And.Tags { + t := replication.Tag{ + Key: key, + Value: value, } - if tag.Value != nil { - t.Value = *tag.Value - } - - result.And.Tags[i] = t + result.And.Tags = append(result.And.Tags, t) } } @@ -595,14 +716,8 @@ func serializeBucketReplicationRule(item replication.Rule) common.StorageReplica } if item.Filter.Tag.Key != "" || item.Filter.Tag.Value != "" { - rule.Filter.Tag = &common.StorageTag{} - - if item.Filter.Tag.Key != "" { - rule.Filter.Tag.Key = &item.Filter.Tag.Key - } - - if item.Filter.Tag.Value != "" { - rule.Filter.Tag.Value = &item.Filter.Tag.Value + rule.Filter.Tag = map[string]string{ + item.Filter.Tag.Key: item.Filter.Tag.Value, } } @@ -612,32 +727,23 @@ func serializeBucketReplicationRule(item replication.Rule) common.StorageReplica rule.Filter.And.Prefix = &item.Filter.Prefix } - rule.Filter.And.Tags = make([]common.StorageTag, len(item.Filter.And.Tags)) - - for i, tag := range item.Filter.And.Tags { - t := common.StorageTag{} - if tag.Key != "" { - t.Key = &tag.Key - } - - if tag.Value != "" { - t.Value = &tag.Value - } + rule.Filter.And.Tags = make(map[string]string) - rule.Filter.And.Tags[i] = t + for _, tag := range item.Filter.And.Tags { + rule.Filter.And.Tags[tag.Key] = tag.Value } } return rule } -func (mc *Client) populateBucket(ctx context.Context, item minio.BucketInfo, options common.BucketOptions) (common.StorageBucketInfo, error) { - bucket := common.StorageBucketInfo{ +func (mc *Client) populateBucket(ctx context.Context, item minio.BucketInfo, options common.BucketOptions) (common.StorageBucket, error) { + bucket := common.StorageBucket{ Name: item.Name, - CreationDate: item.CreationDate, + CreationTime: &item.CreationDate, } - if options.IncludeTags { + if options.Include.Tags { tags, err := mc.GetBucketTagging(ctx, item.Name) if err != nil { return bucket, err @@ -646,5 +752,41 @@ func (mc *Client) populateBucket(ctx context.Context, item minio.BucketInfo, opt bucket.Tags = tags } + if options.Include.Versioning { + versioning, err := mc.GetBucketVersioning(ctx, bucket.Name) + if err != nil { + return bucket, err + } + + bucket.Versioning = versioning + } + + if options.Include.Lifecycle { + lc, err := mc.GetBucketLifecycle(ctx, bucket.Name) + if err != nil { + return bucket, err + } + + bucket.Lifecycle = lc + } + + if options.Include.Encryption { + encryption, err := mc.GetBucketEncryption(ctx, bucket.Name) + if err != nil { + return bucket, err + } + + bucket.Encryption = encryption + } + + if options.Include.ObjectLock { + lock, err := mc.GetObjectLockConfig(ctx, bucket.Name) + if err != nil { + return bucket, err + } + + bucket.ObjectLock = lock + } + return bucket, nil } diff --git a/connector/storage/minio/client.go b/connector/storage/minio/client.go index 0cf0288..0ffa5ab 100644 --- a/connector/storage/minio/client.go +++ b/connector/storage/minio/client.go @@ -16,7 +16,7 @@ import ( var tracer = connector.NewTracer("connector/storage/minio") -// Client prepresents a Minio client wrapper. +// Client represents a Minio client wrapper. type Client struct { publicHost *url.URL providerType common.StorageProviderType @@ -60,6 +60,10 @@ func (mc *Client) startOtelSpan(ctx context.Context, name string, bucketName str spanKind = trace.SpanKindInternal } + return mc.startOtelSpanWithKind(ctx, spanKind, name, bucketName) +} + +func (mc *Client) startOtelSpanWithKind(ctx context.Context, spanKind trace.SpanKind, name string, bucketName string) (context.Context, trace.Span) { ctx, span := tracer.Start(ctx, name, trace.WithSpanKind(spanKind)) span.SetAttributes( common.NewDBSystemAttribute(), diff --git a/connector/storage/minio/config.go b/connector/storage/minio/config.go index d2e4271..9cb4c38 100644 --- a/connector/storage/minio/config.go +++ b/connector/storage/minio/config.go @@ -31,7 +31,7 @@ type ClientConfig struct { func (cc ClientConfig) JSONSchema() *jsonschema.Schema { envStringRef := "#/$defs/EnvString" - result := cc.BaseClientConfig.GetJSONSchema([]any{common.S3, common.GoogleStorage}) + result := cc.BaseClientConfig.GetJSONSchema([]any{common.S3}) result.Required = append(result.Required, "authentication") result.Properties.Set("region", &jsonschema.Schema{ @@ -41,6 +41,10 @@ func (cc ClientConfig) JSONSchema() *jsonschema.Schema { {Ref: envStringRef}, }, }) + result.Properties.Set("publicHost", &jsonschema.Schema{ + Description: "The public host to be used for presigned URL generation", + Ref: envStringRef, + }) result.Properties.Set("authentication", cc.Authentication.JSONSchema()) result.Properties.Set("trailingHeaders", &jsonschema.Schema{ Description: "TrailingHeaders indicates server support of trailing headers. Only supported for v4 signatures", @@ -173,7 +177,7 @@ func (at AuthType) Validate() error { return err } -// AuthCredentials represent the authentication credentials infomartion. +// AuthCredentials represent the authentication credentials information. type AuthCredentials struct { // The authentication type Type AuthType `json:"type" mapstructure:"type" yaml:"type"` diff --git a/connector/storage/minio/lifecycle.go b/connector/storage/minio/lifecycle.go index dcf7924..5dafaf9 100644 --- a/connector/storage/minio/lifecycle.go +++ b/connector/storage/minio/lifecycle.go @@ -10,7 +10,7 @@ import ( ) // SetBucketLifecycle sets lifecycle on bucket or an object prefix. -func (mc *Client) SetBucketLifecycle(ctx context.Context, bucketName string, config common.BucketLifecycleConfiguration) error { +func (mc *Client) SetBucketLifecycle(ctx context.Context, bucketName string, config common.ObjectLifecycleConfiguration) error { ctx, span := mc.startOtelSpan(ctx, "SetBucketLifecycle", bucketName) defer span.End() @@ -28,16 +28,21 @@ func (mc *Client) SetBucketLifecycle(ctx context.Context, bucketName string, con } // GetBucketLifecycle gets lifecycle on a bucket or a prefix. -func (mc *Client) GetBucketLifecycle(ctx context.Context, bucketName string) (*common.BucketLifecycleConfiguration, error) { +func (mc *Client) GetBucketLifecycle(ctx context.Context, bucketName string) (*common.ObjectLifecycleConfiguration, error) { ctx, span := mc.startOtelSpan(ctx, "GetBucketLifecycle", bucketName) defer span.End() rawResult, err := mc.client.GetBucketLifecycle(ctx, bucketName) if err != nil { + respError := evalNotFoundError(err, "NoSuchLifecycleConfiguration") + if respError == nil { + return nil, nil + } + span.SetStatus(codes.Error, err.Error()) span.RecordError(err) - return nil, serializeErrorResponse(err) + return nil, respError } result := serializeLifecycleConfiguration(*rawResult) @@ -45,14 +50,19 @@ func (mc *Client) GetBucketLifecycle(ctx context.Context, bucketName string) (*c return &result, nil } -func validateLifecycleRule(rule common.BucketLifecycleRule) lifecycle.Rule { +func validateLifecycleRule(rule common.ObjectLifecycleRule) lifecycle.Rule { r := lifecycle.Rule{ ID: rule.ID, + Status: "Enabled", Expiration: validateLifecycleExpiration(rule.Expiration), - RuleFilter: validateLifecycleFilter(rule.RuleFilter), + RuleFilter: validateLifecycleFilters(rule.RuleFilter), Transition: validateLifecycleTransition(rule.Transition), } + if !rule.Enabled { + r.Status = "Disabled" + } + if rule.AbortIncompleteMultipartUpload != nil && rule.AbortIncompleteMultipartUpload.DaysAfterInitiation != nil { r.AbortIncompleteMultipartUpload.DaysAfterInitiation = lifecycle.ExpirationDays(*rule.AbortIncompleteMultipartUpload.DaysAfterInitiation) } @@ -99,14 +109,10 @@ func validateLifecycleRule(rule common.BucketLifecycleRule) lifecycle.Rule { r.Prefix = *rule.Prefix } - if rule.Status != nil { - r.Status = *rule.Status - } - return r } -func validateLifecycleExpiration(input *common.LifecycleExpiration) lifecycle.Expiration { +func validateLifecycleExpiration(input *common.ObjectLifecycleExpiration) lifecycle.Expiration { result := lifecycle.Expiration{} if input == nil || input.IsEmpty() { @@ -132,7 +138,7 @@ func validateLifecycleExpiration(input *common.LifecycleExpiration) lifecycle.Ex return result } -func validateLifecycleTransition(input *common.LifecycleTransition) lifecycle.Transition { +func validateLifecycleTransition(input *common.ObjectLifecycleTransition) lifecycle.Transition { result := lifecycle.Transition{} if input == nil { @@ -154,7 +160,7 @@ func validateLifecycleTransition(input *common.LifecycleTransition) lifecycle.Tr return result } -func validateLifecycleConfiguration(input common.BucketLifecycleConfiguration) lifecycle.Configuration { +func validateLifecycleConfiguration(input common.ObjectLifecycleConfiguration) lifecycle.Configuration { result := lifecycle.Configuration{ Rules: make([]lifecycle.Rule, len(input.Rules)), } @@ -167,85 +173,84 @@ func validateLifecycleConfiguration(input common.BucketLifecycleConfiguration) l return result } -func validateLifecycleFilter(input *common.LifecycleFilter) lifecycle.Filter { +func validateLifecycleFilters(input []common.ObjectLifecycleFilter) lifecycle.Filter { result := lifecycle.Filter{} - if input == nil { + inputLen := len(input) + if inputLen == 0 { return result } - if input.Prefix != nil { - result.Prefix = *input.Prefix - } - - if input.ObjectSizeGreaterThan != nil { - result.ObjectSizeGreaterThan = *input.ObjectSizeGreaterThan - } - - if input.ObjectSizeLessThan != nil { - result.ObjectSizeLessThan = *input.ObjectSizeLessThan - } + for key, value := range input[0].Tags { + if result.Tag.Key == "" { + result.Tag.Key = key + result.Tag.Value = value - if input.Tag != nil { - if input.Tag.Key != nil { - result.Tag.Key = *input.Tag.Key + continue } - if input.Tag.Value != nil { - result.Tag.Value = *input.Tag.Value - } + result.And.Tags = append(result.And.Tags, lifecycle.Tag{ + Key: key, + Value: value, + }) } - if input.And != nil { - if input.And.Prefix != nil { - result.And.Prefix = *input.And.Prefix - } + if len(input[0].MatchesPrefix) > 0 { + result.Prefix = input[0].MatchesPrefix[0] + } - if input.And.ObjectSizeGreaterThan != nil { - result.And.ObjectSizeGreaterThan = *input.And.ObjectSizeGreaterThan - } + if input[0].ObjectSizeGreaterThan != nil { + result.ObjectSizeGreaterThan = *input[0].ObjectSizeGreaterThan + } - if input.And.ObjectSizeLessThan != nil { - result.And.ObjectSizeLessThan = *input.And.ObjectSizeLessThan - } + if input[0].ObjectSizeLessThan != nil { + result.ObjectSizeLessThan = *input[0].ObjectSizeLessThan + } - result.And.Tags = make([]lifecycle.Tag, len(input.And.Tags)) + if inputLen == 1 { + return result + } - for i, t := range input.And.Tags { - tag := lifecycle.Tag{} + for key, value := range input[1].Tags { + result.And.Tags = append(result.And.Tags, lifecycle.Tag{ + Key: key, + Value: value, + }) + } - if t.Key != nil { - tag.Key = *t.Key - } + if len(input[1].MatchesPrefix) > 0 { + result.And.Prefix = input[1].MatchesPrefix[0] + } - if t.Value != nil { - tag.Value = *t.Value - } + if input[1].ObjectSizeGreaterThan != nil { + result.And.ObjectSizeGreaterThan = *input[1].ObjectSizeGreaterThan + } - result.And.Tags[i] = tag - } + if input[1].ObjectSizeLessThan != nil { + result.And.ObjectSizeLessThan = *input[1].ObjectSizeLessThan } return result } -func serializeLifecycleRule(rule lifecycle.Rule) common.BucketLifecycleRule { - r := common.BucketLifecycleRule{ +func serializeLifecycleRule(rule lifecycle.Rule) common.ObjectLifecycleRule { + r := common.ObjectLifecycleRule{ ID: rule.ID, + Enabled: rule.Status == "Enabled", RuleFilter: serializeLifecycleFilter(rule.RuleFilter), Transition: serializeLifecycleTransition(rule.Transition), } if !rule.AbortIncompleteMultipartUpload.IsDaysNull() { days := int(rule.AbortIncompleteMultipartUpload.DaysAfterInitiation) - r.AbortIncompleteMultipartUpload = &common.AbortIncompleteMultipartUpload{ + r.AbortIncompleteMultipartUpload = &common.ObjectAbortIncompleteMultipartUpload{ DaysAfterInitiation: &days, } } if !rule.AllVersionsExpiration.IsNull() { deleteMarker := bool(rule.AllVersionsExpiration.DeleteMarker) - r.AllVersionsExpiration = &common.LifecycleAllVersionsExpiration{ + r.AllVersionsExpiration = &common.ObjectLifecycleAllVersionsExpiration{ Days: &rule.AllVersionsExpiration.Days, DeleteMarker: &deleteMarker, } @@ -256,7 +261,7 @@ func serializeLifecycleRule(rule lifecycle.Rule) common.BucketLifecycleRule { } if !rule.Expiration.IsNull() { - r.Expiration = &common.LifecycleExpiration{ + r.Expiration = &common.ObjectLifecycleExpiration{ DeleteMarker: (*bool)(&rule.Expiration.DeleteMarker), } @@ -270,7 +275,7 @@ func serializeLifecycleRule(rule lifecycle.Rule) common.BucketLifecycleRule { } if !rule.NoncurrentVersionExpiration.IsDaysNull() || rule.NoncurrentVersionExpiration.NewerNoncurrentVersions != 0 { - r.NoncurrentVersionExpiration = &common.LifecycleNoncurrentVersionExpiration{} + r.NoncurrentVersionExpiration = &common.ObjectLifecycleNoncurrentVersionExpiration{} if rule.NoncurrentVersionExpiration.NewerNoncurrentVersions != 0 { r.NoncurrentVersionExpiration.NewerNoncurrentVersions = &rule.NoncurrentVersionExpiration.NewerNoncurrentVersions @@ -301,19 +306,15 @@ func serializeLifecycleRule(rule lifecycle.Rule) common.BucketLifecycleRule { r.Prefix = &rule.Prefix } - if rule.Status != "" { - r.Status = &rule.Status - } - return r } -func serializeLifecycleTransition(input lifecycle.Transition) *common.LifecycleTransition { +func serializeLifecycleTransition(input lifecycle.Transition) *common.ObjectLifecycleTransition { if input.IsNull() { return nil } - result := common.LifecycleTransition{} + result := common.ObjectLifecycleTransition{} if input.Days != 0 { result.Days = (*int)(&input.Days) @@ -330,9 +331,9 @@ func serializeLifecycleTransition(input lifecycle.Transition) *common.LifecycleT return &result } -func serializeLifecycleConfiguration(input lifecycle.Configuration) common.BucketLifecycleConfiguration { - result := common.BucketLifecycleConfiguration{ - Rules: make([]common.BucketLifecycleRule, len(input.Rules)), +func serializeLifecycleConfiguration(input lifecycle.Configuration) common.ObjectLifecycleConfiguration { + result := common.ObjectLifecycleConfiguration{ + Rules: make([]common.ObjectLifecycleRule, len(input.Rules)), } for i, rule := range input.Rules { @@ -343,66 +344,58 @@ func serializeLifecycleConfiguration(input lifecycle.Configuration) common.Bucke return result } -func serializeLifecycleFilter(input lifecycle.Filter) *common.LifecycleFilter { - result := common.LifecycleFilter{} +func serializeLifecycleFilter(input lifecycle.Filter) []common.ObjectLifecycleFilter { + result := []common.ObjectLifecycleFilter{} + firstItem := common.ObjectLifecycleFilter{} if input.Prefix != "" { - result.Prefix = &input.Prefix + firstItem.MatchesPrefix = []string{input.Prefix} } if input.ObjectSizeGreaterThan != 0 { - result.ObjectSizeGreaterThan = &input.ObjectSizeGreaterThan + firstItem.ObjectSizeGreaterThan = &input.ObjectSizeGreaterThan } if input.ObjectSizeLessThan != 0 { - result.ObjectSizeLessThan = &input.ObjectSizeLessThan + firstItem.ObjectSizeLessThan = &input.ObjectSizeLessThan } if input.Tag.Key != "" || input.Tag.Value != "" { - if input.Tag.Key != "" { - result.Tag.Key = &input.Tag.Key - } - - if input.Tag.Value != "" { - result.Tag.Value = &input.Tag.Value + firstItem.Tags = map[string]string{ + input.Tag.Key: input.Tag.Value, } } - if !input.And.IsEmpty() { - result.And = &common.LifecycleFilterAnd{} - - if input.And.Prefix != "" { - result.And.Prefix = &input.And.Prefix - } - - if input.And.ObjectSizeGreaterThan != 0 { - result.And.ObjectSizeGreaterThan = &input.And.ObjectSizeGreaterThan - } - - if input.And.ObjectSizeLessThan != 0 { - result.And.ObjectSizeLessThan = &input.And.ObjectSizeLessThan - } + result = append(result, firstItem) - result.And.Tags = make([]common.StorageTag, 0, len(input.And.Tags)) + if input.And.IsEmpty() { + return result + } - for _, t := range input.And.Tags { - if t.IsEmpty() { - continue - } + sndItem := common.ObjectLifecycleFilter{} + if input.And.Prefix != "" { + sndItem.MatchesPrefix = []string{input.And.Prefix} + } - tag := common.StorageTag{} + if input.And.ObjectSizeGreaterThan != 0 { + sndItem.ObjectSizeGreaterThan = &input.And.ObjectSizeGreaterThan + } - if t.Key != "" { - tag.Key = &t.Key - } + if input.And.ObjectSizeLessThan != 0 { + sndItem.ObjectSizeLessThan = &input.And.ObjectSizeLessThan + } - if t.Value != "" { - tag.Value = &t.Value - } + sndItem.Tags = make(map[string]string) - result.And.Tags = append(result.And.Tags, tag) + for _, t := range input.And.Tags { + if t.IsEmpty() { + continue } + + sndItem.Tags[t.Key] = t.Value } - return &result + result = append(result, sndItem) + + return result } diff --git a/connector/storage/minio/object.go b/connector/storage/minio/object.go index 7a33b70..e56f9ed 100644 --- a/connector/storage/minio/object.go +++ b/connector/storage/minio/object.go @@ -161,6 +161,13 @@ func (mc *Client) ListIncompleteUploads(ctx context.Context, bucketName string, return objects, nil } +// ListDeletedObjects list soft-deleted objects in a bucket. +func (mc *Client) ListDeletedObjects(ctx context.Context, bucketName string, opts *common.ListStorageObjectsOptions, predicate func(string) bool) (*common.StorageObjectListResults, error) { + return &common.StorageObjectListResults{ + Objects: []common.StorageObject{}, + }, nil +} + // RemoveIncompleteUpload removes a partially uploaded object. func (mc *Client) RemoveIncompleteUpload(ctx context.Context, bucketName string, objectName string) error { ctx, span := mc.startOtelSpan(ctx, "RemoveIncompleteUpload", bucketName) @@ -210,8 +217,8 @@ func (mc *Client) PutObject(ctx context.Context, bucketName string, objectName s ) options := minio.PutObjectOptions{ - UserMetadata: opts.UserMetadata, - UserTags: opts.UserTags, + UserMetadata: opts.Metadata, + UserTags: opts.Tags, ContentType: opts.ContentType, ContentEncoding: opts.ContentEncoding, ContentDisposition: opts.ContentDisposition, @@ -232,13 +239,14 @@ func (mc *Client) PutObject(ctx context.Context, bucketName string, objectName s options.Expires = *opts.Expires } - if opts.RetainUntilDate != nil { - options.RetainUntilDate = *opts.RetainUntilDate - span.SetAttributes(attribute.String("storage.options.retain_util_date", opts.RetainUntilDate.Format(time.RFC3339))) - } + if opts.Retention != nil { + options.Mode = validateObjectRetentionMode(opts.Retention.Mode) + options.RetainUntilDate = opts.Retention.RetainUntilDate - if opts.Mode != nil { - options.Mode = validateObjectRetentionMode(*opts.Mode) + span.SetAttributes( + attribute.String("storage.options.retention_mode", string(opts.Retention.Mode)), + attribute.String("storage.options.retain_util_date", opts.Retention.RetainUntilDate.Format(time.RFC3339)), + ) } if opts.Checksum != nil { @@ -346,12 +354,12 @@ func (mc *Client) StatObject(ctx context.Context, bucketName, objectName string, result.Bucket = bucketName if opts.Include.Tags { - userTags, err := mc.GetObjectTagging(ctx, bucketName, objectName, opts.VersionID) + userTags, err := mc.GetObjectTags(ctx, bucketName, objectName, opts.VersionID) if err != nil { return nil, err } - result.UserTags = userTags + result.Tags = userTags } if opts.Include.LegalHold { @@ -450,8 +458,45 @@ func (mc *Client) RemoveObjects(ctx context.Context, bucketName string, opts *co return errs } +// UpdateObject updates object configurations. +func (mc *Client) UpdateObject(ctx context.Context, bucketName string, objectName string, opts common.UpdateStorageObjectOptions) error { + ctx, span := mc.startOtelSpanWithKind(ctx, trace.SpanKindInternal, "UpdateObject", bucketName) + defer span.End() + + span.SetAttributes(attribute.String("storage.key", objectName)) + + if opts.VersionID != "" { + span.SetAttributes(attribute.String("storage.options.version", opts.VersionID)) + } + + if opts.LegalHold != nil { + if err := mc.SetObjectLegalHold(ctx, bucketName, objectName, opts.VersionID, opts.LegalHold); err != nil { + return err + } + } + + if opts.Tags != nil { + if err := mc.SetObjectTags(ctx, bucketName, objectName, opts.VersionID, opts.Tags); err != nil { + return err + } + } + + if opts.Retention != nil { + if err := mc.SetObjectRetention(ctx, bucketName, objectName, opts.VersionID, *opts.Retention); err != nil { + return err + } + } + + return nil +} + +// RestoreObject restores a soft-deleted object. +func (mc *Client) RestoreObject(ctx context.Context, bucketName string, objectName string) error { + return schema.NotSupportedError("MinIO does not support this function", nil) +} + // SetObjectRetention applies object retention lock onto an object. -func (mc *Client) SetObjectRetention(ctx context.Context, bucketName string, objectName string, opts common.SetStorageObjectRetentionOptions) error { +func (mc *Client) SetObjectRetention(ctx context.Context, bucketName string, objectName, versionID string, opts common.SetStorageObjectRetentionOptions) error { ctx, span := mc.startOtelSpan(ctx, "SetObjectRetention", bucketName) defer span.End() @@ -460,8 +505,8 @@ func (mc *Client) SetObjectRetention(ctx context.Context, bucketName string, obj attribute.Bool("storage.options.governance_bypass", opts.GovernanceBypass), ) - if opts.VersionID != "" { - span.SetAttributes(attribute.String("storage.options.version", opts.VersionID)) + if versionID != "" { + span.SetAttributes(attribute.String("storage.options.version", versionID)) } if opts.RetainUntilDate != nil { @@ -470,7 +515,7 @@ func (mc *Client) SetObjectRetention(ctx context.Context, bucketName string, obj options := minio.PutObjectRetentionOptions{ GovernanceBypass: opts.GovernanceBypass, - VersionID: opts.VersionID, + VersionID: versionID, RetainUntilDate: opts.RetainUntilDate, } @@ -492,23 +537,23 @@ func (mc *Client) SetObjectRetention(ctx context.Context, bucketName string, obj } // SetObjectLegalHold applies legal-hold onto an object. -func (mc *Client) SetObjectLegalHold(ctx context.Context, bucketName string, objectName string, opts common.SetStorageObjectLegalHoldOptions) error { +func (mc *Client) SetObjectLegalHold(ctx context.Context, bucketName, objectName, versionID string, status *bool) error { ctx, span := mc.startOtelSpan(ctx, "SetObjectLegalHold", bucketName) defer span.End() span.SetAttributes(attribute.String("storage.key", objectName)) options := minio.PutObjectLegalHoldOptions{ - VersionID: opts.VersionID, + VersionID: versionID, } - if opts.VersionID != "" { - span.SetAttributes(attribute.String("storage.options.version", opts.VersionID)) + if versionID != "" { + span.SetAttributes(attribute.String("storage.options.version", versionID)) } - if opts.Status != nil { - span.SetAttributes(attribute.Bool("storage.options.status", *opts.Status)) - legalHold := validateLegalHoldStatus(opts.Status) + if status != nil { + span.SetAttributes(attribute.Bool("storage.options.status", *status)) + legalHold := validateLegalHoldStatus(status) options.Status = &legalHold } @@ -548,24 +593,24 @@ func (mc *Client) GetObjectLegalHold(ctx context.Context, bucketName string, obj return status != nil && *status == minio.LegalHoldEnabled, nil } -// PutObjectTagging sets new object Tags to the given object, replaces/overwrites any existing tags. -func (mc *Client) SetObjectTags(ctx context.Context, bucketName string, objectName string, opts common.SetStorageObjectTagsOptions) error { - if len(opts.Tags) == 0 { - return mc.RemoveObjectTagging(ctx, bucketName, objectName, opts.VersionID) +// SetObjectTags sets new object Tags to the given object, replaces/overwrites any existing tags. +func (mc *Client) SetObjectTags(ctx context.Context, bucketName string, objectName, versionID string, objectTags map[string]string) error { + if len(objectTags) == 0 { + return mc.RemoveObjectTags(ctx, bucketName, objectName, versionID) } - ctx, span := mc.startOtelSpan(ctx, "SetStorageObjectTags", bucketName) + ctx, span := mc.startOtelSpan(ctx, "SetObjectTags", bucketName) defer span.End() span.SetAttributes(attribute.String("storage.key", objectName)) options := minio.PutObjectTaggingOptions{} - if opts.VersionID != "" { - span.SetAttributes(attribute.String("storage.options.version", opts.VersionID)) + if versionID != "" { + span.SetAttributes(attribute.String("storage.options.version", versionID)) } - inputTags, err := tags.NewTags(opts.Tags, false) + inputTags, err := tags.NewTags(objectTags, false) if err != nil { span.SetStatus(codes.Error, "failed to convert minio tags") span.RecordError(err) @@ -584,9 +629,9 @@ func (mc *Client) SetObjectTags(ctx context.Context, bucketName string, objectNa return nil } -// GetObjectTagging fetches Object Tags from the given object. -func (mc *Client) GetObjectTagging(ctx context.Context, bucketName, objectName string, versionID *string) (map[string]string, error) { - ctx, span := mc.startOtelSpan(ctx, "GetObjectTagging", bucketName) +// GetObjectTags fetches Object Tags from the given object. +func (mc *Client) GetObjectTags(ctx context.Context, bucketName, objectName string, versionID *string) (map[string]string, error) { + ctx, span := mc.startOtelSpan(ctx, "GetObjectTags", bucketName) defer span.End() span.SetAttributes(attribute.String("storage.key", objectName)) @@ -609,9 +654,9 @@ func (mc *Client) GetObjectTagging(ctx context.Context, bucketName, objectName s return results.ToMap(), nil } -// RemoveObjectTagging removes Object Tags from the given object. -func (mc *Client) RemoveObjectTagging(ctx context.Context, bucketName, objectName, versionID string) error { - ctx, span := mc.startOtelSpan(ctx, "RemoveObjectTagging", bucketName) +// RemoveObjectTags removes Object Tags from the given object. +func (mc *Client) RemoveObjectTags(ctx context.Context, bucketName, objectName, versionID string) error { + ctx, span := mc.startOtelSpan(ctx, "RemoveObjectTags", bucketName) defer span.End() span.SetAttributes(attribute.String("storage.key", objectName)) @@ -770,16 +815,22 @@ func (mc *Client) GetObjectLockConfig(ctx context.Context, bucketName string) (* ctx, span := mc.startOtelSpan(ctx, "GetObjectLockConfig", bucketName) defer span.End() + // ObjectLockConfigurationNotFoundError objectLock, mode, validity, unit, err := mc.client.GetObjectLockConfig(ctx, bucketName) if err != nil { + respError := evalNotFoundError(err, "ObjectLockConfigurationNotFoundError") + if respError == nil { + return nil, nil + } + span.SetStatus(codes.Error, err.Error()) span.RecordError(err) - return nil, serializeErrorResponse(err) + return nil, respError } result := &common.StorageObjectLockConfig{ - ObjectLock: objectLock, + Enabled: objectLock == "Enabled", SetStorageObjectLockConfig: common.SetStorageObjectLockConfig{ Mode: serializeObjectRetentionMode(mode), Validity: validity, diff --git a/connector/storage/minio/sse.go b/connector/storage/minio/sse.go deleted file mode 100644 index cb43ddb..0000000 --- a/connector/storage/minio/sse.go +++ /dev/null @@ -1,101 +0,0 @@ -package minio - -import ( - "context" - - "github.com/hasura/ndc-storage/connector/storage/common" - "github.com/minio/minio-go/v7/pkg/sse" - "go.opentelemetry.io/otel/codes" -) - -// SetBucketEncryption sets default encryption configuration on a bucket. -func (mc *Client) SetBucketEncryption(ctx context.Context, bucketName string, input common.ServerSideEncryptionConfiguration) error { - ctx, span := mc.startOtelSpan(ctx, "SetBucketEncryption", bucketName) - defer span.End() - - err := mc.client.SetBucketEncryption(ctx, bucketName, validateBucketEncryptionConfiguration(input)) - if err != nil { - span.SetStatus(codes.Error, err.Error()) - span.RecordError(err) - - return serializeErrorResponse(err) - } - - return nil -} - -// GetBucketEncryption gets default encryption configuration set on a bucket. -func (mc *Client) GetBucketEncryption(ctx context.Context, bucketName string) (*common.ServerSideEncryptionConfiguration, error) { - ctx, span := mc.startOtelSpan(ctx, "GetBucketEncryption", bucketName) - defer span.End() - - rawResult, err := mc.client.GetBucketEncryption(ctx, bucketName) - if err != nil { - span.SetStatus(codes.Error, err.Error()) - span.RecordError(err) - - return nil, serializeErrorResponse(err) - } - - return serializeBucketEncryptionConfiguration(rawResult), nil -} - -// RemoveBucketEncryption remove default encryption configuration set on a bucket. -func (mc *Client) RemoveBucketEncryption(ctx context.Context, bucketName string) error { - ctx, span := mc.startOtelSpan(ctx, "RemoveBucketEncryption", bucketName) - defer span.End() - - err := mc.client.RemoveBucketEncryption(ctx, bucketName) - if err != nil { - span.SetStatus(codes.Error, err.Error()) - span.RecordError(err) - - return serializeErrorResponse(err) - } - - return nil -} - -func validateBucketEncryptionConfiguration(input common.ServerSideEncryptionConfiguration) *sse.Configuration { - result := &sse.Configuration{ - Rules: make([]sse.Rule, len(input.Rules)), - } - - for i, rule := range input.Rules { - r := sse.Rule{ - Apply: sse.ApplySSEByDefault{ - SSEAlgorithm: rule.Apply.SSEAlgorithm, - }, - } - - if rule.Apply.KmsMasterKeyID != nil { - r.Apply.KmsMasterKeyID = *rule.Apply.KmsMasterKeyID - } - - result.Rules[i] = r - } - - return result -} - -func serializeBucketEncryptionConfiguration(input *sse.Configuration) *common.ServerSideEncryptionConfiguration { - result := &common.ServerSideEncryptionConfiguration{ - Rules: make([]common.ServerSideEncryptionRule, len(input.Rules)), - } - - for i, rule := range input.Rules { - r := common.ServerSideEncryptionRule{ - Apply: common.StorageApplySSEByDefault{ - SSEAlgorithm: rule.Apply.SSEAlgorithm, - }, - } - - if !isStringNull(rule.Apply.KmsMasterKeyID) { - r.Apply.KmsMasterKeyID = &rule.Apply.KmsMasterKeyID - } - - result.Rules[i] = r - } - - return result -} diff --git a/connector/storage/minio/utils.go b/connector/storage/minio/utils.go index 92b72fb..42efbb5 100644 --- a/connector/storage/minio/utils.go +++ b/connector/storage/minio/utils.go @@ -10,6 +10,7 @@ import ( "github.com/hasura/ndc-storage/connector/storage/common" "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/notification" + "github.com/minio/minio-go/v7/pkg/sse" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" ) @@ -71,9 +72,9 @@ func serializeObjectInfo(obj minio.ObjectInfo, fromList bool) common.StorageObje LastModified: obj.LastModified, Size: &obj.Size, Metadata: map[string]string{}, - UserMetadata: map[string]string{}, - UserTags: obj.UserTags, - UserTagCount: obj.UserTagCount, + RawMetadata: map[string]string{}, + Tags: obj.UserTags, + TagCount: obj.UserTagCount, Grant: grants, IsLatest: &obj.IsLatest, Deleted: &obj.IsDeleteMarker, @@ -81,15 +82,17 @@ func serializeObjectInfo(obj minio.ObjectInfo, fromList bool) common.StorageObje StorageObjectChecksum: checksum, } + if object.TagCount == 0 { + object.TagCount = len(object.Tags) + } + if fromList { - object.Metadata = obj.UserMetadata + object.RawMetadata = obj.UserMetadata for key, value := range obj.UserMetadata { lowerKey := strings.ToLower(key) if strings.HasPrefix(lowerKey, userMetadataHeaderPrefix) { - object.UserMetadata[key[len(userMetadataHeaderPrefix):]] = value - - continue + object.Metadata[key[len(userMetadataHeaderPrefix):]] = value } switch lowerKey { @@ -106,7 +109,7 @@ func serializeObjectInfo(obj minio.ObjectInfo, fromList bool) common.StorageObje } } } else { - object.UserMetadata = obj.UserMetadata + object.Metadata = obj.UserMetadata for key, values := range obj.Metadata { if len(values) == 0 { @@ -114,7 +117,7 @@ func serializeObjectInfo(obj minio.ObjectInfo, fromList bool) common.StorageObje } value := strings.Join(values, ", ") - object.Metadata[key] = value + object.RawMetadata[key] = value switch strings.ToLower(key) { case common.HeaderContentType: @@ -342,10 +345,10 @@ func convertCopyDestOptions(dst common.StorageCopyDestOptions) *minio.CopyDestOp destOptions := minio.CopyDestOptions{ Bucket: dst.Bucket, Object: dst.Object, - UserMetadata: dst.UserMetadata, - ReplaceMetadata: dst.ReplaceMetadata, - UserTags: dst.UserTags, - ReplaceTags: dst.ReplaceTags, + UserMetadata: dst.Metadata, + ReplaceMetadata: dst.Metadata != nil, + UserTags: dst.Tags, + ReplaceTags: dst.Tags != nil, Size: dst.Size, LegalHold: validateLegalHoldStatus(dst.LegalHold), } @@ -624,6 +627,45 @@ func serializeErrorResponse(err error) *schema.ConnectorError { return schema.UnprocessableContentError(err.Error(), nil) } +func evalNotFoundError(err error, notFoundCode string) *schema.ConnectorError { + var errResponse minio.ErrorResponse + if !errors.As(err, &errResponse) { + errRespPtr := &errResponse + if errors.As(err, &errRespPtr) { + errResponse = *errRespPtr + } + } + + if errResponse.Code == notFoundCode { + return nil + } + + if errResponse.StatusCode > 0 { + return evalMinioErrorResponse(errResponse) + } + + return schema.UnprocessableContentError(err.Error(), nil) +} + func isStringNull(input string) bool { return input == "" || input == "null" } + +func validateBucketEncryptionConfiguration(input common.ServerSideEncryptionConfiguration) *sse.Configuration { + if input.SSEAlgorithm == "AES256" { + return sse.NewConfigurationSSES3() + } + + return sse.NewConfigurationSSEKMS(input.KmsMasterKeyID) +} + +func serializeBucketEncryptionConfiguration(input *sse.Configuration) *common.ServerSideEncryptionConfiguration { + if input == nil || len(input.Rules) == 0 { + return nil + } + + return &common.ServerSideEncryptionConfiguration{ + KmsMasterKeyID: input.Rules[0].Apply.KmsMasterKeyID, + SSEAlgorithm: input.Rules[0].Apply.SSEAlgorithm, + } +} diff --git a/connector/storage/object.go b/connector/storage/object.go index d50d8a7..78ad559 100644 --- a/connector/storage/object.go +++ b/connector/storage/object.go @@ -32,6 +32,26 @@ func (m *Manager) ListObjects(ctx context.Context, bucketInfo common.StorageBuck return results, nil } +// ListObjects lists deleted objects in a bucket. +func (m *Manager) ListDeletedObjects(ctx context.Context, bucketInfo common.StorageBucketArguments, opts *common.ListStorageObjectsOptions, predicate func(string) bool) (*common.StorageObjectListResults, error) { + client, bucketName, err := m.GetClientAndBucket(bucketInfo.ClientID, bucketInfo.Bucket) + if err != nil { + return nil, err + } + + results, err := client.ListDeletedObjects(ctx, bucketName, opts, predicate) + if err != nil { + return nil, err + } + + for i := range results.Objects { + results.Objects[i].ClientID = string(client.id) + results.Objects[i].Bucket = bucketName + } + + return results, nil +} + // ListIncompleteUploads list partially uploaded objects in a bucket. func (m *Manager) ListIncompleteUploads(ctx context.Context, bucketInfo common.StorageBucketArguments, opts common.ListIncompleteUploadsOptions) ([]common.StorageObjectMultipartInfo, error) { client, bucketName, err := m.GetClientAndBucket(bucketInfo.ClientID, bucketInfo.Bucket) @@ -152,14 +172,18 @@ func (m *Manager) RemoveObject(ctx context.Context, bucketInfo common.StorageBuc return client.RemoveObject(ctx, bucketName, objectName, opts) } -// SetObjectRetention applies object retention lock onto an object. -func (m *Manager) SetObjectRetention(ctx context.Context, bucketInfo common.StorageBucketArguments, objectName string, opts common.SetStorageObjectRetentionOptions) error { +// UpdateObject updates object configuration. +func (m *Manager) UpdateObject(ctx context.Context, bucketInfo common.StorageBucketArguments, objectName string, opts common.UpdateStorageObjectOptions) error { + if opts.IsEmpty() { + return nil + } + client, bucketName, err := m.GetClientAndBucket(bucketInfo.ClientID, bucketInfo.Bucket) if err != nil { return err } - return client.SetObjectRetention(ctx, bucketName, objectName, opts) + return client.UpdateObject(ctx, bucketName, objectName, opts) } // RemoveObjects remove a list of objects obtained from an input channel. The call sends a delete request to the server up to 1000 objects at a time. @@ -173,24 +197,14 @@ func (m *Manager) RemoveObjects(ctx context.Context, bucketInfo common.StorageBu return client.RemoveObjects(ctx, bucketName, opts, predicate), nil } -// SetObjectLegalHold applies legal-hold onto an object. -func (m *Manager) SetObjectLegalHold(ctx context.Context, bucketInfo common.StorageBucketArguments, objectName string, opts common.SetStorageObjectLegalHoldOptions) error { - client, bucketName, err := m.GetClientAndBucket(bucketInfo.ClientID, bucketInfo.Bucket) - if err != nil { - return err - } - - return client.SetObjectLegalHold(ctx, bucketName, objectName, opts) -} - -// PutObjectTagging sets new object Tags to the given object, replaces/overwrites any existing tags. -func (m *Manager) SetObjectTags(ctx context.Context, bucketInfo common.StorageBucketArguments, objectName string, opts common.SetStorageObjectTagsOptions) error { +// RestoreObject restores a soft-deleted object. +func (m *Manager) RestoreObject(ctx context.Context, bucketInfo common.StorageBucketArguments, objectName string) error { client, bucketName, err := m.GetClientAndBucket(bucketInfo.ClientID, bucketInfo.Bucket) if err != nil { return err } - return client.SetObjectTags(ctx, bucketName, objectName, opts) + return client.RestoreObject(ctx, bucketName, objectName) } // RemoveIncompleteUpload removes a partially uploaded object. diff --git a/connector/testdata/01-setup/mutation/01-createStorageBucket-azblob/request.json b/connector/testdata/01-setup/mutation/01-createStorageBucket-azblob/request.json index abf8fff..7b6dcdc 100644 --- a/connector/testdata/01-setup/mutation/01-createStorageBucket-azblob/request.json +++ b/connector/testdata/01-setup/mutation/01-createStorageBucket-azblob/request.json @@ -7,6 +7,7 @@ "arguments": { "clientId": "azblob", "name": "azblob-bucket-test", + "objectLock": true, "tags": { "provider": "azure" } diff --git a/connector/testdata/01-setup/mutation/02-setStorageBucketLifecycle/expected.json b/connector/testdata/01-setup/mutation/01-createStorageBucket-gcs/expected.json similarity index 100% rename from connector/testdata/01-setup/mutation/02-setStorageBucketLifecycle/expected.json rename to connector/testdata/01-setup/mutation/01-createStorageBucket-gcs/expected.json diff --git a/connector/testdata/01-setup/mutation/01-createStorageBucket-gcs/request.json b/connector/testdata/01-setup/mutation/01-createStorageBucket-gcs/request.json new file mode 100644 index 0000000..7e8bc3e --- /dev/null +++ b/connector/testdata/01-setup/mutation/01-createStorageBucket-gcs/request.json @@ -0,0 +1,17 @@ +{ + "collection_relationships": {}, + "operations": [ + { + "type": "procedure", + "name": "createStorageBucket", + "arguments": { + "clientId": "gcs", + "name": "gcs-bucket", + "objectLock": true, + "tags": { + "provider": "gcp" + } + } + } + ] +} diff --git a/connector/testdata/01-setup/mutation/01-createStorageBucket-lock/request.json b/connector/testdata/01-setup/mutation/01-createStorageBucket-lock/request.json index 9a6c9aa..9ee3ce1 100644 --- a/connector/testdata/01-setup/mutation/01-createStorageBucket-lock/request.json +++ b/connector/testdata/01-setup/mutation/01-createStorageBucket-lock/request.json @@ -7,7 +7,7 @@ "arguments": { "clientId": "minio", "name": "minio-bucket-lock", - "objectLocking": true, + "objectLock": true, "region": "us-east-1" } } diff --git a/connector/testdata/01-setup/mutation/01-createStorageBucket-s3/request.json b/connector/testdata/01-setup/mutation/01-createStorageBucket-s3/request.json index b7bbcd2..7c36c84 100644 --- a/connector/testdata/01-setup/mutation/01-createStorageBucket-s3/request.json +++ b/connector/testdata/01-setup/mutation/01-createStorageBucket-s3/request.json @@ -7,7 +7,7 @@ "arguments": { "clientId": "s3", "name": "s3-bucket-test", - "objectLocking": true, + "objectLock": true, "region": "us-east-1" } } diff --git a/connector/testdata/01-setup/mutation/01-createStorageBucket/request.json b/connector/testdata/01-setup/mutation/01-createStorageBucket/request.json index 3ef0494..ff87e31 100644 --- a/connector/testdata/01-setup/mutation/01-createStorageBucket/request.json +++ b/connector/testdata/01-setup/mutation/01-createStorageBucket/request.json @@ -7,7 +7,7 @@ "arguments": { "clientId": "minio", "name": "minio-bucket-test", - "objectLocking": false, + "objectLock": false, "region": "us-east-1", "tags": { "UserID": "1" diff --git a/connector/testdata/01-setup/mutation/02-setStorageBucketLifecycle/request.json b/connector/testdata/01-setup/mutation/02-setStorageBucketLifecycle/request.json deleted file mode 100644 index f0090f7..0000000 --- a/connector/testdata/01-setup/mutation/02-setStorageBucketLifecycle/request.json +++ /dev/null @@ -1,62 +0,0 @@ -{ - "collection_relationships": {}, - "operations": [ - { - "type": "procedure", - "name": "setStorageBucketLifecycle", - "arguments": { - "bucket": "minio-bucket-test", - "rules": [ - { - "abortIncompleteMultipartUpload": { - "daysAfterInitiation": 1 - }, - "allVersionsExpiration": { - "days": 2, - "deleteMarker": false - }, - "expiration": { - "days": 4, - "expiredObjectAllVersions": false, - "expiredObjectDeleteMarker": false - }, - "filter": { - "and": { - "objectSizeGreaterThan": 1000000, - "objectSizeLessThan": 1, - "prefix": "JSqXP0pJZF", - "tags": [ - { - "key": "7Uw1or2jrq", - "value": "00sA2cRTCB" - } - ] - }, - "objectSizeGreaterThan": 1000000, - "objectSizeLessThan": 1, - "prefix": "30LomArtyT", - "tag": { - "key": "raZx9yPMwi", - "value": "JbBh0wCDaQ" - } - }, - "id": "aXD6eFi5JE", - "noncurrentVersionExpiration": { - "newerNoncurrentVersions": 108, - "noncurrentDays": 12 - }, - "noncurrentVersionTransition": { - "newerNoncurrentVersions": 155, - "noncurrentDays": 7 - }, - "prefix": "GPJel0xa2s", - "status": "Enabled", - "transition": { - "days": 5583 - } - } - ] - } - } - ] -} diff --git a/connector/testdata/01-setup/mutation/02-setStorageBucketNotification/request.json b/connector/testdata/01-setup/mutation/02-setStorageBucketNotification/request.json deleted file mode 100644 index 19e96ba..0000000 --- a/connector/testdata/01-setup/mutation/02-setStorageBucketNotification/request.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "collection_relationships": {}, - "operations": [ - { - "type": "procedure", - "name": "setStorageBucketNotification", - "arguments": { - "bucket": "minio-bucket-test", - "queueConfigurations": [] - } - } - ] -} diff --git a/connector/testdata/01-setup/mutation/02-enableStorageBucketVersioning/expected.json b/connector/testdata/01-setup/mutation/02-updateStorageBucket-azblob/expected.json similarity index 100% rename from connector/testdata/01-setup/mutation/02-enableStorageBucketVersioning/expected.json rename to connector/testdata/01-setup/mutation/02-updateStorageBucket-azblob/expected.json diff --git a/connector/testdata/01-setup/mutation/02-setStorageBucketTags-azblob/request.json b/connector/testdata/01-setup/mutation/02-updateStorageBucket-azblob/request.json similarity index 86% rename from connector/testdata/01-setup/mutation/02-setStorageBucketTags-azblob/request.json rename to connector/testdata/01-setup/mutation/02-updateStorageBucket-azblob/request.json index 8501045..2a03f0e 100644 --- a/connector/testdata/01-setup/mutation/02-setStorageBucketTags-azblob/request.json +++ b/connector/testdata/01-setup/mutation/02-updateStorageBucket-azblob/request.json @@ -3,7 +3,7 @@ "operations": [ { "type": "procedure", - "name": "setStorageBucketTags", + "name": "updateStorageBucket", "arguments": { "clientId": "azblob", "bucket": "azblob-bucket-test", diff --git a/connector/testdata/01-setup/mutation/02-setStorageBucketNotification/expected.json b/connector/testdata/01-setup/mutation/02-updateStorageBucket-gcs/expected.json similarity index 100% rename from connector/testdata/01-setup/mutation/02-setStorageBucketNotification/expected.json rename to connector/testdata/01-setup/mutation/02-updateStorageBucket-gcs/expected.json diff --git a/connector/testdata/01-setup/mutation/02-setStorageBucketTags/request.json b/connector/testdata/01-setup/mutation/02-updateStorageBucket-gcs/request.json similarity index 55% rename from connector/testdata/01-setup/mutation/02-setStorageBucketTags/request.json rename to connector/testdata/01-setup/mutation/02-updateStorageBucket-gcs/request.json index 8687212..8623f28 100644 --- a/connector/testdata/01-setup/mutation/02-setStorageBucketTags/request.json +++ b/connector/testdata/01-setup/mutation/02-updateStorageBucket-gcs/request.json @@ -3,11 +3,12 @@ "operations": [ { "type": "procedure", - "name": "setStorageBucketTags", + "name": "updateStorageBucket", "arguments": { - "bucket": "minio-bucket-test", + "clientId": "gcs", + "bucket": "gcs-bucket", "tags": { - "Foo": "bar" + "Foo": "gcs-bar" } } } diff --git a/connector/testdata/01-setup/mutation/02-setStorageBucketTags-azblob/expected.json b/connector/testdata/01-setup/mutation/02-updateStorageBucket/expected.json similarity index 100% rename from connector/testdata/01-setup/mutation/02-setStorageBucketTags-azblob/expected.json rename to connector/testdata/01-setup/mutation/02-updateStorageBucket/expected.json diff --git a/connector/testdata/01-setup/mutation/02-updateStorageBucket/request.json b/connector/testdata/01-setup/mutation/02-updateStorageBucket/request.json new file mode 100644 index 0000000..57b3d0a --- /dev/null +++ b/connector/testdata/01-setup/mutation/02-updateStorageBucket/request.json @@ -0,0 +1,66 @@ +{ + "collection_relationships": {}, + "operations": [ + { + "type": "procedure", + "name": "updateStorageBucket", + "arguments": { + "bucket": "minio-bucket-test", + "versioningEnabled": true, + "lifecycle": { + "rules": [ + { + "abortIncompleteMultipartUpload": { + "daysAfterInitiation": 1 + }, + "allVersionsExpiration": { + "days": 2, + "deleteMarker": false + }, + "expiration": { + "days": 4, + "expiredObjectAllVersions": false, + "expiredObjectDeleteMarker": false + }, + "filter": [ + { + "objectSizeGreaterThan": 1000000, + "objectSizeLessThan": 1, + "matchesPrefix": ["30LomArtyT"], + "tags": { + "raZx9yPMwi": "JbBh0wCDaQ" + } + }, + { + "objectSizeGreaterThan": 1000000, + "objectSizeLessThan": 1, + "matchesPrefix": ["JSqXP0pJZF"], + "tags": { + "7Uw1or2jrq": "00sA2cRTCB" + } + } + ], + "id": "aXD6eFi5JE", + "noncurrentVersionExpiration": { + "newerNoncurrentVersions": 108, + "noncurrentDays": 12 + }, + "noncurrentVersionTransition": { + "newerNoncurrentVersions": 155, + "noncurrentDays": 7 + }, + "prefix": "GPJel0xa2s", + "enabled": true, + "transition": { + "days": 5583 + } + } + ] + }, + "tags": { + "Foo": "bar" + } + } + } + ] +} diff --git a/connector/testdata/01-setup/mutation/03-uploadStorageObject-2/request.json b/connector/testdata/01-setup/mutation/03-uploadStorageObject-2/request.json index 1851bf2..511e3c4 100644 --- a/connector/testdata/01-setup/mutation/03-uploadStorageObject-2/request.json +++ b/connector/testdata/01-setup/mutation/03-uploadStorageObject-2/request.json @@ -25,10 +25,10 @@ "partSize": 3573915715657741285, "sendContentMd5": false, "storageClass": "STANDARD", - "userMetadata": { + "metadata": { "Foo": "Bar" }, - "userTags": { + "tags": { "UserID": "1" }, "websiteRedirectLocation": "http://localhost:9001" diff --git a/connector/testdata/01-setup/mutation/03-uploadStorageObject-3/request.json b/connector/testdata/01-setup/mutation/03-uploadStorageObject-3/request.json index 1aec54c..d8f8c80 100644 --- a/connector/testdata/01-setup/mutation/03-uploadStorageObject-3/request.json +++ b/connector/testdata/01-setup/mutation/03-uploadStorageObject-3/request.json @@ -25,10 +25,10 @@ "partSize": 3573915715657741285, "sendContentMd5": false, "storageClass": "STANDARD", - "userMetadata": { + "metadata": { "Foo": "Bar" }, - "userTags": { + "tags": { "UserID": "1" }, "websiteRedirectLocation": "http://localhost:9001" diff --git a/connector/testdata/01-setup/mutation/03-uploadStorageObject-azblob/request.json b/connector/testdata/01-setup/mutation/03-uploadStorageObject-azblob/request.json index 44f218d..109b6ad 100644 --- a/connector/testdata/01-setup/mutation/03-uploadStorageObject-azblob/request.json +++ b/connector/testdata/01-setup/mutation/03-uploadStorageObject-azblob/request.json @@ -25,10 +25,10 @@ "partSize": 1024000000, "sendContentMd5": true, "storageClass": "Cool", - "userMetadata": { + "metadata": { "Foo": "Bar" }, - "userTags": { + "tags": { "UserID": "1" }, "websiteRedirectLocation": "http://localhost:9001" diff --git a/connector/testdata/01-setup/mutation/03-uploadStorageObject-gcs/expected.json b/connector/testdata/01-setup/mutation/03-uploadStorageObject-gcs/expected.json new file mode 100644 index 0000000..37d69ad --- /dev/null +++ b/connector/testdata/01-setup/mutation/03-uploadStorageObject-gcs/expected.json @@ -0,0 +1,21 @@ +{ + "operation_results": [ + { + "result": { + "bucket": "gcs-bucket", + "checksumCrc32": null, + "checksumCrc32C": null, + "checksumCrc64Nvme": null, + "checksumSha1": null, + "checksumSha256": null, + "contentMd5": null, + "expiration": null, + "expirationRuleId": null, + "location": null, + "name": "public/hello.txt", + "size": null + }, + "type": "procedure" + } + ] +} diff --git a/connector/testdata/01-setup/mutation/03-uploadStorageObject-gcs/request.json b/connector/testdata/01-setup/mutation/03-uploadStorageObject-gcs/request.json new file mode 100644 index 0000000..8a88a90 --- /dev/null +++ b/connector/testdata/01-setup/mutation/03-uploadStorageObject-gcs/request.json @@ -0,0 +1,93 @@ +{ + "collection_relationships": {}, + "operations": [ + { + "type": "procedure", + "name": "uploadStorageObject", + "arguments": { + "clientId": "gcs", + "bucket": "gcs-bucket", + "data": "SVpHWjhMMHZzVw==", + "object": "public/hello.txt", + "options": { + "autoChecksum": "SHA256", + "cacheControl": "max-age=180, public", + "checksum": "SHA256", + "clientId": "s3", + "concurrentStreamParts": false, + "contentDisposition": "attachment", + "contentEncoding": "gzip", + "contentLanguage": "en-US", + "contentType": "text/plain", + "disableContentSha256": false, + "disableMultipart": false, + "expires": "2099-01-01T00:00:00Z", + "numThreads": 1, + "partSize": 1024000000, + "sendContentMd5": true, + "storageClass": "Cool", + "metadata": { + "Foo": "Bar" + }, + "tags": { + "UserID": "1" + }, + "websiteRedirectLocation": "http://localhost:9001" + } + }, + "fields": { + "fields": { + "bucket": { + "column": "bucket", + "type": "column" + }, + "checksumCrc32": { + "column": "checksumCrc32", + "type": "column" + }, + "checksumCrc32C": { + "column": "checksumCrc32C", + "type": "column" + }, + "checksumCrc64Nvme": { + "column": "checksumCrc64Nvme", + "type": "column" + }, + "checksumSha1": { + "column": "checksumSha1", + "type": "column" + }, + "checksumSha256": { + "column": "checksumSha256", + "type": "column" + }, + "contentMd5": { + "column": "contentMd5", + "type": "column" + }, + "expiration": { + "column": "expiration", + "type": "column" + }, + "expirationRuleId": { + "column": "expirationRuleId", + "type": "column" + }, + "location": { + "column": "location", + "type": "column" + }, + "name": { + "column": "name", + "type": "column" + }, + "size": { + "column": "size", + "type": "column" + } + }, + "type": "object" + } + } + ] +} diff --git a/connector/testdata/01-setup/mutation/03-uploadStorageObject/request.json b/connector/testdata/01-setup/mutation/03-uploadStorageObject/request.json index c34a809..74cd9cd 100644 --- a/connector/testdata/01-setup/mutation/03-uploadStorageObject/request.json +++ b/connector/testdata/01-setup/mutation/03-uploadStorageObject/request.json @@ -25,10 +25,10 @@ "partSize": 3573915715657741285, "sendContentMd5": false, "storageClass": "STANDARD", - "userMetadata": { + "metadata": { "Foo": "Bar" }, - "userTags": { + "tags": { "UserID": "1" }, "websiteRedirectLocation": "http://localhost:9001" diff --git a/connector/testdata/01-setup/mutation/03-uploadStorageObjectText/request.json b/connector/testdata/01-setup/mutation/03-uploadStorageObjectText/request.json index da33776..779ae29 100644 --- a/connector/testdata/01-setup/mutation/03-uploadStorageObjectText/request.json +++ b/connector/testdata/01-setup/mutation/03-uploadStorageObjectText/request.json @@ -22,16 +22,18 @@ "disableMultipart": false, "expires": "2099-01-01T00:00:00Z", "legalHold": true, - "mode": "Unlocked", - "retainUntilDate": "2099-01-01T00:00:02Z", + "retention": { + "mode": "Unlocked", + "retainUntilDate": "2099-01-01T00:00:02Z" + }, "numThreads": 2, "partSize": 35739157156577, "sendContentMd5": false, "storageClass": "STANDARD", - "userMetadata": { + "metadata": { "Foo": "Baz" }, - "userTags": { + "tags": { "UserID": "2" }, "websiteRedirectLocation": "http://localhost:9001" diff --git a/connector/testdata/01-setup/mutation/04-copyStorageObject-azblob/request.json b/connector/testdata/01-setup/mutation/04-copyStorageObject-azblob/request.json index ce19751..1a93986 100644 --- a/connector/testdata/01-setup/mutation/04-copyStorageObject-azblob/request.json +++ b/connector/testdata/01-setup/mutation/04-copyStorageObject-azblob/request.json @@ -8,13 +8,11 @@ "dest": { "bucket": "azblob-bucket-test", "object": "public/hello2.txt", - "replaceMetadata": false, - "replaceTags": true, "size": 8502136105187820186, - "userMetadata": { + "metadata": { "UserID": "2" }, - "userTags": { + "tags": { "Copy": "true" } }, diff --git a/connector/testdata/01-setup/mutation/04-copyStorageObject-gcs/expected.json b/connector/testdata/01-setup/mutation/04-copyStorageObject-gcs/expected.json new file mode 100644 index 0000000..d9cc2c3 --- /dev/null +++ b/connector/testdata/01-setup/mutation/04-copyStorageObject-gcs/expected.json @@ -0,0 +1,21 @@ +{ + "operation_results": [ + { + "result": { + "bucket": "gcs-bucket", + "checksumCrc32": null, + "checksumCrc32C": null, + "checksumCrc64Nvme": null, + "checksumSha1": null, + "checksumSha256": null, + "expiration": null, + "expirationRuleId": null, + "expires": null, + "location": null, + "name": "public/hello2.txt", + "size": 10 + }, + "type": "procedure" + } + ] +} diff --git a/connector/testdata/01-setup/mutation/04-copyStorageObject-gcs/request.json b/connector/testdata/01-setup/mutation/04-copyStorageObject-gcs/request.json new file mode 100644 index 0000000..deeccd7 --- /dev/null +++ b/connector/testdata/01-setup/mutation/04-copyStorageObject-gcs/request.json @@ -0,0 +1,86 @@ +{ + "collection_relationships": {}, + "operations": [ + { + "type": "procedure", + "name": "copyStorageObject", + "arguments": { + "clientId": "gcs", + "dest": { + "bucket": "gcs-bucket", + "object": "public/hello2.txt", + "size": 8502136105187820186, + "metadata": { + "UserID": "2" + }, + "tags": { + "Copy": "true" + } + }, + "source": { + "bucket": "gcs-bucket", + "end": 1730102302880743319, + "matchModifiedSince": "1970-01-01T00:00:01Z", + "matchRange": true, + "matchUnmodifiedSince": "2099-01-01T00:00:01Z", + "noMatchETag": "ic0R0GYP3Z", + "object": "public/hello.txt", + "start": 1 + } + }, + "fields": { + "fields": { + "bucket": { + "column": "bucket", + "type": "column" + }, + "checksumCrc32": { + "column": "checksumCrc32", + "type": "column" + }, + "checksumCrc32C": { + "column": "checksumCrc32C", + "type": "column" + }, + "checksumCrc64Nvme": { + "column": "checksumCrc64Nvme", + "type": "column" + }, + "checksumSha1": { + "column": "checksumSha1", + "type": "column" + }, + "checksumSha256": { + "column": "checksumSha256", + "type": "column" + }, + "expiration": { + "column": "expiration", + "type": "column" + }, + "expirationRuleId": { + "column": "expirationRuleId", + "type": "column" + }, + "expires": { + "column": "expires", + "type": "column" + }, + "location": { + "column": "location", + "type": "column" + }, + "name": { + "column": "name", + "type": "column" + }, + "size": { + "column": "size", + "type": "column" + } + }, + "type": "object" + } + } + ] +} diff --git a/connector/testdata/01-setup/mutation/04-copyStorageObject/request.json b/connector/testdata/01-setup/mutation/04-copyStorageObject/request.json index 7915fdc..55b4b82 100644 --- a/connector/testdata/01-setup/mutation/04-copyStorageObject/request.json +++ b/connector/testdata/01-setup/mutation/04-copyStorageObject/request.json @@ -8,13 +8,11 @@ "dest": { "bucket": "minio-bucket-test", "object": "public/workd2.txt", - "replaceMetadata": false, - "replaceTags": true, "size": 8502136105187820186, - "userMetadata": { + "metadata": { "UserID": "2" }, - "userTags": { + "tags": { "Copy": "true" } }, diff --git a/connector/testdata/01-setup/mutation/05-setStorageObjectLegalHold/request.json b/connector/testdata/01-setup/mutation/05-setStorageObjectLegalHold/request.json deleted file mode 100644 index 9355d3a..0000000 --- a/connector/testdata/01-setup/mutation/05-setStorageObjectLegalHold/request.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "collection_relationships": {}, - "operations": [ - { - "type": "procedure", - "name": "setStorageObjectLegalHold", - "arguments": { - "bucket": "minio-bucket-lock", - "object": "SiKBc6ifDC", - "status": true - } - } - ] -} diff --git a/connector/testdata/01-setup/mutation/05-setStorageObjectLegalHold/expected.json b/connector/testdata/01-setup/mutation/05-updateStorageBucketRetention/expected.json similarity index 100% rename from connector/testdata/01-setup/mutation/05-setStorageObjectLegalHold/expected.json rename to connector/testdata/01-setup/mutation/05-updateStorageBucketRetention/expected.json diff --git a/connector/testdata/01-setup/mutation/05-updateStorageBucketRetention/request.json b/connector/testdata/01-setup/mutation/05-updateStorageBucketRetention/request.json new file mode 100644 index 0000000..fd9f827 --- /dev/null +++ b/connector/testdata/01-setup/mutation/05-updateStorageBucketRetention/request.json @@ -0,0 +1,18 @@ +{ + "collection_relationships": {}, + "operations": [ + { + "type": "procedure", + "name": "updateStorageBucket", + "arguments": { + "bucket": "minio-bucket-lock", + "legalHold": true, + "objectLock": { + "mode": "Locked", + "unit": "DAYS", + "validity": 1 + } + } + } + ] +} diff --git a/connector/testdata/01-setup/mutation/06-setStorageObjectRetention/request.json b/connector/testdata/01-setup/mutation/06-setStorageObjectRetention/request.json index 5a2890c..9de4729 100644 --- a/connector/testdata/01-setup/mutation/06-setStorageObjectRetention/request.json +++ b/connector/testdata/01-setup/mutation/06-setStorageObjectRetention/request.json @@ -3,13 +3,15 @@ "operations": [ { "type": "procedure", - "name": "setStorageObjectRetention", + "name": "updateStorageObject", "arguments": { "bucket": "minio-bucket-lock", - "governanceBypass": true, - "mode": "Locked", "object": "SiKBc6ifDC", - "retainUntilDate": "2099-01-01T00:00:00Z" + "retention": { + "governanceBypass": true, + "mode": "Locked", + "retainUntilDate": "2099-01-01T00:00:00Z" + } } } ] diff --git a/connector/testdata/01-setup/mutation/07-setStorageObjectTags-azblob/request.json b/connector/testdata/01-setup/mutation/07-setStorageObjectTags-azblob/request.json index 4bd91a6..291692c 100644 --- a/connector/testdata/01-setup/mutation/07-setStorageObjectTags-azblob/request.json +++ b/connector/testdata/01-setup/mutation/07-setStorageObjectTags-azblob/request.json @@ -3,7 +3,7 @@ "operations": [ { "type": "procedure", - "name": "setStorageObjectTags", + "name": "updateStorageObject", "arguments": { "bucket": "azblob-bucket-test", "object": "public/hello.txt", diff --git a/connector/testdata/01-setup/mutation/07-setStorageObjectTags/request.json b/connector/testdata/01-setup/mutation/07-setStorageObjectTags/request.json index a9556c7..9fed04a 100644 --- a/connector/testdata/01-setup/mutation/07-setStorageObjectTags/request.json +++ b/connector/testdata/01-setup/mutation/07-setStorageObjectTags/request.json @@ -3,7 +3,7 @@ "operations": [ { "type": "procedure", - "name": "setStorageObjectTags", + "name": "updateStorageObject", "arguments": { "bucket": "minio-bucket-test", "object": "public/workd2.txt", diff --git a/connector/testdata/01-setup/mutation/02-setStorageBucketTags/expected.json b/connector/testdata/01-setup/mutation/07-updateStorageObject-gcs/expected.json similarity index 100% rename from connector/testdata/01-setup/mutation/02-setStorageBucketTags/expected.json rename to connector/testdata/01-setup/mutation/07-updateStorageObject-gcs/expected.json diff --git a/connector/testdata/01-setup/mutation/07-updateStorageObject-gcs/request.json b/connector/testdata/01-setup/mutation/07-updateStorageObject-gcs/request.json new file mode 100644 index 0000000..6919975 --- /dev/null +++ b/connector/testdata/01-setup/mutation/07-updateStorageObject-gcs/request.json @@ -0,0 +1,18 @@ +{ + "collection_relationships": {}, + "operations": [ + { + "type": "procedure", + "name": "updateStorageObject", + "arguments": { + "clientId": "gcs", + "bucket": "gcs-bucket", + "object": "public/hello.txt", + "metadata": { + "Foo": "gcp-baz", + "UserID": "3" + } + } + } + ] +} diff --git a/connector/testdata/01-setup/mutation/08-setStorageObjectLockConfig/request.json b/connector/testdata/01-setup/mutation/08-setStorageObjectLockConfig/request.json deleted file mode 100644 index bfc6b28..0000000 --- a/connector/testdata/01-setup/mutation/08-setStorageObjectLockConfig/request.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "collection_relationships": {}, - "operations": [ - { - "type": "procedure", - "name": "setStorageObjectLockConfig", - "arguments": { - "bucket": "minio-bucket-lock", - "mode": "Locked", - "unit": "DAYS", - "validity": 1 - } - } - ] -} diff --git a/connector/testdata/02-get/query/storageBucket/expected.json b/connector/testdata/02-get/query/storageBucket/expected.json new file mode 100644 index 0000000..0734cbc --- /dev/null +++ b/connector/testdata/02-get/query/storageBucket/expected.json @@ -0,0 +1,81 @@ +[ + { + "rows": [ + { + "__value": { + "autoclass": null, + "cors": [], + "customPlacementConfig": null, + "defaultEventBasedHold": null, + "encryption": null, + "etag": null, + "hierarchicalNamespace": null, + "lifecycle": { + "rules": [ + { + "abortIncompleteMultipartUpload": null, + "allVersionsExpiration": null, + "delMarkerExpiration": null, + "enabled": true, + "expiration": { + "date": null, + "days": 4, + "expiredObjectAllVersions": null, + "expiredObjectDeleteMarker": false + }, + "filter": [ + { + "matchesPrefix": null, + "matchesStorageClasses": null, + "matchesSuffix": null, + "objectSizeGreaterThan": null, + "objectSizeLessThan": null, + "tags": null + }, + { + "matchesPrefix": ["JSqXP0pJZF"], + "matchesStorageClasses": null, + "matchesSuffix": null, + "objectSizeGreaterThan": 1000000, + "objectSizeLessThan": 1, + "tags": { + "7Uw1or2jrq": "00sA2cRTCB" + } + } + ], + "id": "aXD6eFi5JE", + "noncurrentVersionExpiration": { + "newerNoncurrentVersions": 108, + "noncurrentDays": 12 + }, + "noncurrentVersionTransition": null, + "prefix": "GPJel0xa2s", + "transition": null + } + ] + }, + "locationType": null, + "logging": null, + "name": "minio-bucket-test", + "objectLock": null, + "region": null, + "requesterPays": null, + "rpo": null, + "softDeletePolicy": null, + "storageClass": null, + "tags": { + "Foo": "bar" + }, + "lastModified": null, + "versioning": { + "enabled": true, + "excludeFolders": false, + "excludedPrefixes": [], + "mfaDelete": null + }, + "website": null + } + } + ] + } +] diff --git a/connector/testdata/02-get/query/storageBucket/request.json b/connector/testdata/02-get/query/storageBucket/request.json new file mode 100644 index 0000000..7afcec8 --- /dev/null +++ b/connector/testdata/02-get/query/storageBucket/request.json @@ -0,0 +1,460 @@ +{ + "arguments": { + "clientId": { + "type": "literal", + "value": "minio" + }, + "bucket": { + "type": "literal", + "value": "minio-bucket-test" + } + }, + "collection": "storageBucket", + "collection_relationships": {}, + "query": { + "fields": { + "__value": { + "column": "__value", + "fields": { + "fields": { + "autoclass": { + "column": "autoclass", + "fields": { + "fields": { + "enabled": { + "column": "enabled", + "type": "column" + }, + "terminalStorageClass": { + "column": "terminalStorageClass", + "type": "column" + }, + "terminalStorageClassUpdateTime": { + "column": "terminalStorageClassUpdateTime", + "type": "column" + }, + "toggleTime": { + "column": "toggleTime", + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + }, + "cors": { + "column": "cors", + "fields": { + "fields": { + "fields": { + "maxAge": { + "column": "maxAge", + "type": "column" + }, + "methods": { + "column": "methods", + "type": "column" + }, + "origins": { + "column": "origins", + "type": "column" + }, + "responseHeaders": { + "column": "responseHeaders", + "type": "column" + } + }, + "type": "object" + }, + "type": "array" + }, + "type": "column" + }, + "customPlacementConfig": { + "column": "customPlacementConfig", + "fields": { + "fields": { + "DataLocations": { + "column": "DataLocations", + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + }, + "defaultEventBasedHold": { + "column": "defaultEventBasedHold", + "type": "column" + }, + "encryption": { + "column": "encryption", + "fields": { + "fields": { + "kmsMasterKeyId": { + "column": "kmsMasterKeyId", + "type": "column" + }, + "sseAlgorithm": { + "column": "sseAlgorithm", + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + }, + "etag": { + "column": "etag", + "type": "column" + }, + "hierarchicalNamespace": { + "column": "hierarchicalNamespace", + "fields": { + "fields": { + "enabled": { + "column": "enabled", + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + }, + "lifecycle": { + "column": "lifecycle", + "fields": { + "fields": { + "rules": { + "column": "rules", + "fields": { + "fields": { + "fields": { + "abortIncompleteMultipartUpload": { + "column": "abortIncompleteMultipartUpload", + "fields": { + "fields": { + "daysAfterInitiation": { + "column": "daysAfterInitiation", + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + }, + "allVersionsExpiration": { + "column": "allVersionsExpiration", + "fields": { + "fields": { + "days": { + "column": "days", + "type": "column" + }, + "deleteMarker": { + "column": "deleteMarker", + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + }, + "delMarkerExpiration": { + "column": "delMarkerExpiration", + "fields": { + "fields": { + "days": { + "column": "days", + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + }, + "enabled": { + "column": "enabled", + "type": "column" + }, + "expiration": { + "column": "expiration", + "fields": { + "fields": { + "date": { + "column": "date", + "type": "column" + }, + "days": { + "column": "days", + "type": "column" + }, + "expiredObjectAllVersions": { + "column": "expiredObjectAllVersions", + "type": "column" + }, + "expiredObjectDeleteMarker": { + "column": "expiredObjectDeleteMarker", + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + }, + "filter": { + "column": "filter", + "fields": { + "fields": { + "fields": { + "matchesPrefix": { + "column": "matchesPrefix", + "type": "column" + }, + "matchesStorageClasses": { + "column": "matchesStorageClasses", + "type": "column" + }, + "matchesSuffix": { + "column": "matchesSuffix", + "type": "column" + }, + "objectSizeGreaterThan": { + "column": "objectSizeGreaterThan", + "type": "column" + }, + "objectSizeLessThan": { + "column": "objectSizeLessThan", + "type": "column" + }, + "tags": { + "column": "tags", + "type": "column" + } + }, + "type": "object" + }, + "type": "array" + }, + "type": "column" + }, + "id": { + "column": "id", + "type": "column" + }, + "noncurrentVersionExpiration": { + "column": "noncurrentVersionExpiration", + "fields": { + "fields": { + "newerNoncurrentVersions": { + "column": "newerNoncurrentVersions", + "type": "column" + }, + "noncurrentDays": { + "column": "noncurrentDays", + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + }, + "noncurrentVersionTransition": { + "column": "noncurrentVersionTransition", + "fields": { + "fields": { + "newerNoncurrentVersions": { + "column": "newerNoncurrentVersions", + "type": "column" + }, + "noncurrentDays": { + "column": "noncurrentDays", + "type": "column" + }, + "storageClass": { + "column": "storageClass", + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + }, + "prefix": { + "column": "prefix", + "type": "column" + }, + "transition": { + "column": "transition", + "fields": { + "fields": { + "date": { + "column": "date", + "type": "column" + }, + "days": { + "column": "days", + "type": "column" + }, + "storageClass": { + "column": "storageClass", + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + } + }, + "type": "object" + }, + "type": "array" + }, + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + }, + "locationType": { + "column": "locationType", + "type": "column" + }, + "logging": { + "column": "logging", + "fields": { + "fields": { + "logBucket": { + "column": "logBucket", + "type": "column" + }, + "logObjectPrefix": { + "column": "logObjectPrefix", + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + }, + "name": { + "column": "name", + "type": "column" + }, + "objectLock": { + "column": "objectLock", + "fields": { + "fields": { + "enabled": { + "column": "enabled", + "type": "column" + }, + "mode": { + "column": "mode", + "type": "column" + }, + "unit": { + "column": "unit", + "type": "column" + }, + "validity": { + "column": "validity", + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + }, + "region": { + "column": "region", + "type": "column" + }, + "requesterPays": { + "column": "requesterPays", + "type": "column" + }, + "rpo": { + "column": "rpo", + "type": "column" + }, + "softDeletePolicy": { + "column": "softDeletePolicy", + "fields": { + "fields": { + "effectiveTime": { + "column": "effectiveTime", + "type": "column" + }, + "retentionDuration": { + "column": "retentionDuration", + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + }, + "storageClass": { + "column": "storageClass", + "type": "column" + }, + "tags": { + "column": "tags", + "type": "column" + }, + "lastModified": { + "column": "lastModified", + "type": "column" + }, + "versioning": { + "column": "versioning", + "fields": { + "fields": { + "enabled": { + "column": "enabled", + "type": "column" + }, + "excludeFolders": { + "column": "excludeFolders", + "type": "column" + }, + "excludedPrefixes": { + "column": "excludedPrefixes", + "type": "column" + }, + "mfaDelete": { + "column": "mfaDelete", + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + }, + "website": { + "column": "website", + "fields": { + "fields": { + "mainPageSuffix": { + "column": "mainPageSuffix", + "type": "column" + }, + "notFoundPage": { + "column": "notFoundPage", + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + } + } + } +} diff --git a/connector/testdata/02-get/query/storageBucketLifecycle/expected.json b/connector/testdata/02-get/query/storageBucketLifecycle/expected.json deleted file mode 100644 index 98b9664..0000000 --- a/connector/testdata/02-get/query/storageBucketLifecycle/expected.json +++ /dev/null @@ -1,44 +0,0 @@ -[ - { - "rows": [ - { - "__value": { - "rules": [ - { - "abortIncompleteMultipartUpload": null, - "allVersionsExpiration": null, - "delMarkerExpiration": null, - "expiration": { - "date": null, - "days": 4, - "expiredObjectAllVersions": null, - "expiredObjectDeleteMarker": false - }, - "filter": { - "and": { - "objectSizeGreaterThan": 1000000, - "objectSizeLessThan": 1, - "prefix": "JSqXP0pJZF", - "tags": [{ "key": "7Uw1or2jrq", "value": "00sA2cRTCB" }] - }, - "objectSizeGreaterThan": null, - "objectSizeLessThan": null, - "prefix": null, - "tag": null - }, - "id": "aXD6eFi5JE", - "noncurrentVersionExpiration": { - "newerNoncurrentVersions": 108, - "noncurrentDays": 12 - }, - "noncurrentVersionTransition": null, - "prefix": "GPJel0xa2s", - "status": "Enabled", - "transition": null - } - ] - } - } - ] - } -] diff --git a/connector/testdata/02-get/query/storageBucketLifecycle/request.json b/connector/testdata/02-get/query/storageBucketLifecycle/request.json deleted file mode 100644 index 3287e85..0000000 --- a/connector/testdata/02-get/query/storageBucketLifecycle/request.json +++ /dev/null @@ -1,253 +0,0 @@ -{ - "arguments": { - "bucket": { - "type": "literal", - "value": "minio-bucket-test" - } - }, - "collection": "storageBucketLifecycle", - "collection_relationships": {}, - "query": { - "fields": { - "__value": { - "column": "__value", - "fields": { - "fields": { - "rules": { - "column": "rules", - "fields": { - "fields": { - "fields": { - "abortIncompleteMultipartUpload": { - "column": "abortIncompleteMultipartUpload", - "fields": { - "fields": { - "daysAfterInitiation": { - "column": "daysAfterInitiation", - "type": "column" - } - }, - "type": "object" - }, - "type": "column" - }, - "allVersionsExpiration": { - "column": "allVersionsExpiration", - "fields": { - "fields": { - "days": { - "column": "days", - "type": "column" - }, - "deleteMarker": { - "column": "deleteMarker", - "type": "column" - } - }, - "type": "object" - }, - "type": "column" - }, - "delMarkerExpiration": { - "column": "delMarkerExpiration", - "fields": { - "fields": { - "days": { - "column": "days", - "type": "column" - } - }, - "type": "object" - }, - "type": "column" - }, - "expiration": { - "column": "expiration", - "fields": { - "fields": { - "date": { - "column": "date", - "type": "column" - }, - "days": { - "column": "days", - "type": "column" - }, - "expiredObjectAllVersions": { - "column": "expiredObjectAllVersions", - "type": "column" - }, - "expiredObjectDeleteMarker": { - "column": "expiredObjectDeleteMarker", - "type": "column" - } - }, - "type": "object" - }, - "type": "column" - }, - "filter": { - "column": "filter", - "fields": { - "fields": { - "and": { - "column": "and", - "fields": { - "fields": { - "objectSizeGreaterThan": { - "column": "objectSizeGreaterThan", - "type": "column" - }, - "objectSizeLessThan": { - "column": "objectSizeLessThan", - "type": "column" - }, - "prefix": { - "column": "prefix", - "type": "column" - }, - "tags": { - "column": "tags", - "fields": { - "fields": { - "fields": { - "key": { - "column": "key", - "type": "column" - }, - "value": { - "column": "value", - "type": "column" - } - }, - "type": "object" - }, - "type": "array" - }, - "type": "column" - } - }, - "type": "object" - }, - "type": "column" - }, - "objectSizeGreaterThan": { - "column": "objectSizeGreaterThan", - "type": "column" - }, - "objectSizeLessThan": { - "column": "objectSizeLessThan", - "type": "column" - }, - "prefix": { - "column": "prefix", - "type": "column" - }, - "tag": { - "column": "tag", - "fields": { - "fields": { - "key": { - "column": "key", - "type": "column" - }, - "value": { - "column": "value", - "type": "column" - } - }, - "type": "object" - }, - "type": "column" - } - }, - "type": "object" - }, - "type": "column" - }, - "id": { - "column": "id", - "type": "column" - }, - "noncurrentVersionExpiration": { - "column": "noncurrentVersionExpiration", - "fields": { - "fields": { - "newerNoncurrentVersions": { - "column": "newerNoncurrentVersions", - "type": "column" - }, - "noncurrentDays": { - "column": "noncurrentDays", - "type": "column" - } - }, - "type": "object" - }, - "type": "column" - }, - "noncurrentVersionTransition": { - "column": "noncurrentVersionTransition", - "fields": { - "fields": { - "newerNoncurrentVersions": { - "column": "newerNoncurrentVersions", - "type": "column" - }, - "noncurrentDays": { - "column": "noncurrentDays", - "type": "column" - }, - "storageClass": { - "column": "storageClass", - "type": "column" - } - }, - "type": "object" - }, - "type": "column" - }, - "prefix": { - "column": "prefix", - "type": "column" - }, - "status": { - "column": "status", - "type": "column" - }, - "transition": { - "column": "transition", - "fields": { - "fields": { - "date": { - "column": "date", - "type": "column" - }, - "days": { - "column": "days", - "type": "column" - }, - "storageClass": { - "column": "storageClass", - "type": "column" - } - }, - "type": "object" - }, - "type": "column" - } - }, - "type": "object" - }, - "type": "array" - }, - "type": "column" - } - }, - "type": "object" - }, - "type": "column" - } - } - } -} diff --git a/connector/testdata/02-get/query/storageBucketNotification/expected.json b/connector/testdata/02-get/query/storageBucketNotification/expected.json deleted file mode 100644 index 64f4651..0000000 --- a/connector/testdata/02-get/query/storageBucketNotification/expected.json +++ /dev/null @@ -1,13 +0,0 @@ -[ - { - "rows": [ - { - "__value": { - "cloudFunctionConfigurations": [], - "queueConfigurations": [], - "topicConfigurations": [] - } - } - ] - } -] diff --git a/connector/testdata/02-get/query/storageBucketNotification/request.json b/connector/testdata/02-get/query/storageBucketNotification/request.json deleted file mode 100644 index ad9fa73..0000000 --- a/connector/testdata/02-get/query/storageBucketNotification/request.json +++ /dev/null @@ -1,221 +0,0 @@ -{ - "arguments": { - "bucket": { - "type": "literal", - "value": "minio-bucket-test" - } - }, - "collection": "storageBucketNotification", - "collection_relationships": {}, - "query": { - "fields": { - "__value": { - "column": "__value", - "fields": { - "fields": { - "cloudFunctionConfigurations": { - "column": "cloudFunctionConfigurations", - "fields": { - "fields": { - "fields": { - "arn": { - "column": "arn", - "type": "column" - }, - "cloudFunction": { - "column": "cloudFunction", - "type": "column" - }, - "event": { - "column": "event", - "type": "column" - }, - "filter": { - "column": "filter", - "fields": { - "fields": { - "s3Key": { - "column": "s3Key", - "fields": { - "fields": { - "filterRule": { - "column": "filterRule", - "fields": { - "fields": { - "fields": { - "name": { - "column": "name", - "type": "column" - }, - "value": { - "column": "value", - "type": "column" - } - }, - "type": "object" - }, - "type": "array" - }, - "type": "column" - } - }, - "type": "object" - }, - "type": "column" - } - }, - "type": "object" - }, - "type": "column" - }, - "id": { - "column": "id", - "type": "column" - } - }, - "type": "object" - }, - "type": "array" - }, - "type": "column" - }, - "queueConfigurations": { - "column": "queueConfigurations", - "fields": { - "fields": { - "fields": { - "arn": { - "column": "arn", - "type": "column" - }, - "event": { - "column": "event", - "type": "column" - }, - "filter": { - "column": "filter", - "fields": { - "fields": { - "s3Key": { - "column": "s3Key", - "fields": { - "fields": { - "filterRule": { - "column": "filterRule", - "fields": { - "fields": { - "fields": { - "name": { - "column": "name", - "type": "column" - }, - "value": { - "column": "value", - "type": "column" - } - }, - "type": "object" - }, - "type": "array" - }, - "type": "column" - } - }, - "type": "object" - }, - "type": "column" - } - }, - "type": "object" - }, - "type": "column" - }, - "id": { - "column": "id", - "type": "column" - }, - "queue": { - "column": "queue", - "type": "column" - } - }, - "type": "object" - }, - "type": "array" - }, - "type": "column" - }, - "topicConfigurations": { - "column": "topicConfigurations", - "fields": { - "fields": { - "fields": { - "arn": { - "column": "arn", - "type": "column" - }, - "event": { - "column": "event", - "type": "column" - }, - "filter": { - "column": "filter", - "fields": { - "fields": { - "s3Key": { - "column": "s3Key", - "fields": { - "fields": { - "filterRule": { - "column": "filterRule", - "fields": { - "fields": { - "fields": { - "name": { - "column": "name", - "type": "column" - }, - "value": { - "column": "value", - "type": "column" - } - }, - "type": "object" - }, - "type": "array" - }, - "type": "column" - } - }, - "type": "object" - }, - "type": "column" - } - }, - "type": "object" - }, - "type": "column" - }, - "id": { - "column": "id", - "type": "column" - }, - "topic": { - "column": "topic", - "type": "column" - } - }, - "type": "object" - }, - "type": "array" - }, - "type": "column" - } - }, - "type": "object" - }, - "type": "column" - } - } - } -} diff --git a/connector/testdata/02-get/query/storageBucketPolicy/expected.json b/connector/testdata/02-get/query/storageBucketPolicy/expected.json deleted file mode 100644 index 896fcc9..0000000 --- a/connector/testdata/02-get/query/storageBucketPolicy/expected.json +++ /dev/null @@ -1 +0,0 @@ -[{ "rows": [{ "__value": "" }] }] diff --git a/connector/testdata/02-get/query/storageBucketPolicy/request.json b/connector/testdata/02-get/query/storageBucketPolicy/request.json deleted file mode 100644 index 61f71cc..0000000 --- a/connector/testdata/02-get/query/storageBucketPolicy/request.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "arguments": { - "bucket": { - "type": "literal", - "value": "minio-bucket-test" - } - }, - "collection": "storageBucketPolicy", - "collection_relationships": {}, - "query": { - "fields": { - "__value": { - "column": "__value", - "type": "column" - } - } - } -} diff --git a/connector/testdata/02-get/query/storageBucketReplication/expected.json b/connector/testdata/02-get/query/storageBucketReplication/expected.json deleted file mode 100644 index 7f1e655..0000000 --- a/connector/testdata/02-get/query/storageBucketReplication/expected.json +++ /dev/null @@ -1 +0,0 @@ -[{ "rows": [{ "__value": { "role": null, "rules": [] } }] }] diff --git a/connector/testdata/02-get/query/storageBucketReplication/request.json b/connector/testdata/02-get/query/storageBucketReplication/request.json deleted file mode 100644 index a5bef28..0000000 --- a/connector/testdata/02-get/query/storageBucketReplication/request.json +++ /dev/null @@ -1,192 +0,0 @@ -{ - "arguments": { - "bucket": { - "type": "literal", - "value": "minio-bucket-test" - } - }, - "collection": "storageBucketReplication", - "collection_relationships": {}, - "query": { - "fields": { - "__value": { - "column": "__value", - "fields": { - "fields": { - "role": { - "column": "role", - "type": "column" - }, - "rules": { - "column": "rules", - "fields": { - "fields": { - "fields": { - "deleteMarkerReplication": { - "column": "deleteMarkerReplication", - "fields": { - "fields": { - "status": { - "column": "status", - "type": "column" - } - }, - "type": "object" - }, - "type": "column" - }, - "deleteReplication": { - "column": "deleteReplication", - "fields": { - "fields": { - "status": { - "column": "status", - "type": "column" - } - }, - "type": "object" - }, - "type": "column" - }, - "destination": { - "column": "destination", - "fields": { - "fields": { - "bucket": { - "column": "bucket", - "type": "column" - }, - "storageClass": { - "column": "storageClass", - "type": "column" - } - }, - "type": "object" - }, - "type": "column" - }, - "existingObjectReplication": { - "column": "existingObjectReplication", - "fields": { - "fields": { - "status": { - "column": "status", - "type": "column" - } - }, - "type": "object" - }, - "type": "column" - }, - "filter": { - "column": "filter", - "fields": { - "fields": { - "and": { - "column": "and", - "fields": { - "fields": { - "rrefix": { - "column": "rrefix", - "type": "column" - }, - "tag": { - "column": "tag", - "fields": { - "fields": { - "fields": { - "key": { - "column": "key", - "type": "column" - }, - "value": { - "column": "value", - "type": "column" - } - }, - "type": "object" - }, - "type": "array" - }, - "type": "column" - } - }, - "type": "object" - }, - "type": "column" - }, - "rrefix": { - "column": "rrefix", - "type": "column" - }, - "tag": { - "column": "tag", - "fields": { - "fields": { - "key": { - "column": "key", - "type": "column" - }, - "value": { - "column": "value", - "type": "column" - } - }, - "type": "object" - }, - "type": "column" - } - }, - "type": "object" - }, - "type": "column" - }, - "id": { - "column": "id", - "type": "column" - }, - "priority": { - "column": "priority", - "type": "column" - }, - "sourceSelectionCriteria": { - "column": "sourceSelectionCriteria", - "fields": { - "fields": { - "replicaModifications": { - "column": "replicaModifications", - "fields": { - "fields": { - "status": { - "column": "status", - "type": "column" - } - }, - "type": "object" - }, - "type": "column" - } - }, - "type": "object" - }, - "type": "column" - }, - "status": { - "column": "status", - "type": "column" - } - }, - "type": "object" - }, - "type": "array" - }, - "type": "column" - } - }, - "type": "object" - }, - "type": "column" - } - } - } -} diff --git a/connector/testdata/02-get/query/storageBucketVersioning/expected.json b/connector/testdata/02-get/query/storageBucketVersioning/expected.json deleted file mode 100644 index e9d6b6d..0000000 --- a/connector/testdata/02-get/query/storageBucketVersioning/expected.json +++ /dev/null @@ -1,14 +0,0 @@ -[ - { - "rows": [ - { - "__value": { - "excludeFolders": false, - "excludedPrefixes": [], - "mfaDelete": null, - "status": "Enabled" - } - } - ] - } -] diff --git a/connector/testdata/02-get/query/storageBucketVersioning/request.json b/connector/testdata/02-get/query/storageBucketVersioning/request.json deleted file mode 100644 index 374b365..0000000 --- a/connector/testdata/02-get/query/storageBucketVersioning/request.json +++ /dev/null @@ -1,39 +0,0 @@ -{ - "arguments": { - "bucket": { - "type": "literal", - "value": "minio-bucket-test" - } - }, - "collection": "storageBucketVersioning", - "collection_relationships": {}, - "query": { - "fields": { - "__value": { - "column": "__value", - "fields": { - "fields": { - "excludeFolders": { - "column": "excludeFolders", - "type": "column" - }, - "excludedPrefixes": { - "column": "excludedPrefixes", - "type": "column" - }, - "mfaDelete": { - "column": "mfaDelete", - "type": "column" - }, - "status": { - "column": "status", - "type": "column" - } - }, - "type": "object" - }, - "type": "column" - } - } - } -} diff --git a/connector/testdata/02-get/query/storageBuckets-azblob/expected.json b/connector/testdata/02-get/query/storageBuckets-azblob/expected.json index c6e2f8d..2575bd6 100644 --- a/connector/testdata/02-get/query/storageBuckets-azblob/expected.json +++ b/connector/testdata/02-get/query/storageBuckets-azblob/expected.json @@ -2,7 +2,37 @@ { "rows": [ { - "__value": [{ "name": "azblob-bucket-test", "tags": { "foo": "baz" } }] + "__value": { + "buckets": [ + { + "autoclass": null, + "cors": [], + "creationTime": null, + "customPlacementConfig": null, + "defaultEventBasedHold": null, + "encryption": null, + "etag": null, + "hierarchicalNamespace": null, + "lifecycle": null, + "locationType": null, + "logging": null, + "name": "azblob-bucket-test", + "objectLock": null, + "region": null, + "requesterPays": null, + "rpo": null, + "softDeletePolicy": null, + "storageClass": null, + "tags": null, + "versioning": null, + "website": null + } + ], + "pageInfo": { + "cursor": null, + "hasNextPage": false + } + } } ] } diff --git a/connector/testdata/02-get/query/storageBuckets-azblob/request.json b/connector/testdata/02-get/query/storageBuckets-azblob/request.json index 88ef298..16aa3e4 100644 --- a/connector/testdata/02-get/query/storageBuckets-azblob/request.json +++ b/connector/testdata/02-get/query/storageBuckets-azblob/request.json @@ -1,8 +1,32 @@ { "arguments": { - "clientId": { + "maxResults": { "type": "literal", - "value": "azblob" + "value": 10 + }, + "startAfter": { + "type": "literal", + "value": null + }, + "where": { + "type": "literal", + "value": { + "type": "and", + "expressions": [ + { + "type": "binary_comparison_operator", + "column": { "type": "column", "name": "clientId", "path": [] }, + "operator": "_eq", + "value": { "type": "scalar", "value": "azblob" } + }, + { + "type": "binary_comparison_operator", + "column": { "type": "column", "name": "bucket", "path": [] }, + "operator": "_starts_with", + "value": { "type": "scalar", "value": "azblob" } + } + ] + } } }, "collection": "storageBuckets", @@ -13,19 +37,470 @@ "column": "__value", "fields": { "fields": { - "fields": { - "name": { - "column": "name", - "type": "column" + "buckets": { + "column": "buckets", + "fields": { + "fields": { + "fields": { + "autoclass": { + "column": "autoclass", + "fields": { + "fields": { + "enabled": { + "column": "enabled", + "type": "column" + }, + "terminalStorageClass": { + "column": "terminalStorageClass", + "type": "column" + }, + "terminalStorageClassUpdateTime": { + "column": "terminalStorageClassUpdateTime", + "type": "column" + }, + "toggleTime": { + "column": "toggleTime", + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + }, + "cors": { + "column": "cors", + "fields": { + "fields": { + "fields": { + "maxAge": { + "column": "maxAge", + "type": "column" + }, + "methods": { + "column": "methods", + "type": "column" + }, + "origins": { + "column": "origins", + "type": "column" + }, + "responseHeaders": { + "column": "responseHeaders", + "type": "column" + } + }, + "type": "object" + }, + "type": "array" + }, + "type": "column" + }, + "creationTime": { + "column": "creationTime", + "type": "column" + }, + "customPlacementConfig": { + "column": "customPlacementConfig", + "fields": { + "fields": { + "DataLocations": { + "column": "DataLocations", + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + }, + "defaultEventBasedHold": { + "column": "defaultEventBasedHold", + "type": "column" + }, + "encryption": { + "column": "encryption", + "fields": { + "fields": { + "kmsMasterKeyId": { + "column": "kmsMasterKeyId", + "type": "column" + }, + "sseAlgorithm": { + "column": "sseAlgorithm", + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + }, + "etag": { + "column": "etag", + "type": "column" + }, + "hierarchicalNamespace": { + "column": "hierarchicalNamespace", + "fields": { + "fields": { + "enabled": { + "column": "enabled", + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + }, + "lifecycle": { + "column": "lifecycle", + "fields": { + "fields": { + "rules": { + "column": "rules", + "fields": { + "fields": { + "fields": { + "abortIncompleteMultipartUpload": { + "column": "abortIncompleteMultipartUpload", + "fields": { + "fields": { + "daysAfterInitiation": { + "column": "daysAfterInitiation", + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + }, + "allVersionsExpiration": { + "column": "allVersionsExpiration", + "fields": { + "fields": { + "days": { + "column": "days", + "type": "column" + }, + "deleteMarker": { + "column": "deleteMarker", + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + }, + "delMarkerExpiration": { + "column": "delMarkerExpiration", + "fields": { + "fields": { + "days": { + "column": "days", + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + }, + "enabled": { + "column": "enabled", + "type": "column" + }, + "expiration": { + "column": "expiration", + "fields": { + "fields": { + "date": { + "column": "date", + "type": "column" + }, + "days": { + "column": "days", + "type": "column" + }, + "expiredObjectAllVersions": { + "column": "expiredObjectAllVersions", + "type": "column" + }, + "expiredObjectDeleteMarker": { + "column": "expiredObjectDeleteMarker", + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + }, + "filter": { + "column": "filter", + "fields": { + "fields": { + "fields": { + "matchesPrefix": { + "column": "matchesPrefix", + "type": "column" + }, + "matchesStorageClasses": { + "column": "matchesStorageClasses", + "type": "column" + }, + "matchesSuffix": { + "column": "matchesSuffix", + "type": "column" + }, + "objectSizeGreaterThan": { + "column": "objectSizeGreaterThan", + "type": "column" + }, + "objectSizeLessThan": { + "column": "objectSizeLessThan", + "type": "column" + }, + "tags": { + "column": "tags", + "type": "column" + } + }, + "type": "object" + }, + "type": "array" + }, + "type": "column" + }, + "id": { + "column": "id", + "type": "column" + }, + "noncurrentVersionExpiration": { + "column": "noncurrentVersionExpiration", + "fields": { + "fields": { + "newerNoncurrentVersions": { + "column": "newerNoncurrentVersions", + "type": "column" + }, + "noncurrentDays": { + "column": "noncurrentDays", + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + }, + "noncurrentVersionTransition": { + "column": "noncurrentVersionTransition", + "fields": { + "fields": { + "newerNoncurrentVersions": { + "column": "newerNoncurrentVersions", + "type": "column" + }, + "noncurrentDays": { + "column": "noncurrentDays", + "type": "column" + }, + "storageClass": { + "column": "storageClass", + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + }, + "prefix": { + "column": "prefix", + "type": "column" + }, + "transition": { + "column": "transition", + "fields": { + "fields": { + "date": { + "column": "date", + "type": "column" + }, + "days": { + "column": "days", + "type": "column" + }, + "storageClass": { + "column": "storageClass", + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + } + }, + "type": "object" + }, + "type": "array" + }, + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + }, + "locationType": { + "column": "locationType", + "type": "column" + }, + "logging": { + "column": "logging", + "fields": { + "fields": { + "logBucket": { + "column": "logBucket", + "type": "column" + }, + "logObjectPrefix": { + "column": "logObjectPrefix", + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + }, + "name": { + "column": "name", + "type": "column" + }, + "objectLock": { + "column": "objectLock", + "fields": { + "fields": { + "enabled": { + "column": "enabled", + "type": "column" + }, + "mode": { + "column": "mode", + "type": "column" + }, + "unit": { + "column": "unit", + "type": "column" + }, + "validity": { + "column": "validity", + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + }, + "region": { + "column": "region", + "type": "column" + }, + "requesterPays": { + "column": "requesterPays", + "type": "column" + }, + "rpo": { + "column": "rpo", + "type": "column" + }, + "softDeletePolicy": { + "column": "softDeletePolicy", + "fields": { + "fields": { + "effectiveTime": { + "column": "effectiveTime", + "type": "column" + }, + "retentionDuration": { + "column": "retentionDuration", + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + }, + "storageClass": { + "column": "storageClass", + "type": "column" + }, + "tags": { + "column": "tags", + "type": "column" + }, + "versioning": { + "column": "versioning", + "fields": { + "fields": { + "enabled": { + "column": "enabled", + "type": "column" + }, + "excludeFolders": { + "column": "excludeFolders", + "type": "column" + }, + "excludedPrefixes": { + "column": "excludedPrefixes", + "type": "column" + }, + "mfaDelete": { + "column": "mfaDelete", + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + }, + "website": { + "column": "website", + "fields": { + "fields": { + "mainPageSuffix": { + "column": "mainPageSuffix", + "type": "column" + }, + "notFoundPage": { + "column": "notFoundPage", + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + } + }, + "type": "object" + }, + "type": "array" }, - "tags": { - "column": "tags", - "type": "column" - } + "type": "column" }, - "type": "object" + "pageInfo": { + "column": "pageInfo", + "fields": { + "fields": { + "cursor": { + "column": "cursor", + "type": "column" + }, + "hasNextPage": { + "column": "hasNextPage", + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + } }, - "type": "array" + "type": "object" }, "type": "column" } diff --git a/connector/testdata/02-get/query/storageBuckets-gcs/expected.json b/connector/testdata/02-get/query/storageBuckets-gcs/expected.json new file mode 100644 index 0000000..dcfeec6 --- /dev/null +++ b/connector/testdata/02-get/query/storageBuckets-gcs/expected.json @@ -0,0 +1,45 @@ +[ + { + "rows": [ + { + "__value": { + "buckets": [ + { + "autoclass": null, + "cors": [], + "customPlacementConfig": null, + "defaultEventBasedHold": false, + "encryption": null, + "etag": "RVRhZw==", + "hierarchicalNamespace": null, + "lifecycle": { + "rules": [] + }, + "locationType": "region", + "logging": null, + "name": "gcs-bucket", + "objectLock": null, + "region": "US-CENTRAL1", + "requesterPays": false, + "rpo": null, + "softDeletePolicy": null, + "storageClass": "STANDARD", + "tags": null, + "versioning": { + "enabled": false, + "excludeFolders": null, + "excludedPrefixes": null, + "mfaDelete": null + }, + "website": null + } + ], + "pageInfo": { + "cursor": null, + "hasNextPage": false + } + } + } + ] + } +] diff --git a/connector/testdata/02-get/query/storageBuckets-gcs/request.json b/connector/testdata/02-get/query/storageBuckets-gcs/request.json new file mode 100644 index 0000000..61c7b7e --- /dev/null +++ b/connector/testdata/02-get/query/storageBuckets-gcs/request.json @@ -0,0 +1,509 @@ +{ + "arguments": { + "maxResults": { + "type": "literal", + "value": 10 + }, + "prefix": { + "type": "literal", + "value": null + }, + "startAfter": { + "type": "literal", + "value": null + }, + "where": { + "type": "literal", + "value": { + "type": "and", + "expressions": [ + { + "type": "binary_comparison_operator", + "column": { "type": "column", "name": "clientId", "path": [] }, + "operator": "_eq", + "value": { "type": "scalar", "value": "gcs" } + }, + { + "type": "binary_comparison_operator", + "column": { "type": "column", "name": "bucket", "path": [] }, + "operator": "_starts_with", + "value": { "type": "scalar", "value": "gcs" } + } + ] + } + } + }, + "collection": "storageBuckets", + "collection_relationships": {}, + "query": { + "fields": { + "__value": { + "column": "__value", + "fields": { + "fields": { + "buckets": { + "column": "buckets", + "fields": { + "fields": { + "fields": { + "autoclass": { + "column": "autoclass", + "fields": { + "fields": { + "enabled": { + "column": "enabled", + "type": "column" + }, + "terminalStorageClass": { + "column": "terminalStorageClass", + "type": "column" + }, + "terminalStorageClassUpdateTime": { + "column": "terminalStorageClassUpdateTime", + "type": "column" + }, + "toggleTime": { + "column": "toggleTime", + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + }, + "cors": { + "column": "cors", + "fields": { + "fields": { + "fields": { + "maxAge": { + "column": "maxAge", + "type": "column" + }, + "methods": { + "column": "methods", + "type": "column" + }, + "origins": { + "column": "origins", + "type": "column" + }, + "responseHeaders": { + "column": "responseHeaders", + "type": "column" + } + }, + "type": "object" + }, + "type": "array" + }, + "type": "column" + }, + "customPlacementConfig": { + "column": "customPlacementConfig", + "fields": { + "fields": { + "DataLocations": { + "column": "DataLocations", + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + }, + "defaultEventBasedHold": { + "column": "defaultEventBasedHold", + "type": "column" + }, + "encryption": { + "column": "encryption", + "fields": { + "fields": { + "kmsMasterKeyId": { + "column": "kmsMasterKeyId", + "type": "column" + }, + "sseAlgorithm": { + "column": "sseAlgorithm", + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + }, + "etag": { + "column": "etag", + "type": "column" + }, + "hierarchicalNamespace": { + "column": "hierarchicalNamespace", + "fields": { + "fields": { + "enabled": { + "column": "enabled", + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + }, + "lifecycle": { + "column": "lifecycle", + "fields": { + "fields": { + "rules": { + "column": "rules", + "fields": { + "fields": { + "fields": { + "abortIncompleteMultipartUpload": { + "column": "abortIncompleteMultipartUpload", + "fields": { + "fields": { + "daysAfterInitiation": { + "column": "daysAfterInitiation", + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + }, + "allVersionsExpiration": { + "column": "allVersionsExpiration", + "fields": { + "fields": { + "days": { + "column": "days", + "type": "column" + }, + "deleteMarker": { + "column": "deleteMarker", + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + }, + "delMarkerExpiration": { + "column": "delMarkerExpiration", + "fields": { + "fields": { + "days": { + "column": "days", + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + }, + "enabled": { + "column": "enabled", + "type": "column" + }, + "expiration": { + "column": "expiration", + "fields": { + "fields": { + "date": { + "column": "date", + "type": "column" + }, + "days": { + "column": "days", + "type": "column" + }, + "expiredObjectAllVersions": { + "column": "expiredObjectAllVersions", + "type": "column" + }, + "expiredObjectDeleteMarker": { + "column": "expiredObjectDeleteMarker", + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + }, + "filter": { + "column": "filter", + "fields": { + "fields": { + "fields": { + "matchesPrefix": { + "column": "matchesPrefix", + "type": "column" + }, + "matchesStorageClasses": { + "column": "matchesStorageClasses", + "type": "column" + }, + "matchesSuffix": { + "column": "matchesSuffix", + "type": "column" + }, + "objectSizeGreaterThan": { + "column": "objectSizeGreaterThan", + "type": "column" + }, + "objectSizeLessThan": { + "column": "objectSizeLessThan", + "type": "column" + }, + "tags": { + "column": "tags", + "type": "column" + } + }, + "type": "object" + }, + "type": "array" + }, + "type": "column" + }, + "id": { + "column": "id", + "type": "column" + }, + "noncurrentVersionExpiration": { + "column": "noncurrentVersionExpiration", + "fields": { + "fields": { + "newerNoncurrentVersions": { + "column": "newerNoncurrentVersions", + "type": "column" + }, + "noncurrentDays": { + "column": "noncurrentDays", + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + }, + "noncurrentVersionTransition": { + "column": "noncurrentVersionTransition", + "fields": { + "fields": { + "newerNoncurrentVersions": { + "column": "newerNoncurrentVersions", + "type": "column" + }, + "noncurrentDays": { + "column": "noncurrentDays", + "type": "column" + }, + "storageClass": { + "column": "storageClass", + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + }, + "prefix": { + "column": "prefix", + "type": "column" + }, + "transition": { + "column": "transition", + "fields": { + "fields": { + "date": { + "column": "date", + "type": "column" + }, + "days": { + "column": "days", + "type": "column" + }, + "storageClass": { + "column": "storageClass", + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + } + }, + "type": "object" + }, + "type": "array" + }, + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + }, + "locationType": { + "column": "locationType", + "type": "column" + }, + "logging": { + "column": "logging", + "fields": { + "fields": { + "logBucket": { + "column": "logBucket", + "type": "column" + }, + "logObjectPrefix": { + "column": "logObjectPrefix", + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + }, + "name": { + "column": "name", + "type": "column" + }, + "objectLock": { + "column": "objectLock", + "fields": { + "fields": { + "enabled": { + "column": "enabled", + "type": "column" + }, + "mode": { + "column": "mode", + "type": "column" + }, + "unit": { + "column": "unit", + "type": "column" + }, + "validity": { + "column": "validity", + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + }, + "region": { + "column": "region", + "type": "column" + }, + "requesterPays": { + "column": "requesterPays", + "type": "column" + }, + "rpo": { + "column": "rpo", + "type": "column" + }, + "softDeletePolicy": { + "column": "softDeletePolicy", + "fields": { + "fields": { + "effectiveTime": { + "column": "effectiveTime", + "type": "column" + }, + "retentionDuration": { + "column": "retentionDuration", + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + }, + "storageClass": { + "column": "storageClass", + "type": "column" + }, + "tags": { + "column": "tags", + "type": "column" + }, + "versioning": { + "column": "versioning", + "fields": { + "fields": { + "enabled": { + "column": "enabled", + "type": "column" + }, + "excludeFolders": { + "column": "excludeFolders", + "type": "column" + }, + "excludedPrefixes": { + "column": "excludedPrefixes", + "type": "column" + }, + "mfaDelete": { + "column": "mfaDelete", + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + }, + "website": { + "column": "website", + "fields": { + "fields": { + "mainPageSuffix": { + "column": "mainPageSuffix", + "type": "column" + }, + "notFoundPage": { + "column": "notFoundPage", + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + } + }, + "type": "object" + }, + "type": "array" + }, + "type": "column" + }, + "pageInfo": { + "column": "pageInfo", + "fields": { + "fields": { + "cursor": { + "column": "cursor", + "type": "column" + }, + "hasNextPage": { + "column": "hasNextPage", + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + } + } + } +} diff --git a/connector/testdata/02-get/query/storageBuckets/expected.json b/connector/testdata/02-get/query/storageBuckets/expected.json index 095d9e5..1168726 100644 --- a/connector/testdata/02-get/query/storageBuckets/expected.json +++ b/connector/testdata/02-get/query/storageBuckets/expected.json @@ -2,22 +2,83 @@ { "rows": [ { - "__value": [ - { - "name": "minio-bucket-empty", - "tags": null - }, - { - "name": "minio-bucket-lock", - "tags": null - }, - { - "name": "minio-bucket-test", - "tags": { - "Foo": "bar" + "__value": { + "buckets": [ + { + "autoclass": null, + "cors": [], + "customPlacementConfig": null, + "defaultEventBasedHold": null, + "encryption": null, + "etag": null, + "hierarchicalNamespace": null, + "lifecycle": null, + "locationType": null, + "logging": null, + "name": "minio-bucket-empty", + "objectLock": null, + "region": null, + "requesterPays": null, + "rpo": null, + "softDeletePolicy": null, + "storageClass": null, + "tags": null, + "lastModified": null, + "versioning": null, + "website": null + }, + { + "autoclass": null, + "cors": [], + "customPlacementConfig": null, + "defaultEventBasedHold": null, + "encryption": null, + "etag": null, + "hierarchicalNamespace": null, + "lifecycle": null, + "locationType": null, + "logging": null, + "name": "minio-bucket-lock", + "objectLock": null, + "region": null, + "requesterPays": null, + "rpo": null, + "softDeletePolicy": null, + "storageClass": null, + "tags": null, + "lastModified": null, + "versioning": null, + "website": null + }, + { + "autoclass": null, + "cors": [], + "customPlacementConfig": null, + "defaultEventBasedHold": null, + "encryption": null, + "etag": null, + "hierarchicalNamespace": null, + "lifecycle": null, + "locationType": null, + "logging": null, + "name": "minio-bucket-test", + "objectLock": null, + "region": null, + "requesterPays": null, + "rpo": null, + "softDeletePolicy": null, + "storageClass": null, + "tags": null, + "lastModified": null, + "versioning": null, + "website": null } + ], + "pageInfo": { + "cursor": null, + "hasNextPage": false } - ] + } } ] } diff --git a/connector/testdata/02-get/query/storageBuckets/request.json b/connector/testdata/02-get/query/storageBuckets/request.json index 2361587..44bccd7 100644 --- a/connector/testdata/02-get/query/storageBuckets/request.json +++ b/connector/testdata/02-get/query/storageBuckets/request.json @@ -1,5 +1,18 @@ { - "arguments": {}, + "arguments": { + "maxResults": { + "type": "literal", + "value": 10 + }, + "prefix": { + "type": "literal", + "value": null + }, + "startAfter": { + "type": "literal", + "value": null + } + }, "collection": "storageBuckets", "collection_relationships": {}, "query": { @@ -8,19 +21,470 @@ "column": "__value", "fields": { "fields": { - "fields": { - "name": { - "column": "name", - "type": "column" + "buckets": { + "column": "buckets", + "fields": { + "fields": { + "fields": { + "autoclass": { + "column": "autoclass", + "fields": { + "fields": { + "enabled": { + "column": "enabled", + "type": "column" + }, + "terminalStorageClass": { + "column": "terminalStorageClass", + "type": "column" + }, + "terminalStorageClassUpdateTime": { + "column": "terminalStorageClassUpdateTime", + "type": "column" + }, + "toggleTime": { + "column": "toggleTime", + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + }, + "cors": { + "column": "cors", + "fields": { + "fields": { + "fields": { + "maxAge": { + "column": "maxAge", + "type": "column" + }, + "methods": { + "column": "methods", + "type": "column" + }, + "origins": { + "column": "origins", + "type": "column" + }, + "responseHeaders": { + "column": "responseHeaders", + "type": "column" + } + }, + "type": "object" + }, + "type": "array" + }, + "type": "column" + }, + "customPlacementConfig": { + "column": "customPlacementConfig", + "fields": { + "fields": { + "DataLocations": { + "column": "DataLocations", + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + }, + "defaultEventBasedHold": { + "column": "defaultEventBasedHold", + "type": "column" + }, + "encryption": { + "column": "encryption", + "fields": { + "fields": { + "kmsMasterKeyId": { + "column": "kmsMasterKeyId", + "type": "column" + }, + "sseAlgorithm": { + "column": "sseAlgorithm", + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + }, + "etag": { + "column": "etag", + "type": "column" + }, + "hierarchicalNamespace": { + "column": "hierarchicalNamespace", + "fields": { + "fields": { + "enabled": { + "column": "enabled", + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + }, + "lifecycle": { + "column": "lifecycle", + "fields": { + "fields": { + "rules": { + "column": "rules", + "fields": { + "fields": { + "fields": { + "abortIncompleteMultipartUpload": { + "column": "abortIncompleteMultipartUpload", + "fields": { + "fields": { + "daysAfterInitiation": { + "column": "daysAfterInitiation", + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + }, + "allVersionsExpiration": { + "column": "allVersionsExpiration", + "fields": { + "fields": { + "days": { + "column": "days", + "type": "column" + }, + "deleteMarker": { + "column": "deleteMarker", + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + }, + "delMarkerExpiration": { + "column": "delMarkerExpiration", + "fields": { + "fields": { + "days": { + "column": "days", + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + }, + "enabled": { + "column": "enabled", + "type": "column" + }, + "expiration": { + "column": "expiration", + "fields": { + "fields": { + "date": { + "column": "date", + "type": "column" + }, + "days": { + "column": "days", + "type": "column" + }, + "expiredObjectAllVersions": { + "column": "expiredObjectAllVersions", + "type": "column" + }, + "expiredObjectDeleteMarker": { + "column": "expiredObjectDeleteMarker", + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + }, + "filter": { + "column": "filter", + "fields": { + "fields": { + "fields": { + "matchesPrefix": { + "column": "matchesPrefix", + "type": "column" + }, + "matchesStorageClasses": { + "column": "matchesStorageClasses", + "type": "column" + }, + "matchesSuffix": { + "column": "matchesSuffix", + "type": "column" + }, + "objectSizeGreaterThan": { + "column": "objectSizeGreaterThan", + "type": "column" + }, + "objectSizeLessThan": { + "column": "objectSizeLessThan", + "type": "column" + }, + "tags": { + "column": "tags", + "type": "column" + } + }, + "type": "object" + }, + "type": "array" + }, + "type": "column" + }, + "id": { + "column": "id", + "type": "column" + }, + "noncurrentVersionExpiration": { + "column": "noncurrentVersionExpiration", + "fields": { + "fields": { + "newerNoncurrentVersions": { + "column": "newerNoncurrentVersions", + "type": "column" + }, + "noncurrentDays": { + "column": "noncurrentDays", + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + }, + "noncurrentVersionTransition": { + "column": "noncurrentVersionTransition", + "fields": { + "fields": { + "newerNoncurrentVersions": { + "column": "newerNoncurrentVersions", + "type": "column" + }, + "noncurrentDays": { + "column": "noncurrentDays", + "type": "column" + }, + "storageClass": { + "column": "storageClass", + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + }, + "prefix": { + "column": "prefix", + "type": "column" + }, + "transition": { + "column": "transition", + "fields": { + "fields": { + "date": { + "column": "date", + "type": "column" + }, + "days": { + "column": "days", + "type": "column" + }, + "storageClass": { + "column": "storageClass", + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + } + }, + "type": "object" + }, + "type": "array" + }, + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + }, + "locationType": { + "column": "locationType", + "type": "column" + }, + "logging": { + "column": "logging", + "fields": { + "fields": { + "logBucket": { + "column": "logBucket", + "type": "column" + }, + "logObjectPrefix": { + "column": "logObjectPrefix", + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + }, + "name": { + "column": "name", + "type": "column" + }, + "objectLock": { + "column": "objectLock", + "fields": { + "fields": { + "enabled": { + "column": "enabled", + "type": "column" + }, + "mode": { + "column": "mode", + "type": "column" + }, + "unit": { + "column": "unit", + "type": "column" + }, + "validity": { + "column": "validity", + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + }, + "region": { + "column": "region", + "type": "column" + }, + "requesterPays": { + "column": "requesterPays", + "type": "column" + }, + "rpo": { + "column": "rpo", + "type": "column" + }, + "softDeletePolicy": { + "column": "softDeletePolicy", + "fields": { + "fields": { + "effectiveTime": { + "column": "effectiveTime", + "type": "column" + }, + "retentionDuration": { + "column": "retentionDuration", + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + }, + "storageClass": { + "column": "storageClass", + "type": "column" + }, + "tags": { + "column": "tags", + "type": "column" + }, + "lastModified": { + "column": "lastModified", + "type": "column" + }, + "versioning": { + "column": "versioning", + "fields": { + "fields": { + "enabled": { + "column": "enabled", + "type": "column" + }, + "excludeFolders": { + "column": "excludeFolders", + "type": "column" + }, + "excludedPrefixes": { + "column": "excludedPrefixes", + "type": "column" + }, + "mfaDelete": { + "column": "mfaDelete", + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + }, + "website": { + "column": "website", + "fields": { + "fields": { + "mainPageSuffix": { + "column": "mainPageSuffix", + "type": "column" + }, + "notFoundPage": { + "column": "notFoundPage", + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + } + }, + "type": "object" + }, + "type": "array" }, - "tags": { - "column": "tags", - "type": "column" - } + "type": "column" }, - "type": "object" + "pageInfo": { + "column": "pageInfo", + "fields": { + "fields": { + "cursor": { + "column": "cursor", + "type": "column" + }, + "hasNextPage": { + "column": "hasNextPage", + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + } }, - "type": "array" + "type": "object" }, "type": "column" } diff --git a/connector/testdata/02-get/query/storageDeletedObjects-azblob/expected.json b/connector/testdata/02-get/query/storageDeletedObjects-azblob/expected.json new file mode 100644 index 0000000..3745572 --- /dev/null +++ b/connector/testdata/02-get/query/storageDeletedObjects-azblob/expected.json @@ -0,0 +1,15 @@ +[ + { + "rows": [ + { + "__value": { + "objects": [], + "pageInfo": { + "cursor": null, + "hasNextPage": false + } + } + } + ] + } +] diff --git a/connector/testdata/02-get/query/storageDeletedObjects-azblob/request.json b/connector/testdata/02-get/query/storageDeletedObjects-azblob/request.json new file mode 100644 index 0000000..90bfa15 --- /dev/null +++ b/connector/testdata/02-get/query/storageDeletedObjects-azblob/request.json @@ -0,0 +1,402 @@ +{ + "arguments": { + "recursive": { "type": "literal", "value": true }, + "maxResults": { "type": "literal", "value": 1 }, + "where": { + "type": "literal", + "value": { + "type": "and", + "expressions": [ + { + "type": "binary_comparison_operator", + "column": { "type": "column", "name": "bucket", "path": [] }, + "operator": "_eq", + "value": { "type": "scalar", "value": "azblob-bucket-test" } + }, + { + "type": "binary_comparison_operator", + "column": { "type": "column", "name": "object", "path": [] }, + "operator": "_starts_with", + "value": { "type": "scalar", "value": "public" } + } + ] + } + } + }, + "collection": "storageDeletedObjects", + "collection_relationships": {}, + "query": { + "fields": { + "__value": { + "column": "__value", + "fields": { + "fields": { + "objects": { + "column": "objects", + "fields": { + "fields": { + "fields": { + "accessTierChangeTime": { + "column": "accessTierChangeTime", + "type": "column" + }, + "accessTierInferred": { + "column": "accessTierInferred", + "type": "column" + }, + "acl": { + "column": "acl", + "type": "column" + }, + "archiveStatus": { + "column": "archiveStatus", + "type": "column" + }, + "blobSequenceNumber": { + "column": "blobSequenceNumber", + "type": "column" + }, + "blobType": { + "column": "blobType", + "type": "column" + }, + "bucket": { + "column": "bucket", + "type": "column" + }, + "cacheControl": { + "column": "cacheControl", + "type": "column" + }, + "checksumCrc32": { + "column": "checksumCrc32", + "type": "column" + }, + "checksumCrc32C": { + "column": "checksumCrc32C", + "type": "column" + }, + "checksumCrc64Nvme": { + "column": "checksumCrc64Nvme", + "type": "column" + }, + "checksumSha1": { + "column": "checksumSha1", + "type": "column" + }, + "checksumSha256": { + "column": "checksumSha256", + "type": "column" + }, + "clientId": { + "column": "clientId", + "type": "column" + }, + "contentDisposition": { + "column": "contentDisposition", + "type": "column" + }, + "contentEncoding": { + "column": "contentEncoding", + "type": "column" + }, + "contentLanguage": { + "column": "contentLanguage", + "type": "column" + }, + "contentMd5": { + "column": "contentMd5", + "type": "column" + }, + "contentType": { + "column": "contentType", + "type": "column" + }, + "copy": { + "column": "copy", + "fields": { + "fields": { + "completionTime": { + "column": "completionTime", + "type": "column" + }, + "id": { + "column": "id", + "type": "column" + }, + "progress": { + "column": "progress", + "type": "column" + }, + "source": { + "column": "source", + "type": "column" + }, + "status": { + "column": "status", + "type": "column" + }, + "statusDescription": { + "column": "statusDescription", + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + }, + "creationTime": { + "column": "creationTime", + "type": "column" + }, + "customerProvidedKeySha256": { + "column": "customerProvidedKeySha256", + "type": "column" + }, + "deleted": { + "column": "deleted", + "type": "column" + }, + "deletedTime": { + "column": "deletedTime", + "type": "column" + }, + "destinationSnapshot": { + "column": "destinationSnapshot", + "type": "column" + }, + "etag": { + "column": "etag", + "type": "column" + }, + "expiration": { + "column": "expiration", + "type": "column" + }, + "expirationRuleId": { + "column": "expirationRuleId", + "type": "column" + }, + "expires": { + "column": "expires", + "type": "column" + }, + "grant": { + "column": "grant", + "fields": { + "fields": { + "fields": { + "grantee": { + "column": "grantee", + "fields": { + "fields": { + "displayName": { + "column": "displayName", + "type": "column" + }, + "id": { + "column": "id", + "type": "column" + }, + "uri": { + "column": "uri", + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + }, + "permission": { + "column": "permission", + "type": "column" + } + }, + "type": "object" + }, + "type": "array" + }, + "type": "column" + }, + "group": { + "column": "group", + "type": "column" + }, + "incrementalCopy": { + "column": "incrementalCopy", + "type": "column" + }, + "isLatest": { + "column": "isLatest", + "type": "column" + }, + "kmsKeyName": { + "column": "kmsKeyName", + "type": "column" + }, + "lastAccessTime": { + "column": "lastAccessTime", + "type": "column" + }, + "lastModified": { + "column": "lastModified", + "type": "column" + }, + "leaseDuration": { + "column": "leaseDuration", + "type": "column" + }, + "leaseState": { + "column": "leaseState", + "type": "column" + }, + "leaseStatus": { + "column": "leaseStatus", + "type": "column" + }, + "legalHold": { + "column": "legalHold", + "type": "column" + }, + "mediaLink": { + "column": "mediaLink", + "type": "column" + }, + "metadata": { + "column": "metadata", + "type": "column" + }, + "name": { + "column": "name", + "type": "column" + }, + "owner": { + "column": "owner", + "fields": { + "fields": { + "id": { + "column": "id", + "type": "column" + }, + "name": { + "column": "name", + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + }, + "permissions": { + "column": "permissions", + "type": "column" + }, + "rawMetadata": { + "column": "rawMetadata", + "type": "column" + }, + "rehydratePriority": { + "column": "rehydratePriority", + "type": "column" + }, + "remainingRetentionDays": { + "column": "remainingRetentionDays", + "type": "column" + }, + "replicationReady": { + "column": "replicationReady", + "type": "column" + }, + "replicationStatus": { + "column": "replicationStatus", + "type": "column" + }, + "resourceType": { + "column": "resourceType", + "type": "column" + }, + "restore": { + "column": "restore", + "fields": { + "fields": { + "expiryTime": { + "column": "expiryTime", + "type": "column" + }, + "ongoingRestore": { + "column": "ongoingRestore", + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + }, + "retentionMode": { + "column": "retentionMode", + "type": "column" + }, + "retentionUntilDate": { + "column": "retentionUntilDate", + "type": "column" + }, + "sealed": { + "column": "sealed", + "type": "column" + }, + "serverEncrypted": { + "column": "serverEncrypted", + "type": "column" + }, + "size": { + "column": "size", + "type": "column" + }, + "storageClass": { + "column": "storageClass", + "type": "column" + }, + "tagCount": { + "column": "tagCount", + "type": "column" + }, + "tags": { + "column": "tags", + "type": "column" + }, + "versionId": { + "column": "versionId", + "type": "column" + } + }, + "type": "object" + }, + "type": "array" + }, + "type": "column" + }, + "pageInfo": { + "column": "pageInfo", + "fields": { + "fields": { + "cursor": { + "column": "cursor", + "type": "column" + }, + "hasNextPage": { + "column": "hasNextPage", + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + } + } + } +} diff --git a/connector/testdata/02-get/query/storageDeletedObjects-gcs/expected.json b/connector/testdata/02-get/query/storageDeletedObjects-gcs/expected.json new file mode 100644 index 0000000..3745572 --- /dev/null +++ b/connector/testdata/02-get/query/storageDeletedObjects-gcs/expected.json @@ -0,0 +1,15 @@ +[ + { + "rows": [ + { + "__value": { + "objects": [], + "pageInfo": { + "cursor": null, + "hasNextPage": false + } + } + } + ] + } +] diff --git a/connector/testdata/02-get/query/storageDeletedObjects-gcs/request.json b/connector/testdata/02-get/query/storageDeletedObjects-gcs/request.json new file mode 100644 index 0000000..95a8322 --- /dev/null +++ b/connector/testdata/02-get/query/storageDeletedObjects-gcs/request.json @@ -0,0 +1,412 @@ +{ + "arguments": { + "maxResults": { + "type": "literal", + "value": 10 + }, + "prefix": { + "type": "literal", + "value": null + }, + "startAfter": { + "type": "literal", + "value": null + }, + "where": { + "type": "literal", + "value": { + "type": "and", + "expressions": [ + { + "type": "binary_comparison_operator", + "column": { "type": "column", "name": "clientId", "path": [] }, + "operator": "_eq", + "value": { "type": "scalar", "value": "gcs" } + }, + { + "type": "binary_comparison_operator", + "column": { "type": "column", "name": "bucket", "path": [] }, + "operator": "_eq", + "value": { "type": "scalar", "value": "gcs-bucket" } + } + ] + } + } + }, + "collection": "storageDeletedObjects", + "collection_relationships": {}, + "query": { + "fields": { + "__value": { + "column": "__value", + "fields": { + "fields": { + "objects": { + "column": "objects", + "fields": { + "fields": { + "fields": { + "accessTierChangeTime": { + "column": "accessTierChangeTime", + "type": "column" + }, + "accessTierInferred": { + "column": "accessTierInferred", + "type": "column" + }, + "acl": { + "column": "acl", + "type": "column" + }, + "archiveStatus": { + "column": "archiveStatus", + "type": "column" + }, + "blobSequenceNumber": { + "column": "blobSequenceNumber", + "type": "column" + }, + "blobType": { + "column": "blobType", + "type": "column" + }, + "bucket": { + "column": "bucket", + "type": "column" + }, + "cacheControl": { + "column": "cacheControl", + "type": "column" + }, + "checksumCrc32": { + "column": "checksumCrc32", + "type": "column" + }, + "checksumCrc32C": { + "column": "checksumCrc32C", + "type": "column" + }, + "checksumCrc64Nvme": { + "column": "checksumCrc64Nvme", + "type": "column" + }, + "checksumSha1": { + "column": "checksumSha1", + "type": "column" + }, + "checksumSha256": { + "column": "checksumSha256", + "type": "column" + }, + "clientId": { + "column": "clientId", + "type": "column" + }, + "contentDisposition": { + "column": "contentDisposition", + "type": "column" + }, + "contentEncoding": { + "column": "contentEncoding", + "type": "column" + }, + "contentLanguage": { + "column": "contentLanguage", + "type": "column" + }, + "contentMd5": { + "column": "contentMd5", + "type": "column" + }, + "contentType": { + "column": "contentType", + "type": "column" + }, + "copy": { + "column": "copy", + "fields": { + "fields": { + "completionTime": { + "column": "completionTime", + "type": "column" + }, + "id": { + "column": "id", + "type": "column" + }, + "progress": { + "column": "progress", + "type": "column" + }, + "source": { + "column": "source", + "type": "column" + }, + "status": { + "column": "status", + "type": "column" + }, + "statusDescription": { + "column": "statusDescription", + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + }, + "creationTime": { + "column": "creationTime", + "type": "column" + }, + "customerProvidedKeySha256": { + "column": "customerProvidedKeySha256", + "type": "column" + }, + "deleted": { + "column": "deleted", + "type": "column" + }, + "deletedTime": { + "column": "deletedTime", + "type": "column" + }, + "destinationSnapshot": { + "column": "destinationSnapshot", + "type": "column" + }, + "etag": { + "column": "etag", + "type": "column" + }, + "expiration": { + "column": "expiration", + "type": "column" + }, + "expirationRuleId": { + "column": "expirationRuleId", + "type": "column" + }, + "expires": { + "column": "expires", + "type": "column" + }, + "grant": { + "column": "grant", + "fields": { + "fields": { + "fields": { + "grantee": { + "column": "grantee", + "fields": { + "fields": { + "displayName": { + "column": "displayName", + "type": "column" + }, + "id": { + "column": "id", + "type": "column" + }, + "uri": { + "column": "uri", + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + }, + "permission": { + "column": "permission", + "type": "column" + } + }, + "type": "object" + }, + "type": "array" + }, + "type": "column" + }, + "group": { + "column": "group", + "type": "column" + }, + "incrementalCopy": { + "column": "incrementalCopy", + "type": "column" + }, + "isLatest": { + "column": "isLatest", + "type": "column" + }, + "kmsKeyName": { + "column": "kmsKeyName", + "type": "column" + }, + "lastAccessTime": { + "column": "lastAccessTime", + "type": "column" + }, + "lastModified": { + "column": "lastModified", + "type": "column" + }, + "leaseDuration": { + "column": "leaseDuration", + "type": "column" + }, + "leaseState": { + "column": "leaseState", + "type": "column" + }, + "leaseStatus": { + "column": "leaseStatus", + "type": "column" + }, + "legalHold": { + "column": "legalHold", + "type": "column" + }, + "mediaLink": { + "column": "mediaLink", + "type": "column" + }, + "metadata": { + "column": "metadata", + "type": "column" + }, + "name": { + "column": "name", + "type": "column" + }, + "owner": { + "column": "owner", + "fields": { + "fields": { + "id": { + "column": "id", + "type": "column" + }, + "name": { + "column": "name", + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + }, + "permissions": { + "column": "permissions", + "type": "column" + }, + "rawMetadata": { + "column": "rawMetadata", + "type": "column" + }, + "rehydratePriority": { + "column": "rehydratePriority", + "type": "column" + }, + "remainingRetentionDays": { + "column": "remainingRetentionDays", + "type": "column" + }, + "replicationReady": { + "column": "replicationReady", + "type": "column" + }, + "replicationStatus": { + "column": "replicationStatus", + "type": "column" + }, + "resourceType": { + "column": "resourceType", + "type": "column" + }, + "restore": { + "column": "restore", + "fields": { + "fields": { + "expiryTime": { + "column": "expiryTime", + "type": "column" + }, + "ongoingRestore": { + "column": "ongoingRestore", + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + }, + "retentionMode": { + "column": "retentionMode", + "type": "column" + }, + "retentionUntilDate": { + "column": "retentionUntilDate", + "type": "column" + }, + "sealed": { + "column": "sealed", + "type": "column" + }, + "serverEncrypted": { + "column": "serverEncrypted", + "type": "column" + }, + "size": { + "column": "size", + "type": "column" + }, + "storageClass": { + "column": "storageClass", + "type": "column" + }, + "tagCount": { + "column": "tagCount", + "type": "column" + }, + "tags": { + "column": "tags", + "type": "column" + }, + "versionId": { + "column": "versionId", + "type": "column" + } + }, + "type": "object" + }, + "type": "array" + }, + "type": "column" + }, + "pageInfo": { + "column": "pageInfo", + "fields": { + "fields": { + "cursor": { + "column": "cursor", + "type": "column" + }, + "hasNextPage": { + "column": "hasNextPage", + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + } + } + } +} diff --git a/connector/testdata/02-get/query/storageDeletedObjects/expected.json b/connector/testdata/02-get/query/storageDeletedObjects/expected.json new file mode 100644 index 0000000..3745572 --- /dev/null +++ b/connector/testdata/02-get/query/storageDeletedObjects/expected.json @@ -0,0 +1,15 @@ +[ + { + "rows": [ + { + "__value": { + "objects": [], + "pageInfo": { + "cursor": null, + "hasNextPage": false + } + } + } + ] + } +] diff --git a/connector/testdata/02-get/query/storageDeletedObjects/request.json b/connector/testdata/02-get/query/storageDeletedObjects/request.json new file mode 100644 index 0000000..79b7ddc --- /dev/null +++ b/connector/testdata/02-get/query/storageDeletedObjects/request.json @@ -0,0 +1,392 @@ +{ + "arguments": { + "maxResults": { + "type": "literal", + "value": 10 + }, + "prefix": { + "type": "literal", + "value": null + }, + "startAfter": { + "type": "literal", + "value": null + } + }, + "collection": "storageDeletedObjects", + "collection_relationships": {}, + "query": { + "fields": { + "__value": { + "column": "__value", + "fields": { + "fields": { + "objects": { + "column": "objects", + "fields": { + "fields": { + "fields": { + "accessTierChangeTime": { + "column": "accessTierChangeTime", + "type": "column" + }, + "accessTierInferred": { + "column": "accessTierInferred", + "type": "column" + }, + "acl": { + "column": "acl", + "type": "column" + }, + "archiveStatus": { + "column": "archiveStatus", + "type": "column" + }, + "blobSequenceNumber": { + "column": "blobSequenceNumber", + "type": "column" + }, + "blobType": { + "column": "blobType", + "type": "column" + }, + "bucket": { + "column": "bucket", + "type": "column" + }, + "cacheControl": { + "column": "cacheControl", + "type": "column" + }, + "checksumCrc32": { + "column": "checksumCrc32", + "type": "column" + }, + "checksumCrc32C": { + "column": "checksumCrc32C", + "type": "column" + }, + "checksumCrc64Nvme": { + "column": "checksumCrc64Nvme", + "type": "column" + }, + "checksumSha1": { + "column": "checksumSha1", + "type": "column" + }, + "checksumSha256": { + "column": "checksumSha256", + "type": "column" + }, + "clientId": { + "column": "clientId", + "type": "column" + }, + "contentDisposition": { + "column": "contentDisposition", + "type": "column" + }, + "contentEncoding": { + "column": "contentEncoding", + "type": "column" + }, + "contentLanguage": { + "column": "contentLanguage", + "type": "column" + }, + "contentMd5": { + "column": "contentMd5", + "type": "column" + }, + "contentType": { + "column": "contentType", + "type": "column" + }, + "copy": { + "column": "copy", + "fields": { + "fields": { + "completionTime": { + "column": "completionTime", + "type": "column" + }, + "id": { + "column": "id", + "type": "column" + }, + "progress": { + "column": "progress", + "type": "column" + }, + "source": { + "column": "source", + "type": "column" + }, + "status": { + "column": "status", + "type": "column" + }, + "statusDescription": { + "column": "statusDescription", + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + }, + "creationTime": { + "column": "creationTime", + "type": "column" + }, + "customerProvidedKeySha256": { + "column": "customerProvidedKeySha256", + "type": "column" + }, + "deleted": { + "column": "deleted", + "type": "column" + }, + "deletedTime": { + "column": "deletedTime", + "type": "column" + }, + "destinationSnapshot": { + "column": "destinationSnapshot", + "type": "column" + }, + "etag": { + "column": "etag", + "type": "column" + }, + "expiration": { + "column": "expiration", + "type": "column" + }, + "expirationRuleId": { + "column": "expirationRuleId", + "type": "column" + }, + "expires": { + "column": "expires", + "type": "column" + }, + "grant": { + "column": "grant", + "fields": { + "fields": { + "fields": { + "grantee": { + "column": "grantee", + "fields": { + "fields": { + "displayName": { + "column": "displayName", + "type": "column" + }, + "id": { + "column": "id", + "type": "column" + }, + "uri": { + "column": "uri", + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + }, + "permission": { + "column": "permission", + "type": "column" + } + }, + "type": "object" + }, + "type": "array" + }, + "type": "column" + }, + "group": { + "column": "group", + "type": "column" + }, + "incrementalCopy": { + "column": "incrementalCopy", + "type": "column" + }, + "isLatest": { + "column": "isLatest", + "type": "column" + }, + "kmsKeyName": { + "column": "kmsKeyName", + "type": "column" + }, + "lastAccessTime": { + "column": "lastAccessTime", + "type": "column" + }, + "lastModified": { + "column": "lastModified", + "type": "column" + }, + "leaseDuration": { + "column": "leaseDuration", + "type": "column" + }, + "leaseState": { + "column": "leaseState", + "type": "column" + }, + "leaseStatus": { + "column": "leaseStatus", + "type": "column" + }, + "legalHold": { + "column": "legalHold", + "type": "column" + }, + "mediaLink": { + "column": "mediaLink", + "type": "column" + }, + "metadata": { + "column": "metadata", + "type": "column" + }, + "name": { + "column": "name", + "type": "column" + }, + "owner": { + "column": "owner", + "fields": { + "fields": { + "id": { + "column": "id", + "type": "column" + }, + "name": { + "column": "name", + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + }, + "permissions": { + "column": "permissions", + "type": "column" + }, + "rawMetadata": { + "column": "rawMetadata", + "type": "column" + }, + "rehydratePriority": { + "column": "rehydratePriority", + "type": "column" + }, + "remainingRetentionDays": { + "column": "remainingRetentionDays", + "type": "column" + }, + "replicationReady": { + "column": "replicationReady", + "type": "column" + }, + "replicationStatus": { + "column": "replicationStatus", + "type": "column" + }, + "resourceType": { + "column": "resourceType", + "type": "column" + }, + "restore": { + "column": "restore", + "fields": { + "fields": { + "expiryTime": { + "column": "expiryTime", + "type": "column" + }, + "ongoingRestore": { + "column": "ongoingRestore", + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + }, + "retentionMode": { + "column": "retentionMode", + "type": "column" + }, + "retentionUntilDate": { + "column": "retentionUntilDate", + "type": "column" + }, + "sealed": { + "column": "sealed", + "type": "column" + }, + "serverEncrypted": { + "column": "serverEncrypted", + "type": "column" + }, + "size": { + "column": "size", + "type": "column" + }, + "storageClass": { + "column": "storageClass", + "type": "column" + }, + "tagCount": { + "column": "tagCount", + "type": "column" + }, + "tags": { + "column": "tags", + "type": "column" + }, + "versionId": { + "column": "versionId", + "type": "column" + } + }, + "type": "object" + }, + "type": "array" + }, + "type": "column" + }, + "pageInfo": { + "column": "pageInfo", + "fields": { + "fields": { + "cursor": { + "column": "cursor", + "type": "column" + }, + "hasNextPage": { + "column": "hasNextPage", + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + } + } + } +} diff --git a/connector/testdata/02-get/query/storageObject-azblob/expected.json b/connector/testdata/02-get/query/storageObject-azblob/expected.json index b533180..a895bc3 100644 --- a/connector/testdata/02-get/query/storageObject-azblob/expected.json +++ b/connector/testdata/02-get/query/storageObject-azblob/expected.json @@ -21,46 +21,49 @@ "contentLanguage": null, "contentMd5": "KZ0tkTzqqDd9IiRNs5cjyg==", "contentType": "application/octet-stream", - "copyCompletionTime": null, - "copyId": null, - "copyProgress": null, - "copySource": null, - "copyStatus": null, - "copyStatusDescription": null, + "copy": null, "customerProvidedKeySha256": null, "deleted": null, + "deletedTime": null, "destinationSnapshot": null, - "encryptionScope": null, "expiration": null, "expirationRuleId": null, "expires": null, "grant": [], "group": null, - "immutabilityPolicyMode": null, - "immutabilityPolicyUntilDate": null, "incrementalCopy": null, - "lastAccessTime": null, + "isLatest": null, + "kmsKeyName": null, "leaseDuration": null, "leaseState": "available", "leaseStatus": "unlocked", "legalHold": null, - "metadata": { "foo": "Bar" }, + "mediaLink": null, + "metadata": { + "foo": "Bar" + }, "name": "public/hello.txt", "owner": null, "permissions": null, + "rawMetadata": null, "rehydratePriority": null, "remainingRetentionDays": null, "replicationReady": null, "replicationStatus": null, "resourceType": null, "restore": null, + "retentionMode": null, + "retentionUntilDate": null, "sealed": null, "serverEncrypted": true, "size": 10, "storageClass": "Cool", - "userMetadata": {}, - "userTagCount": 2, - "userTags": { "Foo": "baz", "UserID": "3" } + "tagCount": 2, + "tags": { + "Foo": "baz", + "UserID": "3" + }, + "versionId": null } } ] diff --git a/connector/testdata/02-get/query/storageObject-azblob/request.json b/connector/testdata/02-get/query/storageObject-azblob/request.json index 8989f07..ff5ddb9 100644 --- a/connector/testdata/02-get/query/storageObject-azblob/request.json +++ b/connector/testdata/02-get/query/storageObject-azblob/request.json @@ -113,42 +113,55 @@ "column": "contentType", "type": "column" }, - "copyCompletionTime": { - "column": "copyCompletionTime", - "type": "column" - }, - "copyId": { - "column": "copyId", - "type": "column" - }, - "copyProgress": { - "column": "copyProgress", - "type": "column" - }, - "copySource": { - "column": "copySource", + "copy": { + "column": "copy", + "fields": { + "fields": { + "completionTime": { + "column": "completionTime", + "type": "column" + }, + "id": { + "column": "id", + "type": "column" + }, + "progress": { + "column": "progress", + "type": "column" + }, + "source": { + "column": "source", + "type": "column" + }, + "status": { + "column": "status", + "type": "column" + }, + "statusDescription": { + "column": "statusDescription", + "type": "column" + } + }, + "type": "object" + }, "type": "column" }, - "copyStatus": { - "column": "copyStatus", + "customerProvidedKeySha256": { + "column": "customerProvidedKeySha256", "type": "column" }, - "copyStatusDescription": { - "column": "copyStatusDescription", + "deleted": { + "column": "deleted", "type": "column" }, - "customerProvidedKeySha256": { - "column": "customerProvidedKeySha256", + "deletedTime": { + "column": "deletedTime", "type": "column" }, "destinationSnapshot": { "column": "destinationSnapshot", "type": "column" }, - "encryptionScope": { - "column": "encryptionScope", - "type": "column" - }, "expiration": { "column": "expiration", "type": "column" @@ -202,24 +215,16 @@ "column": "group", "type": "column" }, - "immutabilityPolicyMode": { - "column": "immutabilityPolicyMode", - "type": "column" - }, - "immutabilityPolicyUntilDate": { - "column": "immutabilityPolicyUntilDate", - "type": "column" - }, "incrementalCopy": { "column": "incrementalCopy", "type": "column" }, - "deleted": { - "column": "deleted", + "isLatest": { + "column": "isLatest", "type": "column" }, - "lastAccessTime": { - "column": "lastAccessTime", + "kmsKeyName": { + "column": "kmsKeyName", "type": "column" }, "leaseDuration": { @@ -238,6 +243,10 @@ "column": "legalHold", "type": "column" }, + "mediaLink": { + "column": "mediaLink", + "type": "column" + }, "metadata": { "column": "metadata", "type": "column" @@ -267,6 +276,10 @@ "column": "permissions", "type": "column" }, + "rawMetadata": { + "column": "rawMetadata", + "type": "column" + }, "rehydratePriority": { "column": "rehydratePriority", "type": "column" @@ -304,6 +317,14 @@ }, "type": "column" }, + "retentionMode": { + "column": "retentionMode", + "type": "column" + }, + "retentionUntilDate": { + "column": "retentionUntilDate", + "type": "column" + }, "sealed": { "column": "sealed", "type": "column" @@ -320,16 +341,16 @@ "column": "storageClass", "type": "column" }, - "userMetadata": { - "column": "userMetadata", + "tagCount": { + "column": "tagCount", "type": "column" }, - "userTagCount": { - "column": "userTagCount", + "tags": { + "column": "tags", "type": "column" }, - "userTags": { - "column": "userTags", + "versionId": { + "column": "versionId", "type": "column" } }, diff --git a/connector/testdata/02-get/query/storageObject-gcs/expected.json b/connector/testdata/02-get/query/storageObject-gcs/expected.json new file mode 100644 index 0000000..6a17cb1 --- /dev/null +++ b/connector/testdata/02-get/query/storageObject-gcs/expected.json @@ -0,0 +1,76 @@ +[ + { + "rows": [ + { + "__value": { + "accessTierChangeTime": null, + "accessTierInferred": null, + "acl": [ + { + "entity": "projectOwner-test-project", + "projectTeam": {}, + "role": "OWNER" + } + ], + "archiveStatus": null, + "blobSequenceNumber": null, + "blobType": null, + "bucket": "gcs-bucket", + "cacheControl": null, + "checksumCrc32": null, + "checksumCrc32C": null, + "checksumCrc64Nvme": null, + "checksumSha1": null, + "checksumSha256": null, + "clientId": "gcs", + "contentDisposition": "attachment", + "contentEncoding": "gzip", + "contentLanguage": "en-US", + "contentMd5": "KZ0tkTzqqDd9IiRNs5cjyg==", + "contentType": "text/plain", + "copy": null, + "customerProvidedKeySha256": null, + "deleted": null, + "deletedTime": null, + "destinationSnapshot": null, + "etag": "KZ0tkTzqqDd9IiRNs5cjyg==", + "expiration": null, + "expirationRuleId": null, + "expires": null, + "grant": [], + "group": null, + "incrementalCopy": null, + "isLatest": null, + "kmsKeyName": null, + "leaseDuration": null, + "leaseState": null, + "leaseStatus": null, + "legalHold": false, + "mediaLink": "http://0.0.0.0:4443/download/storage/v1/b/gcs-bucket/o/public%2Fhello.txt?alt=media", + "metadata": { + "Foo": "gcp-baz", + "UserID": "3" + }, + "name": "public/hello.txt", + "owner": null, + "permissions": null, + "rawMetadata": null, + "rehydratePriority": null, + "remainingRetentionDays": null, + "replicationReady": null, + "replicationStatus": null, + "resourceType": null, + "restore": null, + "retentionMode": null, + "retentionUntilDate": null, + "sealed": null, + "serverEncrypted": null, + "size": 10, + "storageClass": "Cool", + "tagCount": 0, + "tags": null + } + } + ] + } +] diff --git a/connector/testdata/02-get/query/storageObject-gcs/request.json b/connector/testdata/02-get/query/storageObject-gcs/request.json new file mode 100644 index 0000000..751f4ce --- /dev/null +++ b/connector/testdata/02-get/query/storageObject-gcs/request.json @@ -0,0 +1,367 @@ +{ + "arguments": { + "clientId": { + "type": "literal", + "value": "gcs" + }, + "bucket": { + "type": "literal", + "value": "gcs-bucket" + }, + "object": { + "type": "literal", + "value": "public/hello.txt" + }, + "where": { + "type": "literal", + "value": { + "type": "and", + "expressions": [ + { + "type": "binary_comparison_operator", + "column": { "type": "column", "name": "object", "path": [] }, + "operator": "_starts_with", + "value": { "type": "scalar", "value": "public" } + }, + { + "type": "binary_comparison_operator", + "column": { "type": "column", "name": "bucket", "path": [] }, + "operator": "_eq", + "value": { "type": "scalar", "value": "gcs-bucket" } + } + ] + } + } + }, + "collection": "storageObject", + "collection_relationships": {}, + "query": { + "fields": { + "__value": { + "column": "__value", + "fields": { + "fields": { + "accessTierChangeTime": { + "column": "accessTierChangeTime", + "type": "column" + }, + "accessTierInferred": { + "column": "accessTierInferred", + "type": "column" + }, + "acl": { + "column": "acl", + "type": "column" + }, + "archiveStatus": { + "column": "archiveStatus", + "type": "column" + }, + "blobSequenceNumber": { + "column": "blobSequenceNumber", + "type": "column" + }, + "blobType": { + "column": "blobType", + "type": "column" + }, + "bucket": { + "column": "bucket", + "type": "column" + }, + "cacheControl": { + "column": "cacheControl", + "type": "column" + }, + "checksumCrc32": { + "column": "checksumCrc32", + "type": "column" + }, + "checksumCrc32C": { + "column": "checksumCrc32C", + "type": "column" + }, + "checksumCrc64Nvme": { + "column": "checksumCrc64Nvme", + "type": "column" + }, + "checksumSha1": { + "column": "checksumSha1", + "type": "column" + }, + "checksumSha256": { + "column": "checksumSha256", + "type": "column" + }, + "clientId": { + "column": "clientId", + "type": "column" + }, + "contentDisposition": { + "column": "contentDisposition", + "type": "column" + }, + "contentEncoding": { + "column": "contentEncoding", + "type": "column" + }, + "contentLanguage": { + "column": "contentLanguage", + "type": "column" + }, + "contentMd5": { + "column": "contentMd5", + "type": "column" + }, + "contentType": { + "column": "contentType", + "type": "column" + }, + "copy": { + "column": "copy", + "fields": { + "fields": { + "completionTime": { + "column": "completionTime", + "type": "column" + }, + "id": { + "column": "id", + "type": "column" + }, + "progress": { + "column": "progress", + "type": "column" + }, + "source": { + "column": "source", + "type": "column" + }, + "status": { + "column": "status", + "type": "column" + }, + "statusDescription": { + "column": "statusDescription", + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + }, + "customerProvidedKeySha256": { + "column": "customerProvidedKeySha256", + "type": "column" + }, + "deleted": { + "column": "deleted", + "type": "column" + }, + "deletedTime": { + "column": "deletedTime", + "type": "column" + }, + "destinationSnapshot": { + "column": "destinationSnapshot", + "type": "column" + }, + "etag": { + "column": "etag", + "type": "column" + }, + "expiration": { + "column": "expiration", + "type": "column" + }, + "expirationRuleId": { + "column": "expirationRuleId", + "type": "column" + }, + "expires": { + "column": "expires", + "type": "column" + }, + "grant": { + "column": "grant", + "fields": { + "fields": { + "fields": { + "grantee": { + "column": "grantee", + "fields": { + "fields": { + "displayName": { + "column": "displayName", + "type": "column" + }, + "id": { + "column": "id", + "type": "column" + }, + "uri": { + "column": "uri", + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + }, + "permission": { + "column": "permission", + "type": "column" + } + }, + "type": "object" + }, + "type": "array" + }, + "type": "column" + }, + "group": { + "column": "group", + "type": "column" + }, + "incrementalCopy": { + "column": "incrementalCopy", + "type": "column" + }, + "isLatest": { + "column": "isLatest", + "type": "column" + }, + "kmsKeyName": { + "column": "kmsKeyName", + "type": "column" + }, + "leaseDuration": { + "column": "leaseDuration", + "type": "column" + }, + "leaseState": { + "column": "leaseState", + "type": "column" + }, + "leaseStatus": { + "column": "leaseStatus", + "type": "column" + }, + "legalHold": { + "column": "legalHold", + "type": "column" + }, + "mediaLink": { + "column": "mediaLink", + "type": "column" + }, + "metadata": { + "column": "metadata", + "type": "column" + }, + "name": { + "column": "name", + "type": "column" + }, + "owner": { + "column": "owner", + "fields": { + "fields": { + "id": { + "column": "id", + "type": "column" + }, + "name": { + "column": "name", + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + }, + "permissions": { + "column": "permissions", + "type": "column" + }, + "rawMetadata": { + "column": "rawMetadata", + "type": "column" + }, + "rehydratePriority": { + "column": "rehydratePriority", + "type": "column" + }, + "remainingRetentionDays": { + "column": "remainingRetentionDays", + "type": "column" + }, + "replicationReady": { + "column": "replicationReady", + "type": "column" + }, + "replicationStatus": { + "column": "replicationStatus", + "type": "column" + }, + "resourceType": { + "column": "resourceType", + "type": "column" + }, + "restore": { + "column": "restore", + "fields": { + "fields": { + "expiryTime": { + "column": "expiryTime", + "type": "column" + }, + "ongoingRestore": { + "column": "ongoingRestore", + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + }, + "retentionMode": { + "column": "retentionMode", + "type": "column" + }, + "retentionUntilDate": { + "column": "retentionUntilDate", + "type": "column" + }, + "sealed": { + "column": "sealed", + "type": "column" + }, + "serverEncrypted": { + "column": "serverEncrypted", + "type": "column" + }, + "size": { + "column": "size", + "type": "column" + }, + "storageClass": { + "column": "storageClass", + "type": "column" + }, + "tagCount": { + "column": "tagCount", + "type": "column" + }, + "tags": { + "column": "tags", + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + } + } + } +} diff --git a/connector/testdata/02-get/query/storageObject-null/request.json b/connector/testdata/02-get/query/storageObject-null/request.json index 725a074..03b78cb 100644 --- a/connector/testdata/02-get/query/storageObject-null/request.json +++ b/connector/testdata/02-get/query/storageObject-null/request.json @@ -41,10 +41,38 @@ "column": "__value", "fields": { "fields": { + "accessTierChangeTime": { + "column": "accessTierChangeTime", + "type": "column" + }, + "accessTierInferred": { + "column": "accessTierInferred", + "type": "column" + }, + "acl": { + "column": "acl", + "type": "column" + }, + "archiveStatus": { + "column": "archiveStatus", + "type": "column" + }, + "blobSequenceNumber": { + "column": "blobSequenceNumber", + "type": "column" + }, + "blobType": { + "column": "blobType", + "type": "column" + }, "bucket": { "column": "bucket", "type": "column" }, + "cacheControl": { + "column": "cacheControl", + "type": "column" + }, "checksumCrc32": { "column": "checksumCrc32", "type": "column" @@ -69,10 +97,75 @@ "column": "clientId", "type": "column" }, + "contentDisposition": { + "column": "contentDisposition", + "type": "column" + }, + "contentEncoding": { + "column": "contentEncoding", + "type": "column" + }, + "contentLanguage": { + "column": "contentLanguage", + "type": "column" + }, + "contentMd5": { + "column": "contentMd5", + "type": "column" + }, "contentType": { "column": "contentType", "type": "column" }, + "copy": { + "column": "copy", + "fields": { + "fields": { + "completionTime": { + "column": "completionTime", + "type": "column" + }, + "id": { + "column": "id", + "type": "column" + }, + "progress": { + "column": "progress", + "type": "column" + }, + "source": { + "column": "source", + "type": "column" + }, + "status": { + "column": "status", + "type": "column" + }, + "statusDescription": { + "column": "statusDescription", + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + }, + "customerProvidedKeySha256": { + "column": "customerProvidedKeySha256", + "type": "column" + }, + "deleted": { + "column": "deleted", + "type": "column" + }, + "deletedTime": { + "column": "deletedTime", + "type": "column" + }, + "destinationSnapshot": { + "column": "destinationSnapshot", + "type": "column" + }, "etag": { "column": "etag", "type": "column" @@ -126,14 +219,42 @@ }, "type": "column" }, - "deleted": { - "column": "deleted", + "group": { + "column": "group", + "type": "column" + }, + "incrementalCopy": { + "column": "incrementalCopy", "type": "column" }, "isLatest": { "column": "isLatest", "type": "column" }, + "kmsKeyName": { + "column": "kmsKeyName", + "type": "column" + }, + "leaseDuration": { + "column": "leaseDuration", + "type": "column" + }, + "leaseState": { + "column": "leaseState", + "type": "column" + }, + "leaseStatus": { + "column": "leaseStatus", + "type": "column" + }, + "legalHold": { + "column": "legalHold", + "type": "column" + }, + "mediaLink": { + "column": "mediaLink", + "type": "column" + }, "metadata": { "column": "metadata", "type": "column" @@ -159,6 +280,22 @@ }, "type": "column" }, + "permissions": { + "column": "permissions", + "type": "column" + }, + "rawMetadata": { + "column": "rawMetadata", + "type": "column" + }, + "rehydratePriority": { + "column": "rehydratePriority", + "type": "column" + }, + "remainingRetentionDays": { + "column": "remainingRetentionDays", + "type": "column" + }, "replicationReady": { "column": "replicationReady", "type": "column" @@ -167,6 +304,10 @@ "column": "replicationStatus", "type": "column" }, + "resourceType": { + "column": "resourceType", + "type": "column" + }, "restore": { "column": "restore", "fields": { @@ -184,6 +325,22 @@ }, "type": "column" }, + "retentionMode": { + "column": "retentionMode", + "type": "column" + }, + "retentionUntilDate": { + "column": "retentionUntilDate", + "type": "column" + }, + "sealed": { + "column": "sealed", + "type": "column" + }, + "serverEncrypted": { + "column": "serverEncrypted", + "type": "column" + }, "size": { "column": "size", "type": "column" @@ -192,16 +349,16 @@ "column": "storageClass", "type": "column" }, - "userMetadata": { - "column": "userMetadata", + "tagCount": { + "column": "tagCount", "type": "column" }, - "userTagCount": { - "column": "userTagCount", + "tags": { + "column": "tags", "type": "column" }, - "userTags": { - "column": "userTags", + "versionId": { + "column": "versionId", "type": "column" } }, diff --git a/connector/testdata/02-get/query/storageObject/expected.json b/connector/testdata/02-get/query/storageObject/expected.json index 426b21b..3a9478c 100644 --- a/connector/testdata/02-get/query/storageObject/expected.json +++ b/connector/testdata/02-get/query/storageObject/expected.json @@ -22,31 +22,31 @@ "contentLanguage": "en-US", "contentMd5": null, "contentType": "text/plain", - "copyCompletionTime": null, - "copyId": null, - "copyProgress": null, - "copySource": null, - "copyStatus": null, - "copyStatusDescription": null, + "copy": null, "customerProvidedKeySha256": null, "deleted": false, + "deletedTime": null, "destinationSnapshot": null, - "encryptionScope": null, "etag": "299d2d913ceaa8377d22244db39723ca", "expiration": null, "expirationRuleId": null, "expires": "2099-01-01T00:00:00Z", "grant": [], "group": null, - "immutabilityPolicyMode": null, - "immutabilityPolicyUntilDate": null, "incrementalCopy": null, - "lastAccessTime": null, + "kmsKeyName": null, "leaseDuration": null, "leaseState": null, "leaseStatus": null, "legalHold": null, + "mediaLink": null, "metadata": { + "Foo": "Bar" + }, + "name": "public/hello.txt", + "owner": null, + "permissions": null, + "rawMetadata": { "Cache-Control": "max-age=180, public", "Content-Disposition": "attachment", "Content-Encoding": "gzip", @@ -55,24 +55,20 @@ "X-Amz-Meta-Foo": "Bar", "X-Amz-Tagging-Count": "1" }, - "name": "public/hello.txt", - "owner": null, - "permissions": null, "rehydratePriority": null, "remainingRetentionDays": null, "replicationReady": false, "replicationStatus": null, "resourceType": null, "restore": null, + "retentionMode": null, + "retentionUntilDate": null, "sealed": null, "serverEncrypted": null, "size": 10, "storageClass": null, - "userMetadata": { - "Foo": "Bar" - }, - "userTagCount": 1, - "userTags": { + "tagCount": 1, + "tags": { "UserID": "1" } } diff --git a/connector/testdata/02-get/query/storageObject/request.json b/connector/testdata/02-get/query/storageObject/request.json index 3803aa4..335e24e 100644 --- a/connector/testdata/02-get/query/storageObject/request.json +++ b/connector/testdata/02-get/query/storageObject/request.json @@ -35,7 +35,6 @@ }, "collection": "storageObject", "collection_relationships": {}, - "query": { "fields": { "__value": { @@ -118,42 +117,55 @@ "column": "contentType", "type": "column" }, - "copyCompletionTime": { - "column": "copyCompletionTime", - "type": "column" - }, - "copyId": { - "column": "copyId", - "type": "column" - }, - "copyProgress": { - "column": "copyProgress", - "type": "column" - }, - "copySource": { - "column": "copySource", + "copy": { + "column": "copy", + "fields": { + "fields": { + "completionTime": { + "column": "completionTime", + "type": "column" + }, + "id": { + "column": "id", + "type": "column" + }, + "progress": { + "column": "progress", + "type": "column" + }, + "source": { + "column": "source", + "type": "column" + }, + "status": { + "column": "status", + "type": "column" + }, + "statusDescription": { + "column": "statusDescription", + "type": "column" + } + }, + "type": "object" + }, "type": "column" }, - "copyStatus": { - "column": "copyStatus", + "customerProvidedKeySha256": { + "column": "customerProvidedKeySha256", "type": "column" }, - "copyStatusDescription": { - "column": "copyStatusDescription", + "deleted": { + "column": "deleted", "type": "column" }, - "customerProvidedKeySha256": { - "column": "customerProvidedKeySha256", + "deletedTime": { + "column": "deletedTime", "type": "column" }, "destinationSnapshot": { "column": "destinationSnapshot", "type": "column" }, - "encryptionScope": { - "column": "encryptionScope", - "type": "column" - }, "etag": { "column": "etag", "type": "column" @@ -211,24 +223,12 @@ "column": "group", "type": "column" }, - "immutabilityPolicyMode": { - "column": "immutabilityPolicyMode", - "type": "column" - }, - "immutabilityPolicyUntilDate": { - "column": "immutabilityPolicyUntilDate", - "type": "column" - }, "incrementalCopy": { "column": "incrementalCopy", "type": "column" }, - "deleted": { - "column": "deleted", - "type": "column" - }, - "lastAccessTime": { - "column": "lastAccessTime", + "kmsKeyName": { + "column": "kmsKeyName", "type": "column" }, "leaseDuration": { @@ -247,6 +247,10 @@ "column": "legalHold", "type": "column" }, + "mediaLink": { + "column": "mediaLink", + "type": "column" + }, "metadata": { "column": "metadata", "type": "column" @@ -276,6 +280,10 @@ "column": "permissions", "type": "column" }, + "rawMetadata": { + "column": "rawMetadata", + "type": "column" + }, "rehydratePriority": { "column": "rehydratePriority", "type": "column" @@ -313,6 +321,14 @@ }, "type": "column" }, + "retentionMode": { + "column": "retentionMode", + "type": "column" + }, + "retentionUntilDate": { + "column": "retentionUntilDate", + "type": "column" + }, "sealed": { "column": "sealed", "type": "column" @@ -329,16 +345,12 @@ "column": "storageClass", "type": "column" }, - "userMetadata": { - "column": "userMetadata", - "type": "column" - }, - "userTagCount": { - "column": "userTagCount", + "tagCount": { + "column": "tagCount", "type": "column" }, - "userTags": { - "column": "userTags", + "tags": { + "column": "tags", "type": "column" } }, diff --git a/connector/testdata/02-get/query/storageObjectLockConfig/expected.json b/connector/testdata/02-get/query/storageObjectLockConfig/expected.json deleted file mode 100644 index 705fd75..0000000 --- a/connector/testdata/02-get/query/storageObjectLockConfig/expected.json +++ /dev/null @@ -1,14 +0,0 @@ -[ - { - "rows": [ - { - "__value": { - "mode": "Locked", - "objectLock": "Enabled", - "unit": "DAYS", - "validity": 1 - } - } - ] - } -] diff --git a/connector/testdata/02-get/query/storageObjectLockConfig/request.json b/connector/testdata/02-get/query/storageObjectLockConfig/request.json deleted file mode 100644 index eea15a8..0000000 --- a/connector/testdata/02-get/query/storageObjectLockConfig/request.json +++ /dev/null @@ -1,39 +0,0 @@ -{ - "arguments": { - "bucket": { - "type": "literal", - "value": "minio-bucket-lock" - } - }, - "collection": "storageObjectLockConfig", - "collection_relationships": {}, - "query": { - "fields": { - "__value": { - "column": "__value", - "fields": { - "fields": { - "mode": { - "column": "mode", - "type": "column" - }, - "objectLock": { - "column": "objectLock", - "type": "column" - }, - "unit": { - "column": "unit", - "type": "column" - }, - "validity": { - "column": "validity", - "type": "column" - } - }, - "type": "object" - }, - "type": "column" - } - } - } -} diff --git a/connector/testdata/02-get/query/storageObjects-azblob/expected.json b/connector/testdata/02-get/query/storageObjects-azblob/expected.json index e1ab355..c57af81 100644 --- a/connector/testdata/02-get/query/storageObjects-azblob/expected.json +++ b/connector/testdata/02-get/query/storageObjects-azblob/expected.json @@ -23,57 +23,55 @@ "contentLanguage": null, "contentMd5": "KZ0tkTzqqDd9IiRNs5cjyg==", "contentType": "application/octet-stream", - "copyCompletionTime": null, - "copyId": null, - "copyProgress": null, - "copySource": null, - "copyStatus": null, - "copyStatusDescription": null, + "copy": null, "customerProvidedKeySha256": null, "deleted": null, "deletedTime": null, "destinationSnapshot": null, - "encryptionScope": null, "expiration": null, "expirationRuleId": null, "expires": null, "grant": [], "group": null, - "immutabilityPolicyMode": null, - "immutabilityPolicyUntilDate": null, "incrementalCopy": null, + "isLatest": null, + "kmsKeyName": null, + "lastAccessTime": null, "leaseDuration": null, "leaseState": "available", "leaseStatus": "unlocked", "legalHold": null, + "mediaLink": null, "metadata": { "foo": "Bar" }, "name": "public/hello.txt", "owner": null, "permissions": null, + "rawMetadata": null, "rehydratePriority": null, "remainingRetentionDays": null, "replicationReady": null, "replicationStatus": null, "resourceType": null, "restore": null, + "retentionMode": null, + "retentionUntilDate": null, "sealed": null, "serverEncrypted": true, "size": 10, "storageClass": "Cool", - "userMetadata": {}, - "userTagCount": 2, - "userTags": { + "tagCount": 2, + "tags": { "Foo": "baz", "UserID": "3" - } + }, + "versionId": null } ], "pageInfo": { - "cursor": null, "hasNextPage": true, - "nextCursor": "public/hello.txt" + "cursor": "public/hello.txt" } } } diff --git a/connector/testdata/02-get/query/storageObjects-azblob/request.json b/connector/testdata/02-get/query/storageObjects-azblob/request.json index 57bc5fa..a87f0ad 100644 --- a/connector/testdata/02-get/query/storageObjects-azblob/request.json +++ b/connector/testdata/02-get/query/storageObjects-azblob/request.json @@ -83,28 +83,37 @@ "column": "contentType", "type": "column" }, - "copyCompletionTime": { - "column": "copyCompletionTime", - "type": "column" - }, - "copyId": { - "column": "copyId", - "type": "column" - }, - "copyProgress": { - "column": "copyProgress", - "type": "column" - }, - "copySource": { - "column": "copySource", - "type": "column" - }, - "copyStatus": { - "column": "copyStatus", - "type": "column" - }, - "copyStatusDescription": { - "column": "copyStatusDescription", + "copy": { + "column": "copy", + "fields": { + "fields": { + "completionTime": { + "column": "completionTime", + "type": "column" + }, + "id": { + "column": "id", + "type": "column" + }, + "progress": { + "column": "progress", + "type": "column" + }, + "source": { + "column": "source", + "type": "column" + }, + "status": { + "column": "status", + "type": "column" + }, + "statusDescription": { + "column": "statusDescription", + "type": "column" + } + }, + "type": "object" + }, "type": "column" }, "customerProvidedKeySha256": { @@ -123,10 +132,6 @@ "column": "destinationSnapshot", "type": "column" }, - "encryptionScope": { - "column": "encryptionScope", - "type": "column" - }, "expiration": { "column": "expiration", "type": "column" @@ -180,16 +185,20 @@ "column": "group", "type": "column" }, - "immutabilityPolicyMode": { - "column": "immutabilityPolicyMode", + "incrementalCopy": { + "column": "incrementalCopy", "type": "column" }, - "immutabilityPolicyUntilDate": { - "column": "immutabilityPolicyUntilDate", + "isLatest": { + "column": "isLatest", "type": "column" }, - "incrementalCopy": { - "column": "incrementalCopy", + "kmsKeyName": { + "column": "kmsKeyName", + "type": "column" + }, + "lastAccessTime": { + "column": "lastAccessTime", "type": "column" }, "leaseDuration": { @@ -208,6 +217,10 @@ "column": "legalHold", "type": "column" }, + "mediaLink": { + "column": "mediaLink", + "type": "column" + }, "metadata": { "column": "metadata", "type": "column" @@ -237,6 +250,10 @@ "column": "permissions", "type": "column" }, + "rawMetadata": { + "column": "rawMetadata", + "type": "column" + }, "rehydratePriority": { "column": "rehydratePriority", "type": "column" @@ -274,6 +291,14 @@ }, "type": "column" }, + "retentionMode": { + "column": "retentionMode", + "type": "column" + }, + "retentionUntilDate": { + "column": "retentionUntilDate", + "type": "column" + }, "sealed": { "column": "sealed", "type": "column" @@ -290,16 +315,16 @@ "column": "storageClass", "type": "column" }, - "userMetadata": { - "column": "userMetadata", + "tagCount": { + "column": "tagCount", "type": "column" }, - "userTagCount": { - "column": "userTagCount", + "tags": { + "column": "tags", "type": "column" }, - "userTags": { - "column": "userTags", + "versionId": { + "column": "versionId", "type": "column" } }, @@ -320,10 +345,6 @@ "hasNextPage": { "column": "hasNextPage", "type": "column" - }, - "nextCursor": { - "column": "nextCursor", - "type": "column" } }, "type": "object" diff --git a/connector/testdata/02-get/query/storageObjects-gcs/expected.json b/connector/testdata/02-get/query/storageObjects-gcs/expected.json new file mode 100644 index 0000000..c94423e --- /dev/null +++ b/connector/testdata/02-get/query/storageObjects-gcs/expected.json @@ -0,0 +1,85 @@ +[ + { + "rows": [ + { + "__value": { + "objects": [ + { + "accessTierChangeTime": null, + "accessTierInferred": null, + "acl": [ + { + "entity": "projectOwner-test-project", + "projectTeam": {}, + "role": "OWNER" + } + ], + "archiveStatus": null, + "blobSequenceNumber": null, + "blobType": null, + "bucket": "gcs-bucket", + "cacheControl": null, + "checksumCrc32": null, + "checksumCrc32C": null, + "checksumCrc64Nvme": null, + "checksumSha1": null, + "checksumSha256": null, + "clientId": "gcs", + "contentDisposition": "attachment", + "contentEncoding": "gzip", + "contentLanguage": "en-US", + "contentMd5": "KZ0tkTzqqDd9IiRNs5cjyg==", + "contentType": "text/plain", + "copy": null, + "customerProvidedKeySha256": null, + "deleted": null, + "deletedTime": null, + "destinationSnapshot": null, + "etag": "KZ0tkTzqqDd9IiRNs5cjyg==", + "expiration": null, + "expirationRuleId": null, + "expires": null, + "grant": [], + "group": null, + "incrementalCopy": null, + "isLatest": null, + "kmsKeyName": null, + "lastAccessTime": null, + "leaseDuration": null, + "leaseState": null, + "leaseStatus": null, + "legalHold": false, + "mediaLink": "http://0.0.0.0:4443/download/storage/v1/b/gcs-bucket/o/public%2Fhello.txt?alt=media", + "metadata": { + "Foo": "gcp-baz", + "UserID": "3" + }, + "name": "public/hello.txt", + "owner": null, + "permissions": null, + "rawMetadata": null, + "rehydratePriority": null, + "remainingRetentionDays": null, + "replicationReady": null, + "replicationStatus": null, + "resourceType": null, + "restore": null, + "retentionMode": null, + "retentionUntilDate": null, + "sealed": null, + "serverEncrypted": null, + "size": 10, + "storageClass": "Cool", + "tagCount": 0, + "tags": null + } + ], + "pageInfo": { + "cursor": null, + "hasNextPage": true + } + } + } + ] + } +] diff --git a/connector/testdata/02-get/query/storageObjects-gcs/request.json b/connector/testdata/02-get/query/storageObjects-gcs/request.json new file mode 100644 index 0000000..4167656 --- /dev/null +++ b/connector/testdata/02-get/query/storageObjects-gcs/request.json @@ -0,0 +1,396 @@ +{ + "collection": "storageObjects", + "query": { + "fields": { + "__value": { + "column": "__value", + "fields": { + "fields": { + "objects": { + "column": "objects", + "fields": { + "fields": { + "fields": { + "accessTierChangeTime": { + "column": "accessTierChangeTime", + "type": "column" + }, + "accessTierInferred": { + "column": "accessTierInferred", + "type": "column" + }, + "acl": { + "column": "acl", + "type": "column" + }, + "archiveStatus": { + "column": "archiveStatus", + "type": "column" + }, + "blobSequenceNumber": { + "column": "blobSequenceNumber", + "type": "column" + }, + "blobType": { + "column": "blobType", + "type": "column" + }, + "bucket": { + "column": "bucket", + "type": "column" + }, + "cacheControl": { + "column": "cacheControl", + "type": "column" + }, + "checksumCrc32": { + "column": "checksumCrc32", + "type": "column" + }, + "checksumCrc32C": { + "column": "checksumCrc32C", + "type": "column" + }, + "checksumCrc64Nvme": { + "column": "checksumCrc64Nvme", + "type": "column" + }, + "checksumSha1": { + "column": "checksumSha1", + "type": "column" + }, + "checksumSha256": { + "column": "checksumSha256", + "type": "column" + }, + "clientId": { + "column": "clientId", + "type": "column" + }, + "contentDisposition": { + "column": "contentDisposition", + "type": "column" + }, + "contentEncoding": { + "column": "contentEncoding", + "type": "column" + }, + "contentLanguage": { + "column": "contentLanguage", + "type": "column" + }, + "contentMd5": { + "column": "contentMd5", + "type": "column" + }, + "contentType": { + "column": "contentType", + "type": "column" + }, + "copy": { + "column": "copy", + "fields": { + "fields": { + "completionTime": { + "column": "completionTime", + "type": "column" + }, + "id": { + "column": "id", + "type": "column" + }, + "progress": { + "column": "progress", + "type": "column" + }, + "source": { + "column": "source", + "type": "column" + }, + "status": { + "column": "status", + "type": "column" + }, + "statusDescription": { + "column": "statusDescription", + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + }, + "customerProvidedKeySha256": { + "column": "customerProvidedKeySha256", + "type": "column" + }, + "deleted": { + "column": "deleted", + "type": "column" + }, + "deletedTime": { + "column": "deletedTime", + "type": "column" + }, + "destinationSnapshot": { + "column": "destinationSnapshot", + "type": "column" + }, + "etag": { + "column": "etag", + "type": "column" + }, + "expiration": { + "column": "expiration", + "type": "column" + }, + "expirationRuleId": { + "column": "expirationRuleId", + "type": "column" + }, + "expires": { + "column": "expires", + "type": "column" + }, + "grant": { + "column": "grant", + "fields": { + "fields": { + "fields": { + "grantee": { + "column": "grantee", + "fields": { + "fields": { + "displayName": { + "column": "displayName", + "type": "column" + }, + "id": { + "column": "id", + "type": "column" + }, + "uri": { + "column": "uri", + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + }, + "permission": { + "column": "permission", + "type": "column" + } + }, + "type": "object" + }, + "type": "array" + }, + "type": "column" + }, + "group": { + "column": "group", + "type": "column" + }, + "incrementalCopy": { + "column": "incrementalCopy", + "type": "column" + }, + "isLatest": { + "column": "isLatest", + "type": "column" + }, + "kmsKeyName": { + "column": "kmsKeyName", + "type": "column" + }, + "lastAccessTime": { + "column": "lastAccessTime", + "type": "column" + }, + "leaseDuration": { + "column": "leaseDuration", + "type": "column" + }, + "leaseState": { + "column": "leaseState", + "type": "column" + }, + "leaseStatus": { + "column": "leaseStatus", + "type": "column" + }, + "legalHold": { + "column": "legalHold", + "type": "column" + }, + "mediaLink": { + "column": "mediaLink", + "type": "column" + }, + "metadata": { + "column": "metadata", + "type": "column" + }, + "name": { + "column": "name", + "type": "column" + }, + "owner": { + "column": "owner", + "fields": { + "fields": { + "id": { + "column": "id", + "type": "column" + }, + "name": { + "column": "name", + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + }, + "permissions": { + "column": "permissions", + "type": "column" + }, + "rawMetadata": { + "column": "rawMetadata", + "type": "column" + }, + "rehydratePriority": { + "column": "rehydratePriority", + "type": "column" + }, + "remainingRetentionDays": { + "column": "remainingRetentionDays", + "type": "column" + }, + "replicationReady": { + "column": "replicationReady", + "type": "column" + }, + "replicationStatus": { + "column": "replicationStatus", + "type": "column" + }, + "resourceType": { + "column": "resourceType", + "type": "column" + }, + "restore": { + "column": "restore", + "fields": { + "fields": { + "expiryTime": { + "column": "expiryTime", + "type": "column" + }, + "ongoingRestore": { + "column": "ongoingRestore", + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + }, + "retentionMode": { + "column": "retentionMode", + "type": "column" + }, + "retentionUntilDate": { + "column": "retentionUntilDate", + "type": "column" + }, + "sealed": { + "column": "sealed", + "type": "column" + }, + "serverEncrypted": { + "column": "serverEncrypted", + "type": "column" + }, + "size": { + "column": "size", + "type": "column" + }, + "storageClass": { + "column": "storageClass", + "type": "column" + }, + "tagCount": { + "column": "tagCount", + "type": "column" + }, + "tags": { + "column": "tags", + "type": "column" + } + }, + "type": "object" + }, + "type": "array" + }, + "type": "column" + }, + "pageInfo": { + "column": "pageInfo", + "fields": { + "fields": { + "cursor": { + "column": "cursor", + "type": "column" + }, + "hasNextPage": { + "column": "hasNextPage", + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + } + } + }, + "arguments": { + "recursive": { "type": "literal", "value": true }, + "maxResults": { "type": "literal", "value": 1 }, + "where": { + "type": "literal", + "value": { + "type": "and", + "expressions": [ + { + "type": "binary_comparison_operator", + "column": { "type": "column", "name": "clientId", "path": [] }, + "operator": "_eq", + "value": { "type": "scalar", "value": "gcs" } + }, + { + "type": "binary_comparison_operator", + "column": { "type": "column", "name": "bucket", "path": [] }, + "operator": "_eq", + "value": { "type": "scalar", "value": "gcs-bucket" } + }, + { + "type": "binary_comparison_operator", + "column": { "type": "column", "name": "object", "path": [] }, + "operator": "_starts_with", + "value": { "type": "scalar", "value": "public" } + } + ] + } + } + }, + "collection_relationships": {} +} diff --git a/connector/testdata/02-get/query/storageObjects/expected.json b/connector/testdata/02-get/query/storageObjects/expected.json index 69404a3..0ad8a99 100644 --- a/connector/testdata/02-get/query/storageObjects/expected.json +++ b/connector/testdata/02-get/query/storageObjects/expected.json @@ -24,37 +24,27 @@ "contentLanguage": "en-US", "contentMd5": null, "contentType": "text/plain", - "copyCompletionTime": null, - "copyId": null, - "copyProgress": null, - "copySource": null, - "copyStatus": null, - "copyStatusDescription": null, + "copy": null, "customerProvidedKeySha256": null, "deleted": false, "deletedTime": null, "destinationSnapshot": null, - "encryptionScope": null, "etag": "299d2d913ceaa8377d22244db39723ca", "expiration": null, "expirationRuleId": null, "expires": null, "grant": [], "group": null, - "immutabilityPolicyMode": null, - "immutabilityPolicyUntilDate": null, "incrementalCopy": null, + "kmsKeyName": null, + "lastAccessTime": null, "leaseDuration": null, "leaseState": null, "leaseStatus": null, "legalHold": null, + "mediaLink": null, "metadata": { - "X-Amz-Meta-Foo": "Bar", - "cache-control": "max-age=180, public", - "content-disposition": "attachment", - "content-encoding": "gzip", - "content-language": "en-US", - "content-type": "text/plain" + "Foo": "Bar" }, "name": "public/hello.txt", "owner": { @@ -62,21 +52,28 @@ "name": "02d6176db174dc93cb1b899f7c6078f08654445fe8cf1b6ce98d8855f66bdbf4" }, "permissions": null, + "rawMetadata": { + "X-Amz-Meta-Foo": "Bar", + "cache-control": "max-age=180, public", + "content-disposition": "attachment", + "content-encoding": "gzip", + "content-language": "en-US", + "content-type": "text/plain" + }, "rehydratePriority": null, "remainingRetentionDays": null, "replicationReady": false, "replicationStatus": null, "resourceType": null, "restore": null, + "retentionMode": null, + "retentionUntilDate": null, "sealed": null, "serverEncrypted": null, "size": 10, "storageClass": "STANDARD", - "userMetadata": { - "Foo": "Bar" - }, - "userTagCount": 0, - "userTags": { + "tagCount": 1, + "tags": { "UserID": "1" } }, @@ -100,37 +97,27 @@ "contentLanguage": "en-US", "contentMd5": null, "contentType": "text/plain", - "copyCompletionTime": null, - "copyId": null, - "copyProgress": null, - "copySource": null, - "copyStatus": null, - "copyStatusDescription": null, + "copy": null, "customerProvidedKeySha256": null, "deleted": false, "deletedTime": null, "destinationSnapshot": null, - "encryptionScope": null, "etag": "299d2d913ceaa8377d22244db39723ca", "expiration": null, "expirationRuleId": null, "expires": null, "grant": [], "group": null, - "immutabilityPolicyMode": null, - "immutabilityPolicyUntilDate": null, "incrementalCopy": null, + "kmsKeyName": null, + "lastAccessTime": null, "leaseDuration": null, "leaseState": null, "leaseStatus": null, "legalHold": null, + "mediaLink": null, "metadata": { - "X-Amz-Meta-Foo": "Bar", - "cache-control": "max-age=180, public", - "content-disposition": "attachment", - "content-encoding": "gzip", - "content-language": "en-US", - "content-type": "text/plain" + "Foo": "Bar" }, "name": "public/hello2.txt", "owner": { @@ -138,21 +125,28 @@ "name": "02d6176db174dc93cb1b899f7c6078f08654445fe8cf1b6ce98d8855f66bdbf4" }, "permissions": null, + "rawMetadata": { + "X-Amz-Meta-Foo": "Bar", + "cache-control": "max-age=180, public", + "content-disposition": "attachment", + "content-encoding": "gzip", + "content-language": "en-US", + "content-type": "text/plain" + }, "rehydratePriority": null, "remainingRetentionDays": null, "replicationReady": false, "replicationStatus": null, "resourceType": null, "restore": null, + "retentionMode": null, + "retentionUntilDate": null, "sealed": null, "serverEncrypted": null, "size": 10, "storageClass": "STANDARD", - "userMetadata": { - "Foo": "Bar" - }, - "userTagCount": 0, - "userTags": { + "tagCount": 1, + "tags": { "UserID": "1" } }, @@ -176,37 +170,27 @@ "contentLanguage": "en-US", "contentMd5": null, "contentType": "text/plain", - "copyCompletionTime": null, - "copyId": null, - "copyProgress": null, - "copySource": null, - "copyStatus": null, - "copyStatusDescription": null, + "copy": null, "customerProvidedKeySha256": null, "deleted": false, "deletedTime": null, "destinationSnapshot": null, - "encryptionScope": null, "etag": "299d2d913ceaa8377d22244db39723ca", "expiration": null, "expirationRuleId": null, "expires": null, "grant": [], "group": null, - "immutabilityPolicyMode": null, - "immutabilityPolicyUntilDate": null, "incrementalCopy": null, + "kmsKeyName": null, + "lastAccessTime": null, "leaseDuration": null, "leaseState": null, "leaseStatus": null, "legalHold": null, + "mediaLink": null, "metadata": { - "X-Amz-Meta-Foo": "Bar", - "cache-control": "max-age=180, public", - "content-disposition": "attachment", - "content-encoding": "gzip", - "content-language": "en-US", - "content-type": "text/plain" + "Foo": "Bar" }, "name": "public/hello3.txt", "owner": { @@ -214,21 +198,28 @@ "name": "02d6176db174dc93cb1b899f7c6078f08654445fe8cf1b6ce98d8855f66bdbf4" }, "permissions": null, + "rawMetadata": { + "X-Amz-Meta-Foo": "Bar", + "cache-control": "max-age=180, public", + "content-disposition": "attachment", + "content-encoding": "gzip", + "content-language": "en-US", + "content-type": "text/plain" + }, "rehydratePriority": null, "remainingRetentionDays": null, "replicationReady": false, "replicationStatus": null, "resourceType": null, "restore": null, + "retentionMode": null, + "retentionUntilDate": null, "sealed": null, "serverEncrypted": null, "size": 10, "storageClass": "STANDARD", - "userMetadata": { - "Foo": "Bar" - }, - "userTagCount": 0, - "userTags": { + "tagCount": 1, + "tags": { "UserID": "1" } }, @@ -240,49 +231,39 @@ "blobSequenceNumber": null, "blobType": null, "bucket": "minio-bucket-test", - "cacheControl": "max-age=180, public", + "cacheControl": null, "checksumCrc32": null, "checksumCrc32C": null, "checksumCrc64Nvme": null, "checksumSha1": null, "checksumSha256": null, "clientId": "minio", - "contentDisposition": "attachment", - "contentEncoding": "gzip", - "contentLanguage": "en-US", + "contentDisposition": null, + "contentEncoding": null, + "contentLanguage": null, "contentMd5": null, - "contentType": "text/plain", - "copyCompletionTime": null, - "copyId": null, - "copyProgress": null, - "copySource": null, - "copyStatus": null, - "copyStatusDescription": null, + "contentType": "binary/octet-stream", + "copy": null, "customerProvidedKeySha256": null, "deleted": false, "deletedTime": null, "destinationSnapshot": null, - "encryptionScope": null, "etag": "299d2d913ceaa8377d22244db39723ca", "expiration": null, "expirationRuleId": null, "expires": null, "grant": [], "group": null, - "immutabilityPolicyMode": null, - "immutabilityPolicyUntilDate": null, "incrementalCopy": null, + "kmsKeyName": null, + "lastAccessTime": null, "leaseDuration": null, "leaseState": null, "leaseStatus": null, "legalHold": null, + "mediaLink": null, "metadata": { - "X-Amz-Meta-Foo": "Bar", - "cache-control": "max-age=180, public", - "content-disposition": "attachment", - "content-encoding": "gzip", - "content-language": "en-US", - "content-type": "text/plain" + "Userid": "2" }, "name": "public/workd2.txt", "owner": { @@ -290,21 +271,24 @@ "name": "02d6176db174dc93cb1b899f7c6078f08654445fe8cf1b6ce98d8855f66bdbf4" }, "permissions": null, + "rawMetadata": { + "X-Amz-Meta-Userid": "2", + "content-type": "binary/octet-stream" + }, "rehydratePriority": null, "remainingRetentionDays": null, "replicationReady": false, "replicationStatus": null, "resourceType": null, "restore": null, + "retentionMode": null, + "retentionUntilDate": null, "sealed": null, "serverEncrypted": null, "size": 10, "storageClass": "STANDARD", - "userMetadata": { - "Foo": "Bar" - }, - "userTagCount": 0, - "userTags": { + "tagCount": 2, + "tags": { "Foo": "baz", "UserID": "3" } @@ -312,8 +296,7 @@ ], "pageInfo": { "cursor": null, - "hasNextPage": false, - "nextCursor": null + "hasNextPage": false } } } diff --git a/connector/testdata/02-get/query/storageObjects/request.json b/connector/testdata/02-get/query/storageObjects/request.json index 6969617..9adb09c 100644 --- a/connector/testdata/02-get/query/storageObjects/request.json +++ b/connector/testdata/02-get/query/storageObjects/request.json @@ -87,28 +87,37 @@ "column": "contentType", "type": "column" }, - "copyCompletionTime": { - "column": "copyCompletionTime", - "type": "column" - }, - "copyId": { - "column": "copyId", - "type": "column" - }, - "copyProgress": { - "column": "copyProgress", - "type": "column" - }, - "copySource": { - "column": "copySource", - "type": "column" - }, - "copyStatus": { - "column": "copyStatus", - "type": "column" - }, - "copyStatusDescription": { - "column": "copyStatusDescription", + "copy": { + "column": "copy", + "fields": { + "fields": { + "completionTime": { + "column": "completionTime", + "type": "column" + }, + "id": { + "column": "id", + "type": "column" + }, + "progress": { + "column": "progress", + "type": "column" + }, + "source": { + "column": "source", + "type": "column" + }, + "status": { + "column": "status", + "type": "column" + }, + "statusDescription": { + "column": "statusDescription", + "type": "column" + } + }, + "type": "object" + }, "type": "column" }, "customerProvidedKeySha256": { @@ -127,10 +136,6 @@ "column": "destinationSnapshot", "type": "column" }, - "encryptionScope": { - "column": "encryptionScope", - "type": "column" - }, "etag": { "column": "etag", "type": "column" @@ -188,16 +193,16 @@ "column": "group", "type": "column" }, - "immutabilityPolicyMode": { - "column": "immutabilityPolicyMode", + "incrementalCopy": { + "column": "incrementalCopy", "type": "column" }, - "immutabilityPolicyUntilDate": { - "column": "immutabilityPolicyUntilDate", + "kmsKeyName": { + "column": "kmsKeyName", "type": "column" }, - "incrementalCopy": { - "column": "incrementalCopy", + "lastAccessTime": { + "column": "lastAccessTime", "type": "column" }, "leaseDuration": { @@ -216,6 +221,10 @@ "column": "legalHold", "type": "column" }, + "mediaLink": { + "column": "mediaLink", + "type": "column" + }, "metadata": { "column": "metadata", "type": "column" @@ -245,6 +254,10 @@ "column": "permissions", "type": "column" }, + "rawMetadata": { + "column": "rawMetadata", + "type": "column" + }, "rehydratePriority": { "column": "rehydratePriority", "type": "column" @@ -282,6 +295,14 @@ }, "type": "column" }, + "retentionMode": { + "column": "retentionMode", + "type": "column" + }, + "retentionUntilDate": { + "column": "retentionUntilDate", + "type": "column" + }, "sealed": { "column": "sealed", "type": "column" @@ -298,16 +319,12 @@ "column": "storageClass", "type": "column" }, - "userMetadata": { - "column": "userMetadata", - "type": "column" - }, - "userTagCount": { - "column": "userTagCount", + "tagCount": { + "column": "tagCount", "type": "column" }, - "userTags": { - "column": "userTags", + "tags": { + "column": "tags", "type": "column" } }, @@ -328,10 +345,6 @@ "hasNextPage": { "column": "hasNextPage", "type": "column" - }, - "nextCursor": { - "column": "nextCursor", - "type": "column" } }, "type": "object" diff --git a/connector/testdata/02-get/query/storageObjects2-azblob/expected.json b/connector/testdata/02-get/query/storageObjects2-azblob/expected.json index 317961d..fd74998 100644 --- a/connector/testdata/02-get/query/storageObjects2-azblob/expected.json +++ b/connector/testdata/02-get/query/storageObjects2-azblob/expected.json @@ -5,6 +5,7 @@ "__value": { "objects": [ { + "accessTierChangeTime": null, "accessTierInferred": null, "acl": null, "archiveStatus": null, @@ -23,54 +24,59 @@ "contentLanguage": null, "contentMd5": "KZ0tkTzqqDd9IiRNs5cjyg==", "contentType": "application/octet-stream", - "copyProgress": "10/10", - "copySource": "http://local.hasura.dev:10000/azblob-bucket-test/public%2Fhello.txt", - "copyStatus": "success", - "copyStatusDescription": null, + "copy": { + "progress": "10/10", + "source": "http://local.hasura.dev:10000/azblob-bucket-test/public%2Fhello.txt", + "status": "success", + "statusDescription": null + }, "customerProvidedKeySha256": null, "deleted": null, "deletedTime": null, "destinationSnapshot": null, - "encryptionScope": null, "expiration": null, "expirationRuleId": null, "expires": null, "grant": [], "group": null, - "immutabilityPolicyMode": null, - "immutabilityPolicyUntilDate": null, "incrementalCopy": false, + "isLatest": null, + "kmsKeyName": null, + "lastAccessTime": null, "leaseDuration": null, "leaseState": "available", "leaseStatus": "unlocked", "legalHold": null, + "mediaLink": null, "metadata": { "userid": "2" }, "name": "public/hello2.txt", "owner": null, "permissions": null, + "rawMetadata": null, "rehydratePriority": null, "remainingRetentionDays": null, "replicationReady": null, "replicationStatus": null, "resourceType": null, "restore": null, + "retentionMode": null, + "retentionUntilDate": null, "sealed": null, "serverEncrypted": true, "size": 10, "storageClass": "Cool", - "userMetadata": {}, - "userTagCount": 1, - "userTags": { + "tagCount": 1, + "tags": { "Copy": "true" - } + }, + "versionId": null } ], "pageInfo": { - "cursor": "public/hello.txt", - "hasNextPage": false, - "nextCursor": null + "cursor": null, + "hasNextPage": false } } } diff --git a/connector/testdata/02-get/query/storageObjects2-azblob/request.json b/connector/testdata/02-get/query/storageObjects2-azblob/request.json index 47cbeed..2f0337a 100644 --- a/connector/testdata/02-get/query/storageObjects2-azblob/request.json +++ b/connector/testdata/02-get/query/storageObjects2-azblob/request.json @@ -11,6 +11,10 @@ "fields": { "fields": { "fields": { + "accessTierChangeTime": { + "column": "accessTierChangeTime", + "type": "column" + }, "accessTierInferred": { "column": "accessTierInferred", "type": "column" @@ -83,20 +87,29 @@ "column": "contentType", "type": "column" }, - "copyProgress": { - "column": "copyProgress", - "type": "column" - }, - "copySource": { - "column": "copySource", - "type": "column" - }, - "copyStatus": { - "column": "copyStatus", - "type": "column" - }, - "copyStatusDescription": { - "column": "copyStatusDescription", + "copy": { + "column": "copy", + "fields": { + "fields": { + "progress": { + "column": "progress", + "type": "column" + }, + "source": { + "column": "source", + "type": "column" + }, + "status": { + "column": "status", + "type": "column" + }, + "statusDescription": { + "column": "statusDescription", + "type": "column" + } + }, + "type": "object" + }, "type": "column" }, "customerProvidedKeySha256": { @@ -115,10 +128,6 @@ "column": "destinationSnapshot", "type": "column" }, - "encryptionScope": { - "column": "encryptionScope", - "type": "column" - }, "expiration": { "column": "expiration", "type": "column" @@ -172,16 +181,20 @@ "column": "group", "type": "column" }, - "immutabilityPolicyMode": { - "column": "immutabilityPolicyMode", + "incrementalCopy": { + "column": "incrementalCopy", "type": "column" }, - "immutabilityPolicyUntilDate": { - "column": "immutabilityPolicyUntilDate", + "isLatest": { + "column": "isLatest", "type": "column" }, - "incrementalCopy": { - "column": "incrementalCopy", + "kmsKeyName": { + "column": "kmsKeyName", + "type": "column" + }, + "lastAccessTime": { + "column": "lastAccessTime", "type": "column" }, "leaseDuration": { @@ -200,6 +213,10 @@ "column": "legalHold", "type": "column" }, + "mediaLink": { + "column": "mediaLink", + "type": "column" + }, "metadata": { "column": "metadata", "type": "column" @@ -229,6 +246,10 @@ "column": "permissions", "type": "column" }, + "rawMetadata": { + "column": "rawMetadata", + "type": "column" + }, "rehydratePriority": { "column": "rehydratePriority", "type": "column" @@ -266,6 +287,14 @@ }, "type": "column" }, + "retentionMode": { + "column": "retentionMode", + "type": "column" + }, + "retentionUntilDate": { + "column": "retentionUntilDate", + "type": "column" + }, "sealed": { "column": "sealed", "type": "column" @@ -282,16 +311,16 @@ "column": "storageClass", "type": "column" }, - "userMetadata": { - "column": "userMetadata", + "tagCount": { + "column": "tagCount", "type": "column" }, - "userTagCount": { - "column": "userTagCount", + "tags": { + "column": "tags", "type": "column" }, - "userTags": { - "column": "userTags", + "versionId": { + "column": "versionId", "type": "column" } }, @@ -312,10 +341,6 @@ "hasNextPage": { "column": "hasNextPage", "type": "column" - }, - "nextCursor": { - "column": "nextCursor", - "type": "column" } }, "type": "object" @@ -339,6 +364,12 @@ "value": { "type": "and", "expressions": [ + { + "type": "binary_comparison_operator", + "column": { "type": "column", "name": "clientId", "path": [] }, + "operator": "_eq", + "value": { "type": "scalar", "value": "azblob-connstr" } + }, { "type": "binary_comparison_operator", "column": { "type": "column", "name": "bucket", "path": [] }, diff --git a/connector/testdata/02-get/query/storagePresignedDownloadUrl-gcs/request.json b/connector/testdata/02-get/query/storagePresignedDownloadUrl-gcs/request.json new file mode 100644 index 0000000..41e0764 --- /dev/null +++ b/connector/testdata/02-get/query/storagePresignedDownloadUrl-gcs/request.json @@ -0,0 +1,47 @@ +{ + "arguments": { + "clientId": { + "type": "literal", + "value": "gcs-cred" + }, + "bucket": { + "type": "literal", + "value": "gcs-bucket" + }, + "expiry": { + "type": "literal", + "value": "1h" + }, + "object": { + "type": "literal", + "value": "public/hello.txt" + }, + "requestParams": { + "type": "literal", + "value": null + } + }, + "collection": "storagePresignedDownloadUrl", + "collection_relationships": {}, + "query": { + "fields": { + "__value": { + "column": "__value", + "type": "column", + "fields": { + "fields": { + "url": { + "column": "url", + "type": "column" + }, + "expiredAt": { + "column": "expiredAt", + "type": "column" + } + }, + "type": "object" + } + } + } + } +} diff --git a/connector/testdata/02-get/query/storagePresignedUploadUrl-gcs/request.json b/connector/testdata/02-get/query/storagePresignedUploadUrl-gcs/request.json new file mode 100644 index 0000000..cae2c20 --- /dev/null +++ b/connector/testdata/02-get/query/storagePresignedUploadUrl-gcs/request.json @@ -0,0 +1,43 @@ +{ + "arguments": { + "clientId": { + "type": "literal", + "value": "gcs-cred" + }, + "bucket": { + "type": "literal", + "value": "gcs-bucket" + }, + "expiry": { + "type": "literal", + "value": "1h" + }, + "object": { + "type": "literal", + "value": "8UFsTfY9bI" + } + }, + "collection": "storagePresignedUploadUrl", + "collection_relationships": {}, + "query": { + "fields": { + "__value": { + "column": "__value", + "type": "column", + "fields": { + "fields": { + "url": { + "column": "url", + "type": "column" + }, + "expiredAt": { + "column": "expiredAt", + "type": "column" + } + }, + "type": "object" + } + } + } + } +} diff --git a/connector/testdata/03-cleanup/mutation/02-removeStorageObject-gcs/expected.json b/connector/testdata/03-cleanup/mutation/02-removeStorageObject-gcs/expected.json new file mode 100644 index 0000000..777dfc1 --- /dev/null +++ b/connector/testdata/03-cleanup/mutation/02-removeStorageObject-gcs/expected.json @@ -0,0 +1 @@ +{ "operation_results": [{ "result": true, "type": "procedure" }] } diff --git a/connector/testdata/03-cleanup/mutation/02-removeStorageObject-gcs/request.json b/connector/testdata/03-cleanup/mutation/02-removeStorageObject-gcs/request.json new file mode 100644 index 0000000..96426df --- /dev/null +++ b/connector/testdata/03-cleanup/mutation/02-removeStorageObject-gcs/request.json @@ -0,0 +1,16 @@ +{ + "collection_relationships": {}, + "operations": [ + { + "type": "procedure", + "name": "removeStorageObject", + "arguments": { + "clientId": "gcs", + "bucket": "gcs-bucket", + "forceDelete": true, + "governanceBypass": true, + "object": "public/hello.txt" + } + } + ] +} diff --git a/connector/testdata/03-cleanup/mutation/03-removeStorageObjectTags/request.json b/connector/testdata/03-cleanup/mutation/03-removeStorageObjectTags/request.json index dd47253..bf0c9bf 100644 --- a/connector/testdata/03-cleanup/mutation/03-removeStorageObjectTags/request.json +++ b/connector/testdata/03-cleanup/mutation/03-removeStorageObjectTags/request.json @@ -3,10 +3,11 @@ "operations": [ { "type": "procedure", - "name": "setStorageObjectTags", + "name": "updateStorageObject", "arguments": { "bucket": "minio-bucket-test", - "object": "a9cSGZGzM2" + "object": "a9cSGZGzM2", + "tags": {} } } ] diff --git a/connector/testdata/03-cleanup/mutation/04-removeStorageObjects-gcs/expected.json b/connector/testdata/03-cleanup/mutation/04-removeStorageObjects-gcs/expected.json new file mode 100644 index 0000000..9e77ae4 --- /dev/null +++ b/connector/testdata/03-cleanup/mutation/04-removeStorageObjects-gcs/expected.json @@ -0,0 +1 @@ +{ "operation_results": [{ "result": [], "type": "procedure" }] } diff --git a/connector/testdata/03-cleanup/mutation/04-removeStorageObjects-gcs/request.json b/connector/testdata/03-cleanup/mutation/04-removeStorageObjects-gcs/request.json new file mode 100644 index 0000000..1f43342 --- /dev/null +++ b/connector/testdata/03-cleanup/mutation/04-removeStorageObjects-gcs/request.json @@ -0,0 +1,39 @@ +{ + "collection_relationships": {}, + "operations": [ + { + "type": "procedure", + "name": "removeStorageObjects", + "arguments": { + "clientId": "gcs", + "bucket": "gcs-bucket", + "governanceBypass": false, + "maxKeys": 385252679, + "prefix": "", + "recursive": true, + "withMetadata": true, + "withVersions": true + }, + "fields": { + "fields": { + "fields": { + "error": { + "column": "error", + "type": "column" + }, + "objectName": { + "column": "objectName", + "type": "column" + }, + "versionId": { + "column": "versionId", + "type": "column" + } + }, + "type": "object" + }, + "type": "array" + } + } + ] +} diff --git a/connector/testdata/03-cleanup/mutation/05-suspendStorageBucketVersioning/request.json b/connector/testdata/03-cleanup/mutation/05-suspendStorageBucketVersioning/request.json index ed8ae2f..1ad254b 100644 --- a/connector/testdata/03-cleanup/mutation/05-suspendStorageBucketVersioning/request.json +++ b/connector/testdata/03-cleanup/mutation/05-suspendStorageBucketVersioning/request.json @@ -3,9 +3,11 @@ "operations": [ { "type": "procedure", - "name": "suspendStorageBucketVersioning", + "name": "updateStorageBucket", "arguments": { - "bucket": "minio-bucket-test" + "bucket": "minio-bucket-test", + "versioning": false, + "tags": {} } } ] diff --git a/connector/testdata/03-cleanup/mutation/06-removeStorageBucketTags/expected.json b/connector/testdata/03-cleanup/mutation/06-removeStorageBucketTags/expected.json deleted file mode 100644 index e28c88a..0000000 --- a/connector/testdata/03-cleanup/mutation/06-removeStorageBucketTags/expected.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "operation_results": [ - { - "result": true, - "type": "procedure" - } - ] -} \ No newline at end of file diff --git a/connector/testdata/03-cleanup/mutation/06-removeStorageBucketTags/request.json b/connector/testdata/03-cleanup/mutation/06-removeStorageBucketTags/request.json deleted file mode 100644 index cec4762..0000000 --- a/connector/testdata/03-cleanup/mutation/06-removeStorageBucketTags/request.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "collection_relationships": {}, - "operations": [ - { - "type": "procedure", - "name": "setStorageBucketTags", - "arguments": { - "bucket": "minio-bucket-test", - "tags": null - } - } - ] -} diff --git a/connector/testdata/01-setup/mutation/08-setStorageObjectLockConfig/expected.json b/connector/testdata/03-cleanup/mutation/07-removeStorageBucket-gcs/expected.json similarity index 97% rename from connector/testdata/01-setup/mutation/08-setStorageObjectLockConfig/expected.json rename to connector/testdata/03-cleanup/mutation/07-removeStorageBucket-gcs/expected.json index e28c88a..e62ec74 100644 --- a/connector/testdata/01-setup/mutation/08-setStorageObjectLockConfig/expected.json +++ b/connector/testdata/03-cleanup/mutation/07-removeStorageBucket-gcs/expected.json @@ -5,4 +5,4 @@ "type": "procedure" } ] -} \ No newline at end of file +} diff --git a/connector/testdata/01-setup/mutation/02-enableStorageBucketVersioning/request.json b/connector/testdata/03-cleanup/mutation/07-removeStorageBucket-gcs/request.json similarity index 57% rename from connector/testdata/01-setup/mutation/02-enableStorageBucketVersioning/request.json rename to connector/testdata/03-cleanup/mutation/07-removeStorageBucket-gcs/request.json index 426e032..ee20b56 100644 --- a/connector/testdata/01-setup/mutation/02-enableStorageBucketVersioning/request.json +++ b/connector/testdata/03-cleanup/mutation/07-removeStorageBucket-gcs/request.json @@ -3,9 +3,10 @@ "operations": [ { "type": "procedure", - "name": "enableStorageBucketVersioning", + "name": "removeStorageBucket", "arguments": { - "bucket": "minio-bucket-test" + "clientId": "gcs", + "bucket": "gcs-bucket" } } ] diff --git a/docs/configuration.md b/docs/configuration.md index b55d2e2..8aff7c2 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -6,21 +6,23 @@ The configuration file `configuration.yaml` contains a list of storage clients. Every client has common settings: -- `id`: the unique identity name of the client. This setting is optional unless there are many configured clients. -- `type`: type of the storage provider. Accept one of `s3`, `gs` and `azblob`. +- `id`: the unique identity of the client. This setting is optional unless there are many configured clients. +- `type`: type of the storage provider. Accept one of `s3`, `gcs`, and `azblob`. - `defaultBucket`: the default bucket name. - `authentication`: the authentication setting. - `endpoint`: the base endpoint of the storage server. Required for other S3 compatible services such as MinIO, Cloudflare R2, DigitalOcean Spaces, etc... -- `publicHost`: is used to configure the public host for presigned URL generation if the connector communicates privately with the storage server through an internal DNS. If this setting isn't set the host of the generated URL will be a private DNS that isn't accessible from the internet. +- `publicHost`: is used to configure the public host for pre-signed URL generation if the connector communicates privately with the storage server through an internal DNS. If this setting isn't set the host of the generated URL will be a private DNS that isn't accessible from the internet. - `region`: (optional) region of the bucket going to be created. - `maxRetries`: maximum number of retry times. Default 10. -- `defaultPresignedExpiry`: the default expiry for presigned URL generation in duration format. The maximum expiry is 7 days \(`168h`\) and minimum is 1 second \(`1s`\). -- `trailingHeaders`: indicates server support of trailing headers. Only supported for v4 signatures. -- `allowedBuckets`: the list of allowed bucket names. This setting prevents users to get buckets and objects outside the list. However, it's recommended to restrict the permissions for the IAM credentials. This setting is useful to let the connector know which buckets belong to this client. The empty value means all buckets are allowed. The storage server will handle the validation. +- `defaultPresignedExpiry`: the default expiry for pre-signed URL generation in duration format. The maximum expiry is 7 days \(`168h`\) and minimum is 1 second \(`1s`\). +- `trailingHeaders` indicates server support for trailing headers. Only supported for v4 signatures. +- `allowedBuckets`: the list of allowed bucket names. This setting prevents users from getting buckets and objects outside the list. However, it's recommended that permissions for the IAM credentials be restricted. This setting is useful to let the connector know which buckets belong to this client. The empty value means all buckets are allowed. The storage server will handle the validation. -### Authentication +### S3-Compatible Client -#### Static Credentials (S3) +#### Authentication + +##### Static Credentials Configure the authentication type `static` with `accessKeyId` and `secretAccessKey`. `sessionToken` is also supported for [temporary access](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_use-resources.html) but for testing only. @@ -35,15 +37,19 @@ clients: env: SECRET_ACCESS_KEY ``` -#### IAM (S3) +##### IAM -The IAM authentication retrieves credentials from the AWS EC2, ECS or EKS service, and keeps track if those credentials are expired. This authentication method can be used only if the connector is hosted in the AWS ecosystem. +The IAM authentication retrieves credentials from the AWS EC2, ECS, or EKS service, and keeps track if those credentials are expired. This authentication method can be used only if the connector is hosted in the AWS ecosystem. The following settings are supported: -- `iamAuthEndpoint` : the optional custom endpoint to fetch IAM role credentials. The client can automatically identify the endpoint if not set. +- `iamAuthEndpoint`: the optional custom endpoint to fetch IAM role credentials. The client can automatically identify the endpoint if not set. + +### Azure Blob Storage + +#### Authentication -#### SharedKey (Azure Blob Storage) +#### Shared Key Authorize with an immutable SharedKeyCredential containing the storage account's name and either its primary or secondary key. See [Manage storage account access keys](https://learn.microsoft.com/en-us/azure/storage/common/storage-account-keys-manage?tabs=azure-portal) in Azure Storage docs. @@ -58,7 +64,7 @@ clients: env: AZURE_STORAGE_ACCOUNT_KEY ``` -#### Connection String (Azure Blob Storage) +#### Connection String The connector uses the `endpoint` to authorize for your Azure storage account. See [Azure Storage docs](https://learn.microsoft.com/en-us/azure/storage/common/storage-configure-connection-string?toc=%2Fazure%2Fstorage%2Fblobs%2Ftoc.json&bc=%2Fazure%2Fstorage%2Fblobs%2Fbreadcrumb%2Ftoc.json#configure-a-connection-string-for-an-azure-storage-account) for more context. @@ -70,9 +76,9 @@ clients: type: connectionString ``` -#### Microsoft Entra (or Azure Active Directory) +##### Microsoft Entra (or Azure Active Directory) -The `entra` type supports Microsoft Entra (or Azure Active Directory) that authenticates a service principal with a secret, certificate, user-password or Azure workload identity. You can configure multiple credentials. They can be chained together. +The `entra` type supports Microsoft Entra (or Azure Active Directory) that authenticates a service principal with a secret, certificate, user-password, or Azure workload identity. You can configure multiple credentials. They can be chained together. ```yaml clients: @@ -105,7 +111,7 @@ clients: - ``` -#### Anonymous Access (Azure Blob Storage) +##### Anonymous Access ```yaml clients: @@ -115,18 +121,36 @@ clients: type: anonymous ``` -### Examples +### Google Cloud Storage -> See full configuration examples at [tests/configuration/configuration.yaml](../tests/configuration/configuration.yaml). +#### Authentication -#### AWS S3 +##### Access Credentials -Create [a user access key](https://docs.aws.amazon.com/IAM/latest/UserGuide/access-keys-admin-managed.html) with S3 permissions to configure the Access Key ID and Secret Access Key. +Authorize to Google Cloud Storage with a service account credential in JSON format. The connector supports either an inline value or a file path. ```yaml clients: - - id: s3 - type: s3 + - type: gcs + projectId: + env: GOOGLE_PROJECT_ID + authentication: + type: credentials + # inline credentials + credentials: + value: '{"type": "service_account", ... }' # json string + # or a file path + # credentialsFile: + # env: GOOGLE_STORAGE_CREDENTIALS_FILE +``` + +##### HMAC + +You must use the `s3` client instead. [Generate HMAC key](https://cloud.google.com/storage/docs/authentication/hmackeys) to configure the Access Key ID and Secret Access Key. + +```yaml +clients: + - type: s3 defaultBucket: env: DEFAULT_BUCKET authentication: @@ -137,14 +161,29 @@ clients: env: SECRET_ACCESS_KEY ``` -#### Google Cloud Storage +##### Anonymous -You need to [generate HMAC key](https://cloud.google.com/storage/docs/authentication/hmackeys) to configure the Access Key ID and Secret Access Key. +Use this authentication if the client accesses public buckets and objects only. ```yaml clients: - - id: gs - type: gs + - type: gcs + authentication: + type: anonymous +``` + +### Examples + +> See full configuration examples at [tests/configuration/configuration.yaml](../tests/configuration/configuration.yaml). + +#### AWS S3 + +Create [a user access key](https://docs.aws.amazon.com/IAM/latest/UserGuide/access-keys-admin-managed.html) with S3 permissions to configure the Access Key ID and Secret Access Key. + +```yaml +clients: + - id: s3 + type: s3 defaultBucket: env: DEFAULT_BUCKET authentication: diff --git a/docs/objects.md b/docs/objects.md index c1da8e9..38a28f8 100644 --- a/docs/objects.md +++ b/docs/objects.md @@ -103,12 +103,12 @@ query DownloadObjectText { | Service | Pagination | | -------------------- | ---------- | -| AWS S3 | ❌ | -| MinIO | ❌ | -| Google Cloud Storage | ❌ | -| Cloudflare R2 | ❌ | -| DigitalOcean Spaces | ❌ | +| AWS S3 | ❌ | +| Google Cloud Storage | ✅ | | Azure Blob Storage | ✅ | +| MinIO | ❌ | +| Cloudflare R2 | ❌ | +| DigitalOcean Spaces | ❌ | ```graphql query ListObjects { @@ -116,7 +116,6 @@ query ListObjects { pageInfo { cursor hasNextPage - nextCursor } objects { clientId @@ -126,9 +125,8 @@ query ListObjects { serverEncrypted size storageClass - userMetadata - userTagCount - userTags + tagCount + tags cacheControl checksumCrc32 checksumCrc64Nvme diff --git a/go.mod b/go.mod index e233644..4030536 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/hasura/ndc-storage go 1.23 require ( + cloud.google.com/go/storage v1.50.0 github.com/Azure/azure-sdk-for-go/sdk/azcore v1.16.0 github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.0 github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.5.0 @@ -17,26 +18,45 @@ require ( go.opentelemetry.io/otel v1.29.0 go.opentelemetry.io/otel/trace v1.29.0 golang.org/x/sync v0.10.0 + google.golang.org/api v0.214.0 gopkg.in/yaml.v3 v3.0.1 gotest.tools/v3 v3.5.1 ) require ( + cel.dev/expr v0.16.1 // indirect + cloud.google.com/go v0.116.0 // indirect + cloud.google.com/go/auth v0.13.0 // indirect + cloud.google.com/go/auth/oauth2adapt v0.2.6 // indirect + cloud.google.com/go/compute/metadata v0.6.0 // indirect + cloud.google.com/go/iam v1.2.2 // indirect + cloud.google.com/go/monitoring v1.21.2 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 // indirect github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.25.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.48.1 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.48.1 // indirect github.com/bahlo/generic-list-go v0.2.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/buger/jsonparser v1.1.1 // indirect github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/cncf/xds/go v0.0.0-20240905190251-b4127c9b8d78 // indirect github.com/dustin/go-humanize v1.0.1 // indirect + github.com/envoyproxy/go-control-plane/envoy v1.32.3 // indirect + github.com/envoyproxy/protoc-gen-validate v1.1.0 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect github.com/go-ini/ini v1.67.0 // indirect github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/goccy/go-json v0.10.4 // indirect github.com/golang-jwt/jwt/v5 v5.2.1 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/google/go-cmp v0.6.0 // indirect + github.com/google/s2a-go v0.1.8 // indirect github.com/google/uuid v1.6.0 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.4 // indirect + github.com/googleapis/gax-go/v2 v2.14.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.22.0 // indirect github.com/klauspost/compress v1.17.11 // indirect github.com/klauspost/cpuid/v2 v2.2.9 // indirect @@ -44,13 +64,18 @@ require ( github.com/mailru/easyjson v0.7.7 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect + github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect github.com/prometheus/client_golang v1.20.5 // indirect github.com/prometheus/client_model v0.6.1 // indirect github.com/prometheus/common v0.61.0 // indirect github.com/prometheus/procfs v0.15.1 // indirect github.com/rs/xid v1.6.0 // indirect github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect + go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/contrib/bridges/otelslog v0.4.0 // indirect + go.opentelemetry.io/contrib/detectors/gcp v1.29.0 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 // indirect go.opentelemetry.io/contrib/propagators/b3 v1.29.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.5.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.5.0 // indirect @@ -68,10 +93,13 @@ require ( go.opentelemetry.io/proto/otlp v1.3.1 // indirect golang.org/x/crypto v0.31.0 // indirect golang.org/x/net v0.33.0 // indirect + golang.org/x/oauth2 v0.24.0 // indirect golang.org/x/sys v0.28.0 // indirect golang.org/x/text v0.21.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20240822170219-fc7c04adadcd // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240822170219-fc7c04adadcd // indirect - google.golang.org/grpc v1.65.0 // indirect + golang.org/x/time v0.8.0 // indirect + google.golang.org/genproto v0.0.0-20241118233622-e639e219e697 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20241118233622-e639e219e697 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20241209162323-e6fa225c2576 // indirect + google.golang.org/grpc v1.67.3 // indirect google.golang.org/protobuf v1.35.2 // indirect ) diff --git a/go.sum b/go.sum index a92ff1b..b4bfcfd 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,26 @@ +cel.dev/expr v0.16.1 h1:NR0+oFYzR1CqLFhTAqg3ql59G9VfN8fKq1TCHJ6gq1g= +cel.dev/expr v0.16.1/go.mod h1:AsGA5zb3WruAEQeQng1RZdGEXmBj0jvMWh6l5SnNuC8= +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.116.0 h1:B3fRrSDkLRt5qSHWe40ERJvhvnQwdZiHu0bJOpldweE= +cloud.google.com/go v0.116.0/go.mod h1:cEPSRWPzZEswwdr9BxE6ChEn01dWlTaF05LiC2Xs70U= +cloud.google.com/go/auth v0.13.0 h1:8Fu8TZy167JkW8Tj3q7dIkr2v4cndv41ouecJx0PAHs= +cloud.google.com/go/auth v0.13.0/go.mod h1:COOjD9gwfKNKz+IIduatIhYJQIc0mG3H102r/EMxX6Q= +cloud.google.com/go/auth/oauth2adapt v0.2.6 h1:V6a6XDu2lTwPZWOawrAa9HUK+DB2zfJyTuciBG5hFkU= +cloud.google.com/go/auth/oauth2adapt v0.2.6/go.mod h1:AlmsELtlEBnaNTL7jCj8VQFLy6mbZv0s4Q7NGBeQ5E8= +cloud.google.com/go/compute/metadata v0.6.0 h1:A6hENjEsCDtC1k8byVsgwvVcioamEHvZ4j01OwKxG9I= +cloud.google.com/go/compute/metadata v0.6.0/go.mod h1:FjyFAW1MW0C203CEOMDTu3Dk1FlqW3Rga40jzHL4hfg= +cloud.google.com/go/iam v1.2.2 h1:ozUSofHUGf/F4tCNy/mu9tHLTaxZFLOUiKzjcgWHGIA= +cloud.google.com/go/iam v1.2.2/go.mod h1:0Ys8ccaZHdI1dEUilwzqng/6ps2YB6vRsjIe00/+6JY= +cloud.google.com/go/logging v1.12.0 h1:ex1igYcGFd4S/RZWOCU51StlIEuey5bjqwH9ZYjHibk= +cloud.google.com/go/logging v1.12.0/go.mod h1:wwYBt5HlYP1InnrtYI0wtwttpVU1rifnMT7RejksUAM= +cloud.google.com/go/longrunning v0.6.2 h1:xjDfh1pQcWPEvnfjZmwjKQEcHnpz6lHjfy7Fo0MK+hc= +cloud.google.com/go/longrunning v0.6.2/go.mod h1:k/vIs83RN4bE3YCswdXC5PFfWVILjm3hpEUlSko4PiI= +cloud.google.com/go/monitoring v1.21.2 h1:FChwVtClH19E7pJ+e0xUhJPGksctZNVOk2UhMmblmdU= +cloud.google.com/go/monitoring v1.21.2/go.mod h1:hS3pXvaG8KgWTSz+dAdyzPrGUYmi2Q+WFX8g2hqVEZU= +cloud.google.com/go/storage v1.50.0 h1:3TbVkzTooBvnZsk7WaAQfOsNrdoM8QHusXA1cpk6QJs= +cloud.google.com/go/storage v1.50.0/go.mod h1:l7XeiD//vx5lfqE3RavfmU9yvk5Pp0Zhcv482poyafY= +cloud.google.com/go/trace v1.11.2 h1:4ZmaBdL8Ng/ajrgKqY5jfvzqMXbrDcBsUGXOT9aqTtI= +cloud.google.com/go/trace v1.11.2/go.mod h1:bn7OwXd4pd5rFuAnTrzBuoZ4ax2XQeG3qNgYmfCy0Io= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.16.0 h1:JZg6HRh6W6U4OLl6lk7BZ7BLisIzM9dG1R50zUk9C/M= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.16.0/go.mod h1:YL1xnZ6QejvQHWJrX/AvhFl4WW4rqHVoKspWNVwFk0M= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.0 h1:B/dfvscEQtew9dVuoxqxrUKKv8Ih2f55PydknDamU+g= @@ -18,6 +41,15 @@ github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJ github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE= github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 h1:XHOnouVk1mxXfQidrMEnLlPk9UMeRtyBTnEFtxkV0kU= github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.25.0 h1:3c8yed4lgqTt+oTQ+JNMDo+F4xprBf+O/il4ZC0nRLw= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.25.0/go.mod h1:obipzmGjfSjam60XLwGfqUkJsfiheAl+TUjG+4yzyPM= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.48.1 h1:UQ0AhxogsIRZDkElkblfnwjc3IaltCm2HUMvezQaL7s= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.48.1/go.mod h1:jyqM3eLpJ3IbIFDTKVz2rF9T/xWGW0rIriGwnz8l9Tk= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.48.1 h1:oTX4vsorBZo/Zdum6OKPA4o7544hm6smoRv1QjpTwGo= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.48.1/go.mod h1:0wEl7vrAD8mehJyohS9HZy+WyEOaQO2mJx86Cvh93kM= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.48.1 h1:8nn+rsCvTq9axyEh382S0PFLBeaFwNsT43IrPWzctRU= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.48.1/go.mod h1:viRWSEhtMZqz1rhwmOVKkWl6SwmVowfL9O2YR5gI2PE= github.com/alecthomas/assert/v2 v2.11.0 h1:2Q9r3ki8+JYXvGsDyBXwH3LcJ+WK5D0gc5E8vS6K3D0= github.com/alecthomas/assert/v2 v2.11.0/go.mod h1:Bze95FyfUr7x34QZrjL+XP+0qgp/zg8yS+TtBj1WA3k= github.com/alecthomas/kong v1.6.1 h1:/7bVimARU3uxPD0hbryPE8qWrS3Oz3kPQoxA/H2NKG8= @@ -32,14 +64,34 @@ github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMU github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cncf/xds/go v0.0.0-20240905190251-b4127c9b8d78 h1:QVw89YDxXxEe+l8gU8ETbOasdwEV+avkR75ZzsVV9WI= +github.com/cncf/xds/go v0.0.0-20240905190251-b4127c9b8d78/go.mod h1:W+zGtBO5Y1IgJhy4+A9GOqVhqLpfZi+vwmdNXUehLA8= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/go-control-plane v0.13.4 h1:zEqyPVyku6IvWCFwux4x9RxkLOMUL+1vC9xUFv5l2/M= +github.com/envoyproxy/go-control-plane v0.13.4/go.mod h1:kDfuBlDVsSj2MjrLEtRWtHlsWIFcGyB2RMO44Dc5GZA= +github.com/envoyproxy/go-control-plane/envoy v1.32.3 h1:hVEaommgvzTjTd4xCaFd+kEQ2iYBtGxP6luyLrx6uOk= +github.com/envoyproxy/go-control-plane/envoy v1.32.3/go.mod h1:F6hWupPfh75TBXGKA++MCT/CZHFq5r9/uwt/kQYkZfE= +github.com/envoyproxy/go-control-plane/ratelimit v0.1.0 h1:/G9QYbddjL25KvtKTv3an9lx6VBE2cnb8wp1vEGNYGI= +github.com/envoyproxy/go-control-plane/ratelimit v0.1.0/go.mod h1:Wk+tMFAFbCXaJPzVVHnPgRKdUdwW/KdbRt94AzgRee4= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/envoyproxy/protoc-gen-validate v1.1.0 h1:tntQDh69XqOCOZsDz0lVJQez/2L6Uu2PdjCQwWCJ3bM= +github.com/envoyproxy/protoc-gen-validate v1.1.0/go.mod h1:sXRDRVmzEbkM7CVcM06s9shE/m23dg3wzjl0UWqJ2q4= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= @@ -53,10 +105,39 @@ github.com/goccy/go-json v0.10.4 h1:JSwxQzIqKfmFX1swYPpUThQZp/Ka4wzJdK0LWVytLPM= github.com/goccy/go-json v0.10.4/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk= github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9Fc= +github.com/google/martian/v3 v3.3.3/go.mod h1:iEPrYcgCF7jA9OtScMFQyAlZZ4YXTKEtJ1E6RWzmBA0= +github.com/google/s2a-go v0.1.8 h1:zZDs9gcbt9ZPLV0ndSyQk6Kacx2g/X+SKYovpnz3SMM= +github.com/google/s2a-go v0.1.8/go.mod h1:6iNWHTpQ+nfNRN5E00MSdfDwVesa8hhS32PhPO8deJA= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.3.4 h1:XYIDZApgAnrN1c855gTgghdIA6Stxb52D5RnLI1SLyw= +github.com/googleapis/enterprise-certificate-proxy v0.3.4/go.mod h1:YKe7cfqYXjKGpGvmSg28/fFvhNzinZQm8DGnaburhGA= +github.com/googleapis/gax-go/v2 v2.14.0 h1:f+jMrjBPl+DL9nI4IQzLUxMq7XrAqFYB7hBPqMNIe8o= +github.com/googleapis/gax-go/v2 v2.14.0/go.mod h1:lhBCnjdLrWRaPvLWhmc8IS24m9mr07qSYnHncrgo+zk= github.com/grpc-ecosystem/grpc-gateway/v2 v2.22.0 h1:asbCHRVmodnJTuQ3qamDwqVOIjwqUPTYmYuemVOx+Ys= github.com/grpc-ecosystem/grpc-gateway/v2 v2.22.0/go.mod h1:ggCgvZ2r7uOoQjOyu2Y1NhHmEPPzzuhWgcza5M1Ji1I= github.com/hasura/ndc-sdk-go v1.7.1-0.20250116172618-e81c1199f349 h1:pE/KbhkffaeTilb4wyN/Hehaap+sw8+Q0Q9nxjeyGtw= @@ -91,10 +172,13 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y= github.com/prometheus/client_golang v1.20.5/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= github.com/prometheus/common v0.61.0 h1:3gv/GThfX0cV2lpO7gkTUwZru38mxevy90Bj8YFSRQQ= @@ -107,12 +191,26 @@ github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/wk8/go-ordered-map/v2 v2.1.8 h1:5h/BUHu93oj4gIdvHHHGsScSTMijfx5PeYkE/fJgbpc= github.com/wk8/go-ordered-map/v2 v2.1.8/go.mod h1:5nJHM5DyteebpVlHnWMV0rPz6Zp7+xBAnxjb1X5vnTw= +go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= +go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/contrib/bridges/otelslog v0.4.0 h1:i66F95zqmrf3EyN5gu0E2pjTvCRZo/p8XIYidG3vOP8= go.opentelemetry.io/contrib/bridges/otelslog v0.4.0/go.mod h1:JuCiVizZ6ovLZLnYk1nGRUEAnmRJLKGh5v8DmwiKlhY= +go.opentelemetry.io/contrib/detectors/gcp v1.29.0 h1:TiaiXB4DpGD3sdzNlYQxruQngn5Apwzi1X0DRhuGvDQ= +go.opentelemetry.io/contrib/detectors/gcp v1.29.0/go.mod h1:GW2aWZNwR2ZxDLdv8OyC2G8zkRoQBuURgV7RPQgcPoU= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0 h1:r6I7RJCN86bpD/FQwedZ0vSixDpwuWREjW9oRMsmqDc= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0/go.mod h1:B9yO6b04uB80CzjedvewuqDhxJxi11s7/GtiGa8bAjI= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 h1:TT4fX+nBOA/+LUkobKGW1ydGcn+G3vRw9+g5HwCphpk= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0/go.mod h1:L7UH0GbB0p47T4Rri3uHjbpCFYrVrwc1I25QhNPiGK8= go.opentelemetry.io/contrib/propagators/b3 v1.29.0 h1:hNjyoRsAACnhoOLWupItUjABzeYmX3GTTZLzwJluJlk= go.opentelemetry.io/contrib/propagators/b3 v1.29.0/go.mod h1:E76MTitU1Niwo5NSN+mVxkyLu4h4h7Dp/yh38F2WuIU= go.opentelemetry.io/otel v1.29.0 h1:PdomN/Al4q/lN6iBJEN3AwPvUiHPMlt93c8bqTG5Llw= @@ -135,6 +233,8 @@ go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.29.0 h1:JAv0J go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.29.0/go.mod h1:QNKLmUEAq2QUbPQUfvw4fmv0bgbK7UlOSFCnXyfvSNc= go.opentelemetry.io/otel/exporters/prometheus v0.51.0 h1:G7uexXb/K3T+T9fNLCCKncweEtNEBMTO+46hKX5EdKw= go.opentelemetry.io/otel/exporters/prometheus v0.51.0/go.mod h1:v0mFe5Kk7woIh938mrZBJBmENYquyA0IICrlYm4Y0t4= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.29.0 h1:WDdP9acbMYjbKIyJUhTvtzj601sVJOqgWdUxSdR/Ysc= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.29.0/go.mod h1:BLbf7zbNIONBLPwvFnwNHGj4zge8uTCM/UPIVW1Mq2I= go.opentelemetry.io/otel/log v0.5.0 h1:x1Pr6Y3gnXgl1iFBwtGy1W/mnzENoK0w0ZoaeOI3i30= go.opentelemetry.io/otel/log v0.5.0/go.mod h1:NU/ozXeGuOR5/mjCRXYbTC00NFJ3NYuraV/7O78F0rE= go.opentelemetry.io/otel/metric v1.29.0 h1:vPf/HFWTNkPu1aYeIsc98l4ktOQaL6LeSoeV2g+8YLc= @@ -151,29 +251,87 @@ go.opentelemetry.io/proto/otlp v1.3.1 h1:TrMUixzpM0yuc/znrFTP9MMRh8trP93mkCiDVeX go.opentelemetry.io/proto/otlp v1.3.1/go.mod h1:0X1WI4de4ZsLrrJNLAQbFeLCm3T7yBkR0XqQ7niQU+8= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.24.0 h1:KTBBxWqUa0ykRPLtV69rRto9TLXcqYkeswu48x/gvNE= +golang.org/x/oauth2 v0.24.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= -google.golang.org/genproto/googleapis/api v0.0.0-20240822170219-fc7c04adadcd h1:BBOTEWLuuEGQy9n1y9MhVJ9Qt0BDu21X8qZs71/uPZo= -google.golang.org/genproto/googleapis/api v0.0.0-20240822170219-fc7c04adadcd/go.mod h1:fO8wJzT2zbQbAjbIoos1285VfEIYKDDY+Dt+WpTkh6g= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240822170219-fc7c04adadcd h1:6TEm2ZxXoQmFWFlt1vNxvVOa1Q0dXFQD1m/rYjXmS0E= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240822170219-fc7c04adadcd/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= -google.golang.org/grpc v1.65.0 h1:bs/cUb4lp1G5iImFFd3u5ixQzweKizoZJAwBNLR42lc= -google.golang.org/grpc v1.65.0/go.mod h1:WgYC2ypjlB0EiQi6wdKixMqukr6lBc0Vo+oOgjrM5ZQ= +golang.org/x/time v0.8.0 h1:9i3RxcPv3PZnitoVGMPDKZSq1xW1gK1Xy3ArNOGZfEg= +golang.org/x/time v0.8.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.214.0 h1:h2Gkq07OYi6kusGOaT/9rnNljuXmqPnaig7WGPmKbwA= +google.golang.org/api v0.214.0/go.mod h1:bYPpLG8AyeMWwDU6NXoB00xC0DFkikVvd5MfwoxjLqE= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20241118233622-e639e219e697 h1:ToEetK57OidYuqD4Q5w+vfEnPvPpuTwedCNVohYJfNk= +google.golang.org/genproto v0.0.0-20241118233622-e639e219e697/go.mod h1:JJrvXBWRZaFMxBufik1a4RpFw4HhgVtBBWQeQgUj2cc= +google.golang.org/genproto/googleapis/api v0.0.0-20241118233622-e639e219e697 h1:pgr/4QbFyktUv9CtQ/Fq4gzEE6/Xs7iCXbktaGzLHbQ= +google.golang.org/genproto/googleapis/api v0.0.0-20241118233622-e639e219e697/go.mod h1:+D9ySVjN8nY8YCVjc5O7PZDIdZporIDY3KaGfJunh88= +google.golang.org/genproto/googleapis/rpc v0.0.0-20241209162323-e6fa225c2576 h1:8ZmaLZE4XWrtU3MyClkYqqtl6Oegr3235h7jxsDyqCY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20241209162323-e6fa225c2576/go.mod h1:5uTbfoYQed2U9p3KIj2/Zzm02PYhndfdmML0qC3q3FU= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= +google.golang.org/grpc v1.67.3 h1:OgPcDAFKHnH8X3O4WcO4XUc8GRDeKsKReqbQtiCj7N8= +google.golang.org/grpc v1.67.3/go.mod h1:YGaHCc6Oap+FzBJTZLBzkGSYt/cvGPFTPxkn7QfSU8s= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.35.2 h1:8Ar7bF+apOIoThw1EdZl0p1oWvMqTHmpA2fRTyZO8io= google.golang.org/protobuf v1.35.2/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools/v3 v3.5.1 h1:EENdUnS3pdur5nybKYIh2Vfgc8IUNBjxDPSjtiJcOzU= gotest.tools/v3 v3.5.1/go.mod h1:isy3WKz7GK6uNw/sbHzfKBLvlvXwUyV06n6brMxxopU= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/jsonschema/configuration.schema.json b/jsonschema/configuration.schema.json index 18d643d..ba98a4a 100644 --- a/jsonschema/configuration.schema.json +++ b/jsonschema/configuration.schema.json @@ -10,8 +10,7 @@ "type": { "type": "string", "enum": [ - "s3", - "gs" + "s3" ], "description": "Cloud provider type of the storage client" }, @@ -27,10 +26,6 @@ "$ref": "#/$defs/EnvString", "description": "Endpoint of the storage server. Required for other S3 compatible services such as MinIO, Cloudflare R2, DigitalOcean Spaces, etc..." }, - "publicHost": { - "$ref": "#/$defs/EnvString", - "description": "The public host to be used for presigned URL generation" - }, "maxRetries": { "type": "integer", "minimum": 1, @@ -61,6 +56,10 @@ ], "description": "Optional region" }, + "publicHost": { + "$ref": "#/$defs/EnvString", + "description": "The public host to be used for presigned URL generation" + }, "authentication": { "oneOf": [ { @@ -140,10 +139,6 @@ "$ref": "#/$defs/EnvString", "description": "Endpoint of the storage server. Required for other S3 compatible services such as MinIO, Cloudflare R2, DigitalOcean Spaces, etc..." }, - "publicHost": { - "$ref": "#/$defs/EnvString", - "description": "The public host to be used for presigned URL generation" - }, "maxRetries": { "type": "integer", "minimum": 1, @@ -272,6 +267,10 @@ "$ref": "#/$defs/EnvString", "description": "the path of a file containing a Kubernetes service account token" }, + "audience": { + "type": "#/$defs/EnvString", + "description": "Audience to use when requesting tokens for Azure Active Directory authentication" + }, "disableInstanceDiscovery": { "type": "boolean" }, @@ -308,6 +307,116 @@ "defaultBucket", "authentication" ] + }, + { + "properties": { + "type": { + "type": "string", + "enum": [ + "gcs" + ], + "description": "Cloud provider type of the storage client" + }, + "id": { + "type": "string", + "description": "The unique identity of a client. Use this setting if there are many configured clients" + }, + "defaultBucket": { + "$ref": "#/$defs/EnvString", + "description": "Default bucket name to be set if the user doesn't specify any bucket" + }, + "endpoint": { + "$ref": "#/$defs/EnvString", + "description": "Endpoint of the storage server. Required for other S3 compatible services such as MinIO, Cloudflare R2, DigitalOcean Spaces, etc..." + }, + "maxRetries": { + "type": "integer", + "minimum": 1, + "description": "Maximum number of retry times", + "default": 10 + }, + "defaultPresignedExpiry": { + "type": "string", + "pattern": "[0-9]+(s|m|h)", + "description": "Default bucket name to be set if the user doesn't specify any bucket", + "default": "24h" + }, + "allowedBuckets": { + "items": { + "type": "string" + }, + "type": "array", + "description": "Allowed buckets. This setting prevents users to get buckets and objects outside the list. However, it's recommended to restrict the permissions for the IAM credentials" + }, + "authentication": { + "oneOf": [ + { + "oneOf": [ + { + "required": [ + "credentials" + ] + }, + { + "required": [ + "credentialsFile" + ] + } + ], + "properties": { + "type": { + "type": "string", + "enum": [ + "credentials" + ], + "description": "Authorize with a service account or refresh token JSON credentials" + }, + "credentials": { + "$ref": "#/$defs/EnvString", + "description": "The given service account or refresh token JSON credentials in JSON string format" + }, + "credentialsFile": { + "$ref": "#/$defs/EnvString", + "description": "The given service account or refresh token JSON credentials file" + } + }, + "type": "object", + "required": [ + "type" + ] + }, + { + "properties": { + "type": { + "type": "string", + "enum": [ + "anonymous" + ] + } + }, + "type": "object", + "required": [ + "type" + ] + } + ] + }, + "publicHost": { + "$ref": "#/$defs/EnvString", + "description": "The public host to be used for presigned URL generation" + }, + "projectId": { + "$ref": "#/$defs/EnvString", + "description": "Project ID of the Google Cloud account" + } + }, + "type": "object", + "required": [ + "type", + "defaultBucket", + "authentication", + "projectId" + ] } ] }, diff --git a/scripts/test.sh b/scripts/test.sh index 3dd9dc0..f8f1e24 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -27,7 +27,7 @@ http_wait() { exit 1 } -docker compose up -d --build minio s3mock azurite ndc-storage +docker compose up -d --build minio s3mock azurite gcp-storage-emulator ndc-storage http_wait http://localhost:8080/health http_wait http://localhost:9000/minio/health/live diff --git a/tests/certs/service_account.json b/tests/certs/service_account.json new file mode 100644 index 0000000..ab34f42 --- /dev/null +++ b/tests/certs/service_account.json @@ -0,0 +1,13 @@ +{ + "type": "service_account", + "project_id": "test-local-project", + "private_key_id": "3f80e86d25d5802132e5f51166259f0123456789", + "private_key": "-----BEGIN PRIVATE KEY-----\nMIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCwpR3t3P81TOK7\n3rV25fOjQC8KVwUSVBlVMi3R44aAFAXlirPXlj1daZZ2DeaVfPLJE44s3qPyJaob\ndVCTeSAFAKDYBm4AfCSKg2YVq2ImVVwuxb9sLf1oLijUkgafUi4NUELRDaoJi7GA\nlfHa9ngxuXnwDfAYq/rnpqDuVcCmqTcalSqWdZlK4oOdTouGFM93hkrVs3m9leko\ns4ZD54JS+Dddni1eCVAxNCk8wRe7BTA1DRfKU7dxRlXT0km/c1djLtBTIJFqoABp\nQu95Y5INn71gOOx7+pe8XFF5aGRvbvrBXTxKpeJMu/T9+SSrih5ywCfsvtCYhLHu\ntO9g43TbAgMBAAECggEAAyKYbTmAEWbxGpWBfKqTJZbE3vbxa5mw2L7Aam+9L4vA\nw7SvrB1fWceUeI7sRJidnyz+LSoHGIv2QMYsbRmivwP7rCndDPF8corne8NOxXsR\nXPEnNxklMhKo/duDoOiZUaTW3WqpKxc0H3dJ03GOwOSCV1zf9zLM4WH8jDkFvM96\nEbx3nRtySwsbwPQHZMwOAxBypDu/Bsq6t8xQFSmHr6UWwubSwTLG4nI8X4RCZs0n\nhukNLh05M6eQ6WSiwXMvQHcvA8A4+179uSBSfcT25DYGU5fh2/ZbTsQR2v7bIXzy\nVZDNk9cSJDnsr9D1yKgR304awsLyoMH+aqJyJDV3VQKBgQD3uq08QIVaWQl63qd6\nh2AYgOZnC/T6kIT91Ac4C/VBUPAbPXCL+PGtkZP2J9NM7+6OjTFIdxwxnn/MOiVK\nCE/AWpEHD5LV2Z75tFIg7uE5aGwZO+URl6wfwpUnuwib0zIQs0gtOCrR1XCYfgUc\n3J8UawuYpJC0+Xhbxb1G7M7TvQKBgQC2iuN9KC702bJFEnsedWsjEvz6kGYls5gW\nm91ywj9Rg0X5uHYKvUSD3z3cDe4j6nm1jzpkNyPdhNeBpj9AeduAzu47prApeu19\nrPiFTf6Wc6B3guIUXoSyo5p+uGKd8ohMELFv6pyrKOQ7bvldLVTuqBTxsO7ks1fJ\n5gRvkCeodwKBgDx0ELxoXmKK4dZbKXGsQZYZNqCk+VaAGAQ9TrmtqV8UGlP7bVCr\nIochxILIezcGQigFaqZtcrLlI/i5o8nqN1QQhd28Pi/zmGkFSRmCUfIJen+57w+Y\n/McPXAjDCrnEEy6iIZXvY/uq43xDHIpLiIKaEM1iM3IGgwmaOrNpLJvdAoGBAIoS\nZd3OHi5f16K9RtkUEXSXMTjJLipIPLqlLIxX9xLNy4kGiGq5HHDu09h1xPHvsh2t\nd4pD1uIAUA2l1OQNCAXW/4WpIEhPzgEoa1ZRV0cvsGmVWzHzCEYiz4IYL6euwn9e\nhQfVGqmwJWn7F6zt0/cstaLWpOvh5340eJh2HjkjAoGBANN8GLTrZMqiSb/YfpbW\n+FZilYftnKsaiYX+Lf5gTfAZva7/Sffxwcq8E/qO/PbJg6A1z5eRZLWfGFueCwV1\nug+FYNyfiIIeNoM8WqNqddRFc1pRBYbpzouTbqfC5Ks6lclClO65+5mg51RDVibF\n6KgcmQcvsrsIz1Gt3JnxksH1\n-----END PRIVATE KEY-----\n", + "client_email": "test-account@test-local-project.iam.gserviceaccount.com", + "client_id": "111111111111111111111", + "auth_uri": "https://accounts.google.com/o/oauth2/auth", + "token_uri": "https://oauth2.googleapis.com/token", + "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", + "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/test-account%40test-local-project.iam.gserviceaccount.com", + "universe_domain": "googleapis.com" +} diff --git a/tests/configuration/configuration.yaml b/tests/configuration/configuration.yaml index fb5e2f9..f86dde0 100644 --- a/tests/configuration/configuration.yaml +++ b/tests/configuration/configuration.yaml @@ -56,6 +56,33 @@ clients: connectionString: env: AZURE_STORAGE_CONNECTION_STRING defaultPresignedExpiry: "1h" + - id: gcs + type: gcs + defaultBucket: + env: GOOGLE_STORAGE_DEFAULT_BUCKET + endpoint: + env: GOOGLE_STORAGE_ENDPOINT + publicHost: + env: GOOGLE_STORAGE_PUBLIC_HOST + projectId: + env: GOOGLE_PROJECT_ID + authentication: + type: anonymous + defaultPresignedExpiry: "1h" + - id: gcs-cred + type: gcs + defaultBucket: + env: GOOGLE_STORAGE_DEFAULT_BUCKET + endpoint: + env: GOOGLE_STORAGE_ENDPOINT + publicHost: + env: GOOGLE_STORAGE_PUBLIC_HOST + projectId: + env: GOOGLE_PROJECT_ID + authentication: + type: credentials + credentialsFile: + env: GOOGLE_STORAGE_CREDENTIALS_FILE concurrency: query: 10 diff --git a/tests/engine/app/metadata/ComposeStorageObject.hml b/tests/engine/app/metadata/ComposeStorageObject.hml index 316d43b..18b4a2a 100644 --- a/tests/engine/app/metadata/ComposeStorageObject.hml +++ b/tests/engine/app/metadata/ComposeStorageObject.hml @@ -9,21 +9,17 @@ definition: type: String - name: legalHold type: Boolean + - name: metadata + type: Json - name: mode type: StorageRetentionMode - name: object type: String! - - name: replaceMetadata - type: Boolean - - name: replaceTags - type: Boolean - name: retainUntilDate type: TimestampTz - name: size type: Int64 - - name: userMetadata - type: Json - - name: userTags + - name: tags type: Json graphql: typeName: StorageCopyDestOptions @@ -43,14 +39,12 @@ definition: allowedFields: - bucket - legalHold + - metadata - mode - object - - replaceMetadata - - replaceTags - retainUntilDate - size - - userMetadata - - userTags + - tags --- kind: ObjectType diff --git a/tests/engine/app/metadata/CreateStorageBucket.hml b/tests/engine/app/metadata/CreateStorageBucket.hml index a931c66..31f10c9 100644 --- a/tests/engine/app/metadata/CreateStorageBucket.hml +++ b/tests/engine/app/metadata/CreateStorageBucket.hml @@ -9,7 +9,7 @@ definition: type: StorageClientId - name: name type: String! - - name: objectLocking + - name: objectLock type: Boolean - name: region type: String diff --git a/tests/engine/app/metadata/DownloadStorageObject.hml b/tests/engine/app/metadata/DownloadStorageObject.hml index 59bc961..3c692ab 100644 --- a/tests/engine/app/metadata/DownloadStorageObject.hml +++ b/tests/engine/app/metadata/DownloadStorageObject.hml @@ -2,26 +2,26 @@ kind: ObjectType version: v1 definition: - name: StorageObjectSimple + name: StorageObjectFilter fields: - name: bucket - type: BucketName! + type: StorageBucketName! - name: clientId type: StorageClientId! - name: object - type: ObjectPath! + type: StorageStringFilter! graphql: - typeName: StorageObjectSimple - inputTypeName: StorageObjectSimpleInput + typeName: StorageObjectFilter + inputTypeName: StorageObjectFilterInput dataConnectorTypeMapping: - dataConnectorName: storage - dataConnectorObjectType: StorageObjectSimple + dataConnectorObjectType: StorageObjectFilter --- kind: TypePermissions version: v1 definition: - typeName: StorageObjectSimple + typeName: StorageObjectFilter permissions: - role: admin output: @@ -34,24 +34,24 @@ definition: kind: BooleanExpressionType version: v1 definition: - name: StorageObjectSimpleBoolExp + name: StorageObjectFilterBoolExp operand: object: - type: StorageObjectSimple + type: StorageObjectFilter comparableFields: - fieldName: bucket - booleanExpressionType: BucketNameBoolExp + booleanExpressionType: StorageBucketNameBoolExp - fieldName: clientId booleanExpressionType: StorageClientIdBoolExp - fieldName: object - booleanExpressionType: ObjectPathBoolExp + booleanExpressionType: StorageStringFilterBoolExp comparableRelationships: [] logicalOperators: enable: true isNull: enable: true graphql: - typeName: StorageObjectSimpleBoolExp + typeName: StorageObjectFilterBoolExp --- kind: Command @@ -75,7 +75,7 @@ definition: - name: versionId type: String - name: where - type: StorageObjectSimpleBoolExp + type: StorageObjectFilterBoolExp source: dataConnectorName: storage dataConnectorCommand: diff --git a/tests/engine/app/metadata/DownloadStorageObjectText.hml b/tests/engine/app/metadata/DownloadStorageObjectText.hml index 6c2601c..3312c46 100644 --- a/tests/engine/app/metadata/DownloadStorageObjectText.hml +++ b/tests/engine/app/metadata/DownloadStorageObjectText.hml @@ -20,7 +20,7 @@ definition: - name: versionId type: String - name: where - type: StorageObjectSimpleBoolExp + type: StorageObjectFilterBoolExp source: dataConnectorName: storage dataConnectorCommand: diff --git a/tests/engine/app/metadata/RemoveStorageBucketReplication.hml b/tests/engine/app/metadata/RemoveStorageBucketReplication.hml deleted file mode 100644 index 5caec98..0000000 --- a/tests/engine/app/metadata/RemoveStorageBucketReplication.hml +++ /dev/null @@ -1,28 +0,0 @@ ---- -kind: Command -version: v1 -definition: - name: RemoveStorageBucketReplication - outputType: Boolean! - arguments: - - name: bucket - type: String - - name: clientId - type: StorageClientId - source: - dataConnectorName: storage - dataConnectorCommand: - procedure: removeStorageBucketReplication - graphql: - rootFieldName: removeStorageBucketReplication - rootFieldKind: Mutation - ---- -kind: CommandPermissions -version: v1 -definition: - commandName: RemoveStorageBucketReplication - permissions: - - role: admin - allowExecution: true - diff --git a/tests/engine/app/metadata/RemoveStorageObject.hml b/tests/engine/app/metadata/RemoveStorageObject.hml index 0b6d088..b655e10 100644 --- a/tests/engine/app/metadata/RemoveStorageObject.hml +++ b/tests/engine/app/metadata/RemoveStorageObject.hml @@ -15,10 +15,12 @@ definition: type: Boolean - name: object type: String! + - name: softDelete + type: Boolean - name: versionId type: String - name: where - type: StorageObjectSimpleBoolExp + type: StorageObjectFilterBoolExp source: dataConnectorName: storage dataConnectorCommand: diff --git a/tests/engine/app/metadata/RemoveStorageObjects.hml b/tests/engine/app/metadata/RemoveStorageObjects.hml index b72dfef..4f6f50a 100644 --- a/tests/engine/app/metadata/RemoveStorageObjects.hml +++ b/tests/engine/app/metadata/RemoveStorageObjects.hml @@ -53,7 +53,7 @@ definition: - name: startAfter type: String! - name: where - type: StorageObjectSimpleBoolExp + type: StorageObjectFilterBoolExp source: dataConnectorName: storage dataConnectorCommand: diff --git a/tests/engine/app/metadata/EnableStorageBucketVersioning.hml b/tests/engine/app/metadata/RestoreStorageObject.hml similarity index 57% rename from tests/engine/app/metadata/EnableStorageBucketVersioning.hml rename to tests/engine/app/metadata/RestoreStorageObject.hml index baffcd6..8c53669 100644 --- a/tests/engine/app/metadata/EnableStorageBucketVersioning.hml +++ b/tests/engine/app/metadata/RestoreStorageObject.hml @@ -2,27 +2,31 @@ kind: Command version: v1 definition: - name: EnableStorageBucketVersioning + name: RestoreStorageObject outputType: Boolean! arguments: - name: bucket type: String - name: clientId type: StorageClientId + - name: object + type: String! + - name: where + type: StorageObjectFilterBoolExp source: dataConnectorName: storage dataConnectorCommand: - procedure: enableStorageBucketVersioning + procedure: restoreStorageObject graphql: - rootFieldName: enableStorageBucketVersioning + rootFieldName: restoreStorageObject rootFieldKind: Mutation - description: enables bucket versioning support. + description: restore a soft-deleted object. --- kind: CommandPermissions version: v1 definition: - commandName: EnableStorageBucketVersioning + commandName: RestoreStorageObject permissions: - role: admin allowExecution: true diff --git a/tests/engine/app/metadata/SetStorageBucketEncryption.hml b/tests/engine/app/metadata/SetStorageBucketEncryption.hml deleted file mode 100644 index dc11b15..0000000 --- a/tests/engine/app/metadata/SetStorageBucketEncryption.hml +++ /dev/null @@ -1,31 +0,0 @@ ---- -kind: Command -version: v1 -definition: - name: SetStorageBucketEncryption - outputType: Boolean! - arguments: - - name: bucket - type: String - - name: clientId - type: StorageClientId - - name: rules - type: "[ServerSideEncryptionRule!]!" - source: - dataConnectorName: storage - dataConnectorCommand: - procedure: setStorageBucketEncryption - graphql: - rootFieldName: setStorageBucketEncryption - rootFieldKind: Mutation - description: sets default encryption configuration on a bucket. - ---- -kind: CommandPermissions -version: v1 -definition: - commandName: SetStorageBucketEncryption - permissions: - - role: admin - allowExecution: true - diff --git a/tests/engine/app/metadata/SetStorageBucketLifecycle.hml b/tests/engine/app/metadata/SetStorageBucketLifecycle.hml deleted file mode 100644 index cfed1c1..0000000 --- a/tests/engine/app/metadata/SetStorageBucketLifecycle.hml +++ /dev/null @@ -1,31 +0,0 @@ ---- -kind: Command -version: v1 -definition: - name: SetStorageBucketLifecycle - outputType: Boolean! - arguments: - - name: bucket - type: String - - name: clientId - type: StorageClientId - - name: rules - type: "[BucketLifecycleRule!]!" - source: - dataConnectorName: storage - dataConnectorCommand: - procedure: setStorageBucketLifecycle - graphql: - rootFieldName: setStorageBucketLifecycle - rootFieldKind: Mutation - description: sets lifecycle on bucket or an object prefix. - ---- -kind: CommandPermissions -version: v1 -definition: - commandName: SetStorageBucketLifecycle - permissions: - - role: admin - allowExecution: true - diff --git a/tests/engine/app/metadata/SetStorageBucketNotification.hml b/tests/engine/app/metadata/SetStorageBucketNotification.hml deleted file mode 100644 index 5184e04..0000000 --- a/tests/engine/app/metadata/SetStorageBucketNotification.hml +++ /dev/null @@ -1,35 +0,0 @@ ---- -kind: Command -version: v1 -definition: - name: SetStorageBucketNotification - outputType: Boolean! - arguments: - - name: bucket - type: String - - name: clientId - type: StorageClientId - - name: cloudFunctionConfigurations - type: "[NotificationLambdaConfig!]!" - - name: queueConfigurations - type: "[NotificationQueueConfig!]!" - - name: topicConfigurations - type: "[NotificationTopicConfig!]!" - source: - dataConnectorName: storage - dataConnectorCommand: - procedure: setStorageBucketNotification - graphql: - rootFieldName: setStorageBucketNotification - rootFieldKind: Mutation - description: sets a new notification configuration on a bucket. - ---- -kind: CommandPermissions -version: v1 -definition: - commandName: SetStorageBucketNotification - permissions: - - role: admin - allowExecution: true - diff --git a/tests/engine/app/metadata/SetStorageBucketReplication.hml b/tests/engine/app/metadata/SetStorageBucketReplication.hml deleted file mode 100644 index 4eab61f..0000000 --- a/tests/engine/app/metadata/SetStorageBucketReplication.hml +++ /dev/null @@ -1,35 +0,0 @@ ---- -kind: Command -version: v1 -definition: - name: SetStorageBucketReplication - outputType: Boolean! - arguments: - - name: bucket - type: String - - name: clientId - type: StorageClientId - - name: role - type: String - - name: rules - type: "[StorageReplicationRule!]!" - source: - dataConnectorName: storage - dataConnectorCommand: - procedure: setStorageBucketReplication - graphql: - rootFieldName: setStorageBucketReplication - rootFieldKind: Mutation - description: sets replication configuration on a bucket. Role can be obtained by - first defining the replication target on MinIO to associate the source and - destination buckets for replication with the replication endpoint. - ---- -kind: CommandPermissions -version: v1 -definition: - commandName: SetStorageBucketReplication - permissions: - - role: admin - allowExecution: true - diff --git a/tests/engine/app/metadata/SetStorageBucketTags.hml b/tests/engine/app/metadata/SetStorageBucketTags.hml deleted file mode 100644 index bd55490..0000000 --- a/tests/engine/app/metadata/SetStorageBucketTags.hml +++ /dev/null @@ -1,31 +0,0 @@ ---- -kind: Command -version: v1 -definition: - name: SetStorageBucketTags - outputType: Boolean! - arguments: - - name: bucket - type: String - - name: clientId - type: StorageClientId - - name: tags - type: Json! - source: - dataConnectorName: storage - dataConnectorCommand: - procedure: setStorageBucketTags - graphql: - rootFieldName: setStorageBucketTags - rootFieldKind: Mutation - description: sets tags to a bucket. - ---- -kind: CommandPermissions -version: v1 -definition: - commandName: SetStorageBucketTags - permissions: - - role: admin - allowExecution: true - diff --git a/tests/engine/app/metadata/SetStorageObjectLegalHold.hml b/tests/engine/app/metadata/SetStorageObjectLegalHold.hml deleted file mode 100644 index 06d67bf..0000000 --- a/tests/engine/app/metadata/SetStorageObjectLegalHold.hml +++ /dev/null @@ -1,37 +0,0 @@ ---- -kind: Command -version: v1 -definition: - name: SetStorageObjectLegalHold - outputType: Boolean! - arguments: - - name: bucket - type: String - - name: clientId - type: StorageClientId - - name: object - type: String! - - name: status - type: Boolean - - name: versionId - type: String - - name: where - type: StorageObjectSimpleBoolExp - source: - dataConnectorName: storage - dataConnectorCommand: - procedure: setStorageObjectLegalHold - graphql: - rootFieldName: setStorageObjectLegalHold - rootFieldKind: Mutation - description: applies legal-hold onto an object. - ---- -kind: CommandPermissions -version: v1 -definition: - commandName: SetStorageObjectLegalHold - permissions: - - role: admin - allowExecution: true - diff --git a/tests/engine/app/metadata/SetStorageObjectLockConfig.hml b/tests/engine/app/metadata/SetStorageObjectLockConfig.hml deleted file mode 100644 index e3aedf5..0000000 --- a/tests/engine/app/metadata/SetStorageObjectLockConfig.hml +++ /dev/null @@ -1,34 +0,0 @@ ---- -kind: Command -version: v1 -definition: - name: SetStorageObjectLockConfig - outputType: Boolean! - arguments: - - name: bucket - type: String - - name: clientId - type: StorageClientId - - name: mode - type: StorageRetentionMode - - name: unit - type: StorageRetentionValidityUnit - - name: validity - type: Int32 - source: - dataConnectorName: storage - dataConnectorCommand: - procedure: setStorageObjectLockConfig - graphql: - rootFieldName: setStorageObjectLockConfig - rootFieldKind: Mutation - ---- -kind: CommandPermissions -version: v1 -definition: - commandName: SetStorageObjectLockConfig - permissions: - - role: admin - allowExecution: true - diff --git a/tests/engine/app/metadata/SetStorageObjectRetention.hml b/tests/engine/app/metadata/SetStorageObjectRetention.hml deleted file mode 100644 index 5cc92fb..0000000 --- a/tests/engine/app/metadata/SetStorageObjectRetention.hml +++ /dev/null @@ -1,41 +0,0 @@ ---- -kind: Command -version: v1 -definition: - name: SetStorageObjectRetention - outputType: Boolean! - arguments: - - name: bucket - type: String - - name: clientId - type: StorageClientId - - name: governanceBypass - type: Boolean - - name: mode - type: StorageRetentionMode - - name: object - type: String! - - name: retainUntilDate - type: TimestampTz - - name: versionId - type: String - - name: where - type: StorageObjectSimpleBoolExp - source: - dataConnectorName: storage - dataConnectorCommand: - procedure: setStorageObjectRetention - graphql: - rootFieldName: setStorageObjectRetention - rootFieldKind: Mutation - description: applies object retention lock onto an object. - ---- -kind: CommandPermissions -version: v1 -definition: - commandName: SetStorageObjectRetention - permissions: - - role: admin - allowExecution: true - diff --git a/tests/engine/app/metadata/SetStorageObjectTags.hml b/tests/engine/app/metadata/SetStorageObjectTags.hml deleted file mode 100644 index 67224a8..0000000 --- a/tests/engine/app/metadata/SetStorageObjectTags.hml +++ /dev/null @@ -1,38 +0,0 @@ ---- -kind: Command -version: v1 -definition: - name: SetStorageObjectTags - outputType: Boolean! - arguments: - - name: bucket - type: String - - name: clientId - type: StorageClientId - - name: object - type: String! - - name: tags - type: Json! - - name: versionId - type: String - - name: where - type: StorageObjectSimpleBoolExp - source: - dataConnectorName: storage - dataConnectorCommand: - procedure: setStorageObjectTags - graphql: - rootFieldName: setStorageObjectTags - rootFieldKind: Mutation - description: sets new object Tags to the given object, replaces/overwrites any - existing tags. - ---- -kind: CommandPermissions -version: v1 -definition: - commandName: SetStorageObjectTags - permissions: - - role: admin - allowExecution: true - diff --git a/tests/engine/app/metadata/StorageBucket.hml b/tests/engine/app/metadata/StorageBucket.hml new file mode 100644 index 0000000..04a0c1b --- /dev/null +++ b/tests/engine/app/metadata/StorageBucket.hml @@ -0,0 +1,792 @@ +--- +kind: ObjectType +version: v1 +definition: + name: BucketAutoclass + fields: + - name: enabled + type: Boolean! + - name: terminalStorageClass + type: String! + - name: terminalStorageClassUpdateTime + type: TimestampTz! + - name: toggleTime + type: TimestampTz! + graphql: + typeName: BucketAutoclass + inputTypeName: BucketAutoclassInput + dataConnectorTypeMapping: + - dataConnectorName: storage + dataConnectorObjectType: BucketAutoclass + +--- +kind: TypePermissions +version: v1 +definition: + typeName: BucketAutoclass + permissions: + - role: admin + output: + allowedFields: + - enabled + - terminalStorageClass + - terminalStorageClassUpdateTime + - toggleTime + +--- +kind: ObjectType +version: v1 +definition: + name: BucketCors + description: is the bucket's Cross-Origin Resource Sharing (CORS) configuration. + fields: + - name: maxAge + type: Duration! + - name: methods + type: "[String!]!" + - name: origins + type: "[String!]!" + - name: responseHeaders + type: "[String!]!" + graphql: + typeName: BucketCors + inputTypeName: BucketCorsInput + dataConnectorTypeMapping: + - dataConnectorName: storage + dataConnectorObjectType: BucketCors + +--- +kind: TypePermissions +version: v1 +definition: + typeName: BucketCors + permissions: + - role: admin + output: + allowedFields: + - maxAge + - methods + - origins + - responseHeaders + +--- +kind: ObjectType +version: v1 +definition: + name: CustomPlacementConfig + description: holds the bucket's custom placement configuration for Custom Dual + Regions. See https://cloud.google.com/storage/docs/locations#location-dr for + more information. + fields: + - name: dataLocations + type: "[String!]!" + graphql: + typeName: CustomPlacementConfig + inputTypeName: CustomPlacementConfigInput + dataConnectorTypeMapping: + - dataConnectorName: storage + dataConnectorObjectType: CustomPlacementConfig + fieldMapping: + dataLocations: + column: + name: DataLocations + +--- +kind: TypePermissions +version: v1 +definition: + typeName: CustomPlacementConfig + permissions: + - role: admin + output: + allowedFields: + - dataLocations + +--- +kind: ObjectType +version: v1 +definition: + name: ServerSideEncryptionConfiguration + description: is the default encryption configuration structure. + fields: + - name: kmsMasterKeyId + type: String + - name: sseAlgorithm + type: String + graphql: + typeName: ServerSideEncryptionConfiguration + inputTypeName: ServerSideEncryptionConfigurationInput + dataConnectorTypeMapping: + - dataConnectorName: storage + dataConnectorObjectType: ServerSideEncryptionConfiguration + +--- +kind: TypePermissions +version: v1 +definition: + typeName: ServerSideEncryptionConfiguration + permissions: + - role: admin + output: + allowedFields: + - kmsMasterKeyId + - sseAlgorithm + +--- +kind: ObjectType +version: v1 +definition: + name: BucketHierarchicalNamespace + fields: + - name: enabled + type: Boolean! + graphql: + typeName: BucketHierarchicalNamespace + inputTypeName: BucketHierarchicalNamespaceInput + dataConnectorTypeMapping: + - dataConnectorName: storage + dataConnectorObjectType: BucketHierarchicalNamespace + +--- +kind: TypePermissions +version: v1 +definition: + typeName: BucketHierarchicalNamespace + permissions: + - role: admin + output: + allowedFields: + - enabled + +--- +kind: ObjectType +version: v1 +definition: + name: ObjectAbortIncompleteMultipartUpload + fields: + - name: daysAfterInitiation + type: Int32 + graphql: + typeName: ObjectAbortIncompleteMultipartUpload + inputTypeName: ObjectAbortIncompleteMultipartUploadInput + dataConnectorTypeMapping: + - dataConnectorName: storage + dataConnectorObjectType: ObjectAbortIncompleteMultipartUpload + +--- +kind: TypePermissions +version: v1 +definition: + typeName: ObjectAbortIncompleteMultipartUpload + permissions: + - role: admin + output: + allowedFields: + - daysAfterInitiation + +--- +kind: ObjectType +version: v1 +definition: + name: ObjectLifecycleAllVersionsExpiration + description: represents AllVersionsExpiration actions element in an ILM policy + fields: + - name: days + type: Int32 + - name: deleteMarker + type: Boolean + graphql: + typeName: ObjectLifecycleAllVersionsExpiration + inputTypeName: ObjectLifecycleAllVersionsExpirationInput + dataConnectorTypeMapping: + - dataConnectorName: storage + dataConnectorObjectType: ObjectLifecycleAllVersionsExpiration + +--- +kind: TypePermissions +version: v1 +definition: + typeName: ObjectLifecycleAllVersionsExpiration + permissions: + - role: admin + output: + allowedFields: + - days + - deleteMarker + +--- +kind: ObjectType +version: v1 +definition: + name: ObjectLifecycleDelMarkerExpiration + fields: + - name: days + type: Int32 + graphql: + typeName: ObjectLifecycleDelMarkerExpiration + inputTypeName: ObjectLifecycleDelMarkerExpirationInput + dataConnectorTypeMapping: + - dataConnectorName: storage + dataConnectorObjectType: ObjectLifecycleDelMarkerExpiration + +--- +kind: TypePermissions +version: v1 +definition: + typeName: ObjectLifecycleDelMarkerExpiration + permissions: + - role: admin + output: + allowedFields: + - days + +--- +kind: ObjectType +version: v1 +definition: + name: ObjectLifecycleExpiration + description: expiration details of lifecycle configuration + fields: + - name: date + type: Date + - name: days + type: Int32 + - name: expiredObjectAllVersions + type: Boolean + - name: expiredObjectDeleteMarker + type: Boolean + graphql: + typeName: ObjectLifecycleExpiration + inputTypeName: ObjectLifecycleExpirationInput + dataConnectorTypeMapping: + - dataConnectorName: storage + dataConnectorObjectType: ObjectLifecycleExpiration + +--- +kind: TypePermissions +version: v1 +definition: + typeName: ObjectLifecycleExpiration + permissions: + - role: admin + output: + allowedFields: + - date + - days + - expiredObjectAllVersions + - expiredObjectDeleteMarker + +--- +kind: ObjectType +version: v1 +definition: + name: ObjectLifecycleFilter + description: will be used in selecting rule(s) for lifecycle configuration + fields: + - name: matchesPrefix + type: "[String!]" + - name: matchesStorageClasses + type: "[String!]" + - name: matchesSuffix + type: "[String!]" + - name: objectSizeGreaterThan + type: Int64 + - name: objectSizeLessThan + type: Int64 + - name: tags + type: Json + graphql: + typeName: ObjectLifecycleFilter + inputTypeName: ObjectLifecycleFilterInput + dataConnectorTypeMapping: + - dataConnectorName: storage + dataConnectorObjectType: ObjectLifecycleFilter + +--- +kind: TypePermissions +version: v1 +definition: + typeName: ObjectLifecycleFilter + permissions: + - role: admin + output: + allowedFields: + - matchesPrefix + - matchesStorageClasses + - matchesSuffix + - objectSizeGreaterThan + - objectSizeLessThan + - tags + +--- +kind: ObjectType +version: v1 +definition: + name: ObjectLifecycleNoncurrentVersionExpiration + description: "- Specifies when noncurrent object versions expire. Upon + expiration, server permanently deletes the noncurrent object versions. Set + this lifecycle configuration action on a bucket that has versioning enabled + (or suspended) to request server delete noncurrent object versions at a + specific period in the object's lifetime." + fields: + - name: newerNoncurrentVersions + type: Int32 + - name: noncurrentDays + type: Int32 + graphql: + typeName: ObjectLifecycleNoncurrentVersionExpiration + inputTypeName: ObjectLifecycleNoncurrentVersionExpirationInput + dataConnectorTypeMapping: + - dataConnectorName: storage + dataConnectorObjectType: ObjectLifecycleNoncurrentVersionExpiration + +--- +kind: TypePermissions +version: v1 +definition: + typeName: ObjectLifecycleNoncurrentVersionExpiration + permissions: + - role: admin + output: + allowedFields: + - newerNoncurrentVersions + - noncurrentDays + +--- +kind: ObjectType +version: v1 +definition: + name: ObjectLifecycleNoncurrentVersionTransition + description: sets this action to request server to transition noncurrent object + versions to different set storage classes at a specific period in the + object's lifetime. + fields: + - name: newerNoncurrentVersions + type: Int32 + - name: noncurrentDays + type: Int32 + - name: storageClass + type: String + graphql: + typeName: ObjectLifecycleNoncurrentVersionTransition + inputTypeName: ObjectLifecycleNoncurrentVersionTransitionInput + dataConnectorTypeMapping: + - dataConnectorName: storage + dataConnectorObjectType: ObjectLifecycleNoncurrentVersionTransition + +--- +kind: TypePermissions +version: v1 +definition: + typeName: ObjectLifecycleNoncurrentVersionTransition + permissions: + - role: admin + output: + allowedFields: + - newerNoncurrentVersions + - noncurrentDays + - storageClass + +--- +kind: ObjectType +version: v1 +definition: + name: ObjectLifecycleTransition + description: transition details of lifecycle configuration + fields: + - name: date + type: Date + - name: days + type: Int32 + - name: storageClass + type: String + graphql: + typeName: ObjectLifecycleTransition + inputTypeName: ObjectLifecycleTransitionInput + dataConnectorTypeMapping: + - dataConnectorName: storage + dataConnectorObjectType: ObjectLifecycleTransition + +--- +kind: TypePermissions +version: v1 +definition: + typeName: ObjectLifecycleTransition + permissions: + - role: admin + output: + allowedFields: + - date + - days + - storageClass + +--- +kind: ObjectType +version: v1 +definition: + name: ObjectLifecycleRule + description: represents a single rule in lifecycle configuration + fields: + - name: abortIncompleteMultipartUpload + type: ObjectAbortIncompleteMultipartUpload + - name: allVersionsExpiration + type: ObjectLifecycleAllVersionsExpiration + - name: delMarkerExpiration + type: ObjectLifecycleDelMarkerExpiration + - name: enabled + type: Boolean + - name: expiration + type: ObjectLifecycleExpiration + - name: filter + type: "[ObjectLifecycleFilter!]" + - name: id + type: String + - name: noncurrentVersionExpiration + type: ObjectLifecycleNoncurrentVersionExpiration + - name: noncurrentVersionTransition + type: ObjectLifecycleNoncurrentVersionTransition + - name: prefix + type: String + - name: transition + type: ObjectLifecycleTransition + graphql: + typeName: ObjectLifecycleRule + inputTypeName: ObjectLifecycleRuleInput + dataConnectorTypeMapping: + - dataConnectorName: storage + dataConnectorObjectType: ObjectLifecycleRule + +--- +kind: TypePermissions +version: v1 +definition: + typeName: ObjectLifecycleRule + permissions: + - role: admin + output: + allowedFields: + - abortIncompleteMultipartUpload + - allVersionsExpiration + - delMarkerExpiration + - enabled + - expiration + - filter + - id + - noncurrentVersionExpiration + - noncurrentVersionTransition + - prefix + - transition + +--- +kind: ObjectType +version: v1 +definition: + name: ObjectLifecycleConfiguration + description: is a collection of lifecycle Rule objects. + fields: + - name: rules + type: "[ObjectLifecycleRule!]!" + graphql: + typeName: ObjectLifecycleConfiguration + inputTypeName: ObjectLifecycleConfigurationInput + dataConnectorTypeMapping: + - dataConnectorName: storage + dataConnectorObjectType: ObjectLifecycleConfiguration + +--- +kind: TypePermissions +version: v1 +definition: + typeName: ObjectLifecycleConfiguration + permissions: + - role: admin + output: + allowedFields: + - rules + +--- +kind: ObjectType +version: v1 +definition: + name: BucketLogging + description: holds the bucket's logging configuration, which defines the + destination bucket and optional name prefix for the current bucket's logs. + fields: + - name: logBucket + type: String! + - name: logObjectPrefix + type: String! + graphql: + typeName: BucketLogging + inputTypeName: BucketLoggingInput + dataConnectorTypeMapping: + - dataConnectorName: storage + dataConnectorObjectType: BucketLogging + +--- +kind: TypePermissions +version: v1 +definition: + typeName: BucketLogging + permissions: + - role: admin + output: + allowedFields: + - logBucket + - logObjectPrefix + +--- +kind: ObjectType +version: v1 +definition: + name: StorageObjectLockConfig + description: represents the object lock configuration in given bucket + fields: + - name: enabled + type: Boolean! + - name: mode + type: StorageRetentionMode + - name: unit + type: StorageRetentionValidityUnit + - name: validity + type: Int32 + graphql: + typeName: StorageObjectLockConfig + inputTypeName: StorageObjectLockConfigInput + dataConnectorTypeMapping: + - dataConnectorName: storage + dataConnectorObjectType: StorageObjectLockConfig + +--- +kind: TypePermissions +version: v1 +definition: + typeName: StorageObjectLockConfig + permissions: + - role: admin + output: + allowedFields: + - enabled + - mode + - unit + - validity + +--- +kind: ObjectType +version: v1 +definition: + name: StorageObjectSoftDeletePolicy + description: contains the bucket's soft delete policy, which defines the period + of time that soft-deleted objects will be retained, and cannot be + permanently deleted. + fields: + - name: effectiveTime + type: TimestampTz! + - name: retentionDuration + type: Duration! + graphql: + typeName: StorageObjectSoftDeletePolicy + inputTypeName: StorageObjectSoftDeletePolicyInput + dataConnectorTypeMapping: + - dataConnectorName: storage + dataConnectorObjectType: StorageObjectSoftDeletePolicy + +--- +kind: TypePermissions +version: v1 +definition: + typeName: StorageObjectSoftDeletePolicy + permissions: + - role: admin + output: + allowedFields: + - effectiveTime + - retentionDuration + +--- +kind: ObjectType +version: v1 +definition: + name: StorageBucketVersioningConfiguration + description: is the versioning configuration structure + fields: + - name: enabled + type: Boolean! + - name: excludeFolders + type: Boolean + - name: excludedPrefixes + type: "[String!]" + - name: mfaDelete + type: String + graphql: + typeName: StorageBucketVersioningConfiguration + inputTypeName: StorageBucketVersioningConfigurationInput + dataConnectorTypeMapping: + - dataConnectorName: storage + dataConnectorObjectType: StorageBucketVersioningConfiguration + +--- +kind: TypePermissions +version: v1 +definition: + typeName: StorageBucketVersioningConfiguration + permissions: + - role: admin + output: + allowedFields: + - enabled + - excludeFolders + - excludedPrefixes + - mfaDelete + +--- +kind: ObjectType +version: v1 +definition: + name: BucketWebsite + description: holds the bucket's website configuration, controlling how the + service behaves when accessing bucket contents as a web site. See + https://cloud.google.com/storage/docs/static-website for more information. + fields: + - name: mainPageSuffix + type: String! + - name: notFoundPage + type: String + graphql: + typeName: BucketWebsite + inputTypeName: BucketWebsiteInput + dataConnectorTypeMapping: + - dataConnectorName: storage + dataConnectorObjectType: BucketWebsite + +--- +kind: TypePermissions +version: v1 +definition: + typeName: BucketWebsite + permissions: + - role: admin + output: + allowedFields: + - mainPageSuffix + - notFoundPage + +--- +kind: ObjectType +version: v1 +definition: + name: StorageBucket + description: the container for bucket metadata. + fields: + - name: autoclass + type: BucketAutoclass + - name: cors + type: "[BucketCors!]" + - name: creationTime + type: TimestampTz + - name: customPlacementConfig + type: CustomPlacementConfig + - name: defaultEventBasedHold + type: Boolean + - name: encryption + type: ServerSideEncryptionConfiguration + - name: etag + type: String + - name: hierarchicalNamespace + type: BucketHierarchicalNamespace + - name: lastModified + type: TimestampTz + - name: lifecycle + type: ObjectLifecycleConfiguration + - name: locationType + type: String + - name: logging + type: BucketLogging + - name: name + type: String! + - name: objectLock + type: StorageObjectLockConfig + - name: region + type: String + - name: requesterPays + type: Boolean + - name: rpo + type: GoogleStorageRpo + - name: softDeletePolicy + type: StorageObjectSoftDeletePolicy + - name: storageClass + type: String + - name: tags + type: Json + - name: versioning + type: StorageBucketVersioningConfiguration + - name: website + type: BucketWebsite + graphql: + typeName: StorageBucket + inputTypeName: StorageBucketInput + dataConnectorTypeMapping: + - dataConnectorName: storage + dataConnectorObjectType: StorageBucket + +--- +kind: TypePermissions +version: v1 +definition: + typeName: StorageBucket + permissions: + - role: admin + output: + allowedFields: + - autoclass + - cors + - creationTime + - customPlacementConfig + - defaultEventBasedHold + - encryption + - etag + - hierarchicalNamespace + - lastModified + - lifecycle + - locationType + - logging + - name + - objectLock + - region + - requesterPays + - rpo + - softDeletePolicy + - storageClass + - tags + - versioning + - website + +--- +kind: Command +version: v1 +definition: + name: StorageBucket + outputType: StorageBucket + arguments: + - name: bucket + type: String + - name: clientId + type: StorageClientId + source: + dataConnectorName: storage + dataConnectorCommand: + function: storageBucket + graphql: + rootFieldName: storageBucket + rootFieldKind: Query + description: gets a bucket by name. + +--- +kind: CommandPermissions +version: v1 +definition: + commandName: StorageBucket + permissions: + - role: admin + allowExecution: true + diff --git a/tests/engine/app/metadata/StorageBucketEncryption.hml b/tests/engine/app/metadata/StorageBucketEncryption.hml deleted file mode 100644 index 9706a01..0000000 --- a/tests/engine/app/metadata/StorageBucketEncryption.hml +++ /dev/null @@ -1,115 +0,0 @@ ---- -kind: ObjectType -version: v1 -definition: - name: StorageApplySseByDefault - description: defines default encryption configuration, KMS or SSE. To activate - KMS, SSEAlgoritm needs to be set to `aws:kms“. Minio currently does not - support Kms. - fields: - - name: kmsMasterKeyId - type: String - - name: sseAlgorithm - type: String! - graphql: - typeName: StorageApplySseByDefault - inputTypeName: StorageApplySseByDefaultInput - dataConnectorTypeMapping: - - dataConnectorName: storage - dataConnectorObjectType: StorageApplySSEByDefault - ---- -kind: TypePermissions -version: v1 -definition: - typeName: StorageApplySseByDefault - permissions: - - role: admin - output: - allowedFields: - - kmsMasterKeyId - - sseAlgorithm - ---- -kind: ObjectType -version: v1 -definition: - name: ServerSideEncryptionRule - description: rule layer encapsulates default encryption configuration - fields: - - name: apply - type: StorageApplySseByDefault! - graphql: - typeName: ServerSideEncryptionRule - inputTypeName: ServerSideEncryptionRuleInput - dataConnectorTypeMapping: - - dataConnectorName: storage - dataConnectorObjectType: ServerSideEncryptionRule - ---- -kind: TypePermissions -version: v1 -definition: - typeName: ServerSideEncryptionRule - permissions: - - role: admin - output: - allowedFields: - - apply - ---- -kind: ObjectType -version: v1 -definition: - name: ServerSideEncryptionConfiguration - description: is the default encryption configuration structure. - fields: - - name: rules - type: "[ServerSideEncryptionRule!]!" - graphql: - typeName: ServerSideEncryptionConfiguration - inputTypeName: ServerSideEncryptionConfigurationInput - dataConnectorTypeMapping: - - dataConnectorName: storage - dataConnectorObjectType: ServerSideEncryptionConfiguration - ---- -kind: TypePermissions -version: v1 -definition: - typeName: ServerSideEncryptionConfiguration - permissions: - - role: admin - output: - allowedFields: - - rules - ---- -kind: Command -version: v1 -definition: - name: StorageBucketEncryption - outputType: ServerSideEncryptionConfiguration - arguments: - - name: bucket - type: String - - name: clientId - type: StorageClientId - source: - dataConnectorName: storage - dataConnectorCommand: - function: storageBucketEncryption - graphql: - rootFieldName: storageBucketEncryption - rootFieldKind: Query - description: gets default encryption configuration set on a bucket. - ---- -kind: CommandPermissions -version: v1 -definition: - commandName: StorageBucketEncryption - permissions: - - role: admin - allowExecution: true - diff --git a/tests/engine/app/metadata/StorageBucketLifecycle.hml b/tests/engine/app/metadata/StorageBucketLifecycle.hml deleted file mode 100644 index 2bd6976..0000000 --- a/tests/engine/app/metadata/StorageBucketLifecycle.hml +++ /dev/null @@ -1,440 +0,0 @@ ---- -kind: ObjectType -version: v1 -definition: - name: AbortIncompleteMultipartUpload - description: structure, not supported yet on MinIO - fields: - - name: daysAfterInitiation - type: Int32 - graphql: - typeName: AbortIncompleteMultipartUpload - inputTypeName: AbortIncompleteMultipartUploadInput - dataConnectorTypeMapping: - - dataConnectorName: storage - dataConnectorObjectType: AbortIncompleteMultipartUpload - ---- -kind: TypePermissions -version: v1 -definition: - typeName: AbortIncompleteMultipartUpload - permissions: - - role: admin - output: - allowedFields: - - daysAfterInitiation - ---- -kind: ObjectType -version: v1 -definition: - name: LifecycleAllVersionsExpiration - description: represents AllVersionsExpiration actions element in an ILM policy - fields: - - name: days - type: Int32 - - name: deleteMarker - type: Boolean - graphql: - typeName: LifecycleAllVersionsExpiration - inputTypeName: LifecycleAllVersionsExpirationInput - dataConnectorTypeMapping: - - dataConnectorName: storage - dataConnectorObjectType: LifecycleAllVersionsExpiration - ---- -kind: TypePermissions -version: v1 -definition: - typeName: LifecycleAllVersionsExpiration - permissions: - - role: admin - output: - allowedFields: - - days - - deleteMarker - ---- -kind: ObjectType -version: v1 -definition: - name: LifecycleDelMarkerExpiration - description: represents DelMarkerExpiration actions element in an ILM policy - fields: - - name: days - type: Int32 - graphql: - typeName: LifecycleDelMarkerExpiration - inputTypeName: LifecycleDelMarkerExpirationInput - dataConnectorTypeMapping: - - dataConnectorName: storage - dataConnectorObjectType: LifecycleDelMarkerExpiration - ---- -kind: TypePermissions -version: v1 -definition: - typeName: LifecycleDelMarkerExpiration - permissions: - - role: admin - output: - allowedFields: - - days - ---- -kind: ObjectType -version: v1 -definition: - name: LifecycleExpiration - description: expiration details of lifecycle configuration - fields: - - name: date - type: Date - - name: days - type: Int32 - - name: expiredObjectAllVersions - type: Boolean - - name: expiredObjectDeleteMarker - type: Boolean - graphql: - typeName: LifecycleExpiration - inputTypeName: LifecycleExpirationInput - dataConnectorTypeMapping: - - dataConnectorName: storage - dataConnectorObjectType: LifecycleExpiration - ---- -kind: TypePermissions -version: v1 -definition: - typeName: LifecycleExpiration - permissions: - - role: admin - output: - allowedFields: - - date - - days - - expiredObjectAllVersions - - expiredObjectDeleteMarker - ---- -kind: ObjectType -version: v1 -definition: - name: StorageTag - description: structure key/value pair representing an object tag to apply configuration - fields: - - name: key - type: String - - name: value - type: String - graphql: - typeName: StorageTag - inputTypeName: StorageTagInput - dataConnectorTypeMapping: - - dataConnectorName: storage - dataConnectorObjectType: StorageTag - ---- -kind: TypePermissions -version: v1 -definition: - typeName: StorageTag - permissions: - - role: admin - output: - allowedFields: - - key - - value - ---- -kind: ObjectType -version: v1 -definition: - name: LifecycleFilterAnd - description: the And Rule for LifecycleTag, to be used in LifecycleRuleFilter - fields: - - name: objectSizeGreaterThan - type: Int64 - - name: objectSizeLessThan - type: Int64 - - name: prefix - type: String - - name: tags - type: "[StorageTag!]" - graphql: - typeName: LifecycleFilterAnd - inputTypeName: LifecycleFilterAndInput - dataConnectorTypeMapping: - - dataConnectorName: storage - dataConnectorObjectType: LifecycleFilterAnd - ---- -kind: TypePermissions -version: v1 -definition: - typeName: LifecycleFilterAnd - permissions: - - role: admin - output: - allowedFields: - - objectSizeGreaterThan - - objectSizeLessThan - - prefix - - tags - ---- -kind: ObjectType -version: v1 -definition: - name: LifecycleFilter - description: will be used in selecting rule(s) for lifecycle configuration - fields: - - name: and - type: LifecycleFilterAnd - - name: objectSizeGreaterThan - type: Int64 - - name: objectSizeLessThan - type: Int64 - - name: prefix - type: String - - name: tag - type: StorageTag - graphql: - typeName: LifecycleFilter - inputTypeName: LifecycleFilterInput - dataConnectorTypeMapping: - - dataConnectorName: storage - dataConnectorObjectType: LifecycleFilter - ---- -kind: TypePermissions -version: v1 -definition: - typeName: LifecycleFilter - permissions: - - role: admin - output: - allowedFields: - - and - - objectSizeGreaterThan - - objectSizeLessThan - - prefix - - tag - ---- -kind: ObjectType -version: v1 -definition: - name: LifecycleNoncurrentVersionExpiration - description: "- Specifies when noncurrent object versions expire. Upon - expiration, server permanently deletes the noncurrent object versions. Set - this lifecycle configuration action on a bucket that has versioning enabled - (or suspended) to request server delete noncurrent object versions at a - specific period in the object's lifetime." - fields: - - name: newerNoncurrentVersions - type: Int32 - - name: noncurrentDays - type: Int32 - graphql: - typeName: LifecycleNoncurrentVersionExpiration - inputTypeName: LifecycleNoncurrentVersionExpirationInput - dataConnectorTypeMapping: - - dataConnectorName: storage - dataConnectorObjectType: LifecycleNoncurrentVersionExpiration - ---- -kind: TypePermissions -version: v1 -definition: - typeName: LifecycleNoncurrentVersionExpiration - permissions: - - role: admin - output: - allowedFields: - - newerNoncurrentVersions - - noncurrentDays - ---- -kind: ObjectType -version: v1 -definition: - name: LifecycleNoncurrentVersionTransition - description: sets this action to request server to transition noncurrent object - versions to different set storage classes at a specific period in the - object's lifetime. - fields: - - name: newerNoncurrentVersions - type: Int32 - - name: noncurrentDays - type: Int32 - - name: storageClass - type: String - graphql: - typeName: LifecycleNoncurrentVersionTransition - inputTypeName: LifecycleNoncurrentVersionTransitionInput - dataConnectorTypeMapping: - - dataConnectorName: storage - dataConnectorObjectType: LifecycleNoncurrentVersionTransition - ---- -kind: TypePermissions -version: v1 -definition: - typeName: LifecycleNoncurrentVersionTransition - permissions: - - role: admin - output: - allowedFields: - - newerNoncurrentVersions - - noncurrentDays - - storageClass - ---- -kind: ObjectType -version: v1 -definition: - name: LifecycleTransition - description: transition details of lifecycle configuration - fields: - - name: date - type: Date - - name: days - type: Int32 - - name: storageClass - type: String - graphql: - typeName: LifecycleTransition - inputTypeName: LifecycleTransitionInput - dataConnectorTypeMapping: - - dataConnectorName: storage - dataConnectorObjectType: LifecycleTransition - ---- -kind: TypePermissions -version: v1 -definition: - typeName: LifecycleTransition - permissions: - - role: admin - output: - allowedFields: - - date - - days - - storageClass - ---- -kind: ObjectType -version: v1 -definition: - name: BucketLifecycleRule - description: represents a single rule in lifecycle configuration - fields: - - name: abortIncompleteMultipartUpload - type: AbortIncompleteMultipartUpload - - name: allVersionsExpiration - type: LifecycleAllVersionsExpiration - - name: delMarkerExpiration - type: LifecycleDelMarkerExpiration - - name: expiration - type: LifecycleExpiration - - name: filter - type: LifecycleFilter - - name: id - type: String! - - name: noncurrentVersionExpiration - type: LifecycleNoncurrentVersionExpiration - - name: noncurrentVersionTransition - type: LifecycleNoncurrentVersionTransition - - name: prefix - type: String - - name: status - type: String - - name: transition - type: LifecycleTransition - graphql: - typeName: BucketLifecycleRule - inputTypeName: BucketLifecycleRuleInput - dataConnectorTypeMapping: - - dataConnectorName: storage - dataConnectorObjectType: BucketLifecycleRule - ---- -kind: TypePermissions -version: v1 -definition: - typeName: BucketLifecycleRule - permissions: - - role: admin - output: - allowedFields: - - abortIncompleteMultipartUpload - - allVersionsExpiration - - delMarkerExpiration - - expiration - - filter - - id - - noncurrentVersionExpiration - - noncurrentVersionTransition - - prefix - - status - - transition - ---- -kind: ObjectType -version: v1 -definition: - name: BucketLifecycleConfiguration - description: is a collection of lifecycle Rule objects. - fields: - - name: rules - type: "[BucketLifecycleRule!]!" - graphql: - typeName: BucketLifecycleConfiguration - inputTypeName: BucketLifecycleConfigurationInput - dataConnectorTypeMapping: - - dataConnectorName: storage - dataConnectorObjectType: BucketLifecycleConfiguration - ---- -kind: TypePermissions -version: v1 -definition: - typeName: BucketLifecycleConfiguration - permissions: - - role: admin - output: - allowedFields: - - rules - ---- -kind: Command -version: v1 -definition: - name: StorageBucketLifecycle - outputType: BucketLifecycleConfiguration - arguments: - - name: bucket - type: String - - name: clientId - type: StorageClientId - source: - dataConnectorName: storage - dataConnectorCommand: - function: storageBucketLifecycle - graphql: - rootFieldName: storageBucketLifecycle - rootFieldKind: Query - description: gets lifecycle on a bucket or a prefix. - ---- -kind: CommandPermissions -version: v1 -definition: - commandName: StorageBucketLifecycle - permissions: - - role: admin - allowExecution: true - diff --git a/tests/engine/app/metadata/StorageBucketNotification.hml b/tests/engine/app/metadata/StorageBucketNotification.hml deleted file mode 100644 index d240158..0000000 --- a/tests/engine/app/metadata/StorageBucketNotification.hml +++ /dev/null @@ -1,266 +0,0 @@ ---- -kind: ObjectType -version: v1 -definition: - name: NotificationFilterRule - description: child of S3Key, a tag in the notification xml which carries - suffix/prefix filters - fields: - - name: name - type: String! - - name: value - type: String! - graphql: - typeName: NotificationFilterRule - inputTypeName: NotificationFilterRuleInput - dataConnectorTypeMapping: - - dataConnectorName: storage - dataConnectorObjectType: NotificationFilterRule - ---- -kind: TypePermissions -version: v1 -definition: - typeName: NotificationFilterRule - permissions: - - role: admin - output: - allowedFields: - - name - - value - ---- -kind: ObjectType -version: v1 -definition: - name: NotificationS3Key - description: child of Filter, a tag in the notification xml which carries - suffix/prefix filters - fields: - - name: filterRule - type: "[NotificationFilterRule!]" - graphql: - typeName: NotificationS3Key - inputTypeName: NotificationS3KeyInput - dataConnectorTypeMapping: - - dataConnectorName: storage - dataConnectorObjectType: NotificationS3Key - ---- -kind: TypePermissions -version: v1 -definition: - typeName: NotificationS3Key - permissions: - - role: admin - output: - allowedFields: - - filterRule - ---- -kind: ObjectType -version: v1 -definition: - name: NotificationFilter - description: "- a tag in the notification xml structure which carries - suffix/prefix filters" - fields: - - name: s3Key - type: NotificationS3Key - graphql: - typeName: NotificationFilter - inputTypeName: NotificationFilterInput - dataConnectorTypeMapping: - - dataConnectorName: storage - dataConnectorObjectType: NotificationFilter - ---- -kind: TypePermissions -version: v1 -definition: - typeName: NotificationFilter - permissions: - - role: admin - output: - allowedFields: - - s3Key - ---- -kind: ObjectType -version: v1 -definition: - name: NotificationLambdaConfig - description: carries one single cloudfunction notification configuration - fields: - - name: arn - type: String - - name: cloudFunction - type: String! - - name: event - type: "[String!]!" - - name: filter - type: NotificationFilter - - name: id - type: String - graphql: - typeName: NotificationLambdaConfig - inputTypeName: NotificationLambdaConfigInput - dataConnectorTypeMapping: - - dataConnectorName: storage - dataConnectorObjectType: NotificationLambdaConfig - ---- -kind: TypePermissions -version: v1 -definition: - typeName: NotificationLambdaConfig - permissions: - - role: admin - output: - allowedFields: - - arn - - cloudFunction - - event - - filter - - id - ---- -kind: ObjectType -version: v1 -definition: - name: NotificationQueueConfig - description: carries one single queue notification configuration - fields: - - name: arn - type: String - - name: event - type: "[String!]!" - - name: filter - type: NotificationFilter - - name: id - type: String - - name: queue - type: String! - graphql: - typeName: NotificationQueueConfig - inputTypeName: NotificationQueueConfigInput - dataConnectorTypeMapping: - - dataConnectorName: storage - dataConnectorObjectType: NotificationQueueConfig - ---- -kind: TypePermissions -version: v1 -definition: - typeName: NotificationQueueConfig - permissions: - - role: admin - output: - allowedFields: - - arn - - event - - filter - - id - - queue - ---- -kind: ObjectType -version: v1 -definition: - name: NotificationTopicConfig - description: carries one single topic notification configuration - fields: - - name: arn - type: String - - name: event - type: "[String!]!" - - name: filter - type: NotificationFilter - - name: id - type: String - - name: topic - type: String! - graphql: - typeName: NotificationTopicConfig - inputTypeName: NotificationTopicConfigInput - dataConnectorTypeMapping: - - dataConnectorName: storage - dataConnectorObjectType: NotificationTopicConfig - ---- -kind: TypePermissions -version: v1 -definition: - typeName: NotificationTopicConfig - permissions: - - role: admin - output: - allowedFields: - - arn - - event - - filter - - id - - topic - ---- -kind: ObjectType -version: v1 -definition: - name: NotificationConfig - description: the struct that represents a notification configration object. - fields: - - name: cloudFunctionConfigurations - type: "[NotificationLambdaConfig!]!" - - name: queueConfigurations - type: "[NotificationQueueConfig!]!" - - name: topicConfigurations - type: "[NotificationTopicConfig!]!" - graphql: - typeName: NotificationConfig - inputTypeName: NotificationConfigInput - dataConnectorTypeMapping: - - dataConnectorName: storage - dataConnectorObjectType: NotificationConfig - ---- -kind: TypePermissions -version: v1 -definition: - typeName: NotificationConfig - permissions: - - role: admin - output: - allowedFields: - - cloudFunctionConfigurations - - queueConfigurations - - topicConfigurations - ---- -kind: Command -version: v1 -definition: - name: StorageBucketNotification - outputType: NotificationConfig - arguments: - - name: bucket - type: String - - name: clientId - type: StorageClientId - source: - dataConnectorName: storage - dataConnectorCommand: - function: storageBucketNotification - graphql: - rootFieldName: storageBucketNotification - rootFieldKind: Query - description: gets notification configuration on a bucket. - ---- -kind: CommandPermissions -version: v1 -definition: - commandName: StorageBucketNotification - permissions: - - role: admin - allowExecution: true - diff --git a/tests/engine/app/metadata/StorageBucketPolicy.hml b/tests/engine/app/metadata/StorageBucketPolicy.hml deleted file mode 100644 index 8ba5ca5..0000000 --- a/tests/engine/app/metadata/StorageBucketPolicy.hml +++ /dev/null @@ -1,29 +0,0 @@ ---- -kind: Command -version: v1 -definition: - name: StorageBucketPolicy - outputType: String! - arguments: - - name: bucket - type: String - - name: clientId - type: StorageClientId - source: - dataConnectorName: storage - dataConnectorCommand: - function: storageBucketPolicy - graphql: - rootFieldName: storageBucketPolicy - rootFieldKind: Query - description: gets access permissions on a bucket or a prefix. - ---- -kind: CommandPermissions -version: v1 -definition: - commandName: StorageBucketPolicy - permissions: - - role: admin - allowExecution: true - diff --git a/tests/engine/app/metadata/StorageBucketReplication.hml b/tests/engine/app/metadata/StorageBucketReplication.hml deleted file mode 100644 index dcad2b4..0000000 --- a/tests/engine/app/metadata/StorageBucketReplication.hml +++ /dev/null @@ -1,339 +0,0 @@ ---- -kind: ObjectType -version: v1 -definition: - name: DeleteMarkerReplication - description: whether delete markers are replicated - - https://docs.aws.amazon.com/AmazonS3/latest/dev/replication-add-config.html - fields: - - name: status - type: StorageReplicationRuleStatus! - graphql: - typeName: DeleteMarkerReplication - inputTypeName: DeleteMarkerReplicationInput - dataConnectorTypeMapping: - - dataConnectorName: storage - dataConnectorObjectType: DeleteMarkerReplication - ---- -kind: TypePermissions -version: v1 -definition: - typeName: DeleteMarkerReplication - permissions: - - role: admin - output: - allowedFields: - - status - ---- -kind: ObjectType -version: v1 -definition: - name: DeleteReplication - description: whether versioned deletes are replicated. This is a MinIO specific extension - fields: - - name: status - type: StorageReplicationRuleStatus! - graphql: - typeName: DeleteReplication - inputTypeName: DeleteReplicationInput - dataConnectorTypeMapping: - - dataConnectorName: storage - dataConnectorObjectType: DeleteReplication - ---- -kind: TypePermissions -version: v1 -definition: - typeName: DeleteReplication - permissions: - - role: admin - output: - allowedFields: - - status - ---- -kind: ObjectType -version: v1 -definition: - name: StorageReplicationDestination - fields: - - name: bucket - type: String! - - name: storageClass - type: String - graphql: - typeName: StorageReplicationDestination - inputTypeName: StorageReplicationDestinationInput - dataConnectorTypeMapping: - - dataConnectorName: storage - dataConnectorObjectType: StorageReplicationDestination - ---- -kind: TypePermissions -version: v1 -definition: - typeName: StorageReplicationDestination - permissions: - - role: admin - output: - allowedFields: - - bucket - - storageClass - ---- -kind: ObjectType -version: v1 -definition: - name: ExistingObjectReplication - description: whether existing object replication is enabled - fields: - - name: status - type: StorageReplicationRuleStatus! - graphql: - typeName: ExistingObjectReplication - inputTypeName: ExistingObjectReplicationInput - dataConnectorTypeMapping: - - dataConnectorName: storage - dataConnectorObjectType: ExistingObjectReplication - ---- -kind: TypePermissions -version: v1 -definition: - typeName: ExistingObjectReplication - permissions: - - role: admin - output: - allowedFields: - - status - ---- -kind: ObjectType -version: v1 -definition: - name: StorageReplicationFilterAnd - description: "- a tag to combine a prefix and multiple tags for replication - configuration rule." - fields: - - name: rrefix - type: String - - name: tag - type: "[StorageTag!]" - graphql: - typeName: StorageReplicationFilterAnd - inputTypeName: StorageReplicationFilterAndInput - dataConnectorTypeMapping: - - dataConnectorName: storage - dataConnectorObjectType: StorageReplicationFilterAnd - ---- -kind: TypePermissions -version: v1 -definition: - typeName: StorageReplicationFilterAnd - permissions: - - role: admin - output: - allowedFields: - - rrefix - - tag - ---- -kind: ObjectType -version: v1 -definition: - name: StorageReplicationFilter - description: a filter for a replication configuration Rule. - fields: - - name: and - type: StorageReplicationFilterAnd - - name: rrefix - type: String - - name: tag - type: StorageTag - graphql: - typeName: StorageReplicationFilter - inputTypeName: StorageReplicationFilterInput - dataConnectorTypeMapping: - - dataConnectorName: storage - dataConnectorObjectType: StorageReplicationFilter - ---- -kind: TypePermissions -version: v1 -definition: - typeName: StorageReplicationFilter - permissions: - - role: admin - output: - allowedFields: - - and - - rrefix - - tag - ---- -kind: ObjectType -version: v1 -definition: - name: ReplicaModifications - description: specifies if replica modification sync is enabled - fields: - - name: status - type: StorageReplicationRuleStatus! - graphql: - typeName: ReplicaModifications - inputTypeName: ReplicaModificationsInput - dataConnectorTypeMapping: - - dataConnectorName: storage - dataConnectorObjectType: ReplicaModifications - ---- -kind: TypePermissions -version: v1 -definition: - typeName: ReplicaModifications - permissions: - - role: admin - output: - allowedFields: - - status - ---- -kind: ObjectType -version: v1 -definition: - name: SourceSelectionCriteria - description: specifies additional source selection criteria in ReplicationConfiguration. - fields: - - name: replicaModifications - type: ReplicaModifications - graphql: - typeName: SourceSelectionCriteria - inputTypeName: SourceSelectionCriteriaInput - dataConnectorTypeMapping: - - dataConnectorName: storage - dataConnectorObjectType: SourceSelectionCriteria - ---- -kind: TypePermissions -version: v1 -definition: - typeName: SourceSelectionCriteria - permissions: - - role: admin - output: - allowedFields: - - replicaModifications - ---- -kind: ObjectType -version: v1 -definition: - name: StorageReplicationRule - description: a rule for replication configuration. - fields: - - name: deleteMarkerReplication - type: DeleteMarkerReplication - - name: deleteReplication - type: DeleteReplication - - name: destination - type: StorageReplicationDestination - - name: existingObjectReplication - type: ExistingObjectReplication - - name: filter - type: StorageReplicationFilter! - - name: id - type: String - - name: priority - type: Int32! - - name: sourceSelectionCriteria - type: SourceSelectionCriteria - - name: status - type: StorageReplicationRuleStatus! - graphql: - typeName: StorageReplicationRule - inputTypeName: StorageReplicationRuleInput - dataConnectorTypeMapping: - - dataConnectorName: storage - dataConnectorObjectType: StorageReplicationRule - ---- -kind: TypePermissions -version: v1 -definition: - typeName: StorageReplicationRule - permissions: - - role: admin - output: - allowedFields: - - deleteMarkerReplication - - deleteReplication - - destination - - existingObjectReplication - - filter - - id - - priority - - sourceSelectionCriteria - - status - ---- -kind: ObjectType -version: v1 -definition: - name: StorageReplicationConfig - description: replication configuration specified in - https://docs.aws.amazon.com/AmazonS3/latest/dev/replication-add-config.html - fields: - - name: role - type: String - - name: rules - type: "[StorageReplicationRule!]!" - graphql: - typeName: StorageReplicationConfig - inputTypeName: StorageReplicationConfigInput - dataConnectorTypeMapping: - - dataConnectorName: storage - dataConnectorObjectType: StorageReplicationConfig - ---- -kind: TypePermissions -version: v1 -definition: - typeName: StorageReplicationConfig - permissions: - - role: admin - output: - allowedFields: - - role - - rules - ---- -kind: Command -version: v1 -definition: - name: StorageBucketReplication - outputType: StorageReplicationConfig - arguments: - - name: bucket - type: String - - name: clientId - type: StorageClientId - source: - dataConnectorName: storage - dataConnectorCommand: - function: storageBucketReplication - graphql: - rootFieldName: storageBucketReplication - rootFieldKind: Query - ---- -kind: CommandPermissions -version: v1 -definition: - commandName: StorageBucketReplication - permissions: - - role: admin - allowExecution: true - diff --git a/tests/engine/app/metadata/StorageBucketVersioning.hml b/tests/engine/app/metadata/StorageBucketVersioning.hml deleted file mode 100644 index df60701..0000000 --- a/tests/engine/app/metadata/StorageBucketVersioning.hml +++ /dev/null @@ -1,65 +0,0 @@ ---- -kind: ObjectType -version: v1 -definition: - name: StorageBucketVersioningConfiguration - description: is the versioning configuration structure - fields: - - name: excludeFolders - type: Boolean - - name: excludedPrefixes - type: "[String!]" - - name: mfaDelete - type: String - - name: status - type: String - graphql: - typeName: StorageBucketVersioningConfiguration - inputTypeName: StorageBucketVersioningConfigurationInput - dataConnectorTypeMapping: - - dataConnectorName: storage - dataConnectorObjectType: StorageBucketVersioningConfiguration - ---- -kind: TypePermissions -version: v1 -definition: - typeName: StorageBucketVersioningConfiguration - permissions: - - role: admin - output: - allowedFields: - - excludeFolders - - excludedPrefixes - - mfaDelete - - status - ---- -kind: Command -version: v1 -definition: - name: StorageBucketVersioning - outputType: StorageBucketVersioningConfiguration - arguments: - - name: bucket - type: String - - name: clientId - type: StorageClientId - source: - dataConnectorName: storage - dataConnectorCommand: - function: storageBucketVersioning - graphql: - rootFieldName: storageBucketVersioning - rootFieldKind: Query - description: gets versioning configuration set on a bucket. - ---- -kind: CommandPermissions -version: v1 -definition: - commandName: StorageBucketVersioning - permissions: - - role: admin - allowExecution: true - diff --git a/tests/engine/app/metadata/StorageBuckets.hml b/tests/engine/app/metadata/StorageBuckets.hml index 43e4a83..6cc92fc 100644 --- a/tests/engine/app/metadata/StorageBuckets.hml +++ b/tests/engine/app/metadata/StorageBuckets.hml @@ -2,44 +2,125 @@ kind: ObjectType version: v1 definition: - name: StorageBucketInfo - description: container for bucket metadata. + name: StorageBucketFilter fields: - - name: creationDate - type: TimestampTz! - - name: name - type: String! - - name: tags - type: Json + - name: bucket + type: StorageStringFilter! + - name: clientId + type: StorageClientId! graphql: - typeName: StorageBucketInfo - inputTypeName: StorageBucketInfoInput + typeName: StorageBucketFilter + inputTypeName: StorageBucketFilterInput dataConnectorTypeMapping: - dataConnectorName: storage - dataConnectorObjectType: StorageBucketInfo + dataConnectorObjectType: StorageBucketFilter --- kind: TypePermissions version: v1 definition: - typeName: StorageBucketInfo + typeName: StorageBucketFilter permissions: - role: admin output: allowedFields: - - creationDate - - name - - tags + - bucket + - clientId + +--- +kind: BooleanExpressionType +version: v1 +definition: + name: StorageBucketFilterBoolExp + operand: + object: + type: StorageBucketFilter + comparableFields: + - fieldName: bucket + booleanExpressionType: StorageStringFilterBoolExp + - fieldName: clientId + booleanExpressionType: StorageClientIdBoolExp + comparableRelationships: [] + logicalOperators: + enable: true + isNull: + enable: true + graphql: + typeName: StorageBucketFilterBoolExp + +--- +kind: ObjectType +version: v1 +definition: + name: StoragePaginationInfo + description: holds the pagination information. + fields: + - name: cursor + type: String + - name: hasNextPage + type: Boolean! + graphql: + typeName: StoragePaginationInfo + inputTypeName: StoragePaginationInfoInput + dataConnectorTypeMapping: + - dataConnectorName: storage + dataConnectorObjectType: StoragePaginationInfo + +--- +kind: TypePermissions +version: v1 +definition: + typeName: StoragePaginationInfo + permissions: + - role: admin + output: + allowedFields: + - cursor + - hasNextPage + +--- +kind: ObjectType +version: v1 +definition: + name: StorageBucketListResults + description: hold the paginated results of the storage bucket list. + fields: + - name: buckets + type: "[StorageBucket!]!" + - name: pageInfo + type: StoragePaginationInfo! + graphql: + typeName: StorageBucketListResults + inputTypeName: StorageBucketListResultsInput + dataConnectorTypeMapping: + - dataConnectorName: storage + dataConnectorObjectType: StorageBucketListResults + +--- +kind: TypePermissions +version: v1 +definition: + typeName: StorageBucketListResults + permissions: + - role: admin + output: + allowedFields: + - buckets + - pageInfo --- kind: Command version: v1 definition: name: StorageBuckets - outputType: "[StorageBucketInfo!]!" + outputType: StorageBucketListResults! arguments: - - name: clientId - type: StorageClientId + - name: maxResults + type: Int32 + - name: startAfter + type: String + - name: where + type: StorageBucketFilterBoolExp source: dataConnectorName: storage dataConnectorCommand: diff --git a/tests/engine/app/metadata/StorageDeletedObjects.hml b/tests/engine/app/metadata/StorageDeletedObjects.hml new file mode 100644 index 0000000..7f46a81 --- /dev/null +++ b/tests/engine/app/metadata/StorageDeletedObjects.hml @@ -0,0 +1,435 @@ +--- +kind: ObjectType +version: v1 +definition: + name: StorageObjectCopyInfo + description: holds the copy information if the object was copied from another object. + fields: + - name: completionTime + type: TimestampTz + - name: id + type: String! + - name: progress + type: String + - name: source + type: String + - name: status + type: String + - name: statusDescription + type: String + graphql: + typeName: StorageObjectCopyInfo + inputTypeName: StorageObjectCopyInfoInput + dataConnectorTypeMapping: + - dataConnectorName: storage + dataConnectorObjectType: StorageObjectCopyInfo + +--- +kind: TypePermissions +version: v1 +definition: + typeName: StorageObjectCopyInfo + permissions: + - role: admin + output: + allowedFields: + - completionTime + - id + - progress + - source + - status + - statusDescription + +--- +kind: ObjectType +version: v1 +definition: + name: StorageGrantee + description: represents the person being granted permissions. + fields: + - name: displayName + type: String + - name: id + type: String + - name: uri + type: String + graphql: + typeName: StorageGrantee + inputTypeName: StorageGranteeInput + dataConnectorTypeMapping: + - dataConnectorName: storage + dataConnectorObjectType: StorageGrantee + +--- +kind: TypePermissions +version: v1 +definition: + typeName: StorageGrantee + permissions: + - role: admin + output: + allowedFields: + - displayName + - id + - uri + +--- +kind: ObjectType +version: v1 +definition: + name: StorageGrant + description: holds grant information. + fields: + - name: grantee + type: StorageGrantee + - name: permission + type: String + graphql: + typeName: StorageGrant + inputTypeName: StorageGrantInput + dataConnectorTypeMapping: + - dataConnectorName: storage + dataConnectorObjectType: StorageGrant + +--- +kind: TypePermissions +version: v1 +definition: + typeName: StorageGrant + permissions: + - role: admin + output: + allowedFields: + - grantee + - permission + +--- +kind: ObjectType +version: v1 +definition: + name: StorageOwner + description: name. + fields: + - name: id + type: String + - name: name + type: String + graphql: + typeName: StorageOwner + inputTypeName: StorageOwnerInput + dataConnectorTypeMapping: + - dataConnectorName: storage + dataConnectorObjectType: StorageOwner + +--- +kind: TypePermissions +version: v1 +definition: + typeName: StorageOwner + permissions: + - role: admin + output: + allowedFields: + - id + - name + +--- +kind: ObjectType +version: v1 +definition: + name: StorageRestoreInfo + description: contains information of the restore operation of an archived object. + fields: + - name: expiryTime + type: TimestampTz + - name: ongoingRestore + type: Boolean! + graphql: + typeName: StorageRestoreInfo + inputTypeName: StorageRestoreInfoInput + dataConnectorTypeMapping: + - dataConnectorName: storage + dataConnectorObjectType: StorageRestoreInfo + +--- +kind: TypePermissions +version: v1 +definition: + typeName: StorageRestoreInfo + permissions: + - role: admin + output: + allowedFields: + - expiryTime + - ongoingRestore + +--- +kind: ObjectType +version: v1 +definition: + name: StorageObject + description: container for object metadata. + fields: + - name: accessTierChangeTime + type: TimestampTz + - name: accessTierInferred + type: Boolean + - name: acl + type: Json + - name: archiveStatus + type: String + - name: blobSequenceNumber + type: Int64 + - name: blobType + type: String + - name: bucket + type: String! + - name: cacheControl + type: String + - name: checksumCrc32 + type: String + - name: checksumCrc32C + type: String + - name: checksumCrc64Nvme + type: String + - name: checksumSha1 + type: String + - name: checksumSha256 + type: String + - name: clientId + type: String! + - name: contentDisposition + type: String + - name: contentEncoding + type: String + - name: contentLanguage + type: String + - name: contentMd5 + type: String + - name: contentType + type: String + - name: copy + type: StorageObjectCopyInfo + - name: creationTime + type: TimestampTz + - name: customerProvidedKeySha256 + type: String + - name: deleted + type: Boolean + - name: deletedTime + type: TimestampTz + - name: destinationSnapshot + type: String + - name: etag + type: String + - name: expiration + type: TimestampTz + - name: expirationRuleId + type: String + - name: expires + type: TimestampTz + - name: grant + type: "[StorageGrant!]" + - name: group + type: String + - name: incrementalCopy + type: Boolean + - name: isLatest + type: Boolean + - name: kmsKeyName + type: String + - name: lastAccessTime + type: TimestampTz + - name: lastModified + type: TimestampTz! + - name: leaseDuration + type: String + - name: leaseState + type: String + - name: leaseStatus + type: String + - name: legalHold + type: Boolean + - name: mediaLink + type: String + - name: metadata + type: Json + - name: name + type: String! + - name: owner + type: StorageOwner + - name: permissions + type: String + - name: rawMetadata + type: Json + - name: rehydratePriority + type: String + - name: remainingRetentionDays + type: Int32 + - name: replicationReady + type: Boolean + - name: replicationStatus + type: StorageObjectReplicationStatus + - name: resourceType + type: String + - name: restore + type: StorageRestoreInfo + - name: retentionMode + type: String + - name: retentionUntilDate + type: TimestampTz + - name: sealed + type: Boolean + - name: serverEncrypted + type: Boolean + - name: size + type: Int64 + - name: storageClass + type: String + - name: tagCount + type: Int32 + - name: tags + type: Json + - name: versionId + type: String + graphql: + typeName: StorageObject + inputTypeName: StorageObjectInput + dataConnectorTypeMapping: + - dataConnectorName: storage + dataConnectorObjectType: StorageObject + +--- +kind: TypePermissions +version: v1 +definition: + typeName: StorageObject + permissions: + - role: admin + output: + allowedFields: + - accessTierChangeTime + - accessTierInferred + - acl + - archiveStatus + - blobSequenceNumber + - blobType + - bucket + - cacheControl + - checksumCrc32 + - checksumCrc32C + - checksumCrc64Nvme + - checksumSha1 + - checksumSha256 + - clientId + - contentDisposition + - contentEncoding + - contentLanguage + - contentMd5 + - contentType + - copy + - creationTime + - customerProvidedKeySha256 + - deleted + - deletedTime + - destinationSnapshot + - etag + - expiration + - expirationRuleId + - expires + - grant + - group + - incrementalCopy + - isLatest + - kmsKeyName + - lastAccessTime + - lastModified + - leaseDuration + - leaseState + - leaseStatus + - legalHold + - mediaLink + - metadata + - name + - owner + - permissions + - rawMetadata + - rehydratePriority + - remainingRetentionDays + - replicationReady + - replicationStatus + - resourceType + - restore + - retentionMode + - retentionUntilDate + - sealed + - serverEncrypted + - size + - storageClass + - tagCount + - tags + - versionId + +--- +kind: ObjectType +version: v1 +definition: + name: StorageObjectListResults + description: hold the paginated results of the storage object list. + fields: + - name: objects + type: "[StorageObject!]!" + - name: pageInfo + type: StoragePaginationInfo! + graphql: + typeName: StorageObjectListResults + inputTypeName: StorageObjectListResultsInput + dataConnectorTypeMapping: + - dataConnectorName: storage + dataConnectorObjectType: StorageObjectListResults + +--- +kind: TypePermissions +version: v1 +definition: + typeName: StorageObjectListResults + permissions: + - role: admin + output: + allowedFields: + - objects + - pageInfo + +--- +kind: Command +version: v1 +definition: + name: StorageDeletedObjects + outputType: StorageObjectListResults! + arguments: + - name: maxResults + type: Int32 + - name: recursive + type: Boolean + - name: startAfter + type: String + - name: where + type: StorageObjectFilterBoolExp + source: + dataConnectorName: storage + dataConnectorCommand: + function: storageDeletedObjects + graphql: + rootFieldName: storageDeletedObjects + rootFieldKind: Query + description: list deleted objects in a bucket. + +--- +kind: CommandPermissions +version: v1 +definition: + commandName: StorageDeletedObjects + permissions: + - role: admin + allowExecution: true + diff --git a/tests/engine/app/metadata/StorageObject.hml b/tests/engine/app/metadata/StorageObject.hml index bef61ec..d7f72e5 100644 --- a/tests/engine/app/metadata/StorageObject.hml +++ b/tests/engine/app/metadata/StorageObject.hml @@ -1,345 +1,3 @@ ---- -kind: ObjectType -version: v1 -definition: - name: StorageGrantee - description: represents the person being granted permissions. - fields: - - name: displayName - type: String - - name: id - type: String - - name: uri - type: String - graphql: - typeName: StorageGrantee - inputTypeName: StorageGranteeInput - dataConnectorTypeMapping: - - dataConnectorName: storage - dataConnectorObjectType: StorageGrantee - ---- -kind: TypePermissions -version: v1 -definition: - typeName: StorageGrantee - permissions: - - role: admin - output: - allowedFields: - - displayName - - id - - uri - ---- -kind: ObjectType -version: v1 -definition: - name: StorageGrant - description: holds grant information. - fields: - - name: grantee - type: StorageGrantee - - name: permission - type: String - graphql: - typeName: StorageGrant - inputTypeName: StorageGrantInput - dataConnectorTypeMapping: - - dataConnectorName: storage - dataConnectorObjectType: StorageGrant - ---- -kind: TypePermissions -version: v1 -definition: - typeName: StorageGrant - permissions: - - role: admin - output: - allowedFields: - - grantee - - permission - ---- -kind: ObjectType -version: v1 -definition: - name: StorageOwner - description: name. - fields: - - name: id - type: String - - name: name - type: String - graphql: - typeName: StorageOwner - inputTypeName: StorageOwnerInput - dataConnectorTypeMapping: - - dataConnectorName: storage - dataConnectorObjectType: StorageOwner - ---- -kind: TypePermissions -version: v1 -definition: - typeName: StorageOwner - permissions: - - role: admin - output: - allowedFields: - - id - - name - ---- -kind: ObjectType -version: v1 -definition: - name: StorageRestoreInfo - description: contains information of the restore operation of an archived object. - fields: - - name: expiryTime - type: TimestampTz - - name: ongoingRestore - type: Boolean! - graphql: - typeName: StorageRestoreInfo - inputTypeName: StorageRestoreInfoInput - dataConnectorTypeMapping: - - dataConnectorName: storage - dataConnectorObjectType: StorageRestoreInfo - ---- -kind: TypePermissions -version: v1 -definition: - typeName: StorageRestoreInfo - permissions: - - role: admin - output: - allowedFields: - - expiryTime - - ongoingRestore - ---- -kind: ObjectType -version: v1 -definition: - name: StorageObject - description: container for object metadata. - fields: - - name: accessTierChangeTime - type: TimestampTz - - name: accessTierInferred - type: Boolean - - name: acl - type: String - - name: archiveStatus - type: String - - name: blobSequenceNumber - type: Int64 - - name: blobType - type: String - - name: bucket - type: String! - - name: cacheControl - type: String - - name: checksumCrc32 - type: String - - name: checksumCrc32C - type: String - - name: checksumCrc64Nvme - type: String - - name: checksumSha1 - type: String - - name: checksumSha256 - type: String - - name: clientId - type: String! - - name: contentDisposition - type: String - - name: contentEncoding - type: String - - name: contentLanguage - type: String - - name: contentMd5 - type: String - - name: contentType - type: String - - name: copyCompletionTime - type: TimestampTz - - name: copyId - type: String - - name: copyProgress - type: String - - name: copySource - type: String - - name: copyStatus - type: String - - name: copyStatusDescription - type: String - - name: creationTime - type: TimestampTz - - name: customerProvidedKeySha256 - type: String - - name: deleted - type: Boolean - - name: deletedTime - type: TimestampTz - - name: destinationSnapshot - type: String - - name: encryptionScope - type: String - - name: etag - type: String - - name: expiration - type: TimestampTz - - name: expirationRuleId - type: String - - name: expires - type: TimestampTz - - name: grant - type: "[StorageGrant!]" - - name: group - type: String - - name: immutabilityPolicyMode - type: String - - name: immutabilityPolicyUntilDate - type: TimestampTz - - name: incrementalCopy - type: Boolean - - name: isLatest - type: Boolean - - name: lastAccessTime - type: TimestampTz - - name: lastModified - type: TimestampTz! - - name: leaseDuration - type: String - - name: leaseState - type: String - - name: leaseStatus - type: String - - name: legalHold - type: Boolean - - name: metadata - type: Json - - name: name - type: String! - - name: owner - type: StorageOwner - - name: permissions - type: String - - name: rehydratePriority - type: String - - name: remainingRetentionDays - type: Int32 - - name: replicationReady - type: Boolean - - name: replicationStatus - type: StorageObjectReplicationStatus - - name: resourceType - type: String - - name: restore - type: StorageRestoreInfo - - name: sealed - type: Boolean - - name: serverEncrypted - type: Boolean - - name: size - type: Int64 - - name: storageClass - type: String - - name: userMetadata - type: Json - - name: userTagCount - type: Int32 - - name: userTags - type: Json - - name: versionId - type: String - graphql: - typeName: StorageObject - inputTypeName: StorageObjectInput - dataConnectorTypeMapping: - - dataConnectorName: storage - dataConnectorObjectType: StorageObject - ---- -kind: TypePermissions -version: v1 -definition: - typeName: StorageObject - permissions: - - role: admin - output: - allowedFields: - - accessTierChangeTime - - accessTierInferred - - acl - - archiveStatus - - blobSequenceNumber - - blobType - - bucket - - cacheControl - - checksumCrc32 - - checksumCrc32C - - checksumCrc64Nvme - - checksumSha1 - - checksumSha256 - - clientId - - contentDisposition - - contentEncoding - - contentLanguage - - contentMd5 - - contentType - - copyCompletionTime - - copyId - - copyProgress - - copySource - - copyStatus - - copyStatusDescription - - creationTime - - customerProvidedKeySha256 - - deleted - - deletedTime - - destinationSnapshot - - encryptionScope - - etag - - expiration - - expirationRuleId - - expires - - grant - - group - - immutabilityPolicyMode - - immutabilityPolicyUntilDate - - incrementalCopy - - isLatest - - lastAccessTime - - lastModified - - leaseDuration - - leaseState - - leaseStatus - - legalHold - - metadata - - name - - owner - - permissions - - rehydratePriority - - remainingRetentionDays - - replicationReady - - replicationStatus - - resourceType - - restore - - sealed - - serverEncrypted - - size - - storageClass - - userMetadata - - userTagCount - - userTags - - versionId - --- kind: Command version: v1 @@ -362,7 +20,7 @@ definition: - name: versionId type: String - name: where - type: StorageObjectSimpleBoolExp + type: StorageObjectFilterBoolExp source: dataConnectorName: storage dataConnectorCommand: diff --git a/tests/engine/app/metadata/StorageObjects.hml b/tests/engine/app/metadata/StorageObjects.hml index de4c28d..b48cd2c 100644 --- a/tests/engine/app/metadata/StorageObjects.hml +++ b/tests/engine/app/metadata/StorageObjects.hml @@ -1,66 +1,3 @@ ---- -kind: ObjectType -version: v1 -definition: - name: StorageObjectPaginationInfo - description: holds the pagination information. - fields: - - name: cursor - type: String - - name: hasNextPage - type: Boolean! - - name: nextCursor - type: String - graphql: - typeName: StorageObjectPaginationInfo - inputTypeName: StorageObjectPaginationInfoInput - dataConnectorTypeMapping: - - dataConnectorName: storage - dataConnectorObjectType: StorageObjectPaginationInfo - ---- -kind: TypePermissions -version: v1 -definition: - typeName: StorageObjectPaginationInfo - permissions: - - role: admin - output: - allowedFields: - - cursor - - hasNextPage - - nextCursor - ---- -kind: ObjectType -version: v1 -definition: - name: StorageObjectListResults - description: hold the paginated results of the storage object list. - fields: - - name: objects - type: "[StorageObject!]!" - - name: pageInfo - type: StorageObjectPaginationInfo! - graphql: - typeName: StorageObjectListResults - inputTypeName: StorageObjectListResultsInput - dataConnectorTypeMapping: - - dataConnectorName: storage - dataConnectorObjectType: StorageObjectListResults - ---- -kind: TypePermissions -version: v1 -definition: - typeName: StorageObjectListResults - permissions: - - role: admin - output: - allowedFields: - - objects - - pageInfo - --- kind: Command version: v1 @@ -75,7 +12,7 @@ definition: - name: startAfter type: String - name: where - type: StorageObjectSimpleBoolExp + type: StorageObjectFilterBoolExp source: dataConnectorName: storage dataConnectorCommand: @@ -93,3 +30,4 @@ definition: permissions: - role: admin allowExecution: true + diff --git a/tests/engine/app/metadata/StoragePresignedDownloadUrl.hml b/tests/engine/app/metadata/StoragePresignedDownloadUrl.hml index d362bc0..f28ee3f 100644 --- a/tests/engine/app/metadata/StoragePresignedDownloadUrl.hml +++ b/tests/engine/app/metadata/StoragePresignedDownloadUrl.hml @@ -46,7 +46,7 @@ definition: - name: requestParams type: Json - name: where - type: StorageObjectSimpleBoolExp + type: StorageObjectFilterBoolExp source: dataConnectorName: storage dataConnectorCommand: diff --git a/tests/engine/app/metadata/StoragePresignedUploadUrl.hml b/tests/engine/app/metadata/StoragePresignedUploadUrl.hml index 38a240c..69949a0 100644 --- a/tests/engine/app/metadata/StoragePresignedUploadUrl.hml +++ b/tests/engine/app/metadata/StoragePresignedUploadUrl.hml @@ -14,7 +14,7 @@ definition: - name: object type: String! - name: where - type: StorageObjectSimpleBoolExp + type: StorageObjectFilterBoolExp source: dataConnectorName: storage dataConnectorCommand: diff --git a/tests/engine/app/metadata/SuspendStorageBucketVersioning.hml b/tests/engine/app/metadata/SuspendStorageBucketVersioning.hml deleted file mode 100644 index 459ef51..0000000 --- a/tests/engine/app/metadata/SuspendStorageBucketVersioning.hml +++ /dev/null @@ -1,29 +0,0 @@ ---- -kind: Command -version: v1 -definition: - name: SuspendStorageBucketVersioning - outputType: Boolean! - arguments: - - name: bucket - type: String - - name: clientId - type: StorageClientId - source: - dataConnectorName: storage - dataConnectorCommand: - procedure: suspendStorageBucketVersioning - graphql: - rootFieldName: suspendStorageBucketVersioning - rootFieldKind: Mutation - description: disables bucket versioning support. - ---- -kind: CommandPermissions -version: v1 -definition: - commandName: SuspendStorageBucketVersioning - permissions: - - role: admin - allowExecution: true - diff --git a/tests/engine/app/metadata/StorageObjectLockConfig.hml b/tests/engine/app/metadata/UpdateStorageBucket.hml similarity index 50% rename from tests/engine/app/metadata/StorageObjectLockConfig.hml rename to tests/engine/app/metadata/UpdateStorageBucket.hml index 3cc53cb..c09b47a 100644 --- a/tests/engine/app/metadata/StorageObjectLockConfig.hml +++ b/tests/engine/app/metadata/UpdateStorageBucket.hml @@ -2,34 +2,32 @@ kind: ObjectType version: v1 definition: - name: StorageObjectLockConfig + name: SetStorageObjectLockConfig + description: represents the object lock configuration options in given bucket fields: - name: mode type: StorageRetentionMode - - name: objectLock - type: String! - name: unit type: StorageRetentionValidityUnit - name: validity type: Int32 graphql: - typeName: StorageObjectLockConfig - inputTypeName: StorageObjectLockConfigInput + typeName: SetStorageObjectLockConfig + inputTypeName: SetStorageObjectLockConfigInput dataConnectorTypeMapping: - dataConnectorName: storage - dataConnectorObjectType: StorageObjectLockConfig + dataConnectorObjectType: SetStorageObjectLockConfig --- kind: TypePermissions version: v1 definition: - typeName: StorageObjectLockConfig + typeName: SetStorageObjectLockConfig permissions: - role: admin output: allowedFields: - mode - - objectLock - unit - validity @@ -37,27 +35,37 @@ definition: kind: Command version: v1 definition: - name: StorageObjectLockConfig - outputType: StorageObjectLockConfig + name: UpdateStorageBucket + outputType: Boolean! arguments: - name: bucket type: String - name: clientId type: StorageClientId + - name: encryption + type: ServerSideEncryptionConfiguration + - name: lifecycle + type: ObjectLifecycleConfiguration + - name: objectLock + type: SetStorageObjectLockConfig + - name: tags + type: Json + - name: versioningEnabled + type: Boolean source: dataConnectorName: storage dataConnectorCommand: - function: storageObjectLockConfig + procedure: updateStorageBucket graphql: - rootFieldName: storageObjectLockConfig - rootFieldKind: Query - description: gets object lock configuration of given bucket. + rootFieldName: updateStorageBucket + rootFieldKind: Mutation + description: updates the bucket's configuration. --- kind: CommandPermissions version: v1 definition: - commandName: StorageObjectLockConfig + commandName: UpdateStorageBucket permissions: - role: admin allowExecution: true diff --git a/tests/engine/app/metadata/UpdateStorageObject.hml b/tests/engine/app/metadata/UpdateStorageObject.hml new file mode 100644 index 0000000..472eb4d --- /dev/null +++ b/tests/engine/app/metadata/UpdateStorageObject.hml @@ -0,0 +1,76 @@ +--- +kind: ObjectType +version: v1 +definition: + name: SetStorageObjectRetentionOptions + description: represents options specified by user for PutObject call. + fields: + - name: governanceBypass + type: Boolean + - name: mode + type: StorageRetentionMode + - name: retainUntilDate + type: TimestampTz + graphql: + typeName: SetStorageObjectRetentionOptions + inputTypeName: SetStorageObjectRetentionOptionsInput + dataConnectorTypeMapping: + - dataConnectorName: storage + dataConnectorObjectType: SetStorageObjectRetentionOptions + +--- +kind: TypePermissions +version: v1 +definition: + typeName: SetStorageObjectRetentionOptions + permissions: + - role: admin + output: + allowedFields: + - governanceBypass + - mode + - retainUntilDate + +--- +kind: Command +version: v1 +definition: + name: UpdateStorageObject + outputType: Boolean! + arguments: + - name: bucket + type: String + - name: clientId + type: StorageClientId + - name: legalHold + type: Boolean + - name: metadata + type: Json + - name: object + type: String! + - name: retention + type: SetStorageObjectRetentionOptions + - name: tags + type: Json + - name: versionId + type: String + - name: where + type: StorageObjectFilterBoolExp + source: + dataConnectorName: storage + dataConnectorCommand: + procedure: updateStorageObject + graphql: + rootFieldName: updateStorageObject + rootFieldKind: Mutation + description: updates the object's configuration. + +--- +kind: CommandPermissions +version: v1 +definition: + commandName: UpdateStorageObject + permissions: + - role: admin + allowExecution: true + diff --git a/tests/engine/app/metadata/UploadStorageObject.hml b/tests/engine/app/metadata/UploadStorageObject.hml index 5fe0d09..1328495 100644 --- a/tests/engine/app/metadata/UploadStorageObject.hml +++ b/tests/engine/app/metadata/UploadStorageObject.hml @@ -1,3 +1,36 @@ +--- +kind: ObjectType +version: v1 +definition: + name: PutStorageObjectRetentionOptions + description: represent options of object retention configuration. + fields: + - name: governanceBypass + type: Boolean + - name: mode + type: StorageRetentionMode! + - name: retainUntilDate + type: TimestampTz! + graphql: + typeName: PutStorageObjectRetentionOptions + inputTypeName: PutStorageObjectRetentionOptionsInput + dataConnectorTypeMapping: + - dataConnectorName: storage + dataConnectorObjectType: PutStorageObjectRetentionOptions + +--- +kind: TypePermissions +version: v1 +definition: + typeName: PutStorageObjectRetentionOptions + permissions: + - role: admin + output: + allowedFields: + - governanceBypass + - mode + - retainUntilDate + --- kind: ObjectType version: v1 @@ -29,21 +62,19 @@ definition: type: TimestampTz - name: legalHold type: Boolean - - name: mode - type: StorageRetentionMode + - name: metadata + type: Json - name: numThreads type: Int32 - name: partSize type: Int64 - - name: retainUntilDate - type: TimestampTz + - name: retention + type: PutStorageObjectRetentionOptions - name: sendContentMd5 type: Boolean - name: storageClass type: String - - name: userMetadata - type: Json - - name: userTags + - name: tags type: Json - name: websiteRedirectLocation type: String @@ -75,14 +106,13 @@ definition: - disableMultipart - expires - legalHold - - mode + - metadata - numThreads - partSize - - retainUntilDate + - retention - sendContentMd5 - storageClass - - userMetadata - - userTags + - tags - websiteRedirectLocation --- @@ -103,7 +133,7 @@ definition: - name: options type: PutStorageObjectOptions - name: where - type: StorageObjectSimpleBoolExp + type: StorageObjectFilterBoolExp source: dataConnectorName: storage dataConnectorCommand: diff --git a/tests/engine/app/metadata/UploadStorageObjectText.hml b/tests/engine/app/metadata/UploadStorageObjectText.hml index 74224a3..8a753e5 100644 --- a/tests/engine/app/metadata/UploadStorageObjectText.hml +++ b/tests/engine/app/metadata/UploadStorageObjectText.hml @@ -16,7 +16,7 @@ definition: - name: options type: PutStorageObjectOptions - name: where - type: StorageObjectSimpleBoolExp + type: StorageObjectFilterBoolExp source: dataConnectorName: storage dataConnectorCommand: diff --git a/tests/engine/app/metadata/storage-types.hml b/tests/engine/app/metadata/storage-types.hml index d16e84d..abc4287 100644 --- a/tests/engine/app/metadata/storage-types.hml +++ b/tests/engine/app/metadata/storage-types.hml @@ -128,87 +128,87 @@ definition: kind: ScalarType version: v1 definition: - name: BucketName + name: StorageBucketName graphql: - typeName: BucketName + typeName: StorageBucketName --- kind: BooleanExpressionType version: v1 definition: - name: BucketNameBoolExp + name: StorageBucketNameBoolExp operand: scalar: - type: BucketName + type: StorageBucketName comparisonOperators: - name: _eq - argumentType: BucketName! + argumentType: StorageBucketName! dataConnectorOperatorMapping: - dataConnectorName: storage - dataConnectorScalarType: BucketName + dataConnectorScalarType: StorageBucketName operatorMapping: {} logicalOperators: enable: true isNull: enable: true graphql: - typeName: BucketNameBoolExp + typeName: StorageBucketNameBoolExp --- kind: DataConnectorScalarRepresentation version: v1 definition: dataConnectorName: storage - dataConnectorScalarType: BucketName - representation: BucketName + dataConnectorScalarType: StorageBucketName + representation: StorageBucketName graphql: - comparisonExpressionTypeName: BucketNameComparisonExp + comparisonExpressionTypeName: StorageBucketNameComparisonExp --- kind: ScalarType version: v1 definition: - name: ObjectPath + name: StorageStringFilter graphql: - typeName: ObjectPath + typeName: StorageStringFilter --- kind: BooleanExpressionType version: v1 definition: - name: ObjectPathBoolExp + name: StorageStringFilterBoolExp operand: scalar: - type: ObjectPath + type: StorageStringFilter comparisonOperators: - name: _contains - argumentType: ObjectPath! + argumentType: StorageStringFilter! - name: _eq - argumentType: ObjectPath! + argumentType: StorageStringFilter! - name: _icontains - argumentType: ObjectPath! + argumentType: StorageStringFilter! - name: _starts_with - argumentType: ObjectPath! + argumentType: StorageStringFilter! dataConnectorOperatorMapping: - dataConnectorName: storage - dataConnectorScalarType: ObjectPath + dataConnectorScalarType: StorageStringFilter operatorMapping: {} logicalOperators: enable: true isNull: enable: true graphql: - typeName: ObjectPathBoolExp + typeName: StorageStringFilterBoolExp --- kind: DataConnectorScalarRepresentation version: v1 definition: dataConnectorName: storage - dataConnectorScalarType: ObjectPath - representation: ObjectPath + dataConnectorScalarType: StorageStringFilter + representation: StorageStringFilter graphql: - comparisonExpressionTypeName: ObjectPathComparisonExp + comparisonExpressionTypeName: StorageStringFilterComparisonExp --- kind: ScalarType @@ -262,305 +262,305 @@ definition: kind: ScalarType version: v1 definition: - name: Date + name: TimestampTz graphql: - typeName: Date + typeName: TimestampTz --- kind: BooleanExpressionType version: v1 definition: - name: DateBoolExp + name: TimestampTzBoolExp operand: scalar: - type: Date + type: TimestampTz comparisonOperators: [] dataConnectorOperatorMapping: - dataConnectorName: storage - dataConnectorScalarType: Date + dataConnectorScalarType: TimestampTZ operatorMapping: {} logicalOperators: enable: true isNull: enable: true graphql: - typeName: DateBoolExp + typeName: TimestampTzBoolExp --- kind: DataConnectorScalarRepresentation version: v1 definition: dataConnectorName: storage - dataConnectorScalarType: Date - representation: Date + dataConnectorScalarType: TimestampTZ + representation: TimestampTz graphql: - comparisonExpressionTypeName: DateComparisonExp + comparisonExpressionTypeName: TimestampTzComparisonExp --- kind: ScalarType version: v1 definition: - name: Int64 + name: Duration graphql: - typeName: Int64 + typeName: Duration --- kind: BooleanExpressionType version: v1 definition: - name: Int64BoolExp + name: DurationBoolExp operand: scalar: - type: Int64 + type: Duration comparisonOperators: [] dataConnectorOperatorMapping: - dataConnectorName: storage - dataConnectorScalarType: Int64 + dataConnectorScalarType: Duration operatorMapping: {} logicalOperators: enable: true isNull: enable: true graphql: - typeName: Int64BoolExp + typeName: DurationBoolExp --- kind: DataConnectorScalarRepresentation version: v1 definition: dataConnectorName: storage - dataConnectorScalarType: Int64 - representation: Int64 + dataConnectorScalarType: Duration + representation: Duration graphql: - comparisonExpressionTypeName: Int64ComparisonExp + comparisonExpressionTypeName: DurationComparisonExp --- kind: ScalarType version: v1 definition: - name: StorageReplicationRuleStatus + name: Date graphql: - typeName: StorageReplicationRuleStatus + typeName: Date --- kind: BooleanExpressionType version: v1 definition: - name: StorageReplicationRuleStatusBoolExp + name: DateBoolExp operand: scalar: - type: StorageReplicationRuleStatus + type: Date comparisonOperators: [] dataConnectorOperatorMapping: - dataConnectorName: storage - dataConnectorScalarType: StorageReplicationRuleStatus + dataConnectorScalarType: Date operatorMapping: {} logicalOperators: enable: true isNull: enable: true graphql: - typeName: StorageReplicationRuleStatusBoolExp + typeName: DateBoolExp --- kind: DataConnectorScalarRepresentation version: v1 definition: dataConnectorName: storage - dataConnectorScalarType: StorageReplicationRuleStatus - representation: StorageReplicationRuleStatus + dataConnectorScalarType: Date + representation: Date graphql: - comparisonExpressionTypeName: StorageReplicationRuleStatusComparisonExp + comparisonExpressionTypeName: DateComparisonExp --- kind: ScalarType version: v1 definition: - name: TimestampTz + name: Int64 graphql: - typeName: TimestampTz + typeName: Int64 --- kind: BooleanExpressionType version: v1 definition: - name: TimestampTzBoolExp + name: Int64BoolExp operand: scalar: - type: TimestampTz + type: Int64 comparisonOperators: [] dataConnectorOperatorMapping: - dataConnectorName: storage - dataConnectorScalarType: TimestampTZ + dataConnectorScalarType: Int64 operatorMapping: {} logicalOperators: enable: true isNull: enable: true graphql: - typeName: TimestampTzBoolExp + typeName: Int64BoolExp --- kind: DataConnectorScalarRepresentation version: v1 definition: dataConnectorName: storage - dataConnectorScalarType: TimestampTZ - representation: TimestampTz + dataConnectorScalarType: Int64 + representation: Int64 graphql: - comparisonExpressionTypeName: TimestampTzComparisonExp + comparisonExpressionTypeName: Int64ComparisonExp --- kind: ScalarType version: v1 definition: - name: StorageObjectReplicationStatus + name: StorageRetentionMode graphql: - typeName: StorageObjectReplicationStatus + typeName: StorageRetentionMode --- kind: BooleanExpressionType version: v1 definition: - name: StorageObjectReplicationStatusBoolExp + name: StorageRetentionModeBoolExp operand: scalar: - type: StorageObjectReplicationStatus + type: StorageRetentionMode comparisonOperators: [] dataConnectorOperatorMapping: - dataConnectorName: storage - dataConnectorScalarType: StorageObjectReplicationStatus + dataConnectorScalarType: StorageRetentionMode operatorMapping: {} logicalOperators: enable: true isNull: enable: true graphql: - typeName: StorageObjectReplicationStatusBoolExp + typeName: StorageRetentionModeBoolExp --- kind: DataConnectorScalarRepresentation version: v1 definition: dataConnectorName: storage - dataConnectorScalarType: StorageObjectReplicationStatus - representation: StorageObjectReplicationStatus + dataConnectorScalarType: StorageRetentionMode + representation: StorageRetentionMode graphql: - comparisonExpressionTypeName: StorageObjectReplicationStatusComparisonExp + comparisonExpressionTypeName: StorageRetentionModeComparisonExp --- kind: ScalarType version: v1 definition: - name: StorageRetentionMode + name: StorageRetentionValidityUnit graphql: - typeName: StorageRetentionMode + typeName: StorageRetentionValidityUnit --- kind: BooleanExpressionType version: v1 definition: - name: StorageRetentionModeBoolExp + name: StorageRetentionValidityUnitBoolExp operand: scalar: - type: StorageRetentionMode + type: StorageRetentionValidityUnit comparisonOperators: [] dataConnectorOperatorMapping: - dataConnectorName: storage - dataConnectorScalarType: StorageRetentionMode + dataConnectorScalarType: StorageRetentionValidityUnit operatorMapping: {} logicalOperators: enable: true isNull: enable: true graphql: - typeName: StorageRetentionModeBoolExp + typeName: StorageRetentionValidityUnitBoolExp --- kind: DataConnectorScalarRepresentation version: v1 definition: dataConnectorName: storage - dataConnectorScalarType: StorageRetentionMode - representation: StorageRetentionMode + dataConnectorScalarType: StorageRetentionValidityUnit + representation: StorageRetentionValidityUnit graphql: - comparisonExpressionTypeName: StorageRetentionModeComparisonExp + comparisonExpressionTypeName: StorageRetentionValidityUnitComparisonExp --- kind: ScalarType version: v1 definition: - name: StorageRetentionValidityUnit + name: GoogleStorageRpo graphql: - typeName: StorageRetentionValidityUnit + typeName: GoogleStorageRpo --- kind: BooleanExpressionType version: v1 definition: - name: StorageRetentionValidityUnitBoolExp + name: GoogleStorageRpoBoolExp operand: scalar: - type: StorageRetentionValidityUnit + type: GoogleStorageRpo comparisonOperators: [] dataConnectorOperatorMapping: - dataConnectorName: storage - dataConnectorScalarType: StorageRetentionValidityUnit + dataConnectorScalarType: GoogleStorageRPO operatorMapping: {} logicalOperators: enable: true isNull: enable: true graphql: - typeName: StorageRetentionValidityUnitBoolExp + typeName: GoogleStorageRpoBoolExp --- kind: DataConnectorScalarRepresentation version: v1 definition: dataConnectorName: storage - dataConnectorScalarType: StorageRetentionValidityUnit - representation: StorageRetentionValidityUnit + dataConnectorScalarType: GoogleStorageRPO + representation: GoogleStorageRpo graphql: - comparisonExpressionTypeName: StorageRetentionValidityUnitComparisonExp + comparisonExpressionTypeName: GoogleStorageRpoComparisonExp --- kind: ScalarType version: v1 definition: - name: Duration + name: StorageObjectReplicationStatus graphql: - typeName: Duration + typeName: StorageObjectReplicationStatus --- kind: BooleanExpressionType version: v1 definition: - name: DurationBoolExp + name: StorageObjectReplicationStatusBoolExp operand: scalar: - type: Duration + type: StorageObjectReplicationStatus comparisonOperators: [] dataConnectorOperatorMapping: - dataConnectorName: storage - dataConnectorScalarType: Duration + dataConnectorScalarType: StorageObjectReplicationStatus operatorMapping: {} logicalOperators: enable: true isNull: enable: true graphql: - typeName: DurationBoolExp + typeName: StorageObjectReplicationStatusBoolExp --- kind: DataConnectorScalarRepresentation version: v1 definition: dataConnectorName: storage - dataConnectorScalarType: Duration - representation: Duration + dataConnectorScalarType: StorageObjectReplicationStatus + representation: StorageObjectReplicationStatus graphql: - comparisonExpressionTypeName: DurationComparisonExp + comparisonExpressionTypeName: StorageObjectReplicationStatusComparisonExp --- kind: ScalarType diff --git a/tests/engine/app/metadata/storage.hml b/tests/engine/app/metadata/storage.hml index 89afbc3..35eccc7 100644 --- a/tests/engine/app/metadata/storage.hml +++ b/tests/engine/app/metadata/storage.hml @@ -17,13 +17,6 @@ definition: type: boolean aggregate_functions: {} comparison_operators: {} - BucketName: - representation: - type: string - aggregate_functions: {} - comparison_operators: - _eq: - type: equal Bytes: representation: type: bytes @@ -53,6 +46,14 @@ definition: type: json aggregate_functions: {} comparison_operators: {} + GoogleStorageRPO: + representation: + type: enum + one_of: + - DEFAULT + - ASYNC_TURBO + aggregate_functions: {} + comparison_operators: {} Int32: representation: type: int32 @@ -68,28 +69,13 @@ definition: type: json aggregate_functions: {} comparison_operators: {} - ObjectPath: + StorageBucketName: representation: type: string aggregate_functions: {} comparison_operators: - _contains: - type: custom - argument_type: - type: named - name: ObjectPath _eq: type: equal - _icontains: - type: custom - argument_type: - type: named - name: ObjectPath - _starts_with: - type: custom - argument_type: - type: named - name: ObjectPath StorageClientID: representation: type: enum @@ -98,6 +84,8 @@ definition: - s3 - azblob - azblob-connstr + - gcs + - gcs-cred aggregate_functions: {} comparison_operators: _eq: @@ -112,20 +100,14 @@ definition: - REPLICA aggregate_functions: {} comparison_operators: {} - StorageReplicationRuleStatus: - representation: - type: enum - one_of: - - Enabled - - Disabled - aggregate_functions: {} - comparison_operators: {} StorageRetentionMode: representation: type: enum one_of: - Locked - Unlocked + - Mutable + - Delete aggregate_functions: {} comparison_operators: {} StorageRetentionValidityUnit: @@ -136,6 +118,28 @@ definition: - YEARS aggregate_functions: {} comparison_operators: {} + StorageStringFilter: + representation: + type: string + aggregate_functions: {} + comparison_operators: + _contains: + type: custom + argument_type: + type: named + name: StorageStringFilter + _eq: + type: equal + _icontains: + type: custom + argument_type: + type: named + name: StorageStringFilter + _starts_with: + type: custom + argument_type: + type: named + name: StorageStringFilter String: representation: type: string @@ -147,140 +151,181 @@ definition: aggregate_functions: {} comparison_operators: {} object_types: - AbortIncompleteMultipartUpload: - description: structure, not supported yet on MinIO + BucketAutoclass: fields: - daysAfterInitiation: + enabled: type: - type: nullable - underlying_type: - type: named - name: Int32 - BucketLifecycleConfiguration: - description: is a collection of lifecycle Rule objects. + type: named + name: Boolean + terminalStorageClass: + type: + type: named + name: String + terminalStorageClassUpdateTime: + type: + type: named + name: TimestampTZ + toggleTime: + type: + type: named + name: TimestampTZ + BucketCors: + description: is the bucket's Cross-Origin Resource Sharing (CORS) configuration. fields: - rules: + maxAge: + type: + type: named + name: Duration + methods: type: type: array element_type: type: named - name: BucketLifecycleRule - BucketLifecycleRule: - description: represents a single rule in lifecycle configuration - fields: - abortIncompleteMultipartUpload: + name: String + origins: type: - type: nullable - underlying_type: + type: array + element_type: type: named - name: AbortIncompleteMultipartUpload - allVersionsExpiration: + name: String + responseHeaders: type: - type: nullable - underlying_type: + type: array + element_type: type: named - name: LifecycleAllVersionsExpiration - delMarkerExpiration: + name: String + BucketHierarchicalNamespace: + fields: + enabled: type: - type: nullable - underlying_type: - type: named - name: LifecycleDelMarkerExpiration - expiration: + type: named + name: Boolean + BucketLogging: + description: holds the bucket's logging configuration, which defines the destination bucket and optional name prefix for the current bucket's logs. + fields: + logBucket: type: - type: nullable - underlying_type: - type: named - name: LifecycleExpiration - filter: + type: named + name: String + logObjectPrefix: + type: + type: named + name: String + BucketWebsite: + description: holds the bucket's website configuration, controlling how the service behaves when accessing bucket contents as a web site. See https://cloud.google.com/storage/docs/static-website for more information. + fields: + mainPageSuffix: + type: + type: named + name: String + notFoundPage: type: type: nullable underlying_type: type: named - name: LifecycleFilter - id: + name: String + CustomPlacementConfig: + description: holds the bucket's custom placement configuration for Custom Dual Regions. See https://cloud.google.com/storage/docs/locations#location-dr for more information. + fields: + DataLocations: type: - type: named - name: String - noncurrentVersionExpiration: + type: array + element_type: + type: named + name: String + GetStorageObjectOptions: + description: are used to specify additional headers or options during GET requests. + fields: + headers: type: type: nullable underlying_type: type: named - name: LifecycleNoncurrentVersionExpiration - noncurrentVersionTransition: + name: JSON + partNumber: type: type: nullable underlying_type: type: named - name: LifecycleNoncurrentVersionTransition - prefix: + name: Int32 + requestParams: type: type: nullable underlying_type: type: named - name: String - status: + name: JSON + versionId: type: type: nullable underlying_type: type: named name: String - transition: + ListIncompleteUploadsOptions: + description: the input arguments of the ListIncompleteUploads method. + fields: + prefix: + type: + type: named + name: String + recursive: type: type: nullable underlying_type: type: named - name: LifecycleTransition - DeleteMarkerReplication: - description: whether delete markers are replicated - https://docs.aws.amazon.com/AmazonS3/latest/dev/replication-add-config.html + name: Boolean + ListStorageObjectsOptions: + description: holds all options of a list object request. fields: - status: + maxResults: type: type: named - name: StorageReplicationRuleStatus - DeleteReplication: - description: whether versioned deletes are replicated. This is a MinIO specific extension - fields: - status: + name: Int32 + prefix: type: type: named - name: StorageReplicationRuleStatus - ExistingObjectReplication: - description: whether existing object replication is enabled - fields: - status: + name: String + recursive: type: type: named - name: StorageReplicationRuleStatus - GetStorageObjectOptions: - description: are used to specify additional headers or options during GET requests. + name: Boolean + startAfter: + type: + type: named + name: String + MakeStorageBucketOptions: + description: holds all options to tweak bucket creation. fields: - headers: + name: + type: + type: named + name: String + objectLock: type: type: nullable underlying_type: type: named - name: JSON - partNumber: + name: Boolean + region: type: type: nullable underlying_type: type: named - name: Int32 - requestParams: + name: String + tags: type: type: nullable underlying_type: type: named name: JSON - versionId: + ObjectAbortIncompleteMultipartUpload: + fields: + daysAfterInitiation: type: type: nullable underlying_type: type: named - name: String - LifecycleAllVersionsExpiration: + name: Int32 + ObjectLifecycleAllVersionsExpiration: description: represents AllVersionsExpiration actions element in an ILM policy fields: days: @@ -295,8 +340,16 @@ definition: underlying_type: type: named name: Boolean - LifecycleDelMarkerExpiration: - description: represents DelMarkerExpiration actions element in an ILM policy + ObjectLifecycleConfiguration: + description: is a collection of lifecycle Rule objects. + fields: + rules: + type: + type: array + element_type: + type: named + name: ObjectLifecycleRule + ObjectLifecycleDelMarkerExpiration: fields: days: type: @@ -304,7 +357,7 @@ definition: underlying_type: type: named name: Int32 - LifecycleExpiration: + ObjectLifecycleExpiration: description: expiration details of lifecycle configuration fields: date: @@ -331,42 +384,33 @@ definition: underlying_type: type: named name: Boolean - LifecycleFilter: + ObjectLifecycleFilter: description: will be used in selecting rule(s) for lifecycle configuration fields: - and: - type: - type: nullable - underlying_type: - type: named - name: LifecycleFilterAnd - objectSizeGreaterThan: - type: - type: nullable - underlying_type: - type: named - name: Int64 - objectSizeLessThan: + matchesPrefix: type: type: nullable underlying_type: - type: named - name: Int64 - prefix: + type: array + element_type: + type: named + name: String + matchesStorageClasses: type: type: nullable underlying_type: - type: named - name: String - tag: + type: array + element_type: + type: named + name: String + matchesSuffix: type: type: nullable underlying_type: - type: named - name: StorageTag - LifecycleFilterAnd: - description: the And Rule for LifecycleTag, to be used in LifecycleRuleFilter - fields: + type: array + element_type: + type: named + name: String objectSizeGreaterThan: type: type: nullable @@ -379,21 +423,13 @@ definition: underlying_type: type: named name: Int64 - prefix: - type: - type: nullable - underlying_type: - type: named - name: String tags: type: type: nullable underlying_type: - type: array - element_type: - type: named - name: StorageTag - LifecycleNoncurrentVersionExpiration: + type: named + name: JSON + ObjectLifecycleNoncurrentVersionExpiration: description: '- Specifies when noncurrent object versions expire. Upon expiration, server permanently deletes the noncurrent object versions. Set this lifecycle configuration action on a bucket that has versioning enabled (or suspended) to request server delete noncurrent object versions at a specific period in the object''s lifetime.' fields: newerNoncurrentVersions: @@ -408,7 +444,7 @@ definition: underlying_type: type: named name: Int32 - LifecycleNoncurrentVersionTransition: + ObjectLifecycleNoncurrentVersionTransition: description: sets this action to request server to transition noncurrent object versions to different set storage classes at a specific period in the object's lifetime. fields: newerNoncurrentVersions: @@ -429,256 +465,98 @@ definition: underlying_type: type: named name: String - LifecycleTransition: - description: transition details of lifecycle configuration + ObjectLifecycleRule: + description: represents a single rule in lifecycle configuration fields: - date: + abortIncompleteMultipartUpload: type: type: nullable underlying_type: type: named - name: Date - days: + name: ObjectAbortIncompleteMultipartUpload + allVersionsExpiration: type: type: nullable underlying_type: type: named - name: Int32 - storageClass: + name: ObjectLifecycleAllVersionsExpiration + delMarkerExpiration: type: type: nullable underlying_type: type: named - name: String - ListIncompleteUploadsOptions: - description: the input arguments of the ListIncompleteUploads method. - fields: - prefix: - type: - type: named - name: String - recursive: + name: ObjectLifecycleDelMarkerExpiration + enabled: type: type: nullable underlying_type: type: named name: Boolean - ListStorageObjectsOptions: - description: holds all options of a list object request. - fields: - maxResults: + expiration: type: - type: named - name: Int32 - prefix: - type: - type: named - name: String - recursive: - type: - type: named - name: Boolean - startAfter: - type: - type: named - name: String - MakeStorageBucketOptions: - description: holds all options to tweak bucket creation. - fields: - name: - type: - type: named - name: String - objectLocking: - type: - type: nullable - underlying_type: - type: named - name: Boolean - region: + type: nullable + underlying_type: + type: named + name: ObjectLifecycleExpiration + filter: type: type: nullable underlying_type: - type: named - name: String - tags: - type: - type: nullable - underlying_type: - type: named - name: JSON - NotificationCommonConfig: - description: '- represents one single notification configuration such as topic, queue or lambda configuration.' - fields: - arn: - type: - type: nullable - underlying_type: - type: named - name: String - event: - type: - type: array - element_type: - type: named - name: String - filter: - type: - type: nullable - underlying_type: - type: named - name: NotificationFilter + type: array + element_type: + type: named + name: ObjectLifecycleFilter id: type: type: nullable underlying_type: type: named name: String - NotificationConfig: - description: the struct that represents a notification configration object. - fields: - cloudFunctionConfigurations: - type: - type: array - element_type: - type: named - name: NotificationLambdaConfig - queueConfigurations: - type: - type: array - element_type: - type: named - name: NotificationQueueConfig - topicConfigurations: - type: - type: array - element_type: - type: named - name: NotificationTopicConfig - NotificationFilter: - description: '- a tag in the notification xml structure which carries suffix/prefix filters' - fields: - s3Key: - type: - type: nullable - underlying_type: - type: named - name: NotificationS3Key - NotificationFilterRule: - description: child of S3Key, a tag in the notification xml which carries suffix/prefix filters - fields: - name: - type: - type: named - name: String - value: - type: - type: named - name: String - NotificationLambdaConfig: - description: carries one single cloudfunction notification configuration - fields: - arn: - type: - type: nullable - underlying_type: - type: named - name: String - cloudFunction: - type: - type: named - name: String - event: - type: - type: array - element_type: - type: named - name: String - filter: + noncurrentVersionExpiration: type: type: nullable underlying_type: type: named - name: NotificationFilter - id: + name: ObjectLifecycleNoncurrentVersionExpiration + noncurrentVersionTransition: type: type: nullable underlying_type: type: named - name: String - NotificationQueueConfig: - description: carries one single queue notification configuration - fields: - arn: + name: ObjectLifecycleNoncurrentVersionTransition + prefix: type: type: nullable underlying_type: type: named name: String - event: - type: - type: array - element_type: - type: named - name: String - filter: - type: - type: nullable - underlying_type: - type: named - name: NotificationFilter - id: + transition: type: type: nullable underlying_type: type: named - name: String - queue: - type: - type: named - name: String - NotificationS3Key: - description: child of Filter, a tag in the notification xml which carries suffix/prefix filters - fields: - filterRule: - type: - type: nullable - underlying_type: - type: array - element_type: - type: named - name: NotificationFilterRule - NotificationTopicConfig: - description: carries one single topic notification configuration + name: ObjectLifecycleTransition + ObjectLifecycleTransition: + description: transition details of lifecycle configuration fields: - arn: + date: type: type: nullable underlying_type: type: named - name: String - event: - type: - type: array - element_type: - type: named - name: String - filter: + name: Date + days: type: type: nullable underlying_type: type: named - name: NotificationFilter - id: + name: Int32 + storageClass: type: type: nullable underlying_type: type: named name: String - topic: - type: - type: named - name: String PresignedGetStorageObjectOptions: description: represent the options for the PresignedGetObject method. fields: @@ -735,7 +613,7 @@ definition: type: nullable underlying_type: type: predicate - object_type_name: StorageObjectSimple + object_type_name: StorageObjectFilter PutStorageObjectOptions: description: represents options specified by user for PutObject call. fields: @@ -811,12 +689,12 @@ definition: underlying_type: type: named name: Boolean - mode: + metadata: type: type: nullable underlying_type: type: named - name: StorageRetentionMode + name: JSON numThreads: type: type: nullable @@ -829,12 +707,12 @@ definition: underlying_type: type: named name: Int64 - retainUntilDate: + retention: type: type: nullable underlying_type: type: named - name: TimestampTZ + name: PutStorageObjectRetentionOptions sendContentMd5: type: type: nullable @@ -847,208 +725,296 @@ definition: underlying_type: type: named name: String - userMetadata: + tags: type: type: nullable underlying_type: type: named name: JSON - userTags: + websiteRedirectLocation: + type: + type: nullable + underlying_type: + type: named + name: String + PutStorageObjectRetentionOptions: + description: represent options of object retention configuration. + fields: + governanceBypass: + type: + type: nullable + underlying_type: + type: named + name: Boolean + mode: + type: + type: named + name: StorageRetentionMode + retainUntilDate: + type: + type: named + name: TimestampTZ + RemoveStorageObjectError: + description: the container of Multi Delete S3 API error. + fields: + error: type: type: nullable underlying_type: type: named name: JSON - websiteRedirectLocation: + objectName: + type: + type: named + name: String + versionId: + type: + type: named + name: String + RemoveStorageObjectOptions: + description: represents options specified by user for RemoveObject call. + fields: + forceDelete: + type: + type: nullable + underlying_type: + type: named + name: Boolean + governanceBypass: + type: + type: nullable + underlying_type: + type: named + name: Boolean + softDelete: + type: + type: nullable + underlying_type: + type: named + name: Boolean + versionId: + type: + type: nullable + underlying_type: + type: named + name: String + RemoveStorageObjectsOptions: + description: represents options specified by user for RemoveObjects call. + fields: + governanceBypass: + type: + type: nullable + underlying_type: + type: named + name: Boolean + maxResults: + type: + type: named + name: Int32 + prefix: + type: + type: named + name: String + recursive: + type: + type: named + name: Boolean + startAfter: + type: + type: named + name: String + ServerSideEncryptionConfiguration: + description: is the default encryption configuration structure. + fields: + kmsMasterKeyId: + type: + type: nullable + underlying_type: + type: named + name: String + sseAlgorithm: + type: + type: nullable + underlying_type: + type: named + name: String + SetStorageObjectLockConfig: + description: represents the object lock configuration options in given bucket + fields: + mode: + type: + type: nullable + underlying_type: + type: named + name: StorageRetentionMode + unit: + type: + type: nullable + underlying_type: + type: named + name: StorageRetentionValidityUnit + validity: + type: + type: nullable + underlying_type: + type: named + name: Int32 + SetStorageObjectRetentionOptions: + description: represents options specified by user for PutObject call. + fields: + governanceBypass: + type: + type: nullable + underlying_type: + type: named + name: Boolean + mode: + type: + type: nullable + underlying_type: + type: named + name: StorageRetentionMode + retainUntilDate: + type: + type: nullable + underlying_type: + type: named + name: TimestampTZ + StorageBucket: + description: the container for bucket metadata. + fields: + autoclass: + type: + type: nullable + underlying_type: + type: named + name: BucketAutoclass + cors: + type: + type: nullable + underlying_type: + type: array + element_type: + type: named + name: BucketCors + creationTime: type: type: nullable underlying_type: type: named - name: String - RemoveStorageObjectError: - description: the container of Multi Delete S3 API error. - fields: - error: + name: TimestampTZ + customPlacementConfig: type: type: nullable underlying_type: type: named - name: JSON - objectName: - type: - type: named - name: String - versionId: - type: - type: named - name: String - RemoveStorageObjectOptions: - description: represents options specified by user for RemoveObject call. - fields: - forceDelete: + name: CustomPlacementConfig + defaultEventBasedHold: type: type: nullable underlying_type: type: named name: Boolean - governanceBypass: + encryption: type: type: nullable underlying_type: type: named - name: Boolean - versionId: + name: ServerSideEncryptionConfiguration + etag: type: type: nullable underlying_type: type: named name: String - RemoveStorageObjectsOptions: - description: represents options specified by user for RemoveObjects call. - fields: - governanceBypass: + hierarchicalNamespace: type: type: nullable underlying_type: type: named - name: Boolean - maxResults: - type: - type: named - name: Int32 - prefix: - type: - type: named - name: String - recursive: - type: - type: named - name: Boolean - startAfter: - type: - type: named - name: String - ReplicaModifications: - description: specifies if replica modification sync is enabled - fields: - status: - type: - type: named - name: StorageReplicationRuleStatus - ServerSideEncryptionConfiguration: - description: is the default encryption configuration structure. - fields: - rules: + name: BucketHierarchicalNamespace + lastModified: type: - type: array - element_type: + type: nullable + underlying_type: type: named - name: ServerSideEncryptionRule - ServerSideEncryptionRule: - description: rule layer encapsulates default encryption configuration - fields: - apply: - type: - type: named - name: StorageApplySSEByDefault - SetStorageObjectLegalHoldOptions: - description: represents options specified by user for PutObjectLegalHold call. - fields: - status: + name: TimestampTZ + lifecycle: type: type: nullable underlying_type: type: named - name: Boolean - versionId: + name: ObjectLifecycleConfiguration + locationType: type: type: nullable underlying_type: type: named name: String - SetStorageObjectLockConfig: - description: represents the object lock configuration options in given bucket - fields: - mode: + logging: type: type: nullable underlying_type: type: named - name: StorageRetentionMode - unit: + name: BucketLogging + name: + type: + type: named + name: String + objectLock: type: type: nullable underlying_type: type: named - name: StorageRetentionValidityUnit - validity: + name: StorageObjectLockConfig + region: type: type: nullable underlying_type: type: named - name: Int32 - SetStorageObjectRetentionOptions: - description: represents options specified by user for PutObject call. - fields: - governanceBypass: + name: String + requesterPays: type: type: nullable underlying_type: type: named name: Boolean - mode: + rpo: type: type: nullable underlying_type: type: named - name: StorageRetentionMode - retainUntilDate: + name: GoogleStorageRPO + softDeletePolicy: type: type: nullable underlying_type: type: named - name: TimestampTZ - versionId: + name: StorageObjectSoftDeletePolicy + storageClass: type: type: nullable underlying_type: type: named name: String - SetStorageObjectTagsOptions: - description: holds an object version id to update tag(s) of a specific object version. - fields: tags: - type: - type: named - name: JSON - versionId: type: type: nullable underlying_type: type: named - name: String - SourceSelectionCriteria: - description: specifies additional source selection criteria in ReplicationConfiguration. - fields: - replicaModifications: + name: JSON + versioning: type: type: nullable underlying_type: type: named - name: ReplicaModifications - StorageApplySSEByDefault: - description: defines default encryption configuration, KMS or SSE. To activate KMS, SSEAlgoritm needs to be set to `aws:kms“. Minio currently does not support Kms. - fields: - kmsMasterKeyId: + name: StorageBucketVersioningConfiguration + website: type: type: nullable underlying_type: type: named - name: String - sseAlgorithm: - type: - type: named - name: String + name: BucketWebsite StorageBucketArguments: description: represent the common input arguments for bucket-related methods. fields: @@ -1064,26 +1030,36 @@ definition: underlying_type: type: named name: StorageClientID - StorageBucketInfo: - description: container for bucket metadata. + StorageBucketFilter: fields: - creationDate: + bucket: type: type: named - name: TimestampTZ - name: + name: StorageStringFilter + clientId: type: type: named - name: String - tags: + name: StorageClientID + StorageBucketListResults: + description: hold the paginated results of the storage bucket list. + fields: + buckets: type: - type: nullable - underlying_type: + type: array + element_type: type: named - name: JSON + name: StorageBucket + pageInfo: + type: + type: named + name: StoragePaginationInfo StorageBucketVersioningConfiguration: description: is the versioning configuration structure fields: + enabled: + type: + type: named + name: Boolean excludeFolders: type: type: nullable @@ -1104,12 +1080,6 @@ definition: underlying_type: type: named name: String - status: - type: - type: nullable - underlying_type: - type: named - name: String StorageCopyDestOptions: description: represents options specified by user for CopyObject/ComposeObject APIs. fields: @@ -1125,6 +1095,12 @@ definition: underlying_type: type: named name: Boolean + metadata: + type: + type: nullable + underlying_type: + type: named + name: JSON mode: type: type: nullable @@ -1135,18 +1111,6 @@ definition: type: type: named name: String - replaceMetadata: - type: - type: nullable - underlying_type: - type: named - name: Boolean - replaceTags: - type: - type: nullable - underlying_type: - type: named - name: Boolean retainUntilDate: type: type: nullable @@ -1159,13 +1123,7 @@ definition: underlying_type: type: named name: Int64 - userMetadata: - type: - type: nullable - underlying_type: - type: named - name: JSON - userTags: + tags: type: type: nullable underlying_type: @@ -1288,7 +1246,7 @@ definition: type: nullable underlying_type: type: named - name: String + name: JSON archiveStatus: type: type: nullable @@ -1381,42 +1339,12 @@ definition: underlying_type: type: named name: String - copyCompletionTime: - type: - type: nullable - underlying_type: - type: named - name: TimestampTZ - copyId: - type: - type: nullable - underlying_type: - type: named - name: String - copyProgress: - type: - type: nullable - underlying_type: - type: named - name: String - copySource: - type: - type: nullable - underlying_type: - type: named - name: String - copyStatus: - type: - type: nullable - underlying_type: - type: named - name: String - copyStatusDescription: + copy: type: type: nullable underlying_type: type: named - name: String + name: StorageObjectCopyInfo creationTime: type: type: nullable @@ -1447,12 +1375,6 @@ definition: underlying_type: type: named name: String - encryptionScope: - type: - type: nullable - underlying_type: - type: named - name: String etag: type: type: nullable @@ -1491,30 +1413,24 @@ definition: underlying_type: type: named name: String - immutabilityPolicyMode: - type: - type: nullable - underlying_type: - type: named - name: String - immutabilityPolicyUntilDate: + incrementalCopy: type: type: nullable underlying_type: type: named - name: TimestampTZ - incrementalCopy: + name: Boolean + isLatest: type: type: nullable underlying_type: type: named name: Boolean - isLatest: + kmsKeyName: type: type: nullable underlying_type: type: named - name: Boolean + name: String lastAccessTime: type: type: nullable @@ -1549,6 +1465,12 @@ definition: underlying_type: type: named name: Boolean + mediaLink: + type: + type: nullable + underlying_type: + type: named + name: String metadata: type: type: nullable @@ -1571,6 +1493,12 @@ definition: underlying_type: type: named name: String + rawMetadata: + type: + type: nullable + underlying_type: + type: named + name: JSON rehydratePriority: type: type: nullable @@ -1607,6 +1535,18 @@ definition: underlying_type: type: named name: StorageRestoreInfo + retentionMode: + type: + type: nullable + underlying_type: + type: named + name: String + retentionUntilDate: + type: + type: nullable + underlying_type: + type: named + name: TimestampTZ sealed: type: type: nullable @@ -1631,19 +1571,13 @@ definition: underlying_type: type: named name: String - userMetadata: - type: - type: nullable - underlying_type: - type: named - name: JSON - userTagCount: + tagCount: type: type: nullable underlying_type: type: named name: Int32 - userTags: + tags: type: type: nullable underlying_type: @@ -1670,24 +1604,75 @@ definition: underlying_type: type: named name: String - checksumCrc64Nvme: + checksumCrc64Nvme: + type: + type: nullable + underlying_type: + type: named + name: String + checksumSha1: + type: + type: nullable + underlying_type: + type: named + name: String + checksumSha256: + type: + type: nullable + underlying_type: + type: named + name: String + StorageObjectCopyInfo: + description: holds the copy information if the object was copied from another object. + fields: + completionTime: + type: + type: nullable + underlying_type: + type: named + name: TimestampTZ + id: + type: + type: named + name: String + progress: + type: + type: nullable + underlying_type: + type: named + name: String + source: type: type: nullable underlying_type: type: named name: String - checksumSha1: + status: type: type: nullable underlying_type: type: named name: String - checksumSha256: + statusDescription: type: type: nullable underlying_type: type: named name: String + StorageObjectFilter: + fields: + bucket: + type: + type: named + name: StorageBucketName + clientId: + type: + type: named + name: StorageClientID + object: + type: + type: named + name: StorageStringFilter StorageObjectListResults: description: hold the paginated results of the storage object list. fields: @@ -1700,19 +1685,20 @@ definition: pageInfo: type: type: named - name: StorageObjectPaginationInfo + name: StoragePaginationInfo StorageObjectLockConfig: + description: represents the object lock configuration in given bucket fields: + enabled: + type: + type: named + name: Boolean mode: type: type: nullable underlying_type: type: named name: StorageRetentionMode - objectLock: - type: - type: named - name: String unit: type: type: nullable @@ -1758,39 +1744,17 @@ definition: underlying_type: type: named name: String - StorageObjectPaginationInfo: - description: holds the pagination information. - fields: - cursor: - type: - type: nullable - underlying_type: - type: named - name: String - hasNextPage: - type: - type: named - name: Boolean - nextCursor: - type: - type: nullable - underlying_type: - type: named - name: String - StorageObjectSimple: + StorageObjectSoftDeletePolicy: + description: contains the bucket's soft delete policy, which defines the period of time that soft-deleted objects will be retained, and cannot be permanently deleted. fields: - bucket: - type: - type: named - name: BucketName - clientId: + effectiveTime: type: type: named - name: StorageClientID - object: + name: TimestampTZ + retentionDuration: type: type: named - name: ObjectPath + name: Duration StorageOwner: description: name. fields: @@ -1806,237 +1770,185 @@ definition: underlying_type: type: named name: String - StorageReplicationConfig: - description: replication configuration specified in https://docs.aws.amazon.com/AmazonS3/latest/dev/replication-add-config.html + StoragePaginationInfo: + description: holds the pagination information. fields: - role: + cursor: type: type: nullable underlying_type: type: named name: String - rules: + hasNextPage: type: - type: array - element_type: + type: named + name: Boolean + StorageRestoreInfo: + description: contains information of the restore operation of an archived object. + fields: + expiryTime: + type: + type: nullable + underlying_type: type: named - name: StorageReplicationRule - StorageReplicationDestination: + name: TimestampTZ + ongoingRestore: + type: + type: named + name: Boolean + StorageUploadInfo: + description: represents the information of the uploaded object. fields: bucket: type: type: named name: String - storageClass: + checksumCrc32: type: type: nullable underlying_type: type: named name: String - StorageReplicationFilter: - description: a filter for a replication configuration Rule. - fields: - and: + checksumCrc32C: type: type: nullable underlying_type: type: named - name: StorageReplicationFilterAnd - rrefix: + name: String + checksumCrc64Nvme: type: type: nullable underlying_type: type: named name: String - tag: + checksumSha1: type: type: nullable underlying_type: type: named - name: StorageTag - StorageReplicationFilterAnd: - description: '- a tag to combine a prefix and multiple tags for replication configuration rule.' - fields: - rrefix: + name: String + checksumSha256: type: type: nullable underlying_type: type: named name: String - tag: - type: - type: nullable - underlying_type: - type: array - element_type: - type: named - name: StorageTag - StorageReplicationRule: - description: a rule for replication configuration. - fields: - deleteMarkerReplication: + clientId: type: - type: nullable - underlying_type: - type: named - name: DeleteMarkerReplication - deleteReplication: + type: named + name: String + contentMd5: type: type: nullable underlying_type: type: named - name: DeleteReplication - destination: + name: String + etag: type: type: nullable underlying_type: type: named - name: StorageReplicationDestination - existingObjectReplication: + name: String + expiration: type: type: nullable underlying_type: type: named - name: ExistingObjectReplication - filter: - type: - type: named - name: StorageReplicationFilter - id: + name: TimestampTZ + expirationRuleId: type: type: nullable underlying_type: type: named name: String - priority: - type: - type: named - name: Int32 - sourceSelectionCriteria: - type: - type: nullable - underlying_type: - type: named - name: SourceSelectionCriteria - status: - type: - type: named - name: StorageReplicationRuleStatus - StorageRestoreInfo: - description: contains information of the restore operation of an archived object. - fields: - expiryTime: + lastModified: type: type: nullable underlying_type: type: named name: TimestampTZ - ongoingRestore: - type: - type: named - name: Boolean - StorageTag: - description: structure key/value pair representing an object tag to apply configuration - fields: - key: - type: - type: nullable - underlying_type: - type: named - name: String - value: + location: type: type: nullable underlying_type: type: named name: String - StorageUploadInfo: - description: represents the information of the uploaded object. - fields: - bucket: + name: type: type: named name: String - checksumCrc32: - type: - type: nullable - underlying_type: - type: named - name: String - checksumCrc32C: + size: type: type: nullable underlying_type: type: named - name: String - checksumCrc64Nvme: + name: Int64 + versionId: type: type: nullable underlying_type: type: named name: String - checksumSha1: + UpdateStorageBucketOptions: + description: hold update options for the bucket. + fields: + encryption: type: type: nullable underlying_type: type: named - name: String - checksumSha256: + name: ServerSideEncryptionConfiguration + lifecycle: type: type: nullable underlying_type: type: named - name: String - clientId: - type: - type: named - name: String - contentMd5: + name: ObjectLifecycleConfiguration + objectLock: type: type: nullable underlying_type: type: named - name: String - etag: + name: SetStorageObjectLockConfig + tags: type: type: nullable underlying_type: type: named - name: String - expiration: + name: JSON + versioningEnabled: type: type: nullable underlying_type: type: named - name: TimestampTZ - expirationRuleId: + name: Boolean + UpdateStorageObjectOptions: + description: represents options specified by user for updating object. + fields: + legalHold: type: type: nullable underlying_type: type: named - name: String - lastModified: + name: Boolean + metadata: type: type: nullable underlying_type: type: named - name: TimestampTZ - location: + name: JSON + retention: type: type: nullable underlying_type: type: named - name: String - name: - type: - type: named - name: String - size: + name: SetStorageObjectRetentionOptions + tags: type: type: nullable underlying_type: type: named - name: Int64 + name: JSON versionId: type: type: nullable @@ -2093,7 +2005,7 @@ definition: type: nullable underlying_type: type: predicate - object_type_name: StorageObjectSimple + object_type_name: StorageObjectFilter result_type: type: nullable underlying_type: @@ -2147,14 +2059,14 @@ definition: type: nullable underlying_type: type: predicate - object_type_name: StorageObjectSimple + object_type_name: StorageObjectFilter result_type: type: nullable underlying_type: type: named name: String - - name: storageBucketEncryption - description: gets default encryption configuration set on a bucket. + - name: storageBucket + description: gets a bucket by name. arguments: bucket: type: @@ -2172,7 +2084,7 @@ definition: type: nullable underlying_type: type: named - name: ServerSideEncryptionConfiguration + name: StorageBucket - name: storageBucketExists description: checks if a bucket exists. arguments: @@ -2191,117 +2103,60 @@ definition: result_type: type: named name: Boolean - - name: storageBucketLifecycle - description: gets lifecycle on a bucket or a prefix. - arguments: - bucket: - type: - type: nullable - underlying_type: - type: named - name: String - clientId: - type: - type: nullable - underlying_type: - type: named - name: StorageClientID - result_type: - type: nullable - underlying_type: - type: named - name: BucketLifecycleConfiguration - - name: storageBucketNotification - description: gets notification configuration on a bucket. + - name: storageBuckets + description: list all buckets. arguments: - bucket: - type: - type: nullable - underlying_type: - type: named - name: String - clientId: + maxResults: type: type: nullable underlying_type: type: named - name: StorageClientID - result_type: - type: nullable - underlying_type: - type: named - name: NotificationConfig - - name: storageBucketPolicy - description: gets access permissions on a bucket or a prefix. - arguments: - bucket: + name: Int32 + startAfter: type: type: nullable underlying_type: type: named name: String - clientId: + where: type: type: nullable underlying_type: - type: named - name: StorageClientID + type: predicate + object_type_name: StorageBucketFilter result_type: type: named - name: String - - name: storageBucketReplication + name: StorageBucketListResults + - name: storageDeletedObjects + description: list deleted objects in a bucket. arguments: - bucket: - type: - type: nullable - underlying_type: - type: named - name: String - clientId: + maxResults: type: type: nullable underlying_type: type: named - name: StorageClientID - result_type: - type: nullable - underlying_type: - type: named - name: StorageReplicationConfig - - name: storageBucketVersioning - description: gets versioning configuration set on a bucket. - arguments: - bucket: + name: Int32 + recursive: type: type: nullable underlying_type: type: named - name: String - clientId: + name: Boolean + startAfter: type: type: nullable underlying_type: type: named - name: StorageClientID - result_type: - type: nullable - underlying_type: - type: named - name: StorageBucketVersioningConfiguration - - name: storageBuckets - description: list all buckets. - arguments: - clientId: + name: String + where: type: type: nullable underlying_type: - type: named - name: StorageClientID + type: predicate + object_type_name: StorageObjectFilter result_type: - type: array - element_type: - type: named - name: StorageBucketInfo + type: named + name: StorageObjectListResults - name: storageIncompleteUploads description: list partially uploaded objects in a bucket. arguments: @@ -2380,32 +2235,12 @@ definition: type: nullable underlying_type: type: predicate - object_type_name: StorageObjectSimple + object_type_name: StorageObjectFilter result_type: type: nullable underlying_type: type: named name: StorageObject - - name: storageObjectLockConfig - description: gets object lock configuration of given bucket. - arguments: - bucket: - type: - type: nullable - underlying_type: - type: named - name: String - clientId: - type: - type: nullable - underlying_type: - type: named - name: StorageClientID - result_type: - type: nullable - underlying_type: - type: named - name: StorageObjectLockConfig - name: storageObjects description: lists objects in a bucket. arguments: @@ -2432,7 +2267,7 @@ definition: type: nullable underlying_type: type: predicate - object_type_name: StorageObjectSimple + object_type_name: StorageObjectFilter result_type: type: named name: StorageObjectListResults @@ -2472,7 +2307,7 @@ definition: type: nullable underlying_type: type: predicate - object_type_name: StorageObjectSimple + object_type_name: StorageObjectFilter result_type: type: nullable underlying_type: @@ -2508,7 +2343,7 @@ definition: type: nullable underlying_type: type: predicate - object_type_name: StorageObjectSimple + object_type_name: StorageObjectFilter result_type: type: nullable underlying_type: @@ -2570,7 +2405,7 @@ definition: type: type: named name: String - objectLocking: + objectLock: type: type: nullable underlying_type: @@ -2591,24 +2426,6 @@ definition: result_type: type: named name: Boolean - - name: enableStorageBucketVersioning - description: enables bucket versioning support. - arguments: - bucket: - type: - type: nullable - underlying_type: - type: named - name: String - clientId: - type: - type: nullable - underlying_type: - type: named - name: StorageClientID - result_type: - type: named - name: Boolean - name: removeIncompleteStorageUpload description: removes a partially uploaded object. arguments: @@ -2649,23 +2466,6 @@ definition: result_type: type: named name: Boolean - - name: removeStorageBucketReplication - arguments: - bucket: - type: - type: nullable - underlying_type: - type: named - name: String - clientId: - type: - type: nullable - underlying_type: - type: named - name: StorageClientID - result_type: - type: named - name: Boolean - name: removeStorageObject description: removes an object with some specified options. arguments: @@ -2697,6 +2497,12 @@ definition: type: type: named name: String + softDelete: + type: + type: nullable + underlying_type: + type: named + name: Boolean versionId: type: type: nullable @@ -2708,7 +2514,7 @@ definition: type: nullable underlying_type: type: predicate - object_type_name: StorageObjectSimple + object_type_name: StorageObjectFilter result_type: type: named name: Boolean @@ -2754,38 +2560,14 @@ definition: type: nullable underlying_type: type: predicate - object_type_name: StorageObjectSimple + object_type_name: StorageObjectFilter result_type: type: array element_type: type: named name: RemoveStorageObjectError - - name: setStorageBucketEncryption - description: sets default encryption configuration on a bucket. - arguments: - bucket: - type: - type: nullable - underlying_type: - type: named - name: String - clientId: - type: - type: nullable - underlying_type: - type: named - name: StorageClientID - rules: - type: - type: array - element_type: - type: named - name: ServerSideEncryptionRule - result_type: - type: named - name: Boolean - - name: setStorageBucketLifecycle - description: sets lifecycle on bucket or an object prefix. + - name: restoreStorageObject + description: restore a soft-deleted object. arguments: bucket: type: @@ -2799,53 +2581,21 @@ definition: underlying_type: type: named name: StorageClientID - rules: - type: - type: array - element_type: - type: named - name: BucketLifecycleRule - result_type: - type: named - name: Boolean - - name: setStorageBucketNotification - description: sets a new notification configuration on a bucket. - arguments: - bucket: + object: type: - type: nullable - underlying_type: - type: named - name: String - clientId: + type: named + name: String + where: type: type: nullable underlying_type: - type: named - name: StorageClientID - cloudFunctionConfigurations: - type: - type: array - element_type: - type: named - name: NotificationLambdaConfig - queueConfigurations: - type: - type: array - element_type: - type: named - name: NotificationQueueConfig - topicConfigurations: - type: - type: array - element_type: - type: named - name: NotificationTopicConfig + type: predicate + object_type_name: StorageObjectFilter result_type: type: named name: Boolean - - name: setStorageBucketReplication - description: sets replication configuration on a bucket. Role can be obtained by first defining the replication target on MinIO to associate the source and destination buckets for replication with the replication endpoint. + - name: updateStorageBucket + description: updates the bucket's configuration. arguments: bucket: type: @@ -2859,120 +2609,41 @@ definition: underlying_type: type: named name: StorageClientID - role: + encryption: type: type: nullable underlying_type: type: named - name: String - rules: - type: - type: array - element_type: - type: named - name: StorageReplicationRule - result_type: - type: named - name: Boolean - - name: setStorageBucketTags - description: sets tags to a bucket. - arguments: - bucket: + name: ServerSideEncryptionConfiguration + lifecycle: type: type: nullable underlying_type: type: named - name: String - clientId: + name: ObjectLifecycleConfiguration + objectLock: type: type: nullable underlying_type: type: named - name: StorageClientID + name: SetStorageObjectLockConfig tags: - type: - type: named - name: JSON - result_type: - type: named - name: Boolean - - name: setStorageObjectLegalHold - description: applies legal-hold onto an object. - arguments: - bucket: - type: - type: nullable - underlying_type: - type: named - name: String - clientId: type: type: nullable underlying_type: type: named - name: StorageClientID - object: - type: - type: named - name: String - status: + name: JSON + versioningEnabled: type: type: nullable underlying_type: type: named name: Boolean - versionId: - type: - type: nullable - underlying_type: - type: named - name: String - where: - type: - type: nullable - underlying_type: - type: predicate - object_type_name: StorageObjectSimple - result_type: - type: named - name: Boolean - - name: setStorageObjectLockConfig - arguments: - bucket: - type: - type: nullable - underlying_type: - type: named - name: String - clientId: - type: - type: nullable - underlying_type: - type: named - name: StorageClientID - mode: - type: - type: nullable - underlying_type: - type: named - name: StorageRetentionMode - unit: - type: - type: nullable - underlying_type: - type: named - name: StorageRetentionValidityUnit - validity: - type: - type: nullable - underlying_type: - type: named - name: Int32 result_type: type: named name: Boolean - - name: setStorageObjectRetention - description: applies object retention lock onto an object. + - name: updateStorageObject + description: updates the object's configuration. arguments: bucket: type: @@ -2986,66 +2657,34 @@ definition: underlying_type: type: named name: StorageClientID - governanceBypass: + legalHold: type: type: nullable underlying_type: type: named name: Boolean - mode: + metadata: type: type: nullable underlying_type: type: named - name: StorageRetentionMode + name: JSON object: type: type: named name: String - retainUntilDate: - type: - type: nullable - underlying_type: - type: named - name: TimestampTZ - versionId: - type: - type: nullable - underlying_type: - type: named - name: String - where: - type: - type: nullable - underlying_type: - type: predicate - object_type_name: StorageObjectSimple - result_type: - type: named - name: Boolean - - name: setStorageObjectTags - description: sets new object Tags to the given object, replaces/overwrites any existing tags. - arguments: - bucket: + retention: type: type: nullable underlying_type: type: named - name: String - clientId: + name: SetStorageObjectRetentionOptions + tags: type: type: nullable underlying_type: type: named - name: StorageClientID - object: - type: - type: named - name: String - tags: - type: - type: named - name: JSON + name: JSON versionId: type: type: nullable @@ -3057,25 +2696,7 @@ definition: type: nullable underlying_type: type: predicate - object_type_name: StorageObjectSimple - result_type: - type: named - name: Boolean - - name: suspendStorageBucketVersioning - description: disables bucket versioning support. - arguments: - bucket: - type: - type: nullable - underlying_type: - type: named - name: String - clientId: - type: - type: nullable - underlying_type: - type: named - name: StorageClientID + object_type_name: StorageObjectFilter result_type: type: named name: Boolean @@ -3113,7 +2734,7 @@ definition: type: nullable underlying_type: type: predicate - object_type_name: StorageObjectSimple + object_type_name: StorageObjectFilter result_type: type: named name: StorageUploadInfo @@ -3151,7 +2772,7 @@ definition: type: nullable underlying_type: type: predicate - object_type_name: StorageObjectSimple + object_type_name: StorageObjectFilter result_type: type: named name: StorageUploadInfo