Skip to content

Commit

Permalink
add get fileset context api in python client
Browse files Browse the repository at this point in the history
  • Loading branch information
xiaojiebao committed Aug 5, 2024
1 parent 2608f5a commit ebf6ed7
Show file tree
Hide file tree
Showing 13 changed files with 509 additions and 1 deletion.
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,18 @@ public enum FilesetDataOperation {
GET_DEFAULT_BLOCK_SIZE,
/** This data operation means that set current working directory. */
SET_WORKING_DIR,
/** This data operation means that check a file or a directory exists. */
EXISTS,
/** This data operation means that get the created time of a file. */
CREATED_TIME,
/** This data operation means that get the modified time of a file. */
MODIFIED_TIME,
/** This data operation means that copy a file. */
COPY_FILE,
/** This data operation means that get the content of a file. */
CAT_FILE,
/** This data operation means that copy a remote file to local. */
GET_FILE,
/** This data operation means that it is an unknown data operation. */
UNKNOWN;
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.

from gravitino.api.client_type import ClientType
from gravitino.api.fileset_data_operation import FilesetDataOperation
from gravitino.api.fileset_data_operation_ctx import FilesetDataOperationCtx


class BaseFilesetDataOperationCtx(FilesetDataOperationCtx):
"""Base implementation of FilesetDataOperationCtx."""

_sub_path: str
_operation: FilesetDataOperation
_client_type: ClientType

def __init__(
self, sub_path: str, operation: FilesetDataOperation, client_type: ClientType
):
assert sub_path is not None
assert operation is not None
assert client_type is not None
self._sub_path = sub_path
self._operation = operation
self._client_type = client_type

def sub_path(self) -> str:
return self._sub_path

def operation(self) -> FilesetDataOperation:
return self._operation

def client_type(self) -> ClientType:
return self._client_type
25 changes: 25 additions & 0 deletions clients/client-python/gravitino/api/client_type.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.

from enum import Enum


class ClientType(Enum):
"""An enum class containing fileset data operations client type that supported."""

PYTHON_GVFS = 1
UNKNOWN = 2
44 changes: 44 additions & 0 deletions clients/client-python/gravitino/api/fileset_context.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.

from abc import ABC, abstractmethod

from gravitino.api.fileset import Fileset


class FilesetContext(ABC):
"""An interface representing a fileset context with an existing fileset. This
interface defines some contextual information related to Fileset that can be passed.
"""

@abstractmethod
def fileset(self) -> Fileset:
"""The fileset object.
Returns:
the fileset object.
"""
pass

@abstractmethod
def actual_path(self) -> str:
"""The actual storage path after processing.
Returns:
the actual storage path after processing.
"""
pass
38 changes: 38 additions & 0 deletions clients/client-python/gravitino/api/fileset_data_operation.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.

from enum import Enum


class FilesetDataOperation(Enum):
"""An enum class containing fileset data operations that supported."""

LIST_STATUS = 1
GET_FILE_STATUS = 2
EXISTS = 3
RENAME = 4
APPEND = 5
CREATE = 6
DELETE = 7
OPEN = 8
MKDIRS = 9
CREATED_TIME = 10
MODIFIED_TIME = 11
COPY_FILE = 12
CAT_FILE = 13
GET_FILE = 14
UNKNOWN = 15
54 changes: 54 additions & 0 deletions clients/client-python/gravitino/api/fileset_data_operation_ctx.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.

from abc import ABC, abstractmethod

from gravitino.api.client_type import ClientType
from gravitino.api.fileset_data_operation import FilesetDataOperation


class FilesetDataOperationCtx(ABC):
"""An interface representing a fileset data operation context. This interface defines some
information need to report to the server.
"""

@abstractmethod
def sub_path(self) -> str:
"""The sub path which is operated by the data operation.
Returns:
the sub path which is operated by the data operation.
"""
pass

@abstractmethod
def operation(self) -> FilesetDataOperation:
"""The data operation type.
Returns:
the data operation type.
"""
pass

@abstractmethod
def client_type(self) -> ClientType:
"""The client type of the data operation.
Returns:
the client type of the data operation.
"""
pass
34 changes: 34 additions & 0 deletions clients/client-python/gravitino/catalog/fileset_catalog.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,13 +23,17 @@
from gravitino.api.catalog import Catalog
from gravitino.api.fileset import Fileset
from gravitino.api.fileset_change import FilesetChange
from gravitino.api.fileset_context import FilesetContext
from gravitino.api.fileset_data_operation_ctx import FilesetDataOperationCtx
from gravitino.catalog.base_schema_catalog import BaseSchemaCatalog
from gravitino.dto.audit_dto import AuditDTO
from gravitino.dto.requests.fileset_create_request import FilesetCreateRequest
from gravitino.dto.requests.fileset_update_request import FilesetUpdateRequest
from gravitino.dto.requests.fileset_updates_request import FilesetUpdatesRequest
from gravitino.dto.requests.get_fileset_context_request import GetFilesetContextRequest
from gravitino.dto.responses.drop_response import DropResponse
from gravitino.dto.responses.entity_list_response import EntityListResponse
from gravitino.dto.responses.fileset_context_response import FilesetContextResponse
from gravitino.dto.responses.fileset_response import FilesetResponse
from gravitino.name_identifier import NameIdentifier
from gravitino.namespace import Namespace
Expand Down Expand Up @@ -240,6 +244,36 @@ def drop_fileset(self, ident: NameIdentifier) -> bool:
logger.warning("Failed to drop fileset %s: %s", ident, e)
return False

def get_fileset_context(
self, ident: NameIdentifier, ctx: FilesetDataOperationCtx
) -> FilesetContext:
"""Get a fileset context from the catalog.
Args:
ident: A fileset identifier.
ctx: The data operation context.
Returns:
the fileset context.
"""
self.check_fileset_name_identifier(ident)
req = GetFilesetContextRequest(
ctx.sub_path(), ctx.operation(), ctx.client_type()
)
req.validate()

full_namespace = self._get_fileset_full_namespace(ident.namespace())

resp = self.rest_client.post(
f"{self.format_fileset_request_path(full_namespace)}/{ident.name()}/context",
req,
error_handler=FILESET_ERROR_HANDLER,
)
context_resp = FilesetContextResponse.from_json(resp.body, infer_missing=True)
context_resp.validate()

return context_resp.context()

@staticmethod
def check_fileset_namespace(namespace: Namespace):
Namespace.check(
Expand Down
38 changes: 38 additions & 0 deletions clients/client-python/gravitino/dto/fileset_context_dto.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.

from abc import ABC
from dataclasses import dataclass, field

from dataclasses_json import config, DataClassJsonMixin

from gravitino.api.fileset_context import FilesetContext
from gravitino.dto.fileset_dto import FilesetDTO


@dataclass
class FilesetContextDTO(FilesetContext, DataClassJsonMixin, ABC):
"""Represents a Fileset Context DTO (Data Transfer Object)."""

_fileset: FilesetDTO = field(metadata=config(field_name="fileset"))
_actual_path: str = field(metadata=config(field_name="actualPath"))

def fileset(self) -> FilesetDTO:
return self._fileset

def actual_path(self) -> str:
return self._actual_path
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.

from dataclasses import dataclass, field
from dataclasses_json import config

from gravitino.api.client_type import ClientType
from gravitino.api.fileset_data_operation import FilesetDataOperation
from gravitino.rest.rest_message import RESTRequest


@dataclass
class GetFilesetContextRequest(RESTRequest):
"""Request to represent to get a fileset context."""

_sub_path: str = field(metadata=config(field_name="subPath"))
_operation: FilesetDataOperation = field(metadata=config(field_name="operation"))
_client_type: ClientType = field(metadata=config(field_name="clientType"))

def __init__(
self, sub_path: str, operation: FilesetDataOperation, client_type: ClientType
):
self._sub_path = sub_path
self._operation = operation
self._client_type = client_type
self.validate()

def validate(self):
assert self._sub_path is not None, "subPath is required"
assert self._operation is not None, "operation is required"
assert self._client_type is not None, "clientType is required"
Loading

0 comments on commit ebf6ed7

Please sign in to comment.