Skip to content

Commit

Permalink
feat: export/import for dataset folders
Browse files Browse the repository at this point in the history
  • Loading branch information
@betodealmeida
betodealmeida committed Mar 6, 2025
1 parent 3811473 commit 7756d4f
Show file tree
Hide file tree
Showing 4 changed files with 236 additions and 0 deletions.
1 change: 1 addition & 0 deletions superset/connectors/sqla/models.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1219,6 +1219,7 @@ class SqlaTable(
"extra",
"normalize_columns",
"always_filter_main_dttm",
"folders",
]
update_from_object_fields = [f for f in export_fields if f != "database_id"]
export_parent = "database"
Expand Down
2 changes: 2 additions & 0 deletions tests/integration_tests/datasets/commands_tests.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -171,6 +171,7 @@ def test_export_dataset_command(self, mock_g):
"warning_text": None,
},
],
"folders": None,
"normalize_columns": False,
"always_filter_main_dttm": False,
"offset": 0,
Expand Down Expand Up @@ -235,6 +236,7 @@ def test_export_dataset_command_key_order(self, mock_g):
"extra",
"normalize_columns",
"always_filter_main_dttm",
"folders",
"uuid",
"metrics",
"columns",
Expand Down
67 changes: 67 additions & 0 deletions tests/unit_tests/datasets/commands/export_test.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@
# under the License.
# pylint: disable=import-outside-toplevel, unused-argument, unused-import

from uuid import UUID

from sqlalchemy.orm.session import Session

from superset import db
Expand Down Expand Up @@ -47,20 +49,62 @@ def test_export(session: Session) -> None:
type="INTEGER",
expression="revenue-expenses",
extra=json.dumps({"certified_by": "User"}),
uuid=UUID("00000000-0000-0000-0000-000000000005"),
),
]
metrics = [
SqlMetric(
metric_name="cnt",
expression="COUNT(*)",
extra=json.dumps({"warning_markdown": None}),
uuid=UUID("00000000-0000-0000-0000-000000000004"),
),
]

sqla_table = SqlaTable(
table_name="my_table",
columns=columns,
metrics=metrics,
folders=[
{
"uuid": "00000000-0000-0000-0000-000000000000",
"type": "folder",
"name": "Engineering",
"children": [
{
"uuid": "00000000-0000-0000-0000-000000000001",
"type": "folder",
"name": "Core",
"children": [
{
"uuid": "00000000-0000-0000-0000-000000000004",
"type": "metric",
"name": "cnt",
},
],
},
],
},
{
"uuid": "00000000-0000-0000-0000-000000000002",
"type": "folder",
"name": "Sales",
"children": [
{
"uuid": "00000000-0000-0000-0000-000000000003",
"type": "folder",
"name": "Core",
"children": [
{
"uuid": "00000000-0000-0000-0000-000000000005",
"type": "column",
"name": "profit",
},
],
},
],
},
],
main_dttm_col="ds",
database=database,
offset=-8,
Expand Down Expand Up @@ -126,6 +170,29 @@ def test_export(session: Session) -> None:
warning_markdown: '*WARNING*'
normalize_columns: false
always_filter_main_dttm: false
folders:
- uuid: 00000000-0000-0000-0000-000000000000
type: folder
name: Engineering
children:
- uuid: 00000000-0000-0000-0000-000000000001
type: folder
name: Core
children:
- uuid: 00000000-0000-0000-0000-000000000004
type: metric
name: cnt
- uuid: 00000000-0000-0000-0000-000000000002
type: folder
name: Sales
children:
- uuid: 00000000-0000-0000-0000-000000000003
type: folder
name: Core
children:
- uuid: 00000000-0000-0000-0000-000000000005
type: column
name: profit
uuid: {payload["uuid"]}
metrics:
- metric_name: cnt
Expand Down
166 changes: 166 additions & 0 deletions tests/unit_tests/datasets/commands/importers/v1/import_test.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,7 @@ def test_import_dataset(mocker: MockerFixture, session: Session) -> None:
"d3format": None,
"extra": {"warning_markdown": None},
"warning_text": None,
"uuid": "00000000-0000-0000-0000-000000000001",
}
],
"columns": [
Expand All @@ -103,8 +104,49 @@ def test_import_dataset(mocker: MockerFixture, session: Session) -> None:
"extra": {
"certified_by": "User",
},
"uuid": "00000000-0000-0000-0000-000000000002",
}
],
"folders": [
{
"uuid": "00000000-0000-0000-0000-000000000000",
"type": "folder",
"name": "Engineering",
"children": [
{
"uuid": "00000000-0000-0000-0000-000000000001",
"type": "folder",
"name": "Core",
"children": [
{
"uuid": "00000000-0000-0000-0000-000000000004",
"type": "metric",
"name": "cnt",
},
],
},
],
},
{
"uuid": "00000000-0000-0000-0000-000000000002",
"type": "folder",
"name": "Sales",
"children": [
{
"uuid": "00000000-0000-0000-0000-000000000003",
"type": "folder",
"name": "Core",
"children": [
{
"uuid": "00000000-0000-0000-0000-000000000005",
"type": "column",
"name": "profit",
},
],
},
],
},
],
"database_uuid": database.uuid,
"database_id": database.id,
}
Expand Down Expand Up @@ -136,6 +178,9 @@ def test_import_dataset(mocker: MockerFixture, session: Session) -> None:
assert sqla_table.metrics[0].d3format is None
assert sqla_table.metrics[0].extra == '{"warning_markdown": null}'
assert sqla_table.metrics[0].warning_text is None
assert sqla_table.metrics[0].uuid == uuid.UUID(
"00000000-0000-0000-0000-000000000001"
)
assert len(sqla_table.columns) == 1
assert sqla_table.columns[0].column_name == "profit"
assert sqla_table.columns[0].verbose_name is None
Expand All @@ -148,10 +193,131 @@ def test_import_dataset(mocker: MockerFixture, session: Session) -> None:
assert sqla_table.columns[0].description is None
assert sqla_table.columns[0].python_date_format is None
assert sqla_table.columns[0].extra == '{"certified_by": "User"}'
assert sqla_table.columns[0].uuid == uuid.UUID(
"00000000-0000-0000-0000-000000000002"
)
assert sqla_table.folders == [
{
"uuid": "00000000-0000-0000-0000-000000000000",
"type": "folder",
"name": "Engineering",
"children": [
{
"uuid": "00000000-0000-0000-0000-000000000001",
"type": "folder",
"name": "Core",
"children": [
{
"uuid": "00000000-0000-0000-0000-000000000004",
"type": "metric",
"name": "cnt",
},
],
},
],
},
{
"uuid": "00000000-0000-0000-0000-000000000002",
"type": "folder",
"name": "Sales",
"children": [
{
"uuid": "00000000-0000-0000-0000-000000000003",
"type": "folder",
"name": "Core",
"children": [
{
"uuid": "00000000-0000-0000-0000-000000000005",
"type": "column",
"name": "profit",
},
],
},
],
},
]
assert sqla_table.database.uuid == database.uuid
assert sqla_table.database.id == database.id


def test_import_dataset_no_folder(mocker: MockerFixture, session: Session) -> None:
"""
Test importing a dataset that was exported without folders.
"""
from superset import security_manager
from superset.commands.dataset.importers.v1.utils import import_dataset
from superset.connectors.sqla.models import SqlaTable
from superset.models.core import Database

mocker.patch.object(security_manager, "can_access", return_value=True)

engine = db.session.get_bind()
SqlaTable.metadata.create_all(engine) # pylint: disable=no-member

database = Database(database_name="my_database", sqlalchemy_uri="sqlite://")
db.session.add(database)
db.session.flush()

dataset_uuid = uuid.uuid4()
config = {
"table_name": "my_table",
"main_dttm_col": "ds",
"description": "This is the description",
"default_endpoint": None,
"offset": -8,
"cache_timeout": 3600,
"catalog": "public",
"schema": "my_schema",
"sql": None,
"params": {
"remote_id": 64,
"database_name": "examples",
"import_time": 1606677834,
},
"template_params": {
"answer": "42",
},
"filter_select_enabled": True,
"fetch_values_predicate": "foo IN (1, 2)",
"extra": {"warning_markdown": "*WARNING*"},
"uuid": dataset_uuid,
"metrics": [
{
"metric_name": "cnt",
"verbose_name": None,
"metric_type": None,
"expression": "COUNT(*)",
"description": None,
"d3format": None,
"extra": {"warning_markdown": None},
"warning_text": None,
}
],
"columns": [
{
"column_name": "profit",
"verbose_name": None,
"is_dttm": None,
"is_active": None,
"type": "INTEGER",
"groupby": None,
"filterable": None,
"expression": "revenue-expenses",
"description": None,
"python_date_format": None,
"extra": {
"certified_by": "User",
},
}
],
"database_uuid": database.uuid,
"database_id": database.id,
}

sqla_table = import_dataset(config)
assert sqla_table.folders is None


def test_import_dataset_duplicate_column(
mocker: MockerFixture, session: Session
) -> None:
Expand Down

0 comments on commit 7756d4f

Please sign in to comment.