mirror of
https://github.com/apache/superset.git
synced 2026-05-12 19:35:17 +00:00
feat: migrate examples from Python to YAML format with enhanced CLI
Migrates Superset's example data system from Python-based scripts to YAML configuration files, providing a cleaner, more maintainable approach to managing example datasets, charts, and dashboards. - Converted 9 Python example modules to YAML configurations - Exported existing examples from database and added as YAML files: - 11 dashboards (USA Births Names, World Bank's Data, etc.) - 115 charts - 25 datasets - Moved test-specific fixtures to `tests/fixtures/examples/` - Removed theme_id from dashboard exports for compatibility - **New command group**: `superset examples` with subcommands: - `load` - Load example data (replaces `load-examples`) - `clear-old` - Remove old Python-based examples - `clear` - Placeholder for future YAML clearing - `reload` - Clear and reload in one command - **Backwards compatibility**: `superset load-examples` still works with deprecation warning - **Safety mechanism**: Detects old examples and preserves them to avoid data loss - Fixed JSON data loading - examples can now load `.json.gz` files from CDN - Fixed Docker compose configuration for isolated development - Fixed webpack WebSocket configuration for different ports - Import operations now log what's being created vs updated: - "Creating new dashboard: Sales Dashboard" - "Updating existing chart: World's Population" - Provides clear visibility into the import process - Moved import logging to individual import functions (DRY principle) - Non-destructive migration approach - no user data is deleted - Deterministic UUID generation for consistent example data - Tested migration from old Python examples to new YAML format - Verified safety mechanism prevents accidental data overwrites - Confirmed backwards compatibility with deprecated command - All pre-commit checks pass - Updated installation docs to use new CLI commands - Added deprecation notice to UPDATING.md - Updated development documentation None - the old `load-examples` command continues to work with a deprecation warning. For users with existing Python-based examples: 1. Run `superset examples clear-old --confirm` to remove old examples 2. Run `superset examples load` to load new YAML-based examples
This commit is contained in:
78
tests/fixtures/examples/big_data.py
vendored
Normal file
78
tests/fixtures/examples/big_data.py
vendored
Normal file
@@ -0,0 +1,78 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
import logging
|
||||
import random
|
||||
import string
|
||||
|
||||
import sqlalchemy.sql.sqltypes
|
||||
|
||||
from superset.utils.mock_data import add_data, ColumnInfo
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
COLUMN_TYPES = [
|
||||
sqlalchemy.sql.sqltypes.INTEGER(),
|
||||
sqlalchemy.sql.sqltypes.VARCHAR(length=255),
|
||||
sqlalchemy.sql.sqltypes.TEXT(),
|
||||
sqlalchemy.sql.sqltypes.BOOLEAN(),
|
||||
sqlalchemy.sql.sqltypes.FLOAT(),
|
||||
sqlalchemy.sql.sqltypes.DATE(),
|
||||
sqlalchemy.sql.sqltypes.TIME(),
|
||||
sqlalchemy.sql.sqltypes.TIMESTAMP(),
|
||||
]
|
||||
|
||||
|
||||
def load_big_data() -> None:
|
||||
logger.debug("Creating table `wide_table` with 100 columns")
|
||||
columns: list[ColumnInfo] = []
|
||||
for i in range(100):
|
||||
column: ColumnInfo = {
|
||||
"name": f"col{i}",
|
||||
"type": COLUMN_TYPES[i % len(COLUMN_TYPES)],
|
||||
"nullable": False,
|
||||
"default": None,
|
||||
"autoincrement": "auto",
|
||||
"primary_key": 1 if i == 0 else 0,
|
||||
}
|
||||
columns.append(column)
|
||||
add_data(columns=columns, num_rows=1000, table_name="wide_table")
|
||||
|
||||
logger.debug("Creating 1000 small tables")
|
||||
columns = [
|
||||
{
|
||||
"name": "id",
|
||||
"type": sqlalchemy.sql.sqltypes.INTEGER(),
|
||||
"nullable": False,
|
||||
"default": None,
|
||||
"autoincrement": "auto",
|
||||
"primary_key": 1,
|
||||
},
|
||||
{
|
||||
"name": "value",
|
||||
"type": sqlalchemy.sql.sqltypes.VARCHAR(length=255),
|
||||
"nullable": False,
|
||||
"default": None,
|
||||
"autoincrement": "auto",
|
||||
"primary_key": 0,
|
||||
},
|
||||
]
|
||||
for i in range(1000):
|
||||
add_data(columns=columns, num_rows=10, table_name=f"small_table_{i}")
|
||||
|
||||
logger.debug("Creating table with long name")
|
||||
name = "".join(random.choices(string.ascii_letters + string.digits, k=60)) # noqa: S311
|
||||
add_data(columns=columns, num_rows=10, table_name=name)
|
||||
Reference in New Issue
Block a user