chore: update fixtures scope that are loading data into the analytical db (#17780)

* Update db setup fixture scope

* Load data into db only once

* Update fixture scopes for data loading

* cleanup imports

* try import

* Update scope

* fix cleanup

* Import all fixtures

* Separate data log and metadata creation

* Retain table description

* fix dtype

Co-authored-by: Bogdan Kyryliuk <bogdankyryliuk@dropbox.com>
This commit is contained in:
Bogdan
2021-12-16 16:11:47 -08:00
committed by GitHub
parent d9acb280c4
commit afd2e9f2ca
43 changed files with 247 additions and 138 deletions

View File

@@ -29,35 +29,30 @@ from superset.models.slice import Slice
from superset.utils.core import get_example_default_schema
def create_table_for_dashboard(
df: DataFrame,
def get_table(
table_name: str, database: Database, schema: Optional[str] = None,
):
schema = schema or get_example_default_schema()
table_source = ConnectorRegistry.sources["table"]
return (
db.session.query(table_source)
.filter_by(database_id=database.id, schema=schema, table_name=table_name)
.one_or_none()
)
def create_table_metadata(
table_name: str,
database: Database,
dtype: Dict[str, Any],
table_description: str = "",
fetch_values_predicate: Optional[str] = None,
schema: Optional[str] = None,
) -> SqlaTable:
schema = schema or get_example_default_schema()
df.to_sql(
table_name,
database.get_sqla_engine(),
if_exists="replace",
chunksize=500,
dtype=dtype,
index=False,
method="multi",
schema=schema,
)
table_source = ConnectorRegistry.sources["table"]
table = (
db.session.query(table_source)
.filter_by(database_id=database.id, schema=schema, table_name=table_name)
.one_or_none()
)
table = get_table(table_name, database, schema)
if not table:
table_source = ConnectorRegistry.sources["table"]
table = table_source(schema=schema, table_name=table_name)
if fetch_values_predicate:
table.fetch_values_predicate = fetch_values_predicate