From 398a01f7dc0579599b927ffa1af545bff1e5d2ba Mon Sep 17 00:00:00 2001 From: John Bodley <4567245+john-bodley@users.noreply.github.com> Date: Tue, 12 Oct 2021 21:40:48 -0700 Subject: [PATCH] =?UTF-8?q?chore(engine):=20Translate=20fractional=20time?= =?UTF-8?q?=20grains=E2=80=94requires=20@superset-ui=20bump=20(#17078)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore(engine): Translate fractional time grains * Bump @superset-ui Co-authored-by: John Bodley --- docs/src/resources/openapi.json | 8 +-- .../explore/visualizations/table.test.ts | 6 +- superset/db_engine_specs/ascend.py | 2 +- superset/db_engine_specs/athena.py | 2 +- superset/db_engine_specs/base.py | 4 +- superset/db_engine_specs/bigquery.py | 4 +- superset/db_engine_specs/clickhouse.py | 4 +- superset/db_engine_specs/crate.py | 2 +- superset/db_engine_specs/db2.py | 2 +- superset/db_engine_specs/dremio.py | 2 +- superset/db_engine_specs/drill.py | 4 +- superset/db_engine_specs/druid.py | 14 ++-- superset/db_engine_specs/exasol.py | 2 +- superset/db_engine_specs/firebolt.py | 2 +- superset/db_engine_specs/hana.py | 2 +- superset/db_engine_specs/hive.py | 2 +- superset/db_engine_specs/impala.py | 2 +- superset/db_engine_specs/kylin.py | 2 +- superset/db_engine_specs/mssql.py | 4 +- superset/db_engine_specs/mysql.py | 2 +- superset/db_engine_specs/netezza.py | 2 +- superset/db_engine_specs/oracle.py | 2 +- superset/db_engine_specs/pinot.py | 4 +- superset/db_engine_specs/postgres.py | 4 +- superset/db_engine_specs/presto.py | 2 +- superset/db_engine_specs/rockset.py | 2 +- superset/db_engine_specs/snowflake.py | 4 +- superset/db_engine_specs/sqlite.py | 2 +- superset/db_engine_specs/teradata.py | 2 +- superset/db_engine_specs/trino.py | 2 +- .../configs/charts/Quarterly_Sales.yaml | 2 +- .../Quarterly_Sales_By_Product_Line.yaml | 2 +- superset/examples/utils.py | 2 +- .../32646df09c64_update_time_grain_sqla.py | 72 +++++++++++++++++++ superset/utils/pandas_postprocessing.py | 4 +- .../db_engine_specs/base_engine_spec_tests.py | 6 +- 36 files changed, 129 insertions(+), 57 deletions(-) create mode 100644 superset/migrations/versions/32646df09c64_update_time_grain_sqla.py diff --git a/docs/src/resources/openapi.json b/docs/src/resources/openapi.json index fa21b5ff3a8..cdd51639054 100644 --- a/docs/src/resources/openapi.json +++ b/docs/src/resources/openapi.json @@ -730,13 +730,13 @@ "PT5M", "PT10M", "PT15M", - "PT0.5H", + "PT30M", "PT1H", "PT6H", "P1D", "P1W", "P1M", - "P0.25Y", + "P3M", "P1Y", "1969-12-28T00:00:00Z/P1W", "1969-12-29T00:00:00Z/P1W", @@ -998,13 +998,13 @@ "PT5M", "PT10M", "PT15M", - "PT0.5H", + "PT30M", "PT1H", "PT6H", "P1D", "P1W", "P1M", - "P0.25Y", + "P3M", "P1Y", "1969-12-28T00:00:00Z/P1W", "1969-12-29T00:00:00Z/P1W", diff --git a/superset-frontend/cypress-base/cypress/integration/explore/visualizations/table.test.ts b/superset-frontend/cypress-base/cypress/integration/explore/visualizations/table.test.ts index 2a4258b77c3..6361d93d180 100644 --- a/superset-frontend/cypress-base/cypress/integration/explore/visualizations/table.test.ts +++ b/superset-frontend/cypress-base/cypress/integration/explore/visualizations/table.test.ts @@ -62,7 +62,7 @@ describe('Visualization > Table', () => { ...VIZ_DEFAULTS, include_time: true, granularity_sqla: 'ds', - time_grain_sqla: 'P0.25Y', + time_grain_sqla: 'P3M', metrics: [NUM_METRIC, MAX_DS, MAX_STATE], }); // when format with smart_date, time column use format by granularity @@ -77,7 +77,7 @@ describe('Visualization > Table', () => { ...VIZ_DEFAULTS, include_time: true, granularity_sqla: 'ds', - time_grain_sqla: 'P0.25Y', + time_grain_sqla: 'P3M', table_timestamp_format: '%Y-%m-%d %H:%M', metrics: [NUM_METRIC, MAX_DS, MAX_STATE], }); @@ -111,7 +111,7 @@ describe('Visualization > Table', () => { ...VIZ_DEFAULTS, include_time: true, granularity_sqla: 'ds', - time_grain_sqla: 'P0.25Y', + time_grain_sqla: 'P3M', metrics: [NUM_METRIC, MAX_DS], groupby: ['name'], }); diff --git a/superset/db_engine_specs/ascend.py b/superset/db_engine_specs/ascend.py index ee7bab9facd..759b78ad1c9 100644 --- a/superset/db_engine_specs/ascend.py +++ b/superset/db_engine_specs/ascend.py @@ -35,6 +35,6 @@ class AscendEngineSpec(ImpalaEngineSpec): "P1D": "DATE_TRUNC('day', {col})", "P1W": "DATE_TRUNC('week', {col})", "P1M": "DATE_TRUNC('month', {col})", - "P0.25Y": "DATE_TRUNC('quarter', {col})", + "P3M": "DATE_TRUNC('quarter', {col})", "P1Y": "DATE_TRUNC('year', {col})", } diff --git a/superset/db_engine_specs/athena.py b/superset/db_engine_specs/athena.py index 2952d826777..666049bac61 100644 --- a/superset/db_engine_specs/athena.py +++ b/superset/db_engine_specs/athena.py @@ -41,7 +41,7 @@ class AthenaEngineSpec(BaseEngineSpec): "P1D": "date_trunc('day', CAST({col} AS TIMESTAMP))", "P1W": "date_trunc('week', CAST({col} AS TIMESTAMP))", "P1M": "date_trunc('month', CAST({col} AS TIMESTAMP))", - "P0.25Y": "date_trunc('quarter', CAST({col} AS TIMESTAMP))", + "P3M": "date_trunc('quarter', CAST({col} AS TIMESTAMP))", "P1Y": "date_trunc('year', CAST({col} AS TIMESTAMP))", "P1W/1970-01-03T00:00:00Z": "date_add('day', 5, date_trunc('week', \ date_add('day', 1, CAST({col} AS TIMESTAMP))))", diff --git a/superset/db_engine_specs/base.py b/superset/db_engine_specs/base.py index 74b451ec2da..ed3851efe6c 100644 --- a/superset/db_engine_specs/base.py +++ b/superset/db_engine_specs/base.py @@ -91,13 +91,13 @@ builtin_time_grains: Dict[Optional[str], str] = { "PT5M": __("5 minute"), "PT10M": __("10 minute"), "PT15M": __("15 minute"), - "PT0.5H": __("Half hour"), + "PT30M": __("30 minute"), "PT1H": __("Hour"), "PT6H": __("6 hour"), "P1D": __("Day"), "P1W": __("Week"), "P1M": __("Month"), - "P0.25Y": __("Quarter"), + "P3M": __("Quarter"), "P1Y": __("Year"), "1969-12-28T00:00:00Z/P1W": __("Week starting Sunday"), "1969-12-29T00:00:00Z/P1W": __("Week starting Monday"), diff --git a/superset/db_engine_specs/bigquery.py b/superset/db_engine_specs/bigquery.py index a862df58d9f..5e1ff739cd2 100644 --- a/superset/db_engine_specs/bigquery.py +++ b/superset/db_engine_specs/bigquery.py @@ -131,14 +131,14 @@ class BigQueryEngineSpec(BaseEngineSpec): "PT15M": "CAST(TIMESTAMP_SECONDS(" "15*60 * DIV(UNIX_SECONDS(CAST({col} AS TIMESTAMP)), 15*60)" ") AS {type})", - "PT0.5H": "CAST(TIMESTAMP_SECONDS(" + "PT30M": "CAST(TIMESTAMP_SECONDS(" "30*60 * DIV(UNIX_SECONDS(CAST({col} AS TIMESTAMP)), 30*60)" ") AS {type})", "PT1H": "{func}({col}, HOUR)", "P1D": "{func}({col}, DAY)", "P1W": "{func}({col}, WEEK)", "P1M": "{func}({col}, MONTH)", - "P0.25Y": "{func}({col}, QUARTER)", + "P3M": "{func}({col}, QUARTER)", "P1Y": "{func}({col}, YEAR)", } diff --git a/superset/db_engine_specs/clickhouse.py b/superset/db_engine_specs/clickhouse.py index 60a3584baf4..d9c21882061 100644 --- a/superset/db_engine_specs/clickhouse.py +++ b/superset/db_engine_specs/clickhouse.py @@ -47,12 +47,12 @@ class ClickHouseEngineSpec(BaseEngineSpec): # pylint: disable=abstract-method "PT5M": "toDateTime(intDiv(toUInt32(toDateTime({col})), 300)*300)", "PT10M": "toDateTime(intDiv(toUInt32(toDateTime({col})), 600)*600)", "PT15M": "toDateTime(intDiv(toUInt32(toDateTime({col})), 900)*900)", - "PT0.5H": "toDateTime(intDiv(toUInt32(toDateTime({col})), 1800)*1800)", + "PT30M": "toDateTime(intDiv(toUInt32(toDateTime({col})), 1800)*1800)", "PT1H": "toStartOfHour(toDateTime({col}))", "P1D": "toStartOfDay(toDateTime({col}))", "P1W": "toMonday(toDateTime({col}))", "P1M": "toStartOfMonth(toDateTime({col}))", - "P0.25Y": "toStartOfQuarter(toDateTime({col}))", + "P3M": "toStartOfQuarter(toDateTime({col}))", "P1Y": "toStartOfYear(toDateTime({col}))", } diff --git a/superset/db_engine_specs/crate.py b/superset/db_engine_specs/crate.py index a55d72fd063..67d68c8349e 100644 --- a/superset/db_engine_specs/crate.py +++ b/superset/db_engine_specs/crate.py @@ -37,7 +37,7 @@ class CrateEngineSpec(BaseEngineSpec): "P1D": "DATE_TRUNC('day', {col})", "P1W": "DATE_TRUNC('week', {col})", "P1M": "DATE_TRUNC('month', {col})", - "P0.25Y": "DATE_TRUNC('quarter', {col})", + "P3M": "DATE_TRUNC('quarter', {col})", "P1Y": "DATE_TRUNC('year', {col})", } diff --git a/superset/db_engine_specs/db2.py b/superset/db_engine_specs/db2.py index 1179198bc43..9288a069277 100644 --- a/superset/db_engine_specs/db2.py +++ b/superset/db_engine_specs/db2.py @@ -42,7 +42,7 @@ class Db2EngineSpec(BaseEngineSpec): " - MICROSECOND({col}) MICROSECONDS", "P1W": "{col} - (DAYOFWEEK({col})) DAYS", "P1M": "{col} - (DAY({col})-1) DAYS", - "P0.25Y": "{col} - (DAY({col})-1) DAYS" + "P3M": "{col} - (DAY({col})-1) DAYS" " - (MONTH({col})-1) MONTHS" " + ((QUARTER({col})-1) * 3) MONTHS", "P1Y": "{col} - (DAY({col})-1) DAYS" " - (MONTH({col})-1) MONTHS", diff --git a/superset/db_engine_specs/dremio.py b/superset/db_engine_specs/dremio.py index a76909bf3a8..0d69c8a0f67 100644 --- a/superset/db_engine_specs/dremio.py +++ b/superset/db_engine_specs/dremio.py @@ -34,7 +34,7 @@ class DremioEngineSpec(BaseEngineSpec): "P1D": "DATE_TRUNC('day', {col})", "P1W": "DATE_TRUNC('week', {col})", "P1M": "DATE_TRUNC('month', {col})", - "P0.25Y": "DATE_TRUNC('quarter', {col})", + "P3M": "DATE_TRUNC('quarter', {col})", "P1Y": "DATE_TRUNC('year', {col})", } diff --git a/superset/db_engine_specs/drill.py b/superset/db_engine_specs/drill.py index 7e350b86d5d..5b61282ac49 100644 --- a/superset/db_engine_specs/drill.py +++ b/superset/db_engine_specs/drill.py @@ -36,12 +36,12 @@ class DrillEngineSpec(BaseEngineSpec): "PT1S": "NEARESTDATE({col}, 'SECOND')", "PT1M": "NEARESTDATE({col}, 'MINUTE')", "PT15M": "NEARESTDATE({col}, 'QUARTER_HOUR')", - "PT0.5H": "NEARESTDATE({col}, 'HALF_HOUR')", + "PT30M": "NEARESTDATE({col}, 'HALF_HOUR')", "PT1H": "NEARESTDATE({col}, 'HOUR')", "P1D": "NEARESTDATE({col}, 'DAY')", "P1W": "NEARESTDATE({col}, 'WEEK_SUNDAY')", "P1M": "NEARESTDATE({col}, 'MONTH')", - "P0.25Y": "NEARESTDATE({col}, 'QUARTER')", + "P3M": "NEARESTDATE({col}, 'QUARTER')", "P1Y": "NEARESTDATE({col}, 'YEAR')", } diff --git a/superset/db_engine_specs/druid.py b/superset/db_engine_specs/druid.py index 58545d4f8d7..525c1b78026 100644 --- a/superset/db_engine_specs/druid.py +++ b/superset/db_engine_specs/druid.py @@ -48,14 +48,14 @@ class DruidEngineSpec(BaseEngineSpec): "PT5M": "TIME_FLOOR({col}, 'PT5M')", "PT10M": "TIME_FLOOR({col}, 'PT10M')", "PT15M": "TIME_FLOOR({col}, 'PT15M')", - "PT0.5H": "TIME_FLOOR({col}, 'PT30M')", - "PT1H": "FLOOR({col} TO HOUR)", + "PT30M": "TIME_FLOOR({col}, 'PT30M')", + "PT1H": "TIME_FLOOR({col}, 'PT1H')", "PT6H": "TIME_FLOOR({col}, 'PT6H')", - "P1D": "FLOOR({col} TO DAY)", - "P1W": "FLOOR({col} TO WEEK)", - "P1M": "FLOOR({col} TO MONTH)", - "P0.25Y": "FLOOR({col} TO QUARTER)", - "P1Y": "FLOOR({col} TO YEAR)", + "P1D": "TIME_FLOOR({col}, 'P1D')", + "P1W": "TIME_FLOOR({col}, 'P1W')", + "P1M": "TIME_FLOOR({col}, 'P1M')", + "P3M": "TIME_FLOOR({col}, 'P3M')", + "P1Y": "TIME_FLOOR({col}, 'P1Y')", "P1W/1970-01-03T00:00:00Z": ( "TIMESTAMPADD(DAY, 5, FLOOR(TIMESTAMPADD(DAY, 1, {col}) TO WEEK))" ), diff --git a/superset/db_engine_specs/exasol.py b/superset/db_engine_specs/exasol.py index 327cc3adbbe..c06fbd826df 100644 --- a/superset/db_engine_specs/exasol.py +++ b/superset/db_engine_specs/exasol.py @@ -35,7 +35,7 @@ class ExasolEngineSpec(BaseEngineSpec): # pylint: disable=abstract-method "P1D": "DATE_TRUNC('day', {col})", "P1W": "DATE_TRUNC('week', {col})", "P1M": "DATE_TRUNC('month', {col})", - "P0.25Y": "DATE_TRUNC('quarter', {col})", + "P3M": "DATE_TRUNC('quarter', {col})", "P1Y": "DATE_TRUNC('year', {col})", } diff --git a/superset/db_engine_specs/firebolt.py b/superset/db_engine_specs/firebolt.py index ea5091f69ff..acef2f26b02 100644 --- a/superset/db_engine_specs/firebolt.py +++ b/superset/db_engine_specs/firebolt.py @@ -36,7 +36,7 @@ class FireboltEngineSpec(BaseEngineSpec): "P1D": "date_trunc('day', CAST({col} AS TIMESTAMP))", "P1W": "date_trunc('week', CAST({col} AS TIMESTAMP))", "P1M": "date_trunc('month', CAST({col} AS TIMESTAMP))", - "P0.25Y": "date_trunc('quarter', CAST({col} AS TIMESTAMP))", + "P3M": "date_trunc('quarter', CAST({col} AS TIMESTAMP))", "P1Y": "date_trunc('year', CAST({col} AS TIMESTAMP))", } diff --git a/superset/db_engine_specs/hana.py b/superset/db_engine_specs/hana.py index 11e49d1896a..b2247cf3253 100644 --- a/superset/db_engine_specs/hana.py +++ b/superset/db_engine_specs/hana.py @@ -36,7 +36,7 @@ class HanaEngineSpec(PostgresBaseEngineSpec): "PT1H": "TO_TIMESTAMP(SUBSTRING(TO_TIMESTAMP({col}),0,14) || '00:00')", "P1D": "TO_DATE({col})", "P1M": "TO_DATE(SUBSTRING(TO_DATE({col}),0,7)||'-01')", - "P0.25Y": "TO_DATE(SUBSTRING( \ + "P3M": "TO_DATE(SUBSTRING( \ TO_DATE({col}), 0, 5)|| LPAD(CAST((CAST(SUBSTRING(QUARTER( \ TO_DATE({col}), 1), 7, 1) as int)-1)*3 +1 as text),2,'0') ||'-01')", "P1Y": "TO_DATE(YEAR({col})||'-01-01')", diff --git a/superset/db_engine_specs/hive.py b/superset/db_engine_specs/hive.py index 9735289c7c0..bbfcbd1bb39 100644 --- a/superset/db_engine_specs/hive.py +++ b/superset/db_engine_specs/hive.py @@ -105,7 +105,7 @@ class HiveEngineSpec(PrestoEngineSpec): "P1D": "from_unixtime(unix_timestamp({col}), 'yyyy-MM-dd 00:00:00')", "P1W": "date_format(date_sub({col}, CAST(7-from_unixtime(unix_timestamp({col}),'u') as int)), 'yyyy-MM-dd 00:00:00')", "P1M": "from_unixtime(unix_timestamp({col}), 'yyyy-MM-01 00:00:00')", - "P0.25Y": "date_format(add_months(trunc({col}, 'MM'), -(month({col})-1)%3), 'yyyy-MM-dd 00:00:00')", + "P3M": "date_format(add_months(trunc({col}, 'MM'), -(month({col})-1)%3), 'yyyy-MM-dd 00:00:00')", "P1Y": "from_unixtime(unix_timestamp({col}), 'yyyy-01-01 00:00:00')", "P1W/1970-01-03T00:00:00Z": "date_format(date_add({col}, INT(6-from_unixtime(unix_timestamp({col}), 'u'))), 'yyyy-MM-dd 00:00:00')", "1969-12-28T00:00:00Z/P1W": "date_format(date_add({col}, -INT(from_unixtime(unix_timestamp({col}), 'u'))), 'yyyy-MM-dd 00:00:00')", diff --git a/superset/db_engine_specs/impala.py b/superset/db_engine_specs/impala.py index 9d8dc91374f..f391289ac1d 100644 --- a/superset/db_engine_specs/impala.py +++ b/superset/db_engine_specs/impala.py @@ -36,7 +36,7 @@ class ImpalaEngineSpec(BaseEngineSpec): "P1D": "TRUNC({col}, 'DD')", "P1W": "TRUNC({col}, 'WW')", "P1M": "TRUNC({col}, 'MONTH')", - "P0.25Y": "TRUNC({col}, 'Q')", + "P3M": "TRUNC({col}, 'Q')", "P1Y": "TRUNC({col}, 'YYYY')", } diff --git a/superset/db_engine_specs/kylin.py b/superset/db_engine_specs/kylin.py index ffa7f105b43..2e59e3100c0 100644 --- a/superset/db_engine_specs/kylin.py +++ b/superset/db_engine_specs/kylin.py @@ -35,7 +35,7 @@ class KylinEngineSpec(BaseEngineSpec): # pylint: disable=abstract-method "P1D": "CAST(FLOOR(CAST({col} AS TIMESTAMP) TO DAY) AS DATE)", "P1W": "CAST(FLOOR(CAST({col} AS TIMESTAMP) TO WEEK) AS DATE)", "P1M": "CAST(FLOOR(CAST({col} AS TIMESTAMP) TO MONTH) AS DATE)", - "P0.25Y": "CAST(FLOOR(CAST({col} AS TIMESTAMP) TO QUARTER) AS DATE)", + "P3M": "CAST(FLOOR(CAST({col} AS TIMESTAMP) TO QUARTER) AS DATE)", "P1Y": "CAST(FLOOR(CAST({col} AS TIMESTAMP) TO YEAR) AS DATE)", } diff --git a/superset/db_engine_specs/mssql.py b/superset/db_engine_specs/mssql.py index ff6b5326d39..0572319b2cf 100644 --- a/superset/db_engine_specs/mssql.py +++ b/superset/db_engine_specs/mssql.py @@ -55,12 +55,12 @@ class MssqlEngineSpec(BaseEngineSpec): "PT5M": "DATEADD(minute, DATEDIFF(minute, 0, {col}) / 5 * 5, 0)", "PT10M": "DATEADD(minute, DATEDIFF(minute, 0, {col}) / 10 * 10, 0)", "PT15M": "DATEADD(minute, DATEDIFF(minute, 0, {col}) / 15 * 15, 0)", - "PT0.5H": "DATEADD(minute, DATEDIFF(minute, 0, {col}) / 30 * 30, 0)", + "PT30M": "DATEADD(minute, DATEDIFF(minute, 0, {col}) / 30 * 30, 0)", "PT1H": "DATEADD(hour, DATEDIFF(hour, 0, {col}), 0)", "P1D": "DATEADD(day, DATEDIFF(day, 0, {col}), 0)", "P1W": "DATEADD(week, DATEDIFF(week, 0, {col}), 0)", "P1M": "DATEADD(month, DATEDIFF(month, 0, {col}), 0)", - "P0.25Y": "DATEADD(quarter, DATEDIFF(quarter, 0, {col}), 0)", + "P3M": "DATEADD(quarter, DATEDIFF(quarter, 0, {col}), 0)", "P1Y": "DATEADD(year, DATEDIFF(year, 0, {col}), 0)", } diff --git a/superset/db_engine_specs/mysql.py b/superset/db_engine_specs/mysql.py index 2fb1e9737bb..fbac330696b 100644 --- a/superset/db_engine_specs/mysql.py +++ b/superset/db_engine_specs/mysql.py @@ -109,7 +109,7 @@ class MySQLEngineSpec(BaseEngineSpec, BasicParametersMixin): "P1D": "DATE({col})", "P1W": "DATE(DATE_SUB({col}, " "INTERVAL DAYOFWEEK({col}) - 1 DAY))", "P1M": "DATE(DATE_SUB({col}, " "INTERVAL DAYOFMONTH({col}) - 1 DAY))", - "P0.25Y": "MAKEDATE(YEAR({col}), 1) " + "P3M": "MAKEDATE(YEAR({col}), 1) " "+ INTERVAL QUARTER({col}) QUARTER - INTERVAL 1 QUARTER", "P1Y": "DATE(DATE_SUB({col}, " "INTERVAL DAYOFYEAR({col}) - 1 DAY))", "1969-12-29T00:00:00Z/P1W": "DATE(DATE_SUB({col}, " diff --git a/superset/db_engine_specs/netezza.py b/superset/db_engine_specs/netezza.py index 469ac4fa198..9d3b5ec180c 100644 --- a/superset/db_engine_specs/netezza.py +++ b/superset/db_engine_specs/netezza.py @@ -31,7 +31,7 @@ class NetezzaEngineSpec(PostgresBaseEngineSpec): "P1D": "DATE_TRUNC('day', {col})", "P1W": "DATE_TRUNC('week', {col})", "P1M": "DATE_TRUNC('month', {col})", - "P0.25Y": "DATE_TRUNC('quarter', {col})", + "P3M": "DATE_TRUNC('quarter', {col})", "P1Y": "DATE_TRUNC('year', {col})", } diff --git a/superset/db_engine_specs/oracle.py b/superset/db_engine_specs/oracle.py index 47775973f11..d91ffb4fc9f 100644 --- a/superset/db_engine_specs/oracle.py +++ b/superset/db_engine_specs/oracle.py @@ -36,7 +36,7 @@ class OracleEngineSpec(BaseEngineSpec): "P1D": "TRUNC(CAST({col} as DATE), 'DDD')", "P1W": "TRUNC(CAST({col} as DATE), 'WW')", "P1M": "TRUNC(CAST({col} as DATE), 'MONTH')", - "P0.25Y": "TRUNC(CAST({col} as DATE), 'Q')", + "P3M": "TRUNC(CAST({col} as DATE), 'Q')", "P1Y": "TRUNC(CAST({col} as DATE), 'YEAR')", } diff --git a/superset/db_engine_specs/pinot.py b/superset/db_engine_specs/pinot.py index b07a6256d86..051f42501f9 100644 --- a/superset/db_engine_specs/pinot.py +++ b/superset/db_engine_specs/pinot.py @@ -37,7 +37,7 @@ class PinotEngineSpec(BaseEngineSpec): # pylint: disable=abstract-method "P1D": "1:DAYS", "P1W": "week", "P1M": "month", - "P0.25Y": "quarter", + "P3MY": "quarter", "P1Y": "year", } @@ -57,7 +57,7 @@ class PinotEngineSpec(BaseEngineSpec): # pylint: disable=abstract-method "P1D": False, "P1W": True, "P1M": True, - "P0.25Y": True, + "P3M": True, "P1Y": True, } diff --git a/superset/db_engine_specs/postgres.py b/superset/db_engine_specs/postgres.py index 4d1936310d4..47cf874ccbc 100644 --- a/superset/db_engine_specs/postgres.py +++ b/superset/db_engine_specs/postgres.py @@ -90,7 +90,7 @@ SYNTAX_ERROR_REGEX = re.compile('syntax error at or near "(?P.*?)" class PostgresBaseEngineSpec(BaseEngineSpec): - """ Abstract class for Postgres 'like' databases """ + """Abstract class for Postgres 'like' databases""" engine = "" engine_name = "PostgreSQL" @@ -103,7 +103,7 @@ class PostgresBaseEngineSpec(BaseEngineSpec): "P1D": "DATE_TRUNC('day', {col})", "P1W": "DATE_TRUNC('week', {col})", "P1M": "DATE_TRUNC('month', {col})", - "P0.25Y": "DATE_TRUNC('quarter', {col})", + "P3M": "DATE_TRUNC('quarter', {col})", "P1Y": "DATE_TRUNC('year', {col})", } diff --git a/superset/db_engine_specs/presto.py b/superset/db_engine_specs/presto.py index 9be73045e4e..74364300a7f 100644 --- a/superset/db_engine_specs/presto.py +++ b/superset/db_engine_specs/presto.py @@ -158,7 +158,7 @@ class PrestoEngineSpec(BaseEngineSpec): # pylint: disable=too-many-public-metho "P1D": "date_trunc('day', CAST({col} AS TIMESTAMP))", "P1W": "date_trunc('week', CAST({col} AS TIMESTAMP))", "P1M": "date_trunc('month', CAST({col} AS TIMESTAMP))", - "P0.25Y": "date_trunc('quarter', CAST({col} AS TIMESTAMP))", + "P3M": "date_trunc('quarter', CAST({col} AS TIMESTAMP))", "P1Y": "date_trunc('year', CAST({col} AS TIMESTAMP))", "P1W/1970-01-03T00:00:00Z": "date_add('day', 5, date_trunc('week', " "date_add('day', 1, CAST({col} AS TIMESTAMP))))", diff --git a/superset/db_engine_specs/rockset.py b/superset/db_engine_specs/rockset.py index 7cb3fb0640f..5cbf7e686ba 100644 --- a/superset/db_engine_specs/rockset.py +++ b/superset/db_engine_specs/rockset.py @@ -37,7 +37,7 @@ class RocksetEngineSpec(BaseEngineSpec): "P1D": "DATE_TRUNC('day', {col})", "P1W": "DATE_TRUNC('week', {col})", "P1M": "DATE_TRUNC('month', {col})", - "P0.25Y": "DATE_TRUNC('quarter', {col})", + "P3M": "DATE_TRUNC('quarter', {col})", "P1Y": "DATE_TRUNC('year', {col})", } diff --git a/superset/db_engine_specs/snowflake.py b/superset/db_engine_specs/snowflake.py index 6dd85706562..88ff0e20e09 100644 --- a/superset/db_engine_specs/snowflake.py +++ b/superset/db_engine_specs/snowflake.py @@ -58,13 +58,13 @@ class SnowflakeEngineSpec(PostgresBaseEngineSpec): DATE_TRUNC('HOUR', {col}))", "PT15M": "DATEADD(MINUTE, FLOOR(DATE_PART(MINUTE, {col}) / 15) * 15, \ DATE_TRUNC('HOUR', {col}))", - "PT0.5H": "DATEADD(MINUTE, FLOOR(DATE_PART(MINUTE, {col}) / 30) * 30, \ + "PT30M": "DATEADD(MINUTE, FLOOR(DATE_PART(MINUTE, {col}) / 30) * 30, \ DATE_TRUNC('HOUR', {col}))", "PT1H": "DATE_TRUNC('HOUR', {col})", "P1D": "DATE_TRUNC('DAY', {col})", "P1W": "DATE_TRUNC('WEEK', {col})", "P1M": "DATE_TRUNC('MONTH', {col})", - "P0.25Y": "DATE_TRUNC('QUARTER', {col})", + "P3M": "DATE_TRUNC('QUARTER', {col})", "P1Y": "DATE_TRUNC('YEAR', {col})", } diff --git a/superset/db_engine_specs/sqlite.py b/superset/db_engine_specs/sqlite.py index d2b87a48895..3b6d8b99ecf 100644 --- a/superset/db_engine_specs/sqlite.py +++ b/superset/db_engine_specs/sqlite.py @@ -45,7 +45,7 @@ class SqliteEngineSpec(BaseEngineSpec): "P1D": "DATE({col})", "P1W": "DATE({col}, -strftime('%w', {col}) || ' days')", "P1M": "DATE({col}, -strftime('%d', {col}) || ' days', '+1 day')", - "P0.25Y": ( + "P3M": ( "DATETIME(STRFTIME('%Y-', {col}) || " # year "SUBSTR('00' || " # pad with zeros to 2 chars "((CAST(STRFTIME('%m', {col}) AS INTEGER)) - " # month as integer diff --git a/superset/db_engine_specs/teradata.py b/superset/db_engine_specs/teradata.py index 88bffa5a075..8fd1641064d 100644 --- a/superset/db_engine_specs/teradata.py +++ b/superset/db_engine_specs/teradata.py @@ -32,7 +32,7 @@ class TeradataEngineSpec(BaseEngineSpec): "P1D": "TRUNC(CAST({col} as DATE), 'DDD')", "P1W": "TRUNC(CAST({col} as DATE), 'WW')", "P1M": "TRUNC(CAST({col} as DATE), 'MONTH')", - "P0.25Y": "TRUNC(CAST({col} as DATE), 'Q')", + "P3M": "TRUNC(CAST({col} as DATE), 'Q')", "P1Y": "TRUNC(CAST({col} as DATE), 'YEAR')", } diff --git a/superset/db_engine_specs/trino.py b/superset/db_engine_specs/trino.py index 7d28cfbaa60..6491e661ae9 100644 --- a/superset/db_engine_specs/trino.py +++ b/superset/db_engine_specs/trino.py @@ -37,7 +37,7 @@ class TrinoEngineSpec(BaseEngineSpec): "P1D": "date_trunc('day', CAST({col} AS TIMESTAMP))", "P1W": "date_trunc('week', CAST({col} AS TIMESTAMP))", "P1M": "date_trunc('month', CAST({col} AS TIMESTAMP))", - "P0.25Y": "date_trunc('quarter', CAST({col} AS TIMESTAMP))", + "P3M": "date_trunc('quarter', CAST({col} AS TIMESTAMP))", "P1Y": "date_trunc('year', CAST({col} AS TIMESTAMP))", # "1969-12-28T00:00:00Z/P1W", # Week starting Sunday # "1969-12-29T00:00:00Z/P1W", # Week starting Monday diff --git a/superset/examples/configs/charts/Quarterly_Sales.yaml b/superset/examples/configs/charts/Quarterly_Sales.yaml index 65553d5eff7..dc224318ab0 100644 --- a/superset/examples/configs/charts/Quarterly_Sales.yaml +++ b/superset/examples/configs/charts/Quarterly_Sales.yaml @@ -68,7 +68,7 @@ params: show_legend: false slice_id: 668 time_compare: null - time_grain_sqla: P0.25Y + time_grain_sqla: P3M time_range: No filter time_range_endpoints: - inclusive diff --git a/superset/examples/configs/charts/Quarterly_Sales_By_Product_Line.yaml b/superset/examples/configs/charts/Quarterly_Sales_By_Product_Line.yaml index 66d8733bf58..87fd1d61944 100644 --- a/superset/examples/configs/charts/Quarterly_Sales_By_Product_Line.yaml +++ b/superset/examples/configs/charts/Quarterly_Sales_By_Product_Line.yaml @@ -71,7 +71,7 @@ params: show_legend: true slice_id: 2806 time_compare: null - time_grain_sqla: P0.25Y + time_grain_sqla: P3M time_range: No filter time_range_endpoints: - inclusive diff --git a/superset/examples/utils.py b/superset/examples/utils.py index fc5b78a2304..70ed169bd76 100644 --- a/superset/examples/utils.py +++ b/superset/examples/utils.py @@ -86,7 +86,7 @@ def load_configs_from_directory( # removing "type" from the metadata allows us to import any exported model # from the unzipped directory directly - metadata = yaml.load(contents.get(METADATA_FILE_NAME, "{}")) + metadata = yaml.safe_load(contents.get(METADATA_FILE_NAME, "{}")) if "type" in metadata: del metadata["type"] contents[METADATA_FILE_NAME] = yaml.dump(metadata) diff --git a/superset/migrations/versions/32646df09c64_update_time_grain_sqla.py b/superset/migrations/versions/32646df09c64_update_time_grain_sqla.py new file mode 100644 index 00000000000..db1b87e5460 --- /dev/null +++ b/superset/migrations/versions/32646df09c64_update_time_grain_sqla.py @@ -0,0 +1,72 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""update time grain SQLA + +Revision ID: 32646df09c64 +Revises: 60dc453f4e2e +Create Date: 2021-10-12 11:15:25.559532 + +""" + +# revision identifiers, used by Alembic. +revision = "32646df09c64" +down_revision = "60dc453f4e2e" + +import json +from typing import Dict + +from alembic import op +from sqlalchemy import Column, Integer, Text +from sqlalchemy.ext.declarative import declarative_base + +from superset import db + +Base = declarative_base() + + +class Slice(Base): + __tablename__ = "slices" + + id = Column(Integer, primary_key=True) + params = Column(Text) + + +def migrate(mapping: Dict[str, str]) -> None: + bind = op.get_bind() + session = db.Session(bind=bind) + + for slc in session.query(Slice).all(): + try: + params = json.loads(slc.params) + time_grain_sqla = params.get("time_grain_sqla") + + if time_grain_sqla in mapping: + params["time_grain_sqla"] = mapping[time_grain_sqla] + slc.params = json.dumps(params, sort_keys=True) + except Exception: + pass + + session.commit() + session.close() + + +def upgrade(): + migrate(mapping={"PT0.5H": "PT30M", "P0.25Y": "P3M"}) + + +def downgrade(): + migrate(mapping={"PT30M": "PT0.5H", "P3M": "P0.25Y"}) diff --git a/superset/utils/pandas_postprocessing.py b/superset/utils/pandas_postprocessing.py index aa80cfd3995..76da26dfd6b 100644 --- a/superset/utils/pandas_postprocessing.py +++ b/superset/utils/pandas_postprocessing.py @@ -90,12 +90,12 @@ PROPHET_TIME_GRAIN_MAP = { "PT5M": "5min", "PT10M": "10min", "PT15M": "15min", - "PT0.5H": "30min", + "PT30M": "30min", "PT1H": "H", "P1D": "D", "P1W": "W", "P1M": "M", - "P0.25Y": "Q", + "P3M": "Q", "P1Y": "A", "1969-12-28T00:00:00Z/P1W": "W", "1969-12-29T00:00:00Z/P1W": "W", diff --git a/tests/integration_tests/db_engine_specs/base_engine_spec_tests.py b/tests/integration_tests/db_engine_specs/base_engine_spec_tests.py index 3b69a0729b5..275df7ba654 100644 --- a/tests/integration_tests/db_engine_specs/base_engine_spec_tests.py +++ b/tests/integration_tests/db_engine_specs/base_engine_spec_tests.py @@ -206,7 +206,7 @@ class TestDbEngineSpecs(TestDbEngineSpec): "P1D", "P1W", "P1M", - "P0.25Y", + "P3M", "P1Y", "1969-12-29T00:00:00Z/P1W", ], @@ -314,7 +314,7 @@ def test_time_grain_addons(): def test_get_time_grain_with_config(): - """ Should concatenate from configs and then sort in the proper order """ + """Should concatenate from configs and then sort in the proper order""" config = app.config.copy() app.config["TIME_GRAIN_ADDON_EXPRESSIONS"] = { @@ -345,7 +345,7 @@ def test_get_time_grain_with_config(): "P1D", "P1W", "P1M", - "P0.25Y", + "P3M", "P1Y", "1969-12-29T00:00:00Z/P1W", }