diff --git a/setup.py b/setup.py index 00b56d2df9d..f2ce9081ab9 100644 --- a/setup.py +++ b/setup.py @@ -56,7 +56,7 @@ setup( 'flask-sqlalchemy==2.1', 'flask-testing==0.6.2', 'flask-wtf==0.14.2', - 'flower==0.9.1', + 'flower==0.9.2', 'future>=0.16.0, <0.17', 'python-geohash==0.8.5', 'humanize==0.5.1', diff --git a/superset/assets/javascripts/SqlLab/components/QueryAutoRefresh.jsx b/superset/assets/javascripts/SqlLab/components/QueryAutoRefresh.jsx index 892b7caf529..6f788acba4a 100644 --- a/superset/assets/javascripts/SqlLab/components/QueryAutoRefresh.jsx +++ b/superset/assets/javascripts/SqlLab/components/QueryAutoRefresh.jsx @@ -22,7 +22,7 @@ class QueryAutoRefresh extends React.PureComponent { const queryKeys = Object.keys(queries); const queriesAsArray = queryKeys.map(key => queries[key]); return queriesAsArray.some(q => - ['running', 'started', 'pending', 'fetching'].indexOf(q.state) >= 0); + ['running', 'started', 'pending', 'fetching'].indexOf(q.state) >= 0).length; } startTimer() { if (!(this.timer)) { diff --git a/superset/assets/javascripts/SqlLab/components/ResultSet.jsx b/superset/assets/javascripts/SqlLab/components/ResultSet.jsx index 912e623ce73..f36a1640c79 100644 --- a/superset/assets/javascripts/SqlLab/components/ResultSet.jsx +++ b/superset/assets/javascripts/SqlLab/components/ResultSet.jsx @@ -144,7 +144,8 @@ export default class ResultSet extends React.PureComponent { } render() { const query = this.props.query; - const height = this.props.search ? this.props.height - SEARCH_HEIGHT : this.props.height; + const height = Math.max(0, + (this.props.search ? this.props.height - SEARCH_HEIGHT : this.props.height)); let sql; if (this.props.showSql) { diff --git a/superset/assets/javascripts/SqlLab/components/SqlEditor.jsx b/superset/assets/javascripts/SqlLab/components/SqlEditor.jsx index 4b2e8999a76..cbfaf3597f6 100644 --- a/superset/assets/javascripts/SqlLab/components/SqlEditor.jsx +++ b/superset/assets/javascripts/SqlLab/components/SqlEditor.jsx @@ -1,5 +1,6 @@ import React from 'react'; import PropTypes from 'prop-types'; +import throttle from 'lodash.throttle'; import { Col, FormGroup, @@ -52,6 +53,9 @@ class SqlEditor extends React.PureComponent { autorun: props.queryEditor.autorun, ctas: '', }; + + this.onResize = this.onResize.bind(this); + this.throttledResize = throttle(this.onResize, 250); } componentWillMount() { if (this.state.autorun) { @@ -62,12 +66,16 @@ class SqlEditor extends React.PureComponent { } componentDidMount() { this.onResize(); + window.addEventListener('resize', this.throttledResize); + } + componentWillUnmount() { + window.removeEventListener('resize', this.throttledResize); } onResize() { const height = this.sqlEditorHeight(); this.setState({ - editorPaneHeight: this.refs.ace.clientHeight, - southPaneHeight: height - this.refs.ace.clientHeight, + editorPaneHeight: this.props.queryEditor.height, + southPaneHeight: height - this.props.queryEditor.height, height, }); @@ -252,7 +260,7 @@ class SqlEditor extends React.PureComponent { split="horizontal" defaultSize={defaultNorthHeight} minSize={100} - onChange={this.onResize.bind(this)} + onChange={this.onResize} >
@@ -273,7 +281,7 @@ class SqlEditor extends React.PureComponent { editorQueries={this.props.editorQueries} dataPreviewQueries={this.props.dataPreviewQueries} actions={this.props.actions} - height={this.state.southPaneHeight} + height={this.state.southPaneHeight || 0} />
diff --git a/superset/assets/javascripts/SqlLab/components/VisualizeModal.jsx b/superset/assets/javascripts/SqlLab/components/VisualizeModal.jsx index 7c8b66511d4..23692f9aa0c 100644 --- a/superset/assets/javascripts/SqlLab/components/VisualizeModal.jsx +++ b/superset/assets/javascripts/SqlLab/components/VisualizeModal.jsx @@ -294,7 +294,7 @@ function mapStateToProps(state) { return { datasource: state.datasource, errorMessage: state.errorMessage, - timeout: state.common ? state.common.SUPERSET_WEBSERVER_TIMEOUT : null, + timeout: state.common ? state.common.conf.SUPERSET_WEBSERVER_TIMEOUT : null, }; } diff --git a/superset/assets/javascripts/SqlLab/main.less b/superset/assets/javascripts/SqlLab/main.less index f11329cdba1..fd2e7f29a48 100644 --- a/superset/assets/javascripts/SqlLab/main.less +++ b/superset/assets/javascripts/SqlLab/main.less @@ -289,8 +289,14 @@ a.Link { .tooltip-inner { max-width: 500px; } +.SplitPane.horizontal { + padding-right: 4px; +} .SouthPane { margin-top: 10px; + position: absolute; + width: 100%; + overflow: scroll; } .search-date-filter-container { display: flex; diff --git a/superset/assets/javascripts/components/FilterableTable/FilterableTableStyles.css b/superset/assets/javascripts/components/FilterableTable/FilterableTableStyles.css index c0c3717b635..c74355042f6 100644 --- a/superset/assets/javascripts/components/FilterableTable/FilterableTableStyles.css +++ b/superset/assets/javascripts/components/FilterableTable/FilterableTableStyles.css @@ -1,8 +1,12 @@ +.ReactVirtualized__Grid__innerScrollContainer { + border: 1px solid #ccc; +} .ReactVirtualized__Table__headerRow { font-weight: 700; display: flex; flex-direction: row; align-items: center; + border: 1px solid #ccc; } .ReactVirtualized__Table__row { display: flex; @@ -50,11 +54,6 @@ } .even-row { background: #f2f2f2; } .odd-row { background: #ffffff; } -.even-row, -.odd-row { - border: none; -} .filterable-table-container { overflow: auto; - border: 1px solid #ccc; } diff --git a/superset/assets/javascripts/explore/components/controls/DateFilterControl.jsx b/superset/assets/javascripts/explore/components/controls/DateFilterControl.jsx index 98db93a7e77..3669c9fade0 100644 --- a/superset/assets/javascripts/explore/components/controls/DateFilterControl.jsx +++ b/superset/assets/javascripts/explore/components/controls/DateFilterControl.jsx @@ -69,9 +69,8 @@ export default class DateFilterControl extends React.Component { setType(type) { this.setState({ type }); } - setValue(val) { - this.setState({ type: 'free', free: val }); - this.close(); + setValueAndClose(val) { + this.setState({ type: 'free', free: val }, this.close); } setDatetime(dttm) { this.setState({ dttm: dttm.format().substring(0, 19) }); @@ -175,13 +174,15 @@ export default class DateFilterControl extends React.Component { > diff --git a/superset/assets/javascripts/explore/stores/controls.jsx b/superset/assets/javascripts/explore/stores/controls.jsx index 1ebad1efdfe..2c18f4b0fa4 100644 --- a/superset/assets/javascripts/explore/stores/controls.jsx +++ b/superset/assets/javascripts/explore/stores/controls.jsx @@ -1240,7 +1240,7 @@ export const controls = { type: 'CheckboxControl', label: t('Range Filter'), renderTrigger: true, - default: true, + default: false, description: t('Whether to display the time range interactive selector'), }, @@ -1486,6 +1486,7 @@ export const controls = { point_radius_fixed: { type: 'FixedOrMetricControl', label: t('Point Size'), + default: { type: 'fix', value: 1000 }, description: t('Fixed point radius'), mapStateToProps: state => ({ datasource: state.datasource, @@ -1881,10 +1882,11 @@ export const controls = { }, }, - js_datapoint_mutator: jsFunctionControl( - t('Javascript data point mutator'), - t('Define a javascript function that receives each data point and can alter it ' + - 'before getting sent to the deck.gl layer'), + js_data_mutator: jsFunctionControl( + t('Javascript data interceptor'), + t('Define a javascript function that receives the data array used in the visualization ' + + 'and is expected to return a modified version of that array. This can be used ' + + 'to alter properties of the data, filter, or enrich the array.'), ), js_data: jsFunctionControl( diff --git a/superset/assets/javascripts/explore/stores/visTypes.js b/superset/assets/javascripts/explore/stores/visTypes.js index b175ff3fa88..e1d5e64f440 100644 --- a/superset/assets/javascripts/explore/stores/visTypes.js +++ b/superset/assets/javascripts/explore/stores/visTypes.js @@ -437,7 +437,7 @@ export const visTypes = { label: t('Advanced'), controlSetRows: [ ['js_columns'], - ['js_datapoint_mutator'], + ['js_data_mutator'], ['js_tooltip'], ['js_onclick_href'], ], @@ -509,7 +509,47 @@ export const visTypes = { label: t('Advanced'), controlSetRows: [ ['js_columns'], - ['js_datapoint_mutator'], + ['js_data_mutator'], + ['js_tooltip'], + ['js_onclick_href'], + ], + }, + ], + }, + + deck_polygon: { + label: t('Deck.gl - Polygon'), + requiresTime: true, + controlPanelSections: [ + { + label: t('Query'), + expanded: true, + controlSetRows: [ + ['line_column', 'line_type'], + ['row_limit', null], + ], + }, + { + label: t('Map'), + controlSetRows: [ + ['mapbox_style', 'viewport'], + ['reverse_long_lat', null], + ], + }, + { + label: t('Polygon Settings'), + controlSetRows: [ + ['fill_color_picker', 'stroke_color_picker'], + ['filled', 'stroked'], + ['extruded', null], + ['point_radius_scale', null], + ], + }, + { + label: t('Advanced'), + controlSetRows: [ + ['js_columns'], + ['js_data_mutator'], ['js_tooltip'], ['js_onclick_href'], ], @@ -621,7 +661,7 @@ export const visTypes = { label: t('Advanced'), controlSetRows: [ ['js_columns'], - ['js_datapoint_mutator'], + ['js_data_mutator'], ['js_tooltip'], ['js_onclick_href'], ], diff --git a/superset/assets/spec/javascripts/explore/components/DateFilterControl_spec.jsx b/superset/assets/spec/javascripts/explore/components/DateFilterControl_spec.jsx index e15356e4768..54db5c9cf32 100644 --- a/superset/assets/spec/javascripts/explore/components/DateFilterControl_spec.jsx +++ b/superset/assets/spec/javascripts/explore/components/DateFilterControl_spec.jsx @@ -42,6 +42,14 @@ describe('DateFilterControl', () => { expect(wrapper.state().num).to.equal('90'); }, 10); }); + it('sets now and closes', () => { + const label = wrapper.find('.now').first(); + label.simulate('click'); + setTimeout(() => { + expect(wrapper.state().free).to.equal('now'); + expect(wrapper.find('.popover')).to.have.length(0); + }, 10); + }); it('renders 2 dimmed sections', () => { const label = wrapper.find('.label').first(); label.simulate('click'); diff --git a/superset/assets/spec/javascripts/sqllab/VisualizeModal_spec.jsx b/superset/assets/spec/javascripts/sqllab/VisualizeModal_spec.jsx index 56af8de45e8..dffa8250ba0 100644 --- a/superset/assets/spec/javascripts/sqllab/VisualizeModal_spec.jsx +++ b/superset/assets/spec/javascripts/sqllab/VisualizeModal_spec.jsx @@ -27,7 +27,7 @@ describe('VisualizeModal', () => { const mockStore = configureStore(middlewares); const initialState = sqlLabReducer({}, {}); initialState.common = { - SUPERSET_WEBSERVER_TIMEOUT: 45, + conf: { SUPERSET_WEBSERVER_TIMEOUT: 45 }, }; const store = mockStore(initialState); const mockedProps = { diff --git a/superset/assets/visualizations/cal_heatmap.css b/superset/assets/visualizations/cal_heatmap.css index 0ab20872625..e5e46f6c222 100644 --- a/superset/assets/visualizations/cal_heatmap.css +++ b/superset/assets/visualizations/cal_heatmap.css @@ -1,3 +1,9 @@ .cal_heatmap .slice_container { padding: 10px; + position: static !important; +} + +.cal_heatmap .slice_container .ch-tooltip { + margin-left: 20px; + margin-top: 5px; } diff --git a/superset/assets/visualizations/deckgl/layers/geojson.jsx b/superset/assets/visualizations/deckgl/layers/geojson.jsx index c21ce3bbd19..542ad58686d 100644 --- a/superset/assets/visualizations/deckgl/layers/geojson.jsx +++ b/superset/assets/visualizations/deckgl/layers/geojson.jsx @@ -35,10 +35,10 @@ const alterProps = (props, propOverrides) => { }; }; let features; -const recurseGeoJson = (node, propOverrides, jsFnMutator, extraProps) => { +const recurseGeoJson = (node, propOverrides, extraProps) => { if (node && node.features) { node.features.forEach((obj) => { - recurseGeoJson(obj, propOverrides, jsFnMutator, node.extraProps || extraProps); + recurseGeoJson(obj, propOverrides, node.extraProps || extraProps); }); } if (node && node.geometry) { @@ -46,9 +46,6 @@ const recurseGeoJson = (node, propOverrides, jsFnMutator, extraProps) => { ...node, properties: alterProps(node.properties, propOverrides), }; - if (jsFnMutator) { - jsFnMutator(newNode); - } if (!newNode.extraProps) { newNode.extraProps = extraProps; } @@ -70,10 +67,14 @@ export default function geoJsonLayer(formData, payload, slice) { propOverrides.strokeColor = strokeColor; } + features = []; + recurseGeoJson(payload.data, propOverrides); + let jsFnMutator; - if (fd.js_datapoint_mutator) { + if (fd.js_data_mutator) { // Applying user defined data mutator if defined - jsFnMutator = sandboxedEval(fd.js_datapoint_mutator); + jsFnMutator = sandboxedEval(fd.js_data_mutator); + features = jsFnMutator(features); } features = []; diff --git a/superset/assets/visualizations/deckgl/layers/path.jsx b/superset/assets/visualizations/deckgl/layers/path.jsx index c69f23634a4..df2c1b0c1a7 100644 --- a/superset/assets/visualizations/deckgl/layers/path.jsx +++ b/superset/assets/visualizations/deckgl/layers/path.jsx @@ -14,9 +14,9 @@ export default function getLayer(formData, payload, slice) { color: fixedColor, })); - if (fd.js_datapoint_mutator) { - const jsFnMutator = sandboxedEval(fd.js_datapoint_mutator); - data = data.map(jsFnMutator); + if (fd.js_data_mutator) { + const jsFnMutator = sandboxedEval(fd.js_data_mutator); + data = jsFnMutator(data); } return new PathLayer({ diff --git a/superset/assets/visualizations/deckgl/layers/polygon.jsx b/superset/assets/visualizations/deckgl/layers/polygon.jsx index 05c32c619d7..886282707f4 100644 --- a/superset/assets/visualizations/deckgl/layers/polygon.jsx +++ b/superset/assets/visualizations/deckgl/layers/polygon.jsx @@ -11,10 +11,10 @@ export default function polygonLayer(formData, payload, slice) { fillColor: [fc.r, fc.g, fc.b, 255 * fc.a], })); - if (fd.js_datapoint_mutator) { + if (fd.js_data_mutator) { // Applying user defined data mutator if defined - const jsFnMutator = sandboxedEval(fd.js_datapoint_mutator); - data = data.map(jsFnMutator); + const jsFnMutator = sandboxedEval(fd.js_data_mutator); + data = jsFnMutator(data); } return new PolygonLayer({ diff --git a/superset/assets/visualizations/deckgl/layers/scatter.jsx b/superset/assets/visualizations/deckgl/layers/scatter.jsx index eda1b7c361f..0f591a21233 100644 --- a/superset/assets/visualizations/deckgl/layers/scatter.jsx +++ b/superset/assets/visualizations/deckgl/layers/scatter.jsx @@ -28,10 +28,10 @@ export default function getLayer(formData, payload, slice) { }; }); - if (fd.js_datapoint_mutator) { + if (fd.js_data_mutator) { // Applying user defined data mutator if defined - const jsFnMutator = sandboxedEval(fd.js_datapoint_mutator); - data = data.map(jsFnMutator); + const jsFnMutator = sandboxedEval(fd.js_data_mutator); + data = jsFnMutator(data); } return new ScatterplotLayer({ diff --git a/superset/assets/visualizations/deckgl/multi.jsx b/superset/assets/visualizations/deckgl/multi.jsx index 63f1a880117..a1bac289034 100644 --- a/superset/assets/visualizations/deckgl/multi.jsx +++ b/superset/assets/visualizations/deckgl/multi.jsx @@ -32,11 +32,26 @@ function deckMulti(slice, payload, setControlValue) { }; render(); payload.data.slices.forEach((subslice) => { - const url = getExploreUrl(subslice.form_data, 'json'); + // Filters applied to multi_deck are passed down to underlying charts + // note that dashboard contextual information (filter_immune_slices and such) aren't + // taken into consideration here + let filters = subslice.form_data.filters.concat(fd.filters); + if (fd.extra_filters) { + filters = filters.concat(fd.extra_filters); + } + const subsliceCopy = { + ...subslice, + form_data: { + ...subslice.form_data, + filters, + }, + }; + + const url = getExploreUrl(subsliceCopy.form_data, 'json'); $.get(url, (data) => { // Late import to avoid circular deps - const layer = layerGenerators[subslice.form_data.viz_type](subslice.form_data, data); - slice.subSlicesLayers[subslice.slice_id] = layer; // eslint-disable-line no-param-reassign + const layer = layerGenerators[subsliceCopy.form_data.viz_type](subsliceCopy.form_data, data); + slice.subSlicesLayers[subsliceCopy.slice_id] = layer; // eslint-disable-line no-param-reassign render(); }); }); diff --git a/superset/models/core.py b/superset/models/core.py index 9f26a27d3c9..1b71d42c4b2 100644 --- a/superset/models/core.py +++ b/superset/models/core.py @@ -9,7 +9,6 @@ from datetime import date, datetime import functools import json import logging -import pickle import textwrap from flask import escape, g, Markup, request @@ -395,7 +394,7 @@ class Dashboard(Model, AuditMixinNullable, ImportMixin): be overridden or just copies over. Slices that belong to this dashboard will be wired to existing tables. This function can be used to import/export dashboards between multiple superset instances. - Audit metadata isn't copies over. + Audit metadata isn't copied over. """ def alter_positions(dashboard, old_to_new_slc_id_dict): """ Updates slice_ids in the position json. @@ -533,10 +532,10 @@ class Dashboard(Model, AuditMixinNullable, ImportMixin): make_transient(eager_datasource) eager_datasources.append(eager_datasource) - return pickle.dumps({ + return json.dumps({ 'dashboards': copied_dashboards, 'datasources': eager_datasources, - }) + }, cls=utils.DashboardEncoder, indent=4) class Database(Model, AuditMixinNullable, ImportMixin): diff --git a/superset/sql_lab.py b/superset/sql_lab.py index 63225f3e2d0..87a6b44295b 100644 --- a/superset/sql_lab.py +++ b/superset/sql_lab.py @@ -10,6 +10,7 @@ from time import sleep import uuid from celery.exceptions import SoftTimeLimitExceeded +import numpy as np import pandas as pd import sqlalchemy from sqlalchemy.orm import sessionmaker @@ -85,6 +86,26 @@ def get_session(nullpool): return session +def convert_results_to_df(cursor_description, data): + """Convert raw query results to a DataFrame.""" + column_names = ( + [col[0] for col in cursor_description] if cursor_description else []) + column_names = dedup(column_names) + + # check whether the result set has any nested dict columns + if data: + first_row = data[0] + has_dict_col = any([isinstance(c, dict) for c in first_row]) + df_data = list(data) if has_dict_col else np.array(data) + else: + df_data = [] + + cdf = dataframe.SupersetDataFrame( + pd.DataFrame(df_data, columns=column_names)) + + return cdf + + @celery_app.task(bind=True, soft_time_limit=SQLLAB_TIMEOUT) def get_sql_results( ctask, query_id, return_results=True, store_results=False, @@ -224,11 +245,7 @@ def execute_sql( }, default=utils.json_iso_dttm_ser) - column_names = ( - [col[0] for col in cursor_description] if cursor_description else []) - column_names = dedup(column_names) - cdf = dataframe.SupersetDataFrame( - pd.DataFrame(list(data), columns=column_names)) + cdf = convert_results_to_df(cursor_description, data) query.rows = cdf.size query.progress = 100 diff --git a/superset/templates/appbuilder/navbar.html b/superset/templates/appbuilder/navbar.html index 2a6b65bf245..27f2fee6340 100644 --- a/superset/templates/appbuilder/navbar.html +++ b/superset/templates/appbuilder/navbar.html @@ -3,7 +3,7 @@ {% set WARNING_MSG = appbuilder.app.config.get('WARNING_MSG') %}