0.31.0rc18

(cherry picked from commit 07c340cf82)

UPDATING.md

@@ -27,6 +27,11 @@ assists people when migrating to a new version.
   run `pip install superset[presto]` and/or `pip install superset[hive]` as
   required.
 
+* [5445](https://github.com/apache/incubator-superset/pull/5445) : a change 
+which prevents encoding of empty string from form data in the datanbase. 
+This involves a non-schema changing migration which does potentially impact
+a large number of records. Scheduled downtime may be advised.
+
 ## Superset 0.31.0
 * boto3 / botocore was removed from the dependency list. If you use s3
 as a place to store your SQL Lab result set or Hive uploads, you may

requirements.txt

@@ -43,7 +43,7 @@ mako==1.0.7               # via alembic
 markdown==3.0
 markupsafe==1.0           # via jinja2, mako
 numpy==1.15.2             # via pandas
-pandas==0.23.1
+pandas==0.23.4
 parsedatetime==2.0.0
 pathlib2==2.3.0
 polyline==1.3.2
@@ -60,7 +60,7 @@ requests==2.20.0
 retry==0.9.2
 selenium==3.141.0
 simplejson==3.15.0
-six==1.11.0               # via bleach, cryptography, isodate, pathlib2, polyline, pydruid, python-dateutil, sqlalchemy-utils
+six==1.11.0               # via bleach, cryptography, isodate, pathlib2, polyline, pydruid, python-dateutil, sqlalchemy-utils, wtforms-json
 sqlalchemy-utils==0.32.21
 sqlalchemy==1.2.2
 sqlparse==0.2.4
@@ -69,4 +69,5 @@ urllib3==1.22             # via requests, selenium
 vine==1.1.4               # via amqp
 webencodings==0.5.1       # via bleach
 werkzeug==0.14.1          # via flask
-wtforms==2.2.1            # via flask-wtf
+wtforms-json==0.3.3
+wtforms==2.2.1            # via flask-wtf, wtforms-json

setup.py

@@ -104,6 +104,7 @@ setup(
         'sqlalchemy-utils',
         'sqlparse',
         'unicodecsv',
+        'wtforms-json',
     ],
     extras_require={
         'cors': ['flask-cors>=2.0.0'],

superset/__init__.py

@@ -28,6 +28,7 @@ from flask_compress import Compress
 from flask_migrate import Migrate
 from flask_wtf.csrf import CSRFProtect
 from werkzeug.contrib.fixers import ProxyFix
+import wtforms_json
 
 from superset import config
 from superset.connectors.connector_registry import ConnectorRegistry
@@ -35,6 +36,8 @@ from superset.security import SupersetSecurityManager
 from superset.utils.core import (
     get_update_perms_flag, pessimistic_connection_handling, setup_cache)
 
+wtforms_json.init()
+
 APP_DIR = os.path.dirname(__file__)
 CONFIG_MODULE = os.environ.get('SUPERSET_CONFIG', 'superset.config')
 

superset/assets/cypress/integration/explore/visualizations/area.js

@@ -16,6 +16,8 @@
  * specific language governing permissions and limitations
  * under the License.
  */
+import readResponseBlob from '../../../utils/readResponseBlob';
+
 export default () => describe('Area', () => {
   const AREA_FORM_DATA = {
     datasource: '2__table',
@@ -71,11 +73,12 @@ export default () => describe('Area', () => {
       ...AREA_FORM_DATA,
       groupby: ['region'],
     });
+
     cy.get('.nv-area').should('have.length', 7);
   });
 
   it('should work with groupby and filter', () => {
-    verify({
+    cy.visitChartByParams(JSON.stringify({
       ...AREA_FORM_DATA,
       groupby: ['region'],
       adhoc_filters: [{
@@ -88,6 +91,18 @@ export default () => describe('Area', () => {
         fromFormData: true,
         filterOptionName: 'filter_txje2ikiv6_wxmn0qwd1xo',
       }],
+    }));
+
+    cy.wait('@getJson').then(async (xhr) => {
+      cy.verifyResponseCodes(xhr);
+
+      const responseBody = await readResponseBlob(xhr.response.body);
+
+      // Make sure data is sorted correctly
+      const firstRow = responseBody.data[0].values;
+      const secondRow = responseBody.data[1].values;
+      expect(firstRow[firstRow.length - 1].y).to.be.greaterThan(secondRow[secondRow.length - 1].y);
+      cy.verifySliceContainer('svg');
     });
     cy.get('.nv-area').should('have.length', 2);
   });

superset/assets/cypress/integration/explore/visualizations/line.js

@@ -98,10 +98,40 @@ export default () => describe('Line', () => {
       metrics,
       time_compare: ['1+year'],
       comparison_type: 'values',
+      groupby: ['gender'],
     };
 
     cy.visitChartByParams(JSON.stringify(formData));
     cy.verifySliceSuccess({ waitAlias: '@getJson', chartSelector: 'svg' });
+
+    // Offset color should match original line color
+    cy.get('.nv-legend-text')
+      .contains('boy')
+      .siblings()
+      .first()
+      .should('have.attr', 'style')
+      .then((style) => {
+        cy.get('.nv-legend-text')
+          .contains('boy, 1 year offset')
+          .siblings()
+          .first()
+          .should('have.attr', 'style')
+          .and('eq', style);
+      });
+
+     cy.get('.nv-legend-text')
+      .contains('girl')
+      .siblings()
+      .first()
+      .should('have.attr', 'style')
+      .then((style) => {
+        cy.get('.nv-legend-text')
+          .contains('girl, 1 year offset')
+          .siblings()
+          .first()
+          .should('have.attr', 'style')
+          .and('eq', style);
+      });
   });
 
   it('Test line chart with time shift yoy', () => {

superset/assets/cypress_build.sh

@@ -29,6 +29,7 @@ flask run -p 8081 --with-threads --reload --debugger &
 
 #block on the longer running javascript process
 time npm ci
+time npm run install-cypress
 time npm run build
 echo "[completed js build steps]"
 

superset/assets/package.json

@@ -1,6 +1,6 @@
 {
   "name": "superset",
-  "version": "0.999.0dev",
+  "version": "0.31.0rc18",
   "description": "Superset is a data exploration platform designed to be visual, intuitive, and interactive.",
   "license": "Apache-2.0",
   "directories": {
@@ -19,7 +19,8 @@
     "lint-fix": "eslint --fix --ignore-path=.eslintignore --ext .js,.jsx . && tslint -c tslint.json --fix ./{src,spec}/**/*.ts{,x}",
     "sync-backend": "babel-node --preset=@babel/preset-env src/syncBackend.js",
     "cypress": "cypress",
-    "cypress-debug": "cypress open --config watchForFileChanges=true"
+    "cypress-debug": "cypress open --config watchForFileChanges=true",
+    "install-cypress": "npm install cypress@3.1.5"
   },
   "repository": {
     "type": "git",
@@ -62,6 +63,7 @@
     "@vx/responsive": "0.0.172",
     "@vx/scale": "^0.0.165",
     "abortcontroller-polyfill": "^1.1.9",
+    "bignumber.js": "^8.1.1",
     "bootstrap": "^3.3.6",
     "bootstrap-slider": "^10.0.0",
     "brace": "^0.11.1",
@@ -115,9 +117,10 @@
     "react-select": "1.2.1",
     "react-select-fast-filter-options": "^0.2.1",
     "react-sortable-hoc": "^0.8.3",
-    "react-split-pane": "^0.1.66",
+    "react-split": "^2.0.4",
     "react-sticky": "^6.0.2",
     "react-syntax-highlighter": "^7.0.4",
+    "react-transition-group": "^2.5.3",
     "react-virtualized": "9.19.1",
     "react-virtualized-select": "^3.1.3",
     "reactable-arc": "0.14.42",
@@ -156,7 +159,6 @@
     "cache-loader": "^1.2.2",
     "clean-webpack-plugin": "^0.1.19",
     "css-loader": "^1.0.0",
-    "cypress": "^3.0.3",
     "enzyme": "^3.3.0",
     "enzyme-adapter-react-16": "^1.1.1",
     "eslint": "^4.19.1",
@@ -192,7 +194,7 @@
     "terser-webpack-plugin": "^1.1.0",
     "thread-loader": "^1.2.0",
     "transform-loader": "^0.2.3",
-    "ts-jest": "^23.10.4",
+    "ts-jest": "^24.0.0",
     "ts-loader": "^5.2.0",
     "tslint": "^5.11.0",
     "tslint-react": "^3.6.0",

superset/assets/spec/javascripts/chart/transformBigNumber_spec.js

@@ -0,0 +1,53 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+import BigNumber from 'bignumber.js';
+import transform from 'src/chart/transformBigNumber';
+
+describe('transformBigNumber', () => {
+  it('should transform BigNumber on its own', () => {
+    expect(transform(new BigNumber(123.456))).toBe(123.456);
+  });
+
+  it('should transform BigNumber in objects', () => {
+    expect(transform({
+      foo: new BigNumber(123),
+      bar: 456,
+      baz: null,
+    })).toEqual({ foo: 123, bar: 456, baz: null });
+  });
+
+  it('should transform BigNumber in arrays', () => {
+    expect(transform([
+      { foo: new BigNumber(123) },
+      { bar: 456 },
+    ])).toEqual([{ foo: 123 }, { bar: 456 }]);
+  });
+
+  it('should transform BigNumber in nested structures', () => {
+    expect(transform([{
+      x: new BigNumber(123),
+      y: [{ foo: new BigNumber(456) }, { bar: 'str' }],
+      z: { some: [new BigNumber(789)] },
+    }])).toEqual([{
+      x: 123,
+      y: [{ foo: 456 }, { bar: 'str' }],
+      z: { some: [789] },
+    }]);
+  });
+});

superset/assets/spec/javascripts/sqllab/SouthPane_spec.jsx

@@ -24,7 +24,7 @@ import { shallow } from 'enzyme';
 
 import { STATUS_OPTIONS } from '../../../src/SqlLab/constants';
 import { initialState } from './fixtures';
-import SouthPane from '../../../src/SqlLab/components/SouthPane';
+import SouthPaneContainer, { SouthPane } from '../../../src/SqlLab/components/SouthPane';
 
 describe('SouthPane', () => {
   const middlewares = [thunk];
@@ -42,11 +42,16 @@ describe('SouthPane', () => {
   };
 
   const getWrapper = () => (
-    shallow(<SouthPane {...mockedProps} />, {
+    shallow(<SouthPaneContainer {...mockedProps} />, {
       context: { store },
     }).dive());
 
   let wrapper;
+
+  beforeAll(() => {
+    jest.spyOn(SouthPane.prototype, 'getSouthPaneHeight').mockImplementation(() => 500);
+  });
+
   it('should render offline when the state is offline', () => {
     wrapper = getWrapper();
     wrapper.setProps({ offline: true });

superset/assets/spec/javascripts/sqllab/SqlEditor_spec.jsx

@@ -38,6 +38,11 @@ describe('SqlEditor', () => {
     defaultQueryLimit: 1000,
     maxRow: 100000,
   };
+
+  beforeAll(() => {
+    jest.spyOn(SqlEditor.prototype, 'getSqlEditorHeight').mockImplementation(() => 500);
+  });
+
   it('is valid', () => {
     expect(
       React.isValidElement(<SqlEditor {...mockedProps} />),

superset/assets/spec/javascripts/utils/safeStringify_spec.ts

@@ -0,0 +1,110 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+import { safeStringify } from '../../../src/utils/safeStringify';
+
+class Noise {
+  public next: Noise;
+}
+
+describe('Stringify utility testing', () => {
+  it('correctly parses a simple object just like JSON', () => {
+    const noncircular = {
+      b: 'foo',
+      c: 'bar',
+      d: [
+        {
+          e: 'hello',
+          f: ['world'],
+        },
+        {
+          e: 'hello',
+          f: ['darkness', 'my', 'old', 'friend'],
+        },
+      ],
+    };
+    expect(safeStringify(noncircular)).toEqual(JSON.stringify(noncircular));
+    // Checking that it works with quick-deepish-copies as well.
+    expect(JSON.parse(safeStringify(noncircular))).toEqual(JSON.parse(JSON.stringify(noncircular)));
+  });
+
+  it('handles simple circular json as expected', () => {
+    const ping = new Noise();
+    const pong = new Noise();
+    const pang = new Noise();
+    ping.next = pong;
+    pong.next = ping;
+
+    // ping.next is pong (the circular reference) now
+    const safeString = safeStringify(ping);
+    ping.next = pang;
+
+    // ping.next is pang now, which has no circular reference, so it's safe to use JSON.stringify
+    const ordinaryString = JSON.stringify(ping);
+    expect(safeString).toEqual(ordinaryString);
+  });
+
+  it('creates a parseable object even when the input is circular', () => {
+    const ping = new Noise();
+    const pong = new Noise();
+    ping.next = pong;
+    pong.next = ping;
+
+    const newNoise: Noise = JSON.parse(safeStringify(ping));
+    expect(newNoise).toBeTruthy();
+    expect(newNoise.next).toEqual({});
+  });
+
+  it('does not remove noncircular duplicates', () => {
+    const a = {
+      foo: 'bar',
+    };
+
+    const repeating = {
+      first: a,
+      second: a,
+      third: a,
+    };
+
+    expect(safeStringify(repeating)).toEqual(JSON.stringify(repeating));
+  });
+
+  it('does not remove nodes with empty objects', () => {
+    const emptyObjectValues = {
+      a: {},
+      b: 'foo',
+      c: {
+        d: 'good data here',
+        e: {},
+      },
+    };
+    expect(safeStringify(emptyObjectValues)).toEqual(JSON.stringify(emptyObjectValues));
+  });
+
+  it('does not remove nested same keys', () => {
+    const nestedKeys = {
+      a: 'b',
+      c: {
+        a: 'd',
+        x: 'y',
+      },
+    };
+
+    expect(safeStringify(nestedKeys)).toEqual(JSON.stringify(nestedKeys));
+  });
+});

superset/assets/spec/javascripts/visualizations/deckgl/utils_spec.js

@@ -30,7 +30,7 @@ describe('getBreakPoints', () => {
   });
 
   it('returns sorted break points', () => {
-    const fd = { break_points: ['0', '10', '100', '50', '1000'] };
+    const fd = { breakPoints: ['0', '10', '100', '50', '1000'] };
     const result = getBreakPoints(fd, [], metricAccessor);
     const expected = ['0', '10', '50', '100', '1000'];
     expect(result).toEqual(expected);
@@ -45,7 +45,7 @@ describe('getBreakPoints', () => {
   });
 
   it('formats number with proper precision', () => {
-    const fd = { metric: 'count', num_buckets: 2 };
+    const fd = { metric: 'count', numBuckets: 2 };
     const features = [0, 1 / 3, 2 / 3, 1].map(count => ({ count }));
     const result = getBreakPoints(fd, features, metricAccessor);
     const expected = ['0.0', '0.5', '1.0'];
@@ -53,7 +53,7 @@ describe('getBreakPoints', () => {
   });
 
   it('works with a zero range', () => {
-    const fd = { metric: 'count', num_buckets: 1 };
+    const fd = { metric: 'count', numBuckets: 1 };
     const features = [1, 1, 1].map(count => ({ count }));
     const result = getBreakPoints(fd, features, metricAccessor);
     const expected = ['1', '1'];
@@ -69,7 +69,7 @@ describe('getBreakPointColorScaler', () => {
   it('returns linear color scaler if there are no break points', () => {
     const fd = {
       metric: 'count',
-      linear_color_scheme: ['#000000', '#ffffff'],
+      linearColorScheme: ['#000000', '#ffffff'],
       opacity: 100,
     };
     const features = [10, 20, 30].map(count => ({ count }));
@@ -82,8 +82,8 @@ describe('getBreakPointColorScaler', () => {
   it('returns bucketing scaler if there are break points', () => {
     const fd = {
       metric: 'count',
-      linear_color_scheme: ['#000000', '#ffffff'],
-      break_points: ['0', '1', '10'],
+      linearColorScheme: ['#000000', '#ffffff'],
+      breakPoints: ['0', '1', '10'],
       opacity: 100,
     };
     const features = [];
@@ -97,8 +97,8 @@ describe('getBreakPointColorScaler', () => {
   it('mask values outside the break points', () => {
     const fd = {
       metric: 'count',
-      linear_color_scheme: ['#000000', '#ffffff'],
-      break_points: ['0', '1', '10'],
+      linearColorScheme: ['#000000', '#ffffff'],
+      breakPoints: ['0', '1', '10'],
       opacity: 100,
     };
     const features = [];
@@ -116,8 +116,8 @@ describe('getBuckets', () => {
   it('computes buckets for break points', () => {
     const fd = {
       metric: 'count',
-      linear_color_scheme: ['#000000', '#ffffff'],
-      break_points: ['0', '1', '10'],
+      linearColorScheme: ['#000000', '#ffffff'],
+      breakPoints: ['0', '1', '10'],
       opacity: 100,
     };
     const features = [];

superset/assets/src/SqlLab/components/App.jsx

@@ -84,7 +84,7 @@ class App extends React.PureComponent {
       content = (
         <div>
           <QueryAutoRefresh />
-          <TabbedSqlEditors getHeight={this.getHeight} />
+          <TabbedSqlEditors />
         </div>
       );
     }

superset/assets/src/SqlLab/components/QueryAutoRefresh.jsx

@@ -72,6 +72,8 @@ class QueryAutoRefresh extends React.PureComponent {
         }).catch(() => {
           this.props.actions.setUserOffline(true);
         });
+    } else {
+      this.props.actions.setUserOffline(false);
     }
   }
   render() {

superset/assets/src/SqlLab/components/SaveQuery.jsx

@@ -31,11 +31,13 @@ const propTypes = {
   dbId: PropTypes.number,
   animation: PropTypes.bool,
   onSave: PropTypes.func,
+  saveQueryWarning: PropTypes.string,
 };
 const defaultProps = {
   defaultLabel: t('Undefined'),
   animation: true,
   onSave: () => {},
+  saveQueryWarning: null,
 };
 
 class SaveQuery extends React.PureComponent {
@@ -108,6 +110,18 @@ class SaveQuery extends React.PureComponent {
           </Col>
         </Row>
         <br />
+        {this.props.saveQueryWarning && (
+          <div>
+            <Row>
+              <Col md={12}>
+                <small>
+                  {this.props.saveQueryWarning}
+                </small>
+              </Col>
+            </Row>
+            <br />
+          </div>
+        )}
         <Row>
           <Col md={12}>
             <Button

superset/assets/src/SqlLab/components/SouthPane.jsx

@@ -48,7 +48,24 @@ const defaultProps = {
   offline: false,
 };
 
-class SouthPane extends React.PureComponent {
+export class SouthPane extends React.PureComponent {
+  constructor(props) {
+    super(props);
+    this.state = {
+      height: props.height,
+    };
+    this.southPaneRef = React.createRef();
+    this.getSouthPaneHeight = this.getSouthPaneHeight.bind(this);
+    this.switchTab = this.switchTab.bind(this);
+  }
+  componentWillReceiveProps() {
+    // south pane expands the entire height of the tab content on mount
+    this.setState({ height: this.getSouthPaneHeight() });
+  }
+  // One layer of abstraction for easy spying in unit tests
+  getSouthPaneHeight() {
+    return this.southPaneRef.current ? this.southPaneRef.current.clientHeight : 0;
+  }
   switchTab(id) {
     this.props.actions.setActiveSouthPaneTab(id);
   }
@@ -59,7 +76,7 @@ class SouthPane extends React.PureComponent {
           { STATUS_OPTIONS.offline }
         </Label>);
     }
-    const innerTabHeight = this.props.height - 55;
+    const innerTabHeight = this.state.height - 55;
     let latestQuery;
     const props = this.props;
     if (props.editorQueries.length > 0) {
@@ -98,12 +115,12 @@ class SouthPane extends React.PureComponent {
     ));
 
     return (
-      <div className="SouthPane">
+      <div className="SouthPane" ref={this.southPaneRef}>
         <Tabs
           bsStyle="tabs"
           id={shortid.generate()}
           activeKey={this.props.activeSouthPaneTab}
-          onSelect={this.switchTab.bind(this)}
+          onSelect={this.switchTab}
         >
           <Tab
             title={t('Results')}

superset/assets/src/SqlLab/components/SqlEditor.jsx

@@ -17,21 +17,18 @@
  * under the License.
  */
 import React from 'react';
+import { CSSTransition } from 'react-transition-group';
 import PropTypes from 'prop-types';
-import { throttle } from 'lodash';
 import {
-  Col,
   FormGroup,
   InputGroup,
   Form,
   FormControl,
   Label,
   OverlayTrigger,
-  Row,
   Tooltip,
-  Collapse,
 } from 'react-bootstrap';
-import SplitPane from 'react-split-pane';
+import Split from 'react-split';
 import { t } from '@superset-ui/translation';
 
 import Button from '../../components/Button';
@@ -47,9 +44,13 @@ import AceEditorWrapper from './AceEditorWrapper';
 import { STATE_BSSTYLE_MAP } from '../constants';
 import RunQueryActionButton from './RunQueryActionButton';
 
+const SQL_TOOLBAR_HEIGHT = 51;
+const GUTTER_HEIGHT = 5;
+const INITIAL_NORTH_PERCENT = 30;
+const INITIAL_SOUTH_PERCENT = 70;
+
 const propTypes = {
   actions: PropTypes.object.isRequired,
-  getHeight: PropTypes.func.isRequired,
   database: PropTypes.object,
   latestQuery: PropTypes.object,
   tables: PropTypes.array.isRequired,
@@ -59,12 +60,14 @@ const propTypes = {
   hideLeftBar: PropTypes.bool,
   defaultQueryLimit: PropTypes.number.isRequired,
   maxRow: PropTypes.number.isRequired,
+  saveQueryWarning: PropTypes.string,
 };
 
 const defaultProps = {
   database: null,
   latestQuery: null,
   hideLeftBar: false,
+  saveQueryWarning: null,
 };
 
 class SqlEditor extends React.PureComponent {
@@ -75,13 +78,18 @@ class SqlEditor extends React.PureComponent {
       ctas: '',
       sql: props.queryEditor.sql,
     };
+    this.sqlEditorRef = React.createRef();
+    this.northPaneRef = React.createRef();
 
-    this.onResize = this.onResize.bind(this);
-    this.throttledResize = throttle(this.onResize, 250);
+    this.onResizeStart = this.onResizeStart.bind(this);
+    this.onResizeEnd = this.onResizeEnd.bind(this);
     this.runQuery = this.runQuery.bind(this);
     this.stopQuery = this.stopQuery.bind(this);
     this.onSqlChanged = this.onSqlChanged.bind(this);
     this.setQueryEditorSql = this.setQueryEditorSql.bind(this);
+    this.queryPane = this.queryPane.bind(this);
+    this.getAceEditorAndSouthPaneHeights = this.getAceEditorAndSouthPaneHeights.bind(this);
+    this.getSqlEditorHeight = this.getSqlEditorHeight.bind(this);
   }
   componentWillMount() {
     if (this.state.autorun) {
@@ -91,29 +99,41 @@ class SqlEditor extends React.PureComponent {
     }
   }
   componentDidMount() {
-    this.onResize();
-    window.addEventListener('resize', this.throttledResize);
+    // We need to measure the height of the sql editor post render to figure the height of
+    // the south pane so it gets rendered properly
+    // eslint-disable-next-line react/no-did-mount-set-state
+    this.setState({ height: this.getSqlEditorHeight() });
   }
-  componentWillUnmount() {
-    window.removeEventListener('resize', this.throttledResize);
+  onResizeStart() {
+    // Set the heights on the ace editor and the ace content area after drag starts
+    // to smooth out the visual transition to the new heights when drag ends
+    document.getElementById('brace-editor').style.height = `calc(100% - ${SQL_TOOLBAR_HEIGHT}px)`;
+    document.getElementsByClassName('ace_content')[0].style.height = '100%';
   }
-  onResize() {
-    const height = this.sqlEditorHeight();
-    const editorPaneHeight = this.props.queryEditor.height || 200;
-    const splitPaneHandlerHeight = 8; // 4px of height + 4px of top-margin
-    this.setState({
-      editorPaneHeight,
-      southPaneHeight: height - editorPaneHeight - splitPaneHandlerHeight,
-      height,
-    });
+  onResizeEnd([northPercent, southPercent]) {
+    this.setState(this.getAceEditorAndSouthPaneHeights(
+      this.state.height, northPercent, southPercent));
 
-    if (this.refs.ace && this.refs.ace.clientHeight) {
-      this.props.actions.persistEditorHeight(this.props.queryEditor, this.refs.ace.clientHeight);
+    if (this.northPaneRef.current && this.northPaneRef.current.clientHeight) {
+      this.props.actions.persistEditorHeight(this.props.queryEditor,
+        this.northPaneRef.current.clientHeight);
     }
   }
   onSqlChanged(sql) {
     this.setState({ sql });
   }
+  // One layer of abstraction for easy spying in unit tests
+  getSqlEditorHeight() {
+    return this.sqlEditorRef.current ? this.sqlEditorRef.current.clientHeight : 0;
+  }
+  // Return the heights for the ace editor and the south pane as an object
+  // given the height of the sql editor, north pane percent and south pane percent.
+  getAceEditorAndSouthPaneHeights(height, northPercent, southPercent) {
+    return {
+      aceEditorHeight: height * northPercent / 100 - SQL_TOOLBAR_HEIGHT - GUTTER_HEIGHT / 2,
+      southPaneHeight: height * southPercent / 100,
+    };
+  }
   getHotkeyConfig() {
     return [
       {
@@ -187,9 +207,42 @@ class SqlEditor extends React.PureComponent {
   ctasChanged(event) {
     this.setState({ ctas: event.target.value });
   }
-  sqlEditorHeight() {
-    const horizontalScrollbarHeight = 25;
-    return parseInt(this.props.getHeight(), 10) - horizontalScrollbarHeight;
+  queryPane() {
+    const hotkeys = this.getHotkeyConfig();
+    const { aceEditorHeight, southPaneHeight } = this.getAceEditorAndSouthPaneHeights(
+      this.state.height, INITIAL_NORTH_PERCENT, INITIAL_SOUTH_PERCENT);
+    return (
+      <div className="queryPane">
+        <Split
+          sizes={[INITIAL_NORTH_PERCENT, INITIAL_SOUTH_PERCENT]}
+          minSize={200}
+          direction="vertical"
+          gutterSize={GUTTER_HEIGHT}
+          onDragStart={this.onResizeStart}
+          onDragEnd={this.onResizeEnd}
+        >
+          <div ref={this.northPaneRef}>
+            <AceEditorWrapper
+              actions={this.props.actions}
+              onBlur={this.setQueryEditorSql}
+              onChange={this.onSqlChanged}
+              queryEditor={this.props.queryEditor}
+              sql={this.props.queryEditor.sql}
+              tables={this.props.tables}
+              height={`${this.state.aceEditorHeight || aceEditorHeight}px`}
+              hotkeys={hotkeys}
+            />
+            {this.renderEditorBottomBar(hotkeys)}
+          </div>
+          <SouthPane
+            editorQueries={this.props.editorQueries}
+            dataPreviewQueries={this.props.dataPreviewQueries}
+            actions={this.props.actions}
+            height={this.state.southPaneHeight || southPaneHeight}
+          />
+        </Split>
+      </div>
+    );
   }
   renderEditorBottomBar(hotkeys) {
     let ctasControls;
@@ -258,6 +311,7 @@ class SqlEditor extends React.PureComponent {
                 onSave={this.props.actions.saveQuery}
                 schema={qe.schema}
                 dbId={qe.dbId}
+                saveQueryWarning={this.props.saveQueryWarning}
               />
             </span>
             <span className="m-r-5">
@@ -305,74 +359,23 @@ class SqlEditor extends React.PureComponent {
     );
   }
   render() {
-    const height = this.sqlEditorHeight();
-    const defaultNorthHeight = this.props.queryEditor.height || 200;
-    const hotkeys = this.getHotkeyConfig();
     return (
-      <div
-        className="SqlEditor"
-        style={{
-          height: height + 'px',
-        }}
-      >
-        <Row>
-          <Collapse
-            in={!this.props.hideLeftBar}
-          >
-            <Col
-              xs={6}
-              sm={5}
-              md={4}
-              lg={3}
-            >
-              <SqlEditorLeftBar
-                height={height}
-                database={this.props.database}
-                queryEditor={this.props.queryEditor}
-                tables={this.props.tables}
-                actions={this.props.actions}
-              />
-            </Col>
-          </Collapse>
-          <Col
-            xs={this.props.hideLeftBar ? 12 : 6}
-            sm={this.props.hideLeftBar ? 12 : 7}
-            md={this.props.hideLeftBar ? 12 : 8}
-            lg={this.props.hideLeftBar ? 12 : 9}
-            style={{ height: this.state.height }}
-          >
-            <SplitPane
-              split="horizontal"
-              defaultSize={defaultNorthHeight}
-              minSize={100}
-              onChange={this.onResize}
-            >
-              <div ref="ace" style={{ width: '100%' }}>
-                <div>
-                  <AceEditorWrapper
-                    actions={this.props.actions}
-                    onBlur={this.setQueryEditorSql}
-                    onChange={this.onSqlChanged}
-                    queryEditor={this.props.queryEditor}
-                    sql={this.props.queryEditor.sql}
-                    tables={this.props.tables}
-                    height={((this.state.editorPaneHeight || defaultNorthHeight) - 50) + 'px'}
-                    hotkeys={hotkeys}
-                  />
-                  {this.renderEditorBottomBar(hotkeys)}
-                </div>
-              </div>
-              <div ref="south">
-                <SouthPane
-                  editorQueries={this.props.editorQueries}
-                  dataPreviewQueries={this.props.dataPreviewQueries}
-                  actions={this.props.actions}
-                  height={this.state.southPaneHeight || 0}
-                />
-              </div>
-            </SplitPane>
-          </Col>
-        </Row>
+      <div ref={this.sqlEditorRef} className="SqlEditor">
+        <CSSTransition
+          classNames="schemaPane"
+          in={!this.props.hideLeftBar}
+          timeout={300}
+        >
+          <div className="schemaPane">
+            <SqlEditorLeftBar
+              database={this.props.database}
+              queryEditor={this.props.queryEditor}
+              tables={this.props.tables}
+              actions={this.props.actions}
+            />
+          </div>
+        </CSSTransition>
+        {this.queryPane()}
       </div>
     );
   }

superset/assets/src/SqlLab/components/SqlEditorLeftBar.jsx

@@ -20,7 +20,6 @@ import React from 'react';
 import PropTypes from 'prop-types';
 import { Button } from 'react-bootstrap';
 import { t } from '@superset-ui/translation';
-
 import TableElement from './TableElement';
 import TableSelector from '../../components/TableSelector';
 
@@ -106,7 +105,7 @@ export default class SqlEditorLeftBar extends React.PureComponent {
     const tableMetaDataHeight = this.props.height - 130; // 130 is the height of the selects above
     const qe = this.props.queryEditor;
     return (
-      <div className="clearfix">
+      <div className="sqlEditorLeftBar">
         <TableSelector
           dbId={qe.dbId}
           schema={qe.schema}

superset/assets/src/SqlLab/components/TabbedSqlEditors.jsx

@@ -39,12 +39,13 @@ const propTypes = {
   queryEditors: PropTypes.array,
   tabHistory: PropTypes.array.isRequired,
   tables: PropTypes.array.isRequired,
-  getHeight: PropTypes.func.isRequired,
   offline: PropTypes.bool,
+  saveQueryWarning: PropTypes.string,
 };
 const defaultProps = {
   queryEditors: [],
   offline: false,
+  saveQueryWarning: null,
 };
 
 let queryCount = 1;
@@ -238,7 +239,6 @@ class TabbedSqlEditors extends React.PureComponent {
             <div className="panel-body">
               {isSelected && (
                 <SqlEditor
-                  getHeight={this.props.getHeight}
                   tables={this.props.tables.filter(xt => xt.queryEditorId === qe.id)}
                   queryEditor={qe}
                   editorQueries={this.state.queriesArray}
@@ -249,6 +249,7 @@ class TabbedSqlEditors extends React.PureComponent {
                   hideLeftBar={this.state.hideLeftBar}
                   defaultQueryLimit={this.props.defaultQueryLimit}
                   maxRow={this.props.maxRow}
+                  saveQueryWarning={this.props.saveQueryWarning}
                 />
               )}
             </div>
@@ -291,6 +292,7 @@ function mapStateToProps({ sqlLab, common }) {
     offline: sqlLab.offline,
     defaultQueryLimit: common.conf.DEFAULT_SQLLAB_LIMIT,
     maxRow: common.conf.SQL_MAX_ROW,
+    saveQueryWarning: common.conf.SQLLAB_SAVE_WARNING_MESSAGE,
   };
 }
 function mapDispatchToProps(dispatch) {

superset/assets/src/SqlLab/main.less

@@ -135,7 +135,8 @@ div.Workspace {
     background-color: #e8e8e8;
     display: flex;
     justify-content: space-between;
-    border-bottom: 2px solid #ccc;
+    border: 1px solid #ccc;
+    border-top: 0;
 
     form {
         margin-block-end: 0;
@@ -193,21 +194,67 @@ div.Workspace {
     background-color: transparent !important;
 }
 
-.SqlEditor {
-    .Resizer {
-        -moz-box-sizing: border-box;
-        -webkit-box-sizing: border-box;
-        box-sizing: border-box;
+.SqlLab {
+    .tab-content {
+        height: 100%;
     }
 
-    .Resizer.horizontal {
-        height: 4px;
+    #brace-editor {
+        height: calc(100% - 51px);
+    }
+
+    .ace_content {
+        height: 100%;
+    }
+
+    .SouthPane {
+        height: 100%;
+    }
+}
+
+.SqlEditor {
+    display: flex;
+    flex-direction: row;
+    height: 100%;
+
+    .schemaPane {
+        flex-grow: 1;
+        transition: all .3s ease-in-out;
+    }
+
+    .schemaPane-enter-done, .schemaPane-exit {
+        transform: translateX(0);
+    }
+
+    .schemaPane-enter-active, .schemaPane-exit-active {
+        transform: translateX(-50%);
+    }
+
+    .schemaPane-enter, .schemaPane-exit-done {
+        transform: translateX(-100%);
+        max-width: 0;
+        overflow: hidden;
+    }
+
+    .queryPane {
+        flex-grow: 8;
+        position: relative;
+        margin-left: 15px;
+    }
+
+    .schemaPane-exit-done + .queryPane {
+        margin-left: 0;
+    }
+
+    .gutter {
         border-top: 1px solid #ccc;
         border-bottom: 1px solid #ccc;
+        width: 3%;
+        margin: 3px 47%;
+    }
+
+    .gutter.gutter-vertical {
         cursor: row-resize;
-        width: 4%;
-        margin-top: 4px;
-        margin-left: 47%;
     }
 }
 
@@ -298,9 +345,6 @@ a.Link {
 .tooltip-inner {
     max-width: 500px;
 }
-.SplitPane.horizontal {
-    padding-right: 4px;
-}
 .SouthPane {
     margin-top: 10px;
     position: absolute;

superset/assets/src/chart/ChartRenderer.jsx

@@ -24,6 +24,7 @@ import { ChartProps } from '@superset-ui/chart';
 import { Tooltip } from 'react-bootstrap';
 import { Logger, LOG_ACTIONS_RENDER_CHART } from '../logger';
 import SuperChart from '../visualizations/core/components/SuperChart';
+import transformBigNumber from './transformBigNumber';
 
 const propTypes = {
   annotationData: PropTypes.object,
@@ -67,9 +68,10 @@ class ChartRenderer extends React.Component {
     this.handleAddFilter = this.handleAddFilter.bind(this);
     this.handleRenderSuccess = this.handleRenderSuccess.bind(this);
     this.handleRenderFailure = this.handleRenderFailure.bind(this);
+    this.preTransformProps = this.preTransformProps.bind(this);
   }
 
-  shouldComponentUpdate(nextProps) {
+  shouldComponentUpdate(nextProps, nextState) {
     if (
       nextProps.queryResponse &&
       ['success', 'rendered'].indexOf(nextProps.chartStatus) > -1 &&
@@ -79,6 +81,7 @@ class ChartRenderer extends React.Component {
         nextProps.queryResponse !== this.props.queryResponse ||
         nextProps.height !== this.props.height ||
         nextProps.width !== this.props.width ||
+        nextState.tooltip !== this.state.tooltip ||
         nextProps.triggerRender)
     ) {
       return true;
@@ -149,6 +152,18 @@ class ChartRenderer extends React.Component {
     });
   }
 
+  preTransformProps(chartProps) {
+    const payload = chartProps.payload;
+    const data = transformBigNumber(payload.data);
+    return new ChartProps({
+      ...chartProps,
+      payload: {
+        ...payload,
+        data,
+      },
+    });
+  }
+
   renderTooltip() {
     const { tooltip } = this.state;
     if (tooltip && tooltip.content) {
@@ -192,6 +207,7 @@ class ChartRenderer extends React.Component {
           className={`${snakeCase(vizType)}`}
           chartType={vizType}
           chartProps={skipChartRendering ? null : this.prepareChartProps()}
+          preTransformProps={this.preTransformProps}
           onRenderSuccess={this.handleRenderSuccess}
           onRenderFailure={this.handleRenderFailure}
         />

superset/assets/src/chart/chartAction.js

@@ -152,6 +152,13 @@ export function updateQueryFormData(value, key) {
   return { type: UPDATE_QUERY_FORM_DATA, value, key };
 }
 
+// in the sql lab -> explore flow, user can inline edit chart title,
+// then the chart will be assigned a new slice_id
+export const UPDATE_CHART_ID = 'UPDATE_CHART_ID';
+export function updateChartId(newId, key = 0) {
+  return { type: UPDATE_CHART_ID, newId, key };
+}
+
 export const ADD_CHART = 'ADD_CHART';
 export function addChart(chart, key) {
   return { type: ADD_CHART, chart, key };
@@ -239,7 +246,10 @@ export function runQuery(formData, force = false, timeout = 60, key) {
 export function redirectSQLLab(formData) {
   return (dispatch) => {
     const { url } = getExploreUrlAndPayload({ formData, endpointType: 'query' });
-    return SupersetClient.get({ url })
+    return SupersetClient.post({
+      url,
+      postPayload: { form_data: formData },
+    })
       .then(({ json }) => {
         const redirectUrl = new URL(window.location);
         redirectUrl.pathname = '/superset/sqllab';

superset/assets/src/chart/chartReducer.js

@@ -168,6 +168,14 @@ export default function chartReducer(charts = {}, action) {
   if (action.type === actions.REMOVE_CHART) {
     delete charts[action.key];
     return charts;
+  } else if (action.type === actions.UPDATE_CHART_ID) {
+    const { newId, key } = action;
+    charts[newId] = {
+      ...charts[key],
+      id: newId,
+    };
+    delete charts[key];
+    return charts;
   }
 
   if (action.type in actionHandlers) {

superset/assets/src/chart/transformBigNumber.js

@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+// This method transforms any BigNumber object in the given payload to its
+// 64-bit float representation. It is a temporary fix so charts receive
+// floats instead of BigNumber instances in their props to properly render.
+import BigNumber from 'bignumber.js';
+
+export default function transform(payload) {
+  if (!payload) {
+    return payload;
+  } else if (BigNumber.isBigNumber(payload)) {
+    return payload.toNumber();
+  } else if (payload.constructor === Object) {
+    for (const key in payload) {
+      if (payload.hasOwnProperty(key)) {
+        // Modify in place to prevent creating large payloads
+        // eslint-disable-next-line no-param-reassign
+        payload[key] = transform(payload[key]);
+      }
+    }
+  } else if (payload.constructor === Array) {
+    payload.forEach((elem, idx) => {
+      // Modify in place to prevent creating large payloads
+      // eslint-disable-next-line no-param-reassign
+      payload[idx] = transform(elem);
+    });
+  }
+  return payload;
+}

superset/assets/src/components/AsyncSelect.jsx

@@ -83,17 +83,15 @@ class AsyncSelect extends React.PureComponent {
 
   render() {
     return (
-      <div>
-        <Select
-          placeholder={this.props.placeholder}
-          options={this.state.options}
-          value={this.props.value}
-          isLoading={this.state.isLoading}
-          onChange={this.onChange}
-          valueRenderer={this.props.valueRenderer}
-          {...this.props}
-        />
-      </div>
+      <Select
+        placeholder={this.props.placeholder}
+        options={this.state.options}
+        value={this.props.value}
+        isLoading={this.state.isLoading}
+        onChange={this.onChange}
+        valueRenderer={this.props.valueRenderer}
+        {...this.props}
+      />
     );
   }
 }

superset/assets/src/components/EditableTitle.jsx

@@ -55,7 +55,6 @@ export default class EditableTitle extends React.PureComponent {
     this.handleClick = this.handleClick.bind(this);
     this.handleBlur = this.handleBlur.bind(this);
     this.handleChange = this.handleChange.bind(this);
-    this.handleKeyUp = this.handleKeyUp.bind(this);
     this.handleKeyPress = this.handleKeyPress.bind(this);
 
     // Used so we can access the DOM element if a user clicks on this component.
@@ -112,21 +111,16 @@ export default class EditableTitle extends React.PureComponent {
     }
   }
 
-  handleKeyUp(ev) {
-    // this entire method exists to support using EditableTitle as the title of a
-    // react-bootstrap Tab, as a workaround for this line in react-bootstrap https://goo.gl/ZVLmv4
-    //
-    // tl;dr when a Tab EditableTitle is being edited, typically the Tab it's within has been
-    // clicked and is focused/active. for accessibility, when focused the Tab <a /> intercepts
-    // the ' ' key (among others, including all arrows) and onChange() doesn't fire. somehow
-    // keydown is still called so we can detect this and manually add a ' ' to the current title
-    if (ev.key === ' ') {
-      let title = ev.target.value;
-      const titleLength = (title || '').length;
-      if (title && title[titleLength - 1] !== ' ') {
-        title = `${title} `;
-        this.setState(() => ({ title }));
-      }
+  // this entire method exists to support using EditableTitle as the title of a
+  // react-bootstrap Tab, as a workaround for this line in react-bootstrap https://goo.gl/ZVLmv4
+  //
+  // tl;dr when a Tab EditableTitle is being edited, typically the Tab it's within has been
+  // clicked and is focused/active. for accessibility, when focused the Tab <a /> intercepts
+  // the ' ' key (among others, including all arrows) and onChange() doesn't fire. somehow
+  // keydown is still called so we can detect this and manually add a ' ' to the current title
+  handleKeyDown(event) {
+    if (event.key === ' ') {
+      event.stopPropagation();
     }
   }
 
@@ -170,7 +164,7 @@ export default class EditableTitle extends React.PureComponent {
         required
         value={value}
         className={!title ? 'text-muted' : null}
-        onKeyUp={this.handleKeyUp}
+        onKeyDown={this.handleKeyDown}
         onChange={this.handleChange}
         onBlur={this.handleBlur}
         onClick={this.handleClick}
@@ -184,7 +178,7 @@ export default class EditableTitle extends React.PureComponent {
         type={isEditing ? 'text' : 'button'}
         value={value}
         className={!title ? 'text-muted' : null}
-        onKeyUp={this.handleKeyUp}
+        onKeyDown={this.handleKeyDown}
         onChange={this.handleChange}
         onBlur={this.handleBlur}
         onClick={this.handleClick}

superset/assets/src/components/RefreshLabel.jsx

@@ -18,49 +18,26 @@
  */
 import React from 'react';
 import PropTypes from 'prop-types';
-import { Label } from 'react-bootstrap';
 import TooltipWrapper from './TooltipWrapper';
 
+import './RefreshLabel.less';
+
 const propTypes = {
   onClick: PropTypes.func,
-  className: PropTypes.string,
   tooltipContent: PropTypes.string.isRequired,
 };
 
 class RefreshLabel extends React.PureComponent {
-  constructor(props) {
-    super(props);
-    this.state = {
-      hovered: false,
-    };
-  }
-
-  mouseOver() {
-    this.setState({ hovered: true });
-  }
-
-  mouseOut() {
-    this.setState({ hovered: false });
-  }
-
   render() {
-    const labelStyle = this.state.hovered ? 'primary' : 'default';
-    const tooltip = 'Click to ' + this.props.tooltipContent;
     return (
       <TooltipWrapper
-        tooltip={tooltip}
+        tooltip={this.props.tooltipContent}
         label="cache-desc"
       >
-        <Label
-          className={this.props.className}
-          bsStyle={labelStyle}
-          style={{ fontSize: '13px', marginRight: '5px', cursor: 'pointer' }}
+        <i
+          className="RefreshLabel fa fa-refresh pointer"
           onClick={this.props.onClick}
-          onMouseOver={this.mouseOver.bind(this)}
-          onMouseOut={this.mouseOut.bind(this)}
-        >
-          <i className="fa fa-refresh" />
-        </Label>
+        />
       </TooltipWrapper>);
   }
 }

superset/assets/src/components/RefreshLabel.less

@@ -0,0 +1,27 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+@import "../../stylesheets/less/cosmo/variables.less";
+
+.RefreshLabel:hover {
+  color: @brand-primary;
+}
+
+.RefreshLabel {
+  color: @gray-light;
+}

superset/assets/src/components/TableSelector.css

@@ -0,0 +1,38 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+.TableSelector .fa-refresh {
+  padding-left: 9px;
+}
+.TableSelector .refresh-col {
+  display: flex;
+  align-items: center;
+  width: 30px;
+}
+.TableSelector .section {
+  padding-bottom: 5px;
+  display: flex;
+  flex-direction: row;
+}
+.TableSelector .select {
+  flex-grow: 1;
+}
+.TableSelector .divider {
+  border-bottom: 1px solid #f2f2f2;
+  margin: 10px 0;
+}

superset/assets/src/components/TableSelector.jsx

@@ -20,12 +20,13 @@ import React from 'react';
 import PropTypes from 'prop-types';
 import Select from 'react-virtualized-select';
 import createFilterOptions from 'react-select-fast-filter-options';
-import { ControlLabel, Col, Label } from 'react-bootstrap';
+import { ControlLabel, Label } from 'react-bootstrap';
 import { t } from '@superset-ui/translation';
 import { SupersetClient } from '@superset-ui/connection';
 
 import AsyncSelect from './AsyncSelect';
 import RefreshLabel from './RefreshLabel';
+import './TableSelector.css';
 
 const propTypes = {
   dbId: PropTypes.number.isRequired,
@@ -37,7 +38,6 @@ const propTypes = {
   tableNameSticky: PropTypes.bool,
   tableName: PropTypes.string,
   database: PropTypes.object,
-  horizontal: PropTypes.bool,
   sqlLabMode: PropTypes.bool,
   onChange: PropTypes.func,
   clearable: PropTypes.bool,
@@ -51,7 +51,6 @@ const defaultProps = {
   onTableChange: () => {},
   onChange: () => {},
   tableNameSticky: true,
-  horizontal: false,
   sqlLabMode: true,
   clearable: true,
 };
@@ -112,7 +111,11 @@ export default class TableSelector extends React.PureComponent {
     if (data.result.length === 0) {
       this.props.handleError(t("It seems you don't have access to any database"));
     }
-    return data.result;
+    return data.result.map(row => ({
+      ...row,
+      // label is used for the typeahead
+      label: `${row.backend} ${row.database_name}`,
+    }));
   }
   fetchTables(force, substr) {
     // This can be large so it shouldn't be put in the Redux store
@@ -196,8 +199,16 @@ export default class TableSelector extends React.PureComponent {
         {db.database_name}
       </span>);
   }
-  renderDatabaseSelect() {
+  renderSelectRow(select, refreshBtn) {
     return (
+      <div className="section">
+        <span className="select">{select}</span>
+        <span className="refresh-col">{refreshBtn}</span>
+      </div>
+    );
+  }
+  renderDatabaseSelect() {
+    return this.renderSelectRow(
       <AsyncSelect
         dataEndpoint={
           '/databaseasync/api/' +
@@ -223,33 +234,25 @@ export default class TableSelector extends React.PureComponent {
       />);
   }
   renderSchema() {
-    return (
-      <div className="m-t-5">
-        <div className="row">
-          <div className="col-md-11 col-xs-11 p-r-2">
-            <Select
-              name="select-schema"
-              placeholder={t('Select a schema (%s)', this.state.schemaOptions.length)}
-              options={this.state.schemaOptions}
-              value={this.props.schema}
-              valueRenderer={o => (
-                <div>
-                  <span className="text-muted">{t('Schema:')}</span> {o.label}
-                </div>
-              )}
-              isLoading={this.state.schemaLoading}
-              autosize={false}
-              onChange={this.changeSchema}
-            />
+    return this.renderSelectRow(
+      <Select
+        name="select-schema"
+        placeholder={t('Select a schema (%s)', this.state.schemaOptions.length)}
+        options={this.state.schemaOptions}
+        value={this.props.schema}
+        valueRenderer={o => (
+          <div>
+            <span className="text-muted">{t('Schema:')}</span> {o.label}
           </div>
-          <div className="col-md-1 col-xs-1 p-l-0 p-t-8">
-            <RefreshLabel
-              onClick={() => this.onDatabaseChange({ id: this.props.dbId }, true)}
-              tooltipContent={t('force refresh schema list')}
-            />
-          </div>
-        </div>
-      </div>
+        )}
+        isLoading={this.state.schemaLoading}
+        autosize={false}
+        onChange={this.changeSchema}
+      />,
+      <RefreshLabel
+        onClick={() => this.onDatabaseChange({ id: this.props.dbId }, true)}
+        tooltipContent={t('Force refresh schema list')}
+      />,
     );
   }
   renderTable() {
@@ -262,49 +265,39 @@ export default class TableSelector extends React.PureComponent {
       tableSelectDisabled = true;
     }
     const options = this.addOptionIfMissing(this.state.tableOptions, this.state.tableName);
-    return (
-      <div className="m-t-5">
-        <div className="row">
-          <div className="col-md-11 col-xs-11 p-r-2">
-            {this.props.schema ? (
-              <Select
-                name="select-table"
-                ref="selectTable"
-                isLoading={this.state.tableLoading}
-                placeholder={t('Select table or type table name')}
-                autosize={false}
-                onChange={this.changeTable}
-                filterOptions={this.state.filterOptions}
-                options={options}
-                value={this.state.tableName}
-              />
-            ) : (
-              <Select
-                async
-                name="async-select-table"
-                ref="selectTable"
-                placeholder={tableSelectPlaceholder}
-                disabled={tableSelectDisabled}
-                autosize={false}
-                onChange={this.changeTable}
-                value={this.state.tableName}
-                loadOptions={this.getTableNamesBySubStr}
-              />
-            )}
-          </div>
-          <div className="col-md-1 col-xs-1 p-l-0 p-t-8">
-            <RefreshLabel
-              onClick={() => this.changeSchema({ value: this.props.schema }, true)}
-              tooltipContent={t('force refresh table list')}
-            />
-          </div>
-        </div>
-      </div>);
+    const select = this.props.schema ? (
+      <Select
+        name="select-table"
+        ref="selectTable"
+        isLoading={this.state.tableLoading}
+        placeholder={t('Select table or type table name')}
+        autosize={false}
+        onChange={this.changeTable}
+        filterOptions={this.state.filterOptions}
+        options={options}
+        value={this.state.tableName}
+      />) : (
+        <Select
+          async
+          name="async-select-table"
+          ref="selectTable"
+          placeholder={tableSelectPlaceholder}
+          disabled={tableSelectDisabled}
+          autosize={false}
+          onChange={this.changeTable}
+          value={this.state.tableName}
+          loadOptions={this.getTableNamesBySubStr}
+        />);
+    return this.renderSelectRow(
+      select,
+      <RefreshLabel
+        onClick={() => this.changeSchema({ value: this.props.schema }, true)}
+        tooltipContent={t('Force refresh table list')}
+      />);
   }
   renderSeeTableLabel() {
     return (
-      <div>
-        <hr />
+      <div className="section">
         <ControlLabel>
           {t('See table schema')}{' '}
           <small>
@@ -318,21 +311,15 @@ export default class TableSelector extends React.PureComponent {
       </div>);
   }
   render() {
-    if (this.props.horizontal) {
-      return (
-        <div>
-          <Col md={4}>{this.renderDatabaseSelect()}</Col>
-          <Col md={4}>{this.renderSchema()}</Col>
-          <Col md={4}>{this.renderTable()}</Col>
-        </div>);
-    }
     return (
-      <div>
-        <div>{this.renderDatabaseSelect()}</div>
-        <div className="m-t-5">{this.renderSchema()}</div>
+      <div className="TableSelector">
+        {this.renderDatabaseSelect()}
+        {this.renderSchema()}
+        <div className="divider" />
         {this.props.sqlLabMode && this.renderSeeTableLabel()}
-        <div className="m-t-5">{this.renderTable()}</div>
-      </div>);
+        {this.renderTable()}
+      </div>
+    );
   }
 }
 TableSelector.propTypes = propTypes;

superset/assets/src/dashboard/actions/dashboardState.js

@@ -253,7 +253,10 @@ export function addSliceToDashboard(id) {
         ),
       );
     }
-    const form_data = selectedSlice.form_data;
+    const form_data = {
+      ...selectedSlice.form_data,
+      slice_id: selectedSlice.slice_id,
+    };
     const newChart = {
       ...initChart,
       id,

superset/assets/src/datasource/DatasourceEditor.jsx

@@ -448,6 +448,14 @@ export class DatasourceEditor extends React.PureComponent {
           label={t('Hours offset')}
           control={<TextControl />}
         />
+        { this.state.isSqla &&
+          <Field
+            fieldKey="template_params"
+            label={t('Template parameters')}
+            descr={t('A set of parameters that become available in the query using Jinja templating syntax')}
+            control={<TextControl />}
+          />
+        }
       </Fieldset>);
   }
 

superset/assets/src/datasource/DatasourceModal.jsx

@@ -49,10 +49,8 @@ class DatasourceModal extends React.PureComponent {
     super(props);
     this.state = {
       errors: [],
-      showDatasource: false,
       datasource: props.datasource,
     };
-    this.toggleShowDatasource = this.toggleShowDatasource.bind(this);
     this.setSearchRef = this.setSearchRef.bind(this);
     this.onDatasourceChange = this.onDatasourceChange.bind(this);
     this.onClickSave = this.onClickSave.bind(this);
@@ -111,10 +109,6 @@ class DatasourceModal extends React.PureComponent {
     this.dialog = ref;
   }
 
-  toggleShowDatasource() {
-    this.setState({ showDatasource: !this.state.showDatasource });
-  }
-
   renderSaveDialog() {
     return (
       <div>

superset/assets/src/explore/components/ExploreChartHeader.jsx

@@ -65,6 +65,7 @@ class ExploreChartHeader extends React.PureComponent {
       .then((json) => {
         const { data } = json;
         if (isNewSlice) {
+          this.props.actions.updateChartId(data.slice.slice_id, 0);
           this.props.actions.createNewSlice(
             data.can_add, data.can_download, data.can_overwrite,
             data.slice, data.form_data);

superset/assets/src/explore/components/ExploreViewContainer.jsx

@@ -109,6 +109,9 @@ class ExploreViewContainer extends React.Component {
     const wasRendered =
       ['rendered', 'failed', 'stopped'].indexOf(this.props.chart.chartStatus) > -1;
     const isRendered = ['rendered', 'failed', 'stopped'].indexOf(nextProps.chart.chartStatus) > -1;
+    if (nextProps.chart.id !== this.props.chart.id) {
+      this.loadingLog.sourceId = nextProps.chart.id;
+    }
     if (!wasRendered && isRendered) {
       Logger.send(this.loadingLog);
     }

superset/assets/src/explore/components/controls/AnnotationLayerControl.jsx

@@ -176,10 +176,12 @@ AnnotationLayerControl.defaultProps = defaultProps;
 // directly, could not figure out how to get access to the color_scheme
 function mapStateToProps({ charts, explore }) {
   const chartKey = getChartKey(explore);
+  const chart = charts[chartKey] || charts[0] || {};
+
   return {
     colorScheme: (explore.controls || {}).color_scheme.value,
-    annotationError: charts[chartKey].annotationError,
-    annotationQuery: charts[chartKey].annotationQuery,
+    annotationError: chart.annotationError,
+    annotationQuery: chart.annotationQuery,
     vizType: explore.controls.viz_type.value,
   };
 }

superset/assets/src/explore/components/controls/DatasourceControl.jsx

@@ -19,13 +19,19 @@
 import React from 'react';
 import PropTypes from 'prop-types';
 import {
+  Col,
+  Collapse,
   Label,
   OverlayTrigger,
+  Row,
   Tooltip,
+  Well,
 } from 'react-bootstrap';
 import { t } from '@superset-ui/translation';
 
 import ControlHeader from '../ControlHeader';
+import ColumnOption from '../../../components/ColumnOption';
+import MetricOption from '../../../components/MetricOption';
 import DatasourceModal from '../../../datasource/DatasourceModal';
 
 const propTypes = {
@@ -52,25 +58,51 @@ class DatasourceControl extends React.PureComponent {
     };
     this.toggleShowDatasource = this.toggleShowDatasource.bind(this);
     this.toggleEditDatasourceModal = this.toggleEditDatasourceModal.bind(this);
-  }
-
-  onChange(vizType) {
-    this.props.onChange(vizType);
-    this.setState({ showModal: false });
+    this.renderDatasource = this.renderDatasource.bind(this);
   }
 
   toggleShowDatasource() {
     this.setState(({ showDatasource }) => ({ showDatasource: !showDatasource }));
   }
 
-  toggleModal() {
-    this.setState(({ showModal }) => ({ showModal: !showModal }));
-  }
   toggleEditDatasourceModal() {
     this.setState(({ showEditDatasourceModal }) => ({
       showEditDatasourceModal: !showEditDatasourceModal,
     }));
   }
+  renderDatasource() {
+    const datasource = this.props.datasource;
+    return (
+      <div className="m-t-10">
+        <Well className="m-t-0">
+          <div className="m-b-10">
+            <Label>
+              <i className="fa fa-database" /> {datasource.database.backend}
+            </Label>
+            {` ${datasource.database.name} `}
+          </div>
+          <Row className="datasource-container">
+            <Col md={6}>
+              <strong>Columns</strong>
+              {datasource.columns.map(col => (
+                <div key={col.column_name}>
+                  <ColumnOption showType column={col} />
+                </div>
+              ))}
+            </Col>
+            <Col md={6}>
+              <strong>Metrics</strong>
+              {datasource.metrics.map(m => (
+                <div key={m.metric_name}>
+                  <MetricOption metric={m} showType />
+                </div>
+              ))}
+            </Col>
+          </Row>
+        </Well>
+      </div>
+    );
+  }
   render() {
     return (
       <div>
@@ -85,6 +117,21 @@ class DatasourceControl extends React.PureComponent {
             {this.props.datasource.name}
           </Label>
         </OverlayTrigger>
+        <OverlayTrigger
+          placement="right"
+          overlay={
+            <Tooltip id={'toggle-datasource-tooltip'}>
+              {t('Expand/collapse datasource configuration')}
+            </Tooltip>
+          }
+        >
+          <a href="#">
+            <i
+              className={`fa fa-${this.state.showDatasource ? 'minus' : 'plus'}-square m-r-5`}
+              onClick={this.toggleShowDatasource}
+            />
+          </a>
+        </OverlayTrigger>
         {this.props.datasource.type === 'table' &&
           <OverlayTrigger
             placement="right"
@@ -102,6 +149,7 @@ class DatasourceControl extends React.PureComponent {
               <i className="fa fa-flask m-r-5" />
             </a>
           </OverlayTrigger>}
+        <Collapse in={this.state.showDatasource}>{this.renderDatasource()}</Collapse>
         <DatasourceModal
           datasource={this.props.datasource}
           show={this.state.showEditDatasourceModal}

superset/assets/src/explore/components/controls/MetricsControl.jsx

@@ -60,6 +60,21 @@ function isDictionaryForAdhocMetric(value) {
   return value && !(value instanceof AdhocMetric) && value.expressionType;
 }
 
+function columnsContainAllMetrics(value, nextProps) {
+  const columnNames = new Set(
+    [...nextProps.columns, ...nextProps.savedMetrics]
+    // eslint-disable-next-line camelcase
+      .map(({ column_name, metric_name }) => (column_name || metric_name)),
+  );
+
+  return (Array.isArray(value) ? value : [value])
+    .filter(metric => metric)
+    // find column names
+    .map(metric => metric.column ? metric.column.column_name : metric.column_name || metric)
+    .filter(name => name)
+    .every(name => columnNames.has(name));
+}
+
 // adhoc metrics are stored as dictionaries in URL params. We convert them back into the
 // AdhocMetric class for typechecking, consistency and instance method access.
 function coerceAdhocMetrics(value) {
@@ -135,14 +150,22 @@ export default class MetricsControl extends React.PureComponent {
   }
 
   componentWillReceiveProps(nextProps) {
+    const { value } = this.props;
     if (
-      isEqual(this.props.columns) !== isEqual(nextProps.columns) ||
-      isEqual(this.props.savedMetrics) !== isEqual(nextProps.savedMetrics)
+      !isEqual(this.props.columns, nextProps.columns) ||
+      !isEqual(this.props.savedMetrics, nextProps.savedMetrics)
     ) {
+
       this.setState({ options: this.optionsForSelect(nextProps) });
-      this.props.onChange([]);
+
+      // Remove metrics if selected value no longer a column
+      const containsAllMetrics = columnsContainAllMetrics(value, nextProps);
+
+      if (!containsAllMetrics) {
+        this.props.onChange([]);
+      }
     }
-    if (this.props.value !== nextProps.value) {
+    if (value !== nextProps.value) {
       this.setState({ value: coerceAdhocMetrics(nextProps.value) });
     }
   }

superset/assets/src/explore/controlPanels/extraOverrides.js

@@ -0,0 +1,22 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+// For individual deployments to add custom overrides
+export default function extraOverrides(controlPanelConfigs) {
+  return controlPanelConfigs;
+}

superset/assets/src/explore/controlPanels/index.js

@@ -22,6 +22,7 @@
  */
 import { isFeatureEnabled, FeatureFlag } from 'src/featureFlags';
 import * as sections from './sections';
+import extraOverrides from './extraOverrides';
 
 import Area from './Area';
 import Bar from './Bar';
@@ -72,7 +73,7 @@ import DeckPolygon from './DeckPolygon';
 import DeckScatter from './DeckScatter';
 import DeckScreengrid from './DeckScreengrid';
 
-export const controlPanelConfigs = {
+export const controlPanelConfigs = extraOverrides({
   area: Area,
   bar: Bar,
   big_number: BigNumber,
@@ -122,7 +123,7 @@ export const controlPanelConfigs = {
   deck_scatter: DeckScatter,
   deck_screengrid: DeckScreengrid,
 
-};
+});
 
 export default controlPanelConfigs;
 

superset/assets/src/explore/exploreUtils.js

@@ -19,6 +19,7 @@
 /* eslint camelcase: 0 */
 import URI from 'urijs';
 import { availableDomains } from '../utils/hostNamesConfig';
+import { safeStringify } from '../utils/safeStringify';
 
 const MAX_URL_LENGTH = 8000;
 
@@ -71,7 +72,7 @@ export function getExploreLongUrl(formData, endpointType, allowOverflow = true,
   Object.keys(extraSearch).forEach((key) => {
     search[key] = extraSearch[key];
   });
-  search.form_data = JSON.stringify(formData);
+  search.form_data = safeStringify(formData);
   if (endpointType === 'standalone') {
     search.standalone = 'true';
   }

superset/assets/src/utils/safeStringify.ts

@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/**
+ * A Stringify function that will not crash when it runs into circular JSON references,
+ * unlike JSON.stringify. Any circular references are simply omitted, as if there had
+ * been no data present
+ * @param object any JSON object to be stringified
+ */
+export function safeStringify(object: any): string {
+  const cache = new Set();
+  return JSON.stringify(object, (key, value) => {
+    if (typeof value === 'object' && value !== null) {
+      if (cache.has(value)) {
+        // We've seen this object before
+        try {
+          // Quick deep copy to duplicate if this is a repeat rather than a circle.
+         return JSON.parse(JSON.stringify(value));
+        } catch (err) {
+          // Discard key if value cannot be duplicated.
+         return;
+        }
+      }
+      // Store the value in our cache.
+      cache.add(value);
+    }
+    return value;
+  });
+}

superset/assets/src/visualizations/Iframe/Iframe.jsx

@@ -0,0 +1,58 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+import Mustache from 'mustache';
+import React from 'react';
+import PropTypes from 'prop-types';
+
+const propTypes = {
+  className: PropTypes.string,
+  width: PropTypes.number.isRequired,
+  height: PropTypes.number.isRequired,
+  url: PropTypes.string,
+};
+const defaultProps = {
+  className: '',
+};
+
+class Iframe extends React.PureComponent {
+  render() {
+    const { className, url, width, height } = this.props;
+    const completeUrl = Mustache.render(url, {
+      width,
+      height,
+    });
+
+    return (
+      <iframe
+        className={className}
+        title="superset-iframe"
+        src={completeUrl}
+        style={{
+          width: '100%',
+          height,
+        }}
+      />
+    );
+  }
+}
+
+Iframe.propTypes = propTypes;
+Iframe.defaultProps = defaultProps;
+
+export default Iframe;

superset/assets/src/visualizations/Iframe/IframeChartPlugin.js

@@ -0,0 +1,38 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+import { t } from '@superset-ui/translation';
+import { ChartMetadata, ChartPlugin } from '@superset-ui/chart';
+import thumbnail from './images/thumbnail.png';
+import transformProps from './transformProps';
+
+const metadata = new ChartMetadata({
+  name: t('IFrame'),
+  description: 'HTML Inline Frame',
+  thumbnail,
+});
+
+export default class IframeChartPlugin extends ChartPlugin {
+  constructor() {
+    super({
+      metadata,
+      loadChart: () => import('./Iframe.jsx'),
+      transformProps,
+    });
+  }
+}

superset/assets/src/visualizations/Iframe/transformProps.js

@@ -16,23 +16,13 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-import Mustache from 'mustache';
-
-export default function iframeWidget(slice) {
-  const { selector, formData } = slice;
+export default function transformProps(chartProps) {
+  const { width, height, formData } = chartProps;
   const { url } = formData;
-  const width = slice.width();
-  const height = slice.height();
-  const container = document.querySelector(selector);
 
-  const completedUrl = Mustache.render(url, {
+  return {
     width,
     height,
-  });
-
-  const iframe = document.createElement('iframe');
-  iframe.style.width = '100%';
-  iframe.style.height = height;
-  iframe.setAttribute('src', completedUrl);
-  container.appendChild(iframe);
+    url,
+  };
 }

superset/assets/src/visualizations/Markup/Markup.css

@@ -17,16 +17,16 @@
  * under the License.
  */
 .markup.slice_container {
-    margin: 10px;
+  margin: 10px;
 }
 .separator {
-    background-color: transparent !important;
+  background-color: transparent !important;
 }
 .separator hr {
-    border: 0;
-    height: 1px;
-    background-image: linear-gradient(to right, rgba(0, 0, 0, 1), rgba(0, 0, 0, 1), rgba(0, 0, 0, 1), rgba(0, 0, 0, 0));
+  border: 0;
+  height: 1px;
+  background-image: linear-gradient(to right, rgba(0, 0, 0, 1), rgba(0, 0, 0, 1), rgba(0, 0, 0, 1), rgba(0, 0, 0, 0));
 }
 .separator .chart-header {
-    border: none !important;
+  border: none !important;
 }

superset/assets/src/visualizations/Markup/Markup.jsx

@@ -0,0 +1,76 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+import React from 'react';
+import PropTypes from 'prop-types';
+import './Markup.css';
+
+const propTypes = {
+  className: PropTypes.string,
+  height: PropTypes.number.isRequired,
+  isSeparator: PropTypes.bool,
+  html: PropTypes.string,
+  cssFiles: PropTypes.arrayOf(PropTypes.string),
+};
+const defaultProps = {
+  className: '',
+  isSeparator: false,
+  html: '',
+};
+
+const CONTAINER_STYLE = {
+  position: 'relative',
+  overflow: 'auto',
+};
+
+class Markup extends React.PureComponent {
+  render() {
+    const { className, height, isSeparator, html, cssFiles } = this.props;
+
+    return (
+      <div
+        className={className}
+        style={CONTAINER_STYLE}
+      >
+        <iframe
+          title="superset-markup"
+          frameBorder={0}
+          height={isSeparator ? height - 20 : height}
+          sandbox="allow-forms allow-popups allow-same-origin allow-scripts allow-top-navigation"
+          srcDoc={`
+            <html>
+              <head>
+                ${cssFiles.map(
+                  href => `<link rel="stylesheet" type="text/css" href="${href}" />`,
+                )}
+              </head>
+              <body style="background-color: transparent;">
+                ${html}
+              </body>
+            </html>`
+          }
+        />
+      </div>
+    );
+  }
+}
+
+Markup.propTypes = propTypes;
+Markup.defaultProps = defaultProps;
+
+export default Markup;

superset/assets/src/visualizations/Markup/MarkupChartPlugin.js

@@ -0,0 +1,38 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+import { t } from '@superset-ui/translation';
+import { ChartMetadata, ChartPlugin } from '@superset-ui/chart';
+import thumbnail from './images/thumbnail.png';
+import transformProps from './transformProps';
+
+const metadata = new ChartMetadata({
+  name: t('Markup'),
+  description: 'HTML Markup',
+  thumbnail,
+});
+
+export default class IframeChartPlugin extends ChartPlugin {
+  constructor() {
+    super({
+      metadata,
+      loadChart: () => import('./Markup.jsx'),
+      transformProps,
+    });
+  }
+}

superset/assets/src/visualizations/Markup/transformProps.js

@@ -0,0 +1,33 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+export default function transformProps(chartProps) {
+  const { height, payload, formData } = chartProps;
+  const { vizType } = formData;
+  const {
+    theme_css: cssFiles,
+    html,
+  } = payload.data;
+
+  return {
+    height,
+    cssFiles,
+    html,
+    isSeparator: vizType === 'separator',
+  };
+}

superset/assets/src/visualizations/TimeTable/TimeTable.jsx

@@ -23,6 +23,7 @@ import { scaleLinear } from 'd3-scale';
 import { Table, Thead, Th, Tr, Td } from 'reactable-arc';
 import { formatNumber } from '@superset-ui/number-format';
 import { formatTime } from '@superset-ui/time-format';
+import moment from 'moment';
 
 import MetricOption from '../../components/MetricOption';
 import InfoTooltipWithTrigger from '../../components/InfoTooltipWithTrigger';
@@ -148,7 +149,7 @@ class TimeTable extends React.PureComponent {
           renderTooltip={({ index }) => (
             <div>
               <strong>{formatNumber(column.d3format, sparkData[index])}</strong>
-              <div>{formatTime(column.dateFormat, entries[index].time)}</div>
+              <div>{formatTime(column.dateFormat, moment.utc(entries[index].time).toDate())}</div>
             </div>
           )}
         />

superset/assets/src/visualizations/deckgl/CategoricalDeckGLContainer.jsx

@@ -31,9 +31,9 @@ import { fitViewport } from './layers/common';
 const { getScale } = CategoricalColorNamespace;
 
 function getCategories(fd, data) {
-  const c = fd.color_picker || { r: 0, g: 0, b: 0, a: 1 };
+  const c = fd.colorPicker || { r: 0, g: 0, b: 0, a: 1 };
   const fixedColor = [c.r, c.g, c.b, 255 * c.a];
-  const colorFn = getScale(fd.color_scheme);
+  const colorFn = getScale(fd.colorScheme);
   const categories = {};
   data.forEach((d) => {
     if (d.cat_color != null && !categories.hasOwnProperty(d.cat_color)) {
@@ -70,7 +70,7 @@ export default class CategoricalDeckGLContainer extends React.PureComponent {
    */
   constructor(props) {
     super(props);
-    this.state = this.getInitialStateFromProps(props);
+    this.state = this.getStateFromProps(props);
 
     this.getLayers = this.getLayers.bind(this);
     this.onValuesChange = this.onValuesChange.bind(this);
@@ -78,6 +78,11 @@ export default class CategoricalDeckGLContainer extends React.PureComponent {
     this.toggleCategory = this.toggleCategory.bind(this);
     this.showSingleCategory = this.showSingleCategory.bind(this);
   }
+  componentWillReceiveProps(nextProps) {
+    if (nextProps.payload.form_data !== this.state.formData) {
+      this.setState({ ...this.getStateFromProps(nextProps) });
+    }
+  }
   onValuesChange(values) {
     this.setState({
       values: Array.isArray(values)
@@ -88,7 +93,7 @@ export default class CategoricalDeckGLContainer extends React.PureComponent {
   onViewportChange(viewport) {
     this.setState({ viewport });
   }
-  getInitialStateFromProps(props, state) {
+  getStateFromProps(props, state) {
     const features = props.payload.data.features || [];
     const timestamps = features.map(f => f.__timestamp);
     const categories = getCategories(props.formData, features);
@@ -103,7 +108,7 @@ export default class CategoricalDeckGLContainer extends React.PureComponent {
     // the granularity has to be read from the payload form_data, not the
     // props formData which comes from the instantaneous controls state
     const granularity = (
-      props.payload.form_data.time_grain_sqla ||
+      props.payload.form_data.timeGrainSqla ||
       props.payload.form_data.granularity ||
       'P1D'
     );
@@ -149,8 +154,8 @@ export default class CategoricalDeckGLContainer extends React.PureComponent {
     features = this.addColor(features, fd);
 
     // Apply user defined data mutator if defined
-    if (fd.js_data_mutator) {
-      const jsFnMutator = sandboxedEval(fd.js_data_mutator);
+    if (fd.jsDataMutator) {
+      const jsFnMutator = sandboxedEval(fd.jsDataMutator);
       features = jsFnMutator(features);
     }
 
@@ -175,8 +180,8 @@ export default class CategoricalDeckGLContainer extends React.PureComponent {
     return [getLayer(fd, filteredPayload, onAddFilter, setTooltip)];
   }
   addColor(data, fd) {
-    const c = fd.color_picker || { r: 0, g: 0, b: 0, a: 1 };
-    const colorFn = getScale(fd.color_scheme);
+    const c = fd.colorPicker || { r: 0, g: 0, b: 0, a: 1 };
+    const colorFn = getScale(fd.colorScheme);
     return data.map((d) => {
       let color;
       if (fd.dimension) {
@@ -224,14 +229,14 @@ export default class CategoricalDeckGLContainer extends React.PureComponent {
           viewport={this.state.viewport}
           onViewportChange={this.onViewportChange}
           mapboxApiAccessToken={this.props.mapboxApiKey}
-          mapStyle={this.props.formData.mapbox_style}
+          mapStyle={this.props.formData.mapboxStyle}
           setControlValue={this.props.setControlValue}
         >
           <Legend
             categories={this.state.categories}
             toggleCategory={this.toggleCategory}
             showSingleCategory={this.showSingleCategory}
-            position={this.props.formData.legend_position}
+            position={this.props.formData.legendPosition}
           />
         </AnimatableDeckGLContainer>
       </div>

superset/assets/src/visualizations/deckgl/Multi/Multi.jsx

@@ -56,7 +56,7 @@ class DeckMulti extends React.PureComponent {
       const filters = [
         ...(subslice.form_data.filters || []),
         ...(formData.filters || []),
-        ...(formData.extra_filters || []),
+        ...(formData.extraFilters || []),
       ];
       const subsliceCopy = {
         ...subslice,
@@ -70,7 +70,7 @@ class DeckMulti extends React.PureComponent {
           endpoint: getExploreLongUrl(subsliceCopy.form_data, 'json'),
         })
         .then(({ json }) => {
-          const layer = layerGenerators[subsliceCopy.form_data.viz_type](
+          const layer = layerGenerators[subsliceCopy.form_data.vizType](
             subsliceCopy.form_data,
             json,
           );
@@ -96,7 +96,7 @@ class DeckMulti extends React.PureComponent {
         mapboxApiAccessToken={payload.data.mapboxApiKey}
         viewport={viewport}
         layers={layers}
-        mapStyle={formData.mapbox_style}
+        mapStyle={formData.mapboxStyle}
         setControlValue={setControlValue}
       />
     );

superset/assets/src/visualizations/deckgl/factory.jsx

@@ -90,7 +90,7 @@ export function createDeckGLComponent(getLayer, getPoints) {
           mapboxApiAccessToken={payload.data.mapboxApiKey}
           viewport={viewport}
           layers={[layer]}
-          mapStyle={formData.mapbox_style}
+          mapStyle={formData.mapboxStyle}
           setControlValue={setControlValue}
           onViewportChange={this.onViewportChange}
         />);

superset/assets/src/visualizations/deckgl/layers/Arc/Arc.jsx

@@ -31,14 +31,14 @@ function getPoints(data) {
 
 export function getLayer(fd, payload, onAddFilter, setTooltip) {
   const data = payload.data.features;
-  const sc = fd.color_picker;
-  const tc = fd.target_color_picker;
+  const sc = fd.colorPicker;
+  const tc = fd.targetColorPicker;
   return new ArcLayer({
-    id: `path-layer-${fd.slice_id}`,
+    id: `path-layer-${fd.sliceId}`,
     data,
     getSourceColor: d => d.sourceColor || d.color || [sc.r, sc.g, sc.b, 255 * sc.a],
     getTargetColor: d => d.targetColor || d.color || [tc.r, tc.g, tc.b, 255 * tc.a],
-    strokeWidth: (fd.stroke_width) ? fd.stroke_width : 3,
+    strokeWidth: (fd.strokeWidth) ? fd.strokeWidth : 3,
     ...commonLayerProps(fd, setTooltip),
   });
 }

superset/assets/src/visualizations/deckgl/layers/Geojson/Geojson.jsx

@@ -77,8 +77,8 @@ const recurseGeoJson = (node, propOverrides, extraProps) => {
 
 export function getLayer(formData, payload, onAddFilter, setTooltip) {
   const fd = formData;
-  const fc = fd.fill_color_picker;
-  const sc = fd.stroke_color_picker;
+  const fc = fd.fillColorPicker;
+  const sc = fd.strokeColorPicker;
   const fillColor = [fc.r, fc.g, fc.b, 255 * fc.a];
   const strokeColor = [sc.r, sc.g, sc.b, 255 * sc.a];
   const propOverrides = {};
@@ -93,19 +93,19 @@ export function getLayer(formData, payload, onAddFilter, setTooltip) {
   recurseGeoJson(payload.data, propOverrides);
 
   let jsFnMutator;
-  if (fd.js_data_mutator) {
+  if (fd.jsDataMutator) {
     // Applying user defined data mutator if defined
-    jsFnMutator = sandboxedEval(fd.js_data_mutator);
+    jsFnMutator = sandboxedEval(fd.jsDataMutator);
     features = jsFnMutator(features);
   }
 
   return new GeoJsonLayer({
-    id: `geojson-layer-${fd.slice_id}`,
+    id: `geojson-layer-${fd.sliceId}`,
     filled: fd.filled,
     data: features,
     stroked: fd.stroked,
     extruded: fd.extruded,
-    pointRadiusScale: fd.point_radius_scale,
+    pointRadiusScale: fd.pointRadiusScale,
     ...commonLayerProps(fd, setTooltip),
   });
 }
@@ -145,7 +145,7 @@ function deckGeoJson(props) {
       mapboxApiAccessToken={payload.data.mapboxApiKey}
       viewport={viewport}
       layers={[layer]}
-      mapStyle={formData.mapbox_style}
+      mapStyle={formData.mapboxStyle}
       setControlValue={setControlValue}
     />
   );

superset/assets/src/visualizations/deckgl/layers/Grid/Grid.jsx

@@ -24,24 +24,24 @@ import { createDeckGLComponent } from '../../factory';
 
 export function getLayer(formData, payload, onAddFilter, setTooltip) {
   const fd = formData;
-  const c = fd.color_picker;
+  const c = fd.colorPicker;
   let data = payload.data.features.map(d => ({
     ...d,
     color: [c.r, c.g, c.b, 255 * c.a],
   }));
 
-  if (fd.js_data_mutator) {
+  if (fd.jsDataMutator) {
     // Applying user defined data mutator if defined
-    const jsFnMutator = sandboxedEval(fd.js_data_mutator);
+    const jsFnMutator = sandboxedEval(fd.jsDataMutator);
     data = jsFnMutator(data);
   }
 
-  const aggFunc = getAggFunc(fd.js_agg_function, p => p.weight);
+  const aggFunc = getAggFunc(fd.jsAggFunction, p => p.weight);
   return new GridLayer({
-    id: `grid-layer-${fd.slice_id}`,
+    id: `grid-layer-${fd.sliceId}`,
     data,
     pickable: true,
-    cellSize: fd.grid_size,
+    cellSize: fd.gridSize,
     minColor: [0, 0, 0, 0],
     extruded: fd.extruded,
     maxColor: [c.r, c.g, c.b, 255 * c.a],

superset/assets/src/visualizations/deckgl/layers/Hex/Hex.jsx

@@ -24,23 +24,23 @@ import { createDeckGLComponent } from '../../factory';
 
 export function getLayer(formData, payload, onAddFilter, setTooltip) {
   const fd = formData;
-  const c = fd.color_picker;
+  const c = fd.colorPicker;
   let data = payload.data.features.map(d => ({
     ...d,
     color: [c.r, c.g, c.b, 255 * c.a],
   }));
 
-  if (fd.js_data_mutator) {
+  if (fd.jsDataMutator) {
     // Applying user defined data mutator if defined
-    const jsFnMutator = sandboxedEval(fd.js_data_mutator);
+    const jsFnMutator = sandboxedEval(fd.jsDataMutator);
     data = jsFnMutator(data);
   }
-  const aggFunc = getAggFunc(fd.js_agg_function, p => p.weight);
+  const aggFunc = getAggFunc(fd.jsAggFunction, p => p.weight);
   return new HexagonLayer({
-    id: `hex-layer-${fd.slice_id}`,
+    id: `hex-layer-${fd.sliceId}`,
     data,
     pickable: true,
-    radius: fd.grid_size,
+    radius: fd.gridSize,
     minColor: [0, 0, 0, 0],
     extruded: fd.extruded,
     maxColor: [c.r, c.g, c.b, 255 * c.a],

superset/assets/src/visualizations/deckgl/layers/Path/Path.jsx

@@ -23,22 +23,22 @@ import { createDeckGLComponent } from '../../factory';
 
 export function getLayer(formData, payload, onAddFilter, setTooltip) {
   const fd = formData;
-  const c = fd.color_picker;
+  const c = fd.colorPicker;
   const fixedColor = [c.r, c.g, c.b, 255 * c.a];
   let data = payload.data.features.map(feature => ({
     ...feature,
     path: feature.path,
-    width: fd.line_width,
+    width: fd.lineWidth,
     color: fixedColor,
   }));
 
-  if (fd.js_data_mutator) {
-    const jsFnMutator = sandboxedEval(fd.js_data_mutator);
+  if (fd.jsDataMutator) {
+    const jsFnMutator = sandboxedEval(fd.jsDataMutator);
     data = jsFnMutator(data);
   }
 
   return new PathLayer({
-    id: `path-layer-${fd.slice_id}`,
+    id: `path-layer-${fd.sliceId}`,
     data,
     rounded: true,
     widthScale: 1,

superset/assets/src/visualizations/deckgl/layers/Polygon/Polygon.jsx

@@ -50,8 +50,8 @@ function getElevation(d, colorScaler) {
 
 export function getLayer(formData, payload, setTooltip, selected, onSelect, filters) {
   const fd = formData;
-  const fc = fd.fill_color_picker;
-  const sc = fd.stroke_color_picker;
+  const fc = fd.fillColorPicker;
+  const sc = fd.strokeColorPicker;
   let data = [...payload.data.features];
 
   if (filters != null) {
@@ -60,9 +60,9 @@ export function getLayer(formData, payload, setTooltip, selected, onSelect, filt
     });
   }
 
-  if (fd.js_data_mutator) {
+  if (fd.jsDataMutator) {
     // Applying user defined data mutator if defined
-    const jsFnMutator = sandboxedEval(fd.js_data_mutator);
+    const jsFnMutator = sandboxedEval(fd.jsDataMutator);
     data = jsFnMutator(data);
   }
 
@@ -76,13 +76,13 @@ export function getLayer(formData, payload, setTooltip, selected, onSelect, filt
   // when polygons are selected, reduce the opacity of non-selected polygons
   const colorScaler = (d) => {
     const baseColor = baseColorScaler(d);
-    if (selected.length > 0 && selected.indexOf(d[fd.line_column]) === -1) {
+    if (selected.length > 0 && selected.indexOf(d[fd.lineColumn]) === -1) {
       baseColor[3] /= 2;
     }
     return baseColor;
   };
   return new PolygonLayer({
-    id: `path-layer-${fd.slice_id}`,
+    id: `path-layer-${fd.sliceId}`,
     data,
     pickable: true,
     filled: fd.filled,
@@ -90,7 +90,7 @@ export function getLayer(formData, payload, setTooltip, selected, onSelect, filt
     getPolygon: d => d.polygon,
     getFillColor: colorScaler,
     getLineColor: [sc.r, sc.g, sc.b, 255 * sc.a],
-    getLineWidth: fd.line_width,
+    getLineWidth: fd.lineWidth,
     extruded: fd.extruded,
     getElevation: d => getElevation(d, colorScaler),
     elevationScale: fd.multiplier,
@@ -138,7 +138,7 @@ class DeckGLPolygon extends React.Component {
     // the granularity has to be read from the payload form_data, not the
     // props formData which comes from the instantaneous controls state
     const granularity = (
-      props.payload.form_data.time_grain_sqla ||
+      props.payload.form_data.timeGrainSqla ||
       props.payload.form_data.granularity ||
       'P1D'
     );
@@ -177,7 +177,7 @@ class DeckGLPolygon extends React.Component {
     const selected = [...this.state.selected];
     if (doubleClick) {
       selected.splice(0, selected.length, polygon);
-    } else if (formData.toggle_polygons) {
+    } else if (formData.togglePolygons) {
       const i = selected.indexOf(polygon);
       if (i === -1) {
         selected.push(polygon);
@@ -189,8 +189,8 @@ class DeckGLPolygon extends React.Component {
     }
 
     this.setState({ selected, lastClick: now });
-    if (formData.table_filter) {
-      onAddFilter(formData.line_column, selected, false, true);
+    if (formData.tableFilter) {
+      onAddFilter(formData.lineColumn, selected, false, true);
     }
   }
   onValuesChange(values) {
@@ -249,14 +249,14 @@ class DeckGLPolygon extends React.Component {
           viewport={viewport}
           onViewportChange={this.onViewportChange}
           mapboxApiAccessToken={payload.data.mapboxApiKey}
-          mapStyle={formData.mapbox_style}
+          mapStyle={formData.mapboxStyle}
           setControlValue={setControlValue}
           aggregation
         >
           {formData.metric !== null &&
           <Legend
             categories={buckets}
-            position={formData.legend_position}
+            position={formData.legendPosition}
           />}
         </AnimatableDeckGLContainer>
       </div>

superset/assets/src/visualizations/deckgl/layers/Scatter/Scatter.jsx

@@ -27,24 +27,24 @@ function getPoints(data) {
 
 export function getLayer(fd, payload, onAddFilter, setTooltip) {
   const dataWithRadius = payload.data.features.map((d) => {
-    let radius = unitToRadius(fd.point_unit, d.radius) || 10;
+    let radius = unitToRadius(fd.pointUnit, d.radius) || 10;
     if (fd.multiplier) {
       radius *= fd.multiplier;
     }
     if (d.color) {
       return { ...d, radius };
     }
-    const c = fd.color_picker || { r: 0, g: 0, b: 0, a: 1 };
+    const c = fd.colorPicker || { r: 0, g: 0, b: 0, a: 1 };
     const color = [c.r, c.g, c.b, c.a * 255];
     return { ...d, radius, color };
   });
 
   return new ScatterplotLayer({
-    id: `scatter-layer-${fd.slice_id}`,
+    id: `scatter-layer-${fd.sliceId}`,
     data: dataWithRadius,
     fp64: true,
-    radiusMinPixels: fd.min_radius || null,
-    radiusMaxPixels: fd.max_radius || null,
+    radiusMinPixels: fd.minRadius || null,
+    radiusMaxPixels: fd.maxRadius || null,
     outline: false,
     ...commonLayerProps(fd, setTooltip),
   });

superset/assets/src/visualizations/deckgl/layers/Screengrid/Screengrid.jsx

@@ -32,15 +32,15 @@ function getPoints(data) {
 
 export function getLayer(formData, payload, onAddFilter, setTooltip, filters) {
   const fd = formData;
-  const c = fd.color_picker;
+  const c = fd.colorPicker;
   let data = payload.data.features.map(d => ({
     ...d,
     color: [c.r, c.g, c.b, 255 * c.a],
   }));
 
-  if (fd.js_data_mutator) {
+  if (fd.jsDataMutator) {
     // Applying user defined data mutator if defined
-    const jsFnMutator = sandboxedEval(fd.js_data_mutator);
+    const jsFnMutator = sandboxedEval(fd.jsDataMutator);
     data = jsFnMutator(data);
   }
 
@@ -53,10 +53,10 @@ export function getLayer(formData, payload, onAddFilter, setTooltip, filters) {
   // Passing a layer creator function instead of a layer since the
   // layer needs to be regenerated at each render
   return new ScreenGridLayer({
-    id: `screengrid-layer-${fd.slice_id}`,
+    id: `screengrid-layer-${fd.sliceId}`,
     data,
     pickable: true,
-    cellSizePixels: fd.grid_size,
+    cellSizePixels: fd.gridSize,
     minColor: [c.r, c.g, c.b, 0],
     maxColor: [c.r, c.g, c.b, 255 * c.a],
     outline: false,
@@ -102,7 +102,7 @@ class DeckGLScreenGrid extends React.PureComponent {
     // the granularity has to be read from the payload form_data, not the
     // props formData which comes from the instantaneous controls state
     const granularity = (
-      props.payload.form_data.time_grain_sqla ||
+      props.payload.form_data.timeGrainSqla ||
       props.payload.form_data.granularity ||
       'P1D'
     );
@@ -176,7 +176,7 @@ class DeckGLScreenGrid extends React.PureComponent {
           viewport={this.state.viewport}
           onViewportChange={this.onViewportChange}
           mapboxApiAccessToken={payload.data.mapboxApiKey}
-          mapStyle={formData.mapbox_style}
+          mapStyle={formData.mapboxStyle}
           setControlValue={setControlValue}
           aggregation
         />

superset/assets/src/visualizations/deckgl/layers/common.jsx

@@ -54,13 +54,13 @@ export function commonLayerProps(formData, setTooltip, onSelect) {
   const fd = formData;
   let onHover;
   let tooltipContentGenerator;
-  if (fd.js_tooltip) {
-    tooltipContentGenerator = sandboxedEval(fd.js_tooltip);
-  } else if (fd.line_column && fd.metric && ['geohash', 'zipcode'].indexOf(fd.line_type) >= 0) {
+  if (fd.jsTooltip) {
+    tooltipContentGenerator = sandboxedEval(fd.jsTooltip);
+  } else if (fd.lineColumn && fd.metric && ['geohash', 'zipcode'].indexOf(fd.lineType) >= 0) {
     const metricLabel = fd.metric.label || fd.metric;
     tooltipContentGenerator = o => (
       <div>
-        <div>{fd.line_column}: <strong>{o.object[fd.line_column]}</strong></div>
+        <div>{fd.lineColumn}: <strong>{o.object[fd.lineColumn]}</strong></div>
         {fd.metric &&
           <div>{metricLabel}: <strong>{o.object[metricLabel]}</strong></div>}
       </div>);
@@ -71,7 +71,7 @@ export function commonLayerProps(formData, setTooltip, onSelect) {
         setTooltip({
           content: tooltipContentGenerator(o),
           x: o.x,
-          y: o.y,
+          y: o.y + 30,
         });
       } else {
         setTooltip(null);
@@ -79,13 +79,13 @@ export function commonLayerProps(formData, setTooltip, onSelect) {
     };
   }
   let onClick;
-  if (fd.js_onclick_href) {
+  if (fd.jsOnclickHref) {
     onClick = (o) => {
-      const href = sandboxedEval(fd.js_onclick_href)(o);
+      const href = sandboxedEval(fd.jsOnclickHref)(o);
       window.open(href);
     };
-  } else if (fd.table_filter && onSelect !== undefined) {
-    onClick = o => onSelect(o.object[fd.line_column]);
+  } else if (fd.tableFilter && onSelect !== undefined) {
+    onClick = o => onSelect(o.object[fd.lineColumn]);
   }
   return {
     onClick,

superset/assets/src/visualizations/deckgl/utils.js

@@ -24,8 +24,8 @@ import { hexToRGB } from '../../modules/colors';
 const DEFAULT_NUM_BUCKETS = 10;
 
 export function getBreakPoints({
-    break_points: formDataBreakPoints,
-    num_buckets: formDataNumBuckets,
+    breakPoints: formDataBreakPoints,
+    numBuckets: formDataNumBuckets,
   }, features, accessor) {
   if (!features) {
     return [];
@@ -46,15 +46,15 @@ export function getBreakPoints({
 }
 
 export function getBreakPointColorScaler({
-    break_points: formDataBreakPoints,
-    num_buckets: formDataNumBuckets,
-    linear_color_scheme: linearColorScheme,
+    breakPoints: formDataBreakPoints,
+    numBuckets: formDataNumBuckets,
+    linearColorScheme,
     opacity,
   }, features, accessor) {
   const breakPoints = formDataBreakPoints || formDataNumBuckets
     ? getBreakPoints({
-      break_points: formDataBreakPoints,
-      num_buckets: formDataNumBuckets,
+      breakPoints: formDataBreakPoints,
+      numBuckets: formDataNumBuckets,
     }, features, accessor)
     : null;
   const colorScheme = Array.isArray(linearColorScheme)

superset/assets/src/visualizations/markup.js

@@ -1,58 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-import srcdoc from 'srcdoc-polyfill';
-import './markup.css';
-
-function markupWidget(slice, payload) {
-  const { selector } = slice;
-  const height = slice.height();
-  const headerHeight = slice.headerHeight();
-  const vizType = slice.props.vizType;
-  const { data } = payload;
-
-  const container = document.querySelector(selector);
-  container.style.overflow = 'auto';
-
-  // markup height is slice height - (marginTop + marginBottom)
-  const iframeHeight = vizType === 'separator'
-    ? height - 20
-    : height + headerHeight;
-
-  const html = `
-    <html>
-      <head>
-        ${data.theme_css.map(
-          href => `<link rel="stylesheet" type="text/css" href="${href}" />`,
-        )}
-      </head>
-      <body style="background-color: transparent;">
-        ${data.html}
-      </body>
-    </html>`;
-
-  const iframe = document.createElement('iframe');
-  iframe.setAttribute('frameborder', 0);
-  iframe.setAttribute('height', iframeHeight);
-  iframe.setAttribute('sandbox', 'allow-forms allow-popups allow-same-origin allow-scripts allow-top-navigation');
-  container.appendChild(iframe);
-
-  srcdoc.set(iframe, html);
-}
-
-export default markupWidget;

superset/assets/src/visualizations/nvd3/LineMulti/LineMulti.jsx

@@ -94,9 +94,9 @@ class LineMulti extends React.Component {
       const combinedFormData = {
         ...subslice.form_data,
         filters: (subsliceFormData.filters || [])
-          .concat(filters || [])
-          .concat(extraFilters || []),
+          .concat(filters || []),
         time_range: timeRange,
+        extra_filters: extraFilters || [],
       };
       const addPrefix = prefixMetricWithSliceName;
       return getJson(getExploreLongUrl(combinedFormData, 'json'))

superset/assets/src/visualizations/nvd3/NVD3Vis.js

@@ -641,15 +641,15 @@ function nvd3Vis(element, props) {
         // If x bounds are shown, we need a right margin
         margins.right = Math.max(20, maxXAxisLabelHeight / 2) + marginPad;
       }
-      if (staggerLabels) {
-        margins.bottom = 40;
-      } else {
+      if (xLabelRotation === 45) {
         margins.bottom = (
           maxXAxisLabelHeight * Math.sin(Math.PI * xLabelRotation / 180)
         ) + marginPad;
         margins.right = (
           maxXAxisLabelHeight * Math.cos(Math.PI * xLabelRotation / 180)
         ) + marginPad;
+      } else if (staggerLabels) {
+        margins.bottom = 40;
       }
 
       if (isVizTypes(['dual_line', 'line_multi'])) {
@@ -711,8 +711,9 @@ function nvd3Vis(element, props) {
         .attr('height', height)
         .call(chart);
 
-      // on scroll, hide tooltips. throttle to only 4x/second.
-      window.addEventListener('scroll', throttle(hideTooltips, 250));
+      // On scroll, hide (not remove) tooltips so they can reappear on hover.
+      // Throttle to only 4x/second.
+      window.addEventListener('scroll', throttle(() => hideTooltips(false), 250));
 
       // The below code should be run AFTER rendering because chart is updated in call()
       if (isTimeSeries && activeAnnotationLayers.length > 0) {
@@ -933,10 +934,10 @@ function nvd3Vis(element, props) {
     return chart;
   };
 
-  // hide tooltips before rendering chart, if the chart is being re-rendered sometimes
+  // Remove tooltips before rendering chart, if the chart is being re-rendered sometimes
   // there are left over tooltips in the dom,
   // this will clear them before rendering the chart again.
-  hideTooltips();
+  hideTooltips(true);
 
   nv.addGraph(drawGraph);
 }

superset/assets/src/visualizations/nvd3/utils.js

@@ -165,10 +165,19 @@ export function generateBubbleTooltipContent({
   return s;
 }
 
-export function hideTooltips() {
-  const target = document.querySelector('.nvtooltip');
-  if (target) {
-    target.style.opacity = 0;
+// shouldRemove indicates whether the nvtooltips should be removed from the DOM
+export function hideTooltips(shouldRemove) {
+  const targets = document.querySelectorAll('.nvtooltip');
+  if (targets.length > 0) {
+    // Only set opacity to 0 when hiding tooltips so they would reappear
+    // on hover, which sets the opacity to 1
+    for (const t of targets) {
+      if (shouldRemove) {
+        t.remove();
+      } else {
+        t.style.opacity = 0;
+      }
+    }
   }
 }
 

superset/assets/src/visualizations/presets/LegacyChartPreset.js

@@ -30,7 +30,9 @@ import EventFlowChartPlugin from '../EventFlow/EventFlowChartPlugin';
 import ForceDirectedChartPlugin from '../ForceDirected/ForceDirectedChartPlugin';
 import HeatmapChartPlugin from '../Heatmap/HeatmapChartPlugin';
 import HorizonChartPlugin from '../Horizon/HorizonChartPlugin';
+import IframeChartPlugin from '../Iframe/IframeChartPlugin';
 import LineMultiChartPlugin from '../nvd3/LineMulti/LineMultiChartPlugin';
+import MarkupChartPlugin from '../Markup/MarkupChartPlugin';
 import PairedTTestChartPlugin from '../PairedTTest/PairedTTestChartPlugin';
 import ParallelCoordinatesChartPlugin from '../ParallelCoordinates/ParallelCoordinatesChartPlugin';
 import RoseChartPlugin from '../Rose/RoseChartPlugin';
@@ -57,7 +59,10 @@ export default class LegacyChartPreset extends Preset {
         new ForceDirectedChartPlugin().configure({ key: 'directed_force' }),
         new HeatmapChartPlugin().configure({ key: 'heatmap' }),
         new HorizonChartPlugin().configure({ key: 'horizon' }),
+        new IframeChartPlugin().configure({ key: 'iframe' }),
         new LineMultiChartPlugin().configure({ key: 'line_multi' }),
+        new MarkupChartPlugin().configure({ key: 'markup' }),
+        new MarkupChartPlugin().configure({ key: 'separator' }),
         new PairedTTestChartPlugin().configure({ key: 'paired_ttest' }),
         new ParallelCoordinatesChartPlugin().configure({ key: 'para' }),
         new RoseChartPlugin().configure({ key: 'rose' }),

superset/assets/src/welcome/DashboardTable.jsx

@@ -71,7 +71,7 @@ class DashboardTable extends React.PureComponent {
           {this.state.dashboards.map(o => (
             <Tr key={o.id}>
               <Td column="dashboard" value={o.dashboard_title}>
-                {o.dashboard_title}
+                <a href={o.url}>{o.dashboard_title}</a>
               </Td>
               <Td column="creator" value={o.changed_by_name}>
                 {unsafe(o.creator)}

superset/common/query_context.py

@@ -14,30 +14,247 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-# pylint: disable=R
+# pylint: disable=C,R,W
+from datetime import datetime, timedelta
+import logging
+import pickle as pkl
+import traceback
 from typing import Dict, List
 
+import numpy as np
+import pandas as pd
+
+from superset import app, cache
 from superset import db
 from superset.connectors.connector_registry import ConnectorRegistry
+from superset.utils import core as utils
+from superset.utils.core import DTTM_ALIAS
 from .query_object import QueryObject
 
+config = app.config
+stats_logger = config.get('STATS_LOGGER')
+
 
 class QueryContext:
     """
     The query context contains the query object and additional fields necessary
     to retrieve the data payload for a given viz.
     """
+
+    default_fillna = 0
+    cache_type = 'df'
+    enforce_numerical_metrics = True
+
     # TODO: Type datasource and query_object dictionary with TypedDict when it becomes
     # a vanilla python type https://github.com/python/mypy/issues/5288
     def __init__(
             self,
             datasource: Dict,
             queries: List[Dict],
+            force: bool = False,
+            custom_cache_timeout: int = None,
     ):
         self.datasource = ConnectorRegistry.get_datasource(datasource.get('type'),
                                                            int(datasource.get('id')),
                                                            db.session)
         self.queries = list(map(lambda query_obj: QueryObject(**query_obj), queries))
 
-    def get_data(self):
-        raise NotImplementedError()
+        self.force = force
+
+        self.custom_cache_timeout = custom_cache_timeout
+
+        self.enforce_numerical_metrics = True
+
+    def get_query_result(self, query_object):
+        """Returns a pandas dataframe based on the query object"""
+
+        # Here, we assume that all the queries will use the same datasource, which is
+        # is a valid assumption for current setting. In a long term, we may or maynot
+        # support multiple queries from different data source.
+
+        timestamp_format = None
+        if self.datasource.type == 'table':
+            dttm_col = self.datasource.get_col(query_object.granularity)
+            if dttm_col:
+                timestamp_format = dttm_col.python_date_format
+
+        # The datasource here can be different backend but the interface is common
+        result = self.datasource.query(query_object.to_dict())
+
+        df = result.df
+        # Transform the timestamp we received from database to pandas supported
+        # datetime format. If no python_date_format is specified, the pattern will
+        # be considered as the default ISO date format
+        # If the datetime format is unix, the parse will use the corresponding
+        # parsing logic
+        if df is not None and not df.empty:
+            if DTTM_ALIAS in df.columns:
+                if timestamp_format in ('epoch_s', 'epoch_ms'):
+                    # Column has already been formatted as a timestamp.
+                    df[DTTM_ALIAS] = df[DTTM_ALIAS].apply(pd.Timestamp)
+                else:
+                    df[DTTM_ALIAS] = pd.to_datetime(
+                        df[DTTM_ALIAS], utc=False, format=timestamp_format)
+                if self.datasource.offset:
+                    df[DTTM_ALIAS] += timedelta(hours=self.datasource.offset)
+                df[DTTM_ALIAS] += query_object.time_shift
+
+            if self.enforce_numerical_metrics:
+                self.df_metrics_to_num(df, query_object)
+
+            df.replace([np.inf, -np.inf], np.nan)
+            df = self.handle_nulls(df)
+        return {
+            'query': result.query,
+            'status': result.status,
+            'error_message': result.error_message,
+            'df': df,
+        }
+
+    def df_metrics_to_num(self, df, query_object):
+        """Converting metrics to numeric when pandas.read_sql cannot"""
+        metrics = [metric for metric in query_object.metrics]
+        for col, dtype in df.dtypes.items():
+            if dtype.type == np.object_ and col in metrics:
+                df[col] = pd.to_numeric(df[col], errors='coerce')
+
+    def handle_nulls(self, df):
+        fillna = self.get_fillna_for_columns(df.columns)
+        return df.fillna(fillna)
+
+    def get_fillna_for_col(self, col):
+        """Returns the value to use as filler for a specific Column.type"""
+        if col and col.is_string:
+            return ' NULL'
+        return self.default_fillna
+
+    def get_fillna_for_columns(self, columns=None):
+        """Returns a dict or scalar that can be passed to DataFrame.fillna"""
+        if columns is None:
+            return self.default_fillna
+        columns_dict = {col.column_name: col for col in self.datasource.columns}
+        fillna = {
+            c: self.get_fillna_for_col(columns_dict.get(c))
+            for c in columns
+        }
+        return fillna
+
+    def get_data(self, df):
+        return df.to_dict(orient='records')
+
+    def get_single_payload(self, query_obj):
+        """Returns a payload of metadata and data"""
+        payload = self.get_df_payload(query_obj)
+        df = payload.get('df')
+        status = payload.get('status')
+        if status != utils.QueryStatus.FAILED:
+            if df is not None and df.empty:
+                payload['error'] = 'No data'
+            else:
+                payload['data'] = self.get_data(df)
+        if 'df' in payload:
+            del payload['df']
+        return payload
+
+    def get_payload(self):
+        """Get all the paylaods from the arrays"""
+        return [self.get_single_payload(query_ojbect) for query_ojbect in self.queries]
+
+    @property
+    def cache_timeout(self):
+        if self.custom_cache_timeout is not None:
+            return self.custom_cache_timeout
+        if self.datasource.cache_timeout is not None:
+            return self.datasource.cache_timeout
+        if (
+                hasattr(self.datasource, 'database') and
+                self.datasource.database.cache_timeout) is not None:
+            return self.datasource.database.cache_timeout
+        return config.get('CACHE_DEFAULT_TIMEOUT')
+
+    def get_df_payload(self, query_obj, **kwargs):
+        """Handles caching around the df paylod retrieval"""
+        cache_key = query_obj.cache_key(
+            datasource=self.datasource.uid, **kwargs) if query_obj else None
+        logging.info('Cache key: {}'.format(cache_key))
+        is_loaded = False
+        stacktrace = None
+        df = None
+        cached_dttm = datetime.utcnow().isoformat().split('.')[0]
+        cache_value = None
+        status = None
+        query = ''
+        error_message = None
+        if cache_key and cache and not self.force:
+            cache_value = cache.get(cache_key)
+            if cache_value:
+                stats_logger.incr('loaded_from_cache')
+                try:
+                    cache_value = pkl.loads(cache_value)
+                    df = cache_value['df']
+                    query = cache_value['query']
+                    status = utils.QueryStatus.SUCCESS
+                    is_loaded = True
+                except Exception as e:
+                    logging.exception(e)
+                    logging.error('Error reading cache: ' +
+                                  utils.error_msg_from_exception(e))
+                logging.info('Serving from cache')
+
+        if query_obj and not is_loaded:
+            try:
+                query_result = self.get_query_result(query_obj)
+                status = query_result['status']
+                query = query_result['query']
+                error_message = query_result['error_message']
+                df = query_result['df']
+                if status != utils.QueryStatus.FAILED:
+                    stats_logger.incr('loaded_from_source')
+                    is_loaded = True
+            except Exception as e:
+                logging.exception(e)
+                if not error_message:
+                    error_message = '{}'.format(e)
+                status = utils.QueryStatus.FAILED
+                stacktrace = traceback.format_exc()
+
+            if (
+                    is_loaded and
+                    cache_key and
+                    cache and
+                    status != utils.QueryStatus.FAILED):
+                try:
+                    cache_value = dict(
+                        dttm=cached_dttm,
+                        df=df if df is not None else None,
+                        query=query,
+                    )
+                    cache_value = pkl.dumps(
+                        cache_value, protocol=pkl.HIGHEST_PROTOCOL)
+
+                    logging.info('Caching {} chars at key {}'.format(
+                        len(cache_value), cache_key))
+
+                    stats_logger.incr('set_cache_key')
+                    cache.set(
+                        cache_key,
+                        cache_value,
+                        timeout=self.cache_timeout)
+                except Exception as e:
+                    # cache.set call can fail if the backend is down or if
+                    # the key is too large or whatever other reasons
+                    logging.warning('Could not cache key {}'.format(cache_key))
+                    logging.exception(e)
+                    cache.delete(cache_key)
+        return {
+            'cache_key': cache_key,
+            'cached_dttm': cache_value['dttm'] if cache_value is not None else None,
+            'cache_timeout': self.cache_timeout,
+            'df': df,
+            'error': error_message,
+            'is_cached': cache_key is not None,
+            'query': query,
+            'status': status,
+            'stacktrace': stacktrace,
+            'rowcount': len(df.index) if df is not None else 0,
+        }

superset/common/query_object.py

@@ -15,15 +15,17 @@
 # specific language governing permissions and limitations
 # under the License.
 # pylint: disable=R
-from typing import Dict, List, Optional
+import hashlib
+from typing import Dict, List, Optional, Union
+
+import simplejson as json
 
 from superset import app
 from superset.utils import core as utils
 
+
 # TODO: Type Metrics dictionary with TypedDict when it becomes a vanilla python type
 # https://github.com/python/mypy/issues/5288
-Metric = Dict
-
 
 class QueryObject:
     """
@@ -33,31 +35,87 @@ class QueryObject:
     def __init__(
             self,
             granularity: str,
+            metrics: List[Union[Dict, str]],
             groupby: List[str] = None,
-            metrics: List[Metric] = None,
             filters: List[str] = None,
             time_range: Optional[str] = None,
             time_shift: Optional[str] = None,
             is_timeseries: bool = False,
+            timeseries_limit: int = 0,
             row_limit: int = app.config.get('ROW_LIMIT'),
-            limit: int = 0,
-            timeseries_limit_metric: Optional[Metric] = None,
+            timeseries_limit_metric: Optional[Dict] = None,
             order_desc: bool = True,
             extras: Optional[Dict] = None,
+            prequeries: Optional[Dict] = None,
+            is_prequery: bool = False,
+            columns: List[str] = None,
+            orderby: List[List] = None,
     ):
         self.granularity = granularity
         self.from_dttm, self.to_dttm = utils.get_since_until(time_range, time_shift)
         self.is_timeseries = is_timeseries
-        self.groupby = groupby or []
-        self.metrics = metrics or []
-        self.filter = filters or []
+        self.time_range = time_range
+        self.time_shift = utils.parse_human_timedelta(time_shift)
+        self.groupby = groupby if groupby is not None else []
+
+        # Temporal solution for backward compatability issue
+        # due the new format of non-ad-hoc metric.
+        self.metrics = [metric if 'expressionType' in metric else metric['label']
+                        for metric in metrics]
         self.row_limit = row_limit
-        self.timeseries_limit = int(limit)
+        self.filter = filters if filters is not None else []
+        self.timeseries_limit = timeseries_limit
         self.timeseries_limit_metric = timeseries_limit_metric
         self.order_desc = order_desc
-        self.prequeries = []
-        self.is_prequery = False
-        self.extras = extras
+        self.prequeries = prequeries
+        self.is_prequery = is_prequery
+        self.extras = extras if extras is not None else {}
+        self.columns = columns if columns is not None else []
+        self.orderby = orderby if orderby is not None else []
 
     def to_dict(self):
-        raise NotImplementedError()
+        query_object_dict = {
+            'granularity': self.granularity,
+            'from_dttm': self.from_dttm,
+            'to_dttm': self.to_dttm,
+            'is_timeseries': self.is_timeseries,
+            'groupby': self.groupby,
+            'metrics': self.metrics,
+            'row_limit': self.row_limit,
+            'filter': self.filter,
+            'timeseries_limit': self.timeseries_limit,
+            'timeseries_limit_metric': self.timeseries_limit_metric,
+            'order_desc': self.order_desc,
+            'prequeries': self.prequeries,
+            'is_prequery': self.is_prequery,
+            'extras': self.extras,
+            'columns': self.columns,
+            'orderby': self.orderby,
+        }
+        return query_object_dict
+
+    def cache_key(self, **extra):
+        """
+        The cache key is made out of the key/values in `query_obj`, plus any
+        other key/values in `extra`
+        We remove datetime bounds that are hard values, and replace them with
+        the use-provided inputs to bounds, which may be time-relative (as in
+        "5 days ago" or "now").
+        """
+        cache_dict = self.to_dict()
+        cache_dict.update(extra)
+
+        for k in ['from_dttm', 'to_dttm']:
+            del cache_dict[k]
+        if self.time_range:
+            cache_dict['time_range'] = self.time_range
+        json_data = self.json_dumps(cache_dict, sort_keys=True)
+        return hashlib.md5(json_data.encode('utf-8')).hexdigest()
+
+    def json_dumps(self, obj, sort_keys=False):
+        return json.dumps(
+            obj,
+            default=utils.json_int_dttm_ser,
+            ignore_nan=True,
+            sort_keys=sort_keys,
+        )

superset/config.py

@@ -110,6 +110,7 @@ APP_NAME = 'Superset'
 
 # Uncomment to setup an App icon
 APP_ICON = '/static/assets/images/superset-logo@2x.png'
+APP_ICON_WIDTH = 126
 
 # Druid query timezone
 # tz.tzutc() : Using utc timezone
@@ -325,6 +326,9 @@ DEFAULT_SQLLAB_LIMIT = 1000
 # Maximum number of tables/views displayed in the dropdown window in SQL Lab.
 MAX_TABLE_NAMES = 3000
 
+# Adds a warning message on sqllab save query modal.
+SQLLAB_SAVE_WARNING_MESSAGE = None
+
 # If defined, shows this text in an alert-warning box in the navbar
 # one example use case may be "STAGING" to make it clear that this is
 # not the production version of the site.
@@ -558,6 +562,9 @@ WEBDRIVER_CONFIGURATION = {}
 # The base URL to query for accessing the user interface
 WEBDRIVER_BASEURL = 'http://0.0.0.0:8080/'
 
+# Send user to a link where they can report bugs
+BUG_REPORT_URL = None
+
 
 try:
     if CONFIG_PATH_ENV_VAR in os.environ:

superset/connectors/sqla/models.py

@@ -422,6 +422,7 @@ class SqlaTable(Model, BaseDatasource):
             d['time_grain_sqla'] = grains
             d['main_dttm_col'] = self.main_dttm_col
             d['fetch_values_predicate'] = self.fetch_values_predicate
+            d['template_params'] = self.template_params
         return d
 
     def values_for_column(self, column_name, limit=10000):
@@ -729,15 +730,11 @@ class SqlaTable(Model, BaseDatasource):
 
                 ob = inner_main_metric_expr
                 if timeseries_limit_metric:
-                    if utils.is_adhoc_metric(timeseries_limit_metric):
-                        ob = self.adhoc_metric_to_sqla(timeseries_limit_metric, cols)
-                    elif timeseries_limit_metric in metrics_dict:
-                        timeseries_limit_metric = metrics_dict.get(
-                            timeseries_limit_metric,
-                        )
-                        ob = timeseries_limit_metric.get_sqla_col()
-                    else:
-                        raise Exception(_("Metric '{}' is not valid".format(m)))
+                    ob = self._get_timeseries_orderby(
+                        timeseries_limit_metric,
+                        metrics_dict,
+                        cols,
+                    )
                 direction = desc if order_desc else asc
                 subq = subq.order_by(direction(ob))
                 subq = subq.limit(timeseries_limit)
@@ -752,6 +749,16 @@ class SqlaTable(Model, BaseDatasource):
 
                 tbl = tbl.join(subq.alias(), and_(*on_clause))
             else:
+                if timeseries_limit_metric:
+                    orderby = [(
+                        self._get_timeseries_orderby(
+                            timeseries_limit_metric,
+                            metrics_dict,
+                            cols,
+                        ),
+                        False,
+                    )]
+
                 # run subquery to get top groups
                 subquery_obj = {
                     'prequeries': prequeries,
@@ -780,6 +787,19 @@ class SqlaTable(Model, BaseDatasource):
 
         return qry.select_from(tbl)
 
+    def _get_timeseries_orderby(self, timeseries_limit_metric, metrics_dict, cols):
+        if utils.is_adhoc_metric(timeseries_limit_metric):
+            ob = self.adhoc_metric_to_sqla(timeseries_limit_metric, cols)
+        elif timeseries_limit_metric in metrics_dict:
+            timeseries_limit_metric = metrics_dict.get(
+                timeseries_limit_metric,
+            )
+            ob = timeseries_limit_metric.get_sqla_col()
+        else:
+            raise Exception(_("Metric '{}' is not valid".format(timeseries_limit_metric)))
+
+        return ob
+
     def _get_top_groups(self, df, dimensions):
         cols = {col.column_name: col for col in self.columns}
         groups = []

superset/db_engine_specs.py

@@ -929,16 +929,16 @@ class PrestoEngineSpec(BaseEngineSpec):
         except Exception:
             # table is not partitioned
             return False
-        for c in columns:
-            if c.get('name') == col_name:
-                return qry.where(Column(col_name) == value)
+        if value is not None:
+            for c in columns:
+                if c.get('name') == col_name:
+                    return qry.where(Column(col_name) == value)
         return False
 
     @classmethod
     def _latest_partition_from_df(cls, df):
-        recs = df.to_records(index=False)
-        if recs:
-            return recs[0][0]
+        if not df.empty:
+            return df.to_records(index=False)[0][0]
 
     @classmethod
     def latest_partition(cls, table_name, schema, database, show_first=False):
@@ -955,7 +955,7 @@ class PrestoEngineSpec(BaseEngineSpec):
         :type show_first: bool
 
         >>> latest_partition('foo_table')
-        '2018-01-01'
+        ('ds', '2018-01-01')
         """
         indexes = database.get_indexes(table_name, schema)
         if len(indexes[0]['column_names']) < 1:
@@ -1272,9 +1272,10 @@ class HiveEngineSpec(PrestoEngineSpec):
         except Exception:
             # table is not partitioned
             return False
-        for c in columns:
-            if c.get('name') == col_name:
-                return qry.where(Column(col_name) == value)
+        if value is not None:
+            for c in columns:
+                if c.get('name') == col_name:
+                    return qry.where(Column(col_name) == value)
         return False
 
     @classmethod
@@ -1285,7 +1286,8 @@ class HiveEngineSpec(PrestoEngineSpec):
     @classmethod
     def _latest_partition_from_df(cls, df):
         """Hive partitions look like ds={partition name}"""
-        return df.ix[:, 0].max().split('=')[1]
+        if not df.empty:
+            return df.ix[:, 0].max().split('=')[1]
 
     @classmethod
     def _partition_query(

superset/migrations/versions/c617da68de7d_form_nullable.py

@@ -0,0 +1,191 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""form nullable
+
+Revision ID: c617da68de7d
+Revises: 18dc26817ad2
+Create Date: 2018-07-19 23:41:32.631556
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = 'c617da68de7d'
+down_revision = '18dc26817ad2'
+
+from alembic import op
+from sqlalchemy.ext.declarative import declarative_base
+from sqlalchemy import Column, Integer, String, Text
+
+from superset import db
+from superset.utils.core import MediumText
+
+Base = declarative_base()
+
+
+class BaseColumnMixin(object):
+    id = Column(Integer, primary_key=True)
+    column_name = Column(String(255))
+    description = Column(Text)
+    type = Column(String(32))
+    verbose_name = Column(String(1024))
+
+
+class BaseDatasourceMixin(object):
+    id = Column(Integer, primary_key=True)
+    description = Column(Text)
+
+
+class BaseMetricMixin(object):
+    id = Column(Integer, primary_key=True)
+    d3format = Column(String(128))
+    description = Column(Text)
+    metric_name = Column(String(512))
+    metric_type = Column(String(32))
+    verbose_name = Column(String(1024))
+    warning_text = Column(Text)
+
+
+class Annotation(Base):
+    __tablename__ = 'annotation'
+
+    id = Column(Integer, primary_key=True)
+    long_descr = Column(Text)
+    json_metadata = Column(Text)
+    short_descr = Column(String(500))
+
+
+class Dashboard(Base):
+    __tablename__ = 'dashboards'
+
+    id = Column(Integer, primary_key=True)
+    css = Column(Text)
+    dashboard_title = Column(String(500))
+    description = Column(Text)
+    json_metadata = Column(Text)
+    position_json = Column(MediumText())
+    slug = Column(String(255))
+
+
+class Database(Base):
+    __tablename__ = 'dbs'
+
+    id = Column(Integer, primary_key=True)
+    database_name = Column(String(250))
+    extra = Column(Text)
+    force_ctas_schema = Column(String(250))
+    sqlalchemy_uri = Column(String(1024))
+    verbose_name = Column(String(250))
+
+
+class DruidCluster(Base):
+    __tablename__ = 'clusters'
+
+    id = Column(Integer, primary_key=True)
+    broker_host = Column(String(255))
+    broker_endpoint = Column(String(255))
+    cluster_name = Column(String(250))
+    verbose_name = Column(String(250))
+
+
+class DruidColumn(BaseColumnMixin, Base):
+    __tablename__ = 'columns'
+
+    dimension_spec_json = Column(Text)
+
+
+class DruidDatasource(BaseDatasourceMixin, Base):
+    __tablename__ = 'datasources'
+
+    datasource_name = Column(String(255))
+    default_endpoint = Column(Text)
+    fetch_values_from = Column(String(100))
+
+
+class DruidMetric(BaseMetricMixin, Base):
+    __tablename__ = 'metrics'
+
+    json = Column(Text)
+
+
+class Slice(Base):
+    __tablename__ = 'slices'
+
+    id = Column(Integer, primary_key=True)
+    description = Column(Text)
+    params = Column(Text)
+    slice_name = Column(String(250))
+    viz_type = Column(String(250))
+
+
+class SqlaTable(BaseDatasourceMixin, Base):
+    __tablename__ = 'tables'
+
+    default_endpoint = Column(MediumText())
+    fetch_values_predicate = Column(String(1000))
+    main_dttm_col = Column(String(250))
+    schema = Column(String(255))
+    sql = Column(Text)
+    table_name = Column(String(250))
+    template_params = Column(Text)
+
+
+class SqlMetric(BaseMetricMixin, Base):
+    __tablename__ = 'sql_metrics'
+
+    expression = Column(Text)
+
+
+class TableColumn(BaseColumnMixin, Base):
+    __tablename__ = 'table_columns'
+
+    database_expression = Column(String(255))
+    expression = Column(Text)
+    python_date_format = Column(String(255))
+
+
+def upgrade():
+    bind = op.get_bind()
+    session = db.Session(bind=bind)
+
+    tables = [
+        Annotation,
+        Dashboard,
+        Database,
+        DruidCluster,
+        DruidColumn,
+        DruidDatasource,
+        DruidMetric,
+        Slice,
+        SqlaTable,
+        SqlMetric,
+        TableColumn,
+    ]
+
+    for table in tables:
+        for record in session.query(table).all():
+            for col in record.__table__.columns.values():
+                if not col.primary_key:
+                    if getattr(record, col.name) == '':
+                        setattr(record, col.name, None)
+
+        session.commit()
+
+    session.close()
+
+
+def downgrade():
+    pass

superset/migrations/versions/c82ee8a39623_add_implicit_tags.py

@@ -0,0 +1,75 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""Add implicit tags
+
+Revision ID: c82ee8a39623
+Revises: c18bd4186f15
+Create Date: 2018-07-26 11:10:23.653524
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = 'c82ee8a39623'
+down_revision = 'c617da68de7d'
+
+from alembic import op
+from sqlalchemy import Column, Enum, Integer, ForeignKey, String
+from sqlalchemy.ext.declarative import declarative_base
+
+from superset.models.helpers import AuditMixinNullable
+from superset.models.tags import (
+    ObjectTypes,
+    TagTypes,
+)
+
+
+Base = declarative_base()
+
+
+class Tag(Base, AuditMixinNullable):
+    """A tag attached to an object (query, chart or dashboard)."""
+    __tablename__ = 'tag'
+
+    id = Column(Integer, primary_key=True)
+    name = Column(String(250), unique=True)
+    type = Column(Enum(TagTypes))
+
+
+class TaggedObject(Base, AuditMixinNullable):
+    __tablename__ = 'tagged_object'
+
+    id = Column(Integer, primary_key=True)
+    tag_id = Column(Integer, ForeignKey('tag.id'))
+    object_id = Column(Integer)
+    object_type = Column(Enum(ObjectTypes))
+
+
+class User(Base):
+    """Declarative class to do query in upgrade"""
+    __tablename__ = 'ab_user'
+    id = Column(Integer, primary_key=True)
+
+
+def upgrade():
+    bind = op.get_bind()
+    Tag.__table__.create(bind)
+    TaggedObject.__table__.create(bind)
+
+
+def downgrade():
+    op.drop_table('tag')
+    op.drop_table('tagged_object')

superset/models/core.py

@@ -48,6 +48,7 @@ from superset import app, db, db_engine_specs, security_manager
 from superset.connectors.connector_registry import ConnectorRegistry
 from superset.legacy import update_time_range
 from superset.models.helpers import AuditMixinNullable, ImportMixin
+from superset.models.tags import ChartUpdater, DashboardUpdater, FavStarUpdater
 from superset.models.user_attributes import UserAttribute
 from superset.utils import (
     cache as cache_util,
@@ -359,6 +360,13 @@ class Slice(Model, AuditMixinNullable, ImportMixin):
         session.flush()
         return slc_to_import.id
 
+    @property
+    def url(self):
+        return (
+            '/superset/explore/?form_data=%7B%22slice_id%22%3A%20{0}%7D'
+            .format(self.id)
+        )
+
 
 sqla.event.listen(Slice, 'before_insert', set_related_perm)
 sqla.event.listen(Slice, 'before_update', set_related_perm)
@@ -1251,3 +1259,14 @@ class DatasourceAccessRequest(Model, AuditMixinNullable):
                 href = '{} Role'.format(r.name)
             action_list = action_list + '<li>' + href + '</li>'
         return '<ul>' + action_list + '</ul>'
+
+
+# events for updating tags
+sqla.event.listen(Slice, 'after_insert', ChartUpdater.after_insert)
+sqla.event.listen(Slice, 'after_update', ChartUpdater.after_update)
+sqla.event.listen(Slice, 'after_delete', ChartUpdater.after_delete)
+sqla.event.listen(Dashboard, 'after_insert', DashboardUpdater.after_insert)
+sqla.event.listen(Dashboard, 'after_update', DashboardUpdater.after_update)
+sqla.event.listen(Dashboard, 'after_delete', DashboardUpdater.after_delete)
+sqla.event.listen(FavStar, 'after_insert', FavStarUpdater.after_insert)
+sqla.event.listen(FavStar, 'after_delete', FavStarUpdater.after_delete)

superset/models/sql_lab.py

@@ -29,6 +29,7 @@ from sqlalchemy.orm import backref, relationship
 
 from superset import security_manager
 from superset.models.helpers import AuditMixinNullable, ExtraJSONMixin
+from superset.models.tags import QueryUpdater
 from superset.utils.core import QueryStatus, user_label
 
 
@@ -173,3 +174,12 @@ class SavedQuery(Model, AuditMixinNullable, ExtraJSONMixin):
     @property
     def sqlalchemy_uri(self):
         return self.database.sqlalchemy_uri
+
+    def url(self):
+        return '/superset/sqllab?savedQueryId={0}'.format(self.id)
+
+
+# events for updating tags
+sqla.event.listen(SavedQuery, 'after_insert', QueryUpdater.after_insert)
+sqla.event.listen(SavedQuery, 'after_update', QueryUpdater.after_update)
+sqla.event.listen(SavedQuery, 'after_delete', QueryUpdater.after_delete)

superset/models/tags.py

@@ -0,0 +1,244 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# pylint: disable=C,R,W,no-init
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import enum
+
+from flask_appbuilder import Model
+from sqlalchemy import Column, Enum, ForeignKey, Integer, String
+from sqlalchemy.orm import relationship, sessionmaker
+from sqlalchemy.orm.exc import NoResultFound
+
+from superset.models.helpers import AuditMixinNullable
+
+
+Session = sessionmaker(autoflush=False)
+
+
+class TagTypes(enum.Enum):
+
+    """
+    Types for tags.
+
+    Objects (queries, charts and dashboards) will have with implicit tags based
+    on metadata: types, owners and who favorited them. This way, user "alice"
+    can find all their objects by querying for the tag `owner:alice`.
+    """
+
+    # explicit tags, added manually by the owner
+    custom = 1
+
+    # implicit tags, generated automatically
+    type = 2
+    owner = 3
+    favorited_by = 4
+
+
+class ObjectTypes(enum.Enum):
+
+    """Object types."""
+
+    query = 1
+    chart = 2
+    dashboard = 3
+
+
+class Tag(Model, AuditMixinNullable):
+
+    """A tag attached to an object (query, chart or dashboard)."""
+
+    __tablename__ = 'tag'
+    id = Column(Integer, primary_key=True)  # pylint: disable=invalid-name
+    name = Column(String(250), unique=True)
+    type = Column(Enum(TagTypes))
+
+
+class TaggedObject(Model, AuditMixinNullable):
+
+    """An association between an object and a tag."""
+
+    __tablename__ = 'tagged_object'
+    id = Column(Integer, primary_key=True)  # pylint: disable=invalid-name
+    tag_id = Column(Integer, ForeignKey('tag.id'))
+    object_id = Column(Integer)
+    object_type = Column(Enum(ObjectTypes))
+
+    tag = relationship('Tag')
+
+
+def get_tag(name, session, type_):
+    try:
+        tag = session.query(Tag).filter_by(name=name, type=type_).one()
+    except NoResultFound:
+        tag = Tag(name=name, type=type_)
+        session.add(tag)
+        session.commit()
+
+    return tag
+
+
+def get_object_type(class_name):
+    mapping = {
+        'slice': ObjectTypes.chart,
+        'dashboard': ObjectTypes.dashboard,
+        'query': ObjectTypes.query,
+    }
+    try:
+        return mapping[class_name.lower()]
+    except KeyError:
+        raise Exception('No mapping found for {0}'.format(class_name))
+
+
+class ObjectUpdater(object):
+
+    object_type = None
+
+    @classmethod
+    def get_owners_ids(cls, target):
+        raise NotImplementedError('Subclass should implement `get_owners_ids`')
+
+    @classmethod
+    def _add_owners(cls, session, target):
+        for owner_id in cls.get_owners_ids(target):
+            name = 'owner:{0}'.format(owner_id)
+            tag = get_tag(name, session, TagTypes.owner)
+            tagged_object = TaggedObject(
+                tag_id=tag.id,
+                object_id=target.id,
+                object_type=ObjectTypes.chart,
+            )
+            session.add(tagged_object)
+
+    @classmethod
+    def after_insert(cls, mapper, connection, target):
+        # pylint: disable=unused-argument
+        session = Session(bind=connection)
+
+        # add `owner:` tags
+        cls._add_owners(session, target)
+
+        # add `type:` tags
+        tag = get_tag(
+            'type:{0}'.format(cls.object_type), session, TagTypes.type)
+        tagged_object = TaggedObject(
+            tag_id=tag.id,
+            object_id=target.id,
+            object_type=ObjectTypes.query,
+        )
+        session.add(tagged_object)
+
+        session.commit()
+
+    @classmethod
+    def after_update(cls, mapper, connection, target):
+        # pylint: disable=unused-argument
+        session = Session(bind=connection)
+
+        # delete current `owner:` tags
+        query = session.query(TaggedObject.id).join(Tag).filter(
+            TaggedObject.object_type == cls.object_type,
+            TaggedObject.object_id == target.id,
+            Tag.type == TagTypes.owner,
+        )
+        ids = [row[0] for row in query]
+        session.query(TaggedObject).filter(
+            TaggedObject.id.in_(ids)).delete(
+                synchronize_session=False)
+
+        # add `owner:` tags
+        cls._add_owners(session, target)
+
+        session.commit()
+
+    @classmethod
+    def after_delete(cls, mapper, connection, target):
+        # pylint: disable=unused-argument
+        session = Session(bind=connection)
+
+        # delete row from `tagged_objects`
+        session.query(TaggedObject).filter(
+            TaggedObject.object_type == cls.object_type,
+            TaggedObject.object_id == target.id,
+        ).delete()
+
+        session.commit()
+
+
+class ChartUpdater(ObjectUpdater):
+
+    object_type = 'chart'
+
+    @classmethod
+    def get_owners_ids(cls, target):
+        return [owner.id for owner in target.owners]
+
+
+class DashboardUpdater(ObjectUpdater):
+
+    object_type = 'dashboard'
+
+    @classmethod
+    def get_owners_ids(cls, target):
+        return [owner.id for owner in target.owners]
+
+
+class QueryUpdater(ObjectUpdater):
+
+    object_type = 'query'
+
+    @classmethod
+    def get_owners_ids(cls, target):
+        return [target.user_id]
+
+
+class FavStarUpdater(object):
+
+    @classmethod
+    def after_insert(cls, mapper, connection, target):
+        # pylint: disable=unused-argument
+        session = Session(bind=connection)
+        name = 'favorited_by:{0}'.format(target.user_id)
+        tag = get_tag(name, session, TagTypes.favorited_by)
+        tagged_object = TaggedObject(
+            tag_id=tag.id,
+            object_id=target.obj_id,
+            object_type=get_object_type(target.class_name),
+        )
+        session.add(tagged_object)
+
+        session.commit()
+
+    @classmethod
+    def after_delete(cls, mapper, connection, target):
+        # pylint: disable=unused-argument
+        session = Session(bind=connection)
+        name = 'favorited_by:{0}'.format(target.user_id)
+        query = session.query(TaggedObject.id).join(Tag).filter(
+            TaggedObject.object_id == target.obj_id,
+            Tag.type == TagTypes.favorited_by,
+            Tag.name == name,
+        )
+        ids = [row[0] for row in query]
+        session.query(TaggedObject).filter(
+            TaggedObject.id.in_(ids)).delete(
+                synchronize_session=False)
+
+        session.commit()

superset/sql_parse.py

@@ -23,7 +23,10 @@ from sqlparse.tokens import Keyword, Name
 
 RESULT_OPERATIONS = {'UNION', 'INTERSECT', 'EXCEPT', 'SELECT'}
 ON_KEYWORD = 'ON'
-PRECEDES_TABLE_NAME = {'FROM', 'JOIN', 'DESC', 'DESCRIBE', 'WITH'}
+PRECEDES_TABLE_NAME = {
+    'FROM', 'JOIN', 'DESCRIBE', 'WITH', 'LEFT JOIN', 'RIGHT JOIN',
+}
+CTE_PREFIX = 'CTE__'
 
 
 class ParsedQuery(object):
@@ -71,35 +74,23 @@ class ParsedQuery(object):
                     statements.append(sql)
         return statements
 
-    @staticmethod
-    def __precedes_table_name(token_value):
-        for keyword in PRECEDES_TABLE_NAME:
-            if keyword in token_value:
-                return True
-        return False
-
     @staticmethod
     def __get_full_name(identifier):
-        if len(identifier.tokens) > 1 and identifier.tokens[1].value == '.':
+        if len(identifier.tokens) > 2 and identifier.tokens[1].value == '.':
             return '{}.{}'.format(identifier.tokens[0].value,
                                   identifier.tokens[2].value)
         return identifier.get_real_name()
 
-    @staticmethod
-    def __is_result_operation(keyword):
-        for operation in RESULT_OPERATIONS:
-            if operation in keyword.upper():
-                return True
-        return False
-
     @staticmethod
     def __is_identifier(token):
         return isinstance(token, (IdentifierList, Identifier))
 
     def __process_identifier(self, identifier):
         # exclude subselects
-        if '(' not in '{}'.format(identifier):
-            self._table_names.add(self.__get_full_name(identifier))
+        if '(' not in str(identifier):
+            table_name = self.__get_full_name(identifier)
+            if table_name and not table_name.startswith(CTE_PREFIX):
+                self._table_names.add(table_name)
             return
 
         # store aliases
@@ -129,52 +120,49 @@ class ParsedQuery(object):
         exec_sql += f'CREATE TABLE {table_name} AS \n{sql}'
         return exec_sql
 
-    def __extract_from_token(self, token):
+    def __extract_from_token(self, token, depth=0):
         if not hasattr(token, 'tokens'):
             return
 
         table_name_preceding_token = False
 
         for item in token.tokens:
+            logging.debug(('  ' * depth) + str(item.ttype) + str(item.value))
             if item.is_group and not self.__is_identifier(item):
-                self.__extract_from_token(item)
+                self.__extract_from_token(item, depth=depth + 1)
 
-            if item.ttype in Keyword:
-                if self.__precedes_table_name(item.value.upper()):
-                    table_name_preceding_token = True
-                    continue
-
-            if not table_name_preceding_token:
+            if (
+                    item.ttype in Keyword and (
+                        item.normalized in PRECEDES_TABLE_NAME or
+                        item.normalized.endswith(' JOIN')
+                    )):
+                table_name_preceding_token = True
                 continue
 
-            if item.ttype in Keyword or item.value == ',':
-                if (self.__is_result_operation(item.value) or
-                        item.value.upper() == ON_KEYWORD):
-                    table_name_preceding_token = False
-                    continue
-                # FROM clause is over
-                break
+            if item.ttype in Keyword:
+                table_name_preceding_token = False
+                continue
 
-            if isinstance(item, Identifier):
-                self.__process_identifier(item)
-
-            if isinstance(item, IdentifierList):
-                for token in item.tokens:
-                    if self.__is_identifier(token):
+            if table_name_preceding_token:
+                if isinstance(item, Identifier):
+                    self.__process_identifier(item)
+                elif isinstance(item, IdentifierList):
+                    for token in item.get_identifiers():
                         self.__process_identifier(token)
-
-    def _get_limit_from_token(self, token):
-        if token.ttype == sqlparse.tokens.Literal.Number.Integer:
-            return int(token.value)
-        elif token.is_group:
-            return int(token.get_token_at_offset(1).value)
+            elif isinstance(item, IdentifierList):
+                for token in item.tokens:
+                    if not self.__is_identifier(token):
+                        self.__extract_from_token(item, depth=depth + 1)
 
     def _extract_limit_from_query(self, statement):
-        limit_token = None
-        for pos, item in enumerate(statement.tokens):
-            if item.ttype in Keyword and item.value.lower() == 'limit':
-                limit_token = statement.tokens[pos + 2]
-                return self._get_limit_from_token(limit_token)
+        idx, _ = statement.token_next_by(m=(Keyword, 'LIMIT'))
+        if idx is not None:
+            _, token = statement.token_next(idx=idx)
+            if token:
+                if isinstance(token, IdentifierList):
+                    _, token = token.token_next(idx=-1)
+                if token and token.ttype == sqlparse.tokens.Literal.Number.Integer:
+                    return int(token.value)
 
     def get_query_with_new_limit(self, new_limit):
         """returns the query with the specified limit"""

superset/templates/appbuilder/navbar.html

@@ -19,6 +19,7 @@
 {% set menu = appbuilder.menu %}
 {% set languages = appbuilder.languages %}
 {% set WARNING_MSG = appbuilder.app.config.get('WARNING_MSG') %}
+{% set app_icon_width = appbuilder.app.config.get('APP_ICON_WIDTH', 126) %}
 
 <div class="navbar navbar-static-top {{menu.extra_classes}}" role="navigation">
   <div class="container-fluid">
@@ -30,7 +31,7 @@
       </button>
       <a class="navbar-brand" href="/superset/profile/{{ current_user.username }}/">
         <img
-          width="126"
+          width="{{ app_icon_width }}"
           src="{{ appbuilder.app_icon }}"
           alt="{{ appbuilder.app_name }}"
         />

superset/templates/appbuilder/navbar_right.html

@@ -17,6 +17,7 @@
   under the License.
 #}
 
+{% set bug_report_url = appbuilder.app.config.get('BUG_REPORT_URL') %}
 {% set locale = session['locale'] %}
 {% if not locale %}
     {% set locale = 'en' %}
@@ -34,6 +35,17 @@
         </ul>
     </li>
 {% endif %}
+{% if bug_report_url %}
+<li>
+  <a
+    tabindex="-1"
+    href="{{ bug_report_url }}"
+    title="Report a bug"
+  >
+    <i class="fa fa-bug"></i>&nbsp;
+  </a>
+</li>
+{% endif %}
 {% if languages.keys()|length > 1 %}
 <li class="dropdown">
     <a class="dropdown-toggle" data-toggle="dropdown" href="javascript:void(0)">

superset/utils/core.py

@@ -865,10 +865,12 @@ def get_or_create_main_db():
     logging.info('Creating database reference')
     dbobj = get_main_database(db.session)
     if not dbobj:
-        dbobj = models.Database(database_name='main')
+        dbobj = models.Database(
+            database_name='main',
+            allow_csv_upload=True,
+            expose_in_sqllab=True,
+        )
     dbobj.set_sqlalchemy_uri(conf.get('SQLALCHEMY_DATABASE_URI'))
-    dbobj.expose_in_sqllab = True
-    dbobj.allow_csv_upload = True
     db.session.add(dbobj)
     db.session.commit()
     return dbobj

superset/views/__init__.py

@@ -22,3 +22,4 @@ from . import dashboard # noqa
 from . import annotations # noqa
 from . import datasource # noqa
 from . import schedules # noqa
+from . import tags # noqa

superset/views/api.py

@@ -15,16 +15,18 @@
 # specific language governing permissions and limitations
 # under the License.
 # pylint: disable=R
-import json
-
-from flask import g, request
+from flask import request
 from flask_appbuilder import expose
 from flask_appbuilder.security.decorators import has_access_api
+import simplejson as json
 
-from superset import appbuilder, security_manager
+from superset import appbuilder, db, security_manager
 from superset.common.query_context import QueryContext
+from superset.legacy import update_time_range
+import superset.models.core as models
 from superset.models.core import Log
-from .base import api, BaseSupersetView, data_payload_response, handle_api_exception
+from superset.utils import core as utils
+from .base import api, BaseSupersetView, handle_api_exception
 
 
 class Api(BaseSupersetView):
@@ -37,11 +39,37 @@ class Api(BaseSupersetView):
         """
         Takes a query_obj constructed in the client and returns payload data response
         for the given query_obj.
+        params: query_context: json_blob
         """
         query_context = QueryContext(**json.loads(request.form.get('query_context')))
-        security_manager.assert_datasource_permission(query_context.datasource, g.user)
-        payload_json = query_context.get_data()
-        return data_payload_response(payload_json)
+        security_manager.assert_datasource_permission(query_context.datasource)
+        payload_json = query_context.get_payload()
+        return json.dumps(
+            payload_json,
+            default=utils.json_int_dttm_ser,
+            ignore_nan=True,
+        )
+
+    @Log.log_this
+    @api
+    @handle_api_exception
+    @has_access_api
+    @expose('/v1/form_data/', methods=['GET'])
+    def query_form_data(self):
+        """
+        Get the formdata stored in the database for existing slice.
+        params: slice_id: integer
+        """
+        form_data = {}
+        slice_id = request.args.get('slice_id')
+        if slice_id:
+            slc = db.session.query(models.Slice).filter_by(id=slice_id).one_or_none()
+            if slc:
+                form_data = slc.form_data.copy()
+
+        update_time_range(form_data)
+
+        return json.dumps(form_data)
 
 
 appbuilder.add_view_no_menu(Api)

superset/views/base.py

@@ -43,6 +43,7 @@ FRONTEND_CONF_KEYS = (
     'DEFAULT_SQLLAB_LIMIT',
     'SQL_MAX_ROW',
     'SUPERSET_WEBSERVER_DOMAINS',
+    'SQLLAB_SAVE_WARNING_MESSAGE',
 )
 
 

superset/views/core.py

@@ -378,7 +378,7 @@ class CsvToDatabaseView(SimpleFormView):
             except OSError:
                 pass
             message = 'Table name {} already exists. Please pick another'.format(
-                form.name.data) if isinstance(e, IntegrityError) else e
+                form.name.data) if isinstance(e, IntegrityError) else str(e)
             flash(
                 message,
                 'danger')
@@ -610,8 +610,9 @@ class DashboardModelView(SupersetModelView, DeleteMixin):  # noqa
     }
 
     def pre_add(self, obj):
-        obj.slug = obj.slug.strip() or None
+        obj.slug = obj.slug or None
         if obj.slug:
+            obj.slug = obj.slug.strip()
             obj.slug = obj.slug.replace(' ', '-')
             obj.slug = re.sub(r'[^\w\-]+', '', obj.slug)
         if g.user not in obj.owners:
@@ -1088,7 +1089,7 @@ class Superset(BaseSupersetView):
         if not slc:
             abort(404)
         endpoint = '/superset/explore/?form_data={}'.format(
-            parse.quote(json.dumps(form_data)),
+            parse.quote(json.dumps({'slice_id': slice_id})),
         )
         if request.args.get('standalone') == 'true':
             endpoint += '&standalone=true'
@@ -1524,12 +1525,17 @@ class Superset(BaseSupersetView):
             db.session
             .query(models.Database)
             .filter_by(id=db_id)
-            .one()
+            .first()
         )
-        schemas = database.all_schema_names(cache=database.schema_cache_enabled,
-                                            cache_timeout=database.schema_cache_timeout,
-                                            force=force_refresh)
-        schemas = security_manager.schemas_accessible_by_user(database, schemas)
+        if database:
+            schemas = database.all_schema_names(
+                cache=database.schema_cache_enabled,
+                cache_timeout=database.schema_cache_timeout,
+                force=force_refresh)
+            schemas = security_manager.schemas_accessible_by_user(database, schemas)
+        else:
+            schemas = []
+
         return Response(
             json.dumps({'schemas': schemas}),
             mimetype='application/json')

superset/views/datasource.py

@@ -20,12 +20,11 @@ import json
 from flask import request
 from flask_appbuilder import expose
 from flask_appbuilder.security.decorators import has_access_api
-from flask_babel import gettext as __
 
 from superset import appbuilder, db
 from superset.connectors.connector_registry import ConnectorRegistry
 from superset.models.core import Database
-from .base import BaseSupersetView, check_ownership, json_error_response
+from .base import BaseSupersetView
 
 
 class Datasource(BaseSupersetView):
@@ -39,14 +38,6 @@ class Datasource(BaseSupersetView):
         orm_datasource = ConnectorRegistry.get_datasource(
             datasource_type, datasource_id, db.session)
 
-        if not check_ownership(orm_datasource, raise_if_false=False):
-            return json_error_response(
-                __(
-                    'You are not authorized to modify '
-                    'this data source configuration'),
-                status='401',
-            )
-
         if 'owners' in datasource:
             datasource['owners'] = db.session.query(orm_datasource.owner_class).filter(
                 orm_datasource.owner_class.id.in_(datasource['owners'])).all()

superset/views/tags.py

@@ -0,0 +1,217 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# pylint: disable=C,R,W
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from flask import request, Response
+from flask_appbuilder import expose
+from flask_appbuilder.security.decorators import has_access_api
+from jinja2.sandbox import SandboxedEnvironment
+import simplejson as json
+from sqlalchemy import and_, func
+from werkzeug.routing import BaseConverter
+
+from superset import app, appbuilder, db, utils
+from superset.jinja_context import current_user_id, current_username
+import superset.models.core
+from superset.models.sql_lab import SavedQuery
+from superset.models.tags import ObjectTypes, Tag, TaggedObject, TagTypes
+from .base import BaseSupersetView, json_success
+
+
+class ObjectTypeConverter(BaseConverter):
+
+    """Validate that object_type is indeed an object type."""
+
+    def to_python(self, object_type):
+        return ObjectTypes[object_type]
+
+    def to_url(self, object_type):
+        return object_type.name
+
+
+def process_template(content):
+    env = SandboxedEnvironment()
+    template = env.from_string(content)
+    context = {
+        'current_user_id': current_user_id,
+        'current_username': current_username,
+    }
+    return template.render(context)
+
+
+def get_name(obj):
+    if obj.Dashboard:
+        return obj.Dashboard.dashboard_title
+    elif obj.Slice:
+        return obj.Slice.slice_name
+    elif obj.SavedQuery:
+        return obj.SavedQuery.label
+
+
+def get_creator(obj):
+    if obj.Dashboard:
+        return obj.Dashboard.creator()
+    elif obj.Slice:
+        return obj.Slice.creator()
+    elif obj.SavedQuery:
+        return obj.SavedQuery.creator()
+
+
+def get_attribute(obj, attr):
+    if obj.Dashboard:
+        return getattr(obj.Dashboard, attr)
+    elif obj.Slice:
+        return getattr(obj.Slice, attr)
+    elif obj.SavedQuery:
+        return getattr(obj.SavedQuery, attr)
+
+
+class TagView(BaseSupersetView):
+
+    @has_access_api
+    @expose('/tags/suggestions/', methods=['GET'])
+    def suggestions(self):
+        query = (
+            db.session.query(TaggedObject)
+            .group_by(TaggedObject.tag_id)
+            .order_by(func.count().desc())
+            .all()
+        )
+        tags = [{'id': obj.tag.id, 'name': obj.tag.name} for obj in query]
+        return json_success(json.dumps(tags))
+
+    @has_access_api
+    @expose('/tags/<object_type:object_type>/<int:object_id>/', methods=['GET'])
+    def get(self, object_type, object_id):
+        """List all tags a given object has."""
+        query = db.session.query(TaggedObject).filter(and_(
+            TaggedObject.object_type == object_type,
+            TaggedObject.object_id == object_id))
+        tags = [{'id': obj.tag.id, 'name': obj.tag.name} for obj in query]
+        return json_success(json.dumps(tags))
+
+    @has_access_api
+    @expose('/tags/<object_type:object_type>/<int:object_id>/', methods=['POST'])
+    def post(self, object_type, object_id):
+        """Add new tags to an object."""
+        tagged_objects = []
+        for name in request.get_json(force=True):
+            if ':' in name:
+                type_name = name.split(':', 1)[0]
+                type_ = TagTypes[type_name]
+            else:
+                type_ = TagTypes.custom
+
+            tag = db.session.query(Tag).filter_by(name=name, type=type_).first()
+            if not tag:
+                tag = Tag(name=name, type=type_)
+
+            tagged_objects.append(
+                TaggedObject(
+                    object_id=object_id,
+                    object_type=object_type,
+                    tag=tag,
+                ),
+            )
+
+        db.session.add_all(tagged_objects)
+        db.session.commit()
+
+        return Response(status=201)  # 201 CREATED
+
+    @has_access_api
+    @expose('/tags/<object_type:object_type>/<int:object_id>/', methods=['DELETE'])
+    def delete(self, object_type, object_id):
+        """Remove tags from an object."""
+        tag_names = request.get_json(force=True)
+        if not tag_names:
+            return Response(status=403)
+
+        db.session.query(TaggedObject).filter(and_(
+            TaggedObject.object_type == object_type,
+            TaggedObject.object_id == object_id),
+            TaggedObject.tag.has(Tag.name.in_(tag_names)),
+        ).delete(synchronize_session=False)
+        db.session.commit()
+
+        return Response(status=204)  # 204 NO CONTENT
+
+    @has_access_api
+    @expose('/tagged_objects/', methods=['GET', 'POST'])
+    def tagged_objects(self):
+        query = db.session.query(
+            TaggedObject,
+            superset.models.core.Dashboard,
+            superset.models.core.Slice,
+            SavedQuery,
+        ).join(Tag)
+
+        tags = request.args.get('tags')
+        if not tags:
+            return json_success(json.dumps([]))
+
+        tags = [process_template(tag) for tag in tags.split(',')]
+        query = query.filter(Tag.name.in_(tags))
+
+        # filter types
+        types = request.args.get('types')
+        if types:
+            query = query.filter(TaggedObject.object_type.in_(types.split(',')))
+
+        # get names
+        query = query.outerjoin(
+            superset.models.core.Dashboard,
+            and_(
+                TaggedObject.object_id == superset.models.core.Dashboard.id,
+                TaggedObject.object_type == ObjectTypes.dashboard,
+            ),
+        ).outerjoin(
+            superset.models.core.Slice,
+            and_(
+                TaggedObject.object_id == superset.models.core.Slice.id,
+                TaggedObject.object_type == ObjectTypes.chart,
+            ),
+        ).outerjoin(
+            SavedQuery,
+            and_(
+                TaggedObject.object_id == SavedQuery.id,
+                TaggedObject.object_type == ObjectTypes.query,
+            ),
+        ).group_by(TaggedObject.object_id, TaggedObject.object_type)
+
+        objects = [
+            {
+                'id': get_attribute(obj, 'id'),
+                'type': obj.TaggedObject.object_type.name,
+                'name': get_name(obj),
+                'url': get_attribute(obj, 'url'),
+                'changed_on': get_attribute(obj, 'changed_on'),
+                'created_by': get_attribute(obj, 'created_by_fk'),
+                'creator': get_creator(obj),
+            }
+            for obj in query if get_attribute(obj, 'id')
+        ]
+
+        return json_success(json.dumps(objects, default=utils.core.json_int_dttm_ser))
+
+
+app.url_map.converters['object_type'] = ObjectTypeConverter
+appbuilder.add_view_no_menu(TagView)

superset/viz.py

@@ -1164,10 +1164,6 @@ class NVD3TimeSeriesViz(NVD3Viz):
             dfs.sort_values(ascending=False, inplace=True)
             df = df[dfs.index]
 
-        if fd.get('contribution'):
-            dft = df.T
-            df = (dft / dft.sum()).T
-
         rolling_type = fd.get('rolling_type')
         rolling_periods = int(fd.get('rolling_periods') or 0)
         min_periods = int(fd.get('min_periods') or 0)
@@ -1187,6 +1183,10 @@ class NVD3TimeSeriesViz(NVD3Viz):
         if min_periods:
             df = df[min_periods:]
 
+        if fd.get('contribution'):
+            dft = df.T
+            df = (dft / dft.sum()).T
+
         return df
 
     def run_extra_queries(self):
@@ -1254,7 +1254,9 @@ class NVD3TimeSeriesViz(NVD3Viz):
                     self.to_series(
                         diff, classed='time-shift-{}'.format(i), title_suffix=label))
 
-        return sorted(chart_data, key=lambda x: tuple(x['key']))
+        if not self.sort_series:
+            chart_data = sorted(chart_data, key=lambda x: tuple(x['key']))
+        return chart_data
 
 
 class MultiLineViz(NVD3Viz):

tests/core_tests.py

@@ -646,15 +646,14 @@ class CoreTests(SupersetTestCase):
         main_db_uri = (
             db.session.query(models.Database)
             .filter_by(database_name='main')
-            .all()
+            .one()
         )
-
         test_file = open(filename, 'rb')
         form_data = {
             'csv_file': test_file,
             'sep': ',',
             'name': table_name,
-            'con': main_db_uri[0].id,
+            'con': main_db_uri.id,
             'if_exists': 'append',
             'index_label': 'test_label',
             'mangle_dupe_cols': False,

tests/db_engine_specs_test.py

@@ -141,12 +141,26 @@ class DbEngineSpecsTestCase(SupersetTestCase):
         q2 = 'select * from (select * from my_subquery limit 10) where col=1 limit 20'
         q3 = 'select * from (select * from my_subquery limit 10);'
         q4 = 'select * from (select * from my_subquery limit 10) where col=1 limit 20;'
+        q5 = 'select * from mytable limit 10, 20'
+        q6 = 'select * from mytable limit 10 offset 20'
+        q7 = 'select * from mytable limit'
+        q8 = 'select * from mytable limit 10.0'
+        q9 = 'select * from mytable limit x'
+        q10 = 'select * from mytable limit x, 20'
+        q11 = 'select * from mytable limit x offset 20'
 
         self.assertEqual(engine_spec_class.get_limit_from_sql(q0), None)
         self.assertEqual(engine_spec_class.get_limit_from_sql(q1), 10)
         self.assertEqual(engine_spec_class.get_limit_from_sql(q2), 20)
         self.assertEqual(engine_spec_class.get_limit_from_sql(q3), None)
         self.assertEqual(engine_spec_class.get_limit_from_sql(q4), 20)
+        self.assertEqual(engine_spec_class.get_limit_from_sql(q5), 10)
+        self.assertEqual(engine_spec_class.get_limit_from_sql(q6), 10)
+        self.assertEqual(engine_spec_class.get_limit_from_sql(q7), None)
+        self.assertEqual(engine_spec_class.get_limit_from_sql(q8), None)
+        self.assertEqual(engine_spec_class.get_limit_from_sql(q9), None)
+        self.assertEqual(engine_spec_class.get_limit_from_sql(q10), None)
+        self.assertEqual(engine_spec_class.get_limit_from_sql(q11), None)
 
     def test_wrapped_query(self):
         self.sql_limit_regex(

tests/sql_parse_tests.py

@@ -47,6 +47,11 @@ class SupersetTestCase(unittest.TestCase):
             {'schemaname.tbname'},
             self.extract_tables('SELECT * FROM schemaname.tbname'))
 
+        # Ill-defined schema/table.
+        self.assertEquals(
+            set(),
+            self.extract_tables('SELECT * FROM schemaname.'))
+
         # quotes
         query = 'SELECT field1, field2 FROM tb_name'
         self.assertEquals({'tb_name'}, self.extract_tables(query))
@@ -167,7 +172,6 @@ class SupersetTestCase(unittest.TestCase):
     # DESCRIBE | DESC qualifiedName
     def test_describe(self):
         self.assertEquals({'t1'}, self.extract_tables('DESCRIBE t1'))
-        self.assertEquals({'t1'}, self.extract_tables('DESC t1'))
 
     # SHOW PARTITIONS FROM qualifiedName (WHERE booleanExpression)?
     # (ORDER BY sortItem (',' sortItem)*)? (LIMIT limit=(INTEGER_VALUE | ALL))?
@@ -349,6 +353,32 @@ class SupersetTestCase(unittest.TestCase):
             {'table_a', 'table_b', 'table_c'},
             self.extract_tables(query))
 
+    def test_mixed_from_clause(self):
+        query = """SELECT *
+            FROM table_a AS a, (select * from table_b) AS b, table_c as c
+            WHERE a.id = b.id and b.id = c.id"""
+        self.assertEquals(
+            {'table_a', 'table_b', 'table_c'},
+            self.extract_tables(query))
+
+    def test_nested_selects(self):
+        query = """
+            select (extractvalue(1,concat(0x7e,(select GROUP_CONCAT(TABLE_NAME)
+            from INFORMATION_SCHEMA.COLUMNS
+            WHERE TABLE_SCHEMA like "%bi%"),0x7e)));
+        """
+        self.assertEquals(
+            {'INFORMATION_SCHEMA.COLUMNS'},
+            self.extract_tables(query))
+        query = """
+            select (extractvalue(1,concat(0x7e,(select GROUP_CONCAT(COLUMN_NAME)
+            from INFORMATION_SCHEMA.COLUMNS
+            WHERE TABLE_NAME="bi_achivement_daily"),0x7e)));
+        """
+        self.assertEquals(
+            {'INFORMATION_SCHEMA.COLUMNS'},
+            self.extract_tables(query))
+
     def test_complex_extract_tables3(self):
         query = """SELECT somecol AS somecol
             FROM
@@ -386,6 +416,21 @@ class SupersetTestCase(unittest.TestCase):
             {'a', 'b', 'c', 'd', 'e', 'f'},
             self.extract_tables(query))
 
+    def test_complex_cte_with_prefix(self):
+        query = """
+        WITH CTE__test (SalesPersonID, SalesOrderID, SalesYear)
+        AS (
+            SELECT SalesPersonID, SalesOrderID, YEAR(OrderDate) AS SalesYear
+            FROM SalesOrderHeader
+            WHERE SalesPersonID IS NOT NULL
+        )
+        SELECT SalesPersonID, COUNT(SalesOrderID) AS TotalSales, SalesYear
+        FROM CTE__test
+        GROUP BY SalesYear, SalesPersonID
+        ORDER BY SalesPersonID, SalesYear;
+        """
+        self.assertEquals({'SalesOrderHeader'}, self.extract_tables(query))
+
     def test_basic_breakdown_statements(self):
         multi_sql = """
         SELECT * FROM ab_user;