More goodness

* Refactor the query runner to enable async mode.

* Refactore the sql calling functions into the QueryRunner class.

* Clean up the celery tests.

.gitignore

@@ -18,6 +18,8 @@ dist
 caravel.egg-info/
 app.db
 *.bak
+.idea
+*.sqllite
 
 # Node.js, webpack artifacts
 *.entry.js

caravel/assets/javascripts/SqlLab/TODO.md

@@ -0,0 +1,16 @@
+
+# TODO
+* Figure out how to organize the left panel, integrate Search
+* collapse sql beyond 10 lines
+* Security per-database (dropdown)
+* Get a to work
+
+## Cosmetic
+* SqlEditor buttons
+* use react-bootstrap-prompt for query title input
+* Make tabs look great
+
+# PROJECT
+* Write Runbook
+* Confirm backups
+* merge chef branch

caravel/assets/javascripts/SqlLab/actions.js

@@ -0,0 +1,112 @@
+export const RESET_STATE = 'RESET_STATE';
+export const ADD_QUERY_EDITOR = 'ADD_QUERY_EDITOR';
+export const REMOVE_QUERY_EDITOR = 'REMOVE_QUERY_EDITOR';
+export const ADD_TABLE = 'ADD_TABLE';
+export const REMOVE_TABLE = 'REMOVE_TABLE';
+export const START_QUERY = 'START_QUERY';
+export const STOP_QUERY = 'STOP_QUERY';
+export const END_QUERY = 'END_QUERY';
+export const REMOVE_QUERY = 'REMOVE_QUERY';
+export const EXPAND_TABLE = 'EXPAND_TABLE';
+export const COLLAPSE_TABLE = 'COLLAPSE_TABLE';
+export const QUERY_SUCCESS = 'QUERY_SUCCESS';
+export const QUERY_FAILED = 'QUERY_FAILED';
+export const QUERY_EDITOR_SETDB = 'QUERY_EDITOR_SETDB';
+export const QUERY_EDITOR_SET_SCHEMA = 'QUERY_EDITOR_SET_SCHEMA';
+export const QUERY_EDITOR_SET_TITLE = 'QUERY_EDITOR_SET_TITLE';
+export const QUERY_EDITOR_SET_AUTORUN = 'QUERY_EDITOR_SET_AUTORUN';
+export const QUERY_EDITOR_SET_SQL = 'QUERY_EDITOR_SET_SQL';
+export const SET_WORKSPACE_DB = 'SET_WORKSPACE_DB';
+export const ADD_WORKSPACE_QUERY = 'ADD_WORKSPACE_QUERY';
+export const REMOVE_WORKSPACE_QUERY = 'REMOVE_WORKSPACE_QUERY';
+export const SET_ACTIVE_QUERY_EDITOR = 'SET_ACTIVE_QUERY_EDITOR';
+export const ADD_ALERT = 'ADD_ALERT';
+export const REMOVE_ALERT = 'REMOVE_ALERT';
+
+export function resetState() {
+  return { type: RESET_STATE };
+}
+
+export function addQueryEditor(queryEditor) {
+  return { type: ADD_QUERY_EDITOR, queryEditor };
+}
+
+export function addAlert(alert) {
+  return { type: ADD_ALERT, alert };
+}
+
+export function removeAlert(alert) {
+  return { type: REMOVE_ALERT, alert };
+}
+
+export function setActiveQueryEditor(queryEditor) {
+  return { type: SET_ACTIVE_QUERY_EDITOR, queryEditor };
+}
+
+export function removeQueryEditor(queryEditor) {
+  return { type: REMOVE_QUERY_EDITOR, queryEditor };
+}
+
+export function removeQuery(query) {
+  return { type: REMOVE_QUERY, query };
+}
+
+export function queryEditorSetDb(queryEditor, dbId) {
+  return { type: QUERY_EDITOR_SETDB, queryEditor, dbId };
+}
+
+export function queryEditorSetSchema(queryEditor, schema) {
+  return { type: QUERY_EDITOR_SET_SCHEMA, queryEditor, schema };
+}
+
+export function queryEditorSetAutorun(queryEditor, autorun) {
+  return { type: QUERY_EDITOR_SET_AUTORUN, queryEditor, autorun };
+}
+
+export function queryEditorSetTitle(queryEditor, title) {
+  return { type: QUERY_EDITOR_SET_TITLE, queryEditor, title };
+}
+
+export function queryEditorSetSql(queryEditor, sql) {
+  return { type: QUERY_EDITOR_SET_SQL, queryEditor, sql };
+}
+
+export function addTable(table) {
+  return { type: ADD_TABLE, table };
+}
+
+export function expandTable(table) {
+  return { type: EXPAND_TABLE, table };
+}
+
+export function collapseTable(table) {
+  return { type: COLLAPSE_TABLE, table };
+}
+
+export function removeTable(table) {
+  return { type: REMOVE_TABLE, table };
+}
+
+export function startQuery(query) {
+  return { type: START_QUERY, query };
+}
+
+export function stopQuery(query) {
+  return { type: STOP_QUERY, query };
+}
+
+export function querySuccess(query, results) {
+  return { type: QUERY_SUCCESS, query, results };
+}
+
+export function queryFailed(query, msg) {
+  return { type: QUERY_FAILED, query, msg };
+}
+
+export function addWorkspaceQuery(query) {
+  return { type: ADD_WORKSPACE_QUERY, query };
+}
+
+export function removeWorkspaceQuery(query) {
+  return { type: REMOVE_WORKSPACE_QUERY, query };
+}

caravel/assets/javascripts/SqlLab/components/Alerts.jsx

@@ -0,0 +1,40 @@
+import React from 'react';
+import { Alert } from 'react-bootstrap';
+import { connect } from 'react-redux';
+import { bindActionCreators } from 'redux';
+import * as Actions from '../actions';
+
+class Alerts extends React.Component {
+  removeAlert(alert) {
+    this.props.actions.removeAlert(alert);
+  }
+  render() {
+    const alerts = this.props.alerts.map((alert) =>
+      <Alert
+        bsStyle={alert.bsStyle}
+        style={{ width: '500px', textAlign: 'midddle', margin: '10px auto' }}
+      >
+        {alert.msg}
+        <i
+          className="fa fa-close pull-right"
+          onClick={this.removeAlert.bind(this, alert) }
+          style={{ cursor: 'pointer' }}
+        />
+      </Alert>
+    );
+    return (
+      <div>{alerts}</div>
+    );
+  }
+}
+
+Alerts.propTypes = {
+  alerts: React.PropTypes.array,
+};
+
+function mapDispatchToProps(dispatch) {
+  return {
+    actions: bindActionCreators(Actions, dispatch),
+  };
+}
+export default connect(null, mapDispatchToProps)(Alerts);

caravel/assets/javascripts/SqlLab/components/ButtonWithTooltip.jsx

@@ -0,0 +1,46 @@
+import React from 'react';
+import { Button, OverlayTrigger, Tooltip } from 'react-bootstrap';
+
+const ButtonWithTooltip = (props) => {
+  let tooltip = (
+    <Tooltip id="tooltip">
+      {props.tooltip}
+    </Tooltip>
+  );
+  return (
+    <OverlayTrigger
+      overlay={tooltip}
+      delayShow={300}
+      placement={props.placement}
+      delayHide={150}
+    >
+      <Button
+        onClick={props.onClick}
+        bsStyle={props.bsStyle}
+        disabled={props.disabled}
+        className={props.className}
+      >
+          {props.children}
+      </Button>
+    </OverlayTrigger>
+  );
+};
+
+ButtonWithTooltip.defaultProps = {
+  onClick: () => {},
+  disabled: false,
+  placement: 'top',
+  bsStyle: 'default',
+};
+
+ButtonWithTooltip.propTypes = {
+  bsStyle: React.PropTypes.string,
+  children: React.PropTypes.element,
+  className: React.PropTypes.string,
+  disabled: React.PropTypes.bool,
+  onClick: React.PropTypes.func,
+  placement: React.PropTypes.string,
+  tooltip: React.PropTypes.string,
+};
+
+export default ButtonWithTooltip;

caravel/assets/javascripts/SqlLab/components/LeftPane.jsx

@@ -0,0 +1,78 @@
+import React from 'react';
+import { Alert, Button, Label } from 'react-bootstrap';
+import { connect } from 'react-redux';
+import { bindActionCreators } from 'redux';
+import * as Actions from '../actions';
+import QueryLink from './QueryLink';
+import shortid from 'shortid';
+
+import 'react-select/dist/react-select.css';
+
+const LeftPane = (props) => {
+  let queryElements;
+  if (props.workspaceQueries.length > 0) {
+    queryElements = props.workspaceQueries.map((q) => <QueryLink query={q} />);
+  } else {
+    queryElements = (
+      <Alert bsStyle="info">
+        Use the save button on the SQL editor to save a query into this section for
+        future reference
+      </Alert>
+    );
+  }
+  return (
+    <div className="panel panel-default LeftPane">
+      <div className="panel-heading">
+        <h6 className="m-r-10">
+          <i className="fa fa-flask" />
+          SQL Lab <Label bsStyle="danger">ALPHA</Label>
+        </h6>
+      </div>
+      <div className="panel-body">
+        <div>
+          <h6>
+            <span className="fa-stack">
+              <i className="fa fa-database fa-stack-lg"></i>
+              <i className="fa fa-search fa-stack-1x"></i>
+            </span> Saved Queries
+          </h6>
+          <div>
+            {queryElements}
+          </div>
+          <hr />
+          <Button onClick={props.actions.resetState.bind(this)}>
+            Reset State
+          </Button>
+          <Button onClick={props.actions.addAlert.bind(this, {
+            msg: 'This info alert is a demo alert',
+            bsStyle: 'info',
+          })}>
+            Add Alert
+          </Button>
+        </div>
+      </div>
+    </div>
+  );
+};
+
+LeftPane.propTypes = {
+  workspaceQueries: React.PropTypes.array,
+  actions: React.PropTypes.object,
+};
+
+LeftPane.defaultProps = {
+  workspaceQueries: [],
+};
+
+function mapStateToProps(state) {
+  return {
+    workspaceQueries: state.workspaceQueries,
+  };
+}
+function mapDispatchToProps(dispatch) {
+  return {
+    actions: bindActionCreators(Actions, dispatch),
+  };
+}
+
+export default connect(mapStateToProps, mapDispatchToProps)(LeftPane);

caravel/assets/javascripts/SqlLab/components/Link.jsx

@@ -0,0 +1,52 @@
+import React from 'react';
+import { OverlayTrigger, Tooltip } from 'react-bootstrap';
+
+
+class Link extends React.Component {
+  render() {
+    let tooltip = (
+      <Tooltip id="tooltip">
+        {this.props.tooltip}
+      </Tooltip>
+    );
+    const link = (
+      <a
+        href={this.props.href}
+        onClick={this.props.onClick}
+        className={'Link ' + this.props.className}
+      >
+          {this.props.children}
+      </a>
+    );
+    if (this.props.tooltip) {
+      return (
+        <OverlayTrigger
+          overlay={tooltip}
+          placement={this.props.placement}
+          delayShow={300}
+          delayHide={150}
+        >
+          {link}
+        </OverlayTrigger>
+      );
+    }
+    return link;
+  }
+}
+Link.propTypes = {
+  className: React.PropTypes.string,
+  href: React.PropTypes.string,
+  onClick: React.PropTypes.func,
+  tooltip: React.PropTypes.string,
+  placement: React.PropTypes.string,
+  children: React.PropTypes.object,
+};
+Link.defaultProps = {
+  disabled: false,
+  href: '#',
+  tooltip: null,
+  placement: 'top',
+  onClick: () => {},
+};
+
+export default Link;

caravel/assets/javascripts/SqlLab/components/QueryHistory.jsx

@@ -0,0 +1,50 @@
+import React from 'react';
+
+import { connect } from 'react-redux';
+import { bindActionCreators } from 'redux';
+import * as Actions from '../actions';
+
+import QueryTable from './QueryTable';
+import { Alert } from 'react-bootstrap';
+
+const QueryHistory = (props) => {
+  const activeQeId = props.tabHistory[props.tabHistory.length - 1];
+  const queries = props.queries.filter((q) => (q.sqlEditorId === activeQeId));
+  if (queries.length > 0) {
+    return (
+      <QueryTable
+        columns={['state', 'started', 'duration', 'rows', 'sql', 'actions']}
+        queries={queries}
+      />
+    );
+  }
+  return (
+    <Alert bsStyle="info">
+      No query history yet...
+    </Alert>
+  );
+};
+
+QueryHistory.defaultProps = {
+  queries: [],
+};
+
+QueryHistory.propTypes = {
+  queries: React.PropTypes.array,
+  tabHistory: React.PropTypes.array,
+  actions: React.PropTypes.object,
+};
+
+function mapStateToProps(state) {
+  return {
+    queries: state.queries,
+    tabHistory: state.tabHistory,
+  };
+}
+function mapDispatchToProps(dispatch) {
+  return {
+    actions: bindActionCreators(Actions, dispatch),
+  };
+}
+
+export default connect(mapStateToProps, mapDispatchToProps)(QueryHistory);

caravel/assets/javascripts/SqlLab/components/QueryLink.jsx

@@ -0,0 +1,60 @@
+import React from 'react';
+import { ButtonGroup } from 'react-bootstrap';
+import Link from './Link';
+import { connect } from 'react-redux';
+import { bindActionCreators } from 'redux';
+import * as Actions from '../actions';
+import shortid from 'shortid';
+
+// CSS
+import 'react-select/dist/react-select.css';
+
+class QueryLink extends React.Component {
+  popTab() {
+    const qe = {
+      id: shortid.generate(),
+      title: this.props.query.title,
+      dbId: this.props.query.dbId,
+      autorun: false,
+      sql: this.props.query.sql,
+    };
+    this.props.actions.addQueryEditor(qe);
+  }
+  render() {
+    return (
+      <div className="ws-el">
+        {this.props.query.title}
+        <ButtonGroup className="ws-el-controls pull-right">
+          <Link
+            className="fa fa-plus-circle"
+            onClick={this.popTab.bind(this)}
+            tooltip="Pop this query in a new tab"
+            href="#"
+          />
+          <Link
+            className="fa fa-trash"
+            onClick={this.props.actions.removeWorkspaceQuery.bind(this, this.props.query)}
+            tooltip="Remove query from workspace"
+            href="#"
+          />
+        </ButtonGroup>
+      </div>
+    );
+  }
+}
+
+QueryLink.propTypes = {
+  query: React.PropTypes.object,
+  actions: React.PropTypes.object,
+};
+
+QueryLink.defaultProps = {
+};
+
+function mapDispatchToProps(dispatch) {
+  return {
+    actions: bindActionCreators(Actions, dispatch),
+  };
+}
+export default connect(null, mapDispatchToProps)(QueryLink);
+

caravel/assets/javascripts/SqlLab/components/QueryLog.jsx

@@ -0,0 +1,51 @@
+import React from 'react';
+
+import { connect } from 'react-redux';
+import { bindActionCreators } from 'redux';
+import * as Actions from '../actions';
+
+import QueryTable from './QueryTable';
+import { Alert } from 'react-bootstrap';
+
+class QueryLog extends React.Component {
+  render() {
+    const activeQeId = this.props.tabHistory[this.props.tabHistory.length - 1];
+    const queries = this.props.queries.filter((q) => (q.sqlEditorId === activeQeId));
+    if (queries.length > 0) {
+      return (
+        <QueryTable
+          columns={['state', 'started', 'duration', 'rows', 'sql', 'actions']}
+          queries={queries}
+        />
+      );
+    }
+    return (
+      <Alert bsStyle="info">
+        No query history yet...
+      </Alert>
+    );
+  }
+}
+QueryLog.defaultProps = {
+  queries: [],
+};
+
+QueryLog.propTypes = {
+  queries: React.PropTypes.array,
+  tabHistory: React.PropTypes.array,
+  actions: React.PropTypes.object,
+};
+
+function mapStateToProps(state) {
+  return {
+    queries: state.queries,
+    tabHistory: state.tabHistory,
+  };
+}
+function mapDispatchToProps(dispatch) {
+  return {
+    actions: bindActionCreators(Actions, dispatch),
+  };
+}
+
+export default connect(mapStateToProps, mapDispatchToProps)(QueryLog);

caravel/assets/javascripts/SqlLab/components/QuerySearch.jsx

@@ -0,0 +1,74 @@
+import React from 'react';
+import SplitPane from 'react-split-pane';
+import Select from 'react-select';
+import { Button } from 'react-bootstrap';
+
+import { connect } from 'react-redux';
+import { bindActionCreators } from 'redux';
+import * as Actions from '../actions';
+import QueryTable from './QueryTable';
+
+class QuerySearch extends React.Component {
+  constructor(props) {
+    super(props);
+    this.state = {
+      queryText: '',
+    };
+  }
+  changeQueryText(value) {
+    this.setState({ queryText: value });
+  }
+  render() {
+    const queries = this.props.queries;
+    return (
+      <SplitPane split="vertical" minSize={200} defaultSize={300}>
+        <div className="pane-cell pane-west m-t-5">
+          <div className="panel panel-default Workspace">
+            <div className="panel-heading">
+              <h6>
+                <i className="fa fa-search" /> Search Queries
+              </h6>
+            </div>
+            <div className="panel-body">
+              <input type="text" className="form-control" placeholder="Query Text" />
+              <Select
+                name="select-user"
+                placeholder="[User]"
+                options={['maxime_beauchemin', 'someone else']}
+                value={'maxime_beauchemin'}
+                className="m-t-10"
+                autosize={false}
+              />
+            </div>
+          </div>
+        </div>
+        <div className="pane-cell">
+          <QueryTable
+            columns={['state', 'started', 'duration', 'rows', 'sql', 'actions']}
+            queries={queries}
+          />
+        </div>
+        <Button>Search!</Button>
+      </SplitPane>
+    );
+  }
+}
+QuerySearch.propTypes = {
+  queries: React.PropTypes.array,
+};
+QuerySearch.defaultProps = {
+  queries: [],
+};
+
+function mapStateToProps(state) {
+  return {
+    queries: state.queries,
+  };
+}
+function mapDispatchToProps(dispatch) {
+  return {
+    actions: bindActionCreators(Actions, dispatch),
+  };
+}
+
+export default connect(mapStateToProps, mapDispatchToProps)(QuerySearch);

caravel/assets/javascripts/SqlLab/components/QueryTable.jsx

@@ -0,0 +1,120 @@
+import React from 'react';
+
+import { connect } from 'react-redux';
+import { bindActionCreators } from 'redux';
+import * as Actions from '../actions';
+
+import moment from 'moment';
+import { Table } from 'reactable';
+
+import SyntaxHighlighter from 'react-syntax-highlighter';
+import { github } from 'react-syntax-highlighter/dist/styles';
+
+import Link from './Link';
+import VisualizeModal from './VisualizeModal';
+
+// TODO move to CSS
+const STATE_COLOR_MAP = {
+  failed: 'red',
+  running: 'lime',
+  success: 'green',
+};
+
+class QueryTable extends React.Component {
+  constructor(props) {
+    super(props);
+    this.state = {
+      showVisualizeModal: false,
+      activeQuery: null,
+    };
+  }
+  hideVisualizeModal() {
+    this.setState({ showVisualizeModal: false });
+  }
+  showVisualizeModal(query) {
+    this.setState({ showVisualizeModal: true });
+    this.setState({ activeQuery: query });
+  }
+  render() {
+    const data = this.props.queries.map((query) => {
+      const q = Object.assign({}, query);
+      const since = (q.endDttm) ? q.endDttm : new Date();
+      let duration = since.valueOf() - q.startDttm.valueOf();
+      duration = moment.utc(duration);
+      if (q.endDttm) {
+        q.duration = duration.format('HH:mm:ss.SS');
+      }
+      q.started = moment(q.startDttm).format('HH:mm:ss');
+      q.sql = <SyntaxHighlighter language="sql" style={github}>{q.sql}</SyntaxHighlighter>;
+      q.state = (
+        <span
+          className="label label-default"
+          style={{ backgroundColor: STATE_COLOR_MAP[q.state] }}
+        >
+          {q.state}
+        </span>
+      );
+      q.actions = (
+        <div>
+          <Link
+            className="fa fa-line-chart fa-lg"
+            tooltip="Visualize the data out of this query"
+            onClick={this.showVisualizeModal.bind(this, query)}
+            href="#"
+          />
+          <Link
+            className="fa fa-plus-circle"
+            tooltip="Pop a tab containing this query"
+            href="#"
+          />
+          <Link
+            className="fa fa-trash"
+            href="#"
+            tooltip="Remove query from log"
+            onClick={this.props.actions.removeQuery.bind(this, query)}
+          />
+          <Link
+            className="fa fa-map-pin"
+            tooltip="Pin this query to the top of this query log"
+            href="#"
+          />
+        </div>
+      );
+
+      return q;
+    }).reverse();
+    return (
+      <div>
+        <VisualizeModal
+          show={this.state.showVisualizeModal}
+          query={this.state.activeQuery}
+          onHide={this.hideVisualizeModal.bind(this)}
+        />
+        <Table
+          columns={['state', 'started', 'duration', 'rows', 'sql', 'actions']}
+          className="table table-condensed"
+          data={data}
+        />
+      </div>
+    );
+  }
+}
+QueryTable.propTypes = {
+  columns: React.PropTypes.array,
+  actions: React.PropTypes.object,
+  queries: React.PropTypes.object,
+};
+QueryTable.defaultProps = {
+  columns: ['state', 'started', 'duration', 'rows', 'sql', 'actions'],
+  queries: [],
+};
+
+function mapStateToProps() {
+  return {};
+}
+function mapDispatchToProps(dispatch) {
+  return {
+    actions: bindActionCreators(Actions, dispatch),
+  };
+}
+export default connect(mapStateToProps, mapDispatchToProps)(QueryTable);

caravel/assets/javascripts/SqlLab/components/ResultSet.jsx

@@ -0,0 +1,94 @@
+import React from 'react';
+import { Alert, Button } from 'react-bootstrap';
+import { Table } from 'reactable';
+
+import VisualizeModal from './VisualizeModal';
+
+
+class ResultSet extends React.Component {
+  constructor(props) {
+    super(props);
+    this.state = {
+      searchText: '',
+      showModal: false,
+    };
+  }
+  shouldComponentUpdate(nextProps, nextState) {
+    return (
+      this.state.searchText !== nextState.searchText ||
+      this.state.showModal !== nextState.showModal
+    );
+  }
+  changeSearch(event) {
+    this.setState({ searchText: event.target.value });
+  }
+  showModal() {
+    this.setState({ showModal: true });
+  }
+  hideModal() {
+    this.setState({ showModal: false });
+  }
+  render() {
+    const results = this.props.query.results;
+    let controls = <div className="noControls" />;
+    if (this.props.showControls) {
+      controls = (
+        <div className="ResultSetControls">
+          <div className="clearfix">
+            <div className="pull-left">
+              <Button className="m-r-5" onClick={this.showModal.bind(this)}>
+                <i className="fa fa-line-chart m-l-1" /> Visualize
+              </Button>
+              <Button className="m-r-5"><i className="fa fa-file-text-o" /> .CSV</Button>
+            </div>
+            <div className="pull-right">
+              <input
+                type="text"
+                onChange={this.changeSearch.bind(this)}
+                className="form-control"
+                placeholder="Search Results"
+              />
+            </div>
+          </div>
+        </div>
+      );
+    }
+    if (results.data.length > 0) {
+      return (
+        <div>
+          <VisualizeModal
+            show={this.state.showModal}
+            query={this.props.query}
+            onHide={this.hideModal.bind(this)}
+          />
+          {controls}
+          <div className="ResultSet">
+            <Table
+              data={results.data}
+              columns={results.columns}
+              sortable
+              className="table table-condensed table-bordered"
+              filterBy={this.state.searchText}
+              filterable={results.columns}
+              hideFilterInput
+            />
+          </div>
+        </div>
+      );
+    }
+    return (<Alert bsStyle="warning">The query returned no data</Alert>);
+  }
+}
+ResultSet.propTypes = {
+  query: React.PropTypes.object,
+  showControls: React.PropTypes.boolean,
+  search: React.PropTypes.boolean,
+  searchText: React.PropTypes.string,
+};
+ResultSet.defaultProps = {
+  showControls: true,
+  search: true,
+  searchText: '',
+};
+
+export default ResultSet;

caravel/assets/javascripts/SqlLab/components/SouthPane.jsx

@@ -0,0 +1,42 @@
+import { Tab, Tabs } from 'react-bootstrap';
+import QueryHistory from './QueryHistory';
+import ResultSet from './ResultSet';
+import React from 'react';
+
+const SouthPane = (props) => {
+  let results;
+  if (props.latestQuery) {
+    if (props.latestQuery.state === 'running') {
+      results = (
+        <img className="loading" alt="Loading.." src="/static/assets/images/loading.gif" />
+      );
+    } else if (props.latestQuery.state === 'failed') {
+      results = <div className="alert alert-danger">{props.latestQuery.msg}</div>;
+    } else if (props.latestQuery.state === 'success') {
+      results = <ResultSet showControls query={props.latestQuery} />;
+    }
+  } else {
+    results = <div className="alert alert-info">Run a query to display results here</div>;
+  }
+  return (
+    <Tabs bsStyle="pills">
+      <Tab title="Results" eventKey={1}>
+        <div style={{ overflow: 'auto' }}>
+          {results}
+        </div>
+      </Tab>
+      <Tab title="Query History" eventKey={2}>
+        <QueryHistory />
+      </Tab>
+    </Tabs>
+  );
+};
+
+SouthPane.propTypes = {
+  latestQuery: React.PropTypes.object,
+};
+
+SouthPane.defaultProps = {
+};
+
+export default SouthPane;

caravel/assets/javascripts/SqlLab/components/SqlEditor.jsx

@@ -0,0 +1,227 @@
+const $ = window.$ = require('jquery');
+import React from 'react';
+import {
+  Button,
+  ButtonGroup,
+  DropdownButton,
+  Label,
+  MenuItem,
+  OverlayTrigger,
+  Tooltip,
+} from 'react-bootstrap';
+
+import AceEditor from 'react-ace';
+import 'brace/mode/sql';
+import 'brace/theme/github';
+import 'brace/ext/language_tools';
+
+import { bindActionCreators } from 'redux';
+import { connect } from 'react-redux';
+import * as Actions from '../actions';
+import shortid from 'shortid';
+import ButtonWithTooltip from './ButtonWithTooltip';
+import SouthPane from './SouthPane';
+import Timer from './Timer';
+
+import SqlEditorTopToolbar from './SqlEditorTopToolbar';
+
+// CSS
+import 'react-select/dist/react-select.css';
+
+class SqlEditor extends React.Component {
+  constructor(props) {
+    super(props);
+    this.state = {
+      autorun: props.queryEditor.autorun,
+      sql: props.queryEditor.sql,
+    };
+  }
+  componentDidMount() {
+    this.onMount();
+  }
+  onMount() {
+    if (this.state.autorun) {
+      this.setState({ autorun: false });
+      this.props.actions.queryEditorSetAutorun(this.props.queryEditor, false);
+      this.startQuery();
+    }
+  }
+  startQuery() {
+    const that = this;
+    const query = {
+      id: shortid.generate(),
+      sqlEditorId: this.props.queryEditor.id,
+      sql: this.props.queryEditor.sql,
+      state: 'running',
+      tab: this.props.queryEditor.title,
+      dbId: this.props.queryEditor.dbId,
+      startDttm: new Date(),
+    };
+    const url = '/caravel/sql_json/';
+    const data = {
+      sql: this.props.queryEditor.sql,
+      database_id: this.props.queryEditor.dbId,
+      schema: this.props.queryEditor.schema,
+      json: true,
+    };
+    this.props.actions.startQuery(query);
+    $.ajax({
+      type: 'POST',
+      dataType: 'json',
+      url,
+      data,
+      success(results) {
+        try {
+          that.props.actions.querySuccess(query, results);
+        } catch (e) {
+          that.props.actions.queryFailed(query, e);
+        }
+      },
+      error(err) {
+        let msg = '';
+        try {
+          msg = err.responseJSON.error;
+        } catch (e) {
+          msg = (err.responseText) ? err.responseText : e;
+        }
+        that.props.actions.queryFailed(query, msg);
+      },
+    });
+  }
+  stopQuery() {
+    this.props.actions.stopQuery(this.props.latestQuery);
+  }
+  textChange(text) {
+    this.setState({ sql: text });
+    this.props.actions.queryEditorSetSql(this.props.queryEditor, text);
+  }
+  addWorkspaceQuery() {
+    this.props.actions.addWorkspaceQuery({
+      id: shortid.generate(),
+      sql: this.state.sql,
+      dbId: this.props.queryEditor.dbId,
+      schema: this.props.queryEditor.schema,
+      title: this.props.queryEditor.title,
+    });
+  }
+  ctasChange() {}
+  visualize() {}
+  render() {
+    let runButtons = (
+      <ButtonGroup className="inline m-r-5">
+        <Button onClick={this.startQuery.bind(this)} disabled={!(this.props.queryEditor.dbId)}>
+          <i className="fa fa-table" /> Run
+        </Button>
+      </ButtonGroup>
+    );
+    if (this.props.latestQuery && this.props.latestQuery.state === 'running') {
+      runButtons = (
+        <ButtonGroup className="inline m-r-5">
+          <Button onClick={this.stopQuery.bind(this)}>
+            <a className="fa fa-stop" /> Stop
+          </Button>
+        </ButtonGroup>
+      );
+    }
+    const rightButtons = (
+      <ButtonGroup className="inlineblock">
+        <ButtonWithTooltip
+          tooltip="Save this query in your workspace"
+          placement="left"
+          onClick={this.addWorkspaceQuery.bind(this)}
+        >
+          <i className="fa fa-save" />&nbsp;
+        </ButtonWithTooltip>
+        <DropdownButton id="ddbtn-export" pullRight title={<i className="fa fa-file-o" />}>
+          <MenuItem
+            onClick={this.notImplemented}
+          >
+            <i className="fa fa-file-text-o" /> export to .csv
+          </MenuItem>
+          <MenuItem
+            onClick={this.notImplemented}
+          >
+            <i className="fa fa-file-code-o" /> export to .json
+          </MenuItem>
+        </DropdownButton>
+      </ButtonGroup>
+    );
+    let limitWarning = null;
+    const rowLimit = 1000;
+    if (this.props.latestQuery && this.props.latestQuery.rows === rowLimit) {
+      const tooltip = (
+        <Tooltip id="tooltip">
+          It appears that the number of rows in the query results displayed
+          was limited on the server side to the {rowLimit} limit.
+        </Tooltip>
+      );
+      limitWarning = (
+        <OverlayTrigger placement="left" overlay={tooltip}>
+          <Label bsStyle="warning" className="m-r-5">LIMIT</Label>
+        </OverlayTrigger>
+      );
+    }
+    const editorBottomBar = (
+      <div className="clearfix sql-toolbar padded">
+        <div className="pull-left">
+          {runButtons}
+          <span className="inlineblock valignTop" style={{ height: '20px' }}>
+            <input type="text" className="form-control" placeholder="CREATE TABLE AS" />
+          </span>
+        </div>
+        <div className="pull-right">
+          {limitWarning}
+          <Timer query={this.props.latestQuery} />
+          {rightButtons}
+        </div>
+      </div>
+    );
+    return (
+      <div className="SqlEditor">
+        <div>
+          <div>
+            <SqlEditorTopToolbar queryEditor={this.props.queryEditor} />
+            <AceEditor
+              mode="sql"
+              name={this.props.queryEditor.title}
+              theme="github"
+              minLines={5}
+              maxLines={30}
+              onChange={this.textChange.bind(this)}
+              height="200px"
+              width="100%"
+              editorProps={{ $blockScrolling: true }}
+              enableBasicAutocompletion
+              value={this.props.queryEditor.sql}
+            />
+            {editorBottomBar}
+            <div className="padded">
+              <SouthPane latestQuery={this.props.latestQuery} sqlEditor={this} />
+            </div>
+          </div>
+        </div>
+      </div>
+    );
+  }
+}
+
+SqlEditor.propTypes = {
+  queryEditor: React.PropTypes.object,
+  actions: React.PropTypes.object,
+  latestQuery: React.PropTypes.object,
+};
+
+SqlEditor.defaultProps = {
+};
+
+function mapStateToProps() {
+  return {};
+}
+
+function mapDispatchToProps(dispatch) {
+  return {
+    actions: bindActionCreators(Actions, dispatch),
+  };
+}
+
+export default connect(mapStateToProps, mapDispatchToProps)(SqlEditor);

caravel/assets/javascripts/SqlLab/components/SqlEditorTopToolbar.jsx

@@ -0,0 +1,266 @@
+const $ = window.$ = require('jquery');
+import React from 'react';
+import { Label, OverlayTrigger, Popover } from 'react-bootstrap';
+
+import { bindActionCreators } from 'redux';
+import { connect } from 'react-redux';
+import * as Actions from '../actions';
+import shortid from 'shortid';
+import Select from 'react-select';
+import Link from './Link';
+
+// CSS
+import 'react-select/dist/react-select.css';
+
+class SqlEditorTopToolbar extends React.Component {
+  constructor(props) {
+    super(props);
+    this.state = {
+      databaseLoading: false,
+      databaseOptions: [],
+      schemaLoading: false,
+      schemaOptions: [],
+      tableLoading: false,
+      tableOptions: [],
+    };
+  }
+  componentWillMount() {
+    this.fetchDatabaseOptions();
+    this.fetchSchemas();
+    this.fetchTables();
+  }
+  getSql(table) {
+    let cols = '';
+    table.columns.forEach(function (col, i) {
+      cols += col.name;
+      if (i < table.columns.length - 1) {
+        cols += ', ';
+      }
+    });
+    return `SELECT ${cols}\nFROM ${table.name}`;
+  }
+  selectStar(table) {
+    this.props.actions.queryEditorSetSql(this.props.queryEditor, this.getSql(table));
+  }
+  popTab(table) {
+    const qe = {
+      id: shortid.generate(),
+      title: table.name,
+      dbId: table.dbId,
+      schema: table.schema,
+      autorun: true,
+      sql: this.getSql(table),
+    };
+    this.props.actions.addQueryEditor(qe);
+  }
+  fetchTables(dbId, schema) {
+    const actualDbId = dbId || this.props.queryEditor.dbId;
+    if (actualDbId) {
+      const actualSchema = schema || this.props.queryEditor.schema;
+      this.setState({ tableLoading: true });
+      this.setState({ tableOptions: [] });
+      const url = `/caravel/tables/${actualDbId}/${actualSchema}`;
+      $.get(url, (data) => {
+        let tableOptions = data.tables.map((s) => ({ value: s, label: s }));
+        const views = data.views.map((s) => ({ value: s, label: '[view] ' + s }));
+        tableOptions = [...tableOptions, ...views];
+        this.setState({ tableOptions });
+        this.setState({ tableLoading: false });
+      });
+    }
+  }
+  changeSchema(schemaOpt) {
+    const schema = (schemaOpt) ? schemaOpt.value : null;
+    this.props.actions.queryEditorSetSchema(this.props.queryEditor, schema);
+    this.fetchTables(this.props.queryEditor.dbId, schema);
+  }
+  fetchSchemas(dbId) {
+    const actualDbId = dbId || this.props.queryEditor.dbId;
+    if (actualDbId) {
+      this.setState({ schemaLoading: true });
+      const url = `/databasetablesasync/api/read?_flt_0_id=${actualDbId}`;
+      $.get(url, (data) => {
+        const schemas = data.result[0].all_schema_names;
+        const schemaOptions = schemas.map((s) => ({ value: s, label: s }));
+        this.setState({ schemaOptions });
+        this.setState({ schemaLoading: false });
+      });
+    }
+  }
+  changeDb(db) {
+    const val = (db) ? db.value : null;
+    this.setState({ schemaOptions: [] });
+    this.props.actions.queryEditorSetDb(this.props.queryEditor, val);
+    if (!(db)) {
+      this.setState({ tableOptions: [] });
+      return;
+    }
+    this.fetchTables(val, this.props.queryEditor.schema);
+    this.fetchSchemas(val);
+  }
+  fetchDatabaseOptions() {
+    this.setState({ databaseLoading: true });
+    const url = '/databaseasync/api/read';
+    $.get(url, (data) => {
+      const options = data.result.map((db) => ({ value: db.id, label: db.database_name }));
+      this.setState({ databaseOptions: options });
+      this.setState({ databaseLoading: false });
+    });
+  }
+  closePopover(ref) {
+    this.refs[ref].hide();
+  }
+  changeTable(tableOpt) {
+    const tableName = tableOpt.value;
+    const qe = this.props.queryEditor;
+    const url = `/caravel/table/${qe.dbId}/${tableName}/${qe.schema}/`;
+    $.get(url, (data) => {
+      this.props.actions.addTable({
+        id: shortid.generate(),
+        dbId: this.props.queryEditor.dbId,
+        queryEditorId: this.props.queryEditor.id,
+        name: data.name,
+        schema: qe.schema,
+        columns: data.columns,
+        expanded: true,
+        showPopup: false,
+      });
+    })
+    .fail((err) => {
+      this.props.actions.addAlert({
+        msg: 'Error occurred while fetching metadata',
+        bsStyle: 'danger',
+      });
+    });
+  }
+  render() {
+    const tables = this.props.tables.filter((t) => (t.queryEditorId === this.props.queryEditor.id));
+    const tablesEls = tables.map((table) => {
+      let cols = [];
+      if (table.columns) {
+        cols = table.columns.map((col) => (
+          <div className="clearfix">
+            <div className="pull-left m-r-10">{col.name}</div>
+            <div className="pull-right text-muted"> {col.type}</div>
+          </div>
+        ));
+      }
+      const popoverId = 'tblPopover_' + table.name;
+      const popoverTop = (
+        <div className="clearfix">
+          <div className="pull-left">
+            <Link
+              className="fa fa-pencil"
+              onClick={this.selectStar.bind(this, table)}
+              tooltip="Overwrite text in editor with a query on this table"
+              placement="left"
+              href="#"
+            />
+            <Link
+              className="fa fa-plus-circle"
+              onClick={this.popTab.bind(this, table)}
+              tooltip="Run query in a new tab"
+              placement="left"
+              href="#"
+            />
+          </div>
+          <div className="pull-right">
+            <Link
+              className="fa fa-close"
+              onClick={this.closePopover.bind(this, popoverId)}
+              href="#"
+            />
+          </div>
+        </div>
+      );
+      const popover = (
+        <Popover
+          id={popoverId}
+          className="tablePopover"
+          title={popoverTop}
+        >
+          {cols}
+        </Popover>
+      );
+      return (
+        <Label className="m-r-5 table-label" style={{ fontSize: '100%' }}>
+          <OverlayTrigger trigger="click" placement="bottom" overlay={popover} ref={popoverId}>
+            <span className="m-r-5" style={{ cursor: 'pointer' }}>
+              {table.name}
+            </span>
+          </OverlayTrigger>
+          <i
+            className="fa fa-close"
+            style={{ cursor: 'pointer' }}
+            onClick={this.props.actions.removeTable.bind(this, table)}
+          />
+        </Label>
+      );
+    });
+    return (
+      <div className="clearfix sql-toolbar padded">
+        <div className="pull-left m-r-5">
+          <Select
+            name="select-db"
+            placeholder="[Database]"
+            options={this.state.databaseOptions}
+            value={this.props.queryEditor.dbId}
+            isLoading={this.state.databaseLoading}
+            autosize={false}
+            onChange={this.changeDb.bind(this)}
+          />
+        </div>
+        <div className="pull-left m-r-5">
+          <Select
+            name="select-schema"
+            placeholder="[Schema]"
+            options={this.state.schemaOptions}
+            value={this.props.queryEditor.schema}
+            isLoading={this.state.schemaLoading}
+            autosize={false}
+            onChange={this.changeSchema.bind(this)}
+          />
+        </div>
+        <div className="pull-left m-r-5">
+          <Select
+            name="select-table"
+            ref="selectTable"
+            isLoading={this.state.tableLoading}
+            placeholder="Add a table"
+            autosize={false}
+            value={this.state.tableName}
+            onChange={this.changeTable.bind(this)}
+            options={this.state.tableOptions}
+          />
+        </div>
+        <div className="pull-left m-r-5">
+          {tablesEls}
+        </div>
+      </div>
+    );
+  }
+}
+
+SqlEditorTopToolbar.propTypes = {
+  queryEditor: React.PropTypes.object,
+  tables: React.PropTypes.array,
+  actions: React.PropTypes.object,
+};
+
+SqlEditorTopToolbar.defaultProps = {
+  tables: [],
+};
+
+function mapStateToProps(state) {
+  return {
+    tables: state.tables,
+  };
+}
+
+function mapDispatchToProps(dispatch) {
+  return {
+    actions: bindActionCreators(Actions, dispatch),
+  };
+}
+
+export default connect(mapStateToProps, mapDispatchToProps)(SqlEditorTopToolbar);

caravel/assets/javascripts/SqlLab/components/TabbedSqlEditors.jsx

@@ -0,0 +1,125 @@
+import React from 'react';
+import { DropdownButton, MenuItem, Panel, Tab, Tabs } from 'react-bootstrap';
+import { connect } from 'react-redux';
+import { bindActionCreators } from 'redux';
+import * as Actions from '../actions';
+import SqlEditor from './SqlEditor';
+import shortid from 'shortid';
+
+let queryCount = 1;
+
+class QueryEditors extends React.Component {
+  renameTab(qe) {
+    const newTitle = prompt('Enter a new title for the tab');
+    if (newTitle) {
+      this.props.actions.queryEditorSetTitle(qe, newTitle);
+    }
+  }
+  activeQueryEditor() {
+    const qeid = this.props.tabHistory[this.props.tabHistory.length - 1];
+    for (let i = 0; i < this.props.queryEditors.length; i++) {
+      const qe = this.props.queryEditors[i]
+      if (qe.id === qeid) {
+        return qe;
+      }
+    }
+  }
+  newQueryEditor() {
+    queryCount++;
+    const activeQueryEditor = this.activeQueryEditor();
+    console.log(activeQueryEditor);
+    const qe = {
+      id: shortid.generate(),
+      title: `Query ${queryCount}`,
+      dbId: (activeQueryEditor) ? activeQueryEditor.dbId : null,
+      schema: (activeQueryEditor) ? activeQueryEditor.schema : null,
+      autorun: false,
+      sql: 'SELECT ...',
+    };
+    this.props.actions.addQueryEditor(qe);
+  }
+  handleSelect(key) {
+    if (key === 'add_tab') {
+      this.newQueryEditor();
+    } else {
+      this.props.actions.setActiveQueryEditor({ id: key });
+    }
+  }
+  render() {
+    const editors = this.props.queryEditors.map((qe, i) => {
+      let latestQuery;
+      this.props.queries.forEach((q) => {
+        if (q.id === qe.latestQueryId) {
+          latestQuery = q;
+        }
+      });
+      const state = (latestQuery) ? latestQuery.state : '';
+      const tabTitle = (
+        <div>
+          <div className={'circle ' + state} /> {qe.title} {' '}
+          <DropdownButton
+            bsSize="small"
+            id={'ddbtn-tab-' + i}
+            className="no-shadow tab-caret"
+            id="bg-vertical-dropdown-1"
+          >
+            <MenuItem eventKey="1" onClick={this.props.actions.removeQueryEditor.bind(this, qe)}>
+              <i className="fa fa-close" /> close tab
+            </MenuItem>
+            <MenuItem eventKey="2" onClick={this.renameTab.bind(this, qe)}>
+              <i className="fa fa-i-cursor" /> rename tab
+            </MenuItem>
+          </DropdownButton>
+        </div>
+      );
+      return (
+        <Tab
+          key={qe.id}
+          title={tabTitle}
+          eventKey={qe.id}
+        >
+          <Panel className="nopadding">
+            <SqlEditor
+              queryEditor={qe}
+              latestQuery={latestQuery}
+            />
+          </Panel>
+        </Tab>);
+    });
+    return (
+      <Tabs
+        bsStyle="tabs"
+        activeKey={this.props.tabHistory[this.props.tabHistory.length - 1]}
+        onSelect={this.handleSelect.bind(this)}
+      >
+        {editors}
+        <Tab title={<div><i className="fa fa-plus-circle" />&nbsp;</div>} eventKey="add_tab" />
+      </Tabs>
+    );
+  }
+}
+QueryEditors.propTypes = {
+  actions: React.PropTypes.object,
+  queries: React.PropTypes.array,
+  queryEditors: React.PropTypes.array,
+  tabHistory: React.PropTypes.array,
+};
+QueryEditors.defaultProps = {
+  tabHistory: [],
+  queryEditors: [],
+};
+
+function mapStateToProps(state) {
+  return {
+    queryEditors: state.queryEditors,
+    queries: state.queries,
+    tabHistory: state.tabHistory,
+  };
+}
+function mapDispatchToProps(dispatch) {
+  return {
+    actions: bindActionCreators(Actions, dispatch),
+  };
+}
+
+export default connect(mapStateToProps, mapDispatchToProps)(QueryEditors);

caravel/assets/javascripts/SqlLab/components/TableMetadata.jsx

@@ -0,0 +1,23 @@
+import React from 'react';
+import { BootstrapTable, TableHeaderColumn } from 'react-bootstrap-table';
+
+const TableMetadata = function (props) {
+  return (
+    <BootstrapTable
+      condensed
+      data={props.table.columns}
+    >
+      <TableHeaderColumn dataField="id" isKey hidden>
+        id
+      </TableHeaderColumn>
+      <TableHeaderColumn dataField="name">Name</TableHeaderColumn>
+      <TableHeaderColumn dataField="type">Type</TableHeaderColumn>
+    </BootstrapTable>
+  );
+};
+
+TableMetadata.propTypes = {
+  table: React.PropTypes.object,
+};
+
+export default TableMetadata;

caravel/assets/javascripts/SqlLab/components/TableWorkspaceElement.jsx

@@ -0,0 +1,102 @@
+import React from 'react';
+import { ButtonGroup } from 'react-bootstrap';
+import Link from './Link';
+import { connect } from 'react-redux';
+import { bindActionCreators } from 'redux';
+import * as Actions from '../actions';
+import shortid from 'shortid';
+
+// CSS
+import 'react-select/dist/react-select.css';
+
+class TableWorkspaceElement extends React.Component {
+  selectStar() {
+    let cols = '';
+    this.props.table.columns.forEach((col, i) => {
+      cols += col.name;
+      if (i < this.props.table.columns.length - 1) {
+        cols += ', ';
+      }
+    });
+    const sql = `SELECT ${cols}\nFROM ${this.props.table.name}`;
+    const qe = {
+      id: shortid.generate(),
+      title: this.props.table.name,
+      dbId: this.props.table.dbId,
+      autorun: true,
+      sql,
+    };
+    this.props.actions.addQueryEditor(qe);
+  }
+  render() {
+    let metadata = null;
+    let buttonToggle;
+    if (!this.props.table.expanded) {
+      buttonToggle = (
+        <Link
+          href="#"
+          onClick={this.props.actions.expandTable.bind(this, this.props.table)}
+          placement="right"
+          tooltip="Collapse the table's structure information"
+        >
+          <i className="fa fa-minus" /> {this.props.table.name}
+        </Link>
+      );
+      metadata = this.props.table.columns.map((col) =>
+        <div className="clearfix">
+          <span className="pull-left">{col.name}</span>
+          <span className="pull-right">{col.type}</span>
+        </div>
+      );
+      metadata = (
+        <div style={{ 'margin-bottom': '5px' }}>{metadata}</div>
+      );
+    } else {
+      buttonToggle = (
+        <Link
+          href="#"
+          onClick={this.props.actions.collapseTable.bind(this, this.props.table)}
+          placement="right"
+          tooltip="Expand the table's structure information"
+        >
+          <i className="fa fa-plus" /> {this.props.table.name}
+        </Link>
+      );
+    }
+    return (
+      <div className="ws-el">
+        {buttonToggle}
+        <ButtonGroup className="ws-el-controls pull-right">
+          <Link
+            className="fa fa-play"
+            onClick={this.selectStar.bind(this)}
+            tooltip="Run query in a new tab"
+            href="#"
+          />
+          <Link
+            className="fa fa-trash"
+            onClick={this.props.actions.removeTable.bind(this, this.props.table)}
+            tooltip="Remove from workspace"
+            href="#"
+          />
+        </ButtonGroup>
+        {metadata}
+      </div>
+    );
+  }
+}
+TableWorkspaceElement.propTypes = {
+  table: React.PropTypes.object,
+  actions: React.PropTypes.object,
+};
+TableWorkspaceElement.defaultProps = {
+  table: null,
+};
+
+function mapDispatchToProps(dispatch) {
+  return {
+    actions: bindActionCreators(Actions, dispatch),
+  };
+}
+export default connect(null, mapDispatchToProps)(TableWorkspaceElement);
+

caravel/assets/javascripts/SqlLab/components/Timer.jsx

@@ -0,0 +1,62 @@
+import React from 'react';
+import moment from 'moment';
+
+
+class Timer extends React.Component {
+  constructor(props) {
+    super(props);
+    this.state = {
+      clockStr: '',
+    };
+  }
+  componentWillMount() {
+    this.startTimer();
+  }
+  componentWillUnmount() {
+    this.stopTimer();
+  }
+  startTimer() {
+    if (!(this.timer)) {
+      this.timer = setInterval(this.stopwatch.bind(this), 30);
+    }
+  }
+  stopTimer() {
+    clearInterval(this.timer);
+    this.timer = null;
+  }
+  stopwatch() {
+    if (this.props && this.props.query) {
+      let fromDttm = this.props.query.endDttm || new Date();
+      fromDttm = moment(fromDttm);
+      let duration = fromDttm - moment(this.props.query.startDttm).valueOf();
+      duration = moment.utc(duration);
+      const clockStr = duration.format('HH:mm:ss.SS');
+      this.setState({ clockStr });
+      if (this.props.query.state !== 'running') {
+        this.stopTimer();
+      }
+    }
+  }
+  render() {
+    if (this.props.query && this.props.query.state === 'running') {
+      this.startTimer();
+    }
+    let timerSpan = null;
+    if (this.props && this.props.query) {
+      timerSpan = (
+        <span className={'label label-warning inlineBlock m-r-5 ' + this.props.query.state}>
+          {this.state.clockStr}
+        </span>
+      );
+    }
+    return timerSpan;
+  }
+}
+Timer.propTypes = {
+  query: React.PropTypes.object,
+};
+Timer.defaultProps = {
+  query: null,
+};
+
+export default Timer;

caravel/assets/javascripts/SqlLab/components/VisualizeModal.jsx

@@ -0,0 +1,173 @@
+import React from 'react';
+import { Alert, Button, Grid, Row, Col, Modal } from 'react-bootstrap';
+
+import { connect } from 'react-redux';
+import { bindActionCreators } from 'redux';
+import * as Actions from '../actions';
+
+import Select from 'react-select';
+import { Table } from 'reactable';
+import shortid from 'shortid';
+
+const $ = require('jquery');
+
+class VisualizeModal extends React.Component {
+  constructor(props) {
+    super(props);
+    this.state = {
+      chartType: 'line',
+      datasourceName: shortid.generate(),
+      columns: {},
+    };
+  }
+  changeChartType(option) {
+    this.setState({ chartType: (option) ? option.value : null });
+  }
+  mergedColumns() {
+    const columns = Object.assign({}, this.state.columns);
+    if (this.props.query && this.props.query.results.columns) {
+      this.props.query.results.columns.forEach((col) => {
+        if (columns[col] === undefined) {
+          columns[col] = {};
+        }
+      });
+    }
+    return columns;
+  }
+  visualize() {
+    const vizOptions = {
+      chartType: this.state.chartType,
+      datasourceName: this.state.datasourceName,
+      columns: this.state.columns,
+      sql: this.props.query.sql,
+    };
+    window.open('/caravel/sqllab_viz/?' + $.param(vizOptions));
+  }
+  changeDatasourceName(event) {
+    this.setState({ datasourceName: event.target.value });
+  }
+  changeCheckbox(attr, col, event) {
+    console.log([attr, col, event]);
+    let columns = this.mergedColumns();
+    const column = Object.assign({}, columns[col], { [attr]: event.target.checked });
+    columns = Object.assign({}, columns, { [col]: column });
+    this.setState({ columns });
+  }
+  changeAggFunction(col, option) {
+    let columns = this.mergedColumns();
+    const val = (option) ? option.value : null;
+    const column = Object.assign({}, columns[col], { agg: option.value });
+    columns = Object.assign({}, columns, { [col]: column });
+    this.setState({ columns });
+  }
+  render() {
+    console.log(this.state);
+    if (!(this.props.query)) {
+      return <div />;
+    }
+    const tableData = this.props.query.results.columns.map((col) => ({
+      column: col,
+      is_dimension: (
+        <input
+          type="checkbox"
+          onChange={this.changeCheckbox.bind(this, 'is_dim', col)}
+          checked={(this.state.columns[col]) ? this.state.columns[col].is_dim : false}
+          className="form-control"
+        />
+      ),
+      is_date: (
+        <input
+          type="checkbox"
+          className="form-control"
+          onChange={this.changeCheckbox.bind(this, 'is_date', col)}
+          checked={(this.state.columns[col]) ? this.state.columns[col].is_date : false}
+        />
+      ),
+      agg_func: (
+        <Select
+          options={[
+            { value: 'sum', label: 'SUM(x)' },
+            { value: 'min', label: 'MIN(x)' },
+            { value: 'max', label: 'MAX(x)' },
+            { value: 'avg', label: 'AVG(x)' },
+            { value: 'count_distinct', label: 'COUNT(DISTINCT x)' },
+          ]}
+          onChange={this.changeAggFunction.bind(this, col)}
+          value={(this.state.columns[col]) ? this.state.columns[col].agg : null}
+        />
+      ),
+    }))
+    const modal = (
+      <div className="VisualizeModal">
+        <Modal show={this.props.show} onHide={this.props.onHide}>
+          <Modal.Header closeButton>
+            <Modal.Title>
+              Visualize <span className="alert alert-danger">under construction</span>
+            </Modal.Title>
+          </Modal.Header>
+          <Modal.Body>
+            <div className="row">
+              <Col md={6}>
+                Chart Type
+                <Select
+                  name="select-chart-type"
+                  placeholder="[Chart Type]"
+                  options={[
+                    { value: 'line', label: 'Time Series - Line Chart' },
+                    { value: 'bar', label: 'Time Series - Bar Chart' },
+                    { value: 'bar_dist', label: 'Distribution - Bar Chart' },
+                    { value: 'pie', label: 'Pie Chart' },
+                  ]}
+                  value={this.state.chartType}
+                  autosize={false}
+                  onChange={this.changeChartType.bind(this)}
+                />
+              </Col>
+              <Col md={6}>
+                Datasource Name
+                <input
+                  type="text"
+                  className="form-control"
+                  placeholder="datasource name"
+                  onChange={this.changeDatasourceName.bind(this)}
+                  value={this.state.datasourceName}
+                />
+              </Col>
+            </div>
+            <hr/>
+            <Table
+              className="table table-condensed"
+              columns={['column', 'is_dimension', 'is_date', 'agg_func']}
+              data={tableData}
+            />
+            <Button
+              onClick={this.visualize.bind(this)}
+              bsStyle="primary"
+            >
+              Visualize
+            </Button>
+          </Modal.Body>
+        </Modal>
+      </div>
+    );
+    return modal;
+  }
+}
+VisualizeModal.propTypes = {
+  query: React.PropTypes.object,
+  show: React.PropTypes.boolean,
+  onHide: React.PropTypes.function,
+};
+VisualizeModal.defaultProps = {
+  show: false,
+};
+
+function mapStateToProps() {
+  return {};
+}
+function mapDispatchToProps(dispatch) {
+  return {
+    actions: bindActionCreators(Actions, dispatch),
+  };
+}
+export default connect(mapStateToProps, mapDispatchToProps)(VisualizeModal);

caravel/assets/javascripts/SqlLab/index.jsx

@@ -0,0 +1,70 @@
+var $ = window.$ = require('jquery');
+var jQuery = window.jQuery = $;
+require('bootstrap');
+
+import React from 'react';
+import { render } from 'react-dom';
+import { connect } from 'react-redux';
+import { bindActionCreators } from 'redux';
+import * as Actions from './actions';
+
+import SplitPane from 'react-split-pane';
+import { Label, Tab, Tabs } from 'react-bootstrap';
+
+import LeftPane from './components/LeftPane';
+import TabbedSqlEditors from './components/TabbedSqlEditors';
+import Alerts from './components/Alerts';
+
+import { compose, createStore } from 'redux';
+import { Provider } from 'react-redux';
+
+import { initialState, sqlLabReducer } from './reducers';
+import persistState from 'redux-localstorage';
+
+require('./main.css');
+
+let store = createStore(sqlLabReducer, initialState, compose(persistState(), window.devToolsExtension && window.devToolsExtension()));
+
+// jquery hack to highlight the navbar menu
+$('a[href="/caravel/sqllab"]').parent().addClass('active');
+
+class App extends React.Component {
+  render() {
+    return (
+      <div className="App SqlLab">
+        <div className="container-fluid">
+          <Alerts alerts={this.props.alerts} />
+          <SplitPane split="vertical" minSize={200} defaultSize={300}>
+            <div className="pane-cell pane-west m-t-5">
+              <LeftPane />
+            </div>
+            <div className="pane-cell">
+              <TabbedSqlEditors />
+            </div>
+          </SplitPane>
+        </div>
+      </div>
+    );
+  }
+}
+
+function mapStateToProps(state) {
+  return {
+    alerts: state.alerts,
+  };
+}
+function mapDispatchToProps(dispatch) {
+  return {
+    actions: bindActionCreators(Actions, dispatch),
+  };
+}
+
+App = connect(mapStateToProps, mapDispatchToProps)(App);
+
+render(
+  <Provider store={store}>
+    <App />
+  </Provider>,
+  document.getElementById('app')
+);
+

caravel/assets/javascripts/SqlLab/main.css

@@ -0,0 +1,256 @@
+#app {
+    position: absolute;
+    top: 65;
+    left: 0;
+    right: 0;
+    bottom: 0;
+}
+.inlineBlock {
+    display: inline-block;
+}
+.valignTop {
+    vertical-align: top;
+}
+.inline {
+    display: inline;
+}
+.nopadding {
+    padding: 0px;
+}
+.panel.nopadding .panel-body {
+    padding: 0px;
+}
+.panel {
+    width: 100%;
+    overflow: auto;
+    margin-bottom: 10px;
+}
+.SqlEditor .panel-heading {
+    padding: 5px;
+}
+.window.panel-heading {
+    padding: 1px 5px;
+}
+.loading {
+    width: 50px;
+    margin-top: 15px;
+}
+.pane-cell {
+    padding: 10px;
+    overflow: auto;
+    width: 100%;
+    height: 100%;
+}
+.SqlEditor .header {
+    padding-top: 5px;
+    padding-bottom: 5px;
+}
+.Workspace .btn-sm {
+    box-shadow: 1px 1px 2px rgba(0, 0, 0, 0.1);
+    margin-top: 2px;
+    padding: 4px;
+}
+.Workspace hr {
+    margin-top: 10px;
+    margin-bottom: 10px;
+}
+div.Workspace {
+    height: 100%;
+    margin: 0px;
+}
+.SqlEditor .clock {
+    background-color: orange;
+    padding: 5px;
+}
+.sql-toolbar {
+    border-bottom: 1px solid #DDD;
+    border-top: 1px solid #DDD;
+}
+.padded {
+    padding: 10px;
+}
+.nav-pills {
+    padding-bottom: 5px;
+}
+.p-t-10 {
+    padding-top: 10px;
+}
+.p-t-5 {
+    padding-top: 5px;
+}
+.m-r-5 {
+    margin-right: 5px;
+}
+.m-l-1 {
+    margin-left: 1px;
+}
+.m-r-10 {
+    margin-right: 10px;
+}
+.m-b-10 {
+    margin-bottom: 10px;
+}
+.m-t-5 {
+    margin-top: 5px;
+}
+.m-t-10 {
+    margin-top: 10px;
+}
+.p-t-10 {
+    padding-top: 10px;
+}
+.sqllab-toolbar {
+    padding-top: 5px;
+    border-bottom: 1px solid #DDD;
+}
+.no-shadow {
+    box-shadow: none;
+    background-color: transparent;
+}
+.pane-west {
+    height: 100%;
+    overflow: auto;
+}
+.ws-el .ws-el-controls { display: none; }
+.ws-el:hover .ws-el-controls { display: block; }
+.ws-el {
+    border-radius: 4px;
+    padding: 1px 6px;
+    border: 1px solid transparent;
+}
+.circle {
+    border-radius: 50%;
+    width: 10px;
+    height: 10px;
+    display: inline-block;
+    border: 1px solid #444;
+}
+.Pane2 {
+    width: 0;
+}
+.running {
+    background-color: lime;
+    color: black;
+}
+.success {
+    background-color: green;
+}
+.failed {
+    background-color: red;
+}
+.ws-el:hover { border: 1px solid #DDD; }
+
+.handle {
+    cursor: move;
+}
+.window {
+    z-index: 1000;
+    position: absolute;
+    width: 300px;
+    opacity: 0.85;
+    border: 1px solid #AAA;
+    max-height: 600px;
+    box-shadow: rgba(0, 0, 0, 0.8) 5px 5px 25px
+}
+
+.list-group-item {
+    padding: 5px 10px;
+}
+
+table {
+    font-size: 12px;
+    margin: 0px;
+}
+
+.SqlLab pre {
+    padding: 0px !important;
+    margin: 0px;
+    border: none;
+    font-size: 11px;
+    line-height: 125%;
+    background-color: transparent !important;
+}
+
+.Resizer {
+    background: #000;
+    opacity: .2;
+    z-index: 1;
+    -moz-box-sizing: border-box;
+    -webkit-box-sizing: border-box;
+    box-sizing: border-box;
+    -moz-background-clip: padding;
+    -webkit-background-clip: padding;
+    background-clip: padding-box;
+}
+
+.Resizer:hover {
+    -webkit-transition: all 2s ease;
+    transition: all 2s ease;
+}
+
+.Resizer.horizontal {
+    height: 10px;
+    margin: -5px 0;
+    border-top: 5px solid rgba(255, 255, 255, 0);
+    border-bottom: 5px solid rgba(255, 255, 255, 0);
+    cursor: row-resize;
+    width: 100%;
+    padding: 1px;
+}
+
+.Resizer.horizontal:hover {
+    border-top: 5px solid rgba(0, 0, 0, 0.5);
+    border-bottom: 5px solid rgba(0, 0, 0, 0.5);
+}
+
+.Resizer.vertical {
+    width: 9px;
+    margin: 0 -5px;
+    border-left: 5px solid rgba(255, 255, 255, 0);
+    border-right: 5px solid rgba(255, 255, 255, 0);
+    cursor: col-resize;
+}
+
+.Resizer.vertical:hover {
+    border-left: 5px solid rgba(0, 0, 0, 0.5);
+    border-right: 5px solid rgba(0, 0, 0, 0.5);
+}
+Resizer.disabled {
+    cursor: not-allowed;
+}
+Resizer.disabled:hover {
+    border-color: transparent;
+}
+
+a.Link {
+    padding: 7px 3px;
+}
+
+table .label {
+    margin-top: 5px;
+}
+.popover{
+    max-width:400px;
+}
+.Select-menu-outer {
+    z-index: 1000;
+}
+.table-label {
+    margin-top: 5px;
+    margin-right: 10px;
+    float: left;
+}
+div.tablePopover {
+    opacity: 0.7 !important;
+}
+div.tablePopover:hover {
+    opacity: 1 !important;
+}
+.ResultSetControls {
+    padding-bottom: 3px;
+    padding-top: 3px;
+}
+button.tab-caret {
+    padding: 5px !important;
+    border-color: transparent;
+}

caravel/assets/javascripts/SqlLab/reducers.js

@@ -0,0 +1,155 @@
+import moment from 'moment';
+import shortid from 'shortid';
+import * as actions from './actions';
+
+const defaultQueryEditor = {
+  id: shortid.generate(),
+  title: 'Query 1',
+  sql: 'SELECT *\nFROM\nWHERE',
+  latestQueryId: null,
+  autorun: false,
+  dbId: null,
+};
+
+export const initialState = {
+  alerts: [],
+  queries: [],
+  queryEditors: [defaultQueryEditor],
+  tabHistory: [defaultQueryEditor.id],
+  tables: [],
+  workspaceQueries: [],
+};
+
+
+function alterInArr(state, arrKey, obj, alterations) {
+  // Finds an item in an array in the state and replaces it with a
+  // new object with an altered property
+  const idKey = 'id';
+  const newArr = [];
+  state[arrKey].forEach((arrItem) => {
+    if (obj[idKey] === arrItem[idKey]) {
+      newArr.push(Object.assign({}, arrItem, alterations));
+    } else {
+      newArr.push(arrItem);
+    }
+  });
+  return Object.assign({}, state, { [arrKey]: newArr });
+}
+
+function removeFromArr(state, arrKey, obj, idKey = 'id') {
+  const newArr = [];
+  state[arrKey].forEach((arrItem) => {
+    if (!(obj[idKey] === arrItem[idKey])) {
+      newArr.push(arrItem);
+    }
+  });
+  return Object.assign({}, state, { [arrKey]: newArr });
+}
+
+function addToArr(state, arrKey, obj) {
+  if (!(obj.id)) {
+    obj.id = shortid.generate();
+  }
+  const newState = {};
+  newState[arrKey] = [...state[arrKey], Object.assign({}, obj)];
+  return Object.assign({}, state, newState);
+}
+
+export const sqlLabReducer = function (state, action) {
+  const actionHandlers = {
+    [actions.ADD_QUERY_EDITOR]() {
+      const tabHistory = state.tabHistory.slice();
+      tabHistory.push(action.queryEditor.id);
+      const newState = Object.assign({}, state, { tabHistory });
+      return addToArr(newState, 'queryEditors', action.queryEditor);
+    },
+    [actions.REMOVE_QUERY_EDITOR]() {
+      let newState = removeFromArr(state, 'queryEditors', action.queryEditor);
+      // List of remaining queryEditor ids
+      const qeIds = newState.queryEditors.map((qe) => qe.id);
+      let th = state.tabHistory.slice();
+      th = th.filter((id) => qeIds.includes(id));
+      newState = Object.assign({}, newState, { tabHistory: th });
+      return newState;
+    },
+    [actions.REMOVE_QUERY]() {
+      return removeFromArr(state, 'queries', action.query);
+    },
+    [actions.RESET_STATE]() {
+      return Object.assign({}, initialState);
+    },
+    [actions.ADD_TABLE]() {
+      return addToArr(state, 'tables', action.table);
+    },
+    [actions.EXPAND_TABLE]() {
+      return alterInArr(state, 'tables', action.table, { expanded: true });
+    },
+    [actions.COLLAPSE_TABLE]() {
+      return alterInArr(state, 'tables', action.table, { expanded: false });
+    },
+    [actions.REMOVE_TABLE]() {
+      return removeFromArr(state, 'tables', action.table);
+    },
+    [actions.START_QUERY]() {
+      const newState = addToArr(state, 'queries', action.query);
+      const sqlEditor = { id: action.query.sqlEditorId };
+      return alterInArr(newState, 'queryEditors', sqlEditor, { latestQueryId: action.query.id });
+    },
+    [actions.STOP_QUERY]() {
+      return alterInArr(state, 'queries', action.query, { state: 'stopped' });
+    },
+    [actions.QUERY_SUCCESS]() {
+      const alts = {
+        state: 'success',
+        results: action.results,
+        rows: action.results.data.length,
+        endDttm: moment(),
+      };
+      return alterInArr(state, 'queries', action.query, alts);
+    },
+    [actions.QUERY_FAILED]() {
+      const alts = { state: 'failed', msg: action.msg, endDttm: moment() };
+      return alterInArr(state, 'queries', action.query, alts);
+    },
+    [actions.SET_ACTIVE_QUERY_EDITOR]() {
+      const qeIds = state.queryEditors.map((qe) => qe.id);
+      if (qeIds.includes(action.queryEditor.id)) {
+        const tabHistory = state.tabHistory.slice();
+        tabHistory.push(action.queryEditor.id);
+        return Object.assign({}, state, { tabHistory });
+      }
+      return state;
+    },
+    [actions.QUERY_EDITOR_SETDB]() {
+      return alterInArr(state, 'queryEditors', action.queryEditor, { dbId: action.dbId });
+    },
+    [actions.QUERY_EDITOR_SET_SCHEMA]() {
+      return alterInArr(state, 'queryEditors', action.queryEditor, { schema: action.schema });
+    },
+    [actions.QUERY_EDITOR_SET_TITLE]() {
+      return alterInArr(state, 'queryEditors', action.queryEditor, { title: action.title });
+    },
+    [actions.QUERY_EDITOR_SET_SQL]() {
+      return alterInArr(state, 'queryEditors', action.queryEditor, { sql: action.sql });
+    },
+    [actions.QUERY_EDITOR_SET_AUTORUN]() {
+      return alterInArr(state, 'queryEditors', action.queryEditor, { autorun: action.autorun });
+    },
+    [actions.ADD_WORKSPACE_QUERY]() {
+      return addToArr(state, 'workspaceQueries', action.query);
+    },
+    [actions.REMOVE_WORKSPACE_QUERY]() {
+      return removeFromArr(state, 'workspaceQueries', action.query);
+    },
+    [actions.ADD_ALERT]() {
+      return addToArr(state, 'alerts', action.alert);
+    },
+    [actions.REMOVE_ALERT]() {
+      return removeFromArr(state, 'alerts', action.alert);
+    },
+  };
+  if (action.type in actionHandlers) {
+    return actionHandlers[action.type]();
+  }
+  return state;
+};

caravel/assets/javascripts/dashboard/Dashboard.jsx

@@ -50,6 +50,7 @@ function dashboardContainer(dashboardData) {
       const sliceObjects = [];
       const dash = this;
       dashboard.slices.forEach((data) => {
+        console.log(data);
         if (data.error) {
           const html = '<div class="alert alert-danger">' + data.error + '</div>';
           $('#slice_' + data.slice_id).find('.token').html(html);

caravel/assets/package.json

@@ -8,7 +8,7 @@
   },
   "scripts": {
     "test": "npm run lint && mocha --compilers js:babel-core/register --required spec/helpers/browser.js spec/**/*_spec.*",
-    "dev": "NODE_ENV=dev webpack -d --watch --colors",
+    "dev": "NODE_ENV=dev webpack -d --watch --colors --progress",
     "prod": "NODE_ENV=production webpack -p --colors --progress",
     "lint": "npm run --silent lint:js",
     "lint:js": "eslint --ignore-path=.eslintignore --ext .js ."
@@ -55,19 +55,30 @@
     "jquery": "^2.2.1",
     "jquery-ui": "1.10.5",
     "mapbox-gl": "^0.20.0",
+    "moment": "^2.14.1",
+    "moments": "0.0.2",
     "mustache": "^2.2.1",
     "nvd3": "1.8.4",
     "react": "^15.2.1",
-    "react-bootstrap": "^0.28.3",
-    "react-bootstrap-datetimepicker": "0.0.22",
-    "react-bootstrap-table": "^2.3.7",
+    "react-ace": "^3.4.1",
+    "react-bootstrap": "^0.30.1",
+    "react-bootstrap-table": "^2.3.8",
     "react-dom": "^0.14.8",
-    "react-grid-layout": "^0.12.3",
+    "react-draggable": "^2.1.2",
+    "react-grid-layout": "^0.12.4",
     "react-map-gl": "^1.0.0-beta-10",
+    "react-redux": "^4.4.5",
     "react-resizable": "^1.3.3",
     "react-select": "^1.0.0-beta14",
+    "react-split-pane": "^0.1.42",
+    "react-syntax-highlighter": "^2.1.1",
+    "reactable": "^0.13.2",
+    "redux": "^3.5.2",
+    "redux-localstorage": "^0.4.1",
     "select2": "3.5",
     "select2-bootstrap-css": "^1.4.6",
+    "shortid": "^2.2.6",
+    "style-loader": "^0.13.0",
     "supercluster": "https://github.com/georgeke/supercluster/tarball/ac3492737e7ce98e07af679623aad452373bbc40",
     "topojson": "^1.6.22",
     "viewport-mercator-project": "^2.1.0"

caravel/assets/stylesheets/less/variables.less

@@ -142,9 +142,9 @@
 @border-radius-small:       2px;
 
 //** Global color for active items (e.g., navs or dropdowns).
-@component-active-color:    #fff;
+@component-active-color:    black;
 //** Global background color for active items (e.g., navs or dropdowns).
-@component-active-bg:       @brand-primary;
+@component-active-bg:       #DDD;
 
 //** Width of the `border` for generating carets that indicator dropdowns.
 @caret-width-base:          4px;

caravel/assets/webpack.config.js

@@ -16,6 +16,7 @@ const config = {
     sql: APP_DIR + '/javascripts/sql.js',
     standalone: APP_DIR + '/javascripts/standalone.js',
     common: APP_DIR + '/javascripts/common.js',
+    sqllab: APP_DIR + '/javascripts/SqlLab/index.jsx',
   },
   output: {
     path: BUILD_DIR,

caravel/bin/caravel

@@ -5,6 +5,8 @@ from __future__ import print_function
 from __future__ import unicode_literals
 
 import logging
+import celery
+from celery.bin import worker as celery_worker
 from datetime import datetime
 from subprocess import Popen
 
@@ -126,5 +128,24 @@ def refresh_druid():
     session.commit()
 
 
+@manager.command
+def worker():
+    """Starts a Caravel worker for async SQL query execution."""
+    # celery -A tasks worker --loglevel=info
+    print("Starting SQL Celery worker.")
+    if config.get('CELERY_CONFIG'):
+        print("Celery broker url: ")
+        print(config.get('CELERY_CONFIG').BROKER_URL)
+
+    application = celery.current_app._get_current_object()
+    c_worker = celery_worker.worker(app=application)
+    options = {
+        'broker': config.get('CELERY_CONFIG').BROKER_URL,
+        'loglevel': 'INFO',
+        'traceback': True,
+    }
+    c_worker.run(**options)
+
+
 if __name__ == "__main__":
     manager.run()

caravel/config.py

@@ -179,11 +179,30 @@ BACKUP_COUNT = 30
 # Set this API key to enable Mapbox visualizations
 MAPBOX_API_KEY = ""
 
+# Maximum number of rows returned in the SQL editor
+SQL_MAX_ROW = 1000
+
 # If defined, shows this text in an alert-warning box in the navbar
 # one example use case may be "STAGING" to make it clear that this is
 # not the production version of the site.
 WARNING_MSG = None
 
+# Default celery config is to use SQLA as a broker, in a production setting
+# you'll want to use a proper broker as specified here:
+# http://docs.celeryproject.org/en/latest/getting-started/brokers/index.html
+"""
+# Example:
+class CeleryConfig(object):
+  BROKER_URL = 'sqla+sqlite:///celerydb.sqlite'
+  CELERY_IMPORTS = ('caravel.tasks', )
+  CELERY_RESULT_BACKEND = 'db+sqlite:///celery_results.sqlite'
+  CELERY_ANNOTATIONS = {'tasks.add': {'rate_limit': '10/s'}}
+CELERY_CONFIG = CeleryConfig
+"""
+CELERY_CONFIG = None
+
+# The db id here results in selecting this one as a default in SQL Lab
+DEFAULT_DB_ID = None
 
 try:
     from caravel_config import *  # noqa

caravel/extract_table_names.py

@@ -0,0 +1,60 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2016 Andi Albrecht, albrecht.andi@gmail.com
+#
+# This example is part of python-sqlparse and is released under
+# the BSD License: http://www.opensource.org/licenses/bsd-license.php
+#
+# This example illustrates how to extract table names from nested
+# SELECT statements.
+#
+# See:
+# http://groups.google.com/group/sqlparse/browse_thread/thread/b0bd9a022e9d4895
+
+import sqlparse
+from sqlparse.sql import IdentifierList, Identifier
+from sqlparse.tokens import Keyword, DML
+
+
+def is_subselect(parsed):
+    if not parsed.is_group():
+        return False
+    for item in parsed.tokens:
+        if item.ttype is DML and item.value.upper() == 'SELECT':
+            return True
+    return False
+
+
+def extract_from_part(parsed):
+    from_seen = False
+    for item in parsed.tokens:
+        if from_seen:
+            if is_subselect(item):
+                for x in extract_from_part(item):
+                    yield x
+            elif item.ttype is Keyword:
+                raise StopIteration
+            else:
+                yield item
+        elif item.ttype is Keyword and item.value.upper() == 'FROM':
+            from_seen = True
+
+
+def extract_table_identifiers(token_stream):
+    for item in token_stream:
+        if isinstance(item, IdentifierList):
+            for identifier in item.get_identifiers():
+                yield identifier.get_name()
+        elif isinstance(item, Identifier):
+            yield item.get_name()
+        # It's a bug to check for Keyword here, but in the example
+        # above some tables names are identified as keywords...
+        elif item.ttype is Keyword:
+            yield item.value
+
+
+# TODO(bkyryliuk): add logic to support joins and unions.
+def extract_tables(sql):
+    stream = extract_from_part(sqlparse.parse(sql)[0])
+    return list(extract_table_identifiers(stream))

caravel/migrations/versions/ad82a75afd82_add_query_model.py

@@ -0,0 +1,49 @@
+"""Update models to support storing the queries.
+
+Revision ID: ad82a75afd82
+Revises: f162a1dea4c4
+Create Date: 2016-07-25 17:48:12.771103
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = 'ad82a75afd82'
+down_revision = 'f162a1dea4c4'
+
+from alembic import op
+import sqlalchemy as sa
+
+
+def upgrade():
+    op.create_table('query',
+        sa.Column('id', sa.Integer(), nullable=False),
+        sa.Column('database_id', sa.Integer(), nullable=False),
+        sa.Column('tmp_table_name', sa.String(length=256), nullable=True),
+        sa.Column('tab_name', sa.String(length=256),nullable=True),
+        sa.Column('user_id', sa.Integer(), nullable=True),
+        sa.Column('status', sa.String(length=16), nullable=True),
+        sa.Column('name', sa.String(length=256), nullable=True),
+        sa.Column('schema', sa.String(length=256), nullable=True),
+        sa.Column('sql', sa.Text(), nullable=True),
+        sa.Column('select_sql', sa.Text(), nullable=True),
+        sa.Column('executed_sql', sa.Text(), nullable=True),
+        sa.Column('limit', sa.Integer(), nullable=True),
+        sa.Column('limit_used', sa.Boolean(), nullable=True),
+        sa.Column('select_as_cta', sa.Boolean(), nullable=True),
+        sa.Column('select_as_cta_used', sa.Boolean(), nullable=True),
+        sa.Column('progress', sa.Integer(), nullable=True),
+        sa.Column('rows', sa.Integer(), nullable=True),
+        sa.Column('error_message', sa.Text(), nullable=True),
+        sa.Column('start_time', sa.DateTime(), nullable=True),
+        sa.Column('end_time', sa.DateTime(), nullable=True),
+        sa.ForeignKeyConstraint(['database_id'], [u'dbs.id'], ),
+        sa.ForeignKeyConstraint(['user_id'], [u'ab_user.id'], ),
+        sa.PrimaryKeyConstraint('id')
+    )
+    op.add_column('dbs', sa.Column('select_as_create_table_as', sa.Boolean(),
+                                   nullable=True))
+
+
+def downgrade():
+    op.drop_table('query')
+    op.drop_column('dbs', 'select_as_create_table_as')

caravel/models.py

@@ -16,6 +16,7 @@ import humanize
 import pandas as pd
 import requests
 import sqlalchemy as sqla
+from sqlalchemy.engine.url import make_url
 import sqlparse
 from dateutil.parser import parse
 
@@ -31,8 +32,9 @@ from pydruid.utils.postaggregator import Postaggregator
 from pydruid.utils.having import Aggregation
 from six import string_types
 from sqlalchemy import (
-    Column, Integer, String, ForeignKey, Text, Boolean, DateTime, Date,
-    Table, create_engine, MetaData, desc, asc, select, and_, func)
+    Column, Integer, String, ForeignKey, Text, Boolean, DateTime, Date, Table,
+    create_engine, MetaData, desc, asc, select, and_, func
+)
 from sqlalchemy.engine import reflection
 from sqlalchemy.ext.declarative import declared_attr
 from sqlalchemy.orm import relationship
@@ -310,6 +312,11 @@ class Dashboard(Model, AuditMixinNullable):
         else:
             return {}
 
+    @property
+    def sqla_metadata(self):
+        metadata = MetaData(bind=self.get_sqla_engine())
+        return metadata.reflect()
+
     def dashboard_link(self):
         return '<a href="{obj.url}">{obj.dashboard_title}</a>'.format(obj=self)
 
@@ -372,6 +379,7 @@ class Database(Model, AuditMixinNullable):
     sqlalchemy_uri = Column(String(1024))
     password = Column(EncryptedType(String(1024), config.get('SECRET_KEY')))
     cache_timeout = Column(Integer)
+    select_as_create_table_as = Column(Boolean, default=False)
     extra = Column(Text, default=textwrap.dedent("""\
     {
         "metadata_params": {},
@@ -382,14 +390,49 @@ class Database(Model, AuditMixinNullable):
     def __repr__(self):
         return self.database_name
 
-    def get_sqla_engine(self):
+    def get_sqla_engine(self, schema=None):
         extra = self.get_extra()
         params = extra.get('engine_params', {})
-        return create_engine(self.sqlalchemy_uri_decrypted, **params)
+        url = make_url(self.sqlalchemy_uri_decrypted)
+        backend = url.get_backend_name()
+        if backend == 'presto' and schema:
+            if '/' in url.database:
+                url.database = url.database.split('/')[0] + '/' + schema
+            else:
+                url.database += '/' + schema
+        elif schema:
+            url.database = schema
+        return create_engine(url, **params)
+
+    def get_df(self, sql, schema):
+        eng = self.get_sqla_engine(schema=schema)
+        cur = eng.execute(sql, schema=schema)
+        cols = [col[0] for col in cur.cursor.description]
+        df = pd.DataFrame(cur.fetchall(), columns=cols)
+        return df
 
     def safe_sqlalchemy_uri(self):
         return self.sqlalchemy_uri
 
+    @property
+    def inspector(self):
+        engine = self.get_sqla_engine()
+        return sqla.inspect(engine)
+
+    def all_table_names(self, schema=None):
+        return sorted(self.inspector.get_table_names(schema))
+
+    def all_view_names(self, schema=None):
+        views = []
+        try:
+            views = self.inspector.get_view_names(schema)
+        except Exception as e:
+            pass
+        return views
+
+    def all_schema_names(self):
+        return sorted(self.inspector.get_schema_names())
+
     def grains(self):
         """Defines time granularity database-specific expressions.
 
@@ -508,10 +551,8 @@ class Database(Model, AuditMixinNullable):
             autoload=True,
             autoload_with=self.get_sqla_engine())
 
-    def get_columns(self, table_name):
-        engine = self.get_sqla_engine()
-        insp = reflection.Inspector.from_engine(engine)
-        return insp.get_columns(table_name)
+    def get_columns(self, table_name, schema=None):
+        return self.inspector.get_columns(table_name, schema)
 
     @property
     def sqlalchemy_uri_decrypted(self):
@@ -1690,3 +1731,61 @@ class FavStar(Model):
     class_name = Column(String(50))
     obj_id = Column(Integer)
     dttm = Column(DateTime, default=func.now())
+
+
+class QueryStatus:
+    def from_presto_states(self, presto_status):
+        if presto_status.lower() == 'running':
+            return QueryStatus.IN_PROGRESS
+        if presto_status.lower() == 'running':
+            return QueryStatus.IN_PROGRESS
+        if presto_status.lower() == 'running':
+            return QueryStatus.IN_PROGRESS
+        if presto_status.lower() == 'running':
+            return QueryStatus.IN_PROGRESS
+
+    SCHEDULED = 'SCHEDULED'
+    CANCELLED = 'CANCELLED'
+    IN_PROGRESS = 'IN_PROGRESS'
+    FINISHED = 'FINISHED'
+    TIMED_OUT = 'TIMED_OUT'
+    FAILED = 'FAILED'
+
+
+class Query(Model):
+
+    """ORM model for SQL query"""
+
+    __tablename__ = 'query'
+    id = Column(Integer, primary_key=True)
+
+    database_id = Column(Integer, ForeignKey('dbs.id'), nullable=False)
+
+    # Store the tmp table into the DB only if the user asks for it.
+    tmp_table_name = Column(String(256))
+    user_id = Column(Integer, ForeignKey('ab_user.id'), nullable=True)
+
+    # models.QueryStatus
+    status = Column(String(16))
+
+    name = Column(String(256))
+    tab_name = Column(String(256))
+    schema = Column(String(256))
+    sql = Column(Text)
+    # Query to retrieve the results,
+    # used only in case of select_as_cta_used is true.
+    select_sql = Column(Text)
+    executed_sql = Column(Text)
+    # Could be configured in the caravel config.
+    limit = Column(Integer)
+    limit_used = Column(Boolean)
+    select_as_cta = Column(Boolean)
+    select_as_cta_used = Column(Boolean)
+
+    # 1..100
+    progress = Column(Integer)
+    # # of rows in the result set or rows modified.
+    rows = Column(Integer)
+    error_message = Column(Text)
+    start_time = Column(DateTime)
+    end_time = Column(DateTime)

caravel/sql_lab.py

@@ -0,0 +1,153 @@
+import celery
+from caravel import models, app, utils, sql_lab_utils
+from datetime import datetime
+
+celery_app = celery.Celery(config_source=app.config.get('CELERY_CONFIG'))
+
+
+@celery_app.task
+def get_sql_results(query_id):
+    """Executes the sql query returns the results."""
+    sql_manager = QueryRunner(query_id)
+    sql_manager.run_sql()
+    # Return the result for the sync call.
+    # if self.request.called_directly:
+    if sql_manager.query().status == models.QueryStatus.FINISHED:
+        return {
+            'query_id': sql_manager.query().id,
+            'status': sql_manager.query().status,
+            'data': sql_manager.data(),
+            'columns': sql_manager.columns(),
+        }
+    else:
+        return {
+            'query_id': sql_manager.query().id,
+            'status': sql_manager.query().status,
+            'error': sql_manager.query().error_message,
+        }
+
+
+class QueryRunner:
+    def __init__(self, query_id):
+        self._query_id = query_id
+        # Creates a separate session, reusing the db.session leads to the
+        # concurrency issues.
+        self._session = sql_lab_utils.create_scoped_session()
+        self._query = self._session.query(models.Query).filter_by(
+            id=query_id).first()
+        self._db_to_query = self._session.query(models.Database).filter_by(
+            id=self._query.database_id).first()
+        # Query result.
+        self._data = None
+        self._columns = None
+
+    def _sanity_check(self):
+        if not self._query:
+            self._query.error_message = "Query with id {0} not found.".format(
+                self._query_id)
+        if not self._db_to_query:
+            self._query.error_message = (
+                "Database with id {0} is missing.".format(
+                    self._query.database_id)
+            )
+
+        if self._query.error_message:
+            self._query.status = models.QueryStatus.FAILED
+            self._session.flush()
+            return False
+        return True
+
+    def query(self):
+        return self._query
+
+    def data(self):
+        return self._data
+
+    def columns(self):
+        return self._columns
+
+    def run_sql(self):
+        if not self._sanity_check():
+            return self._query.status
+
+        # TODO(bkyryliuk): dump results somewhere for the webserver.
+        engine = self._db_to_query.get_sqla_engine(schema=self._query.schema)
+        self._query.executed_sql = self._query.sql.strip().strip(';')
+
+        # Limit enforced only for retrieving the data, not for the CTA queries.
+        self._query.select_as_cta_used = False
+        self._query.limit_used = False
+        if sql_lab_utils.is_query_select(self._query.sql):
+            if self._query.select_as_cta:
+                if not self._query.tmp_table_name:
+                    self._query.tmp_table_name = 'tmp_{}_table_{}'.format(
+                        self._query.user_id,
+                        self._query.start_time.strftime('%Y_%m_%d_%H_%M_%S'))
+                self._query.executed_sql = sql_lab_utils.create_table_as(
+                    self._query.executed_sql, self._query.tmp_table_name)
+                self._query.select_as_cta_used = True
+            elif self._query.limit:
+                self._query.executed_sql = sql_lab_utils.add_limit_to_the_sql(
+                    self._query.executed_sql, self._query.limit, engine)
+                self._query.limit_used = True
+
+        # TODO(bkyryliuk): ensure that tmp table was created.
+        # Do not set tmp table name if table wasn't created.
+        if not self._query.select_as_cta_used:
+            self._query.tmp_table_name = None
+        self._get_sql_results(engine)
+
+        self._query.end_time = datetime.now()
+        self._session.flush()
+        return self._query.status
+
+    def _get_sql_results(self, engine):
+        try:
+            result_proxy = engine.execute(
+                self._query.executed_sql, schema=self._query.schema)
+        except Exception as e:
+            self._query.error_message = utils.error_msg_from_exception(e)
+            self._query.status = models.QueryStatus.FAILED
+            return
+
+        cursor = result_proxy.cursor
+        if hasattr(cursor, "poll"):
+            query_stats = cursor.poll()
+            self._query.status = models.QueryStatus.IN_PROGRESS
+            self._session.flush()
+            # poll returns dict -- JSON status information or ``None``
+            # if the query is done
+            # https://github.com/dropbox/PyHive/blob/
+            # b34bdbf51378b3979eaf5eca9e956f06ddc36ca0/pyhive/presto.py#L178
+            while query_stats:
+                # Update the object and wait for the kill signal.
+                self._session.refresh(self._query)
+                completed_splits = int(query_stats['stats']['completedSplits'])
+                total_splits = int(query_stats['stats']['totalSplits'])
+                progress = 100 * completed_splits / total_splits
+                if progress > self._query.progress:
+                    self._query.progress = progress
+
+                self._session.flush()
+                query_stats = cursor.poll()
+                # TODO(b.kyryliuk): check for the kill signal.
+
+        sql_results = sql_lab_utils.fetch_response_from_cursor(
+            result_proxy)
+        self._columns = sql_results['columns']
+        self._data = sql_results['data']
+        self._query.rows = result_proxy.rowcount
+        self._query.progress = 100
+        self._query.status = models.QueryStatus.FINISHED
+        if self._query.rows == -1 and self._data:
+            # Presto doesn't provide result_proxy.row_count
+            self._query.rows = len(self._data)
+
+        # CTAs queries result in 1 cell having the # of the added rows.
+        if self._query.select_as_cta_used:
+            self._query.select_sql = sql_lab_utils.select_star(
+                engine, self._query.tmp_table_name, self._query.limit)
+        else:
+            self._query.tmp_table = None
+
+

caravel/sql_lab_utils.py

@@ -0,0 +1,125 @@
+# SQL Lab Utils
+import pandas as pd
+
+import sqlparse
+from caravel import models, app
+from sqlalchemy import create_engine
+from sqlalchemy.orm import scoped_session, sessionmaker
+from sqlalchemy import select, text
+from sqlalchemy.sql.expression import TextAsFrom
+
+
+def create_scoped_session():
+    """Creates new SQLAlchemy scoped_session."""
+    engine = create_engine(
+        app.config.get('SQLALCHEMY_DATABASE_URI'), convert_unicode=True)
+    return scoped_session(sessionmaker(
+        autocommit=True, autoflush=False, bind=engine))
+
+
+def fetch_response_from_cursor(result_proxy):
+    columns = None
+    data = None
+    if result_proxy.cursor:
+        cols = [col[0] for col in result_proxy.cursor.description]
+        data = result_proxy.fetchall()
+        df = pd.DataFrame(data, columns=cols)
+        df = df.fillna(0)
+        columns = [c for c in df.columns]
+        data = df.to_dict(orient='records')
+    return {
+        'columns': columns,
+        'data': data,
+    }
+
+
+def is_query_select(sql):
+    try:
+        return sqlparse.parse(sql)[0].get_type() == 'SELECT'
+    # Capture sqlparse exceptions, worker shouldn't fail here.
+    except Exception:
+        # TODO(bkyryliuk): add logging here.
+        return False
+
+
+# if sqlparse provides the stream of tokens but don't provide the API
+# to access the table names, more on it:
+# https://groups.google.com/forum/#!topic/sqlparse/sL2aAi6dSJU
+# https://github.com/andialbrecht/sqlparse/blob/master/examples/
+# extract_table_names.py
+#
+# Another approach would be to run the EXPLAIN on the sql statement:
+# https://prestodb.io/docs/current/sql/explain.html
+# https://cwiki.apache.org/confluence/display/Hive/LanguageManual+Explain
+def get_tables():
+    """Retrieves the query names from the query."""
+    # TODO(bkyryliuk): implement parsing the sql statement.
+    pass
+
+
+def select_star(engine, table_name, limit):
+    if limit:
+        select_star_sql = select('*').select_from(table_name).limit(limit)
+    else:
+        select_star_sql = select('*').select_from(table_name)
+
+    # SQL code to preview the results
+    return '{}'.format(select_star_sql.compile(
+        engine, compile_kwargs={"literal_binds": True}))
+
+
+def add_limit_to_the_sql(sql, limit, eng):
+    # Treat as single sql statement in case of failure.
+    try:
+        sql_statements = [s for s in sqlparse.split(sql) if s]
+    except Exception as e:
+        app.logger.info(
+            "Statement " + sql + "failed to be transformed to have the limit "
+
+
+            "with the exception" + e.message)
+        return sql
+    if len(sql_statements) == 1 and is_query_select(sql):
+        qry = select('*').select_from(
+            TextAsFrom(text(sql_statements[0]), ['*']).alias(
+                'inner_qry')).limit(limit)
+        sql_statement = str(qry.compile(
+            eng, compile_kwargs={"literal_binds": True}))
+        return sql_statement
+    return sql
+
+
+# create table works only for the single statement.
+# TODO(bkyryliuk): enforce that all the columns have names. Presto requires it
+#                  for the CTA operation.
+def create_table_as(sql, table_name, override=False):
+    """Reformats the query into the create table as query.
+
+    Works only for the single select SQL statements, in all other cases
+    the sql query is not modified.
+    :param sql: string, sql query that will be executed
+    :param table_name: string, will contain the results of the query execution
+    :param override, boolean, table table_name will be dropped if true
+    :return: string, create table as query
+    """
+    # TODO(bkyryliuk): drop table if allowed, check the namespace and
+    #                  the permissions.
+    # Treat as single sql statement in case of failure.
+    try:
+        # Filter out empty statements.
+        sql_statements = [s for s in sqlparse.split(sql) if s]
+    except Exception as e:
+        app.logger.info(
+            "Statement " + sql + "failed to be transformed as create table as "
+            "with the exception" + e.message)
+        return sql
+    if len(sql_statements) == 1 and is_query_select(sql):
+        updated_sql = ''
+        # TODO(bkyryliuk): use sqlalchemy statements for the
+        #  the drop and create operations.
+        if override:
+            updated_sql = 'DROP TABLE IF EXISTS {};\n'.format(table_name)
+        updated_sql += "CREATE TABLE %s AS %s" % (
+            table_name, sql_statements[0])
+        return updated_sql
+    return sql

caravel/templates/caravel/basic.html

@@ -29,7 +29,7 @@
     {% block body %}
       {% include 'caravel/flash_wrapper.html' %}
       <div id="app">
-        Oops! React.js is not working properly.
+        <img src="/static/assets/images/loading.gif" style="width: 50px; margin: 10px;">
       </div>
     {% endblock %}
 

caravel/templates/caravel/sqllab.html

@@ -0,0 +1,6 @@
+{% extends "caravel/basic.html" %}
+
+{% block tail_js %}
+  {{ super() }}
+  <script src="/static/assets/javascripts/dist/sqllab.entry.js"></script>
+{% endblock %}

caravel/utils.py

@@ -4,7 +4,7 @@ from __future__ import division
 from __future__ import print_function
 from __future__ import unicode_literals
 
-from datetime import datetime, date
+from datetime import date, datetime
 import decimal
 import functools
 import json
@@ -203,8 +203,11 @@ def init(caravel):
         if perm.permission.name in ('datasource_access', 'database_access'):
             continue
         if perm.view_menu and perm.view_menu.name not in (
-                'UserDBModelView', 'RoleModelView', 'ResetPasswordView',
-                'Security'):
+                'ResetPasswordView',
+                'RoleModelView',
+                'Security',
+                'UserDBModelView',
+                'SQL Lab'):
             sm.add_permission_role(alpha, perm)
         sm.add_permission_role(admin, perm)
     gamma = sm.add_role("Gamma")
@@ -217,6 +220,7 @@ def init(caravel):
                     'ResetPasswordView',
                     'RoleModelView',
                     'UserDBModelView',
+                    'SQL Lab',
                     'Security') and
                 perm.permission.name not in (
                     'all_datasource_access',
@@ -304,6 +308,8 @@ def json_iso_dttm_ser(obj):
         return val
     if isinstance(obj, datetime):
         obj = obj.isoformat()
+    elif isinstance(obj, date):
+        obj = obj.isoformat()
     else:
         raise TypeError(
             "Unserializable object {} of type {}".format(obj, type(obj))
@@ -329,13 +335,21 @@ def json_int_dttm_ser(obj):
 
 def error_msg_from_exception(e):
     """Translate exception into error message
+
     Database have different ways to handle exception. This function attempts
     to make sense of the exception object and construct a human readable
     sentence.
+
+    TODO(bkyryliuk): parse the Presto error message from the connection
+                     created via create_engine.
+    engine = create_engine('presto://localhost:3506/silver') -
+      gives an e.message as the str(dict)
+    presto.connect("localhost", port=3506, catalog='silver') - as a dict.
+    The latter version is parsed correctly by this function.
     """
     msg = ''
     if hasattr(e, 'message'):
-        if (type(e.message) is dict):
+        if type(e.message) is dict:
             msg = e.message.get('message')
         elif e.message:
             msg = "{}".format(e.message)

caravel/views.py

@@ -32,7 +32,9 @@ from werkzeug.routing import BaseConverter
 from wtforms.validators import ValidationError
 
 import caravel
-from caravel import appbuilder, db, models, viz, utils, app, sm, ascii_art
+from caravel import (
+    appbuilder, db, models, viz, utils, app, sm, ascii_art, sql_lab
+)
 
 config = app.config
 log_this = models.Log.log_this
@@ -452,6 +454,18 @@ appbuilder.add_view(
     category_icon='fa-database',)
 
 
+class DatabaseAsync(DatabaseView):
+    list_columns = ['id', 'database_name']
+
+appbuilder.add_view_no_menu(DatabaseAsync)
+
+
+class DatabaseTablesAsync(DatabaseView):
+    list_columns = ['id', 'all_table_names', 'all_schema_names']
+
+appbuilder.add_view_no_menu(DatabaseTablesAsync)
+
+
 class TableModelView(CaravelModelView, DeleteMixin):  # noqa
     datamodel = SQLAInterface(models.SqlaTable)
     list_columns = [
@@ -610,7 +624,8 @@ class SliceModelView(CaravelModelView, DeleteMixin):  # noqa
             url = "/druiddatasourcemodelview/list/"
             msg = _(
                 "Click on a datasource link to create a Slice, "
-                "or click on a table link <a href='/tablemodelview/list/'>here</a> "
+                "or click on a table link "
+                "<a href='/tablemodelview/list/'>here</a> "
                 "to create a Slice for a table"
             )
         else:
@@ -892,7 +907,8 @@ class Caravel(BaseCaravelView):
         datasource_access = self.can_access(
             'datasource_access', datasource.perm)
         if not (all_datasource_access or datasource_access):
-            flash(__("You don't seem to have access to this datasource"), "danger")
+            flash(__("You don't seem to have access to this datasource"),
+                  "danger")
             return redirect(error_redirect)
 
         action = request.args.get('action')
@@ -911,7 +927,7 @@ class Caravel(BaseCaravelView):
                 form_data=request.args,
                 slice_=slc)
         except Exception as e:
-            flash(str(e), "danger")
+            flash(utils.error_msg_from_exception(e), "danger")
             return redirect(error_redirect)
         if request.args.get("json") == "true":
             status = 200
@@ -923,7 +939,7 @@ class Caravel(BaseCaravelView):
                     payload = obj.get_json()
                 except Exception as e:
                     logging.exception(e)
-                    payload = str(e)
+                    payload = utils.error_msg_from_exception(e)
                     status = 500
             resp = Response(
                 payload,
@@ -953,7 +969,7 @@ class Caravel(BaseCaravelView):
                 if config.get("DEBUG"):
                     raise(e)
                 return Response(
-                    str(e),
+                    utils.error_msg_from_exception(e),
                     status=500,
                     mimetype="application/json")
             return resp
@@ -969,7 +985,8 @@ class Caravel(BaseCaravelView):
         del d['action']
         del d['previous_viz_type']
 
-        as_list = ('metrics', 'groupby', 'columns', 'all_columns', 'mapbox_label', 'order_by_cols')
+        as_list = ('metrics', 'groupby', 'columns', 'all_columns',
+                   'mapbox_label', 'order_by_cols')
         for k in d:
             v = d.get(k)
             if k in as_list and not isinstance(v, list):
@@ -1080,9 +1097,29 @@ class Caravel(BaseCaravelView):
             .group_by(Log.dt)
             .all()
         )
-        payload = {str(time.mktime(dt.timetuple())): ccount for dt, ccount in qry if dt}
+        payload = {str(time.mktime(dt.timetuple())):
+                   ccount for dt, ccount in qry if dt}
         return Response(json.dumps(payload), mimetype="application/json")
 
+    @api
+    @has_access_api
+    @expose("/tables/<db_id>/<schema>")
+    def tables(self, db_id, schema):
+        """endpoint to power the calendar heatmap on the welcome page"""
+        schema = None if schema in ('null', 'undefined') else schema
+        database = (
+            db.session
+            .query(models.Database)
+            .filter_by(id=db_id)
+            .one()
+        )
+        payload = {
+            'tables': database.all_table_names(schema),
+            'views': database.all_view_names(schema),
+        }
+        return Response(
+            json.dumps(payload), mimetype="application/json")
+
     @api
     @has_access_api
     @expose("/save_dash/<dashboard_id>/", methods=['GET', 'POST'])
@@ -1117,9 +1154,11 @@ class Caravel(BaseCaravelView):
         data = json.loads(request.form.get('data'))
         session = db.session()
         Slice = models.Slice # noqa
-        dash = session.query(models.Dashboard).filter_by(id=dashboard_id).first()
+        dash = (
+            session.query(models.Dashboard).filter_by(id=dashboard_id).first())
         check_ownership(dash, raise_if_false=True)
-        new_slices = session.query(Slice).filter(Slice.id.in_(data['slice_ids']))
+        new_slices = session.query(Slice).filter(
+            Slice.id.in_(data['slice_ids']))
         dash.slices += new_slices
         session.merge(dash)
         session.commit()
@@ -1153,13 +1192,18 @@ class Caravel(BaseCaravelView):
         FavStar = models.FavStar  # noqa
         count = 0
         favs = session.query(FavStar).filter_by(
-            class_name=class_name, obj_id=obj_id, user_id=g.user.get_id()).all()
+            class_name=class_name, obj_id=obj_id,
+            user_id=g.user.get_id()).all()
         if action == 'select':
             if not favs:
                 session.add(
                     FavStar(
-                        class_name=class_name, obj_id=obj_id, user_id=g.user.get_id(),
-                        dttm=datetime.now()))
+                        class_name=class_name,
+                        obj_id=obj_id,
+                        user_id=g.user.get_id(),
+                        dttm=datetime.now()
+                    )
+                )
             count = 1
         elif action == 'unselect':
             for fav in favs:
@@ -1214,10 +1258,23 @@ class Caravel(BaseCaravelView):
             dash_save_perm=dash_save_perm,
             dash_edit_perm=dash_edit_perm)
 
+    @has_access
+    @expose("/sqllab_viz/")
+    @log_this
+    def sqllab_viz(self):
+        return json.dumps(request.args.to_dict(), indent=4)
+
     @has_access
     @expose("/sql/<database_id>/")
     @log_this
     def sql(self, database_id):
+        if (
+                not self.can_access(
+                    'all_datasource_access', 'all_datasource_access')):
+            flash(
+                "SQL Lab requires the `all_datasource_access` "
+                "permission", "danger")
+            return redirect("/tablemodelview/list/")
         mydb = db.session.query(
             models.Database).filter_by(id=database_id).first()
 
@@ -1240,23 +1297,35 @@ class Caravel(BaseCaravelView):
             db=mydb)
 
     @has_access
-    @expose("/table/<database_id>/<table_name>/")
+    @expose("/table/<database_id>/<table_name>/<schema>/")
     @log_this
-    def table(self, database_id, table_name):
-        mydb = db.session.query(
-            models.Database).filter_by(id=database_id).first()
-        cols = mydb.get_columns(table_name)
-        df = pd.DataFrame([(c['name'], c['type']) for c in cols])
-        df.columns = ['col', 'type']
-        tbl_cls = (
-            "dataframe table table-striped table-bordered "
-            "table-condensed sql_results").split(' ')
-        return self.render_template(
-            "caravel/ajah.html",
-            content=df.to_html(
-                index=False,
-                na_rep='',
-                classes=tbl_cls))
+    def table(self, database_id, table_name, schema):
+        schema = None if schema in ('null', 'undefined') else schema
+        mydb = db.session.query(models.Database).filter_by(id=database_id).one()
+        cols = []
+        t = mydb.get_columns(table_name, schema)
+        try:
+            t = mydb.get_columns(table_name, schema)
+        except Exception as e:
+            return Response(
+                json.dumps({'error': utils.error_msg_from_exception(e)}),
+                mimetype="application/json")
+        for col in t:
+            dtype = ""
+            try:
+                dtype = '{}'.format(col['type'])
+            except:
+                pass
+            cols.append({
+                'name': col['name'],
+                'type': dtype.split('(')[0] if '(' in dtype else dtype,
+                'longType': dtype,
+            })
+        tbl = {
+            'name': table_name,
+            'columns': cols,
+        }
+        return Response(json.dumps(tbl), mimetype="application/json")
 
     @has_access
     @expose("/select_star/<database_id>/<table_name>/")
@@ -1330,7 +1399,7 @@ class Caravel(BaseCaravelView):
                 content = (
                     '<div class="alert alert-danger">'
                     "{}</div>"
-                ).format(e.message)
+                ).format(utils.error_msg_from_exception(e))
         session.commit()
         return content
 
@@ -1343,12 +1412,27 @@ class Caravel(BaseCaravelView):
     @log_this
     def sql_json(self):
         """Runs arbitrary sql and returns and json"""
-        session = db.session()
-        limit = 1000
         sql = request.form.get('sql')
         database_id = request.form.get('database_id')
+        schema = request.form.get('schema')
+        tab_name = request.form.get('tab_name')
+
+        async = request.form.get('async') == 'True'
+        tmp_table_name = request.form.get('tmp_table_name', None)
+        select_as_cta = request.form.get('select_as_cta') == 'True'
+
+        session = db.session()
         mydb = session.query(models.Database).filter_by(id=database_id).first()
 
+        if not mydb:
+            return Response(
+                json.dumps({
+                    'error': 'Database with id 0 is missing.',
+                    'status': models.QueryStatus.FAILED,
+                }),
+                status=500,
+                mimetype="application/json")
+
         if not (self.can_access(
                 'all_datasource_access', 'all_datasource_access') or
                 self.can_access('database_access', mydb.perm)):
@@ -1356,43 +1440,132 @@ class Caravel(BaseCaravelView):
                 "SQL Lab requires the `all_datasource_access` or "
                 "specific DB permission"))
 
-        error_msg = ""
-        if not mydb:
-            error_msg = "The database selected doesn't seem to exist"
-        else:
-            eng = mydb.get_sqla_engine()
-            if limit:
-                sql = sql.strip().strip(';')
-                qry = (
-                    select('*')
-                    .select_from(TextAsFrom(text(sql), ['*'])
-                                 .alias('inner_qry'))
-                    .limit(limit)
-                )
-                sql = '{}'.format(qry.compile(
-                    eng, compile_kwargs={"literal_binds": True}))
-            try:
-                df = pd.read_sql_query(sql=sql, con=eng)
-                df = df.fillna(0)  # TODO make sure NULL
-            except Exception as e:
-                logging.exception(e)
-                error_msg = utils.error_msg_from_exception(e)
+        start_time = datetime.now()
+        query_name = '{}_{}_{}'.format(
+            g.user.get_id(), tab_name, start_time.strftime('%M:%S:%f'))
 
+        query = models.Query(
+            database_id=database_id,
+            limit=app.config.get('SQL_MAX_ROW', None),
+            name=query_name,
+            sql=sql,
+            schema=schema,
+            # TODO(bkyryliuk): consider it being DB property.
+            select_as_cta=select_as_cta,
+            start_time=start_time,
+            status=models.QueryStatus.SCHEDULED,
+            tab_name=tab_name,
+            tmp_table_name=tmp_table_name,
+            user_id=g.user.get_id(),
+        )
+        session.add(query)
         session.commit()
-        if error_msg:
+
+        # Async request.
+        if async:
+            # Ignore the celery future object and the request may time out.
+            sql_lab.get_sql_results.delay(query.id)
+            return Response(json.dumps(
+                {
+                    'query_id': query.id,
+                    'status': query.status,
+                },
+                default=utils.json_int_dttm_ser, allow_nan=False),
+                status=202,  # Accepted
+                mimetype="application/json")
+
+        # Sync request.
+        data = sql_lab.get_sql_results(query.id)
+        if data['status'] == models.QueryStatus.FAILED:
+                return Response(
+                    json.dumps(
+                        data, default=utils.json_int_dttm_ser, allow_nan=False),
+                    status=500,
+                    mimetype="application/json")
+        return Response(
+            json.dumps(
+                data, default=utils.json_int_dttm_ser, allow_nan=False),
+            status=200,
+            mimetype="application/json")
+
+    @has_access
+    @expose("/queries/", methods=['GET'])
+    @log_this
+    def queries(self):
+        """Runs arbitrary sql and returns and json"""
+        last_updated = request.form.get('timestamp')
+        s = db.session()
+        query = s.query(models.Query).filter_by(id=query_id).first()
+        mydb = s.query(models.Database).filter_by(id=query.database_id).first()
+
+        if not (self.can_access(
+                'all_datasource_access', 'all_datasource_access') or
+                self.can_access('database_access', mydb.perm)):
+            raise utils.CaravelSecurityException(_(
+                "SQL Lab requires the `all_datasource_access` or "
+                "specific DB permission"))
+
+        if query:
             return Response(
                 json.dumps({
-                    'error': error_msg,
+                    'status': query.status,
+                    'progress': query.progress
                 }),
+                status=200,
+                mimetype="application/json")
+
+        return Response(
+            json.dumps({
+                'error': "Query with id {} wasn't found".format(query_id),
+            }),
+            status=404,
+            mimetype="application/json")
+
+    @has_access
+    @expose("/cta_query_results/", methods=['GET'])
+    @log_this
+    def cta_query_results(self):
+        """Runs arbitrary sql and returns and json"""
+        query_id = request.form.get('query_id')
+        s = db.session()
+        query = s.query(models.Query).filter_by(id=query_id).first()
+        mydb = s.query(models.Database).filter_by(id=query.database_id).first()
+
+        if not (self.can_access(
+                'all_datasource_access', 'all_datasource_access') or
+                self.can_access('database_access', mydb.perm)):
+            raise utils.CaravelSecurityException(_(
+                "SQL Lab requires the `all_datasource_access` or "
+                "specific DB permission"))
+
+        if not query:
+            return Response(
+                json.dumps({
+                    'error': "Query with id {} wasn't found".format(query_id),
+                }),
+                status=404,
+                mimetype="application/json")
+
+        if query.status != models.QueryStatus.FINISHED:
+            return Response(
+                json.dumps({
+                    'error': "Query with id {} not finished yet".format(
+                        query_id),
+                }),
+                status=400,
+                mimetype="application/json")
+        try:
+            data = mydb.get_df(query.select_sql, query.schema)
+            return Response(
+                json.dumps(
+                    data, default=utils.json_int_dttm_ser, allow_nan=False),
+                status=200,
+                mimetype="application/json")
+        except Exception as e:
+            return Response(
+                json.dumps('error', utils.error_msg_from_exception(e)),
                 status=500,
                 mimetype="application/json")
-        else:
-            data = {
-                'columns': [c for c in df.columns],
-                'data': df.to_dict(orient='records'),
-            }
-            return json.dumps(
-                data, default=utils.json_int_dttm_ser, allow_nan=False)
 
     @has_access
     @expose("/refresh_datasources/")
@@ -1433,6 +1606,11 @@ class Caravel(BaseCaravelView):
         """Personalized welcome page"""
         return self.render_template('caravel/welcome.html', utils=utils)
 
+    @has_access
+    @expose("/sqllab")
+    def sqlanvil(self):
+        """SQL Editor"""
+        return self.render_template('caravel/sqllab.html')
 
 appbuilder.add_view_no_menu(Caravel)
 
@@ -1462,6 +1640,11 @@ appbuilder.add_view(
     category_label=__("Sources"),
     category_icon='')
 
+appbuilder.add_link(
+    "SQL Lab",
+    href='/caravel/sqllab',
+    icon="fa-flask")
+
 
 # ---------------------------------------------------------------------
 # Redirecting URL from previous names

caravel/viz.py

@@ -285,7 +285,8 @@ class BaseViz(object):
                     cached_data = cached_data.decode('utf-8')
                 payload = json.loads(cached_data)
             except Exception as e:
-                logging.error("Error reading cache")
+                logging.error("Error reading cache: " +
+                              utils.error_msg_from_exception(e))
                 payload = None
             logging.info("Serving from cache")
 

run_tests.sh

@@ -1,6 +1,8 @@
 #!/usr/bin/env bash
 echo $DB
 rm ~/.caravel/caravel_unittests.db
+rm ~/.caravel/celerydb.sqlite
+rm ~/.caravel/celery_results.sqlite
 rm -f .coverage
 export CARAVEL_CONFIG=tests.caravel_test_config
 set -e

setup.py

@@ -16,6 +16,7 @@ setup(
     zip_safe=False,
     scripts=['caravel/bin/caravel'],
     install_requires=[
+        'celery==3.1.23',
         'cryptography==1.4',
         'flask-appbuilder==1.8.1',
         'flask-cache==0.13.1',
@@ -29,6 +30,7 @@ setup(
         'pandas==0.18.1',
         'parsedatetime==2.0.0',
         'pydruid==0.3.0',
+        'PyHive>=0.2.1',
         'python-dateutil==2.5.3',
         'requests==2.10.0',
         'simplejson==3.8.2',
@@ -36,6 +38,8 @@ setup(
         'sqlalchemy==1.0.13',
         'sqlalchemy-utils==0.32.7',
         'sqlparse==0.1.19',
+        'thrift>=0.9.3',
+        'thrift-sasl>=0.2.1',
         'werkzeug==0.11.10',
     ],
     extras_require={

tests/caravel_test_config.py

@@ -9,3 +9,17 @@ CARAVEL_WEBSERVER_PORT = 8081
 # continuous integration
 if 'CARAVEL__SQLALCHEMY_DATABASE_URI' in os.environ:
     SQLALCHEMY_DATABASE_URI = os.environ.get('CARAVEL__SQLALCHEMY_DATABASE_URI')
+
+SQL_CELERY_DB_FILE_PATH =  os.path.join(DATA_DIR, 'celerydb.sqlite')
+SQL_CELERY_RESULTS_DB_FILE_PATH =  os.path.join(DATA_DIR, 'celery_results.sqlite')
+SQL_SELECT_AS_CTA = True
+SQL_MAX_ROW = 666
+
+
+class CeleryConfig(object):
+    BROKER_URL = 'sqla+sqlite:///' + SQL_CELERY_DB_FILE_PATH
+    CELERY_IMPORTS = ('caravel.sql_lab', )
+    CELERY_RESULT_BACKEND = 'db+sqlite:///' + SQL_CELERY_RESULTS_DB_FILE_PATH
+    CELERY_ANNOTATIONS = {'sql_lab.add': {'rate_limit': '10/s'}}
+    CONCURRENCY = 1
+CELERY_CONFIG = CeleryConfig

tests/celery_tests.py

@@ -0,0 +1,428 @@
+"""Unit tests for Caravel Celery worker"""
+import imp
+import json
+import subprocess
+
+import time
+
+import os
+
+import pandas as pd
+import unittest
+
+import caravel
+from caravel import app, appbuilder, db, models, sql_lab, sql_lab_utils, utils
+
+
+BASE_DIR = app.config.get('BASE_DIR')
+cli = imp.load_source('cli', BASE_DIR + '/bin/caravel')
+
+
+SQL_CELERY_DB_FILE_PATH = '/tmp/celerydb.sqlite'
+SQL_CELERY_RESULTS_DB_FILE_PATH = '/tmp/celery_results.sqlite'
+
+
+class CeleryConfig(object):
+    BROKER_URL = 'sqla+sqlite:///' + SQL_CELERY_DB_FILE_PATH
+    CELERY_IMPORTS = ('caravel.sql_lab', )
+    CELERY_RESULT_BACKEND = 'db+sqlite:///' + SQL_CELERY_RESULTS_DB_FILE_PATH
+    CELERY_ANNOTATIONS = {'sql_lab.add': {'rate_limit': '10/s'}}
+    CONCURRENCY = 1
+app.config['CELERY_CONFIG'] = CeleryConfig
+
+# TODO(bkyryliuk): add ability to run this test separately.
+
+class UtilityFunctionTests(unittest.TestCase):
+    def test_create_table_as(self):
+        select_query = "SELECT * FROM outer_space;"
+        updated_select_query = sql_lab_utils.create_table_as(
+            select_query, "tmp")
+        self.assertEqual(
+            "CREATE TABLE tmp AS SELECT * FROM outer_space;",
+            updated_select_query)
+
+        updated_select_query_with_drop = sql_lab_utils.create_table_as(
+            select_query, "tmp", override=True)
+        self.assertEqual(
+            "DROP TABLE IF EXISTS tmp;\n"
+            "CREATE TABLE tmp AS SELECT * FROM outer_space;",
+            updated_select_query_with_drop)
+
+        select_query_no_semicolon = "SELECT * FROM outer_space"
+        updated_select_query_no_semicolon = sql_lab_utils.create_table_as(
+            select_query_no_semicolon, "tmp")
+        self.assertEqual(
+            "CREATE TABLE tmp AS SELECT * FROM outer_space",
+            updated_select_query_no_semicolon)
+
+        incorrect_query = "SMTH WRONG SELECT * FROM outer_space"
+        updated_incorrect_query = sql_lab_utils.create_table_as(
+            incorrect_query, "tmp")
+        self.assertEqual(incorrect_query, updated_incorrect_query)
+
+        insert_query = "INSERT INTO stomach VALUES (beer, chips);"
+        updated_insert_query = sql_lab_utils.create_table_as(
+            insert_query, "tmp")
+        self.assertEqual(insert_query, updated_insert_query)
+
+        multi_line_query = (
+            "SELECT * FROM planets WHERE\n"
+            "Luke_Father = 'Darth Vader';")
+        updated_multi_line_query = sql_lab_utils.create_table_as(
+            multi_line_query, "tmp")
+        expected_updated_multi_line_query = (
+            "CREATE TABLE tmp AS SELECT * FROM planets WHERE\n"
+            "Luke_Father = 'Darth Vader';")
+        self.assertEqual(
+            expected_updated_multi_line_query,
+            updated_multi_line_query)
+
+        updated_multi_line_query_with_drop = sql_lab_utils.create_table_as(
+            multi_line_query, "tmp", override=True)
+        expected_updated_multi_line_query_with_drop = (
+            "DROP TABLE IF EXISTS tmp;\n"
+            "CREATE TABLE tmp AS SELECT * FROM planets WHERE\n"
+            "Luke_Father = 'Darth Vader';")
+        self.assertEqual(
+            expected_updated_multi_line_query_with_drop,
+            updated_multi_line_query_with_drop)
+
+        delete_query = "DELETE FROM planet WHERE name = 'Earth'"
+        updated_delete_query = sql_lab_utils.create_table_as(delete_query, "tmp")
+        self.assertEqual(delete_query, updated_delete_query)
+
+        create_table_as = (
+            "CREATE TABLE pleasure AS SELECT chocolate FROM lindt_store;\n")
+        updated_create_table_as = sql_lab_utils.create_table_as(
+            create_table_as, "tmp")
+        self.assertEqual(create_table_as, updated_create_table_as)
+
+        sql_procedure = (
+            "CREATE PROCEDURE MyMarriage\n                "
+            "BrideGroom Male (25) ,\n                "
+            "Bride Female(20) AS\n                "
+            "BEGIN\n                "
+            "SELECT Bride FROM ukraine_ Brides\n                "
+            "WHERE\n                  "
+            "FatherInLaw = 'Millionaire' AND Count(Car) > 20\n"
+            "                  AND HouseStatus ='ThreeStoreyed'\n"
+            "                  AND BrideEduStatus IN "
+            "(B.TECH ,BE ,Degree ,MCA ,MiBA)\n                  "
+            "AND Having Brothers= Null AND Sisters =Null"
+        )
+        updated_sql_procedure = sql_lab_utils.create_table_as(sql_procedure, "tmp")
+        self.assertEqual(sql_procedure, updated_sql_procedure)
+
+        multiple_statements = """
+          DROP HUSBAND;
+          SELECT * FROM politicians WHERE clue > 0;
+          INSERT INTO MyCarShed VALUES('BUGATTI');
+          SELECT standard_disclaimer, witty_remark FROM company_requirements;
+          select count(*) from developer_brain;
+        """
+        updated_multiple_statements = sql_lab_utils.create_table_as(
+            multiple_statements, "tmp")
+        self.assertEqual(multiple_statements, updated_multiple_statements)
+
+
+class CeleryTestCase(unittest.TestCase):
+    def __init__(self, *args, **kwargs):
+        super(CeleryTestCase, self).__init__(*args, **kwargs)
+        self.client = app.test_client()
+
+    def get_query_by_name(self, sql):
+        session = db.create_scoped_session()
+        query = session.query(models.Query).filter_by(sql=sql).first()
+        session.close()
+        return query
+
+    def get_query_by_id(self, id):
+        session = db.create_scoped_session()
+        query = session.query(models.Query).filter_by(id=id).first()
+        session.close()
+        return query
+
+    @classmethod
+    def setUpClass(cls):
+        try:
+            os.remove(app.config.get('SQL_CELERY_DB_FILE_PATH'))
+        except OSError as e:
+            app.logger.warn(str(e))
+        try:
+            os.remove(app.config.get('SQL_CELERY_RESULTS_DB_FILE_PATH'))
+        except OSError as e:
+            app.logger.warn(str(e))
+
+        utils.init(caravel)
+
+        worker_command = BASE_DIR + '/bin/caravel worker'
+        subprocess.Popen(
+            worker_command, shell=True, stdout=subprocess.PIPE)
+
+        admin = appbuilder.sm.find_user('admin')
+        if not admin:
+            appbuilder.sm.add_user(
+                'admin', 'admin', ' user', 'admin@fab.org',
+                appbuilder.sm.find_role('Admin'),
+                password='general')
+        cli.load_examples(load_test_data=True)
+
+
+    @classmethod
+    def tearDownClass(cls):
+        subprocess.call(
+            "ps auxww | grep 'celeryd' | awk '{print $2}' | xargs kill -9",
+            shell=True
+        )
+        subprocess.call(
+            "ps auxww | grep 'caravel worker' | awk '{print $2}' | "
+            "xargs kill -9",
+            shell=True
+        )
+
+    def setUp(self):
+        pass
+
+    def tearDown(self):
+        pass
+
+    def login(self, username='admin', password='general'):
+        resp = self.client.post(
+            '/login/',
+            data=dict(username=username, password=password),
+            follow_redirects=True)
+        assert 'Welcome' in resp.data.decode('utf-8')
+
+    def logout(self):
+        self.client.get('/logout/', follow_redirects=True)
+
+    def run_sql(self, dbid, sql, cta='False', tmp_table='tmp',
+                async='False'):
+        self.login()
+        resp = self.client.post(
+            '/caravel/sql_json/',
+            data=dict(
+                database_id=dbid,
+                sql=sql,
+                async=async,
+                select_as_cta=cta,
+                tmp_table_name=tmp_table,
+            ),
+        )
+        self.logout()
+        return json.loads(resp.data.decode('utf-8'))
+
+    def test_add_limit_to_the_query(self):
+        query_session = sql_lab_utils.create_scoped_session()
+        db_to_query = query_session.query(models.Database).filter_by(
+            id=1).first()
+        eng = db_to_query.get_sqla_engine()
+
+        select_query = "SELECT * FROM outer_space;"
+        updated_select_query = sql_lab_utils.add_limit_to_the_sql(
+            select_query, 100, eng)
+        # Different DB engines have their own spacing while compiling
+        # the queries, that's why ' '.join(query.split()) is used.
+        # In addition some of the engines do not include OFFSET 0.
+        self.assertTrue(
+            "SELECT * FROM (SELECT * FROM outer_space;) AS inner_qry "
+            "LIMIT 100" in ' '.join(updated_select_query.split())
+        )
+
+        select_query_no_semicolon = "SELECT * FROM outer_space"
+        updated_select_query_no_semicolon = sql_lab_utils.add_limit_to_the_sql(
+            select_query_no_semicolon, 100, eng)
+        self.assertTrue(
+            "SELECT * FROM (SELECT * FROM outer_space) AS inner_qry "
+            "LIMIT 100" in
+            ' '.join(updated_select_query_no_semicolon.split())
+        )
+
+        incorrect_query = "SMTH WRONG SELECT * FROM outer_space"
+        updated_incorrect_query = sql_lab_utils.add_limit_to_the_sql(
+            incorrect_query, 100, eng)
+        self.assertEqual(incorrect_query, updated_incorrect_query)
+
+        insert_query = "INSERT INTO stomach VALUES (beer, chips);"
+        updated_insert_query = sql_lab_utils.add_limit_to_the_sql(
+            insert_query, 100, eng)
+        self.assertEqual(insert_query, updated_insert_query)
+
+        multi_line_query = (
+            "SELECT * FROM planets WHERE\n Luke_Father = 'Darth Vader';"
+        )
+        updated_multi_line_query = sql_lab_utils.add_limit_to_the_sql(
+            multi_line_query, 100, eng)
+        self.assertTrue(
+            "SELECT * FROM (SELECT * FROM planets WHERE "
+            "Luke_Father = 'Darth Vader';) AS inner_qry LIMIT 100" in
+            ' '.join(updated_multi_line_query.split())
+        )
+
+        delete_query = "DELETE FROM planet WHERE name = 'Earth'"
+        updated_delete_query = sql_lab_utils.add_limit_to_the_sql(
+            delete_query, 100, eng)
+        self.assertEqual(delete_query, updated_delete_query)
+
+        create_table_as = (
+            "CREATE TABLE pleasure AS SELECT chocolate FROM lindt_store;\n")
+        updated_create_table_as = sql_lab_utils.add_limit_to_the_sql(
+            create_table_as, 100, eng)
+        self.assertEqual(create_table_as, updated_create_table_as)
+
+        sql_procedure = (
+            "CREATE PROCEDURE MyMarriage\n                "
+            "BrideGroom Male (25) ,\n                "
+            "Bride Female(20) AS\n                "
+            "BEGIN\n                "
+            "SELECT Bride FROM ukraine_ Brides\n                "
+            "WHERE\n                  "
+            "FatherInLaw = 'Millionaire' AND Count(Car) > 20\n"
+            "                  AND HouseStatus ='ThreeStoreyed'\n"
+            "                  AND BrideEduStatus IN "
+            "(B.TECH ,BE ,Degree ,MCA ,MiBA)\n                  "
+            "AND Having Brothers= Null AND Sisters = Null"
+        )
+        updated_sql_procedure = sql_lab_utils.add_limit_to_the_sql(
+            sql_procedure, 100, eng)
+        self.assertEqual(sql_procedure, updated_sql_procedure)
+
+    def test_run_sync_query(self):
+        main_db = db.session.query(models.Database).filter_by(
+            database_name="main").first()
+        eng = main_db.get_sqla_engine()
+
+        # Case 1.
+        # DB #0 doesn't exist.
+        sql_dont_exist = 'SELECT * FROM dontexist'
+        result1 = self.run_sql(0, sql_dont_exist, cta='True')
+        self.assertEqual(models.QueryStatus.FAILED, result1[u'status'])
+        self.assertFalse(u'query_id' in result1)
+        self.assertEqual('Database with id 0 is missing.', result1['error'])
+        self.assertIsNone(self.get_query_by_name(sql_dont_exist))
+
+        # Case 2.
+        # Table doesn't exist.
+        result2 = self.run_sql(1, sql_dont_exist, cta='True', )
+        self.assertTrue('error' in result2)
+        self.assertEqual(models.QueryStatus.FAILED, result1[u'status'])
+        query2 = self.get_query_by_id(result2[u'query_id'])
+        self.assertEqual(models.QueryStatus.FAILED, query2.status)
+
+        # Case 3.
+        # Table and DB exists, CTA call to the backend.
+        sql_where = "SELECT name FROM ab_permission WHERE name='can_sql'"
+        result3 = self.run_sql(
+            1, sql_where, tmp_table='tmp_table_3', cta='True')
+        self.assertEqual(models.QueryStatus.FINISHED, result3[u'status'])
+        self.assertIsNone(result3[u'data'])
+        self.assertIsNone(result3[u'columns'])
+        query3 = self.get_query_by_id(result3[u'query_id'])
+
+        # Check the data in the tmp table.
+        df3 = pd.read_sql_query(sql=query3.select_sql, con=eng)
+        data3 = df3.to_dict(orient='records')
+        self.assertEqual([{'name': 'can_sql'}], data3)
+
+        # Case 4.
+        # Table and DB exists, CTA call to the backend, no data.
+        sql_empty_result = 'SELECT * FROM ab_user WHERE id=666'
+        result4 = self.run_sql(
+            1, sql_empty_result, tmp_table='tmp_table_4', cta='True',)
+        self.assertEqual(models.QueryStatus.FINISHED, result4[u'status'])
+        self.assertIsNone(result4[u'data'])
+        self.assertIsNone(result4[u'columns'])
+
+        query4 = self.get_query_by_id(result4[u'query_id'])
+        self.assertEqual(models.QueryStatus.FINISHED, query4.status)
+        self.assertTrue("SELECT * \nFROM tmp_table_4" in query4.select_sql)
+        self.assertTrue("LIMIT 666" in query4.select_sql)
+        self.assertEqual(
+            "CREATE TABLE tmp_table_4 AS SELECT * FROM ab_user WHERE id=666",
+            query4.executed_sql)
+        self.assertEqual("SELECT * FROM ab_user WHERE id=666", query4.sql)
+        if eng.name != 'sqlite':
+            self.assertEqual(0, query4.rows)
+        self.assertEqual(666, query4.limit)
+        self.assertEqual(False, query4.limit_used)
+        self.assertEqual(True, query4.select_as_cta)
+        self.assertEqual(True, query4.select_as_cta_used)
+
+        # Check the data in the tmp table.
+        df4 = pd.read_sql_query(sql=query4.select_sql, con=eng)
+        data4 = df4.to_dict(orient='records')
+        self.assertEqual([], data4)
+
+        # Case 5.
+        # Table and DB exists, select without CTA.
+        result5 = self.run_sql(1, sql_where, tmp_table='tmp_table_5')
+        self.assertEqual(models.QueryStatus.FINISHED, result5[u'status'])
+        self.assertEqual([u'name'], result5[u'columns'])
+        self.assertEqual([{u'name': u'can_sql'}], result5[u'data'])
+
+        query5 = self.get_query_by_id(result5[u'query_id'])
+        self.assertEqual(sql_where, query5.sql)
+        if eng.name != 'sqlite':
+            self.assertEqual(1, query5.rows)
+        self.assertEqual(666, query5.limit)
+        self.assertEqual(True, query5.limit_used)
+        self.assertEqual(False, query5.select_as_cta)
+        self.assertEqual(False, query5.select_as_cta_used)
+
+    def test_run_async_query(self):
+        main_db = db.session.query(models.Database).filter_by(
+            database_name="main").first()
+        eng = main_db.get_sqla_engine()
+
+        # Schedule queries
+
+        # Case 1.
+        # Table and DB exists, async CTA call to the backend.
+        sql_where = "SELECT name FROM ab_role WHERE name='Admin'"
+        result1 = self.run_sql(
+            1, sql_where, async='True', tmp_table='tmp_async_1', cta='True')
+        self.assertEqual(models.QueryStatus.SCHEDULED, result1[u'status'])
+
+        # Case 2.
+        # Table and DB exists, async insert query, no CTAs.
+        insert_query = "INSERT INTO ab_role VALUES (9, 'fake_role')"
+        result2 = self.run_sql(1, insert_query, async='True')
+        self.assertEqual(models.QueryStatus.SCHEDULED, result2[u'status'])
+
+        time.sleep(2)
+
+        # Case 1.
+        query1 = self.get_query_by_id(result1[u'query_id'])
+        df1 = pd.read_sql_query(query1.select_sql, con=eng)
+        self.assertEqual(models.QueryStatus.FINISHED, query1.status)
+        self.assertEqual([{'name': 'Admin'}], df1.to_dict(orient='records'))
+        self.assertEqual(models.QueryStatus.FINISHED, query1.status)
+        self.assertTrue("SELECT * \nFROM tmp_async_1" in query1.select_sql)
+        self.assertTrue("LIMIT 666" in query1.select_sql)
+        self.assertEqual(
+            "CREATE TABLE tmp_async_1 AS SELECT name FROM ab_role "
+            "WHERE name='Admin'", query1.executed_sql)
+        self.assertEqual(sql_where, query1.sql)
+        if eng.name != 'sqlite':
+            self.assertEqual(1, query1.rows)
+        self.assertEqual(666, query1.limit)
+        self.assertEqual(False, query1.limit_used)
+        self.assertEqual(True, query1.select_as_cta)
+        self.assertEqual(True, query1.select_as_cta_used)
+
+        # Case 2.
+        query2 = self.get_query_by_id(result2[u'query_id'])
+        self.assertEqual(models.QueryStatus.FINISHED, query2.status)
+        self.assertIsNone(query2.select_sql)
+        self.assertEqual(insert_query, query2.executed_sql)
+        self.assertEqual(insert_query, query2.sql)
+        if eng.name != 'sqlite':
+            self.assertEqual(1, query2.rows)
+        self.assertEqual(666, query2.limit)
+        self.assertEqual(False, query2.limit_used)
+        self.assertEqual(False, query2.select_as_cta)
+        self.assertEqual(False, query2.select_as_cta_used)
+
+
+if __name__ == '__main__':
+    unittest.main()

tests/core_tests.py

@@ -10,6 +10,7 @@ import json
 import imp
 import os
 import unittest
+
 from mock import Mock, patch
 
 from flask import escape
@@ -21,11 +22,15 @@ from caravel.models import DruidCluster, DruidDatasource
 
 os.environ['CARAVEL_CONFIG'] = 'tests.caravel_test_config'
 
-app.config['TESTING'] = True
+# Disable celery.
+app.config['CELERY_CONFIG'] = None
 app.config['CSRF_ENABLED'] = False
-app.config['SECRET_KEY'] = 'thisismyscretkey'
-app.config['WTF_CSRF_ENABLED'] = False
 app.config['PUBLIC_ROLE_LIKE_GAMMA'] = True
+app.config['SECRET_KEY'] = 'thisismyscretkey'
+app.config['SQL_SELECT_AS_CTA'] = False
+app.config['TESTING'] = True
+app.config['WTF_CSRF_ENABLED'] = False
+
 BASE_DIR = app.config.get("BASE_DIR")
 cli = imp.load_source('cli', BASE_DIR + "/bin/caravel")
 
@@ -40,7 +45,7 @@ class CaravelTestCase(unittest.TestCase):
         admin = appbuilder.sm.find_user('admin')
         if not admin:
             appbuilder.sm.add_user(
-                'admin', 'admin',' user', 'admin@fab.org',
+                'admin', 'admin', ' user', 'admin@fab.org',
                 appbuilder.sm.find_role('Admin'),
                 password='general')
 
@@ -79,7 +84,7 @@ class CaravelTestCase(unittest.TestCase):
         public_role = appbuilder.sm.find_role('Public')
         perms = db.session.query(ab_models.PermissionView).all()
         for perm in perms:
-            if (    perm.permission.name == 'datasource_access' and
+            if (perm.permission.name == 'datasource_access' and
                     perm.view_menu and table_name in perm.view_menu.name):
                 appbuilder.sm.add_permission_role(public_role, perm)
 
@@ -87,7 +92,7 @@ class CaravelTestCase(unittest.TestCase):
         public_role = appbuilder.sm.find_role('Public')
         perms = db.session.query(ab_models.PermissionView).all()
         for perm in perms:
-            if (    perm.permission.name == 'datasource_access' and
+            if (perm.permission.name == 'datasource_access' and
                     perm.view_menu and table_name in perm.view_menu.name):
                 appbuilder.sm.del_permission_role(public_role, perm)
 
@@ -95,15 +100,15 @@ class CaravelTestCase(unittest.TestCase):
 class CoreTests(CaravelTestCase):
 
     def __init__(self, *args, **kwargs):
-        # Load examples first, so that we setup proper permission-view relations
-        # for all example data sources.
+        # Load examples first, so that we setup proper permission-view
+        # relations for all example data sources.
         super(CoreTests, self).__init__(*args, **kwargs)
 
     @classmethod
     def setUpClass(cls):
         cli.load_examples(load_test_data=True)
         utils.init(caravel)
-        cls.table_ids = {tbl.table_name: tbl.id  for tbl in (
+        cls.table_ids = {tbl.table_name: tbl.id for tbl in (
             db.session
             .query(models.SqlaTable)
             .all()
@@ -126,7 +131,12 @@ class CoreTests(CaravelTestCase):
 
         copy_name = "Test Sankey Save"
         tbl_id = self.table_ids.get('energy_usage')
-        url = "/caravel/explore/table/{}/?viz_type=sankey&groupby=source&groupby=target&metric=sum__value&row_limit=5000&where=&having=&flt_col_0=source&flt_op_0=in&flt_eq_0=&slice_id={}&slice_name={}&collapsed_fieldsets=&action={}&datasource_name=energy_usage&datasource_id=1&datasource_type=table&previous_viz_type=sankey"
+        url = (
+            "/caravel/explore/table/{}/?viz_type=sankey&groupby=source&"
+            "groupby=target&metric=sum__value&row_limit=5000&where=&having=&"
+            "flt_col_0=source&flt_op_0=in&flt_eq_0=&slice_id={}&slice_name={}&"
+            "collapsed_fieldsets=&action={}&datasource_name=energy_usage&"
+            "datasource_id=1&datasource_type=table&previous_viz_type=sankey")
 
         db.session.commit()
         resp = self.client.get(
@@ -146,6 +156,8 @@ class CoreTests(CaravelTestCase):
         for slc in db.session.query(Slc).all():
             urls += [
                 (slc.slice_name, 'slice_url', slc.slice_url),
+                (slc.slice_name, 'slice_id_endpoint', '/caravel/slices/{}'.
+                 format(slc.id)),
                 (slc.slice_name, 'json_endpoint', slc.viz.json_endpoint),
                 (slc.slice_name, 'csv_endpoint', slc.viz.csv_endpoint),
             ]
@@ -210,13 +222,20 @@ class CoreTests(CaravelTestCase):
 
     def test_shortner(self):
         self.login(username='admin')
-        data = "//caravel/explore/table/1/?viz_type=sankey&groupby=source&groupby=target&metric=sum__value&row_limit=5000&where=&having=&flt_col_0=source&flt_op_0=in&flt_eq_0=&slice_id=78&slice_name=Energy+Sankey&collapsed_fieldsets=&action=&datasource_name=energy_usage&datasource_id=1&datasource_type=table&previous_viz_type=sankey"
+        data = (
+            "//caravel/explore/table/1/?viz_type=sankey&groupby=source&"
+            "groupby=target&metric=sum__value&row_limit=5000&where=&having=&"
+            "flt_col_0=source&flt_op_0=in&flt_eq_0=&slice_id=78&slice_name="
+            "Energy+Sankey&collapsed_fieldsets=&action=&datasource_name="
+            "energy_usage&datasource_id=1&datasource_type=table&"
+            "previous_viz_type=sankey")
         resp = self.client.post('/r/shortner/', data=data)
         assert '/r/' in resp.data.decode('utf-8')
 
     def test_save_dash(self, username='admin'):
         self.login(username=username)
-        dash = db.session.query(models.Dashboard).filter_by(slug="births").first()
+        dash = db.session.query(models.Dashboard).filter_by(
+            slug="births").first()
         positions = []
         for i, slc in enumerate(dash.slices):
             d = {
@@ -237,18 +256,24 @@ class CoreTests(CaravelTestCase):
 
     def test_add_slices(self, username='admin'):
         self.login(username=username)
-        dash = db.session.query(models.Dashboard).filter_by(slug="births").first()
-        new_slice = db.session.query(models.Slice).filter_by(slice_name="Mapbox Long/Lat").first()
-        existing_slice = db.session.query(models.Slice).filter_by(slice_name="Name Cloud").first()
+        dash = db.session.query(models.Dashboard).filter_by(
+            slug="births").first()
+        new_slice = db.session.query(models.Slice).filter_by(
+            slice_name="Mapbox Long/Lat").first()
+        existing_slice = db.session.query(models.Slice).filter_by(
+            slice_name="Name Cloud").first()
         data = {
-            "slice_ids": [new_slice.data["slice_id"], existing_slice.data["slice_id"]]
+            "slice_ids": [new_slice.data["slice_id"],
+                          existing_slice.data["slice_id"]]
         }
         url = '/caravel/add_slices/{}/'.format(dash.id)
         resp = self.client.post(url, data=dict(data=json.dumps(data)))
         assert "SLICES ADDED" in resp.data.decode('utf-8')
 
-        dash = db.session.query(models.Dashboard).filter_by(slug="births").first()
-        new_slice = db.session.query(models.Slice).filter_by(slice_name="Mapbox Long/Lat").first()
+        dash = db.session.query(models.Dashboard).filter_by(
+            slug="births").first()
+        new_slice = db.session.query(models.Slice).filter_by(
+            slice_name="Mapbox Long/Lat").first()
         assert new_slice in dash.slices
         assert len(set(dash.slices)) == len(dash.slices)
 
@@ -256,7 +281,10 @@ class CoreTests(CaravelTestCase):
         self.login(username=username)
         url = '/slicemodelview/add'
         resp = self.client.get(url, follow_redirects=True)
-        assert "Click on a table link to create a Slice" in resp.data.decode('utf-8')
+        assert (
+            "Click on a table link to create a Slice" in
+            resp.data.decode('utf-8')
+        )
 
     def test_add_slice_redirect_to_druid(self, username='admin'):
         datasource = DruidDatasource(
@@ -268,7 +296,10 @@ class CoreTests(CaravelTestCase):
         self.login(username=username)
         url = '/slicemodelview/add'
         resp = self.client.get(url, follow_redirects=True)
-        assert "Click on a datasource link to create a Slice" in resp.data.decode('utf-8')
+        assert (
+            "Click on a datasource link to create a Slice"
+            in resp.data.decode('utf-8')
+        )
 
         db.session.delete(datasource)
         db.session.commit()
@@ -290,7 +321,7 @@ class CoreTests(CaravelTestCase):
         )
         resp = self.client.post(
             '/caravel/sql_json/',
-            data=dict(database_id=dbid, sql=sql),
+            data=dict(database_id=dbid, sql=sql, select_as_create_as=False),
         )
         self.logout()
         return json.loads(resp.data.decode('utf-8'))
@@ -309,9 +340,9 @@ class CoreTests(CaravelTestCase):
         db.session.commit()
         main_db_permission_view = (
             db.session.query(ab_models.PermissionView)
-                .join(ab_models.ViewMenu)
-                .filter(ab_models.ViewMenu.name == '[main].(id:1)')
-                .first()
+            .join(ab_models.ViewMenu)
+            .filter(ab_models.ViewMenu.name == '[main].(id:1)')
+            .first()
         )
         astronaut = sm.add_role("Astronaut")
         sm.add_permission_role(astronaut, main_db_permission_view)
@@ -365,6 +396,10 @@ class CoreTests(CaravelTestCase):
         assert 'Births' in data
 
         # Confirm that public doesn't have access to other datasets.
+        resp = self.client.get('/slicemodelview/list/')
+        data = resp.data.decode('utf-8')
+        assert 'wb_health_population</a>' not in data
+
         resp = self.client.get('/dashboardmodelview/list/')
         data = resp.data.decode('utf-8')
         assert "/caravel/dashboard/world_health/" not in data
@@ -400,26 +435,26 @@ class CoreTests(CaravelTestCase):
 
 SEGMENT_METADATA = [{
   "id": "some_id",
-  "intervals": [ "2013-05-13T00:00:00.000Z/2013-05-14T00:00:00.000Z" ],
+  "intervals": ["2013-05-13T00:00:00.000Z/2013-05-14T00:00:00.000Z"],
   "columns": {
     "__time": {
         "type": "LONG", "hasMultipleValues": False,
-        "size": 407240380, "cardinality": None, "errorMessage": None },
+        "size": 407240380, "cardinality": None, "errorMessage": None},
     "dim1": {
         "type": "STRING", "hasMultipleValues": False,
-        "size": 100000, "cardinality": 1944, "errorMessage": None },
+        "size": 100000, "cardinality": 1944, "errorMessage": None},
     "dim2": {
         "type": "STRING", "hasMultipleValues": True,
-        "size": 100000, "cardinality": 1504, "errorMessage": None },
+        "size": 100000, "cardinality": 1504, "errorMessage": None},
     "metric1": {
         "type": "FLOAT", "hasMultipleValues": False,
-        "size": 100000, "cardinality": None, "errorMessage": None }
+        "size": 100000, "cardinality": None, "errorMessage": None}
   },
   "aggregators": {
     "metric1": {
         "type": "longSum",
         "name": "metric1",
-        "fieldName": "metric1" }
+        "fieldName": "metric1"}
   },
   "size": 300000,
   "numRows": 5000000
@@ -485,7 +520,8 @@ class DruidTests(CaravelTestCase):
         datasource_id = cluster.datasources[0].id
         db.session.commit()
 
-        resp = self.client.get('/caravel/explore/druid/{}/'.format(datasource_id))
+        resp = self.client.get('/caravel/explore/druid/{}/'.format(
+            datasource_id))
         assert "[test_cluster].[test_datasource]" in resp.data.decode('utf-8')
 
         nres = [
@@ -497,9 +533,15 @@ class DruidTests(CaravelTestCase):
         instance.export_pandas.return_value = df
         instance.query_dict = {}
         instance.query_builder.last_query.query_dict = {}
-        resp = self.client.get('/caravel/explore/druid/{}/?viz_type=table&granularity=one+day&druid_time_origin=&since=7+days+ago&until=now&row_limit=5000&include_search=false&metrics=count&groupby=name&flt_col_0=dim1&flt_op_0=in&flt_eq_0=&slice_id=&slice_name=&collapsed_fieldsets=&action=&datasource_name=test_datasource&datasource_id={}&datasource_type=druid&previous_viz_type=table&json=true&force=true'.format(datasource_id, datasource_id))
+        resp = self.client.get(
+            '/caravel/explore/druid/{}/?viz_type=table&granularity=one+day&'
+            'druid_time_origin=&since=7+days+ago&until=now&row_limit=5000&'
+            'include_search=false&metrics=count&groupby=name&flt_col_0=dim1&'
+            'flt_op_0=in&flt_eq_0=&slice_id=&slice_name=&collapsed_fieldsets=&'
+            'action=&datasource_name=test_datasource&datasource_id={}&'
+            'datasource_type=druid&previous_viz_type=table&json=true&'
+            'force=true'.format(datasource_id, datasource_id))
         assert "Canada" in resp.data.decode('utf-8')
 
-
 if __name__ == '__main__':
     unittest.main()