[sqllab] add support for results backends (#1377)

* [sqllab] add support for results backends

Long running SQL queries (beyond the scope of a web request) can now use
a k/v store to hold their result sets.

* Addressing comments, fixed js tests

* Fixing mysql has gone away

* Adressing more comments

* Touchups
This commit is contained in:
Maxime Beauchemin
2016-10-20 23:40:24 -07:00
committed by GitHub
parent 7dfe891cc1
commit 6fb3b305ad
27 changed files with 788 additions and 365 deletions

View File

@@ -5,6 +5,7 @@ import TabbedSqlEditors from './TabbedSqlEditors';
import QueryAutoRefresh from './QueryAutoRefresh';
import QuerySearch from './QuerySearch';
import Alerts from './Alerts';
import DataPreviewModal from './DataPreviewModal';
import { bindActionCreators } from 'redux';
import { connect } from 'react-redux';
@@ -26,8 +27,9 @@ class App extends React.Component {
this.setState({ hash: window.location.hash });
}
render() {
let content;
if (this.state.hash) {
return (
content = (
<div className="container-fluid">
<div className="row">
<div className="col-md-12">
@@ -37,16 +39,18 @@ class App extends React.Component {
</div>
);
}
content = (
<div>
<QueryAutoRefresh />
<TabbedSqlEditors />
</div>
);
return (
<div className="App SqlLab">
<Alerts alerts={this.props.alerts} />
<DataPreviewModal />
<div className="container-fluid">
<QueryAutoRefresh />
<Alerts alerts={this.props.alerts} />
<div className="row">
<div className="col-md-12">
<TabbedSqlEditors />
</div>
</div>
{content}
</div>
</div>
);

View File

@@ -0,0 +1,58 @@
import * as Actions from '../actions';
import React from 'react';
import { bindActionCreators } from 'redux';
import { connect } from 'react-redux';
import { Modal } from 'react-bootstrap';
import ResultSet from './ResultSet';
const propTypes = {
queries: React.PropTypes.object,
actions: React.PropTypes.object,
showDataPreviewModal: React.PropTypes.bool,
dataPreviewQueryId: React.PropTypes.string,
};
class DataPreviewModal extends React.Component {
hide() {
this.props.actions.hideDataPreview();
}
render() {
if (this.props.showDataPreviewModal && this.props.dataPreviewQueryId) {
const query = this.props.queries[this.props.dataPreviewQueryId];
return (
<Modal
show={this.props.showDataPreviewModal}
onHide={this.hide.bind(this)}
bsStyle="lg"
>
<Modal.Header closeButton>
<Modal.Title>
Data preview for <strong>{query.tableName}</strong>
</Modal.Title>
</Modal.Header>
<Modal.Body>
<ResultSet query={query} visualize={false} csv={false} />
</Modal.Body>
</Modal>
);
}
return null;
}
}
DataPreviewModal.propTypes = propTypes;
function mapStateToProps(state) {
return {
queries: state.queries,
showDataPreviewModal: state.showDataPreviewModal,
dataPreviewQueryId: state.dataPreviewQueryId,
};
}
function mapDispatchToProps(dispatch) {
return {
actions: bindActionCreators(Actions, dispatch),
};
}
export default connect(mapStateToProps, mapDispatchToProps)(DataPreviewModal);

View File

@@ -2,38 +2,43 @@ import React from 'react';
import SyntaxHighlighter from 'react-syntax-highlighter';
import { github } from 'react-syntax-highlighter/dist/styles';
const SqlShrink = (props) => {
const HighlightedSql = (props) => {
const sql = props.sql || '';
let lines = sql.split('\n');
if (lines.length >= props.maxLines) {
lines = lines.slice(0, props.maxLines);
lines.push('{...}');
}
const shrunk = lines.map((line) => {
if (line.length > props.maxWidth) {
return line.slice(0, props.maxWidth) + '{...}';
}
return line;
})
.join('\n');
let shownSql = sql;
if (props.shrink) {
shownSql = lines.map((line) => {
if (line.length > props.maxWidth) {
return line.slice(0, props.maxWidth) + '{...}';
}
return line;
})
.join('\n');
}
return (
<div>
<SyntaxHighlighter language="sql" style={github}>
{shrunk}
{shownSql}
</SyntaxHighlighter>
</div>
);
};
SqlShrink.defaultProps = {
HighlightedSql.defaultProps = {
maxWidth: 60,
maxLines: 6,
shrink: false,
};
SqlShrink.propTypes = {
HighlightedSql.propTypes = {
sql: React.PropTypes.string,
maxWidth: React.PropTypes.number,
maxLines: React.PropTypes.number,
shrink: React.PropTypes.bool,
};
export default SqlShrink;
export default HighlightedSql;

View File

@@ -6,14 +6,30 @@ import * as Actions from '../actions';
import moment from 'moment';
import { Table } from 'reactable';
import { ProgressBar } from 'react-bootstrap';
import { Label, ProgressBar } from 'react-bootstrap';
import Link from './Link';
import VisualizeModal from './VisualizeModal';
import SqlShrink from './SqlShrink';
import ResultSet from './ResultSet';
import ModalTrigger from '../../components/ModalTrigger';
import HighlightedSql from './HighlightedSql';
import { STATE_BSSTYLE_MAP } from '../common';
import { fDuration } from '../../modules/dates';
import { getLink } from '../../../utils/common';
const propTypes = {
columns: React.PropTypes.array,
actions: React.PropTypes.object,
queries: React.PropTypes.array,
onUserClicked: React.PropTypes.func,
onDbClicked: React.PropTypes.func,
};
const defaultProps = {
columns: ['started', 'duration', 'rows'],
queries: [],
onUserClicked: () => {},
onDbClicked: () => {},
};
class QueryTable extends React.Component {
constructor(props) {
@@ -45,6 +61,12 @@ class QueryTable extends React.Component {
openQueryInNewTab(query) {
this.props.actions.cloneQueryToNewTab(query);
}
openAsyncResults(query) {
this.props.actions.fetchQueryResults(query);
}
clearQueryResults(query) {
this.props.actions.clearQueryResults(query);
}
render() {
const data = this.props.queries.map((query) => {
@@ -71,9 +93,30 @@ class QueryTable extends React.Component {
q.started = moment(q.startDttm).format('HH:mm:ss');
const source = (q.ctas) ? q.executedSql : q.sql;
q.sql = (
<SqlShrink sql={source} maxWidth={100} />
<HighlightedSql sql={source} shrink maxWidth={100} />
);
q.output = q.tempTable;
if (q.resultsKey) {
q.output = (
<ModalTrigger
bsSize="large"
className="ResultsModal"
triggerNode={(
<Label
bsStyle="info"
style={{ cursor: 'pointer' }}
>
view results
</Label>
)}
modalTitle={'Data preview'}
beforeOpen={this.openAsyncResults.bind(this, query)}
onExit={this.clearQueryResults.bind(this, query)}
modalBody={<ResultSet showSql query={query} />}
/>
);
} else {
q.output = q.tempTable;
}
q.progress = (
<ProgressBar
style={{ width: '75px' }}
@@ -137,7 +180,7 @@ class QueryTable extends React.Component {
return q;
}).reverse();
return (
<div>
<div className="QueryTable">
<VisualizeModal
show={this.state.showVisualizeModal}
query={this.state.activeQuery}
@@ -152,19 +195,8 @@ class QueryTable extends React.Component {
);
}
}
QueryTable.propTypes = {
columns: React.PropTypes.array,
actions: React.PropTypes.object,
queries: React.PropTypes.array,
onUserClicked: React.PropTypes.func,
onDbClicked: React.PropTypes.func,
};
QueryTable.defaultProps = {
columns: ['started', 'duration', 'rows'],
queries: [],
onUserClicked: () => {},
onDbClicked: () => {},
};
QueryTable.propTypes = propTypes;
QueryTable.defaultProps = defaultProps;
function mapStateToProps() {
return {};
@@ -174,4 +206,5 @@ function mapDispatchToProps(dispatch) {
actions: bindActionCreators(Actions, dispatch),
};
}
export { QueryTable };
export default connect(mapStateToProps, mapDispatchToProps)(QueryTable);

View File

@@ -1,8 +1,32 @@
import React from 'react';
import { Alert, Button, ButtonGroup } from 'react-bootstrap';
import { Alert, Button, ButtonGroup, ProgressBar } from 'react-bootstrap';
import { Table } from 'reactable';
import shortid from 'shortid';
import { connect } from 'react-redux';
import { bindActionCreators } from 'redux';
import * as Actions from '../actions';
import VisualizeModal from './VisualizeModal';
import HighlightedSql from './HighlightedSql';
const propTypes = {
actions: React.PropTypes.object,
csv: React.PropTypes.bool,
query: React.PropTypes.object,
search: React.PropTypes.bool,
searchText: React.PropTypes.string,
showSql: React.PropTypes.bool,
visualize: React.PropTypes.bool,
};
const defaultProps = {
search: true,
visualize: true,
showSql: false,
csv: true,
searchText: '',
actions: {},
};
class ResultSet extends React.Component {
@@ -13,8 +37,65 @@ class ResultSet extends React.Component {
showModal: false,
};
}
changeSearch(event) {
this.setState({ searchText: event.target.value });
getControls() {
if (this.props.search || this.props.visualize || this.props.csv) {
let csvButton;
if (this.props.csv) {
csvButton = (
<Button bsSize="small" href={'/caravel/csv/' + this.props.query.id}>
<i className="fa fa-file-text-o" /> .CSV
</Button>
);
}
let visualizeButton;
if (this.props.visualize) {
visualizeButton = (
<Button
bsSize="small"
onClick={this.showModal.bind(this)}
>
<i className="fa fa-line-chart m-l-1" /> Visualize
</Button>
);
}
let searchBox;
if (this.props.search) {
searchBox = (
<input
type="text"
onChange={this.changeSearch.bind(this)}
className="form-control input-sm"
placeholder="Search Results"
/>
);
}
return (
<div className="ResultSetControls">
<div className="clearfix">
<div className="pull-left">
<ButtonGroup>
{visualizeButton}
{csvButton}
</ButtonGroup>
</div>
<div className="pull-right">
{searchBox}
</div>
</div>
</div>
);
}
return <div className="noControls" />;
}
popSelectStar() {
const qe = {
id: shortid.generate(),
title: this.props.query.tempTable,
autorun: false,
dbId: this.props.query.dbId,
sql: `SELECT * FROM ${this.props.query.tempTable}`,
};
this.props.actions.addQueryEditor(qe);
}
showModal() {
this.setState({ showModal: true });
@@ -22,74 +103,100 @@ class ResultSet extends React.Component {
hideModal() {
this.setState({ showModal: false });
}
changeSearch(event) {
this.setState({ searchText: event.target.value });
}
fetchResults(query) {
this.props.actions.fetchQueryResults(query);
}
render() {
const results = this.props.query.results;
let controls = <div className="noControls" />;
if (this.props.showControls) {
controls = (
<div className="ResultSetControls">
<div className="clearfix">
<div className="pull-left">
<ButtonGroup>
<Button
bsSize="small"
onClick={this.showModal.bind(this)}
>
<i className="fa fa-line-chart m-l-1" /> Visualize
</Button>
<Button bsSize="small" href={'/caravel/csv/' + this.props.query.id}>
<i className="fa fa-file-text-o" /> .CSV
</Button>
</ButtonGroup>
</div>
<div className="pull-right">
<input
type="text"
onChange={this.changeSearch.bind(this)}
className="form-control input-sm"
placeholder="Search Results"
const query = this.props.query;
const results = query.results;
let sql;
if (this.props.showSql) {
sql = <HighlightedSql sql={query.sql} />;
}
if (['running', 'pending', 'fetching'].includes(query.state)) {
let progressBar;
if (query.progress > 0 && query.state === 'running') {
progressBar = (
<ProgressBar
striped
now={query.progress}
label={`${query.progress}%`}
/>);
}
return (
<div>
<img className="loading" alt="Loading..." src="/static/assets/images/loading.gif" />
{progressBar}
</div>
);
} else if (query.state === 'failed') {
return <Alert bsStyle="danger">{query.errorMessage}</Alert>;
} else if (query.state === 'success' && query.ctas) {
return (
<div>
<Alert bsStyle="info">
Table [<strong>{query.tempTable}</strong>] was
created &nbsp;
<Button
bsSize="small"
className="m-r-5"
onClick={this.popSelectStar.bind(this)}
>
Query in a new tab
</Button>
</Alert>
</div>);
} else if (query.state === 'success') {
if (results && results.data && results.data.length > 0) {
return (
<div>
<VisualizeModal
show={this.state.showModal}
query={this.props.query}
onHide={this.hideModal.bind(this)}
/>
{this.getControls.bind(this)()}
{sql}
<div className="ResultSet">
<Table
data={results.data}
columns={results.columns.map((col) => col.name)}
sortable
className="table table-condensed table-bordered"
filterBy={this.state.searchText}
filterable={results.columns.map((c) => c.name)}
hideFilterInput
/>
</div>
</div>
</div>
);
}
if (results && results.data && results.data.length > 0) {
return (
<div>
<VisualizeModal
show={this.state.showModal}
query={this.props.query}
onHide={this.hideModal.bind(this)}
/>
{controls}
<div className="ResultSet">
<Table
data={results.data}
columns={results.columns.map((col) => col.name)}
sortable
className="table table-condensed table-bordered"
filterBy={this.state.searchText}
filterable={results.columns}
hideFilterInput
/>
);
} else if (query.resultsKey) {
return (
<div>
<Alert bsStyle="warning">This query was run asynchronously &nbsp;
<Button bsSize="sm" onClick={this.fetchResults.bind(this, query)}>
Fetch results
</Button>
</Alert>
</div>
</div>
);
);
}
}
return (<Alert bsStyle="warning">The query returned no data</Alert>);
}
}
ResultSet.propTypes = {
query: React.PropTypes.object,
showControls: React.PropTypes.bool,
search: React.PropTypes.bool,
searchText: React.PropTypes.string,
};
ResultSet.defaultProps = {
showControls: true,
search: true,
searchText: '',
};
ResultSet.propTypes = propTypes;
ResultSet.defaultProps = defaultProps;
export default ResultSet;
function mapStateToProps() {
return {};
}
function mapDispatchToProps(dispatch) {
return {
actions: bindActionCreators(Actions, dispatch),
};
}
export default connect(mapStateToProps, mapDispatchToProps)(ResultSet);

View File

@@ -1,4 +1,4 @@
import { Alert, Button, Tab, Tabs } from 'react-bootstrap';
import { Alert, Tab, Tabs } from 'react-bootstrap';
import QueryHistory from './QueryHistory';
import ResultSet from './ResultSet';
import React from 'react';
@@ -8,66 +8,29 @@ import { bindActionCreators } from 'redux';
import * as Actions from '../actions';
import shortid from 'shortid';
class SouthPane extends React.Component {
popSelectStar() {
const qe = {
id: shortid.generate(),
title: this.props.latestQuery.tempTable,
autorun: false,
dbId: this.props.latestQuery.dbId,
sql: `SELECT * FROM ${this.props.latestQuery.tempTable}`,
};
this.props.actions.addQueryEditor(qe);
const SouthPane = function (props) {
let results = <div />;
const latestQuery = props.latestQuery;
if (latestQuery) {
results = <ResultSet showControls search query={latestQuery} />;
} else {
results = <Alert bsStyle="info">Run a query to display results here</Alert>;
}
render() {
let results = <div />;
const latestQuery = this.props.latestQuery;
if (latestQuery) {
if (['running', 'pending'].includes(latestQuery.state)) {
results = (
<img className="loading" alt="Loading.." src="/static/assets/images/loading.gif" />
);
} else if (latestQuery.state === 'failed') {
results = <Alert bsStyle="danger">{latestQuery.errorMessage}</Alert>;
} else if (latestQuery.state === 'success' && latestQuery.ctas) {
results = (
<div>
<Alert bsStyle="info">
Table [<strong>{latestQuery.tempTable}</strong>] was created
</Alert>
<p>
<Button
bsSize="small"
className="m-r-5"
onClick={this.popSelectStar.bind(this)}
>
Query in a new tab
</Button>
<Button bsSize="small">Visualize</Button>
</p>
</div>);
} else if (latestQuery.state === 'success') {
results = <ResultSet showControls search query={latestQuery} />;
}
} else {
results = <Alert bsStyle="info">Run a query to display results here</Alert>;
}
return (
<div className="SouthPane">
<Tabs bsStyle="tabs" id={shortid.generate()}>
<Tab title="Results" eventKey={1}>
<div style={{ overflow: 'auto' }}>
{results}
</div>
</Tab>
<Tab title="Query History" eventKey={2}>
<QueryHistory />
</Tab>
</Tabs>
</div>
);
}
}
return (
<div className="SouthPane">
<Tabs bsStyle="tabs" id={shortid.generate()}>
<Tab title="Results" eventKey={1}>
<div style={{ overflow: 'auto' }}>
{results}
</div>
</Tab>
<Tab title="Query History" eventKey={2}>
<QueryHistory />
</Tab>
</Tabs>
</div>
);
};
SouthPane.propTypes = {
latestQuery: React.PropTypes.object,

View File

@@ -1,5 +1,3 @@
const $ = require('jquery');
import { now } from '../../modules/dates';
import React from 'react';
import {
Button,
@@ -53,64 +51,16 @@ class SqlEditor extends React.Component {
this.startQuery(runAsync);
}
startQuery(runAsync = false, ctas = false) {
const that = this;
const query = {
dbId: this.props.queryEditor.dbId,
id: shortid.generate(),
progress: 0,
sql: this.props.queryEditor.sql,
sqlEditorId: this.props.queryEditor.id,
startDttm: now(),
state: 'running',
tab: this.props.queryEditor.title,
};
if (runAsync) {
query.state = 'pending';
}
// Execute the Query
that.props.actions.startQuery(query);
const sqlJsonUrl = '/caravel/sql_json/';
const sqlJsonRequest = {
client_id: query.id,
database_id: this.props.queryEditor.dbId,
json: true,
tempTableName: this.state.ctas,
runAsync,
schema: this.props.queryEditor.schema,
select_as_cta: ctas,
sql: this.props.queryEditor.sql,
sql_editor_id: this.props.queryEditor.id,
tab: this.props.queryEditor.title,
tmp_table_name: this.state.ctas,
ctas,
};
$.ajax({
type: 'POST',
dataType: 'json',
url: sqlJsonUrl,
data: sqlJsonRequest,
success(results) {
if (!runAsync) {
that.props.actions.querySuccess(query, results);
}
},
error(err, textStatus, errorThrown) {
let msg;
try {
msg = err.responseJSON.error;
} catch (e) {
if (err.responseText !== undefined) {
msg = err.responseText;
}
}
if (textStatus === 'error' && errorThrown === '') {
msg = 'Could not connect to server';
} else if (msg === null) {
msg = `[${textStatus}] ${errorThrown}`;
}
that.props.actions.queryFailed(query, msg);
},
});
this.props.actions.runQuery(query);
}
stopQuery() {
this.props.actions.stopQuery(this.props.latestQuery);
@@ -180,7 +130,7 @@ class SqlEditor extends React.Component {
{runButtons}
</ButtonGroup>
);
if (this.props.latestQuery && this.props.latestQuery.state === 'running') {
if (this.props.latestQuery && ['running', 'pending'].includes(this.props.latestQuery.state)) {
runButtons = (
<ButtonGroup bsSize="small" className="inline m-r-5 pull-left">
<Button

View File

@@ -8,6 +8,18 @@ import shortid from 'shortid';
import { getParamFromQuery, getLink } from '../../../utils/common';
import CopyQueryTabUrl from './CopyQueryTabUrl';
const propTypes = {
actions: React.PropTypes.object,
databases: React.PropTypes.object,
queries: React.PropTypes.object,
queryEditors: React.PropTypes.array,
tabHistory: React.PropTypes.array,
};
const defaultProps = {
tabHistory: [],
queryEditors: [],
};
let queryCount = 1;
class TabbedSqlEditors extends React.Component {
@@ -141,17 +153,8 @@ class TabbedSqlEditors extends React.Component {
);
}
}
TabbedSqlEditors.propTypes = {
actions: React.PropTypes.object,
databases: React.PropTypes.object,
queries: React.PropTypes.object,
queryEditors: React.PropTypes.array,
tabHistory: React.PropTypes.array,
};
TabbedSqlEditors.defaultProps = {
tabHistory: [],
queryEditors: [],
};
TabbedSqlEditors.propTypes = propTypes;
TabbedSqlEditors.defaultProps = defaultProps;
function mapStateToProps(state) {
return {

View File

@@ -1,12 +1,15 @@
import React from 'react';
import { ButtonGroup, Well } from 'react-bootstrap';
import Link from './Link';
import { connect } from 'react-redux';
import { bindActionCreators } from 'redux';
import * as Actions from '../actions';
import { ButtonGroup, Well } from 'react-bootstrap';
import shortid from 'shortid';
import ModalTrigger from '../../components/ModalTrigger';
import { DATA_PREVIEW_ROW_COUNT } from '../common';
import CopyToClipboard from '../../components/CopyToClipboard';
import Link from './Link';
import ModalTrigger from '../../components/ModalTrigger';
const propTypes = {
table: React.PropTypes.object,
@@ -24,7 +27,7 @@ class TableElement extends React.Component {
this.props.actions.queryEditorSetSql(this.props.queryEditor, this.selectStar());
}
selectStar() {
selectStar(useStar = false, limit = 0) {
let cols = '';
this.props.table.columns.forEach((col, i) => {
cols += col.name;
@@ -36,7 +39,16 @@ class TableElement extends React.Component {
if (this.props.table.schema) {
tableName = this.props.table.schema + '.' + tableName;
}
return `SELECT ${cols}\nFROM ${tableName}`;
let sql;
if (useStar) {
sql = `SELECT * FROM ${tableName}`;
} else {
sql = `SELECT ${cols}\nFROM ${tableName}`;
}
if (limit > 0) {
sql += `\nLIMIT ${limit}`;
}
return sql;
}
popSelectStar() {
@@ -63,6 +75,18 @@ class TableElement extends React.Component {
removeTable() {
this.props.actions.removeTable(this.props.table);
}
dataPreviewModal() {
const query = {
dbId: this.props.queryEditor.dbId,
sql: this.selectStar(true, DATA_PREVIEW_ROW_COUNT),
tableName: this.props.table.name,
sqlEditorId: null,
tab: '',
runAsync: false,
ctas: false,
};
this.props.actions.runQuery(query);
}
render() {
const table = this.props.table;
@@ -175,16 +199,18 @@ class TableElement extends React.Component {
<ButtonGroup className="ws-el-controls pull-right">
{keyLink}
<Link
className="fa fa-pencil pull-left m-l-2"
onClick={this.setSelectStar.bind(this)}
tooltip="Run query in this tab"
className="fa fa-search-plus pull-left m-l-2"
onClick={this.dataPreviewModal.bind(this)}
tooltip="Data preview"
href="#"
/>
<Link
className="fa fa-plus-circle pull-left m-l-2"
onClick={this.popSelectStar.bind(this)}
tooltip="Run query in a new tab"
href="#"
<CopyToClipboard
copyNode={
<a className="fa fa-clipboard pull-left m-l-2" />
}
text={this.selectStar()}
shouldShowText={false}
tooltipText="Copy SELECT statement to clipboard"
/>
<Link
className="fa fa-trash pull-left m-l-2"

View File

@@ -26,8 +26,6 @@ class VisualizeModal extends React.Component {
columns: {},
hints: [],
};
// update columns if possible
this.setStateFromProps();
}
componentWillMount() {
this.setStateFromProps();