[geo] provide more flexible Spatial controls

Before this PR the only way to query lat/long is in the shape of 2
columns that contains lat and long.

Now we're adding 2 more options:
* a single column that has lat and long with a delimiter in between
* support for geohashes - geohashes are cool

(cherry picked from commit 3aeefda11fcdf590a7778dc837596e65b41dec09)

README.md

@@ -1,5 +1,5 @@
 Superset
-=========
+========== 
 
 [![Build Status](https://travis-ci.org/apache/incubator-superset.svg?branch=master)](https://travis-ci.org/apache/incubator-superset)
 [![PyPI version](https://badge.fury.io/py/superset.svg)](https://badge.fury.io/py/superset)

setup.py

@@ -58,6 +58,7 @@ setup(
         'flask-wtf==0.14.2',
         'flower==0.9.1',
         'future>=0.16.0, <0.17',
+        'python-geohash==0.8.5',
         'humanize==0.5.1',
         'gunicorn==19.7.1',
         'idna==2.5',

superset/assets/javascripts/components/PopoverSection.jsx

@@ -23,7 +23,7 @@ export default function PopoverSection({ title, isSelected, children, onSelect,
          
         <i className={isSelected ? 'fa fa-check text-primary' : ''} />
       </div>
-      <div>
+      <div className="m-t-5 m-l-5">
         {children}
       </div>
     </div>);

superset/assets/javascripts/explore/components/controls/DateFilterControl.jsx

@@ -91,7 +91,7 @@ export default class DateFilterControl extends React.Component {
   renderPopover() {
     return (
       <Popover id="filter-popover">
-        <div style={{ width: '240px' }}>
+        <div style={{ width: '250px' }}>
           <PopoverSection
             title="Fixed"
             isSelected={this.state.type === 'fix'}

superset/assets/javascripts/explore/components/controls/SelectControl.jsx

@@ -24,6 +24,7 @@ const propTypes = {
   valueRenderer: PropTypes.func,
   valueKey: PropTypes.string,
   options: PropTypes.array,
+  placeholder: PropTypes.string,
 };
 
 const defaultProps = {
@@ -105,10 +106,11 @@ export default class SelectControl extends React.PureComponent {
   }
   render() {
     //  Tab, comma or Enter will trigger a new option created for FreeFormSelect
+    const placeholder = this.props.placeholder || t('Select %s', this.state.options.length);
     const selectProps = {
       multi: this.props.multi,
       name: `select-${this.props.name}`,
-      placeholder: t('Select %s', this.state.options.length),
+      placeholder,
       options: this.state.options,
       value: this.props.value,
       labelKey: 'label',

superset/assets/javascripts/explore/components/controls/SpatialControl.jsx

@@ -0,0 +1,222 @@
+import React from 'react';
+import PropTypes from 'prop-types';
+import {
+  Row, Col, Button, FormControl, Label, OverlayTrigger, Popover,
+} from 'react-bootstrap';
+import 'react-datetime/css/react-datetime.css';
+
+import ControlHeader from '../ControlHeader';
+import SelectControl from './SelectControl';
+import PopoverSection from '../../../components/PopoverSection';
+import Checkbox from '../../../components/Checkbox';
+import { t } from '../../../locales';
+
+const spatialTypes = {
+  latlong: 'latlong',
+  delimited: 'delimited',
+  geohash: 'geohash',
+};
+
+const propTypes = {
+  onChange: PropTypes.func,
+  value: PropTypes.object,
+  animation: PropTypes.bool,
+  choices: PropTypes.array,
+};
+
+const defaultProps = {
+  onChange: () => {},
+  animation: true,
+  choices: [],
+};
+
+export default class SpatialControl extends React.Component {
+  constructor(props) {
+    super(props);
+    const v = props.value || {};
+    let defaultCol;
+    if (props.choices.length > 0) {
+      defaultCol = props.choices[0][0];
+    }
+    this.state = {
+      type: v.type || spatialTypes.latlong,
+      delimiter: v.delimiter || ',',
+      latCol: v.latCol || defaultCol,
+      lonCol: v.lonCol || defaultCol,
+      lonlatCol: v.lonlatCol || defaultCol,
+      reverseCheckbox: v.reverseCheckbox || false,
+      geohashCol: v.geohashCol || defaultCol,
+      value: null,
+      errors: [],
+    };
+    this.onDelimiterChange = this.onDelimiterChange.bind(this);
+    this.toggleCheckbox = this.toggleCheckbox.bind(this);
+    this.onChange = this.onChange.bind(this);
+  }
+  componentDidMount() {
+    this.onChange();
+  }
+  onChange() {
+    const type = this.state.type;
+    const value = { type };
+    const errors = [];
+    const errMsg = t('Invalid lat/long configuration.');
+    if (type === spatialTypes.latlong) {
+      value.latCol = this.state.latCol;
+      value.lonCol = this.state.lonCol;
+      if (!value.lonCol || !value.latCol) {
+        errors.push(errMsg);
+      }
+    } else if (type === spatialTypes.delimited) {
+      value.lonlatCol = this.state.lonlatCol;
+      value.delimiter = this.state.delimiter;
+      value.reverseCheckbox = this.state.reverseCheckbox;
+      if (!value.lonlatCol || !value.delimiter) {
+        errors.push(errMsg);
+      }
+    } else if (type === spatialTypes.geohash) {
+      value.geohashCol = this.state.geohashCol;
+      if (!value.geohashCol) {
+        errors.push(errMsg);
+      }
+    }
+    this.setState({ value, errors });
+    this.props.onChange(value, errors);
+  }
+  onDelimiterChange(event) {
+    this.setState({ delimiter: event.target.value }, this.onChange);
+  }
+  setType(type) {
+    this.setState({ type }, this.onChange);
+  }
+  close() {
+    this.refs.trigger.hide();
+  }
+  toggleCheckbox() {
+    this.setState({ reverseCheckbox: !this.state.reverseCheckbox }, this.onChange);
+  }
+  renderLabelContent() {
+    if (this.state.errors.length > 0) {
+      return 'N/A';
+    }
+    if (this.state.type === spatialTypes.latlong) {
+      return `${this.state.lonCol} | ${this.state.latCol}`;
+    } else if (this.state.type === spatialTypes.delimited) {
+      return `${this.state.lonlatCol}`;
+    } else if (this.state.type === spatialTypes.geohash) {
+      return `${this.state.geohashCol}`;
+    }
+    return null;
+  }
+  renderSelect(name, type) {
+    return (
+      <SelectControl
+        name={name}
+        choices={this.props.choices}
+        value={this.state[name]}
+        clearable={false}
+        onFocus={() => {
+          this.setType(type);
+        }}
+        onChange={(value) => {
+          this.setState({ [name]: value }, this.onChange);
+        }}
+      />
+    );
+  }
+  renderPopover() {
+    return (
+      <Popover id="filter-popover">
+        <div style={{ width: '300px' }}>
+          <PopoverSection
+            title="Longitude & Latitude columns"
+            isSelected={this.state.type === spatialTypes.latlong}
+            onSelect={this.setType.bind(this, spatialTypes.latlong)}
+          >
+            <Row>
+              <Col md={6}>
+                Longitude
+                {this.renderSelect('lonCol', spatialTypes.latlong)}
+              </Col>
+              <Col md={6}>
+                Latitude
+                {this.renderSelect('latCol', spatialTypes.latlong)}
+              </Col>
+            </Row>
+          </PopoverSection>
+          <PopoverSection
+            title="Delimited long & lat single column"
+            isSelected={this.state.type === spatialTypes.delimited}
+            onSelect={this.setType.bind(this, spatialTypes.delimited)}
+          >
+            <Row>
+              <Col md={6}>
+                Column
+                {this.renderSelect('lonlatCol', spatialTypes.delimited)}
+              </Col>
+              <Col md={6}>
+                Delimiter
+                <FormControl
+                  onFocus={this.setType.bind(this, spatialTypes.delimited)}
+                  value={this.state.delimiter}
+                  onChange={this.onDelimiterChange}
+                  placeholder="delimiter"
+                  bsSize="small"
+                />
+              </Col>
+            </Row>
+            <div>
+              {t('Reverse lat/long ')}
+              <Checkbox checked={this.state.reverseCheckbox} onChange={this.toggleCheckbox} />
+            </div>
+          </PopoverSection>
+          <PopoverSection
+            title="Geohash"
+            isSelected={this.state.type === spatialTypes.geohash}
+            onSelect={this.setType.bind(this, spatialTypes.geohash)}
+          >
+            <Row>
+              <Col md={6}>
+                Column
+                {this.renderSelect('geohashCol', spatialTypes.geohash)}
+              </Col>
+            </Row>
+          </PopoverSection>
+          <div className="clearfix">
+            <Button
+              bsSize="small"
+              className="float-left ok"
+              bsStyle="primary"
+              onClick={this.close.bind(this)}
+            >
+              Ok
+            </Button>
+          </div>
+        </div>
+      </Popover>
+    );
+  }
+  render() {
+    return (
+      <div>
+        <ControlHeader {...this.props} />
+        <OverlayTrigger
+          animation={this.props.animation}
+          container={document.body}
+          trigger="click"
+          rootClose
+          ref="trigger"
+          placement="right"
+          overlay={this.renderPopover()}
+        >
+          <Label style={{ cursor: 'pointer' }}>
+            {this.renderLabelContent()}
+          </Label>
+        </OverlayTrigger>
+      </div>
+    );
+  }
+}
+
+SpatialControl.propTypes = propTypes;
+SpatialControl.defaultProps = defaultProps;

superset/assets/javascripts/explore/components/controls/index.js

@@ -10,6 +10,7 @@ import FixedOrMetricControl from './FixedOrMetricControl';
 import HiddenControl from './HiddenControl';
 import SelectAsyncControl from './SelectAsyncControl';
 import SelectControl from './SelectControl';
+import SpatialControl from './SpatialControl';
 import TextAreaControl from './TextAreaControl';
 import TextControl from './TextControl';
 import TimeSeriesColumnControl from './TimeSeriesColumnControl';
@@ -29,6 +30,7 @@ const controlMap = {
   HiddenControl,
   SelectAsyncControl,
   SelectControl,
+  SpatialControl,
   TextAreaControl,
   TextControl,
   TimeSeriesColumnControl,

superset/assets/javascripts/explore/stores/controls.jsx

@@ -482,6 +482,16 @@ export const controls = {
     }),
   },
 
+  spatial: {
+    type: 'SpatialControl',
+    label: t('Longitude & Latitude'),
+    validators: [v.nonEmpty],
+    description: t('Point to your spatial columns'),
+    mapStateToProps: state => ({
+      choices: (state.datasource) ? state.datasource.all_cols : [],
+    }),
+  },
+
   longitude: {
     type: 'SelectControl',
     label: t('Longitude'),

superset/assets/javascripts/explore/stores/visTypes.js

@@ -346,9 +346,8 @@ export const visTypes = {
         label: t('Query'),
         expanded: true,
         controlSetRows: [
-          ['longitude', 'latitude'],
-          ['groupby', 'size'],
-          ['row_limit'],
+          ['spatial', 'size'],
+          ['groupby', 'row_limit'],
         ],
       },
       {
@@ -377,9 +376,8 @@ export const visTypes = {
         label: t('Query'),
         expanded: true,
         controlSetRows: [
-          ['longitude', 'latitude'],
-          ['groupby', 'size'],
-          ['row_limit'],
+          ['spatial', 'size'],
+          ['groupby', 'row_limit'],
         ],
       },
       {
@@ -408,9 +406,8 @@ export const visTypes = {
         label: t('Query'),
         expanded: true,
         controlSetRows: [
-          ['longitude', 'latitude'],
-          ['groupby', 'size'],
-          ['row_limit'],
+          ['spatial', 'size'],
+          ['groupby', 'row_limit'],
         ],
       },
       {
@@ -443,9 +440,8 @@ export const visTypes = {
         label: t('Query'),
         expanded: true,
         controlSetRows: [
-          ['longitude', 'latitude'],
-          ['groupby'],
-          ['row_limit'],
+          ['spatial', 'size'],
+          ['groupby', 'row_limit'],
         ],
       },
       {
@@ -470,14 +466,6 @@ export const visTypes = {
       },
     ],
     controlOverrides: {
-      all_columns_x: {
-        label: t('Longitude Column'),
-        validators: [v.nonEmpty],
-      },
-      all_columns_y: {
-        label: t('Latitude Column'),
-        validators: [v.nonEmpty],
-      },
       dimension: {
         label: t('Categorical Color'),
         description: t('Pick a dimension from which categorical colors are defined'),

superset/assets/package.json

@@ -1,6 +1,6 @@
 {
   "name": "superset",
-  "version": "0.21.0dev",
+  "version": "0.21.0",
   "description": "Superset is a data exploration platform designed to be visual, intuitive, and interactive.",
   "license": "Apache-2.0",
   "directories": {

superset/assets/stylesheets/superset.less

@@ -249,9 +249,15 @@ table.table-no-hover tr:hover {
 .m-t-5 {
     margin-top: 5px;
 }
+.m-t-10 {
+    margin-top: 10px;
+}
 .m-l-5 {
     margin-left: 5px;
 }
+.m-l-10 {
+    margin-left: 10px;
+}
 .m-l-25 {
     margin-left: 25px;
 }

superset/connectors/sqla/models.py

@@ -472,7 +472,7 @@ class SqlaTable(Model, BaseDatasource):
                         # For backwards compatibility and edge cases
                         # where a column data type might have changed
                         if isinstance(v, basestring):
-                            v = v.strip("'").strip('"')
+                            v = v.strip(""").strip(""")
                             if col_obj.is_num:
                                 v = utils.string_to_num(v)
 

superset/data/__init__.py

@@ -13,6 +13,7 @@ import textwrap
 
 import pandas as pd
 from sqlalchemy import BigInteger, Date, DateTime, Float, String
+import geohash
 
 from superset import app, db, utils
 from superset.connectors.connector_registry import ConnectorRegistry
@@ -1017,6 +1018,9 @@ def load_long_lat_data():
     pdf['date'] = datetime.datetime.now().date()
     pdf['occupancy'] = [random.randint(1, 6) for _ in range(len(pdf))]
     pdf['radius_miles'] = [random.uniform(1, 3) for _ in range(len(pdf))]
+    pdf['geohash'] = pdf[['LAT', 'LON']].apply(
+        lambda x: geohash.encode(*x), axis=1)
+    pdf['delimited'] = pdf['LAT'].map(str).str.cat(pdf['LON'].map(str), sep=',')
     pdf.to_sql(  # pylint: disable=no-member
         'long_lat',
         db.engine,
@@ -1036,6 +1040,8 @@ def load_long_lat_data():
             'date': Date(),
             'occupancy': Float(),
             'radius_miles': Float(),
+            'geohash': String(12),
+            'delimited': String(60),
         },
         index=False)
     print("Done loading table!")
@@ -1233,8 +1239,11 @@ def load_deck_dash():
     slices = []
     tbl = db.session.query(TBL).filter_by(table_name='long_lat').first()
     slice_data = {
-        "longitude": "LON",
-        "latitude": "LAT",
+        "spatial": {
+            "type": "latlong",
+            "lonCol": "LON",
+            "latCol": "LAT",
+        },
         "color_picker": {
             "r": 205,
             "g": 0,
@@ -1281,8 +1290,11 @@ def load_deck_dash():
         "point_unit": "square_m",
         "filters": [],
         "row_limit": 5000,
-        "longitude": "LON",
-        "latitude": "LAT",
+        "spatial": {
+            "type": "latlong",
+            "lonCol": "LON",
+            "latCol": "LAT",
+        },
         "mapbox_style": "mapbox://styles/mapbox/dark-v9",
         "granularity_sqla": "date",
         "size": "count",
@@ -1290,10 +1302,12 @@ def load_deck_dash():
         "since": "2014-01-01",
         "point_radius": "Auto",
         "until": "now",
-        "color_picker": {"a": 1,
-        "r": 14,
-        "b": 0,
-        "g": 255},
+        "color_picker": {
+            "a": 1,
+            "r": 14,
+            "b": 0,
+            "g": 255,
+        },
         "grid_size": 20,
         "where": "",
         "having": "",
@@ -1321,10 +1335,13 @@ def load_deck_dash():
     slices.append(slc)
 
     slice_data = {
+        "spatial": {
+            "type": "latlong",
+            "lonCol": "LON",
+            "latCol": "LAT",
+        },
         "filters": [],
         "row_limit": 5000,
-        "longitude": "LON",
-        "latitude": "LAT",
         "mapbox_style": "mapbox://styles/mapbox/streets-v9",
         "granularity_sqla": "date",
         "size": "count",
@@ -1367,10 +1384,13 @@ def load_deck_dash():
     slices.append(slc)
 
     slice_data = {
+        "spatial": {
+            "type": "latlong",
+            "lonCol": "LON",
+            "latCol": "LAT",
+        },
         "filters": [],
         "row_limit": 5000,
-        "longitude": "LON",
-        "latitude": "LAT",
         "mapbox_style": "mapbox://styles/mapbox/satellite-streets-v9",
         "granularity_sqla": "date",
         "size": "count",

superset/migrations/versions/67a6ac9b727b_update_spatial_params.py

@@ -0,0 +1,51 @@
+"""update_spatial_params
+
+Revision ID: 67a6ac9b727b
+Revises: 4736ec66ce19
+Create Date: 2017-12-08 08:19:21.148775
+
+"""
+import json
+
+from alembic import op
+from sqlalchemy.ext.declarative import declarative_base
+from sqlalchemy import Column, Integer, String, Text
+
+from superset import db
+
+# revision identifiers, used by Alembic.
+revision = '67a6ac9b727b'
+down_revision = '4736ec66ce19'
+
+Base = declarative_base()
+
+
+class Slice(Base):
+    __tablename__ = 'slices'
+    id = Column(Integer, primary_key=True)
+    viz_type = Column(String(250))
+    params = Column(Text)
+
+
+def upgrade():
+    bind = op.get_bind()
+    session = db.Session(bind=bind)
+
+    for slc in session.query(Slice).filter(Slice.viz_type.like('deck_%')):
+        params = json.loads(slc.params)
+        if params.get('latitude'):
+            params['spatial'] = {
+                'lonCol': params.get('longitude'),
+                'latCol': params.get('latitude'),
+                'type': 'latlong',
+            }
+            del params['latitude']
+            del params['longitude']
+        slc.params = json.dumps(params)
+        session.merge(slc)
+        session.commit()
+    session.close()
+
+
+def downgrade():
+    pass

superset/templates/appbuilder/general/security/login_oauth.html

@@ -0,0 +1,15 @@
+{% extends "appbuilder/base.html" %}
+
+{% block content %}
+
+<div class="container">
+  <div id="loginbox" style="margin-top:50px;" class="mainbox col-md-6 col-md-offset-3 col-sm-8 col-sm-offset-2">
+    <center>
+      <a href="/login/google">
+        <img width="300" src="https://developers.google.com/accounts/images/sign-in-with-google.png">
+      </a>
+    </center>
+  </div>
+</div>
+
+{% endblock %}

superset/views/core.py

@@ -1049,9 +1049,9 @@ class Superset(BaseSupersetView):
             status=200,
             mimetype='application/json')
 
+    #@has_access_api # remove until we figure out auth
     @log_this
-    @has_access_api
-    @expose('/explore_json/<datasource_type>/<datasource_id>/')
+    @expose("/explore_json/<datasource_type>/<datasource_id>/")
     def explore_json(self, datasource_type, datasource_id):
         try:
             viz_obj = self.get_viz(
@@ -1064,7 +1064,7 @@ class Superset(BaseSupersetView):
                 utils.error_msg_from_exception(e),
                 stacktrace=traceback.format_exc())
 
-        if not self.datasource_access(viz_obj.datasource):
+        if not self.datasource_access(viz_obj.datasource) and request.args.get("restful") != "true":
             return json_error_response(DATASOURCE_ACCESS_ERR, status=404)
 
         if request.args.get('csv') == 'true':
@@ -1840,6 +1840,7 @@ class Superset(BaseSupersetView):
     @expose('/dashboard/<dashboard_id>/')
     def dashboard(self, dashboard_id):
         """Server side rendering for a dashboard"""
+        logging.info("in dashboard")
         session = db.session()
         qry = session.query(models.Dashboard)
         if dashboard_id.isdigit():
@@ -1897,6 +1898,61 @@ class Superset(BaseSupersetView):
             bootstrap_data=json.dumps(bootstrap_data),
         )
 
+    @expose("/dashboard_json/<dashboard_id>/")
+    def dashboard_json(self, dashboard_id):
+        """Server side rendering for a dashboard"""
+        session = db.session()
+        qry = session.query(models.Dashboard)
+        if dashboard_id.isdigit():
+            qry = qry.filter_by(id=int(dashboard_id))
+        else:
+            qry = qry.filter_by(slug=dashboard_id)
+
+        dash = qry.one()
+        datasources = set()
+        for slc in dash.slices:
+            datasource = slc.datasource
+            if datasource:
+                datasources.add(datasource)
+
+        # Commenting until we figure out Authentication from a service
+        # for datasource in datasources:
+        #     if datasource and not self.datasource_access(datasource):
+        #         flash(
+        #             __(get_datasource_access_error_msg(datasource.name)),
+        #             "danger")
+        #         return redirect(
+        #             'superset/request_access/?'
+        #             'dashboard_id={dash.id}&'.format(**locals()))
+
+        # Hack to log the dashboard_id properly, even when getting a slug
+        @log_this
+        def dashboard(**kwargs):  # noqa
+            pass
+        dashboard(dashboard_id=dash.id)
+
+        dash_edit_perm = check_ownership(dash, raise_if_false=False)
+        dash_save_perm = \
+            dash_edit_perm and self.can_access('can_save_dash', 'Superset')
+
+        standalone_mode = request.args.get("standalone") == "true"
+
+        dashboard_data = dash.data
+        dashboard_data.update({
+            'standalone_mode': standalone_mode,
+            'dash_save_perm': dash_save_perm,
+            'dash_edit_perm': dash_edit_perm,
+        })
+
+        bootstrap_data = {
+            'user_id': g.user.get_id(),
+            'dashboard_data': dashboard_data,
+            'datasources': {ds.uid: ds.data for ds in datasources},
+            'common': self.common_bootsrap_payload(),
+        }
+
+        return json_success(json.dumps(bootstrap_data))
+
     @has_access
     @expose('/sync_druid/', methods=['POST'])
     @log_this

superset/viz.py

@@ -22,6 +22,7 @@ import zlib
 from dateutil import relativedelta as rdelta
 from flask import request
 from flask_babel import lazy_gettext as _
+import geohash
 from markdown import markdown
 import numpy as np
 import pandas as pd
@@ -1799,22 +1800,56 @@ class BaseDeckGLViz(BaseViz):
 
     def get_position(self, d):
         return [
-            d.get(self.form_data.get('longitude')),
-            d.get(self.form_data.get('latitude')),
+            d.get('lon'),
+            d.get('lat'),
         ]
 
     def query_obj(self):
         d = super(BaseDeckGLViz, self).query_obj()
         fd = self.form_data
 
-        d['groupby'] = [fd.get('longitude'), fd.get('latitude')]
+        gb = []
+
+        spatial = fd.get('spatial')
+        if spatial.get('type') == 'latlong':
+            gb += [spatial.get('lonCol')]
+            gb += [spatial.get('latCol')]
+        elif spatial.get('type') == 'delimited':
+            gb += [spatial.get('lonlatCol')]
+        elif spatial.get('type') == 'geohash':
+            gb += [spatial.get('geohashCol')]
+
         if fd.get('dimension'):
             d['groupby'] += [fd.get('dimension')]
 
+        d['groupby'] = gb
         d['metrics'] = self.get_metrics()
         return d
 
     def get_data(self, df):
+        fd = self.form_data
+        spatial = fd.get('spatial')
+        if spatial.get('type') == 'latlong':
+            df = df.rename(columns={
+                spatial.get('lonCol'): 'lon',
+                spatial.get('latCol'): 'lat'})
+        elif spatial.get('type') == 'delimited':
+            cols = ['lon', 'lat']
+            if spatial.get('reverseCheckbox'):
+                cols.reverse()
+            df[cols] = (
+                df[spatial.get('lonlatCol')]
+                .str
+                .split(spatial.get('delimiter'), expand=True)
+                .astype(np.float64)
+            )
+            del df[spatial.get('lonlatCol')]
+        elif spatial.get('type') == 'geohash':
+            latlong = df[spatial.get('geohashCol')].map(geohash.decode)
+            df['lat'] = latlong.apply(lambda x: x[0])
+            df['lon'] = latlong.apply(lambda x: x[1])
+            del df['geohash']
+
         features = []
         for d in df.to_dict(orient='records'):
             d = dict(position=self.get_position(d), **self.get_properties(d))