Using user-defined Javascript to customize geospatial visualization (#4173)

* Using JS to customize spatial viz and tooltips

* Add missing deck_multi.png

* Improve GeoJSON layer with JS support and extra controls

* Addressing comments

superset/assets/javascripts/chart/Chart.jsx

@@ -2,6 +2,7 @@
 import React from 'react';
 import PropTypes from 'prop-types';
 import Mustache from 'mustache';
+import { Tooltip } from 'react-bootstrap';
 
 import { d3format } from '../modules/utils';
 import ChartBody from './ChartBody';
@@ -9,6 +10,7 @@ import Loading from '../components/Loading';
 import StackTraceMessage from '../components/StackTraceMessage';
 import visMap from '../../visualizations/main';
 import sandboxedEval from '../modules/sandbox';
+import './chart.css';
 
 const propTypes = {
   annotationData: PropTypes.object,
@@ -49,6 +51,7 @@ const defaultProps = {
 class Chart extends React.PureComponent {
   constructor(props) {
     super(props);
+    this.state = {};
     // these properties are used by visualizations
     this.annotationData = props.annotationData;
     this.containerId = props.containerId;
@@ -99,6 +102,10 @@ class Chart extends React.PureComponent {
     return this.props.getFilters();
   }
 
+  setTooltip(tooltip) {
+    this.setState({ tooltip });
+  }
+
   addFilter(col, vals, merge = true, refresh = true) {
     this.props.addFilter(col, vals, merge, refresh);
   }
@@ -140,6 +147,26 @@ class Chart extends React.PureComponent {
     return Mustache.render(s, context);
   }
 
+  renderTooltip() {
+    if (this.state.tooltip) {
+      /* eslint-disable react/no-danger */
+      return (
+        <Tooltip
+          className="chart-tooltip"
+          id="chart-tooltip"
+          placement="right"
+          positionTop={this.state.tooltip.y - 10}
+          positionLeft={this.state.tooltip.x + 30}
+          arrowOffsetTop={10}
+        >
+          <div dangerouslySetInnerHTML={{ __html: this.state.tooltip.content }} />
+        </Tooltip>
+      );
+      /* eslint-enable react/no-danger */
+    }
+    return null;
+  }
+
   renderViz() {
     const viz = visMap[this.props.vizType];
     const fd = this.props.formData;
@@ -160,10 +187,10 @@ class Chart extends React.PureComponent {
     const isLoading = this.props.chartStatus === 'loading';
     return (
       <div className={`token col-md-12 ${isLoading ? 'is-loading' : ''}`}>
+        {this.renderTooltip()}
         {isLoading &&
           <Loading size={25} />
         }
-
         {this.props.chartAlert &&
         <StackTraceMessage
           message={this.props.chartAlert}

superset/assets/javascripts/chart/chart.css

@@ -0,0 +1,4 @@
+.chart-tooltip {
+  opacity: 0.75;
+  font-size: 12px;
+}

superset/assets/javascripts/explore/stores/controls.jsx

@@ -46,15 +46,6 @@ const sortAxisChoices = [
   ['value_desc', 'sum(value) descending'],
 ];
 
-const sandboxUrl = 'https://github.com/apache/incubator-superset/blob/master/superset/assets/javascripts/modules/sandbox.js';
-const sandboxedEvalInfo = (
-  <span>
-    {t('While this runs in a ')}
-    <a href="https://nodejs.org/api/vm.html#vm_script_runinnewcontext_sandbox_options">sandboxed vm</a>
-    , {t('a set of')}<a href={sandboxUrl}> useful objects are in context </a>
-    {t('to be used where necessary.')}
-  </span>);
-
 const groupByControl = {
   type: 'SelectControl',
   multi: true,
@@ -77,6 +68,35 @@ const groupByControl = {
   },
 };
 
+const sandboxUrl = (
+  'https://github.com/apache/incubator-superset/' +
+  'blob/master/superset/assets/javascripts/modules/sandbox.js');
+const jsFunctionInfo = (
+  <div>
+    {t('For more information about objects are in context in the scope of this function, refer to the')}
+    <a href={sandboxUrl}>
+      {t(" source code of Superset's sandboxed parser")}.
+    </a>.
+  </div>
+);
+function jsFunctionControl(label, description, extraDescr = null, height = 100, defaultText = '') {
+  return {
+    type: 'TextAreaControl',
+    language: 'javascript',
+    label,
+    description,
+    height,
+    default: defaultText,
+    aboveEditorSection: (
+      <div>
+        <p>{description}</p>
+        <p>{jsFunctionInfo}</p>
+        {extraDescr}
+      </div>
+    ),
+  };
+}
+
 export const controls = {
   datasource: {
     type: 'DatasourceControl',
@@ -1181,14 +1201,14 @@ export const controls = {
     type: 'CheckboxControl',
     label: t('Range Filter'),
     renderTrigger: true,
-    default: false,
+    default: true,
     description: t('Whether to display the time range interactive selector'),
   },
 
   date_filter: {
     type: 'CheckboxControl',
     label: t('Date Filter'),
-    default: false,
+    default: true,
     description: t('Whether to include a time filter'),
   },
 
@@ -1399,7 +1419,7 @@ export const controls = {
       ['mapbox://styles/mapbox/satellite-v9', 'Satellite'],
       ['mapbox://styles/mapbox/outdoors-v9', 'Outdoors'],
     ],
-    default: 'mapbox://styles/mapbox/streets-v9',
+    default: 'mapbox://styles/mapbox/light-v9',
     description: t('Base layer map style'),
   },
 
@@ -1804,20 +1824,6 @@ export const controls = {
     default: false,
   },
 
-  js_data: {
-    type: 'TextAreaControl',
-    label: t('Javascript data mutator'),
-    description: t('Define a function that receives intercepts the data objects and can mutate it'),
-    language: 'javascript',
-    default: '',
-    height: 100,
-    aboveEditorSection: (
-      <p>
-        Define a function that intercepts the <code>data</code> object passed to the visualization
-        and returns a similarly shaped object. {sandboxedEvalInfo}
-      </p>),
-  },
-
   deck_slices: {
     type: 'SelectAsyncControl',
     multi: true,
@@ -1835,5 +1841,49 @@ export const controls = {
       return data.result.map(o => ({ value: o.id, label: o.slice_name }));
     },
   },
+
+  js_datapoint_mutator: jsFunctionControl(
+    t('Javascript data point mutator'),
+    t('Define a javascript function that receives each data point and can alter it ' +
+      'before getting sent to the deck.gl layer'),
+  ),
+
+  js_data: jsFunctionControl(
+    t('Javascript data mutator'),
+    t('Define a function that receives intercepts the data objects and can mutate it'),
+  ),
+
+  js_tooltip: jsFunctionControl(
+    t('Javascript tooltip generator'),
+    t('Define a function that receives the input and outputs the content for a tooltip'),
+  ),
+
+  js_onclick_href: jsFunctionControl(
+    t('Javascript onClick href'),
+    t('Define a function that returns a URL to navigate to when user clicks'),
+  ),
+
+  js_columns: {
+    ...groupByControl,
+    label: t('Extra data for JS'),
+    default: [],
+    description: t('List of extra columns made available in Javascript functions'),
+  },
+
+  stroked: {
+    type: 'CheckboxControl',
+    label: t('Stroked'),
+    renderTrigger: true,
+    description: t('Whether to display the stroke'),
+    default: false,
+  },
+
+  filled: {
+    type: 'CheckboxControl',
+    label: t('Filled'),
+    renderTrigger: true,
+    description: t('Whether to fill the objects'),
+    default: false,
+  },
 };
 export default controls;

superset/assets/javascripts/explore/stores/visTypes.js

@@ -433,6 +433,15 @@ export const visTypes = {
           ['reverse_long_lat', null],
         ],
       },
+      {
+        label: t('Advanced'),
+        controlSetRows: [
+          ['js_columns'],
+          ['js_datapoint_mutator'],
+          ['js_tooltip'],
+          ['js_onclick_href'],
+        ],
+      },
     ],
   },
 
@@ -491,9 +500,20 @@ export const visTypes = {
         label: t('GeoJson Settings'),
         controlSetRows: [
           ['fill_color_picker', 'stroke_color_picker'],
+          ['filled', 'stroked'],
+          ['extruded', null],
           ['point_radius_scale', null],
         ],
       },
+      {
+        label: t('Advanced'),
+        controlSetRows: [
+          ['js_columns'],
+          ['js_datapoint_mutator'],
+          ['js_tooltip'],
+          ['js_onclick_href'],
+        ],
+      },
     ],
   },
 
@@ -529,6 +549,15 @@ export const visTypes = {
           ['dimension', 'color_scheme'],
         ],
       },
+      {
+        label: t('Advanced'),
+        controlSetRows: [
+          ['js_columns'],
+          ['js_datapoint_mutator'],
+          ['js_tooltip'],
+          ['js_onclick_href'],
+        ],
+      },
     ],
     controlOverrides: {
       dimension: {

superset/assets/javascripts/modules/sandbox.js

@@ -1,6 +1,7 @@
 // A safe alternative to JS's eval
 import vm from 'vm';
 import _ from 'underscore';
+import * as colors from './colors';
 
 // Objects exposed here should be treated like a public API
 // if `underscore` had backwards incompatible changes in a future release, we'd
@@ -8,6 +9,7 @@ import _ from 'underscore';
 const GLOBAL_CONTEXT = {
   console,
   _,
+  colors,
 };
 
 // Copied/modified from https://github.com/hacksparrow/safe-eval/blob/master/index.js

superset/assets/package.json

@@ -56,15 +56,16 @@
     "d3-tip": "^0.6.7",
     "datamaps": "^0.5.8",
     "datatables.net-bs": "^1.10.15",
-    "deck.gl": "^4.1.5",
+    "deck.gl": "^5.0.1",
     "distributions": "^1.0.0",
+    "dompurify": "^1.0.3",
     "fastdom": "^1.0.6",
     "geolib": "^2.0.24",
     "immutable": "^3.8.2",
     "jed": "^1.1.1",
     "jquery": "3.1.1",
     "lodash.throttle": "^4.1.1",
-    "luma.gl": "^4.0.5",
+    "luma.gl": "^5.0.1",
     "mathjs": "^3.16.3",
     "moment": "2.18.1",
     "mustache": "^2.2.1",

superset/assets/visualizations/deckgl/factory.jsx

@@ -6,7 +6,7 @@ import layerGenerators from './layers';
 
 export default function deckglFactory(slice, payload, setControlValue) {
   const fd = slice.formData;
-  const layer = layerGenerators[fd.viz_type](fd, payload);
+  const layer = layerGenerators[fd.viz_type](fd, payload, slice);
   const viewport = {
     ...fd.viewport,
     width: slice.width(),

superset/assets/visualizations/deckgl/layers/common.js

@@ -0,0 +1,33 @@
+import dompurify from 'dompurify';
+import sandboxedEval from '../../../javascripts/modules/sandbox';
+
+export function commonLayerProps(formData, slice) {
+  const fd = formData;
+  let onHover;
+  if (fd.js_tooltip) {
+    const jsTooltip = sandboxedEval(fd.js_tooltip);
+    onHover = (o) => {
+      if (o.picked) {
+        slice.setTooltip({
+          content: dompurify.sanitize(jsTooltip(o)),
+          x: o.x,
+          y: o.y,
+        });
+      } else {
+        slice.setTooltip(null);
+      }
+    };
+  }
+  let onClick;
+  if (fd.js_onclick_href) {
+    onClick = (o) => {
+      const href = sandboxedEval(fd.js_onclick_href)(o);
+      window.open(href);
+    };
+  }
+  return {
+    onClick,
+    onHover,
+    pickable: Boolean(onHover),
+  };
+}

superset/assets/visualizations/deckgl/layers/geojson.jsx

@@ -1,6 +1,8 @@
 import { GeoJsonLayer } from 'deck.gl';
-import { hexToRGB } from '../../../javascripts/modules/colors';
 
+import * as common from './common';
+import { hexToRGB } from '../../../javascripts/modules/colors';
+import sandboxedEval from '../../../javascripts/modules/sandbox';
 
 const propertyMap = {
   fillColor: 'fillColor',
@@ -23,11 +25,11 @@ const convertGeoJsonColorProps = (p, colors) => {
   };
 };
 
-export default function geoJsonLayer(formData, payload) {
+export default function geoJsonLayer(formData, payload, slice) {
   const fd = formData;
   const fc = fd.fill_color_picker;
   const sc = fd.stroke_color_picker;
-  const data = payload.data.geojson.features.map(d => ({
+  let data = payload.data.geojson.features.map(d => ({
     ...d,
     properties: convertGeoJsonColorProps(
       d.properties, {
@@ -36,12 +38,19 @@ export default function geoJsonLayer(formData, payload) {
       }),
   }));
 
+  if (fd.js_datapoint_mutator) {
+    // Applying user defined data mutator if defined
+    const jsFnMutator = sandboxedEval(fd.js_datapoint_mutator);
+    data = data.map(jsFnMutator);
+  }
+
   return new GeoJsonLayer({
     id: `path-layer-${fd.slice_id}`,
     data,
-    filled: true,
-    stroked: false,
-    extruded: true,
+    filled: fd.filled,
+    stroked: fd.stroked,
+    extruded: fd.extruded,
     pointRadiusScale: fd.point_radius_scale,
+    ...common.commonLayerProps(fd, slice),
   });
 }

superset/assets/visualizations/deckgl/layers/path.jsx

@@ -1,19 +1,29 @@
 import { PathLayer } from 'deck.gl';
 
-export default function getLayer(formData, payload) {
+import * as common from './common';
+import sandboxedEval from '../../../javascripts/modules/sandbox';
+
+export default function getLayer(formData, payload, slice) {
   const fd = formData;
   const c = fd.color_picker;
   const fixedColor = [c.r, c.g, c.b, 255 * c.a];
-  const data = payload.data.paths.map(path => ({
-    path,
+  let data = payload.data.features.map(feature => ({
+    ...feature,
+    path: feature.path,
     width: fd.line_width,
     color: fixedColor,
   }));
 
+  if (fd.js_datapoint_mutator) {
+    const jsFnMutator = sandboxedEval(fd.js_datapoint_mutator);
+    data = data.map(jsFnMutator);
+  }
+
   return new PathLayer({
     id: `path-layer-${fd.slice_id}`,
     data,
     rounded: true,
     widthScale: 1,
+    ...common.commonLayerProps(fd, slice),
   });
 }

superset/assets/visualizations/deckgl/layers/scatter.jsx

@@ -1,14 +1,16 @@
 import { ScatterplotLayer } from 'deck.gl';
 
+import * as common from './common';
 import { getColorFromScheme, hexToRGB } from '../../../javascripts/modules/colors';
 import { unitToRadius } from '../../../javascripts/modules/geo';
+import sandboxedEval from '../../../javascripts/modules/sandbox';
 
-export default function getLayer(formData, payload) {
+export default function getLayer(formData, payload, slice) {
   const fd = formData;
   const c = fd.color_picker || { r: 0, g: 0, b: 0, a: 1 };
   const fixedColor = [c.r, c.g, c.b, 255 * c.a];
 
-  const data = payload.data.features.map((d) => {
+  let data = payload.data.features.map((d) => {
     let radius = unitToRadius(fd.point_unit, d.radius) || 10;
     if (fd.multiplier) {
       radius *= fd.multiplier;
@@ -25,11 +27,18 @@ export default function getLayer(formData, payload) {
       color,
     };
   });
+
+  if (fd.js_datapoint_mutator) {
+    // Applying user defined data mutator if defined
+    const jsFnMutator = sandboxedEval(fd.js_datapoint_mutator);
+    data = data.map(jsFnMutator);
+  }
+
   return new ScatterplotLayer({
     id: `scatter-layer-${fd.slice_id}`,
     data,
-    pickable: true,
     fp64: true,
     outline: false,
+    ...common.commonLayerProps(fd, slice),
   });
 }

superset/viz.py

@@ -1841,6 +1841,8 @@ class BaseDeckGLViz(BaseViz):
         if fd.get('dimension'):
             gb += [fd.get('dimension')]
 
+        if fd.get('js_columns'):
+            gb += fd.get('js_columns')
         metrics = self.get_metrics()
         if metrics:
             d['groupby'] = gb
@@ -1849,6 +1851,10 @@ class BaseDeckGLViz(BaseViz):
             d['columns'] = gb
         return d
 
+    def get_js_columns(self, d):
+        cols = self.form_data.get('js_columns') or []
+        return {col: d.get(col) for col in cols}
+
     def get_data(self, df):
         fd = self.form_data
         spatial = fd.get('spatial')
@@ -1876,8 +1882,11 @@ class BaseDeckGLViz(BaseViz):
 
         features = []
         for d in df.to_dict(orient='records'):
-            d = dict(position=self.get_position(d), **self.get_properties(d))
-            features.append(d)
+            feature = dict(
+                position=self.get_position(d),
+                props=self.get_js_columns(d),
+                **self.get_properties(d))
+            features.append(feature)
         return {
             'features': features,
             'mapboxApiKey': config.get('MAPBOX_API_KEY'),
@@ -1949,22 +1958,22 @@ class DeckPathViz(BaseDeckGLViz):
 
     def query_obj(self):
         d = super(DeckPathViz, self).query_obj()
-        d['groupby'] = []
-        d['metrics'] = []
-        d['columns'] = [self.form_data.get('line_column')]
+        line_col = self.form_data.get('line_column')
+        if d['metrics']:
+            d['groupby'].append(line_col)
+        else:
+            d['columns'].append(line_col)
         return d
 
-    def get_data(self, df):
+    def get_properties(self, d):
         fd = self.form_data
         deser = self.deser_map[fd.get('line_type')]
-        paths = [deser(s) for s in df[fd.get('line_column')]]
+        path = deser(d[fd.get('line_column')])
         if fd.get('reverse_long_lat'):
-            paths = [[(point[1], point[0]) for point in path] for path in paths]
-        d = {
-            'mapboxApiKey': config.get('MAPBOX_API_KEY'),
-            'paths': paths,
+            path = (path[1], path[0])
+        return {
+            'path': path,
         }
-        return d
 
 
 class DeckHex(BaseDeckGLViz):