Improvements to the polygon spatial viz (#6178)

* WIP

* WIP

* WIP

* WIP

* Fix color bucketing

* Fixed colors

* Fix no num categories selected

* Colors working

* Fix no metric selected

* Visual cues for selection

* Add unit tests

* Remove jest from deps

* Rename category to bucket

* Small fixes

* Fix lint

* Fix unit tests

* Remove duplicate hexToRGB

* Fix import

* Change order of arguments in getBuckets

* Refactor function signature
This commit is contained in:
Beto Dealmeida
2018-10-24 18:40:57 -07:00
committed by GitHub
parent ca5be1c1e2
commit f1089c40a4
17 changed files with 2372 additions and 42 deletions

View File

@@ -1,12 +1,29 @@
import { hexToRGB } from '../../../src/modules/colors';
describe('colors', () => {
describe('hexToRGB', () => {
it('is a function', () => {
expect(typeof hexToRGB).toBe('function');
});
it('hexToRGB converts properly', () => {
expect(hexToRGB('#FFFFFF')).toEqual(expect.arrayContaining([255, 255, 255, 255]));
expect(hexToRGB('#000000')).toEqual(expect.arrayContaining([0, 0, 0, 255]));
expect(hexToRGB('#FF0000')).toEqual(expect.arrayContaining([255, 0, 0, 255]));
expect(hexToRGB('#00FF00')).toEqual(expect.arrayContaining([0, 255, 0, 255]));
expect(hexToRGB('#0000FF')).toEqual(expect.arrayContaining([0, 0, 255, 255]));
});
it('works with falsy values', () => {
expect(hexToRGB()).toEqual([0, 0, 0, 255]);
/* eslint-disable quotes */
[false, 0, -0, 0.0, '', "", ``, null, undefined, NaN].forEach((value) => {
expect(hexToRGB(value)).toEqual(expect.arrayContaining([0, 0, 0, 255]));
});
});
it('takes and alpha argument', () => {
expect(hexToRGB('#FF0000', 128)).toEqual(expect.arrayContaining([255, 0, 0, 128]));
expect(hexToRGB('#000000', 100)).toEqual(expect.arrayContaining([0, 0, 0, 100]));
expect(hexToRGB('#ffffff', 0)).toEqual(expect.arrayContaining([255, 255, 255, 0]));
});
});

View File

@@ -0,0 +1,111 @@
import {
getBreakPoints,
getBreakPointColorScaler,
getBuckets,
} from '../../../../src/visualizations/deckgl/utils';
describe('getBreakPoints', () => {
it('is a function', () => {
expect(typeof getBreakPoints).toBe('function');
});
it('returns sorted break points', () => {
const fd = { break_points: ['0', '10', '100', '50', '1000'] };
const result = getBreakPoints(fd);
const expected = ['0', '10', '50', '100', '1000'];
expect(result).toEqual(expected);
});
it('returns evenly distributed break points when no break points are specified', () => {
const fd = { metric: 'count' };
const features = [0, 1, 2, 10].map(count => ({ count }));
const result = getBreakPoints(fd, features);
const expected = ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '10'];
expect(result).toEqual(expected);
});
it('formats number with proper precision', () => {
const fd = { metric: 'count', num_buckets: 2 };
const features = [0, 1 / 3, 2 / 3, 1].map(count => ({ count }));
const result = getBreakPoints(fd, features);
const expected = ['0.0', '0.5', '1.0'];
expect(result).toEqual(expected);
});
it('works with a zero range', () => {
const fd = { metric: 'count', num_buckets: 1 };
const features = [1, 1, 1].map(count => ({ count }));
const result = getBreakPoints(fd, features);
const expected = ['1', '1'];
expect(result).toEqual(expected);
});
});
describe('getBreakPointColorScaler', () => {
it('is a function', () => {
expect(typeof getBreakPointColorScaler).toBe('function');
});
it('returns linear color scaler if there are no break points', () => {
const fd = {
metric: 'count',
linear_color_scheme: ['#000000', '#ffffff'],
opacity: 100,
};
const features = [10, 20, 30].map(count => ({ count }));
const scaler = getBreakPointColorScaler(fd, features);
expect(scaler({ count: 10 })).toEqual([0, 0, 0, 255]);
expect(scaler({ count: 15 })).toEqual([64, 64, 64, 255]);
expect(scaler({ count: 30 })).toEqual([255, 255, 255, 255]);
});
it('returns bucketing scaler if there are break points', () => {
const fd = {
metric: 'count',
linear_color_scheme: ['#000000', '#ffffff'],
break_points: ['0', '1', '10'],
opacity: 100,
};
const features = [];
const scaler = getBreakPointColorScaler(fd, features);
expect(scaler({ count: 0 })).toEqual([0, 0, 0, 255]);
expect(scaler({ count: 0.5 })).toEqual([0, 0, 0, 255]);
expect(scaler({ count: 1 })).toEqual([255, 255, 255, 255]);
expect(scaler({ count: 5 })).toEqual([255, 255, 255, 255]);
});
it('mask values outside the break points', () => {
const fd = {
metric: 'count',
linear_color_scheme: ['#000000', '#ffffff'],
break_points: ['0', '1', '10'],
opacity: 100,
};
const features = [];
const scaler = getBreakPointColorScaler(fd, features);
expect(scaler({ count: -1 })).toEqual([0, 0, 0, 0]);
expect(scaler({ count: 11 })).toEqual([0, 0, 0, 0]);
});
});
describe('getBuckets', () => {
it('is a function', () => {
expect(typeof getBuckets).toBe('function');
});
it('computes buckets for break points', () => {
const fd = {
metric: 'count',
linear_color_scheme: ['#000000', '#ffffff'],
break_points: ['0', '1', '10'],
opacity: 100,
};
const features = [];
const result = getBuckets(fd, features);
const expected = {
'0 - 1': { color: [0, 0, 0, 255], enabled: true },
'1 - 10': { color: [255, 255, 255, 255], enabled: true },
};
expect(result).toEqual(expected);
});
});

View File

@@ -1467,6 +1467,35 @@ export const controls = {
description: t('Send range filter events to other charts'),
},
toggle_polygons: {
type: 'CheckboxControl',
label: t('Multiple filtering'),
renderTrigger: true,
default: true,
description: t('Allow sending multiple polygons as a filter event'),
},
num_buckets: {
type: 'SelectControl',
multi: false,
freeForm: true,
label: t('Number of buckets to group data'),
default: 5,
choices: formatSelectOptions([2, 3, 5, 10]),
description: t('How many buckets should the data be grouped in.'),
renderTrigger: true,
},
break_points: {
type: 'SelectControl',
multi: true,
freeForm: true,
label: t('Bucket break points'),
choices: formatSelectOptions([]),
description: t('List of n+1 values for bucketing metric into n buckets.'),
renderTrigger: true,
},
show_labels: {
type: 'CheckboxControl',
label: t('Show Labels'),

View File

@@ -723,7 +723,7 @@ export const visTypes = {
expanded: true,
controlSetRows: [
['adhoc_filters'],
['metric'],
['metric', 'point_radius_fixed'],
['row_limit', null],
['line_column', 'line_type'],
['reverse_long_lat', 'filter_nulls'],
@@ -743,10 +743,12 @@ export const visTypes = {
controlSetRows: [
['fill_color_picker', 'stroke_color_picker'],
['filled', 'stroked'],
['extruded', null],
['extruded', 'multiplier'],
['line_width', null],
['linear_color_scheme', 'opacity'],
['table_filter', null],
['num_buckets', 'break_points'],
['table_filter', 'toggle_polygons'],
['legend_position', null],
],
},
{
@@ -769,6 +771,10 @@ export const visTypes = {
line_type: {
label: t('Polygon Encoding'),
},
point_radius_fixed: {
label: t('Elevation'),
},
time_grain_sqla: timeGrainSqlaAnimationOverrides,
},
},

View File

@@ -15,7 +15,7 @@ export function hexToRGB(hex, alpha = 255) {
}
export const colorScalerFactory = function (colors, data, accessor, extents, outputRGBA = false) {
// Returns a linear scaler our of an array of color
// Returns a linear scaler out of an array of color
if (!Array.isArray(colors)) {
/* eslint no-param-reassign: 0 */
colors = getSequentialSchemeRegistry().get(colors).colors;

View File

@@ -21,7 +21,7 @@ export function getLayer(fd, payload, onAddFilter, setTooltip) {
getSourceColor: d => d.sourceColor || d.color || [sc.r, sc.g, sc.b, 255 * sc.a],
getTargetColor: d => d.targetColor || d.color || [tc.r, tc.g, tc.b, 255 * tc.a],
strokeWidth: (fd.stroke_width) ? fd.stroke_width : 3,
...commonLayerProps(fd, onAddFilter, setTooltip),
...commonLayerProps(fd, setTooltip),
});
}

View File

@@ -88,7 +88,7 @@ export function getLayer(formData, payload, onAddFilter, setTooltip) {
stroked: fd.stroked,
extruded: fd.extruded,
pointRadiusScale: fd.point_radius_scale,
...commonLayerProps(fd, onAddFilter, setTooltip),
...commonLayerProps(fd, setTooltip),
});
}

View File

@@ -28,7 +28,7 @@ export function getLayer(formData, payload, onAddFilter, setTooltip) {
outline: false,
getElevationValue: points => points.reduce((sum, point) => sum + point.weight, 0),
getColorValue: points => points.reduce((sum, point) => sum + point.weight, 0),
...commonLayerProps(fd, onAddFilter, setTooltip),
...commonLayerProps(fd, setTooltip),
});
}

View File

@@ -28,7 +28,7 @@ export function getLayer(formData, payload, onAddFilter, setTooltip) {
outline: false,
getElevationValue: points => points.reduce((sum, point) => sum + point.weight, 0),
getColorValue: points => points.reduce((sum, point) => sum + point.weight, 0),
...commonLayerProps(fd, onAddFilter, setTooltip),
...commonLayerProps(fd, setTooltip),
});
}

View File

@@ -24,7 +24,7 @@ export function getLayer(formData, payload, onAddFilter, setTooltip) {
data,
rounded: true,
widthScale: 1,
...commonLayerProps(fd, onAddFilter, setTooltip),
...commonLayerProps(fd, setTooltip),
});
}

View File

@@ -1,31 +1,41 @@
import d3 from 'd3';
import { PolygonLayer } from 'deck.gl';
import { flatten } from 'lodash';
import { colorScalerFactory } from '../../../../modules/colors';
import { commonLayerProps } from '../common';
import sandboxedEval from '../../../../modules/sandbox';
import { createDeckGLComponent } from '../../factory';
/* eslint no-underscore-dangle: ["error", { "allow": ["", "__timestamp"] }] */
function getPoints(features) {
return flatten(features.map(d => d.polygon), true);
import React from 'react';
import PropTypes from 'prop-types';
import { PolygonLayer } from 'deck.gl';
import AnimatableDeckGLContainer from '../../AnimatableDeckGLContainer';
import Legend from '../../../Legend';
import { getBuckets, getBreakPointColorScaler } from '../../utils';
import { commonLayerProps } from '../common';
import { getPlaySliderParams } from '../../../../modules/time';
import sandboxedEval from '../../../../modules/sandbox';
const DOUBLE_CLICK_TRESHOLD = 250; // milliseconds
function getElevation(d, colorScaler) {
/* in deck.gl 5.3.4 (used in Superset as of 2018-10-24), if a polygon has
* opacity zero it will make everything behind it have opacity zero,
* effectively showing the map layer no matter what other polygons are
* behind it.
*/
return colorScaler(d)[3] === 0
? 0
: d.elevation;
}
export function getLayer(formData, payload, onAddFilter, setTooltip) {
export function getLayer(formData, payload, setTooltip, selected, onSelect, filters) {
const fd = formData;
const fc = fd.fill_color_picker;
const sc = fd.stroke_color_picker;
let data = [...payload.data.features];
const mainMetric = payload.data.metricLabels.length ? payload.data.metricLabels[0] : null;
let colorScaler;
if (mainMetric) {
const ext = d3.extent(data, d => d[mainMetric]);
const scaler = colorScalerFactory(fd.linear_color_scheme, null, null, ext, true);
colorScaler = (d) => {
const c = scaler(d[mainMetric]);
c[3] = (fd.opacity / 100.0) * 255;
return c;
};
if (filters != null) {
filters.forEach((f) => {
data = data.filter(f);
});
}
if (fd.js_data_mutator) {
@@ -34,18 +44,169 @@ export function getLayer(formData, payload, onAddFilter, setTooltip) {
data = jsFnMutator(data);
}
// base color for the polygons
const baseColorScaler = fd.metric === null
? () => [fc.r, fc.g, fc.b, 255 * fc.a]
: getBreakPointColorScaler(fd, data);
// when polygons are selected, reduce the opacity of non-selected polygons
const colorScaler = (d) => {
const baseColor = baseColorScaler(d);
if (selected.length > 0 && selected.indexOf(d[fd.line_column]) === -1) {
baseColor[3] /= 2;
}
return baseColor;
};
return new PolygonLayer({
id: `path-layer-${fd.slice_id}`,
data,
pickable: true,
filled: fd.filled,
stroked: fd.stroked,
getFillColor: colorScaler || [fc.r, fc.g, fc.b, 255 * fc.a],
getPolygon: d => d.polygon,
getFillColor: colorScaler,
getLineColor: [sc.r, sc.g, sc.b, 255 * sc.a],
getLineWidth: fd.line_width,
extruded: fd.extruded,
getElevation: d => getElevation(d, colorScaler),
elevationScale: fd.multiplier,
fp64: true,
...commonLayerProps(fd, onAddFilter, setTooltip),
...commonLayerProps(fd, setTooltip, onSelect),
});
}
export default createDeckGLComponent(getLayer, getPoints);
const propTypes = {
formData: PropTypes.object.isRequired,
payload: PropTypes.object.isRequired,
setControlValue: PropTypes.func.isRequired,
viewport: PropTypes.object.isRequired,
onAddFilter: PropTypes.func,
setTooltip: PropTypes.func,
};
const defaultProps = {
onAddFilter() {},
setTooltip() {},
};
class DeckGLPolygon extends React.PureComponent {
constructor(props) {
super(props);
const fd = props.formData;
const timeGrain = fd.time_grain_sqla || fd.granularity || 'PT1M';
const timestamps = props.payload.data.features.map(f => f.__timestamp);
const { start, end, getStep, values, disabled } = getPlaySliderParams(timestamps, timeGrain);
this.state = {
start,
end,
getStep,
values,
disabled,
viewport: props.viewport,
selected: [],
lastClick: 0,
};
this.getLayers = this.getLayers.bind(this);
this.onSelect = this.onSelect.bind(this);
this.onValuesChange = this.onValuesChange.bind(this);
this.onViewportChange = this.onViewportChange.bind(this);
}
onSelect(polygon) {
const { formData, onAddFilter } = this.props;
const now = new Date();
const doubleClick = (now - this.state.lastClick) <= DOUBLE_CLICK_TRESHOLD;
// toggle selected polygons
const selected = [...this.state.selected];
if (doubleClick) {
selected.splice(0, selected.length, polygon);
} else if (formData.toggle_polygons) {
const i = selected.indexOf(polygon);
if (i === -1) {
selected.push(polygon);
} else {
selected.splice(i, 1);
}
} else {
selected.splice(0, 1, polygon);
}
this.setState({ selected, lastClick: now });
if (formData.table_filter) {
onAddFilter(formData.line_column, selected, false, true);
}
}
onValuesChange(values) {
this.setState({
values: Array.isArray(values)
? values
: [values, values + this.state.getStep(values)],
});
}
onViewportChange(viewport) {
this.setState({ viewport });
}
getLayers(values) {
if (this.props.payload.data.features === undefined) {
return [];
}
const filters = [];
// time filter
if (values[0] === values[1] || values[1] === this.end) {
filters.push(d => d.__timestamp >= values[0] && d.__timestamp <= values[1]);
} else {
filters.push(d => d.__timestamp >= values[0] && d.__timestamp < values[1]);
}
const layer = getLayer(
this.props.formData,
this.props.payload,
this.props.setTooltip,
this.state.selected,
this.onSelect,
filters);
return [layer];
}
render() {
const { payload, formData, setControlValue } = this.props;
const { start, end, getStep, values, disabled, viewport } = this.state;
const buckets = getBuckets(formData, payload.data.features);
return (
<div style={{ position: 'relative' }}>
<AnimatableDeckGLContainer
getLayers={this.getLayers}
start={start}
end={end}
getStep={getStep}
values={values}
onValuesChange={this.onValuesChange}
disabled={disabled}
viewport={viewport}
onViewportChange={this.onViewportChange}
mapboxApiAccessToken={payload.data.mapboxApiKey}
mapStyle={formData.mapbox_style}
setControlValue={setControlValue}
aggregation
>
{formData.metric !== null &&
<Legend
categories={buckets}
position={formData.legend_position}
/>}
</AnimatableDeckGLContainer>
</div>
);
}
}
DeckGLPolygon.propTypes = propTypes;
DeckGLPolygon.defaultProps = defaultProps;
export default DeckGLPolygon;

View File

@@ -7,7 +7,7 @@ function getPoints(data) {
return data.map(d => d.position);
}
export function getLayer(fd, payload, slice) {
export function getLayer(fd, payload, onAddFilter, setTooltip) {
const dataWithRadius = payload.data.features.map((d) => {
let radius = unitToRadius(fd.point_unit, d.radius) || 10;
if (fd.multiplier) {
@@ -28,7 +28,7 @@ export function getLayer(fd, payload, slice) {
radiusMinPixels: fd.min_radius || null,
radiusMaxPixels: fd.max_radius || null,
outline: false,
...commonLayerProps(fd, slice),
...commonLayerProps(fd, setTooltip),
});
}

View File

@@ -43,7 +43,7 @@ export function getLayer(formData, payload, onAddFilter, setTooltip, filters) {
maxColor: [c.r, c.g, c.b, 255 * c.a],
outline: false,
getWeight: d => d.weight || 0,
...commonLayerProps(fd, onAddFilter, setTooltip),
...commonLayerProps(fd, setTooltip),
});
}

View File

@@ -31,7 +31,7 @@ export function fitViewport(viewport, points, padding = 10) {
}
}
export function commonLayerProps(formData, onAddFilter, setTooltip) {
export function commonLayerProps(formData, setTooltip, onSelect) {
const fd = formData;
let onHover;
let tooltipContentGenerator;
@@ -64,8 +64,8 @@ export function commonLayerProps(formData, onAddFilter, setTooltip) {
const href = sandboxedEval(fd.js_onclick_href)(o);
window.open(href);
};
} else if (fd.table_filter && fd.line_type === 'geohash') {
onClick = o => onAddFilter(fd.line_column, [o.object[fd.line_column]], false);
} else if (fd.table_filter && onSelect !== undefined) {
onClick = o => onSelect(o.object[fd.line_column]);
}
return {
onClick,

View File

@@ -0,0 +1,92 @@
import d3 from 'd3';
import getSequentialSchemeRegistry from '../../modules/colors/SequentialSchemeRegistrySingleton';
import { colorScalerFactory, hexToRGB } from '../../modules/colors';
export function getBreakPoints({
break_points: formDataBreakPoints,
num_buckets: formDataNumBuckets,
metric,
}, features) {
if (formDataBreakPoints === undefined || formDataBreakPoints.length === 0) {
// compute evenly distributed break points based on number of buckets
const numBuckets = formDataNumBuckets
? parseInt(formDataNumBuckets, 10)
: 10;
const [minValue, maxValue] = d3.extent(features, d => d[metric]);
const delta = (maxValue - minValue) / numBuckets;
const precision = delta === 0
? 0
: Math.max(0, Math.ceil(Math.log10(1 / delta)));
return Array(numBuckets + 1)
.fill()
.map((_, i) => (minValue + i * delta).toFixed(precision));
}
return formDataBreakPoints.sort((a, b) => parseFloat(a) - parseFloat(b));
}
export function getBreakPointColorScaler({
break_points: formDataBreakPoints,
num_buckets: formDataNumBuckets,
linear_color_scheme: linearColorScheme,
metric,
opacity,
}, features) {
const breakPoints = formDataBreakPoints || formDataNumBuckets
? getBreakPoints({
break_points: formDataBreakPoints,
num_buckets: formDataNumBuckets,
metric,
}, features)
: null;
const colors = Array.isArray(linearColorScheme)
? linearColorScheme
: getSequentialSchemeRegistry().get(linearColorScheme).colors;
let scaler;
let maskPoint;
if (breakPoints !== null) {
// bucket colors into discrete colors
const colorScaler = colorScalerFactory(colors);
const n = breakPoints.length - 1;
const bucketedColors = n > 1
? [...Array(n).keys()].map(d => colorScaler(d / (n - 1)))
: [colors[colors.length - 1]];
// repeat ends
bucketedColors.unshift(bucketedColors[0]);
bucketedColors.push(bucketedColors[n - 1]);
const points = breakPoints.map(p => parseFloat(p));
scaler = d3.scale.threshold().domain(points).range(bucketedColors);
maskPoint = value => value > breakPoints[n] || value < breakPoints[0];
} else {
// interpolate colors linearly
scaler = colorScalerFactory(colors, features, d => d[metric]);
maskPoint = () => false;
}
return (d) => {
const c = hexToRGB(scaler(d[metric]));
if (maskPoint(d[metric])) {
c[3] = 0;
} else {
c[3] = (opacity / 100.0) * 255;
}
return c;
};
}
export function getBuckets(fd, features) {
const breakPoints = getBreakPoints(fd, features, true);
const colorScaler = getBreakPointColorScaler(fd, features);
const buckets = {};
breakPoints.slice(1).forEach((value, i) => {
const range = breakPoints[i] + ' - ' + breakPoints[i + 1];
const mid = 0.5 * (parseInt(breakPoints[i], 10) + parseInt(breakPoints[i + 1], 10));
buckets[range] = {
color: colorScaler({ [fd.metric]: mid }),
enabled: true,
};
});
return buckets;
}

File diff suppressed because it is too large Load Diff

View File

@@ -2319,6 +2319,7 @@ class DeckPathViz(BaseDeckGLViz):
viz_type = 'deck_path'
verbose_name = _('Deck.gl - Paths')
deck_viz_key = 'path'
is_timeseries = True
deser_map = {
'json': json.loads,
'polyline': polyline.decode,
@@ -2326,8 +2327,11 @@ class DeckPathViz(BaseDeckGLViz):
}
def query_obj(self):
fd = self.form_data
self.is_timeseries = fd.get('time_grain_sqla') or fd.get('granularity')
d = super(DeckPathViz, self).query_obj()
line_col = self.form_data.get('line_column')
self.metric = fd.get('metric')
line_col = fd.get('line_column')
if d['metrics']:
self.has_metrics = True
d['groupby'].append(line_col)
@@ -2347,8 +2351,13 @@ class DeckPathViz(BaseDeckGLViz):
d[self.deck_viz_key] = path
if line_type != 'geohash':
del d[line_column]
d['__timestamp'] = d.get(DTTM_ALIAS) or d.get('__time')
return d
def get_data(self, df):
self.metric_label = self.get_metric_label(self.metric)
return super(DeckPathViz, self).get_data(df)
class DeckPolygon(DeckPathViz):
@@ -2358,6 +2367,26 @@ class DeckPolygon(DeckPathViz):
deck_viz_key = 'polygon'
verbose_name = _('Deck.gl - Polygon')
def query_obj(self):
fd = self.form_data
self.elevation = (
fd.get('point_radius_fixed') or {'type': 'fix', 'value': 500})
return super(DeckPolygon, self).query_obj()
def get_metrics(self):
metrics = [self.form_data.get('metric')]
if self.elevation.get('type') == 'metric':
metrics.append(self.elevation.get('value'))
return [metric for metric in metrics if metric]
def get_properties(self, d):
super(DeckPolygon, self).get_properties(d)
fd = self.form_data
elevation = fd['point_radius_fixed']['value']
type_ = fd['point_radius_fixed']['type']
d['elevation'] = d.get(elevation) if type_ == 'metric' else elevation
return d
class DeckHex(BaseDeckGLViz):