Time Series Annotation Layers (#3521)

* Adding annotations to backend

* Auto fetching Annotations on the backend

* Closing the loop

* Adding missing files

* annotation layers UI

for https://github.com/apache/incubator-superset/issues/3502

* a few fixes per code review.

- add annotation input sanity check before add and before update.
- make SelectAsyncControl component statelesis, and generic
- add annotation description in d3 tool tip
- use less variable to replace hard-coded color

superset/assets/javascripts/components/AsyncSelect.jsx

@@ -10,7 +10,10 @@ const propTypes = {
   onChange: PropTypes.func.isRequired,
   mutator: PropTypes.func.isRequired,
   onAsyncError: PropTypes.func,
-  value: PropTypes.number,
+  value: PropTypes.oneOfType([
+    PropTypes.number,
+    PropTypes.arrayOf(PropTypes.number),
+  ]),
   valueRenderer: PropTypes.func,
   placeholder: PropTypes.string,
   autoSelect: PropTypes.bool,
@@ -63,6 +66,7 @@ class AsyncSelect extends React.PureComponent {
           isLoading={this.state.isLoading}
           onChange={this.onChange.bind(this)}
           valueRenderer={this.props.valueRenderer}
+          {...this.props}
         />
       </div>
     );

superset/assets/javascripts/explore/components/Control.jsx

@@ -3,15 +3,16 @@ import PropTypes from 'prop-types';
 
 import BoundsControl from './controls/BoundsControl';
 import CheckboxControl from './controls/CheckboxControl';
+import ColorSchemeControl from './controls/ColorSchemeControl';
 import DatasourceControl from './controls/DatasourceControl';
 import DateFilterControl from './controls/DateFilterControl';
 import FilterControl from './controls/FilterControl';
 import HiddenControl from './controls/HiddenControl';
+import SelectAsyncControl from './controls/SelectAsyncControl';
 import SelectControl from './controls/SelectControl';
 import TextAreaControl from './controls/TextAreaControl';
 import TextControl from './controls/TextControl';
 import VizTypeControl from './controls/VizTypeControl';
-import ColorSchemeControl from './controls/ColorSchemeControl';
 
 const controlMap = {
   BoundsControl,
@@ -25,6 +26,7 @@ const controlMap = {
   TextControl,
   VizTypeControl,
   ColorSchemeControl,
+  SelectAsyncControl,
 };
 const controlTypes = Object.keys(controlMap);
 

superset/assets/javascripts/explore/components/controls/SelectAsyncControl.jsx

@@ -0,0 +1,53 @@
+/* global notify */
+import React from 'react';
+import PropTypes from 'prop-types';
+import Select from '../../../components/AsyncSelect';
+import { t } from '../../../locales';
+
+const propTypes = {
+  dataEndpoint: PropTypes.string.isRequired,
+  multi: PropTypes.bool,
+  mutator: PropTypes.func,
+  onAsyncErrorMessage: PropTypes.string,
+  onChange: PropTypes.func,
+  placeholder: PropTypes.string,
+  value: PropTypes.oneOfType([
+    PropTypes.string,
+    PropTypes.number,
+    PropTypes.arrayOf(PropTypes.string),
+    PropTypes.arrayOf(PropTypes.number),
+  ]),
+};
+
+const defaultProps = {
+  multi: true,
+  onAsyncErrorMessage: t('Error while fetching data'),
+  onChange: () => {},
+  placeholder: t('Select ...'),
+};
+
+const SelectAsyncControl = ({ value, onChange, dataEndpoint,
+                              multi, mutator, placeholder, onAsyncErrorMessage }) => {
+  const onSelectionChange = (options) => {
+    const optionValues = options.map(option => option.value);
+    onChange(optionValues);
+  };
+
+  return (
+    <Select
+      dataEndpoint={dataEndpoint}
+      onChange={onSelectionChange}
+      onAsyncError={() => notify.error(onAsyncErrorMessage)}
+      mutator={mutator}
+      multi={multi}
+      value={value}
+      placeholder={placeholder}
+      valueRenderer={v => (<div>{v.label}</div>)}
+    />
+  );
+};
+
+SelectAsyncControl.propTypes = propTypes;
+SelectAsyncControl.defaultProps = defaultProps;
+
+export default SelectAsyncControl;

superset/assets/javascripts/explore/main.css

@@ -28,9 +28,12 @@
 }
 
 .control-panel-section {
-  margin-bottom: 0px;
+  margin-bottom: 0;
   box-shadow: none;
 }
+.control-panel-section:last-child {
+  padding-bottom: 40px;
+ }
 
 .control-panel-section .Select-multi-value-wrapper .Select-input > input {
   width: 100px;

superset/assets/javascripts/explore/stores/controls.jsx

@@ -119,6 +119,23 @@ export const controls = {
     }),
   },
 
+  annotation_layers: {
+    type: 'SelectAsyncControl',
+    multi: true,
+    label: t('Annotation Layers'),
+    default: [],
+    description: t('Annotation layers to overlay on the visualization'),
+    dataEndpoint: '/annotationlayermodelview/api/read?',
+    placeholder: t('Select a annotation layer'),
+    onAsyncErrorMessage: t('Error while fetching annotation layers'),
+    mutator: (data) => {
+      if (!data || !data.result) {
+        return [];
+      }
+      return data.result.map(layer => ({ value: layer.id, label: layer.name }));
+    },
+  },
+
   metric: {
     type: 'SelectControl',
     label: t('Metric'),

superset/assets/javascripts/explore/stores/visTypes.js

@@ -44,6 +44,13 @@ export const sections = {
     ],
     description: t('This section exposes ways to include snippets of SQL in your query'),
   },
+  annotations: {
+    label: t('Annotations'),
+    expanded: true,
+    controlSetRows: [
+      ['annotation_layers'],
+    ],
+  },
   NVD3TimeSeries: [
     {
       label: t('Query'),
@@ -177,6 +184,7 @@ export const visTypes = {
         ],
       },
       sections.NVD3TimeSeries[1],
+      sections.annotations,
     ],
     controlOverrides: {
       x_axis_format: {
@@ -209,6 +217,7 @@ export const visTypes = {
           ['metric_2', 'y_axis_2_format'],
         ],
       },
+      sections.annotations,
     ],
     controlOverrides: {
       metric: {
@@ -251,6 +260,7 @@ export const visTypes = {
         ],
       },
       sections.NVD3TimeSeries[1],
+      sections.annotations,
     ],
     controlOverrides: {
       x_axis_format: {
@@ -273,6 +283,7 @@ export const visTypes = {
         ],
       },
       sections.NVD3TimeSeries[1],
+      sections.annotations,
     ],
     controlOverrides: {
       x_axis_format: {
@@ -306,6 +317,7 @@ export const visTypes = {
         ],
       },
       sections.NVD3TimeSeries[1],
+      sections.annotations,
     ],
     controlOverrides: {
       x_axis_format: {

superset/assets/stylesheets/less/index.less

@@ -3,3 +3,5 @@
 @import "~bootstrap/less/bootstrap.less";
 @import "./cosmo/variables.less";
 @import "./cosmo/bootswatch.less";
+
+@stroke-primary:  @brand-primary;

superset/assets/stylesheets/superset.less

@@ -1,3 +1,5 @@
+@import './less/index.less';
+
 body {
     margin: 0px !important;
 }
@@ -368,3 +370,15 @@ iframe {
 .float-right {
   float: right;
 }
+
+g.annotation-container {
+  line {
+    stroke: @stroke-primary;
+  }
+
+  rect.annotation {
+    stroke: @stroke-primary;
+    fill-opacity: 0.1;
+    stroke-width: 1;
+  }
+}

superset/assets/visualizations/nvd3_vis.js

@@ -3,6 +3,7 @@ import $ from 'jquery';
 import throttle from 'lodash.throttle';
 import d3 from 'd3';
 import nv from 'nvd3';
+import d3tip from 'd3-tip';
 
 import { getColorFromScheme } from '../javascripts/modules/colors';
 import { customizeToolTip, d3TimeFormatPreset, d3FormatPreset, tryNumify } from '../javascripts/modules/utils';
@@ -503,6 +504,78 @@ function nvd3Vis(slice, payload) {
       .attr('height', height)
       .attr('width', width)
       .call(chart);
+
+      // add annotation_layer
+      if (isTimeSeries && payload.annotations.length) {
+        const tip = d3tip()
+          .attr('class', 'd3-tip')
+          .direction('n')
+          .offset([-5, 0])
+          .html((d) => {
+            if (!d || !d.layer) {
+              return '';
+            }
+
+            const title = d.short_descr ?
+              d.short_descr + ' - ' + d.layer :
+              d.layer;
+            const body = d.long_descr;
+            return '<div><strong>' + title + '</strong></div><br/>' +
+            '<div>' + body + '</div>';
+          });
+
+        const hh = chart.yAxis.scale().range()[0];
+
+        let annotationLayer;
+        let xScale;
+        let minStep;
+        if (vizType === 'bar') {
+          const xMax = d3.max(payload.data[0].values, d => (d.x));
+          const xMin = d3.min(payload.data[0].values, d => (d.x));
+          minStep = chart.xAxis.range()[1] - chart.xAxis.range()[0];
+          annotationLayer = svg.select('.nv-barsWrap')
+            .insert('g', ':first-child');
+          xScale = d3.scale.quantile()
+            .domain([xMin, xMax])
+            .range(chart.xAxis.range());
+        } else {
+          minStep = 1;
+          annotationLayer = svg.select('.nv-background')
+            .append('g');
+          xScale = chart.xScale();
+        }
+
+        annotationLayer
+          .attr('class', 'annotation-container')
+          .append('defs')
+          .append('pattern')
+          .attr('id', 'diagonal')
+          .attr('patternUnits', 'userSpaceOnUse')
+          .attr('width', 8)
+          .attr('height', 10)
+          .attr('patternTransform', 'rotate(45 50 50)')
+          .append('line')
+          .attr('stroke-width', 7)
+          .attr('y2', 10);
+
+        annotationLayer.selectAll('rect')
+          .data(payload.annotations)
+          .enter()
+          .append('rect')
+          .attr('class', 'annotation')
+          .attr('x', d => (xScale(d.start_dttm)))
+          .attr('y', 0)
+          .attr('width', (d) => {
+            const w = xScale(d.end_dttm) - xScale(d.start_dttm);
+            return w === 0 ? minStep : w;
+          })
+          .attr('height', hh)
+          .attr('fill', 'url(#diagonal)')
+          .on('mouseover', tip.show)
+          .on('mouseout', tip.hide);
+
+        annotationLayer.selectAll('rect').call(tip);
+      }
     }
 
     // on scroll, hide tooltips. throttle to only 4x/second.

superset/migrations/versions/d39b1e37131d_.py

@@ -0,0 +1,22 @@
+"""empty message
+
+Revision ID: d39b1e37131d
+Revises: ('a9c47e2c1547', 'ddd6ebdd853b')
+Create Date: 2017-09-19 15:09:14.292633
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = 'd39b1e37131d'
+down_revision = ('a9c47e2c1547', 'ddd6ebdd853b')
+
+from alembic import op
+import sqlalchemy as sa
+
+
+def upgrade():
+    pass
+
+
+def downgrade():
+    pass

superset/migrations/versions/ddd6ebdd853b_annotations.py

@@ -0,0 +1,56 @@
+"""annotations
+
+Revision ID: ddd6ebdd853b
+Revises: ca69c70ec99b
+Create Date: 2017-09-13 16:36:39.144489
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = 'ddd6ebdd853b'
+down_revision = 'ca69c70ec99b'
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table(
+        'annotation_layer',
+        sa.Column('created_on', sa.DateTime(), nullable=True),
+        sa.Column('changed_on', sa.DateTime(), nullable=True),
+        sa.Column('id', sa.Integer(), nullable=False),
+        sa.Column('name', sa.String(length=250), nullable=True),
+        sa.Column('descr', sa.Text(), nullable=True),
+        sa.Column('changed_by_fk', sa.Integer(), nullable=True),
+        sa.Column('created_by_fk', sa.Integer(), nullable=True),
+        sa.ForeignKeyConstraint(['changed_by_fk'], ['ab_user.id'], ),
+        sa.ForeignKeyConstraint(['created_by_fk'], ['ab_user.id'], ),
+        sa.PrimaryKeyConstraint('id')
+    )
+    op.create_table(
+        'annotation',
+        sa.Column('created_on', sa.DateTime(), nullable=True),
+        sa.Column('changed_on', sa.DateTime(), nullable=True),
+        sa.Column('id', sa.Integer(), nullable=False),
+        sa.Column('start_dttm', sa.DateTime(), nullable=True),
+        sa.Column('end_dttm', sa.DateTime(), nullable=True),
+        sa.Column('layer_id', sa.Integer(), nullable=True),
+        sa.Column('short_descr', sa.String(length=500), nullable=True),
+        sa.Column('long_descr', sa.Text(), nullable=True),
+        sa.Column('changed_by_fk', sa.Integer(), nullable=True),
+        sa.Column('created_by_fk', sa.Integer(), nullable=True),
+        sa.ForeignKeyConstraint(['changed_by_fk'], ['ab_user.id'], ),
+        sa.ForeignKeyConstraint(['created_by_fk'], ['ab_user.id'], ),
+        sa.ForeignKeyConstraint(['layer_id'], [u'annotation_layer.id'], ),
+        sa.PrimaryKeyConstraint('id')
+    )
+    op.create_index(
+        'ti_dag_state',
+        'annotation', ['layer_id', 'start_dttm', 'end_dttm'], unique=False)
+
+
+def downgrade():
+    op.drop_index('ti_dag_state', table_name='annotation')
+    op.drop_table('annotation')
+    op.drop_table('annotation_layer')

superset/migrations/versions/f959a6652acd_.py

@@ -0,0 +1,22 @@
+"""empty message
+
+Revision ID: f959a6652acd
+Revises: ('472d2f73dfd4', 'd39b1e37131d')
+Create Date: 2017-09-24 20:18:35.791707
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = 'f959a6652acd'
+down_revision = ('472d2f73dfd4', 'd39b1e37131d')
+
+from alembic import op
+import sqlalchemy as sa
+
+
+def upgrade():
+    pass
+
+
+def downgrade():
+    pass

superset/models/annotations.py

@@ -0,0 +1,57 @@
+"""a collection of Annotation-related models"""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from sqlalchemy import (
+    Column, Integer, String, ForeignKey, Text,
+    DateTime, Index,
+)
+from sqlalchemy.orm import relationship
+from flask_appbuilder import Model
+
+from superset.models.helpers import AuditMixinNullable
+
+
+class AnnotationLayer(Model, AuditMixinNullable):
+
+    """A logical namespace for a set of annotations"""
+
+    __tablename__ = 'annotation_layer'
+    id = Column(Integer, primary_key=True)
+    name = Column(String(250))
+    descr = Column(Text)
+
+    def __repr__(self):
+        return self.name
+
+
+class Annotation(Model, AuditMixinNullable):
+
+    """Time-related annotation"""
+
+    __tablename__ = 'annotation'
+    id = Column(Integer, primary_key=True)
+    start_dttm = Column(DateTime)
+    end_dttm = Column(DateTime)
+    layer_id = Column(Integer, ForeignKey('annotation_layer.id'))
+    short_descr = Column(String(500))
+    long_descr = Column(Text)
+    layer = relationship(
+        AnnotationLayer,
+        backref='annotation')
+
+    __table_args__ = (
+        Index('ti_dag_state', layer_id, start_dttm, end_dttm),
+    )
+
+    @property
+    def data(self):
+        return {
+            'start_dttm': self.start_dttm,
+            'end_dttm': self.end_dttm,
+            'short_descr': self.short_descr,
+            'long_descr': self.long_descr,
+            'layer': self.layer.name if self.layer else None,
+        }

superset/models/helpers.py

@@ -1,3 +1,9 @@
+"""a collection of model-related helper classes and functions"""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
 from datetime import datetime
 import humanize
 import json

superset/views/__init__.py

@@ -1,3 +1,4 @@
 from . import base  # noqa
 from . import core  # noqa
 from . import sql_lab  # noqa
+from . import annotations # noqa

superset/views/annotations.py

@@ -0,0 +1,59 @@
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from flask_babel import gettext as __
+from flask_appbuilder.models.sqla.interface import SQLAInterface
+
+from superset.models.annotations import Annotation, AnnotationLayer
+from superset import appbuilder
+from .base import SupersetModelView, DeleteMixin
+
+
+class AnnotationModelView(SupersetModelView, DeleteMixin):  # noqa
+    datamodel = SQLAInterface(Annotation)
+    list_columns = ['layer', 'short_descr', 'start_dttm', 'end_dttm']
+    edit_columns = [
+        'layer', 'short_descr', 'long_descr', 'start_dttm', 'end_dttm']
+    add_columns = edit_columns
+
+    def pre_add(self, obj):
+        if not obj.layer:
+            raise Exception("Annotation layer is required.")
+        if not obj.start_dttm and not obj.end_dttm:
+            raise Exception("Annotation start time or end time is required.")
+        elif not obj.start_dttm:
+            obj.start_dttm = obj.end_dttm
+        elif not obj.end_dttm:
+            obj.end_dttm = obj.start_dttm
+        elif obj.end_dttm < obj.start_dttm:
+            raise Exception("Annotation end time must be no earlier than start time.")
+
+    def pre_update(self, obj):
+        self.pre_add(obj)
+
+
+class AnnotationLayerModelView(SupersetModelView, DeleteMixin):
+    datamodel = SQLAInterface(AnnotationLayer)
+    list_columns = ['id', 'name']
+    edit_columns = ['name', 'descr']
+    add_columns = edit_columns
+
+
+appbuilder.add_view(
+    AnnotationLayerModelView,
+    "Annotation Layers",
+    label=__("Annotation Layers"),
+    icon="fa-comment",
+    category="Manage",
+    category_label=__("Manage"),
+    category_icon='')
+appbuilder.add_view(
+    AnnotationModelView,
+    "Annotations",
+    label=__("Annotations"),
+    icon="fa-comments",
+    category="Manage",
+    category_label=__("Manage"),
+    category_icon='')

superset/views/core.py

@@ -17,7 +17,7 @@ import sqlalchemy as sqla
 
 from flask import (
     g, request, redirect, flash, Response, render_template, Markup,
-    abort, url_for)
+    url_for)
 from flask_appbuilder import expose
 from flask_appbuilder.actions import action
 from flask_appbuilder.models.sqla.interface import SQLAInterface
@@ -2367,6 +2367,7 @@ appbuilder.add_view(
     category_label=__("Manage"),
     category_icon='')
 
+
 appbuilder.add_view_no_menu(CssTemplateAsyncModelView)
 
 appbuilder.add_link(

superset/viz.py

@@ -58,6 +58,7 @@ class BaseViz(object):
             'token', 'token_' + uuid.uuid4().hex[:8])
         self.metrics = self.form_data.get('metrics') or []
         self.groupby = self.form_data.get('groupby') or []
+        self.annotation_layers = []
 
         self.status = None
         self.error_message = None
@@ -179,6 +180,10 @@ class BaseViz(object):
         if from_dttm and to_dttm and from_dttm > to_dttm:
             raise Exception(_("From date cannot be larger than to date"))
 
+        self.from_dttm = from_dttm
+        self.to_dttm = to_dttm
+        self.annotation_layers = form_data.get("annotation_layers") or []
+
         # extras are used to query elements specific to a datasource type
         # for instance the extra where clause that applies only to Tables
         extras = {
@@ -238,6 +243,23 @@ class BaseViz(object):
         s = str([(k, self.form_data[k]) for k in sorted(self.form_data.keys())])
         return hashlib.md5(s.encode('utf-8')).hexdigest()
 
+    def get_annotations(self):
+        """Fetches the annotations for the specified layers and date range"""
+        annotations = []
+        if self.annotation_layers:
+            from superset.models.annotations import Annotation
+            from superset import db
+            qry = (
+                db.session
+                .query(Annotation)
+                .filter(Annotation.layer_id.in_(self.annotation_layers)))
+            if self.from_dttm:
+                qry = qry.filter(Annotation.start_dttm >= self.from_dttm)
+            if self.to_dttm:
+                qry = qry.filter(Annotation.end_dttm <= self.to_dttm)
+            annotations = [o.data for o in qry.all()]
+        return annotations
+
     def get_payload(self, force=False):
         """Handles caching around the json payload retrieval"""
         cache_key = self.cache_key
@@ -258,6 +280,7 @@ class BaseViz(object):
                 logging.error("Error reading cache: " +
                               utils.error_msg_from_exception(e))
                 payload = None
+                return []
             logging.info("Serving from cache")
 
         if not payload:
@@ -266,10 +289,12 @@ class BaseViz(object):
             is_cached = False
             cache_timeout = self.cache_timeout
             stacktrace = None
+            annotations = []
             try:
                 df = self.get_df()
                 if not self.error_message:
                     data = self.get_data(df)
+                annotations = self.get_annotations()
             except Exception as e:
                 logging.exception(e)
                 if not self.error_message:
@@ -286,6 +311,7 @@ class BaseViz(object):
                 'query': self.query,
                 'status': self.status,
                 'stacktrace': stacktrace,
+                'annotations': annotations,
             }
             payload['cached_dttm'] = datetime.utcnow().isoformat().split('.')[0]
             logging.info("Caching for the next {} seconds".format(