Merge pull request #92 from mistercrunch/nogroupby

Allowing not to group by on table view

TODO.md

@@ -2,13 +2,10 @@
 List of TODO items for Panoramix
 
 ## Improvments
-* GROUPED boolean for table view
 * datasource in explore mode could be a dropdown
 * [sql] make "Test Connection" test further
-* in/notin filters autocomplete
 * [druid] Allow for post aggregations (ratios!)
 * in/notin filters autocomplete
-* [sql] make "Test Connection" test further
 
 ## Better Javascript enables
 * Async on Druidify! in exploration page

panoramix/forms.py

@@ -128,6 +128,10 @@ class FormFactory(object):
                 'Columns',
                 choices=self.choicify(datasource.groupby_column_names),
                 description="One or many fields to pivot as columns"),
+            'all_columns': SelectMultipleSortableField(
+                'Columns',
+                choices=self.choicify(datasource.column_names),
+                description="Columns to display"),
             'granularity': FreeFormSelectField(
                 'Time Granularity', default="one day",
                 choices=self.choicify([
@@ -322,6 +326,10 @@ class FormFactory(object):
                 "Range Filter", default=True,
                 description=(
                     "Whether to display the time range interactive selector")),
+            'include_search': BetterBooleanField(
+                "Search Box", default=False,
+                description=(
+                    "Whether to include a client side search box")),
             'show_bubbles': BetterBooleanField(
                 "Show Bubbles", default=False,
                 description=(
@@ -335,10 +343,14 @@ class FormFactory(object):
                     "Whether to display the min and max values of the axis")),
             'rich_tooltip': BetterBooleanField(
                 "Rich Tooltip", default=True,
-                description="The rich tooltip shows a list of all series for that point in time"),
+                description=(
+                    "The rich tooltip shows a list of all series for that"
+                    " point in time")),
             'y_axis_zero': BetterBooleanField(
                 "Y Axis Zero", default=False,
-                description="Force the Y axis to start at 0 instead of the minimum value"),
+                description=(
+                    "Force the Y axis to start at 0 instead of the minimum "
+                    "value")),
             'y_log_scale': BetterBooleanField(
                 "Y Log", default=False,
                 description="Use a log scale for the Y axis"),

panoramix/models.py

@@ -377,7 +377,8 @@ class SqlaTable(Model, Queryable, AuditMixinNullable):
             is_timeseries=True,
             timeseries_limit=15, row_limit=None,
             inner_from_dttm=None, inner_to_dttm=None,
-            extras=None):
+            extras=None,
+            columns=None):
 
         # For backward compatibility
         if granularity not in self.dttm_cols:
@@ -427,15 +428,20 @@ class SqlaTable(Model, Queryable, AuditMixinNullable):
                 select_exprs.append(outer)
                 inner_groupby_exprs.append(inner)
                 inner_select_exprs.append(inner)
+        elif columns:
+            for s in columns:
+                select_exprs.append(s)
+            metrics_exprs = []
 
-        if is_timeseries:
+        if is_timeseries and groupby:
             select_exprs += [timestamp]
             groupby_exprs += [timestamp]
 
         select_exprs += metrics_exprs
         qry = select(select_exprs)
         from_clause = table(self.table_name)
-        qry = qry.group_by(*groupby_exprs)
+        if groupby:
+            qry = qry.group_by(*groupby_exprs)
 
         time_filter = [
             timestamp >= from_dttm.isoformat(),
@@ -466,7 +472,8 @@ class SqlaTable(Model, Queryable, AuditMixinNullable):
             having_clause_and += [text(extras['having'])]
         qry = qry.where(and_(*(time_filter + where_clause_and)))
         qry = qry.having(and_(*having_clause_and))
-        qry = qry.order_by(desc(main_metric_expr))
+        if groupby:
+            qry = qry.order_by(desc(main_metric_expr))
         qry = qry.limit(row_limit)
 
         if timeseries_limit and groupby:
@@ -492,6 +499,7 @@ class SqlaTable(Model, Queryable, AuditMixinNullable):
             con=engine
         )
         sql = sqlparse.format(sql, reindent=True)
+        print(sql)
         return QueryResult(
             df=df, duration=datetime.now() - qry_start_dttm, query=sql)
 
@@ -779,7 +787,8 @@ class Datasource(Model, AuditMixinNullable, Queryable):
             timeseries_limit=None,
             row_limit=None,
             inner_from_dttm=None, inner_to_dttm=None,
-            extras=None):
+            extras=None,
+            select=None):
         qry_start_dttm = datetime.now()
 
         inner_from_dttm = inner_from_dttm or from_dttm

panoramix/static/panoramix.css

@@ -1,3 +1,9 @@
+html>body{
+    margin: 0px; !important
+}
+.container-fluid {
+    text-align: left;
+}
 input[type="checkbox"] {
   display: inline-block;
   width: 16px;

panoramix/static/widgets/viz_pivot_table.js

@@ -4,26 +4,24 @@ px.registerWidget('pivot_table', function(data_attribute) {
   var form_data = data_attribute.form_data;
 
   function refresh(done) {
-    token.load(data_attribute.json_endpoint, function(response, status, xhr){
-      if(status=="error"){
+    $.getJSON(data_attribute.json_endpoint, function(json){
+      token.html(json.data);
+      if (form_data.groupby.length == 1){
+        var table = token.find('table').DataTable({
+          paging: false,
+          searching: false,
+        });
+        table.column('-1').order( 'desc' ).draw();
+      }
+      token.show();
+      done(json);
+    }).fail(function(xhr){
         var err = '<div class="alert alert-danger">' + xhr.responseText  + '</div>';
         token.html(err);
         token.show();
-      }
-      else{
-        if (form_data.groupby.length == 1){
-          var table = token.find('table').DataTable({
-            paging: false,
-            searching: false,
-          });
-          table.column('-1').order( 'desc' ).draw();
-        }
-      }
-      token.show();
-      done();
+        done();
     });
   }
-
   return {
     render: refresh,
     resize: refresh,

panoramix/static/widgets/viz_table.js

@@ -4,20 +4,56 @@ px.registerWidget('table', function(data_attribute) {
   var token = $('#' + token_name);
 
   function refresh(done) {
-    token.load(data_attribute.json_endpoint, function(response, status, xhr){
-      if(status=="error"){
-        var err = '<div class="alert alert-danger">' + xhr.responseText  + '</div>';
-        token.html(err);
-        token.show();
-        done();
+    $.getJSON(data_attribute.json_endpoint, function(json){
+      var data = json.data;
+      var metrics = json.form_data.metrics;
+      function col(c){
+        arr = [];
+        for (var i=0; i<data.records.length; i++){
+          arr.push(json.data.records[i][c]);
+        }
+        return arr;
       }
-      else{
-        var table = token.find('table').DataTable({
-          paging: false,
-          searching: false,
-        });
-        table.column('-1').order( 'desc' ).draw();
+      maxes = {};
+      for (var i=0; i<metrics.length; i++){
+        maxes[metrics[i]] = d3.max(col(metrics[i]));
       }
+
+      var table = d3.select('#' + token_name).append('table')
+        .attr('class', 'dataframe table table-striped table-bordered table-condensed table-hover');
+      table.append('thead').append('tr')
+       .selectAll('th')
+       .data(data.columns).enter()
+       .append('th')
+       .text(function(d){return d});
+      table.append('tbody')
+       .selectAll('tr')
+       .data(data.records).enter()
+       .append('tr')
+       .selectAll('td')
+       .data(function(row, i) {
+          return data.columns.map(function(c) {return [c, row];});
+       }).enter()
+       .append('td')
+       .style('background-image', function(d){
+          var perc = Math.round((d[1][d[0]] / maxes[d[0]]) * 100);
+          if (perc !== NaN)
+            return "linear-gradient(to right, lightgrey, lightgrey " + perc + "%, rgba(0,0,0,0) " + perc + "%"
+       })
+       .html(function(d){return d[1][d[0]]});
+      var datatable = token.find('table').DataTable({
+        paging: false,
+        searching: data_attribute.form_data.include_search,
+      });
+      // Sorting table by main column
+      if (data_attribute.form_data.metrics.length > 0) {
+        var main_metric = data_attribute.form_data.metrics[0];
+        datatable.column(data.columns.indexOf(main_metric)).order( 'desc' ).draw();
+      }
+      done(json);
+    }).fail(function(xhr){
+      var err = '<div class="alert alert-danger">' + xhr.responseText  + '</div>';
+      token.html(err);
       token.show();
       done();
     });
@@ -25,7 +61,6 @@ px.registerWidget('table', function(data_attribute) {
 
   return {
     render: refresh,
-    resize: refresh,
+    resize: function(){},
   };
-
 });

panoramix/templates/panoramix/viz_pivot_table.html

@@ -1,18 +1,7 @@
 {% macro viz_html(viz) %}
-  {% if viz.request.args.get("async") == "true" %}
-    {{ viz.get_df().to_html(na_rep='', classes="dataframe table table-striped table-bordered table-condensed")|safe }}
-  {% else %}
-    <div id="{{ viz.token }}" style="display: none;overflow: auto; height: 100%;"></div>
-  {% endif %}
 {% endmacro %}
 
 {% macro viz_js(viz) %}
-  {% if viz.form_data.get("async") != "true" %}
-  <script>
-  $( document ).ready(function() {
-  });
-  </script>
-  {% endif %}
 {% endmacro %}
 
 {% macro viz_css(viz) %}

panoramix/templates/panoramix/viz_table.html

@@ -2,7 +2,7 @@
   {% if viz.request.args.get("async") == "true" %}
     {% set df = viz.get_df() %}
     <div class="table">
-      <table class="dataframe table table-striped table-bordered table-condensed">
+      <table class="dataframe table table-striped table-bordered table-condensed table-hover">
         <thead>
           <tr>
             {% for col in df.columns if not col.endswith('__perc') %}

panoramix/viz.py

@@ -242,25 +242,43 @@ class TableViz(BaseViz):
         'fields': (
             'granularity',
             ('since', 'until'),
-            'metrics', 'groupby',
-            'row_limit'
+            'row_limit',
+            ('include_search', None),
+        )
+    },
+    {
+        'label': "GROUP BY",
+        'fields': (
+            'groupby',
+            'metrics',
+        )
+    },
+    {
+        'label': "NOT GROUPED BY",
+        'fields': (
+            'all_columns',
         )
     },)
     css_files = ['lib/dataTables/dataTables.bootstrap.css']
     is_timeseries = False
     js_files = [
+        'lib/d3.min.js',
         'lib/dataTables/jquery.dataTables.min.js',
         'lib/dataTables/dataTables.bootstrap.js',
         'widgets/viz_table.js',
     ]
     css_files = ['widgets/viz_table.css']
 
-    @property
-    def json_endpoint(self):
-        return self.get_url(async='true', standalone='true', skip_libs='true')
-
     def query_obj(self):
         d = super(TableViz, self).query_obj()
+        fd = self.form_data
+        if fd.get('all_columns') and (fd.get('groupby') or fd.get('metrics')):
+            raise Exception(
+                "Choose either fields to [Group By] and [Metrics] or "
+                "[Columns], not both")
+        if fd.get('all_columns'):
+            d['columns'] = fd.get('all_columns')
+            d['groupby'] = []
         d['is_timeseries'] = False
         d['timeseries_limit'] = None
         return d
@@ -271,10 +289,15 @@ class TableViz(BaseViz):
                 self.form_data.get("granularity") == "all" and
                 'timestamp' in df):
             del df['timestamp']
-        for m in self.metrics:
-            df[m + '__perc'] = np.rint((df[m] / np.max(df[m])) * 100)
         return df
 
+    def get_json_data(self):
+        df = self.get_df()
+        return dumps(dict(
+            records=df.to_dict(orient="records"),
+            columns=df.columns,
+        ))
+
 
 class PivotTableViz(BaseViz):
     viz_type = "pivot_table"
@@ -301,10 +324,6 @@ class PivotTableViz(BaseViz):
         )
     },)
 
-    @property
-    def json_endpoint(self):
-        return self.get_url(async='true', standalone='true', skip_libs='true')
-
     def query_obj(self):
         d = super(PivotTableViz, self).query_obj()
         groupby = self.form_data.get('groupby')
@@ -343,6 +362,13 @@ class PivotTableViz(BaseViz):
         )
         return df
 
+    def get_json_data(self):
+        return dumps(self.get_df().to_html(
+            na_rep='',
+            classes=(
+                "dataframe table table-striped table-bordered "
+                "table-condensed table-hover")))
+
 
 class MarkupViz(BaseViz):
     viz_type = "markup"