Table output

zeppelin-web/src/app/notebook/paragraph/paragraph-graph.html

@@ -22,11 +22,6 @@ limitations under the License.
        ng-show="getGraphMode()==viz.id">
   </div>
 
-  <div ng-if="getGraphMode()=='multiBarChart'"
-       id="p{{paragraph.id}}_multiBarChart">
-    <svg></svg>
-  </div>
-
   <div ng-if="getGraphMode()=='pieChart'"
        id="p{{paragraph.id}}_pieChart">
     <svg></svg>

zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js

@@ -99,24 +99,6 @@
 
     var angularObjectRegistry = {};
 
-    // Controller init
-    $scope.init = function(newParagraph, note) {
-      $scope.paragraph = newParagraph;
-      $scope.parentNote = note;
-      $scope.originalText = angular.copy(newParagraph.text);
-      $scope.chart = {};
-      $scope.colWidthOption = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12];
-      $scope.paragraphFocused = false;
-      if (newParagraph.focus) {
-        $scope.paragraphFocused = true;
-      }
-      if (!$scope.paragraph.config) {
-        $scope.paragraph.config = {};
-      }
-    };
-
-    var angularObjectRegistry = {};
-
     /**
      * Built-in visualizations
      */

zeppelin-web/src/app/tabledata/tabledata.js

@@ -0,0 +1,362 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+'use strict';
+
+var zeppelin = zeppelin || {};
+
+/**
+ * Create table data object from paragraph table type result
+ */
+zeppelin.TableData = function(columns, rows, comment) {
+  this.columns = columns || [];
+  this.rows = rows || [];
+  this.comment = comment || '';
+};
+
+zeppelin.TableData.prototype.loadParagraphResult = function(paragraphResult) {
+  if (!paragraphResult || paragraphResult.type !== 'TABLE') {
+    console.log('Can not load paragraph result');
+    return;
+  }
+
+  var columnNames = [];
+  var rows = [];
+  var array = [];
+  var textRows = paragraphResult.msg.split('\n');
+  var comment = '';
+  var commentRow = false;
+
+  for (var i = 0; i < textRows.length; i++) {
+    var textRow = textRows[i];
+    if (commentRow) {
+      comment += textRow;
+      continue;
+    }
+
+    if (textRow === '') {
+      if (rows.length > 0) {
+        commentRow = true;
+      }
+      continue;
+    }
+    var textCols = textRow.split('\t');
+    var cols = [];
+    var cols2 = [];
+    for (var j = 0; j < textCols.length; j++) {
+      var col = textCols[j];
+      if (i === 0) {
+        columnNames.push({name: col, index: j, aggr: 'sum'});
+      } else {
+        var parsedCol = this.parseTableCell(col);
+        cols.push(parsedCol);
+        cols2.push({key: (columnNames[i]) ? columnNames[i].name : undefined, value: parsedCol});
+      }
+    }
+    if (i !== 0) {
+      rows.push(cols);
+      array.push(cols2);
+    }
+  }
+  this.comment = comment;
+  this.columns = columnNames;
+  this.rows = rows;
+};
+
+zeppelin.TableData.prototype.parseTableCell = function(cell) {
+  if (!isNaN(cell)) {
+    if (cell.length === 0 || Number(cell) > Number.MAX_SAFE_INTEGER || Number(cell) < Number.MIN_SAFE_INTEGER) {
+      return cell;
+    } else {
+      return Number(cell);
+    }
+  }
+  var d = moment(cell);
+  if (d.isValid()) {
+    return d;
+  }
+  return cell;
+};
+
+zeppelin.TableData.prototype.pivot = function(keys, groups, values, allowTextXAxis, fillMissingValues, chartType) {
+  var aggrFunc = {
+    sum: function(a, b) {
+      var varA = (a !== undefined) ? (isNaN(a) ? 1 : parseFloat(a)) : 0;
+      var varB = (b !== undefined) ? (isNaN(b) ? 1 : parseFloat(b)) : 0;
+      return varA + varB;
+    },
+    count: function(a, b) {
+      var varA = (a !== undefined) ? parseInt(a) : 0;
+      var varB = (b !== undefined) ? 1 : 0;
+      return varA + varB;
+    },
+    min: function(a, b) {
+      var varA = (a !== undefined) ? (isNaN(a) ? 1 : parseFloat(a)) : 0;
+      var varB = (b !== undefined) ? (isNaN(b) ? 1 : parseFloat(b)) : 0;
+      return Math.min(varA,varB);
+    },
+    max: function(a, b) {
+      var varA = (a !== undefined) ? (isNaN(a) ? 1 : parseFloat(a)) : 0;
+      var varB = (b !== undefined) ? (isNaN(b) ? 1 : parseFloat(b)) : 0;
+      return Math.max(varA,varB);
+    },
+    avg: function(a, b, c) {
+      var varA = (a !== undefined) ? (isNaN(a) ? 1 : parseFloat(a)) : 0;
+      var varB = (b !== undefined) ? (isNaN(b) ? 1 : parseFloat(b)) : 0;
+      return varA + varB;
+    }
+  };
+
+  var aggrFuncDiv = {
+    sum: false,
+    count: false,
+    min: false,
+    max: false,
+    avg: true
+  };
+
+  var schema = {};
+  var rows = {};
+
+  for (var i = 0; i < this.rows.length; i++) {
+    var row = this.rows[i];
+    var s = schema;
+    var p = rows;
+
+    for (var k = 0; k < keys.length; k++) {
+      var key = keys[k];
+
+      // add key to schema
+      if (!s[key.name]) {
+        s[key.name] = {
+          order: k,
+          index: key.index,
+          type: 'key',
+          children: {}
+        };
+      }
+      s = s[key.name].children;
+
+      // add key to row
+      var keyKey = row[key.index];
+      if (!p[keyKey]) {
+        p[keyKey] = {};
+      }
+      p = p[keyKey];
+    }
+
+    for (var g = 0; g < groups.length; g++) {
+      var group = groups[g];
+      var groupKey = row[group.index];
+
+      // add group to schema
+      if (!s[groupKey]) {
+        s[groupKey] = {
+          order: g,
+          index: group.index,
+          type: 'group',
+          children: {}
+        };
+      }
+      s = s[groupKey].children;
+
+      // add key to row
+      if (!p[groupKey]) {
+        p[groupKey] = {};
+      }
+      p = p[groupKey];
+    }
+
+    for (var v = 0; v < values.length; v++) {
+      var value = values[v];
+      var valueKey = value.name + '(' + value.aggr + ')';
+
+      // add value to schema
+      if (!s[valueKey]) {
+        s[valueKey] = {
+          type: 'value',
+          order: v,
+          index: value.index
+        };
+      }
+
+      // add value to row
+      if (!p[valueKey]) {
+        p[valueKey] = {
+          value: (value.aggr !== 'count') ? row[value.index] : 1,
+          count: 1
+        };
+      } else {
+        p[valueKey] = {
+          value: aggrFunc[value.aggr](p[valueKey].value, row[value.index], p[valueKey].count + 1),
+          count: (aggrFuncDiv[value.aggr]) ?  p[valueKey].count + 1 : p[valueKey].count
+        };
+      }
+    }
+  }
+
+  console.log("schema=%o, rows=%o", schema, rows);
+  return {
+    schema: schema,
+    rows: rows
+  }
+};
+
+zeppelin.TableData.prototype._buildTableDataFromPivot = function(schema, rows, keys, groups, values, values, allowTextXAxis, fillMissingValues, chartType) {
+  // construct table data
+  var d3g = [];
+
+  var concat = function(o, n) {
+    if (!o) {
+      return n;
+    } else {
+      return o + '.' + n;
+    }
+  };
+
+  var getSchemaUnderKey = function(key, s) {
+    for (var c in key.children) {
+      s[c] = {};
+      getSchemaUnderKey(key.children[c], s[c]);
+    }
+  };
+
+  var traverse = function(sKey, s, rKey, r, func, rowName, rowValue, colName) {
+    //console.log("TRAVERSE sKey=%o, s=%o, rKey=%o, r=%o, rowName=%o, rowValue=%o, colName=%o", sKey, s, rKey, r, rowName, rowValue, colName);
+
+    if (s.type === 'key') {
+      rowName = concat(rowName, sKey);
+      rowValue = concat(rowValue, rKey);
+    } else if (s.type === 'group') {
+      colName = concat(colName, rKey);
+    } else if (s.type === 'value' && sKey === rKey || valueOnly) {
+      colName = concat(colName, rKey);
+      func(rowName, rowValue, colName, r);
+    }
+
+    for (var c in s.children) {
+      if (fillMissingValues && s.children[c].type === 'group' && r[c] === undefined) {
+        var cs = {};
+        getSchemaUnderKey(s.children[c], cs);
+        traverse(c, s.children[c], c, cs, func, rowName, rowValue, colName);
+        continue;
+      }
+
+      for (var j in r) {
+        if (s.children[c].type === 'key' || c === j) {
+          traverse(c, s.children[c], j, r[j], func, rowName, rowValue, colName);
+        }
+      }
+    }
+  };
+
+  var valueOnly = (keys.length === 0 && groups.length === 0 && values.length > 0);
+  var noKey = (keys.length === 0);
+  var isMultiBarChart = (chartType === 'multiBarChart');
+
+  var sKey = Object.keys(schema)[0];
+
+  var rowNameIndex = {};
+  var rowIdx = 0;
+  var colNameIndex = {};
+  var colIdx = 0;
+  var rowIndexValue = {};
+
+  for (var k in rows) {
+    traverse(sKey, schema[sKey], k, rows[k], function(rowName, rowValue, colName, value) {
+      //console.log("RowName=%o, row=%o, col=%o, value=%o", rowName, rowValue, colName, value);
+      if (rowNameIndex[rowValue] === undefined) {
+        rowIndexValue[rowIdx] = rowValue;
+        rowNameIndex[rowValue] = rowIdx++;
+      }
+
+      if (colNameIndex[colName] === undefined) {
+        colNameIndex[colName] = colIdx++;
+      }
+      var i = colNameIndex[colName];
+      if (noKey && isMultiBarChart) {
+        i = 0;
+      }
+
+      if (!d3g[i]) {
+        d3g[i] = {
+          values: [],
+          key: (noKey && isMultiBarChart) ? 'values' : colName
+        };
+      }
+
+      var xVar = isNaN(rowValue) ? ((allowTextXAxis) ? rowValue : rowNameIndex[rowValue]) : parseFloat(rowValue);
+      var yVar = 0;
+      if (xVar === undefined) { xVar = colName; }
+      if (value !== undefined) {
+        yVar = isNaN(value.value) ? 0 : parseFloat(value.value) / parseFloat(value.count);
+      }
+      d3g[i].values.push({
+        x: xVar,
+        y: yVar
+      });
+    });
+  }
+
+  // clear aggregation name, if possible
+  var namesWithoutAggr = {};
+  var colName;
+  var withoutAggr;
+  // TODO - This part could use som refactoring - Weird if/else with similar actions and variable names
+  for (colName in colNameIndex) {
+    withoutAggr = colName.substring(0, colName.lastIndexOf('('));
+    if (!namesWithoutAggr[withoutAggr]) {
+      namesWithoutAggr[withoutAggr] = 1;
+    } else {
+      namesWithoutAggr[withoutAggr]++;
+    }
+  }
+
+  if (valueOnly) {
+    for (var valueIndex = 0; valueIndex < d3g[0].values.length; valueIndex++) {
+      colName = d3g[0].values[valueIndex].x;
+      if (!colName) {
+        continue;
+      }
+
+      withoutAggr = colName.substring(0, colName.lastIndexOf('('));
+      if (namesWithoutAggr[withoutAggr] <= 1) {
+        d3g[0].values[valueIndex].x = withoutAggr;
+      }
+    }
+  } else {
+    for (var d3gIndex = 0; d3gIndex < d3g.length; d3gIndex++) {
+      colName = d3g[d3gIndex].key;
+      withoutAggr = colName.substring(0, colName.lastIndexOf('('));
+      if (namesWithoutAggr[withoutAggr] <= 1) {
+        d3g[d3gIndex].key = withoutAggr;
+      }
+    }
+
+    // use group name instead of group.value as a column name, if there're only one group and one value selected.
+    if (groups.length === 1 && values.length === 1) {
+      for (d3gIndex = 0; d3gIndex < d3g.length; d3gIndex++) {
+        colName = d3g[d3gIndex].key;
+        colName = colName.split('.').slice(0, -1).join('.');
+        d3g[d3gIndex].key = colName;
+      }
+    }
+
+  }
+
+  return {
+    xLabels: rowIndexValue,
+    d3g: d3g
+  };
+};

zeppelin-web/src/app/tabledata/transformation.js

@@ -0,0 +1,30 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+'use strict';
+
+var zeppelin = zeppelin || {};
+
+/**
+ * Base class for visualization
+ */
+zeppelin.Transformation = function() {
+};
+
+/**
+ * Method will be invoked when tableData or config changes
+ */
+zeppelin.Transformation.prototype.transform = function(tableData, config) {
+  // override this
+};

zeppelin-web/src/app/visualization/builtins/visualization-barchart.js

@@ -0,0 +1,34 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+'use strict';
+
+/**
+ * Visualize data in table format
+ */
+zeppelin.BarchartVisualization = function(targetEl) {
+  zeppelin.Nvd3ChartVisualization.call(this, targetEl);
+};
+
+zeppelin.BarchartVisualization.prototype = Object.create(zeppelin.Nvd3ChartVisualization.prototype);
+
+zeppelin.BarchartVisualization.prototype.type = function() {
+  return 'multiBarChart';
+};
+
+zeppelin.BarchartVisualization.prototype.configureChart = function(chart) {
+  // override this to configure chart
+  chart.yAxis.axisLabelDistance(50);
+  chart.yAxis.tickFormat(function(d) {return this.yAxisTickFormat(d);});
+};

zeppelin-web/src/app/visualization/builtins/visualization-nvd3chart.js

@@ -0,0 +1,72 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+'use strict';
+
+/**
+ * Visualize data in table format
+ */
+zeppelin.Nvd3ChartVisualization = function(targetEl) {
+  zeppelin.Visualization.call(this, targetEl);
+};
+
+zeppelin.Nvd3ChartVisualization.prototype = Object.create(zeppelin.Visualization.prototype);
+
+zeppelin.Nvd3ChartVisualization.prototype.render = function(tableData) {
+  var type = this.type();
+  console.log('chart type = %o', type);
+
+  if (!this.chart) {
+    this.chart = nv.models[type]();
+  }
+
+  tableData.pivot(['key'],[],['value']);
+
+  this.configureChart(this.chart);
+};
+
+zeppelin.Nvd3ChartVisualization.prototype.type = function() {
+  // override this and return chart type
+};
+
+zeppelin.Nvd3ChartVisualization.prototype.configureChart = function(chart) {
+  // override this to configure chart
+};
+
+zeppelin.Nvd3ChartVisualization.prototype.groupedThousandsWith3DigitsFormatter = function(x) {
+  return d3.format(',')(d3.round(x, 3));
+};
+
+zeppelin.Nvd3ChartVisualization.prototype.customAbbrevFormatter = function(x) {
+  var s = d3.format('.3s')(x);
+  switch (s[s.length - 1]) {
+    case 'G': return s.slice(0, -1) + 'B';
+  }
+  return s;
+};
+
+zeppelin.Nvd3ChartVisualization.prototype.xAxisTickFormat = function(d, xLabels) {
+  if (xLabels[d] && (isNaN(parseFloat(xLabels[d])) || !isFinite(xLabels[d]))) { // to handle string type xlabel
+    return xLabels[d];
+  } else {
+    return d;
+  }
+};
+
+zeppelin.Nvd3ChartVisualization.prototype.yAxisTickFormat = function(d) {
+  if (Math.abs(d) >= Math.pow(10,6)) {
+    return this.customAbbrevFormatter(d);
+  }
+  return this.groupedThousandsWith3DigitsFormatter(d);
+};

zeppelin-web/src/app/visualization/builtins/visualization-table.js

@@ -23,13 +23,13 @@ zeppelin.TableVisualization = function(targetEl) {
   targetEl.addClass('table');
 };
 
-zeppelin.TableVisualization.prototype = Object.create(zeppelin.Visualization);
+zeppelin.TableVisualization.prototype = Object.create(zeppelin.Visualization.prototype);
 
 zeppelin.TableVisualization.prototype.render = function(tableData) {
   var height = this.targetEl.height();
   var container = this.targetEl.css('height', height).get(0);
   var resultRows = tableData.rows;
-  var columnNames = _.pluck(tableData.columnNames, 'name');
+  var columnNames = _.pluck(tableData.columns, 'name');
 
   if (this.hot) {
     this.hot.destroy();

zeppelin-web/src/app/visualization/visualization.js

@@ -14,7 +14,8 @@
 
 'use strict';
 
-var zeppelin = {};
+var zeppelin = zeppelin || {};
+
 /**
  * Base class for visualization
  */

zeppelin-web/src/index.html

@@ -153,8 +153,11 @@ limitations under the License.
     <script src="app/app.js"></script>
     <script src="app/app.controller.js"></script>
     <script src="app/home/home.controller.js"></script>
+    <script src="app/tabledata/tabledata.js"></script>
     <script src="app/visualization/visualization.js"></script>
     <script src="app/visualization/builtins/visualization-table.js"></script>
+    <script src="app/visualization/builtins/visualization-nvd3chart.js"></script>
+    <script src="app/visualization/builtins/visualization-barchart.js"></script>
     <script src="app/notebook/notebook.controller.js"></script>
     <script src="app/jobmanager/jobmanager.controller.js"></script>
     <script src="app/jobmanager/jobs/job.controller.js"></script>

zeppelin-web/test/karma.conf.js

@@ -69,14 +69,15 @@ module.exports = function(config) {
       // endbower
       'src/app/app.js',
       'src/app/app.controller.js',
-      'src/app/visualization/visualization.js',
       'src/app/**/*.js',
       'src/components/**/*.js',
       'test/spec/**/*.js'
     ],
 
     // list of files / patterns to exclude
-    exclude: [],
+    exclude: [
+      'src/app/visualization/builtins/*.js'
+    ],
 
     // web server port
     port: 9002,

zeppelin-web/test/spec/controllers/paragraph.js

@@ -36,7 +36,7 @@ describe('Controller: ParagraphCtrl', function() {
     'closeTable', 'openTable', 'showTitle', 'hideTitle', 'setTitle', 'showLineNumbers', 'hideLineNumbers',
     'changeColWidth', 'columnWidthClass', 'toggleGraphOption', 'toggleOutput', 'loadForm',
     'aceChanged', 'aceLoaded', 'getEditorValue', 'getProgress', 'getExecutionTime', 'isResultOutdated',
-    'getResultType', 'loadTableData', 'setGraphMode', 'isGraphMode', 'onGraphOptionChange',
+    'getResultType', 'setGraphMode', 'isGraphMode', 'onGraphOptionChange',
     'removeGraphOptionKeys', 'removeGraphOptionValues', 'removeGraphOptionGroups', 'setGraphOptionValueAggr',
     'removeScatterOptionXaxis', 'removeScatterOptionYaxis', 'removeScatterOptionGroup',
     'removeScatterOptionSize'];
@@ -64,10 +64,8 @@ describe('Controller: ParagraphCtrl', function() {
       scope.getResultType = jasmine.createSpy('getResultType spy').andCallFake(function() {
         return 'TABLE';
       });
-      spyOn(scope, 'loadTableData');
       spyOn(scope, 'setGraphMode');
       scope.init(paragraphMock);
-      expect(scope.loadTableData).toHaveBeenCalled();
       expect(scope.setGraphMode).toHaveBeenCalled();
       expect(scope.getGraphMode()).toEqual('table');
     });