Merge remote-tracking branch 'upstream/master' into ZEPPELIN-2403

# Conflicts:
#	docs/install/upgrade.md

alluxio/src/main/java/org/apache/zeppelin/alluxio/AlluxioInterpreter.java

@@ -23,9 +23,9 @@ import java.io.PrintStream;
 import java.io.ByteArrayOutputStream;
 import java.util.*;
 
+import org.apache.zeppelin.completer.CompletionType;
 import org.apache.zeppelin.interpreter.Interpreter;
 import org.apache.zeppelin.interpreter.InterpreterContext;
-import org.apache.zeppelin.interpreter.InterpreterPropertyBuilder;
 import org.apache.zeppelin.interpreter.InterpreterResult;
 import org.apache.zeppelin.interpreter.InterpreterResult.Code;
 import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
@@ -166,7 +166,8 @@ public class AlluxioInterpreter extends Interpreter {
   }
 
   @Override
-  public List<InterpreterCompletion> completion(String buf, int cursor) {
+  public List<InterpreterCompletion> completion(String buf, int cursor,
+      InterpreterContext interpreterContext) {
     String[] words = splitAndRemoveEmpty(splitAndRemoveEmpty(buf, "\n"), " ");
     String lastWord = "";
     if (words.length > 0) {
@@ -176,7 +177,7 @@ public class AlluxioInterpreter extends Interpreter {
     List<InterpreterCompletion>  voices = new LinkedList<>();
     for (String command : keywords) {
       if (command.startsWith(lastWord)) {
-        voices.add(new InterpreterCompletion(command, command));
+        voices.add(new InterpreterCompletion(command, command, CompletionType.command.name()));
       }
     }
     return voices;

alluxio/src/test/java/org/apache/zeppelin/alluxio/AlluxioInterpreterTest.java

@@ -29,6 +29,8 @@ import java.util.Properties;
 
 import alluxio.client.WriteType;
 import alluxio.client.file.URIStatus;
+
+import org.apache.zeppelin.completer.CompletionType;
 import org.apache.zeppelin.interpreter.InterpreterResult;
 import org.apache.zeppelin.interpreter.InterpreterResult.Code;
 import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
@@ -78,28 +80,28 @@ public class AlluxioInterpreterTest {
   @Test
   public void testCompletion() {
     List expectedResultOne = Arrays.asList(
-      new InterpreterCompletion("cat", "cat"),
-      new InterpreterCompletion("chgrp", "chgrp"),
-      new InterpreterCompletion("chmod", "chmod"),
-      new InterpreterCompletion("chown", "chown"),
-      new InterpreterCompletion("copyFromLocal", "copyFromLocal"),
-      new InterpreterCompletion("copyToLocal", "copyToLocal"),
-      new InterpreterCompletion("count", "count"),
-      new InterpreterCompletion("createLineage", "createLineage"));
+      new InterpreterCompletion("cat", "cat", CompletionType.command.name()),
+      new InterpreterCompletion("chgrp", "chgrp", CompletionType.command.name()),
+      new InterpreterCompletion("chmod", "chmod", CompletionType.command.name()),
+      new InterpreterCompletion("chown", "chown", CompletionType.command.name()),
+      new InterpreterCompletion("copyFromLocal", "copyFromLocal", CompletionType.command.name()),
+      new InterpreterCompletion("copyToLocal", "copyToLocal", CompletionType.command.name()),
+      new InterpreterCompletion("count", "count", CompletionType.command.name()),
+      new InterpreterCompletion("createLineage", "createLineage", CompletionType.command.name()));
     List expectedResultTwo = Arrays.asList(
-      new InterpreterCompletion("copyFromLocal", "copyFromLocal"),
-      new InterpreterCompletion("copyToLocal", "copyToLocal"),
-      new InterpreterCompletion("count", "count"));
+      new InterpreterCompletion("copyFromLocal", "copyFromLocal", CompletionType.command.name()),
+      new InterpreterCompletion("copyToLocal", "copyToLocal", CompletionType.command.name()),
+      new InterpreterCompletion("count", "count", CompletionType.command.name()));
     List expectedResultThree = Arrays.asList(
-      new InterpreterCompletion("copyFromLocal", "copyFromLocal"),
-      new InterpreterCompletion("copyToLocal", "copyToLocal"));
+      new InterpreterCompletion("copyFromLocal", "copyFromLocal", CompletionType.command.name()),
+      new InterpreterCompletion("copyToLocal", "copyToLocal", CompletionType.command.name()));
     List expectedResultNone = new ArrayList<>();
 
-    List<InterpreterCompletion> resultOne = alluxioInterpreter.completion("c", 0);
-    List<InterpreterCompletion> resultTwo = alluxioInterpreter.completion("co", 0);
-    List<InterpreterCompletion> resultThree = alluxioInterpreter.completion("copy", 0);
-    List<InterpreterCompletion> resultNotMatch = alluxioInterpreter.completion("notMatch", 0);
-    List<InterpreterCompletion> resultAll = alluxioInterpreter.completion("", 0);
+    List<InterpreterCompletion> resultOne = alluxioInterpreter.completion("c", 0, null);
+    List<InterpreterCompletion> resultTwo = alluxioInterpreter.completion("co", 0, null);
+    List<InterpreterCompletion> resultThree = alluxioInterpreter.completion("copy", 0, null);
+    List<InterpreterCompletion> resultNotMatch = alluxioInterpreter.completion("notMatch", 0, null);
+    List<InterpreterCompletion> resultAll = alluxioInterpreter.completion("", 0, null);
 
     Assert.assertEquals(expectedResultOne, resultOne);
     Assert.assertEquals(expectedResultTwo, resultTwo);

angular/src/main/java/org/apache/zeppelin/angular/AngularInterpreter.java

@@ -67,7 +67,8 @@ public class AngularInterpreter extends Interpreter {
   }
 
   @Override
-  public List<InterpreterCompletion> completion(String buf, int cursor) {
+  public List<InterpreterCompletion> completion(String buf, int cursor,
+      InterpreterContext interpreterContext) {
     return new LinkedList<>();
   }
 

beam/src/main/java/org/apache/zeppelin/beam/BeamInterpreter.java

@@ -92,7 +92,8 @@ public class BeamInterpreter extends Interpreter {
   }
 
   @Override
-  public List<InterpreterCompletion> completion(String buf, int cursor) {
+  public List<InterpreterCompletion> completion(String buf, int cursor,
+      InterpreterContext interpreterContext) {
     return Collections.emptyList();
   }
 

bigquery/src/main/java/org/apache/zeppelin/bigquery/BigQueryInterpreter.java

@@ -332,7 +332,8 @@ public class BigQueryInterpreter extends Interpreter {
   }
 
   @Override
-  public List<InterpreterCompletion> completion(String buf, int cursor) {
+  public List<InterpreterCompletion> completion(String buf, int cursor,
+      InterpreterContext interpreterContext) {
     return NO_COMPLETION;
   }
 }

cassandra/src/main/java/org/apache/zeppelin/cassandra/CassandraInterpreter.java

@@ -216,7 +216,8 @@ public class CassandraInterpreter extends Interpreter {
   }
 
   @Override
-  public List<InterpreterCompletion> completion(String buf, int cursor) {
+  public List<InterpreterCompletion> completion(String buf, int cursor,
+      InterpreterContext interpreterContext) {
     return NO_COMPLETION;
   }
 

docs/_includes/themes/zeppelin/_navigation.html

@@ -117,10 +117,11 @@
                 <li><a href="{{BASE_PATH}}/security/notebook_authorization.html">Notebook Authorization</a></li>
                 <li><a href="{{BASE_PATH}}/security/datasource_authorization.html">Data Source Authorization</a></li>
                 <li role="separator" class="divider"></li>
-                <li class="title"><span><b>Helium Framework</b><span></li>
-                <li><a href="{{BASE_PATH}}/development/writingzeppelinapplication.html">Writing Zeppelin Application (Experimental)</a></li>
-                <li><a href="{{BASE_PATH}}/development/writingzeppelinspell.html">Writing Zeppelin Spell (Experimental)</a></li>
-                <li><a href="{{BASE_PATH}}/development/writingzeppelinvisualization.html">Writing Zeppelin Visualization (Experimental)</a></li>
+                <li class="title"><span><b>Helium Framework (Experimental)</b></span></li>
+                <li><a href="{{BASE_PATH}}/development/writingzeppelinapplication.html">Writing Zeppelin Application</a></li>
+                <li><a href="{{BASE_PATH}}/development/writingzeppelinspell.html">Writing Zeppelin Spell</a></li>
+                <li><a href="{{BASE_PATH}}/development/writingzeppelinvisualization.html">Writing Zeppelin Visualization: Basics</a></li>
+                <li><a href="{{BASE_PATH}}/development/writingzeppelinvisualization_transformation.html">Writing Zeppelin Visualization: Transformation</a></li>
                 <li role="separator" class="divider"></li>
                 <li class="title"><span><b>Advanced</b><span></li>
                 <li><a href="{{BASE_PATH}}/install/virtual_machine.html">Zeppelin on Vagrant VM</a></li>

docs/development/writingzeppelinapplication.md

@@ -1,6 +1,6 @@
 ---
 layout: page
-title: "Writing a new Application(Experimental)"
+title: "Writing a new Application"
 description: "Apache Zeppelin Application is a package that runs on Interpreter process and displays it's output inside of the notebook. Make your own Application in Apache Zeppelin is quite easy."
 group: development
 ---
@@ -19,7 +19,7 @@ limitations under the License.
 -->
 {% include JB/setup %}
 
-# Writing a new Application (Experimental)
+# Writing a new Application
 
 <div id="toc"></div>
 

docs/development/writingzeppelinspell.md

@@ -1,6 +1,6 @@
 ---
 layout: page
-title: "Writing a new Spell(Experimental)"
+title: "Writing a new Spell"
 description: "Spell is a kind of interpreter that runs on browser not on backend. So, technically it's the frontend interpreter. "
 group: development
 ---
@@ -19,7 +19,7 @@ limitations under the License.
 -->
 {% include JB/setup %}
 
-# Writing a new Spell (Experimental)
+# Writing a new Spell
 
 <div id="toc"></div>
 

docs/development/writingzeppelinvisualization.md

@@ -1,6 +1,6 @@
 ---
 layout: page
-title: "Writing a new Visualization(Experimental)"
+title: "Writing a new Visualization"
 description: "Apache Zeppelin Visualization is a pluggable package that can be loaded/unloaded on runtime through Helium framework in Zeppelin. A Visualization is a javascript npm package and user can use them just like any other built-in visualization in a note."
 group: development
 ---
@@ -19,7 +19,7 @@ limitations under the License.
 -->
 {% include JB/setup %}
 
-# Writing a new Visualization (Experimental)
+# Writing a new Visualization
 
 <div id="toc"></div>
 

docs/development/writingzeppelinvisualization_transformation.md

@@ -0,0 +1,281 @@
+---
+layout: page
+title: "Transformations for Zeppelin Visualization"
+description: "Description for Transformations"
+group: development
+---
+<!--
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+{% include JB/setup %}
+
+# Transformations for Zeppelin Visualization
+
+<div id="toc"></div>
+
+## Overview 
+
+Transformations 
+
+- **renders** setting which allows users to set columns and 
+- **transforms** table rows according to the configured columns.
+
+Zeppelin provides 4 types of transformations.
+
+## 1. PassthroughTransformation
+
+`PassthroughTransformation` is the simple transformation which does not convert original tabledata at all.
+
+See [passthrough.js](https://github.com/apache/zeppelin/blob/master/zeppelin-web/src/app/tabledata/passthrough.js)
+
+## 2. ColumnselectorTransformation
+
+`ColumnselectorTransformation` is uses when you need `N` axes but do not need aggregation. 
+
+See [columnselector.js](https://github.com/apache/zeppelin/blob/master/zeppelin-web/src/app/tabledata/columnselector.js)
+
+## 3. PivotTransformation
+
+`PivotTransformation` provides group by and aggregation. Every chart using `PivotTransformation` has 3 axes. `Keys`, `Groups` and `Values`.
+
+See [pivot.js](https://github.com/apache/zeppelin/blob/master/zeppelin-web/src/app/tabledata/pivot.js)
+
+## 4. AdvancedTransformation
+
+`AdvancedTransformation` has more detailed options while providing existing features of `PivotTransformation` and `ColumnselectorTransformation`
+
+- multiple sub charts
+- configurable chart axes
+- parameter widgets: `input`, `checkbox`, `option`, `textarea`
+- parsing parameters automatically based on their types
+- expand / fold axis and parameter panels
+- multiple transformation methods while supporting lazy converting 
+- re-initialize the whole configuration based on spec hash.
+
+### Spec 
+
+`AdvancedTransformation` requires `spec` which includes axis and parameter details for charts.
+
+Let's create 2 sub-charts called `line` and `no-group`. Each sub chart can have different axis and parameter depending on their requirements.
+
+<br/>
+
+```js
+class AwesomeVisualization extends Visualization {
+  constructor(targetEl, config) {
+    super(targetEl, config)
+  
+    const spec = {
+      charts: {
+        'line': {
+          transform: { method: 'object', },
+          sharedAxis: false, /** set if you want to share axes between sub charts, default is `false` */
+          axis: {
+            'xAxis': { dimension: 'multiple', axisType: 'key', description: 'serial', },
+            'yAxis': { dimension: 'multiple', axisType: 'aggregator', description: 'serial', },
+            'category': { dimension: 'multiple', axisType: 'group', description: 'categorical', },
+          },
+          parameter: {
+            'xAxisUnit': { valueType: 'string', defaultValue: '', description: 'unit of xAxis', },
+            'yAxisUnit': { valueType: 'string', defaultValue: '', description: 'unit of yAxis', },
+            'lineWidth': { valueType: 'int', defaultValue: 0, description: 'width of line', },
+          },
+        },
+  
+        'no-group': {
+          transform: { method: 'object', },
+          sharedAxis: false,
+          axis: {
+            'xAxis': { dimension: 'single', axisType: 'key', },
+            'yAxis': { dimension: 'multiple', axisType: 'value', },
+          },
+          parameter: {
+            'xAxisUnit': { valueType: 'string', defaultValue: '', description: 'unit of xAxis', },
+            'yAxisUnit': { valueType: 'string', defaultValue: '', description: 'unit of yAxis', },
+        },
+      },
+    }
+
+    this.transformation = new AdvancedTransformation(config, spec)
+  }
+  
+  ...
+  
+  // `render` will be called whenever `axis` or `parameter` is changed 
+  render(data) {
+    const { chart, parameter, column, transformer, } = data
+   
+    if (chart === 'line') {
+      const transformed = transformer()
+      // draw line chart 
+    } else if (chart === 'no-group') {
+      const transformed = transformer()
+      // draw no-group chart 
+    }
+  }
+}
+```
+
+<br/>
+
+### Spec: `axis` 
+
+| Field Name | Available Values (type) | Description |
+| --- | --- | --- |
+|`dimension` | `single` | Axis can contains only 1 column |
+|`dimension` | `multiple` | Axis can contains multiple columns |
+|`axisType` | `key` | Column(s) in this axis will be used as `key` like in `PivotTransformation`. These columns will be served in `column.key` |
+|`axisType` | `aggregator` | Column(s) in this axis will be used as `value` like in `PivotTransformation`. These columns will be served in `column.aggregator` |
+|`axisType` | `group` | Column(s) in this axis will be used as `group` like in `PivotTransformation`. These columns will be served in `column.group` |
+|`axisType` | (string) | Any string value can be used here. These columns will be served in `column.custom` |
+|`maxAxisCount` (optional) | (int) | The max number of columns that this axis can contain. (unlimited if `undefined`) |
+|`minAxisCount` (optional) | (int) | The min number of columns that this axis should contain to draw chart. (`1` in case of single dimension) |
+|`description` (optional) | (string) | Description for the axis. |
+
+<br/>
+
+Here is an example.
+
+```js
+axis: {
+  'xAxis': { dimension: 'multiple', axisType: 'key',  },
+  'yAxis': { dimension: 'multiple', axisType: 'aggregator'},
+  'category': { dimension: 'multiple', axisType: 'group', maxAxisCount: 2, valueType: 'string', },
+},
+```
+
+<br/>
+
+### Spec: `sharedAxis` 
+
+If you set `sharedAxis: false` for sub charts, then their axes are persisted in global space (shared). It's useful for when you creating multiple sub charts sharing their axes but have different parameters. For example, 
+
+- `basic-column`, `stacked-column`, `percent-column`
+- `pie` and `donut`
+
+<br/>
+
+Here is an example.
+
+```js
+    const spec = {
+      charts: {
+        'column': {
+          transform: { method: 'array', },
+          sharedAxis: true,
+          axis: { ... },
+          parameter: { ... },
+        },
+
+        'stacked': {
+          transform: { method: 'array', },
+          sharedAxis: true,
+          axis: { ... }
+          parameter: { ... },
+        },
+```
+
+<br/>
+
+### Spec: `parameter` 
+
+| Field Name | Available Values (type) | Description |
+| --- | --- | --- |
+|`valueType` | `string` | Parameter which has string value |
+|`valueType` | `int` | Parameter which has int value |
+|`valueType` | `float` | Parameter which has float value |
+|`valueType` | `boolean` | Parameter which has boolean value used with `checkbox` widget usually |
+|`valueType` | `JSON` | Parameter which has JSON value used with `textarea` widget usually. `defaultValue` should be `""` (empty string). This ||`defaultValue` | (any) | Default value of this parameter. `JSON` type should have `""` (empty string) |
+|`description` | (string) | Description of this parameter. This value will be parsed as HTML for pretty output |
+|`widget` | `input` |  Use [input](https://developer.mozilla.org/en/docs/Web/HTML/Element/input) widget. This is the default widget (if `widget` is undefined)|
+|`widget` | `checkbox` |  Use [checkbox](https://developer.mozilla.org/en-US/docs/Web/HTML/Element/input/checkbox) widget. |
+|`widget` | `textarea` |  Use [textarea](https://developer.mozilla.org/en-US/docs/Web/HTML/Element/textarea) widget. |
+|`widget` | `option` |  Use [select + option](https://developer.mozilla.org/en-US/docs/Web/HTML/Element/select) widget. This parameter should have `optionValues` field as well. |
+|`optionValues` | (Array<string>) |  Available option values used with the `option` widget |
+
+<br/>
+
+Here is an example.
+
+```js
+parameter: {
+  // string type, input widget
+  'xAxisUnit': { valueType: 'string', defaultValue: '', description: 'unit of xAxis', },
+
+  // boolean type, checkbox widget
+  'inverted': { widget: 'checkbox', valueType: 'boolean', defaultValue: false, description: 'invert x and y axes', },
+
+  // string type, option widget with `optionValues`
+  'graphType': { widget: 'option', valueType: 'string', defaultValue: 'line', description: 'graph type', optionValues: [ 'line', 'smoothedLine', 'step', ], },
+
+  // HTML in `description`
+  'dateFormat': { valueType: 'string', defaultValue: '', description: 'format of date (<a href="https://docs.amcharts.com/3/javascriptcharts/AmGraph#dateFormat">doc</a>) (e.g YYYY-MM-DD)', },
+ 
+  // JSON type, textarea widget
+  'yAxisGuides': { widget: 'textarea', valueType: 'JSON', defaultValue: '', description: 'guides of yAxis ', },
+```
+
+<br/>
+
+### Spec: `transform`
+
+| Field Name | Available Values (type) | Description |
+| --- | --- | --- |
+|`method` | `object` |  designed for rows requiring object manipulation | 
+|`method` | `array` |  designed for rows requiring array manipulation | 
+|`method` | `array:2-key` |  designed for xyz charts (e.g bubble chart) | 
+|`method` | `drill-down` |  designed for drill-down charts | 
+|`method` | `raw` | will return the original `tableData.rows` | 
+
+<br/>
+
+Whatever you specified as `transform.method`, the `transformer` value will be always function for lazy computation. 
+
+```js
+// advanced-transformation.util#getTransformer
+
+if (transformSpec.method === 'raw') {
+  transformer = () => { return rows; }
+} else if (transformSpec.method === 'array') {
+  transformer = () => {
+    ...
+    return { ... }
+  }
+}
+```
+
+Here is actual usage.
+
+```js
+class AwesomeVisualization extends Visualization {
+  constructor(...) { /** setup your spec */ }
+  
+  ... 
+  
+  // `render` will be called whenever `axis` or `parameter` are changed
+  render(data) {
+    const { chart, parameter, column, transformer, } = data
+   
+    if (chart === 'line') {
+      const transformed = transformer()
+      // draw line chart 
+    } else if (chart === 'no-group') {
+      const transformed = transformer()
+      // draw no-group chart 
+    }
+  }
+  
+  ...
+}
+```
+

docs/index.md

@@ -172,10 +172,11 @@ Join to our [Mailing list](https://zeppelin.apache.org/community.html) and repor
   * [Shiro Authentication](./security/shiroauthentication.html)
   * [Notebook Authorization](./security/notebook_authorization.html)
   * [Data Source Authorization](./security/datasource_authorization.html)
-* Helium Framework
-  * [Writing Zeppelin Application (Experimental)](./development/writingzeppelinapplication.html)
-  * [Writing Zeppelin Spell (Experimental)](./development/writingzeppelinspell.html)
-  * [Writing Zeppelin Visualization (Experimental)](./development/writingzeppelinvisualization.html)
+* Helium Framework (Experimental)
+  * [Writing Zeppelin Application](./development/writingzeppelinapplication.html)
+  * [Writing Zeppelin Spell](./development/writingzeppelinspell.html)
+  * [Writing Zeppelin Visualization: Basic](./development/writingzeppelinvisualization.html)
+  * [Writing Zeppelin Visualization: Transformation](./development/writingzeppelinvisualization_transformation.html)
 * Advanced
   * [Apache Zeppelin on Vagrant VM](./install/virtual_machine.html)
   * [Zeppelin on Spark Cluster Mode (Standalone via Docker)](./install/spark_cluster_mode.html#spark-standalone-mode)

docs/install/upgrade.md

@@ -60,39 +60,39 @@ So, copying `notebook` and `conf` directory should be enough.
 ### Upgrading from Zeppelin 0.7 to 0.8
 
  - From 0.8, we recommend to use `PYSPARK_PYTHON` and `PYSPARK_DRIVER_PYTHON` instead of `zeppelin.pyspark.python` as `zeppelin.pyspark.python` only effects driver. You can use `PYSPARK_PYTHON` and `PYSPARK_DRIVER_PYTHON` as using them in spark.
+ - From 0.8, depending on your device, the keyboard shortcut `Ctrl-L` or `Command-L` which goes to the line somewhere user wants is not supported.
  - From 0.8, changed the format settings for interpreters (`interpreter.json`)
- 
+
    old format:
    ```
      "interpreterSettings": {
-         "2CD8TH1XV": {
-           "id": "2CD8TH1XV",
-           "name": "spark",
-           "group": "spark",
-           "properties": {
-             "spark.executor.memory": "",
-             "zeppelin.spark.concurrentSQL": "false",
-             ...
-           }
+       "2CD8TH1XV": {
+         "id": "2CD8TH1XV",
+         "name": "spark",
+         "group": "spark",
+         "properties": {
+           "spark.executor.memory": "",
+           "zeppelin.spark.concurrentSQL": "false",
+           ...
+         }
    ```
    new format:
    ```
      "interpreterSettings": {
-         "2CD8TH1XV": {
-           "id": "2CD8TH1XV",
-           "name": "spark",
-           "group": "spark",
-           "properties": {
-             "spark.executor.memory": {
-               "name": "spark.executor.memory",
-               "value": "",
-               "type": "text"
-              },
-             "zeppelin.spark.concurrentSQL": {
-               "name": "zeppelin.spark.concurrentSQL", 
-               "value": "false",
-               "type": "text"
-             ...
-           }
-   ```
- 
+       "2CD8TH1XV": {
+         "id": "2CD8TH1XV",
+         "name": "spark",
+         "group": "spark",
+         "properties": {
+           "spark.executor.memory": {
+             "name": "spark.executor.memory",
+             "value": "",
+             "type": "text"
+            },
+           "zeppelin.spark.concurrentSQL": {
+             "name": "zeppelin.spark.concurrentSQL", 
+             "value": false,
+             "type": "checkbox"
+           ...
+         }
+   ```

docs/interpreter/jdbc.md

@@ -123,6 +123,11 @@ The JDBC interpreter properties are defined by default like below.
     <td></td>
     <td>Some SQL which executes while opening connection</td>
   </tr>
+  <tr>
+    <td>default.completer.schemaFilters</td>
+    <td></td>
+    <td>Сomma separated schema (schema = catalog = database) filters to get metadata for completions. Supports '%' symbol is equivalent to any set of characters. (ex. prod_v_%,public%,info)</td>
+  </tr>
 </table>
 
 If you want to connect other databases such as `Mysql`, `Redshift` and `Hive`, you need to edit the property values.

elasticsearch/src/main/java/org/apache/zeppelin/elasticsearch/ElasticsearchInterpreter.java

@@ -33,6 +33,7 @@ import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
 import org.apache.commons.lang3.StringUtils;
+import org.apache.zeppelin.completer.CompletionType;
 import org.apache.zeppelin.elasticsearch.action.ActionResponse;
 import org.apache.zeppelin.elasticsearch.action.AggWrapper;
 import org.apache.zeppelin.elasticsearch.action.HitWrapper;
@@ -239,12 +240,13 @@ public class ElasticsearchInterpreter extends Interpreter {
   }
 
   @Override
-  public List<InterpreterCompletion> completion(String s, int i) {
+  public List<InterpreterCompletion> completion(String s, int i,
+      InterpreterContext interpreterContext) {
     final List suggestions = new ArrayList<>();
 
     for (final String cmd : COMMANDS) {
       if (cmd.toLowerCase().contains(s)) {
-        suggestions.add(new InterpreterCompletion(cmd, cmd));
+        suggestions.add(new InterpreterCompletion(cmd, cmd, CompletionType.command.name()));
       }
     }
     return suggestions;

elasticsearch/src/test/java/org/apache/zeppelin/elasticsearch/ElasticsearchInterpreterTest.java

@@ -31,6 +31,7 @@ import java.util.UUID;
 import java.util.concurrent.atomic.AtomicInteger;
 
 import org.apache.commons.lang.math.RandomUtils;
+import org.apache.zeppelin.completer.CompletionType;
 import org.apache.zeppelin.display.AngularObjectRegistry;
 import org.apache.zeppelin.interpreter.InterpreterContext;
 import org.apache.zeppelin.interpreter.InterpreterResult;
@@ -305,12 +306,12 @@ public class ElasticsearchInterpreterTest {
 
   @Theory
   public void testCompletion(ElasticsearchInterpreter interpreter) {
-    final List<InterpreterCompletion> expectedResultOne = Arrays.asList(new InterpreterCompletion("count", "count"));
-    final List<InterpreterCompletion> expectedResultTwo = Arrays.asList(new InterpreterCompletion("help", "help"));
+    final List<InterpreterCompletion> expectedResultOne = Arrays.asList(new InterpreterCompletion("count", "count", CompletionType.command.name()));
+    final List<InterpreterCompletion> expectedResultTwo = Arrays.asList(new InterpreterCompletion("help", "help", CompletionType.command.name()));
 
-    final List<InterpreterCompletion> resultOne = interpreter.completion("co", 0);
-    final List<InterpreterCompletion> resultTwo = interpreter.completion("he", 0);
-    final List<InterpreterCompletion> resultAll = interpreter.completion("", 0);
+    final List<InterpreterCompletion> resultOne = interpreter.completion("co", 0, null);
+    final List<InterpreterCompletion> resultTwo = interpreter.completion("he", 0, null);
+    final List<InterpreterCompletion> resultAll = interpreter.completion("", 0, null);
 
     Assert.assertEquals(expectedResultOne, resultOne);
     Assert.assertEquals(expectedResultTwo, resultTwo);

file/src/main/java/org/apache/zeppelin/file/FileInterpreter.java

@@ -166,7 +166,8 @@ public abstract class FileInterpreter extends Interpreter {
   }
 
   @Override
-  public List<InterpreterCompletion> completion(String buf, int cursor) {
+  public List<InterpreterCompletion> completion(String buf, int cursor,
+      InterpreterContext interpreterContext) {
     return null;
   }
 }

file/src/main/java/org/apache/zeppelin/file/HDFSFileInterpreter.java

@@ -23,6 +23,8 @@ import java.util.*;
 
 import com.google.gson.Gson;
 import org.apache.commons.lang.StringUtils;
+import org.apache.zeppelin.completer.CompletionType;
+import org.apache.zeppelin.interpreter.InterpreterContext;
 import org.apache.zeppelin.interpreter.InterpreterException;
 import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
 
@@ -247,21 +249,25 @@ public class HDFSFileInterpreter extends FileInterpreter {
 
 
   @Override
-  public List<InterpreterCompletion> completion(String buf, int cursor) {
+  public List<InterpreterCompletion> completion(String buf, int cursor,
+      InterpreterContext interpreterContext) {
     logger.info("Completion request at position\t" + cursor + " in string " + buf);
     final List<InterpreterCompletion> suggestions = new ArrayList<>();
     if (StringUtils.isEmpty(buf)) {
-      suggestions.add(new InterpreterCompletion("ls", "ls"));
-      suggestions.add(new InterpreterCompletion("cd", "cd"));
-      suggestions.add(new InterpreterCompletion("pwd", "pwd"));
+      suggestions.add(new InterpreterCompletion("ls", "ls", CompletionType.command.name()));
+      suggestions.add(new InterpreterCompletion("cd", "cd", CompletionType.command.name()));
+      suggestions.add(new InterpreterCompletion("pwd", "pwd", CompletionType.command.name()));
       return suggestions;
     }
 
     //part of a command == no spaces
     if (buf.split(" ").length == 1){
-      if ("cd".contains(buf)) suggestions.add(new InterpreterCompletion("cd", "cd"));
-      if ("ls".contains(buf)) suggestions.add(new InterpreterCompletion("ls", "ls"));
-      if ("pwd".contains(buf)) suggestions.add(new InterpreterCompletion("pwd", "pwd"));
+      if ("cd".contains(buf)) suggestions.add(new InterpreterCompletion("cd", "cd",
+          CompletionType.command.name()));
+      if ("ls".contains(buf)) suggestions.add(new InterpreterCompletion("ls", "ls",
+          CompletionType.command.name()));
+      if ("pwd".contains(buf)) suggestions.add(new InterpreterCompletion("pwd", "pwd",
+          CompletionType.command.name()));
 
       return suggestions;
     }
@@ -298,7 +304,8 @@ public class HDFSFileInterpreter extends FileInterpreter {
                 String beforeLastPeriod = unfinished.substring(0, unfinished.lastIndexOf('.') + 1);
                 //beforeLastPeriod should be the start of fs.pathSuffix, so take the end of it.
                 String suggestedFinish = fs.pathSuffix.substring(beforeLastPeriod.length());
-                suggestions.add(new InterpreterCompletion(suggestedFinish, suggestedFinish));
+                suggestions.add(new InterpreterCompletion(suggestedFinish, suggestedFinish,
+                    CompletionType.path.name()));
               }
             }
             return suggestions;

file/src/test/java/org/apache/zeppelin/file/HDFSFileInterpreterTest.java

@@ -21,6 +21,8 @@ package org.apache.zeppelin.file;
 import com.google.gson.Gson;
 import junit.framework.TestCase;
 import static org.junit.Assert.*;
+
+import org.apache.zeppelin.completer.CompletionType;
 import org.apache.zeppelin.interpreter.InterpreterResult;
 import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
 import org.junit.Test;
@@ -106,11 +108,11 @@ public class HDFSFileInterpreterTest extends TestCase {
 
       // auto completion test
       List expectedResultOne = Arrays.asList(
-        new InterpreterCompletion("ls", "ls"));
+        new InterpreterCompletion("ls", "ls", CompletionType.command.name()));
       List expectedResultTwo = Arrays.asList(
-        new InterpreterCompletion("pwd", "pwd"));
-      List<InterpreterCompletion> resultOne = t.completion("l", 0);
-      List<InterpreterCompletion> resultTwo = t.completion("p", 0);
+        new InterpreterCompletion("pwd", "pwd", CompletionType.command.name()));
+      List<InterpreterCompletion> resultOne = t.completion("l", 0, null);
+      List<InterpreterCompletion> resultTwo = t.completion("p", 0, null);
 
       assertEquals(expectedResultOne, resultOne);
       assertEquals(expectedResultTwo, resultTwo);

flink/src/main/java/org/apache/zeppelin/flink/FlinkInterpreter.java

@@ -373,7 +373,8 @@ public class FlinkInterpreter extends Interpreter {
   }
 
   @Override
-  public List<InterpreterCompletion> completion(String buf, int cursor) {
+  public List<InterpreterCompletion> completion(String buf, int cursor,
+      InterpreterContext interpreterContext) {
     return new LinkedList<>();
   }
 

geode/src/main/java/org/apache/zeppelin/geode/GeodeOqlInterpreter.java

@@ -282,7 +282,8 @@ public class GeodeOqlInterpreter extends Interpreter {
   }
 
   @Override
-  public List<InterpreterCompletion> completion(String buf, int cursor) {
+  public List<InterpreterCompletion> completion(String buf, int cursor,
+      InterpreterContext interpreterContext) {
     return null;
   }
 

hbase/src/main/java/org/apache/zeppelin/hbase/HbaseInterpreter.java

@@ -145,7 +145,8 @@ public class HbaseInterpreter extends Interpreter {
   }
 
   @Override
-  public List<InterpreterCompletion> completion(String buf, int cursor) {
+  public List<InterpreterCompletion> completion(String buf, int cursor,
+      InterpreterContext interpreterContext) {
     return null;
   }
 

helium-dev/src/main/java/org/apache/zeppelin/helium/DevInterpreter.java

@@ -98,7 +98,8 @@ public class DevInterpreter extends Interpreter {
   }
 
   @Override
-  public List<InterpreterCompletion> completion(String buf, int cursor) {
+  public List<InterpreterCompletion> completion(String buf, int cursor,
+      InterpreterContext interpreterContext) {
     return new LinkedList<>();
   }
 

ignite/src/main/java/org/apache/zeppelin/ignite/IgniteInterpreter.java

@@ -331,7 +331,8 @@ public class IgniteInterpreter extends Interpreter {
   }
 
   @Override
-  public List<InterpreterCompletion> completion(String buf, int cursor) {
+  public List<InterpreterCompletion> completion(String buf, int cursor,
+      InterpreterContext interpreterContext) {
     return new LinkedList<>();
   }
 

ignite/src/main/java/org/apache/zeppelin/ignite/IgniteSqlInterpreter.java

@@ -184,7 +184,8 @@ public class IgniteSqlInterpreter extends Interpreter {
   }
 
   @Override
-  public List<InterpreterCompletion> completion(String buf, int cursor) {
+  public List<InterpreterCompletion> completion(String buf, int cursor,
+      InterpreterContext interpreterContext) {
     return new LinkedList<>();
   }
 }

jdbc/pom.xml

@@ -35,7 +35,6 @@
   <properties>
     <!--library versions-->
     <postgresql.version>9.4-1201-jdbc41</postgresql.version>
-    <jline.version>2.12.1</jline.version>
     <hadoop.common.version>2.7.2</hadoop.common.version>
     <h2.version>1.4.190</h2.version>
     <commons.dbcp2.version>2.0.1</commons.dbcp2.version>
@@ -68,17 +67,6 @@
       <artifactId>slf4j-log4j12</artifactId>
     </dependency>
 	
-	<dependency>
-      <groupId>com.google.guava</groupId>
-      <artifactId>guava</artifactId>
-    </dependency>
-
-    <dependency>
-      <groupId>jline</groupId>
-      <artifactId>jline</artifactId>
-      <version>${jline.version}</version>
-    </dependency>
-
     <dependency>
       <groupId>com.h2database</groupId>
       <artifactId>h2</artifactId>

jdbc/src/main/java/org/apache/zeppelin/jdbc/JDBCInterpreter.java

@@ -57,9 +57,7 @@ import org.apache.zeppelin.user.UsernamePassword;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import com.google.common.base.Function;
 import com.google.common.base.Throwables;
-import com.google.common.collect.Lists;
 
 import static org.apache.commons.lang.StringUtils.containsIgnoreCase;
 import static org.apache.commons.lang.StringUtils.isEmpty;
@@ -103,6 +101,7 @@ public class JDBCInterpreter extends Interpreter {
   static final String USER_KEY = "user";
   static final String PASSWORD_KEY = "password";
   static final String PRECODE_KEY = "precode";
+  static final String COMPLETER_SCHEMA_FILTERS_KEY = "completer.schemaFilters";
   static final String JDBC_JCEKS_FILE = "jceks.file";
   static final String JDBC_JCEKS_CREDENTIAL_KEY = "jceks.credentialKey";
   static final String PRECODE_KEY_TEMPLATE = "%s.precode";
@@ -130,22 +129,12 @@ public class JDBCInterpreter extends Interpreter {
 
   private final HashMap<String, Properties> basePropretiesMap;
   private final HashMap<String, JDBCUserConfigurations> jdbcUserConfigurationsMap;
-  private final Map<String, SqlCompleter> propertyKeySqlCompleterMap;
 
-  private static final Function<CharSequence, InterpreterCompletion> sequenceToStringTransformer =
-      new Function<CharSequence, InterpreterCompletion>() {
-        public InterpreterCompletion apply(CharSequence seq) {
-          return new InterpreterCompletion(seq.toString(), seq.toString());
-        }
-      };
-
-  private static final List<InterpreterCompletion> NO_COMPLETION = new ArrayList<>();
   private int maxLineResults;
 
   public JDBCInterpreter(Properties property) {
     super(property);
     jdbcUserConfigurationsMap = new HashMap<>();
-    propertyKeySqlCompleterMap = new HashMap<>();
     basePropretiesMap = new HashMap<>();
     maxLineResults = MAX_LINE_DEFAULT;
   }
@@ -193,9 +182,7 @@ public class JDBCInterpreter extends Interpreter {
     if (!isEmpty(property.getProperty("zeppelin.jdbc.auth.type"))) {
       JDBCSecurityImpl.createSecureConfiguration(property);
     }
-    for (String propertyKey : basePropretiesMap.keySet()) {
-      propertyKeySqlCompleterMap.put(propertyKey, createSqlCompleter(null));
-    }
+
     setMaxLineResults();
   }
 
@@ -206,10 +193,11 @@ public class JDBCInterpreter extends Interpreter {
     }
   }
 
-  private SqlCompleter createSqlCompleter(Connection jdbcConnection) {
-
+  private SqlCompleter createSqlCompleter(Connection jdbcConnection, String propertyKey) {
+    String schemaFiltersKey = String.format("%s.%s", propertyKey, COMPLETER_SCHEMA_FILTERS_KEY);
+    String filters = getProperty(schemaFiltersKey);
     SqlCompleter completer = new SqlCompleter();
-    completer.initFromConnection(jdbcConnection, "");
+    completer.initFromConnection(jdbcConnection, filters);
     return completer;
   }
 
@@ -425,7 +413,7 @@ public class JDBCInterpreter extends Interpreter {
             connection = getConnectionFromPool(url, user, propertyKey, properties);
       }
     }
-    propertyKeySqlCompleterMap.put(propertyKey, createSqlCompleter(connection));
+
     return connection;
   }
 
@@ -794,18 +782,26 @@ public class JDBCInterpreter extends Interpreter {
   }
 
   @Override
-  public List<InterpreterCompletion> completion(String buf, int cursor) {
-    List<CharSequence> candidates = new ArrayList<>();
-    SqlCompleter sqlCompleter = propertyKeySqlCompleterMap.get(getPropertyKey(buf));
-    // It's strange but here cursor comes with additional +1 (even if buf is "" cursor = 1)
-    if (sqlCompleter != null && sqlCompleter.complete(buf, cursor - 1, candidates) >= 0) {
-      List<InterpreterCompletion> completion;
-      completion = Lists.transform(candidates, sequenceToStringTransformer);
-
-      return completion;
-    } else {
-      return NO_COMPLETION;
+  public List<InterpreterCompletion> completion(String buf, int cursor,
+      InterpreterContext interpreterContext) {
+    List<InterpreterCompletion> candidates = new ArrayList<>();
+    String propertyKey = getPropertyKey(buf);
+    Connection connection = null;
+    try {
+      if (interpreterContext != null) {
+        connection = getConnection(propertyKey, interpreterContext);
+      }
+    } catch (ClassNotFoundException | SQLException | IOException e) {
+      logger.warn("SQLCompleter will created without use connection");
     }
+
+    SqlCompleter sqlCompleter = createSqlCompleter(connection, propertyKey);
+
+    if (sqlCompleter != null) {
+      sqlCompleter.complete(buf, cursor - 1, candidates);
+    }
+
+    return candidates;
   }
 
   public int getMaxResult() {

jdbc/src/main/java/org/apache/zeppelin/jdbc/SqlCompleter.java

@@ -4,15 +4,6 @@ package org.apache.zeppelin.jdbc;
  * This source file is based on code taken from SQLLine 1.0.2 See SQLLine notice in LICENSE
  */
 
-import com.google.common.base.Joiner;
-import com.google.common.collect.Sets;
-import com.google.common.collect.Sets.SetView;
-import jline.console.completer.ArgumentCompleter.ArgumentList;
-import jline.console.completer.ArgumentCompleter.WhitespaceArgumentDelimiter;
-import jline.console.completer.StringsCompleter;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
 import java.io.BufferedReader;
 import java.io.IOException;
 import java.io.InputStreamReader;
@@ -20,15 +11,34 @@ import java.sql.Connection;
 import java.sql.DatabaseMetaData;
 import java.sql.ResultSet;
 import java.sql.SQLException;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.StringTokenizer;
+import java.util.TreeSet;
 import java.util.regex.Pattern;
 
+import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang.math.NumberUtils;
+import org.apache.zeppelin.completer.CompletionType;
+import org.apache.zeppelin.completer.StringsCompleter;
+import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import jline.console.completer.ArgumentCompleter.ArgumentList;
+import jline.console.completer.ArgumentCompleter.WhitespaceArgumentDelimiter;
+
 import static org.apache.commons.lang.StringUtils.isBlank;
 
 /**
  * SQL auto complete functionality for the JdbcInterpreter.
  */
-public class SqlCompleter extends StringsCompleter {
+public class SqlCompleter {
 
   private static Logger logger = LoggerFactory.getLogger(SqlCompleter.class);
 
@@ -67,8 +77,7 @@ public class SqlCompleter extends StringsCompleter {
    */
   private StringsCompleter keywordCompleter = new StringsCompleter();
 
-  @Override
-  public int complete(String buffer, int cursor, List<CharSequence> candidates) {
+  public int complete(String buffer, int cursor, List<InterpreterCompletion> candidates) {
 
     logger.debug("Complete with buffer = " + buffer + ", cursor = " + cursor);
 
@@ -76,21 +85,36 @@ public class SqlCompleter extends StringsCompleter {
     // white spaces.
     ArgumentList argumentList = sqlDelimiter.delimit(buffer, cursor);
 
-    String beforeCursorBuffer = buffer.substring(0,
-            Math.min(cursor, buffer.length())).toUpperCase();
+    Pattern whitespaceEndPatter = Pattern.compile("\\s$");
+    String cursorArgument = null;
+    int argumentPosition;
+    if (buffer.length() == 0 || whitespaceEndPatter.matcher(buffer).find()) {
+      argumentPosition = buffer.length() - 1;
+    } else {
+      cursorArgument = argumentList.getCursorArgument();
+      argumentPosition = argumentList.getArgumentPosition();
+    }
 
-    // check what sql is and where cursor is to allow column completion or not
     boolean isColumnAllowed = true;
-    if (beforeCursorBuffer.contains("SELECT ") && beforeCursorBuffer.contains(" FROM ")
-            && !beforeCursorBuffer.contains(" WHERE "))
-      isColumnAllowed = false;
+    if (buffer.length() > 0) {
+      String beforeCursorBuffer = buffer.substring(0,
+          Math.min(cursor, buffer.length())).toUpperCase();
+      // check what sql is and where cursor is to allow column completion or not
+      if (beforeCursorBuffer.contains("SELECT ") && beforeCursorBuffer.contains(" FROM ")
+          && !beforeCursorBuffer.contains(" WHERE "))
+        isColumnAllowed = false;
+    }
 
-    int complete = completeName(argumentList.getCursorArgument(),
-            argumentList.getArgumentPosition(), candidates,
+    int complete = completeName(cursorArgument, argumentPosition, candidates,
             findAliasesInSQL(argumentList.getArguments()), isColumnAllowed);
 
+    if (candidates.size() == 1) {
+      InterpreterCompletion interpreterCompletion = candidates.get(0);
+      interpreterCompletion.setName(interpreterCompletion.getName() + " ");
+      interpreterCompletion.setValue(interpreterCompletion.getValue() + " ");
+      candidates.set(0, interpreterCompletion);
+    }
     logger.debug("complete:" + complete + ", size:" + candidates.size());
-
     return complete;
   }
 
@@ -98,24 +122,26 @@ public class SqlCompleter extends StringsCompleter {
    * Return list of schema names within the database
    *
    * @param meta metadata from connection to database
-   * @param schemaFilter a schema name pattern; must match the schema name
+   * @param schemaFilters a schema name patterns; must match the schema name
    *        as it is stored in the database; "" retrieves those without a schema;
    *        <code>null</code> means that the schema name should not be used to narrow
-   *        the search; supports '%' and '_' symbols; for example "prod_v_%"
+   *        the search; supports '%'; for example "prod_v_%"
    * @return set of all schema names in the database
    */
-  private static Set<String> getSchemaNames(DatabaseMetaData meta, String schemaFilter) {
+  private static Set<String> getSchemaNames(DatabaseMetaData meta, List<String> schemaFilters) {
     Set<String> res = new HashSet<>();
     try {
       ResultSet schemas = meta.getSchemas();
       try {
         while (schemas.next()) {
           String schemaName = schemas.getString("TABLE_SCHEM");
-          if (schemaName == null)
+          if (schemaName == null) {
             schemaName = "";
-          if (schemaFilter.equals("") || schemaFilter == null || schemaName.matches(
-                  schemaFilter.replace("_", ".").replace("%", ".*?"))) {
-            res.add(schemaName);
+          }
+          for (String schemaFilter : schemaFilters) {
+            if (schemaFilter.equals("") || schemaName.matches(schemaFilter.replace("%", ".*?"))) {
+              res.add(schemaName);
+            }
           }
         }
       } finally {
@@ -131,22 +157,23 @@ public class SqlCompleter extends StringsCompleter {
    * Return list of catalog names within the database
    *
    * @param meta metadata from connection to database
-   * @param schemaFilter a catalog name pattern; must match the catalog name
+   * @param schemaFilters a catalog name patterns; must match the catalog name
    *        as it is stored in the database; "" retrieves those without a catalog;
    *        <code>null</code> means that the schema name should not be used to narrow
-   *        the search; supports '%' and '_' symbols; for example "prod_v_%"
+   *        the search; supports '%'; for example "prod_v_%"
    * @return set of all catalog names in the database
    */
-  private static Set<String> getCatalogNames(DatabaseMetaData meta, String schemaFilter) {
+  private static Set<String> getCatalogNames(DatabaseMetaData meta, List<String> schemaFilters) {
     Set<String> res = new HashSet<>();
     try {
       ResultSet schemas = meta.getCatalogs();
       try {
         while (schemas.next()) {
           String schemaName = schemas.getString("TABLE_CAT");
-          if (schemaFilter.equals("") || schemaFilter == null || schemaName.matches(
-                  schemaFilter.replace("_", ".").replace("%", ".*?"))) {
-            res.add(schemaName);
+          for (String schemaFilter : schemaFilters) {
+            if (schemaFilter.equals("") || schemaName.matches(schemaFilter.replace("%", ".*?"))) {
+              res.add(schemaName);
+            }
           }
         }
       } finally {
@@ -166,7 +193,7 @@ public class SqlCompleter extends StringsCompleter {
    * @param schemaFilter a schema name pattern; must match the schema name
    *        as it is stored in the database; "" retrieves those without a schema;
    *        <code>null</code> means that the schema name should not be used to narrow
-   *        the search; supports '%' and '_' symbols; for example "prod_v_%"
+   *        the search; supports '%'; for example "prod_v_%"
    * @param tables function fills this map, for every schema name adds
    *        set of table names within the schema
    * @param columns function fills this map, for every table name adds set
@@ -177,19 +204,27 @@ public class SqlCompleter extends StringsCompleter {
                                               Map<String, Set<String>> tables,
                                               Map<String, Set<String>> columns)  {
     try {
-      ResultSet cols = meta.getColumns(catalogName, schemaFilter, "%",
-              "%");
+      ResultSet cols = meta.getColumns(catalogName, StringUtils.EMPTY, "%", "%");
       try {
         while (cols.next()) {
           String schema = cols.getString("TABLE_SCHEM");
-          if (schema == null) schema = cols.getString("TABLE_CAT");
+          if (schema == null) {
+            schema = cols.getString("TABLE_CAT");
+          }
+          if (!schemaFilter.equals("") && !schema.matches(schemaFilter.replace("%", ".*?"))) {
+            continue;
+          }
           String table = cols.getString("TABLE_NAME");
           String column = cols.getString("COLUMN_NAME");
           if (!isBlank(table)) {
             String schemaTable = schema + "." + table;
-            if (!columns.containsKey(schemaTable)) columns.put(schemaTable, new HashSet<String>());
+            if (!columns.containsKey(schemaTable)) {
+              columns.put(schemaTable, new HashSet<String>());
+            }
             columns.get(schemaTable).add(column);
-            if (!tables.containsKey(schema)) tables.put(schema, new HashSet<String>());
+            if (!tables.containsKey(schema)) {
+              tables.put(schema, new HashSet<String>());
+            }
             tables.get(schema).add(table);
           }
         }
@@ -327,33 +362,31 @@ public class SqlCompleter extends StringsCompleter {
    * Initializes all local completers from database connection
    *
    * @param connection database connection
-   * @param schemaFilter a schema name pattern; must match the schema name
-   *        as it is stored in the database; "" retrieves those without a schema;
-   *        <code>null</code> means that the schema name should not be used to narrow
-   *        the search; supports '%' and '_' symbols; for example "prod_v_%"
+   * @param schemaFiltersString a comma separated schema name patterns; supports '%'  symbol;
+   * for example "prod_v_%,prod_t_%"
    */
-  public void initFromConnection(Connection connection, String schemaFilter) {
+  public void initFromConnection(Connection connection, String schemaFiltersString) {
+    if (schemaFiltersString == null) {
+      schemaFiltersString = StringUtils.EMPTY;
+    }
+    List<String> schemaFilters = Arrays.asList(schemaFiltersString.split(","));
 
-    try {
+    try (Connection c = connection) {
       Map<String, Set<String>> tables = new HashMap<>();
       Map<String, Set<String>> columns = new HashMap<>();
       Set<String> schemas = new HashSet<>();
       Set<String> catalogs = new HashSet<>();
       Set<String> keywords = getSqlKeywordsCompletions(connection);
       if (connection != null) {
-        schemas = getSchemaNames(connection.getMetaData(), schemaFilter);
-        catalogs = getCatalogNames(connection.getMetaData(), schemaFilter);
-
-        if (!"".equals(connection.getCatalog())) {
-          if (schemas.size() == 0 )
-            schemas.add(connection.getCatalog());
-          fillTableAndColumnNames(connection.getCatalog(), connection.getMetaData(), schemaFilter,
-                  tables, columns);
-        } else {
-          if (schemas.size() == 0) schemas.addAll(catalogs);
-          for (String catalog : catalogs) {
-            fillTableAndColumnNames(catalog, connection.getMetaData(), schemaFilter, tables,
-                    columns);
+        schemas = getSchemaNames(connection.getMetaData(), schemaFilters);
+        catalogs = getCatalogNames(connection.getMetaData(), schemaFilters);
+        if (schemas.size() == 0) {
+          schemas.addAll(catalogs);
+        }
+        for (String schema : schemas) {
+          for (String schemaFilter : schemaFilters) {
+            fillTableAndColumnNames(schema, connection.getMetaData(), schemaFilter, tables,
+                columns);
           }
         }
       }
@@ -408,8 +441,18 @@ public class SqlCompleter extends StringsCompleter {
    */
   private int completeTable(String schema, String buffer, int cursor,
                             List<CharSequence> candidates) {
+    if (schema == null) {
+      int res = -1;
+      Set<CharSequence> candidatesSet = new HashSet<>();
+      for (StringsCompleter stringsCompleter : tablesCompleters.values()) {
+        int resTable = stringsCompleter.complete(buffer, cursor, candidatesSet);
+        res = Math.max(res, resTable);
+      }
+      candidates.addAll(candidatesSet);
+      return res;
+    }
     // Wrong schema
-    if (!tablesCompleters.containsKey(schema))
+    if (!tablesCompleters.containsKey(schema) && schema != null)
       return -1;
     else
       return tablesCompleters.get(schema).complete(buffer, cursor, candidates);
@@ -422,12 +465,23 @@ public class SqlCompleter extends StringsCompleter {
    */
   private int completeColumn(String schema, String table, String buffer, int cursor,
                              List<CharSequence> candidates) {
+    if (table == null && schema == null) {
+      int res = -1;
+      Set<CharSequence> candidatesSet = new HashSet<>();
+      for (StringsCompleter stringsCompleter : columnsCompleters.values()) {
+        int resColumn = stringsCompleter.complete(buffer, cursor, candidatesSet);
+        res = Math.max(res, resColumn);
+      }
+      candidates.addAll(candidatesSet);
+      return res;
+    }
     // Wrong schema or wrong table
     if (!tablesCompleters.containsKey(schema) ||
-            !columnsCompleters.containsKey(schema + "." + table))
+        !columnsCompleters.containsKey(schema + "." + table)) {
       return -1;
-    else
+    } else {
       return columnsCompleters.get(schema + "." + table).complete(buffer, cursor, candidates);
+    }
   }
 
   /**
@@ -438,32 +492,43 @@ public class SqlCompleter extends StringsCompleter {
    * @param isColumnAllowed if false the function will not search and complete columns
    * @return -1 in case of no candidates found, 0 otherwise
    */
-  public int completeName(String buffer, int cursor, List<CharSequence> candidates,
+  public int completeName(String buffer, int cursor, List<InterpreterCompletion> candidates,
                           Map<String, String> aliases, boolean isColumnAllowed) {
 
-    if (buffer == null) buffer = "";
-
-    // no need to process after first point after cursor
-    int nextPointPos = buffer.indexOf('.', cursor);
-    if (nextPointPos != -1) buffer = buffer.substring(0, nextPointPos);
-
     // points divide the name to the schema, table and column - find them
-    int pointPos1 = buffer.indexOf('.');
-    int pointPos2 = buffer.indexOf('.', pointPos1 + 1);
+    int pointPos1 = -1;
+    int pointPos2 = -1;
 
+    if (StringUtils.isNotEmpty(buffer)) {
+      if (buffer.length() > cursor) {
+        buffer = buffer.substring(0, cursor + 1);
+      }
+      pointPos1 = buffer.indexOf('.');
+      pointPos2 = buffer.indexOf('.', pointPos1 + 1);
+    }
     // find schema and table name if they are
     String schema;
     String table;
     String column;
-    if (pointPos1 == -1) {             // process only schema or keyword case
-      schema = buffer;
-      int keywordsRes = completeKeyword(buffer, cursor, candidates);
+
+    if (pointPos1 == -1) {             // process all
+      List<CharSequence> keywordsCandidates = new ArrayList();
       List<CharSequence> schemaCandidates = new ArrayList<>();
-      int schemaRes = completeSchema(schema, cursor, schemaCandidates);
-      candidates.addAll(schemaCandidates);
-      return Math.max(keywordsRes, schemaRes);
-    }
-    else {
+      List<CharSequence> tableCandidates = new ArrayList<>();
+      List<CharSequence> columnCandidates = new ArrayList<>();
+      int keywordsRes = completeKeyword(buffer, cursor, keywordsCandidates);
+      int schemaRes = completeSchema(buffer, cursor, schemaCandidates);
+      int tableRes = completeTable(null, buffer, cursor, tableCandidates);
+      int columnRes = -1;
+      if (isColumnAllowed) {
+        columnRes = completeColumn(null, null, buffer, cursor, columnCandidates);
+      }
+      addCompletions(candidates, keywordsCandidates, CompletionType.keyword.name());
+      addCompletions(candidates, schemaCandidates, CompletionType.schema.name());
+      addCompletions(candidates, tableCandidates, CompletionType.table.name());
+      addCompletions(candidates, columnCandidates, CompletionType.column.name());
+      return NumberUtils.max(new int[]{keywordsRes, schemaRes, tableRes, columnRes});
+    } else {
       schema = buffer.substring(0, pointPos1);
       if (aliases.containsKey(schema)) {  // process alias case
         String alias = aliases.get(schema);
@@ -471,26 +536,40 @@ public class SqlCompleter extends StringsCompleter {
         schema = alias.substring(0, pointPos);
         table = alias.substring(pointPos + 1);
         column = buffer.substring(pointPos1 + 1);
-      }
-      else if (pointPos2 == -1) {        // process schema.table case
+      } else if (pointPos2 == -1) {        // process schema.table case
+        List<CharSequence> tableCandidates = new ArrayList();
         table = buffer.substring(pointPos1 + 1);
-        return completeTable(schema, table, cursor - pointPos1 - 1, candidates);
-      }
-      else {
+        int tableRes = completeTable(schema, table, cursor - pointPos1 - 1, tableCandidates);
+        addCompletions(candidates, tableCandidates, CompletionType.table.name());
+        return tableRes;
+      } else {
         table = buffer.substring(pointPos1 + 1, pointPos2);
         column = buffer.substring(pointPos2 + 1);
       }
     }
 
     // here in case of column
-    if (isColumnAllowed)
-      return completeColumn(schema, table, column, cursor - pointPos2 - 1, candidates);
-    else
-      return -1;
+    if (table != null && isColumnAllowed) {
+      List<CharSequence> columnCandidates = new ArrayList();
+      int columnRes = completeColumn(schema, table, column, cursor - pointPos2 - 1,
+          columnCandidates);
+      addCompletions(candidates, columnCandidates, CompletionType.column.name());
+      return columnRes;
+    }
+
+    return -1;
   }
 
   // test purpose only
   WhitespaceArgumentDelimiter getSqlDelimiter() {
     return this.sqlDelimiter;
   }
+
+  private void addCompletions(List<InterpreterCompletion> interpreterCompletions,
+      List<CharSequence> candidates, String meta) {
+    for (CharSequence candidate : candidates) {
+      interpreterCompletions.add(new InterpreterCompletion(candidate.toString(),
+          candidate.toString(), meta));
+    }
+  }
 }

jdbc/src/main/resources/ansi.sql.keywords

@@ -1 +1 @@
-ABSOLUTE,ACTION,ADD,ALL,ALLOCATE,ALTER,AND,ANY,ARE,AS,ASC,ASSERTION,AT,AUTHORIZATION,AVG,BEGIN,BETWEEN,BIT,BIT_LENGTH,BOTH,BY,CASCADE,CASCADED,CASE,CAST,CATALOG,CHAR,CHARACTER,CHAR_LENGTH,CHARACTER_LENGTH,CHECK,CLOSE,CLUSTER,COALESCE,COLLATE,COLLATION,COLUMN,COMMIT,CONNECT,CONNECTION,CONSTRAINT,CONSTRAINTS,CONTINUE,CONVERT,CORRESPONDING,COUNT,CREATE,CROSS,CURRENT,CURRENT_DATE,CURRENT_TIME,CURRENT_TIMESTAMP,CURRENT_USER,CURSOR,DATE,DAY,DEALLOCATE,DEC,DECIMAL,DECLARE,DEFAULT,DEFERRABLE,DEFERRED,DELETE,DESC,DESCRIBE,DESCRIPTOR,DIAGNOSTICS,DISCONNECT,DISTINCT,DOMAIN,DOUBLE,DROP,ELSE,END,END-EXEC,ESCAPE,EXCEPT,EXCEPTION,EXEC,EXECUTE,EXISTS,EXTERNAL,EXTRACT,FALSE,FETCH,FIRST,FLOAT,FOR,FOREIGN,FOUND,FROM,FULL,GET,GLOBAL,GO,GOTO,GRANT,GROUP,HAVING,HOUR,IDENTITY,IMMEDIATE,IN,INDICATOR,INITIALLY,INNER,INPUT,INSENSITIVE,INSERT,INT,INTEGER,INTERSECT,INTERVAL,INTO,IS,ISOLATION,JOIN,KEY,LANGUAGE,LAST,LEADING,LEFT,LEVEL,LIKE,LOCAL,LOWER,MATCH,MAX,MIN,MINUTE,MODULE,MONTH,NAMES,NATIONAL,NATURAL,NCHAR,NEXT,NO,NOT,NULL,NULLIF,NUMERIC,OCTET_LENGTH,OF,ON,ONLY,OPEN,OPTION,OR,ORDER,OUTER,OUTPUT,OVERLAPS,OVERWRITE,PAD,PARTIAL,PARTITION,POSITION,PRECISION,PREPARE,PRESERVE,PRIMARY,PRIOR,PRIVILEGES,PROCEDURE,PUBLIC,READ,REAL,REFERENCES,RELATIVE,RESTRICT,REVOKE,RIGHT,ROLLBACK,ROWS,SCHEMA,SCROLL,SECOND,SECTION,SELECT,SESSION,SESSION_USER,SET,SIZE,SMALLINT,SOME,SPACE,SQL,SQLCODE,SQLERROR,SQLSTATE,SUBSTRING,SUM,SYSTEM_USER,TABLE,TEMPORARY,THEN,TIME,TIMESTAMP,TIMEZONE_HOUR,TIMEZONE_MINUTE,TO,TRAILING,TRANSACTION,TRANSLATE,TRANSLATION,TRIM,TRUE,UNION,UNIQUE,UNKNOWN,UPDATE,UPPER,USAGE,USER,USING,VALUE,VALUES,VARCHAR,VARYING,VIEW,WHEN,WHENEVER,WHERE,WITH,WORK,WRITE,YEAR,ZONE,ADA,C,CATALOG_NAME,CHARACTER_SET_CATALOG,CHARACTER_SET_NAME,CHARACTER_SET_SCHEMA,CLASS_ORIGIN,COBOL,COLLATION_CATALOG,COLLATION_NAME,COLLATION_SCHEMA,COLUMN_NAME,COMMAND_FUNCTION,COMMITTED,CONDITION_NUMBER,CONNECTION_NAME,CONSTRAINT_CATALOG,CONSTRAINT_NAME,CONSTRAINT_SCHEMA,CURSOR_NAME,DATA,DATETIME_INTERVAL_CODE,DATETIME_INTERVAL_PRECISION,DYNAMIC_FUNCTION,FORTRAN,LENGTH,MESSAGE_LENGTH,MESSAGE_OCTET_LENGTH,MESSAGE_TEXT,MORE,MUMPS,NAME,NULLABLE,NUMBER,PASCAL,PLI,REPEATABLE,RETURNED_LENGTH,RETURNED_OCTET_LENGTH,RETURNED_SQLSTATE,ROW_COUNT,SCALE,SCHEMA_NAME,SERIALIZABLE,SERVER_NAME,SUBCLASS_ORIGIN,TABLE_NAME,TYPE,UNCOMMITTED,UNNAMED,LIMIT
+absolute,action,add,all,allocate,alter,and,any,are,as,asc,assertion,at,authorization,avg,begin,between,bit,bit_length,both,by,cascade,cascaded,case,cast,catalog,char,character,char_length,character_length,check,close,cluster,coalesce,collate,collation,column,commit,connect,connection,constraint,constraints,continue,convert,corresponding,count,create,cross,current,current_date,current_time,current_timestamp,current_user,cursor,date,day,deallocate,dec,decimal,declare,default,deferrable,deferred,delete,desc,describe,descriptor,diagnostics,disconnect,distinct,domain,double,drop,else,end,end-exec,escape,except,exception,exec,execute,exists,external,extract,false,fetch,first,float,for,foreign,found,from,full,get,global,go,goto,grant,group,having,hour,identity,immediate,in,indicator,initially,inner,input,insensitive,insert,int,integer,intersect,interval,into,is,isolation,join,key,language,last,leading,left,level,like,local,lower,match,max,min,minute,module,month,names,national,natural,nchar,next,no,not,null,nullif,numeric,octet_length,of,on,only,open,option,or,order,outer,output,overlaps,overwrite,pad,partial,partition,position,precision,prepare,preserve,primary,prior,privileges,procedure,public,read,real,references,relative,restrict,revoke,right,rollback,rows,schema,scroll,second,section,select,session,session_user,set,size,smallint,some,space,sql,sqlcode,sqlerror,sqlstate,substring,sum,system_user,table,temporary,then,time,timestamp,timezone_hour,timezone_minute,to,trailing,transaction,translate,translation,trim,true,union,unique,unknown,update,upper,usage,user,using,value,values,varchar,varying,view,when,whenever,where,with,work,write,year,zone,ada,c,catalog_name,character_set_catalog,character_set_name,character_set_schema,class_origin,cobol,collation_catalog,collation_name,collation_schema,column_name,command_function,committed,condition_number,connection_name,constraint_catalog,constraint_name,constraint_schema,cursor_name,data,datetime_interval_code,datetime_interval_precision,dynamic_function,fortran,length,message_length,message_octet_length,message_text,more,mumps,name,nullable,number,pascal,pli,repeatable,returned_length,returned_octet_length,returned_sqlstate,row_count,scale,schema_name,serializable,server_name,subclass_origin,table_name,type,uncommitted,unnamed,limit

jdbc/src/main/resources/interpreter-setting.json

@@ -32,6 +32,12 @@
         "description": "JDBC Driver Name",
         "type": "text"
       },
+      "default.completer.schemaFilters": {
+        "envName": null,
+        "propertyName": "default.completer.schemaFilters",
+        "defaultValue": "",
+        "description": "Сomma separated schema (schema = catalog = database) filters to get metadata for completions. Supports '%' symbol is equivalent to any set of characters. (ex. prod_v_%,public%,info)"
+      },
       "default.precode": {
         "envName": null,
         "propertyName": "zeppelin.jdbc.precode",

jdbc/src/test/java/org/apache/zeppelin/jdbc/JDBCInterpreterTest.java

@@ -32,6 +32,7 @@ import java.util.ArrayList;
 import java.util.List;
 import java.util.Properties;
 
+import org.apache.zeppelin.completer.CompletionType;
 import org.apache.zeppelin.interpreter.InterpreterContext;
 import org.apache.zeppelin.interpreter.InterpreterResult;
 import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
@@ -295,9 +296,9 @@ public class JDBCInterpreterTest extends BasicJDBCTestCaseAdapter {
 
     jdbcInterpreter.interpret("", interpreterContext);
 
-    List<InterpreterCompletion> completionList = jdbcInterpreter.completion("sel", 1);
+    List<InterpreterCompletion> completionList = jdbcInterpreter.completion("sel", 3, null);
 
-    InterpreterCompletion correctCompletionKeyword = new InterpreterCompletion("select ", "select ");
+    InterpreterCompletion correctCompletionKeyword = new InterpreterCompletion("select ", "select ", CompletionType.keyword.name());
 
     assertEquals(1, completionList.size());
     assertEquals(true, completionList.contains(correctCompletionKeyword));

jdbc/src/test/java/org/apache/zeppelin/jdbc/SqlCompleterTest.java

@@ -14,18 +14,26 @@
  */
 package org.apache.zeppelin.jdbc;
 
-import com.google.common.base.Joiner;
-import com.mockrunner.jdbc.BasicJDBCTestCaseAdapter;
-import jline.console.completer.ArgumentCompleter;
-import jline.console.completer.Completer;
+import java.io.IOException;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.zeppelin.completer.CompletionType;
+import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
+import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.io.IOException;
-import java.sql.SQLException;
-import java.util.*;
+import com.google.common.base.Joiner;
+import jline.console.completer.ArgumentCompleter;
 
 import static com.google.common.collect.Sets.newHashSet;
 import static org.junit.Assert.assertEquals;
@@ -34,18 +42,18 @@ import static org.junit.Assert.assertTrue;
 /**
  * SQL completer unit tests
  */
-public class SqlCompleterTest extends BasicJDBCTestCaseAdapter {
+public class SqlCompleterTest {
 
   public class CompleterTester {
 
-    private Completer completer;
+    private SqlCompleter completer;
 
     private String buffer;
     private int fromCursor;
     private int toCursor;
-    private Set<String> expectedCompletions;
+    private Set<InterpreterCompletion> expectedCompletions;
 
-    public CompleterTester(Completer completer) {
+    public CompleterTester(SqlCompleter completer) {
       this.completer = completer;
     }
 
@@ -64,7 +72,7 @@ public class SqlCompleterTest extends BasicJDBCTestCaseAdapter {
       return this;
     }
 
-    public CompleterTester expect(Set<String> expectedCompletions) {
+    public CompleterTester expect(Set<InterpreterCompletion> expectedCompletions) {
       this.expectedCompletions = expectedCompletions;
       return this;
     }
@@ -75,9 +83,13 @@ public class SqlCompleterTest extends BasicJDBCTestCaseAdapter {
       }
     }
 
-    private void expectedCompletions(String buffer, int cursor, Set<String> expected) {
+    private void expectedCompletions(String buffer, int cursor,
+        Set<InterpreterCompletion> expected) {
+      if (StringUtils.isNotEmpty(buffer) && buffer.length() > cursor) {
+        buffer = buffer.substring(0, cursor + 1);
+      }
 
-      ArrayList<CharSequence> candidates = new ArrayList<>();
+      List<InterpreterCompletion> candidates = new ArrayList<>();
 
       completer.complete(buffer, cursor, candidates);
 
@@ -85,11 +97,15 @@ public class SqlCompleterTest extends BasicJDBCTestCaseAdapter {
 
       logger.info(explain);
 
-      assertEquals("Buffer [" + buffer.replace(" ", ".") + "] and Cursor[" + cursor + "] "
-              + explain, expected, newHashSet(candidates));
+      Assert.assertEquals("Buffer [" + buffer.replace(" ", ".") + "] and Cursor[" + cursor + "] "
+          + explain, expected, newHashSet(candidates));
     }
 
-    private String explain(String buffer, int cursor, ArrayList<CharSequence> candidates) {
+    private String explain(String buffer, int cursor, List<InterpreterCompletion> candidates) {
+      List<String> cndidateStrings = new ArrayList<>();
+      for (InterpreterCompletion candidate : candidates) {
+        cndidateStrings.add(candidate.getValue());
+      }
       StringBuffer sb = new StringBuffer();
 
       for (int i = 0; i <= Math.max(cursor, buffer.length()); i++) {
@@ -109,7 +125,7 @@ public class SqlCompleterTest extends BasicJDBCTestCaseAdapter {
           sb.append(")");
         }
       }
-      sb.append(" >> [").append(Joiner.on(",").join(candidates)).append("]");
+      sb.append(" >> [").append(Joiner.on(",").join(cndidateStrings)).append("]");
 
       return sb.toString();
     }
@@ -122,7 +138,7 @@ public class SqlCompleterTest extends BasicJDBCTestCaseAdapter {
   private CompleterTester tester;
 
   private ArgumentCompleter.WhitespaceArgumentDelimiter delimiter =
-          new ArgumentCompleter.WhitespaceArgumentDelimiter();
+      new ArgumentCompleter.WhitespaceArgumentDelimiter();
 
   private SqlCompleter sqlCompleter = new SqlCompleter();
 
@@ -178,7 +194,7 @@ public class SqlCompleterTest extends BasicJDBCTestCaseAdapter {
   }
 
   @Test
-  public void testFindAliasesInSQL_Simple(){
+  public void testFindAliasesInSQL_Simple() {
     String sql = "select * from prod_emart.financial_account a";
     Map<String, String> res = sqlCompleter.findAliasesInSQL(delimiter.delimit(sql, 0).getArguments());
     assertEquals(1, res.size());
@@ -186,7 +202,7 @@ public class SqlCompleterTest extends BasicJDBCTestCaseAdapter {
   }
 
   @Test
-  public void testFindAliasesInSQL_Two(){
+  public void testFindAliasesInSQL_Two() {
     String sql = "select * from prod_dds.financial_account a, prod_dds.customer b";
     Map<String, String> res = sqlCompleter.findAliasesInSQL(sqlCompleter.getSqlDelimiter().delimit(sql, 0).getArguments());
     assertEquals(2, res.size());
@@ -195,7 +211,7 @@ public class SqlCompleterTest extends BasicJDBCTestCaseAdapter {
   }
 
   @Test
-  public void testFindAliasesInSQL_WrongTables(){
+  public void testFindAliasesInSQL_WrongTables() {
     String sql = "select * from prod_ddsxx.financial_account a, prod_dds.customerxx b";
     Map<String, String> res = sqlCompleter.findAliasesInSQL(sqlCompleter.getSqlDelimiter().delimit(sql, 0).getArguments());
     assertEquals(0, res.size());
@@ -205,116 +221,145 @@ public class SqlCompleterTest extends BasicJDBCTestCaseAdapter {
   public void testCompleteName_Empty() {
     String buffer = "";
     int cursor = 0;
-    List<CharSequence> candidates = new ArrayList<>();
+    List<InterpreterCompletion> candidates = new ArrayList<>();
     Map<String, String> aliases = new HashMap<>();
-    sqlCompleter.completeName(buffer, cursor, candidates, aliases, false);
-    assertEquals(9, candidates.size());
-    assertTrue(candidates.contains("prod_dds"));
-    assertTrue(candidates.contains("prod_emart"));
-    assertTrue(candidates.contains("SUM"));
-    assertTrue(candidates.contains("SUBSTRING"));
-    assertTrue(candidates.contains("SUBCLASS_ORIGIN"));
-    assertTrue(candidates.contains("SELECT"));
-    assertTrue(candidates.contains("ORDER"));
-    assertTrue(candidates.contains("LIMIT"));
-    assertTrue(candidates.contains("FROM"));
+    sqlCompleter.completeName(buffer, cursor, candidates, aliases, true);
+    assertEquals(17, candidates.size());
+    assertTrue(candidates.contains(new InterpreterCompletion("prod_dds", "prod_dds", CompletionType.schema.name())));
+    assertTrue(candidates.contains(new InterpreterCompletion("prod_emart", "prod_emart", CompletionType.schema.name())));
+    assertTrue(candidates.contains(new InterpreterCompletion("SUM", "SUM", CompletionType.keyword.name())));
+    assertTrue(candidates.contains(new InterpreterCompletion("SUBSTRING", "SUBSTRING", CompletionType.keyword.name())));
+    assertTrue(candidates.contains(new InterpreterCompletion("SUBCLASS_ORIGIN", "SUBCLASS_ORIGIN", CompletionType.keyword.name())));
+    assertTrue(candidates.contains(new InterpreterCompletion("SELECT", "SELECT", CompletionType.keyword.name())));
+    assertTrue(candidates.contains(new InterpreterCompletion("ORDER", "ORDER", CompletionType.keyword.name())));
+    assertTrue(candidates.contains(new InterpreterCompletion("LIMIT", "LIMIT", CompletionType.keyword.name())));
+    assertTrue(candidates.contains(new InterpreterCompletion("FROM", "FROM", CompletionType.keyword.name())));
+    assertTrue(candidates.contains(new InterpreterCompletion("financial_account", "financial_account", CompletionType.table.name())));
+    assertTrue(candidates.contains(new InterpreterCompletion("customer", "customer", CompletionType.table.name())));
+    assertTrue(candidates.contains(new InterpreterCompletion("account_id", "account_id", CompletionType.column.name())));
+    assertTrue(candidates.contains(new InterpreterCompletion("customer_rk", "customer_rk", CompletionType.column.name())));
+    assertTrue(candidates.contains(new InterpreterCompletion("account_rk", "account_rk", CompletionType.column.name())));
+    assertTrue(candidates.contains(new InterpreterCompletion("name", "name", CompletionType.column.name())));
+    assertTrue(candidates.contains(new InterpreterCompletion("birth_dt", "birth_dt", CompletionType.column.name())));
+    assertTrue(candidates.contains(new InterpreterCompletion("balance_amt", "balance_amt", CompletionType.column.name())));
   }
 
   @Test
   public void testCompleteName_SimpleSchema() {
     String buffer = "prod_";
     int cursor = 3;
-    List<CharSequence> candidates = new ArrayList<>();
+    List<InterpreterCompletion> candidates = new ArrayList<>();
     Map<String, String> aliases = new HashMap<>();
     sqlCompleter.completeName(buffer, cursor, candidates, aliases, false);
     assertEquals(2, candidates.size());
-    assertTrue(candidates.contains("prod_dds"));
-    assertTrue(candidates.contains("prod_emart"));
+    assertTrue(candidates.contains(new InterpreterCompletion("prod_dds", "prod_dds", CompletionType.schema.name())));
+    assertTrue(candidates.contains(new InterpreterCompletion("prod_emart", "prod_emart", CompletionType.schema.name())));
   }
 
   @Test
   public void testCompleteName_SimpleTable() {
     String buffer = "prod_dds.fin";
     int cursor = 11;
-    List<CharSequence> candidates = new ArrayList<>();
+    List<InterpreterCompletion> candidates = new ArrayList<>();
     Map<String, String> aliases = new HashMap<>();
     sqlCompleter.completeName(buffer, cursor, candidates, aliases, false);
     assertEquals(1, candidates.size());
-    assertTrue(candidates.contains("financial_account "));
+    assertTrue(candidates.contains(new InterpreterCompletion("financial_account", "financial_account", CompletionType.table.name())));
   }
 
   @Test
   public void testCompleteName_SimpleColumn() {
     String buffer = "prod_dds.financial_account.acc";
     int cursor = 30;
-    List<CharSequence> candidates = new ArrayList<>();
+    List<InterpreterCompletion> candidates = new ArrayList<>();
     Map<String, String> aliases = new HashMap<>();
     sqlCompleter.completeName(buffer, cursor, candidates, aliases, true);
     assertEquals(2, candidates.size());
-    assertTrue(candidates.contains("account_rk"));
-    assertTrue(candidates.contains("account_id"));
+    assertTrue(candidates.contains(new InterpreterCompletion("account_rk", "account_rk", CompletionType.column.name())));
+    assertTrue(candidates.contains(new InterpreterCompletion("account_id", "account_id", CompletionType.column.name())));
   }
 
   @Test
   public void testCompleteName_WithAlias() {
     String buffer = "a.acc";
     int cursor = 4;
-    List<CharSequence> candidates = new ArrayList<>();
+    List<InterpreterCompletion> candidates = new ArrayList<>();
     Map<String, String> aliases = new HashMap<>();
     aliases.put("a", "prod_dds.financial_account");
     sqlCompleter.completeName(buffer, cursor, candidates, aliases, true);
     assertEquals(2, candidates.size());
-    assertTrue(candidates.contains("account_rk"));
-    assertTrue(candidates.contains("account_id"));
+    assertTrue(candidates.contains(new InterpreterCompletion("account_rk", "account_rk", CompletionType.column.name())));
+    assertTrue(candidates.contains(new InterpreterCompletion("account_id", "account_id", CompletionType.column.name())));
   }
 
   @Test
   public void testCompleteName_WithAliasAndPoint() {
     String buffer = "a.";
     int cursor = 2;
-    List<CharSequence> candidates = new ArrayList<>();
+    List<InterpreterCompletion> candidates = new ArrayList<>();
     Map<String, String> aliases = new HashMap<>();
     aliases.put("a", "prod_dds.financial_account");
     sqlCompleter.completeName(buffer, cursor, candidates, aliases, true);
     assertEquals(2, candidates.size());
-    assertTrue(candidates.contains("account_rk"));
-    assertTrue(candidates.contains("account_id"));
+    assertTrue(candidates.contains(new InterpreterCompletion("account_rk", "account_rk", CompletionType.column.name())));
+    assertTrue(candidates.contains(new InterpreterCompletion("account_id", "account_id", CompletionType.column.name())));
   }
 
+  @Test
   public void testSchemaAndTable() {
-    String buffer = "select * from prod_v_emart.fi";
-    tester.buffer(buffer).from(15).to(26).expect(newHashSet("prod_v_emart ")).test();
-    tester.buffer(buffer).from(27).to(29).expect(newHashSet("financial_account ")).test();
+    String buffer = "select * from prod_emart.fi";
+    tester.buffer(buffer).from(19).to(23).expect(newHashSet(new InterpreterCompletion("prod_emart ", "prod_emart ", CompletionType.schema.name()))).test();
+    tester.buffer(buffer).from(25).to(27).expect(newHashSet(new InterpreterCompletion("financial_account ", "financial_account ", CompletionType.table.name()))).test();
   }
 
   @Test
   public void testEdges() {
     String buffer = "  ORDER  ";
-    tester.buffer(buffer).from(0).to(7).expect(newHashSet("ORDER ")).test();
-    tester.buffer(buffer).from(8).to(15).expect(newHashSet("ORDER", "SUBCLASS_ORIGIN", "SUBSTRING",
-            "prod_emart", "LIMIT", "SUM", "prod_dds", "SELECT", "FROM")).test();
+    tester.buffer(buffer).from(2).to(6).expect(newHashSet(new InterpreterCompletion("ORDER ", "ORDER ", CompletionType.keyword.name()))).test();
+    tester.buffer(buffer).from(0).to(1).expect(newHashSet(
+        new InterpreterCompletion("ORDER", "ORDER", CompletionType.keyword.name()),
+        new InterpreterCompletion("SUBCLASS_ORIGIN", "SUBCLASS_ORIGIN", CompletionType.keyword.name()),
+        new InterpreterCompletion("SUBSTRING", "SUBSTRING", CompletionType.keyword.name()),
+        new InterpreterCompletion("prod_emart", "prod_emart", CompletionType.schema.name()),
+        new InterpreterCompletion("LIMIT", "LIMIT", CompletionType.keyword.name()),
+        new InterpreterCompletion("SUM", "SUM", CompletionType.keyword.name()),
+        new InterpreterCompletion("prod_dds", "prod_dds", CompletionType.schema.name()),
+        new InterpreterCompletion("SELECT", "SELECT", CompletionType.keyword.name()),
+        new InterpreterCompletion("FROM", "FROM", CompletionType.keyword.name()),
+        new InterpreterCompletion("financial_account", "financial_account", CompletionType.table.name()),
+        new InterpreterCompletion("customer", "customer", CompletionType.table.name()),
+        new InterpreterCompletion("account_rk", "account_rk", CompletionType.column.name()),
+        new InterpreterCompletion("account_id", "account_id", CompletionType.column.name()),
+        new InterpreterCompletion("customer_rk", "customer_rk", CompletionType.column.name()),
+        new InterpreterCompletion("name", "name", CompletionType.column.name()),
+        new InterpreterCompletion("birth_dt", "birth_dt", CompletionType.column.name()),
+        new InterpreterCompletion("balance_amt", "balance_amt", CompletionType.column.name())
+    )).test();
   }
 
   @Test
   public void testMultipleWords() {
     String buffer = "SELE FRO LIM";
-    tester.buffer(buffer).from(0).to(4).expect(newHashSet("SELECT ")).test();
-    tester.buffer(buffer).from(5).to(8).expect(newHashSet("FROM ")).test();
-    tester.buffer(buffer).from(9).to(12).expect(newHashSet("LIMIT ")).test();
+    tester.buffer(buffer).from(1).to(3).expect(newHashSet(new InterpreterCompletion("SELECT ", "SELECT ", CompletionType.keyword.name()))).test();
+    tester.buffer(buffer).from(6).to(7).expect(newHashSet(new InterpreterCompletion("FROM ", "FROM ", CompletionType.keyword.name()))).test();
+    tester.buffer(buffer).from(9).to(12).expect(newHashSet(new InterpreterCompletion("LIMIT ", "LIMIT ", CompletionType.keyword.name()))).test();
   }
 
   @Test
   public void testMultiLineBuffer() {
     String buffer = " \n SELE\nFRO";
-    tester.buffer(buffer).from(0).to(7).expect(newHashSet("SELECT ")).test();
-    tester.buffer(buffer).from(8).to(11).expect(newHashSet("FROM ")).test();
+    tester.buffer(buffer).from(4).to(6).expect(newHashSet(new InterpreterCompletion("SELECT ", "SELECT ", CompletionType.keyword.name()))).test();
+    tester.buffer(buffer).from(9).to(11).expect(newHashSet(new InterpreterCompletion("FROM ", "FROM ", CompletionType.keyword.name()))).test();
   }
 
   @Test
   public void testMultipleCompletionSuggestions() {
     String buffer = "SU";
-    tester.buffer(buffer).from(0).to(2).expect(newHashSet("SUBCLASS_ORIGIN", "SUM", "SUBSTRING"))
-            .test();
+    tester.buffer(buffer).from(1).to(2).expect(newHashSet(
+        new InterpreterCompletion("SUBCLASS_ORIGIN", "SUBCLASS_ORIGIN", CompletionType.keyword.name()),
+        new InterpreterCompletion("SUM", "SUM", CompletionType.keyword.name()),
+        new InterpreterCompletion("SUBSTRING", "SUBSTRING", CompletionType.keyword.name()))
+    ).test();
   }
 
   @Test

kylin/src/main/java/org/apache/zeppelin/kylin/KylinInterpreter.java

@@ -95,7 +95,8 @@ public class KylinInterpreter extends Interpreter {
   }
 
   @Override
-  public List<InterpreterCompletion> completion(String buf, int cursor) {
+  public List<InterpreterCompletion> completion(String buf, int cursor,
+      InterpreterContext interpreterContext) {
     return null;
   }
 

lens/src/main/java/org/apache/zeppelin/lens/LensInterpreter.java

@@ -420,7 +420,8 @@ public class LensInterpreter extends Interpreter {
   }
 
   @Override
-  public List<InterpreterCompletion> completion(String buf, int cursor) {
+  public List<InterpreterCompletion> completion(String buf, int cursor,
+      InterpreterContext interpreterContext) {
     return null;
   }
   

livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterprereter.java

@@ -43,13 +43,17 @@ import javax.net.ssl.SSLContext;
 import java.io.FileInputStream;
 import java.io.IOException;
 import java.security.KeyStore;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Properties;
+import java.util.Set;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.ConcurrentHashMap;
 
+
+
 /**
  * Base class for livy interpreters.
  */
@@ -68,9 +72,8 @@ public abstract class BaseLivyInterprereter extends Interpreter {
   protected LivyVersion livyVersion;
   private RestTemplate restTemplate;
 
-  // keep tracking the mapping between paragraphId and statementId, so that we can cancel the
-  // statement after we execute it.
-  private ConcurrentHashMap<String, Integer> paragraphId2StmtIdMap = new ConcurrentHashMap<>();
+  Set<Object> paragraphsToCancel = Collections.newSetFromMap(
+      new ConcurrentHashMap<Object, Boolean>());
   private ConcurrentHashMap<String, Integer> paragraphId2StmtProgressMap =
       new ConcurrentHashMap<>();
 
@@ -163,21 +166,8 @@ public abstract class BaseLivyInterprereter extends Interpreter {
 
   @Override
   public void cancel(InterpreterContext context) {
-    if (livyVersion.isCancelSupported()) {
-      String paraId = context.getParagraphId();
-      Integer stmtId = paragraphId2StmtIdMap.get(paraId);
-      try {
-        if (stmtId != null) {
-          cancelStatement(stmtId);
-        }
-      } catch (LivyException e) {
-        LOGGER.error("Fail to cancel statement " + stmtId + " for paragraph " + paraId, e);
-      } finally {
-        paragraphId2StmtIdMap.remove(paraId);
-      }
-    } else {
-      LOGGER.warn("cancel is not supported for this version of livy: " + livyVersion);
-    }
+    paragraphsToCancel.add(context.getParagraphId());
+    LOGGER.info("Added paragraph " + context.getParagraphId() + " for cancellation.");
   }
 
   @Override
@@ -261,11 +251,12 @@ public abstract class BaseLivyInterprereter extends Interpreter {
         }
         stmtInfo = executeStatement(new ExecuteRequest(code));
       }
-      if (paragraphId != null) {
-        paragraphId2StmtIdMap.put(paragraphId, stmtInfo.id);
-      }
       // pull the statement status
       while (!stmtInfo.isAvailable()) {
+        if (paragraphId != null && paragraphsToCancel.contains(paragraphId)) {
+          cancel(stmtInfo.id, paragraphId);
+          return new InterpreterResult(InterpreterResult.Code.ERROR, "Job is cancelled");
+        }
         try {
           Thread.sleep(pullStatusInterval);
         } catch (InterruptedException e) {
@@ -285,12 +276,29 @@ public abstract class BaseLivyInterprereter extends Interpreter {
       }
     } finally {
       if (paragraphId != null) {
-        paragraphId2StmtIdMap.remove(paragraphId);
         paragraphId2StmtProgressMap.remove(paragraphId);
+        paragraphsToCancel.remove(paragraphId);
       }
     }
   }
 
+  private void cancel(int id, String paragraphId) {
+    if (livyVersion.isCancelSupported()) {
+      try {
+        LOGGER.info("Cancelling statement " + id);
+        cancelStatement(id);
+      } catch (LivyException e) {
+        LOGGER.error("Fail to cancel statement " + id + " for paragraph " + paragraphId, e);
+      }
+      finally {
+        paragraphsToCancel.remove(paragraphId);
+      }
+    } else {
+      LOGGER.warn("cancel is not supported for this version of livy: " + livyVersion);
+      paragraphsToCancel.clear();
+    }
+  }
+
   protected LivyVersion getLivyVersion() throws LivyException {
     return new LivyVersion((LivyVersionResponse.fromJson(callRestAPI("/version", "GET")).version));
   }
@@ -371,7 +379,7 @@ public abstract class BaseLivyInterprereter extends Interpreter {
 
       if (displayAppInfo) {
         InterpreterResult interpreterResult = new InterpreterResult(InterpreterResult.Code.SUCCESS);
-        interpreterResult.add(InterpreterResult.Type.TEXT, result);
+        interpreterResult.add(result);
         String appInfoHtml = "<hr/>Spark Application Id: " + sessionInfo.appId + "<br/>"
             + "Spark WebUI: <a href=\"" + sessionInfo.webUIAddress + "\">"
             + sessionInfo.webUIAddress + "</a>";

livy/src/main/java/org/apache/zeppelin/livy/LivySparkSQLInterpreter.java

@@ -229,6 +229,11 @@ public class LivySparkSQLInterpreter extends BaseLivyInterprereter {
     }
   }
 
+  @Override
+  public void cancel(InterpreterContext context) {
+    sparkInterpreter.cancel(context);
+  }
+
   @Override
   public void close() {
     this.sparkInterpreter.close();

livy/src/test/java/org/apache/zeppelin/livy/LivyInterpreterIT.java

@@ -145,6 +145,13 @@ public class LivyInterpreterIT {
         assertTrue(result.message().get(0).getData().contains("defined object Person"));
       }
 
+      // html output
+      String htmlCode = "println(\"%html <h1> hello </h1>\")";
+      result = sparkInterpreter.interpret(htmlCode, context);
+      assertEquals(InterpreterResult.Code.SUCCESS, result.code());
+      assertEquals(1, result.message().size());
+      assertEquals(InterpreterResult.Type.HTML, result.message().get(0).getType());
+
       // error
       result = sparkInterpreter.interpret("println(a)", context);
       assertEquals(InterpreterResult.Code.ERROR, result.code());
@@ -162,9 +169,9 @@ public class LivyInterpreterIT {
         Thread cancelThread = new Thread() {
           @Override
           public void run() {
-            // invoke cancel after 3 seconds to wait job starting
+            // invoke cancel after 1 millisecond to wait job starting
             try {
-              Thread.sleep(3000);
+              Thread.sleep(1);
             } catch (InterruptedException e) {
               e.printStackTrace();
             }
@@ -306,6 +313,88 @@ public class LivyInterpreterIT {
     }
   }
 
+
+  @Test
+  public void testSparkSQLCancellation() {
+    if (!checkPreCondition()) {
+      return;
+    }
+    InterpreterGroup interpreterGroup = new InterpreterGroup("group_1");
+    interpreterGroup.put("session_1", new ArrayList<Interpreter>());
+    LivySparkInterpreter sparkInterpreter = new LivySparkInterpreter(properties);
+    sparkInterpreter.setInterpreterGroup(interpreterGroup);
+    interpreterGroup.get("session_1").add(sparkInterpreter);
+    AuthenticationInfo authInfo = new AuthenticationInfo("user1");
+    MyInterpreterOutputListener outputListener = new MyInterpreterOutputListener();
+    InterpreterOutput output = new InterpreterOutput(outputListener);
+    final InterpreterContext context = new InterpreterContext("noteId", "paragraphId", "livy.spark",
+        "title", "text", authInfo, null, null, null, null, null, output);
+    sparkInterpreter.open();
+
+    final LivySparkSQLInterpreter sqlInterpreter = new LivySparkSQLInterpreter(properties);
+    interpreterGroup.get("session_1").add(sqlInterpreter);
+    sqlInterpreter.setInterpreterGroup(interpreterGroup);
+    sqlInterpreter.open();
+
+    try {
+      // detect spark version
+      InterpreterResult result = sparkInterpreter.interpret("sc.version", context);
+      assertEquals(InterpreterResult.Code.SUCCESS, result.code());
+      assertEquals(1, result.message().size());
+
+      boolean isSpark2 = isSpark2(sparkInterpreter, context);
+
+      // test DataFrame api
+      if (!isSpark2) {
+        result = sparkInterpreter.interpret(
+            "val df=sqlContext.createDataFrame(Seq((\"hello\",20))).toDF(\"col_1\", \"col_2\")\n"
+                + "df.collect()", context);
+        assertEquals(InterpreterResult.Code.SUCCESS, result.code());
+        assertEquals(1, result.message().size());
+        assertTrue(result.message().get(0).getData()
+            .contains("Array[org.apache.spark.sql.Row] = Array([hello,20])"));
+      } else {
+        result = sparkInterpreter.interpret(
+            "val df=spark.createDataFrame(Seq((\"hello\",20))).toDF(\"col_1\", \"col_2\")\n"
+                + "df.collect()", context);
+        assertEquals(InterpreterResult.Code.SUCCESS, result.code());
+        assertEquals(1, result.message().size());
+        assertTrue(result.message().get(0).getData()
+            .contains("Array[org.apache.spark.sql.Row] = Array([hello,20])"));
+      }
+      sparkInterpreter.interpret("df.registerTempTable(\"df\")", context);
+
+      // cancel
+      if (sqlInterpreter.getLivyVersion().newerThanEquals(LivyVersion.LIVY_0_3_0)) {
+        Thread cancelThread = new Thread() {
+          @Override
+          public void run() {
+            sqlInterpreter.cancel(context);
+          }
+        };
+        cancelThread.start();
+        //sleep so that cancelThread performs a cancel.
+        try {
+          Thread.sleep(1);
+        } catch (InterruptedException e) {
+          e.printStackTrace();
+        }
+        result = sqlInterpreter
+            .interpret("select count(1) from df", context);
+        if (result.code().equals(InterpreterResult.Code.ERROR)) {
+          String message = result.message().get(0).getData();
+          // 2 possibilities, sometimes livy doesn't return the real cancel exception
+          assertTrue(message.contains("cancelled part of cancelled job group") ||
+              message.contains("Job is cancelled"));
+        }
+      }
+    } catch (LivyException e) {
+    } finally {
+      sparkInterpreter.close();
+      sqlInterpreter.close();
+    }
+  }
+
   @Test
   public void testStringWithTruncation() {
     if (!checkPreCondition()) {
@@ -495,9 +584,9 @@ public class LivyInterpreterIT {
         Thread cancelThread = new Thread() {
           @Override
           public void run() {
-            // invoke cancel after 3 seconds to wait job starting
+            // invoke cancel after 1 millisecond to wait job starting
             try {
-              Thread.sleep(3000);
+              Thread.sleep(1);
             } catch (InterruptedException e) {
               e.printStackTrace();
             }
@@ -544,8 +633,15 @@ public class LivyInterpreterIT {
       InterpreterResult result = sparkInterpreter.interpret("sc.version", context);
       assertEquals(InterpreterResult.Code.SUCCESS, result.code());
       assertEquals(2, result.message().size());
-
       assertTrue(result.message().get(1).getData().contains("Spark Application Id"));
+
+      // html output
+      String htmlCode = "println(\"%html <h1> hello </h1>\")";
+      result = sparkInterpreter.interpret(htmlCode, context);
+      assertEquals(InterpreterResult.Code.SUCCESS, result.code());
+      assertEquals(2, result.message().size());
+      assertEquals(InterpreterResult.Type.HTML, result.message().get(0).getType());
+
     } finally {
       sparkInterpreter.close();
     }
@@ -586,9 +682,9 @@ public class LivyInterpreterIT {
         Thread cancelThread = new Thread() {
           @Override
           public void run() {
-            // invoke cancel after 3 seconds to wait job starting
+            // invoke cancel after 1 millisecond to wait job starting
             try {
-              Thread.sleep(3000);
+              Thread.sleep(1);
             } catch (InterruptedException e) {
               e.printStackTrace();
             }

markdown/src/main/java/org/apache/zeppelin/markdown/Markdown.java

@@ -124,7 +124,8 @@ public class Markdown extends Interpreter {
   }
 
   @Override
-  public List<InterpreterCompletion> completion(String buf, int cursor) {
+  public List<InterpreterCompletion> completion(String buf, int cursor,
+      InterpreterContext interpreterContext) {
     return null;
   }
 }

pig/pom.xml

@@ -47,6 +47,12 @@
             <artifactId>zeppelin-interpreter</artifactId>
             <version>${project.version}</version>
             <scope>provided</scope>
+            <exclusions>
+              <exclusion>
+                <groupId>jline</groupId>
+                <artifactId>jline</artifactId>
+              </exclusion>
+            </exclusions>
         </dependency>
 
         <dependency>

python/src/main/java/org/apache/zeppelin/python/PythonInterpreter.java

@@ -222,7 +222,7 @@ public class PythonInterpreter extends Interpreter implements ExecuteResultHandl
     // Add matplotlib display hook
     InterpreterGroup intpGroup = getInterpreterGroup();
     if (intpGroup != null && intpGroup.getInterpreterHookRegistry() != null) {
-      registerHook(HookType.POST_EXEC_DEV, "z._displayhook()");
+      registerHook(HookType.POST_EXEC_DEV, "__zeppelin__._displayhook()");
     }
     // Add matplotlib display hook
     try {
@@ -435,7 +435,8 @@ public class PythonInterpreter extends Interpreter implements ExecuteResultHandl
   }
 
   @Override
-  public List<InterpreterCompletion> completion(String buf, int cursor) {
+  public List<InterpreterCompletion> completion(String buf, int cursor,
+      InterpreterContext interpreterContext) {
     return null;
   }
 

python/src/main/java/org/apache/zeppelin/python/PythonInterpreterPandasSql.java

@@ -87,7 +87,8 @@ public class PythonInterpreterPandasSql extends Interpreter {
     LOG.info("Running SQL query: '{}' over Pandas DataFrame", st);
     Interpreter python = getPythonInterpreter();
 
-    return python.interpret("z.show(pysqldf('" + st + "'))\nz._displayhook()", context);
+    return python.interpret(
+            "__zeppelin__.show(pysqldf('" + st + "'))\n__zeppelin__._displayhook()", context);
   }
 
   @Override

python/src/main/resources/python/zeppelin_python.py

@@ -195,6 +195,7 @@ host = "127.0.0.1"
 if len(sys.argv) >= 3:
   host = sys.argv[2]
 
+_zcUserQueryNameSpace = {}
 client = GatewayClient(address=host, port=int(sys.argv[1]))
 
 #gateway = JavaGateway(client, auto_convert = True)
@@ -204,8 +205,11 @@ intp = gateway.entry_point
 intp.onPythonScriptInitialized(os.getpid())
 
 java_import(gateway.jvm, "org.apache.zeppelin.display.Input")
-z = PyZeppelinContext(intp)
-z._setup_matplotlib()
+z = __zeppelin__ = PyZeppelinContext(intp)
+__zeppelin__._setup_matplotlib()
+
+_zcUserQueryNameSpace["__zeppelin__"] = __zeppelin__
+_zcUserQueryNameSpace["z"] = z
 
 output = Logger()
 sys.stdout = output
@@ -227,7 +231,7 @@ while True :
       global_hook = None
 
     try:
-      user_hook = z.getHook('post_exec')
+      user_hook = __zeppelin__.getHook('post_exec')
     except:
       user_hook = None
       
@@ -263,17 +267,17 @@ while True :
         for node in to_run_exec:
           mod = ast.Module([node])
           code = compile(mod, '<stdin>', 'exec')
-          exec(code)
+          exec(code, _zcUserQueryNameSpace)
 
         for node in to_run_single:
           mod = ast.Interactive([node])
           code = compile(mod, '<stdin>', 'single')
-          exec(code)
+          exec(code, _zcUserQueryNameSpace)
 
         for node in to_run_hooks:
           mod = ast.Module([node])
           code = compile(mod, '<stdin>', 'exec')
-          exec(code)
+          exec(code, _zcUserQueryNameSpace)
       except:
         raise Exception(traceback.format_exc())
 

python/src/test/java/org/apache/zeppelin/python/PythonInterpreterTest.java

@@ -106,6 +106,19 @@ public class PythonInterpreterTest implements InterpreterOutputListener {
     assertTrue(new String(out.getOutputAt(0).toByteArray()).contains("hi\nhi\nhi"));
  }
 
+  @Test
+  public void testRedefinitionZeppelinContext() {
+    String pyRedefinitionCode = "z = 1\n";
+    String pyRestoreCode = "z = __zeppelin__\n";
+    String pyValidCode = "z.input(\"test\")\n";
+
+    assertEquals(InterpreterResult.Code.SUCCESS, pythonInterpreter.interpret(pyValidCode, context).code());
+    assertEquals(InterpreterResult.Code.SUCCESS, pythonInterpreter.interpret(pyRedefinitionCode, context).code());
+    assertEquals(InterpreterResult.Code.ERROR, pythonInterpreter.interpret(pyValidCode, context).code());
+    assertEquals(InterpreterResult.Code.SUCCESS, pythonInterpreter.interpret(pyRestoreCode, context).code());
+    assertEquals(InterpreterResult.Code.SUCCESS, pythonInterpreter.interpret(pyValidCode, context).code());
+  }
+
   @Override
   public void onUpdateAll(InterpreterOutput out) {
 

r/src/main/java/org/apache/zeppelin/rinterpreter/KnitR.java

@@ -77,8 +77,9 @@ public class KnitR extends Interpreter implements WrappedInterpreter {
   }
 
   @Override
-  public List<InterpreterCompletion> completion(String s, int i) {
-    List completion = intp.completion(s, i);
+  public List<InterpreterCompletion> completion(String s, int i,
+      InterpreterContext interpreterContext) {
+    List completion = intp.completion(s, i, interpreterContext);
     return completion;
   }
 

r/src/main/java/org/apache/zeppelin/rinterpreter/RRepl.java

@@ -77,8 +77,9 @@ public class RRepl extends Interpreter implements WrappedInterpreter {
   }
 
   @Override
-  public List<InterpreterCompletion> completion(String s, int i) {
-    List completion = intp.completion(s, i);
+  public List<InterpreterCompletion> completion(String s, int i,
+      InterpreterContext interpreterContext) {
+    List completion = intp.completion(s, i, interpreterContext);
     return completion;
   }
 

scalding/src/main/java/org/apache/zeppelin/scalding/ScaldingInterpreter.java

@@ -270,7 +270,8 @@ public class ScaldingInterpreter extends Interpreter {
   }
 
   @Override
-  public List<InterpreterCompletion> completion(String buf, int cursor) {
+  public List<InterpreterCompletion> completion(String buf, int cursor,
+      InterpreterContext interpreterContext) {
     return NO_COMPLETION;
   }
 

shell/src/main/java/org/apache/zeppelin/shell/ShellInterpreter.java

@@ -138,7 +138,8 @@ public class ShellInterpreter extends Interpreter {
   }
 
   @Override
-  public List<InterpreterCompletion> completion(String buf, int cursor) {
+  public List<InterpreterCompletion> completion(String buf, int cursor,
+      InterpreterContext interpreterContext) {
     return null;
   }
 

spark/src/main/java/org/apache/zeppelin/spark/DepInterpreter.java

@@ -33,6 +33,8 @@ import java.util.Properties;
 
 import com.google.common.reflect.TypeToken;
 import com.google.gson.Gson;
+
+import org.apache.commons.lang.StringUtils;
 import org.apache.spark.repl.SparkILoop;
 import org.apache.zeppelin.interpreter.Interpreter;
 import org.apache.zeppelin.interpreter.InterpreterContext;
@@ -284,7 +286,8 @@ public class DepInterpreter extends Interpreter {
   }
 
   @Override
-  public List<InterpreterCompletion> completion(String buf, int cursor) {
+  public List<InterpreterCompletion> completion(String buf, int cursor,
+      InterpreterContext interpreterContext) {
     if (Utils.isScala2_10()) {
       ScalaCompleter c = (ScalaCompleter) Utils.invokeMethod(completer, "completer");
       Candidates ret = c.complete(buf, cursor);
@@ -293,7 +296,7 @@ public class DepInterpreter extends Interpreter {
       List<InterpreterCompletion> completions = new LinkedList<>();
 
       for (String candidate : candidates) {
-        completions.add(new InterpreterCompletion(candidate, candidate));
+        completions.add(new InterpreterCompletion(candidate, candidate, StringUtils.EMPTY));
       }
 
       return completions;

spark/src/main/java/org/apache/zeppelin/spark/PySparkInterpreter.java

@@ -42,6 +42,7 @@ import org.apache.commons.exec.ExecuteResultHandler;
 import org.apache.commons.exec.ExecuteWatchdog;
 import org.apache.commons.exec.PumpStreamHandler;
 import org.apache.commons.exec.environment.EnvironmentUtils;
+import org.apache.commons.lang.StringUtils;
 import org.apache.spark.SparkConf;
 import org.apache.spark.api.java.JavaSparkContext;
 import org.apache.spark.sql.SQLContext;
@@ -113,7 +114,7 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand
     // Add matplotlib display hook
     InterpreterGroup intpGroup = getInterpreterGroup();
     if (intpGroup != null && intpGroup.getInterpreterHookRegistry() != null) {
-      registerHook(HookType.POST_EXEC_DEV, "z._displayhook()");
+      registerHook(HookType.POST_EXEC_DEV, "__zeppelin__._displayhook()");
     }
     DepInterpreter depInterpreter = getDepInterpreter();
 
@@ -390,9 +391,9 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand
       return new InterpreterResult(Code.ERROR, errorMessage);
     }
     String jobGroup = Utils.buildJobGroupId(context);
-    ZeppelinContext z = sparkInterpreter.getZeppelinContext();
-    z.setInterpreterContext(context);
-    z.setGui(context.getGui());
+    ZeppelinContext __zeppelin__ = sparkInterpreter.getZeppelinContext();
+    __zeppelin__.setInterpreterContext(context);
+    __zeppelin__.setGui(context.getGui());
     pythonInterpretRequest = new PythonInterpretRequest(st, jobGroup);
     statementOutput = null;
 
@@ -457,7 +458,8 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand
 
 
   @Override
-  public List<InterpreterCompletion> completion(String buf, int cursor) {
+  public List<InterpreterCompletion> completion(String buf, int cursor,
+      InterpreterContext interpreterContext) {
     if (buf.length() < cursor) {
       cursor = buf.length();
     }
@@ -508,7 +510,7 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand
 
     List<InterpreterCompletion> results = new LinkedList<>();
     for (String name: completionList) {
-      results.add(new InterpreterCompletion(name, name));
+      results.add(new InterpreterCompletion(name, name, StringUtils.EMPTY));
     }
     return results;
   }

spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java

@@ -255,7 +255,7 @@ public class SparkInterpreter extends Interpreter {
    */
   private boolean hiveClassesArePresent() {
     try {
-      this.getClass().forName("org.apache.spark.sql.hive.HiveSessionState");
+      this.getClass().forName("org.apache.spark.sql.hive.execution.InsertIntoHiveTable");
       this.getClass().forName("org.apache.hadoop.hive.conf.HiveConf");
       return true;
     } catch (ClassNotFoundException | NoClassDefFoundError e) {
@@ -1083,7 +1083,8 @@ public class SparkInterpreter extends Interpreter {
   }
 
   @Override
-  public List<InterpreterCompletion> completion(String buf, int cursor) {
+  public List<InterpreterCompletion> completion(String buf, int cursor,
+      InterpreterContext interpreterContext) {
     if (completer == null) {
       logger.warn("Can't find completer");
       return new LinkedList<>();
@@ -1105,7 +1106,7 @@ public class SparkInterpreter extends Interpreter {
     List<InterpreterCompletion> completions = new LinkedList<>();
 
     for (String candidate : candidates) {
-      completions.add(new InterpreterCompletion(candidate, candidate));
+      completions.add(new InterpreterCompletion(candidate, candidate, StringUtils.EMPTY));
     }
 
     return completions;

spark/src/main/java/org/apache/zeppelin/spark/SparkRInterpreter.java

@@ -212,7 +212,8 @@ public class SparkRInterpreter extends Interpreter {
   }
 
   @Override
-  public List<InterpreterCompletion> completion(String buf, int cursor) {
+  public List<InterpreterCompletion> completion(String buf, int cursor,
+      InterpreterContext interpreterContext) {
     return new ArrayList<>();
   }
 

spark/src/main/java/org/apache/zeppelin/spark/SparkSqlInterpreter.java

@@ -177,7 +177,8 @@ public class SparkSqlInterpreter extends Interpreter {
   }
 
   @Override
-  public List<InterpreterCompletion> completion(String buf, int cursor) {
+  public List<InterpreterCompletion> completion(String buf, int cursor,
+      InterpreterContext interpreterContext) {
     return null;
   }
 }

spark/src/main/resources/python/zeppelin_pyspark.py

@@ -271,19 +271,37 @@ else:
 
 java_import(gateway.jvm, "scala.Tuple2")
 
+_zcUserQueryNameSpace = {}
+
 jconf = intp.getSparkConf()
 conf = SparkConf(_jvm = gateway.jvm, _jconf = jconf)
-sc = SparkContext(jsc=jsc, gateway=gateway, conf=conf)
-if sparkVersion.isSpark2():
-  spark = SparkSession(sc, intp.getSparkSession())
-  sqlc = spark._wrapped
-else:
-  sqlc = SQLContext(sparkContext=sc, sqlContext=intp.getSQLContext())
-sqlContext = sqlc
+sc = _zsc_ = SparkContext(jsc=jsc, gateway=gateway, conf=conf)
+_zcUserQueryNameSpace["_zsc_"] = _zsc_
+_zcUserQueryNameSpace["sc"] = sc
 
-completion = PySparkCompletion(intp)
-z = PyZeppelinContext(intp.getZeppelinContext())
-z._setup_matplotlib()
+if sparkVersion.isSpark2():
+  spark = __zSpark__ = SparkSession(sc, intp.getSparkSession())
+  sqlc = __zSqlc__ = __zSpark__._wrapped
+  _zcUserQueryNameSpace["sqlc"] = sqlc
+  _zcUserQueryNameSpace["__zSqlc__"] = __zSqlc__
+  _zcUserQueryNameSpace["spark"] = spark
+  _zcUserQueryNameSpace["__zSpark__"] = __zSpark__
+else:
+  sqlc = __zSqlc__ = SQLContext(sparkContext=sc, sqlContext=intp.getSQLContext())
+  _zcUserQueryNameSpace["sqlc"] = sqlc
+  _zcUserQueryNameSpace["__zSqlc__"] = sqlc
+
+sqlContext = __zSqlc__
+_zcUserQueryNameSpace["sqlContext"] = sqlContext
+
+completion = __zeppelin_completion__ = PySparkCompletion(intp)
+_zcUserQueryNameSpace["completion"] = completion
+_zcUserQueryNameSpace["__zeppelin_completion__"] = __zeppelin_completion__
+
+z = __zeppelin__ = PyZeppelinContext(intp.getZeppelinContext())
+__zeppelin__._setup_matplotlib()
+_zcUserQueryNameSpace["z"] = z
+_zcUserQueryNameSpace["__zeppelin__"] = __zeppelin__
 
 while True :
   req = intp.getStatements()
@@ -299,7 +317,7 @@ while True :
       global_hook = None
       
     try:
-      user_hook = z.getHook('post_exec')
+      user_hook = __zeppelin__.getHook('post_exec')
     except:
       user_hook = None
       
@@ -334,17 +352,17 @@ while True :
         for node in to_run_exec:
           mod = ast.Module([node])
           code = compile(mod, '<stdin>', 'exec')
-          exec(code)
+          exec(code, _zcUserQueryNameSpace)
 
         for node in to_run_single:
           mod = ast.Interactive([node])
           code = compile(mod, '<stdin>', 'single')
-          exec(code)
+          exec(code, _zcUserQueryNameSpace)
           
         for node in to_run_hooks:
           mod = ast.Module([node])
           code = compile(mod, '<stdin>', 'exec')
-          exec(code)
+          exec(code, _zcUserQueryNameSpace)
       except:
         raise Exception(traceback.format_exc())
 

spark/src/test/java/org/apache/zeppelin/spark/PySparkInterpreterTest.java

@@ -118,11 +118,26 @@ public class PySparkInterpreterTest {
   @Test
   public void testCompletion() {
     if (getSparkVersionNumber() > 11) {
-      List<InterpreterCompletion> completions = pySparkInterpreter.completion("sc.", "sc.".length());
+      List<InterpreterCompletion> completions = pySparkInterpreter.completion("sc.", "sc.".length(), null);
       assertTrue(completions.size() > 0);
     }
   }
 
+  @Test
+  public void testRedefinitionZeppelinContext() {
+    if (getSparkVersionNumber() > 11) {
+      String redefinitionCode = "z = 1\n";
+      String restoreCode = "z = __zeppelin__\n";
+      String validCode = "z.input(\"test\")\n";
+
+      assertEquals(InterpreterResult.Code.SUCCESS, pySparkInterpreter.interpret(validCode, context).code());
+      assertEquals(InterpreterResult.Code.SUCCESS, pySparkInterpreter.interpret(redefinitionCode, context).code());
+      assertEquals(InterpreterResult.Code.ERROR, pySparkInterpreter.interpret(validCode, context).code());
+      assertEquals(InterpreterResult.Code.SUCCESS, pySparkInterpreter.interpret(restoreCode, context).code());
+      assertEquals(InterpreterResult.Code.SUCCESS, pySparkInterpreter.interpret(validCode, context).code());
+    }
+  }
+
   private class infinityPythonJob implements Runnable {
     @Override
     public void run() {

spark/src/test/java/org/apache/zeppelin/spark/SparkInterpreterTest.java

@@ -301,7 +301,7 @@ public class SparkInterpreterTest {
 
   @Test
   public void testCompletion() {
-    List<InterpreterCompletion> completions = repl.completion("sc.", "sc.".length());
+    List<InterpreterCompletion> completions = repl.completion("sc.", "sc.".length(), null);
     assertTrue(completions.size() > 0);
   }
 

zeppelin-interpreter/pom.xml

@@ -43,6 +43,7 @@
     <aether.version>1.12</aether.version>
     <maven.aeither.provider.version>3.0.3</maven.aeither.provider.version>
     <wagon.version>1.0</wagon.version>
+    <jline.version>2.12.1</jline.version>
 
     <!--plugin versions-->
     <plugin.shade.version>2.3</plugin.shade.version>
@@ -202,6 +203,17 @@
       <version>${wagon.version}</version>
     </dependency>
 
+    <dependency>
+      <groupId>jline</groupId>
+      <artifactId>jline</artifactId>
+      <version>${jline.version}</version>
+    </dependency>
+
+    <dependency>
+      <groupId>com.google.guava</groupId>
+      <artifactId>guava</artifactId>
+    </dependency>
+
     <dependency>
       <groupId>junit</groupId>
       <artifactId>junit</artifactId>

zeppelin-interpreter/src/main/java/org/apache/zeppelin/completer/CompletionType.java

@@ -0,0 +1,28 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.zeppelin.completer;
+
+/**
+ * Types of completion
+ */
+public enum CompletionType {
+  schema,
+  table,
+  column,
+  setting,
+  command,
+  keyword,
+  path
+}

zeppelin-interpreter/src/main/java/org/apache/zeppelin/completer/StringsCompleter.java

@@ -0,0 +1,77 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.zeppelin.completer;
+
+import java.util.Collection;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Set;
+import java.util.SortedSet;
+import java.util.TreeSet;
+
+import jline.console.completer.Completer;
+import jline.internal.Preconditions;
+
+/**
+ * Case-insensitive completer for a set of strings.
+ */
+public class StringsCompleter implements Completer {
+  private final SortedSet<String> strings = new TreeSet<String>(new Comparator<String>() {
+    @Override
+    public int compare(String o1, String o2) {
+      return o1.compareToIgnoreCase(o2);
+    }
+  });
+
+  public StringsCompleter() {
+  }
+
+  public StringsCompleter(final Collection<String> strings) {
+    Preconditions.checkNotNull(strings);
+    getStrings().addAll(strings);
+  }
+
+  public Collection<String> getStrings() {
+    return strings;
+  }
+
+  public int complete(final String buffer, final int cursor, final List<CharSequence> candidates) {
+    return completeCollection(buffer, cursor, candidates);
+  }
+
+  public int complete(final String buffer, final int cursor, final Set<CharSequence> candidates) {
+    return completeCollection(buffer, cursor, candidates);
+  }
+
+  private int completeCollection(final String buffer, final int cursor,
+      final Collection<CharSequence> candidates) {
+    Preconditions.checkNotNull(candidates);
+    if (buffer == null) {
+      candidates.addAll(strings);
+    } else {
+      String bufferTmp = buffer.toUpperCase();
+      for (String match : strings.tailSet(buffer)) {
+        String matchTmp = match.toUpperCase();
+        if (!matchTmp.startsWith(bufferTmp)) {
+          break;
+        }
+
+        candidates.add(match);
+      }
+    }
+
+    return candidates.isEmpty() ? -1 : 0;
+  }
+}

zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/ClassloaderInterpreter.java

@@ -152,11 +152,12 @@ public class ClassloaderInterpreter
   }
 
   @Override
-  public List<InterpreterCompletion> completion(String buf, int cursor) {
+  public List<InterpreterCompletion> completion(String buf, int cursor,
+      InterpreterContext interpreterContext) {
     ClassLoader oldcl = Thread.currentThread().getContextClassLoader();
     Thread.currentThread().setContextClassLoader(cl);
     try {
-      List completion = intp.completion(buf, cursor);
+      List completion = intp.completion(buf, cursor, interpreterContext);
       return completion;
     } catch (Exception e) {
       throw new InterpreterException(e);

zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/Interpreter.java

@@ -102,10 +102,12 @@ public abstract class Interpreter {
    *
    * @param buf statements
    * @param cursor cursor position in statements
+   * @param interpreterContext
    * @return list of possible completion. Return empty list if there're nothing to return.
    */
   @ZeppelinApi
-  public List<InterpreterCompletion> completion(String buf, int cursor) {
+  public List<InterpreterCompletion> completion(String buf, int cursor,
+      InterpreterContext interpreterContext)  {
     return null;
   }
 

zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/LazyOpenInterpreter.java

@@ -121,9 +121,10 @@ public class LazyOpenInterpreter
   }
 
   @Override
-  public List<InterpreterCompletion> completion(String buf, int cursor) {
+  public List<InterpreterCompletion> completion(String buf, int cursor,
+      InterpreterContext interpreterContext) {
     open();
-    List completion = intp.completion(buf, cursor);
+    List completion = intp.completion(buf, cursor, interpreterContext);
     return completion;
   }
 

zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java

@@ -481,7 +481,8 @@ public class RemoteInterpreter extends Interpreter {
 
 
   @Override
-  public List<InterpreterCompletion> completion(String buf, int cursor) {
+  public List<InterpreterCompletion> completion(String buf, int cursor,
+      InterpreterContext interpreterContext) {
     RemoteInterpreterProcess interpreterProcess = getInterpreterProcess();
     Client client = null;
     try {
@@ -492,7 +493,8 @@ public class RemoteInterpreter extends Interpreter {
 
     boolean broken = false;
     try {
-      List completion = client.completion(sessionKey, className, buf, cursor);
+      List completion = client.completion(sessionKey, className, buf, cursor,
+          convert(interpreterContext));
       return completion;
     } catch (TException e) {
       broken = true;

zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java

@@ -562,10 +562,10 @@ public class RemoteInterpreterServer
 
   @Override
   public List<InterpreterCompletion> completion(String noteId,
-      String className, String buf, int cursor)
+      String className, String buf, int cursor, RemoteInterpreterContext remoteInterpreterContext)
       throws TException {
     Interpreter intp = getInterpreter(noteId, className);
-    List completion = intp.completion(buf, cursor);
+    List completion = intp.completion(buf, cursor, convert(remoteInterpreterContext, null));
     return completion;
   }
 

zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/InterpreterCompletion.java

@@ -51,12 +51,13 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 @SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"})
-@Generated(value = "Autogenerated by Thrift Compiler (0.9.2)", date = "2017-1-25")
+@Generated(value = "Autogenerated by Thrift Compiler (0.9.2)", date = "2017-3-27")
 public class InterpreterCompletion implements org.apache.thrift.TBase<InterpreterCompletion, InterpreterCompletion._Fields>, java.io.Serializable, Cloneable, Comparable<InterpreterCompletion> {
   private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("InterpreterCompletion");
 
   private static final org.apache.thrift.protocol.TField NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("name", org.apache.thrift.protocol.TType.STRING, (short)1);
   private static final org.apache.thrift.protocol.TField VALUE_FIELD_DESC = new org.apache.thrift.protocol.TField("value", org.apache.thrift.protocol.TType.STRING, (short)2);
+  private static final org.apache.thrift.protocol.TField META_FIELD_DESC = new org.apache.thrift.protocol.TField("meta", org.apache.thrift.protocol.TType.STRING, (short)3);
 
   private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
   static {
@@ -66,11 +67,13 @@ public class InterpreterCompletion implements org.apache.thrift.TBase<Interprete
 
   public String name; // required
   public String value; // required
+  public String meta; // required
 
   /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
   public enum _Fields implements org.apache.thrift.TFieldIdEnum {
     NAME((short)1, "name"),
-    VALUE((short)2, "value");
+    VALUE((short)2, "value"),
+    META((short)3, "meta");
 
     private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
 
@@ -89,6 +92,8 @@ public class InterpreterCompletion implements org.apache.thrift.TBase<Interprete
           return NAME;
         case 2: // VALUE
           return VALUE;
+        case 3: // META
+          return META;
         default:
           return null;
       }
@@ -136,6 +141,8 @@ public class InterpreterCompletion implements org.apache.thrift.TBase<Interprete
         new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
     tmpMap.put(_Fields.VALUE, new org.apache.thrift.meta_data.FieldMetaData("value", org.apache.thrift.TFieldRequirementType.DEFAULT, 
         new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
+    tmpMap.put(_Fields.META, new org.apache.thrift.meta_data.FieldMetaData("meta", org.apache.thrift.TFieldRequirementType.DEFAULT, 
+        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
     metaDataMap = Collections.unmodifiableMap(tmpMap);
     org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(InterpreterCompletion.class, metaDataMap);
   }
@@ -145,11 +152,13 @@ public class InterpreterCompletion implements org.apache.thrift.TBase<Interprete
 
   public InterpreterCompletion(
     String name,
-    String value)
+    String value,
+    String meta)
   {
     this();
     this.name = name;
     this.value = value;
+    this.meta = meta;
   }
 
   /**
@@ -162,6 +171,9 @@ public class InterpreterCompletion implements org.apache.thrift.TBase<Interprete
     if (other.isSetValue()) {
       this.value = other.value;
     }
+    if (other.isSetMeta()) {
+      this.meta = other.meta;
+    }
   }
 
   public InterpreterCompletion deepCopy() {
@@ -172,6 +184,7 @@ public class InterpreterCompletion implements org.apache.thrift.TBase<Interprete
   public void clear() {
     this.name = null;
     this.value = null;
+    this.meta = null;
   }
 
   public String getName() {
@@ -222,6 +235,30 @@ public class InterpreterCompletion implements org.apache.thrift.TBase<Interprete
     }
   }
 
+  public String getMeta() {
+    return this.meta;
+  }
+
+  public InterpreterCompletion setMeta(String meta) {
+    this.meta = meta;
+    return this;
+  }
+
+  public void unsetMeta() {
+    this.meta = null;
+  }
+
+  /** Returns true if field meta is set (has been assigned a value) and false otherwise */
+  public boolean isSetMeta() {
+    return this.meta != null;
+  }
+
+  public void setMetaIsSet(boolean value) {
+    if (!value) {
+      this.meta = null;
+    }
+  }
+
   public void setFieldValue(_Fields field, Object value) {
     switch (field) {
     case NAME:
@@ -240,6 +277,14 @@ public class InterpreterCompletion implements org.apache.thrift.TBase<Interprete
       }
       break;
 
+    case META:
+      if (value == null) {
+        unsetMeta();
+      } else {
+        setMeta((String)value);
+      }
+      break;
+
     }
   }
 
@@ -251,6 +296,9 @@ public class InterpreterCompletion implements org.apache.thrift.TBase<Interprete
     case VALUE:
       return getValue();
 
+    case META:
+      return getMeta();
+
     }
     throw new IllegalStateException();
   }
@@ -266,6 +314,8 @@ public class InterpreterCompletion implements org.apache.thrift.TBase<Interprete
       return isSetName();
     case VALUE:
       return isSetValue();
+    case META:
+      return isSetMeta();
     }
     throw new IllegalStateException();
   }
@@ -301,6 +351,15 @@ public class InterpreterCompletion implements org.apache.thrift.TBase<Interprete
         return false;
     }
 
+    boolean this_present_meta = true && this.isSetMeta();
+    boolean that_present_meta = true && that.isSetMeta();
+    if (this_present_meta || that_present_meta) {
+      if (!(this_present_meta && that_present_meta))
+        return false;
+      if (!this.meta.equals(that.meta))
+        return false;
+    }
+
     return true;
   }
 
@@ -318,6 +377,11 @@ public class InterpreterCompletion implements org.apache.thrift.TBase<Interprete
     if (present_value)
       list.add(value);
 
+    boolean present_meta = true && (isSetMeta());
+    list.add(present_meta);
+    if (present_meta)
+      list.add(meta);
+
     return list.hashCode();
   }
 
@@ -349,6 +413,16 @@ public class InterpreterCompletion implements org.apache.thrift.TBase<Interprete
         return lastComparison;
       }
     }
+    lastComparison = Boolean.valueOf(isSetMeta()).compareTo(other.isSetMeta());
+    if (lastComparison != 0) {
+      return lastComparison;
+    }
+    if (isSetMeta()) {
+      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.meta, other.meta);
+      if (lastComparison != 0) {
+        return lastComparison;
+      }
+    }
     return 0;
   }
 
@@ -384,6 +458,14 @@ public class InterpreterCompletion implements org.apache.thrift.TBase<Interprete
       sb.append(this.value);
     }
     first = false;
+    if (!first) sb.append(", ");
+    sb.append("meta:");
+    if (this.meta == null) {
+      sb.append("null");
+    } else {
+      sb.append(this.meta);
+    }
+    first = false;
     sb.append(")");
     return sb.toString();
   }
@@ -443,6 +525,14 @@ public class InterpreterCompletion implements org.apache.thrift.TBase<Interprete
               org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
             }
             break;
+          case 3: // META
+            if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
+              struct.meta = iprot.readString();
+              struct.setMetaIsSet(true);
+            } else { 
+              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+            }
+            break;
           default:
             org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
         }
@@ -468,6 +558,11 @@ public class InterpreterCompletion implements org.apache.thrift.TBase<Interprete
         oprot.writeString(struct.value);
         oprot.writeFieldEnd();
       }
+      if (struct.meta != null) {
+        oprot.writeFieldBegin(META_FIELD_DESC);
+        oprot.writeString(struct.meta);
+        oprot.writeFieldEnd();
+      }
       oprot.writeFieldStop();
       oprot.writeStructEnd();
     }
@@ -492,19 +587,25 @@ public class InterpreterCompletion implements org.apache.thrift.TBase<Interprete
       if (struct.isSetValue()) {
         optionals.set(1);
       }
-      oprot.writeBitSet(optionals, 2);
+      if (struct.isSetMeta()) {
+        optionals.set(2);
+      }
+      oprot.writeBitSet(optionals, 3);
       if (struct.isSetName()) {
         oprot.writeString(struct.name);
       }
       if (struct.isSetValue()) {
         oprot.writeString(struct.value);
       }
+      if (struct.isSetMeta()) {
+        oprot.writeString(struct.meta);
+      }
     }
 
     @Override
     public void read(org.apache.thrift.protocol.TProtocol prot, InterpreterCompletion struct) throws org.apache.thrift.TException {
       TTupleProtocol iprot = (TTupleProtocol) prot;
-      BitSet incoming = iprot.readBitSet(2);
+      BitSet incoming = iprot.readBitSet(3);
       if (incoming.get(0)) {
         struct.name = iprot.readString();
         struct.setNameIsSet(true);
@@ -513,6 +614,10 @@ public class InterpreterCompletion implements org.apache.thrift.TBase<Interprete
         struct.value = iprot.readString();
         struct.setValueIsSet(true);
       }
+      if (incoming.get(2)) {
+        struct.meta = iprot.readString();
+        struct.setMetaIsSet(true);
+      }
     }
   }
 

zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteApplicationResult.java

@@ -51,7 +51,7 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 @SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"})
-@Generated(value = "Autogenerated by Thrift Compiler (0.9.2)", date = "2017-1-25")
+@Generated(value = "Autogenerated by Thrift Compiler (0.9.2)", date = "2017-3-27")
 public class RemoteApplicationResult implements org.apache.thrift.TBase<RemoteApplicationResult, RemoteApplicationResult._Fields>, java.io.Serializable, Cloneable, Comparable<RemoteApplicationResult> {
   private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("RemoteApplicationResult");
 

zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterContext.java

@@ -51,7 +51,7 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 @SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"})
-@Generated(value = "Autogenerated by Thrift Compiler (0.9.2)", date = "2017-1-25")
+@Generated(value = "Autogenerated by Thrift Compiler (0.9.2)", date = "2017-3-27")
 public class RemoteInterpreterContext implements org.apache.thrift.TBase<RemoteInterpreterContext, RemoteInterpreterContext._Fields>, java.io.Serializable, Cloneable, Comparable<RemoteInterpreterContext> {
   private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("RemoteInterpreterContext");
 

zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterEvent.java

@@ -51,7 +51,7 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 @SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"})
-@Generated(value = "Autogenerated by Thrift Compiler (0.9.2)", date = "2017-1-25")
+@Generated(value = "Autogenerated by Thrift Compiler (0.9.2)", date = "2017-3-27")
 public class RemoteInterpreterEvent implements org.apache.thrift.TBase<RemoteInterpreterEvent, RemoteInterpreterEvent._Fields>, java.io.Serializable, Cloneable, Comparable<RemoteInterpreterEvent> {
   private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("RemoteInterpreterEvent");
 

zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterResult.java

@@ -51,7 +51,7 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 @SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"})
-@Generated(value = "Autogenerated by Thrift Compiler (0.9.2)", date = "2017-1-25")
+@Generated(value = "Autogenerated by Thrift Compiler (0.9.2)", date = "2017-3-27")
 public class RemoteInterpreterResult implements org.apache.thrift.TBase<RemoteInterpreterResult, RemoteInterpreterResult._Fields>, java.io.Serializable, Cloneable, Comparable<RemoteInterpreterResult> {
   private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("RemoteInterpreterResult");
 

zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterResultMessage.java

@@ -51,7 +51,7 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 @SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"})
-@Generated(value = "Autogenerated by Thrift Compiler (0.9.2)", date = "2017-1-25")
+@Generated(value = "Autogenerated by Thrift Compiler (0.9.2)", date = "2017-3-27")
 public class RemoteInterpreterResultMessage implements org.apache.thrift.TBase<RemoteInterpreterResultMessage, RemoteInterpreterResultMessage._Fields>, java.io.Serializable, Cloneable, Comparable<RemoteInterpreterResultMessage> {
   private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("RemoteInterpreterResultMessage");
 

zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterService.java

@@ -51,7 +51,7 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 @SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"})
-@Generated(value = "Autogenerated by Thrift Compiler (0.9.2)", date = "2017-1-25")
+@Generated(value = "Autogenerated by Thrift Compiler (0.9.2)", date = "2017-3-27")
 public class RemoteInterpreterService {
 
   public interface Iface {
@@ -70,7 +70,7 @@ public class RemoteInterpreterService {
 
     public String getFormType(String sessionKey, String className) throws org.apache.thrift.TException;
 
-    public List<InterpreterCompletion> completion(String sessionKey, String className, String buf, int cursor) throws org.apache.thrift.TException;
+    public List<InterpreterCompletion> completion(String sessionKey, String className, String buf, int cursor, RemoteInterpreterContext interpreterContext) throws org.apache.thrift.TException;
 
     public void shutdown() throws org.apache.thrift.TException;
 
@@ -126,7 +126,7 @@ public class RemoteInterpreterService {
 
     public void getFormType(String sessionKey, String className, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException;
 
-    public void completion(String sessionKey, String className, String buf, int cursor, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException;
+    public void completion(String sessionKey, String className, String buf, int cursor, RemoteInterpreterContext interpreterContext, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException;
 
     public void shutdown(org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException;
 
@@ -349,19 +349,20 @@ public class RemoteInterpreterService {
       throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "getFormType failed: unknown result");
     }
 
-    public List<InterpreterCompletion> completion(String sessionKey, String className, String buf, int cursor) throws org.apache.thrift.TException
+    public List<InterpreterCompletion> completion(String sessionKey, String className, String buf, int cursor, RemoteInterpreterContext interpreterContext) throws org.apache.thrift.TException
     {
-      send_completion(sessionKey, className, buf, cursor);
+      send_completion(sessionKey, className, buf, cursor, interpreterContext);
       return recv_completion();
     }
 
-    public void send_completion(String sessionKey, String className, String buf, int cursor) throws org.apache.thrift.TException
+    public void send_completion(String sessionKey, String className, String buf, int cursor, RemoteInterpreterContext interpreterContext) throws org.apache.thrift.TException
     {
       completion_args args = new completion_args();
       args.setSessionKey(sessionKey);
       args.setClassName(className);
       args.setBuf(buf);
       args.setCursor(cursor);
+      args.setInterpreterContext(interpreterContext);
       sendBase("completion", args);
     }
 
@@ -1064,9 +1065,9 @@ public class RemoteInterpreterService {
       }
     }
 
-    public void completion(String sessionKey, String className, String buf, int cursor, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException {
+    public void completion(String sessionKey, String className, String buf, int cursor, RemoteInterpreterContext interpreterContext, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException {
       checkReady();
-      completion_call method_call = new completion_call(sessionKey, className, buf, cursor, resultHandler, this, ___protocolFactory, ___transport);
+      completion_call method_call = new completion_call(sessionKey, className, buf, cursor, interpreterContext, resultHandler, this, ___protocolFactory, ___transport);
       this.___currentMethod = method_call;
       ___manager.call(method_call);
     }
@@ -1076,12 +1077,14 @@ public class RemoteInterpreterService {
       private String className;
       private String buf;
       private int cursor;
-      public completion_call(String sessionKey, String className, String buf, int cursor, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
+      private RemoteInterpreterContext interpreterContext;
+      public completion_call(String sessionKey, String className, String buf, int cursor, RemoteInterpreterContext interpreterContext, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
         super(client, protocolFactory, transport, resultHandler, false);
         this.sessionKey = sessionKey;
         this.className = className;
         this.buf = buf;
         this.cursor = cursor;
+        this.interpreterContext = interpreterContext;
       }
 
       public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
@@ -1091,6 +1094,7 @@ public class RemoteInterpreterService {
         args.setClassName(className);
         args.setBuf(buf);
         args.setCursor(cursor);
+        args.setInterpreterContext(interpreterContext);
         args.write(prot);
         prot.writeMessageEnd();
       }
@@ -1933,7 +1937,7 @@ public class RemoteInterpreterService {
 
       public completion_result getResult(I iface, completion_args args) throws org.apache.thrift.TException {
         completion_result result = new completion_result();
-        result.success = iface.completion(args.sessionKey, args.className, args.buf, args.cursor);
+        result.success = iface.completion(args.sessionKey, args.className, args.buf, args.cursor, args.interpreterContext);
         return result;
       }
     }
@@ -2742,7 +2746,7 @@ public class RemoteInterpreterService {
       }
 
       public void start(I iface, completion_args args, org.apache.thrift.async.AsyncMethodCallback<List<InterpreterCompletion>> resultHandler) throws TException {
-        iface.completion(args.sessionKey, args.className, args.buf, args.cursor,resultHandler);
+        iface.completion(args.sessionKey, args.className, args.buf, args.cursor, args.interpreterContext,resultHandler);
       }
     }
 
@@ -9809,6 +9813,7 @@ public class RemoteInterpreterService {
     private static final org.apache.thrift.protocol.TField CLASS_NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("className", org.apache.thrift.protocol.TType.STRING, (short)2);
     private static final org.apache.thrift.protocol.TField BUF_FIELD_DESC = new org.apache.thrift.protocol.TField("buf", org.apache.thrift.protocol.TType.STRING, (short)3);
     private static final org.apache.thrift.protocol.TField CURSOR_FIELD_DESC = new org.apache.thrift.protocol.TField("cursor", org.apache.thrift.protocol.TType.I32, (short)4);
+    private static final org.apache.thrift.protocol.TField INTERPRETER_CONTEXT_FIELD_DESC = new org.apache.thrift.protocol.TField("interpreterContext", org.apache.thrift.protocol.TType.STRUCT, (short)5);
 
     private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
     static {
@@ -9820,13 +9825,15 @@ public class RemoteInterpreterService {
     public String className; // required
     public String buf; // required
     public int cursor; // required
+    public RemoteInterpreterContext interpreterContext; // required
 
     /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
     public enum _Fields implements org.apache.thrift.TFieldIdEnum {
       SESSION_KEY((short)1, "sessionKey"),
       CLASS_NAME((short)2, "className"),
       BUF((short)3, "buf"),
-      CURSOR((short)4, "cursor");
+      CURSOR((short)4, "cursor"),
+      INTERPRETER_CONTEXT((short)5, "interpreterContext");
 
       private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
 
@@ -9849,6 +9856,8 @@ public class RemoteInterpreterService {
             return BUF;
           case 4: // CURSOR
             return CURSOR;
+          case 5: // INTERPRETER_CONTEXT
+            return INTERPRETER_CONTEXT;
           default:
             return null;
         }
@@ -9902,6 +9911,8 @@ public class RemoteInterpreterService {
           new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
       tmpMap.put(_Fields.CURSOR, new org.apache.thrift.meta_data.FieldMetaData("cursor", org.apache.thrift.TFieldRequirementType.DEFAULT, 
           new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32)));
+      tmpMap.put(_Fields.INTERPRETER_CONTEXT, new org.apache.thrift.meta_data.FieldMetaData("interpreterContext", org.apache.thrift.TFieldRequirementType.DEFAULT, 
+          new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, RemoteInterpreterContext.class)));
       metaDataMap = Collections.unmodifiableMap(tmpMap);
       org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(completion_args.class, metaDataMap);
     }
@@ -9913,7 +9924,8 @@ public class RemoteInterpreterService {
       String sessionKey,
       String className,
       String buf,
-      int cursor)
+      int cursor,
+      RemoteInterpreterContext interpreterContext)
     {
       this();
       this.sessionKey = sessionKey;
@@ -9921,6 +9933,7 @@ public class RemoteInterpreterService {
       this.buf = buf;
       this.cursor = cursor;
       setCursorIsSet(true);
+      this.interpreterContext = interpreterContext;
     }
 
     /**
@@ -9938,6 +9951,9 @@ public class RemoteInterpreterService {
         this.buf = other.buf;
       }
       this.cursor = other.cursor;
+      if (other.isSetInterpreterContext()) {
+        this.interpreterContext = new RemoteInterpreterContext(other.interpreterContext);
+      }
     }
 
     public completion_args deepCopy() {
@@ -9951,6 +9967,7 @@ public class RemoteInterpreterService {
       this.buf = null;
       setCursorIsSet(false);
       this.cursor = 0;
+      this.interpreterContext = null;
     }
 
     public String getSessionKey() {
@@ -10048,6 +10065,30 @@ public class RemoteInterpreterService {
       __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __CURSOR_ISSET_ID, value);
     }
 
+    public RemoteInterpreterContext getInterpreterContext() {
+      return this.interpreterContext;
+    }
+
+    public completion_args setInterpreterContext(RemoteInterpreterContext interpreterContext) {
+      this.interpreterContext = interpreterContext;
+      return this;
+    }
+
+    public void unsetInterpreterContext() {
+      this.interpreterContext = null;
+    }
+
+    /** Returns true if field interpreterContext is set (has been assigned a value) and false otherwise */
+    public boolean isSetInterpreterContext() {
+      return this.interpreterContext != null;
+    }
+
+    public void setInterpreterContextIsSet(boolean value) {
+      if (!value) {
+        this.interpreterContext = null;
+      }
+    }
+
     public void setFieldValue(_Fields field, Object value) {
       switch (field) {
       case SESSION_KEY:
@@ -10082,6 +10123,14 @@ public class RemoteInterpreterService {
         }
         break;
 
+      case INTERPRETER_CONTEXT:
+        if (value == null) {
+          unsetInterpreterContext();
+        } else {
+          setInterpreterContext((RemoteInterpreterContext)value);
+        }
+        break;
+
       }
     }
 
@@ -10099,6 +10148,9 @@ public class RemoteInterpreterService {
       case CURSOR:
         return Integer.valueOf(getCursor());
 
+      case INTERPRETER_CONTEXT:
+        return getInterpreterContext();
+
       }
       throw new IllegalStateException();
     }
@@ -10118,6 +10170,8 @@ public class RemoteInterpreterService {
         return isSetBuf();
       case CURSOR:
         return isSetCursor();
+      case INTERPRETER_CONTEXT:
+        return isSetInterpreterContext();
       }
       throw new IllegalStateException();
     }
@@ -10171,6 +10225,15 @@ public class RemoteInterpreterService {
           return false;
       }
 
+      boolean this_present_interpreterContext = true && this.isSetInterpreterContext();
+      boolean that_present_interpreterContext = true && that.isSetInterpreterContext();
+      if (this_present_interpreterContext || that_present_interpreterContext) {
+        if (!(this_present_interpreterContext && that_present_interpreterContext))
+          return false;
+        if (!this.interpreterContext.equals(that.interpreterContext))
+          return false;
+      }
+
       return true;
     }
 
@@ -10198,6 +10261,11 @@ public class RemoteInterpreterService {
       if (present_cursor)
         list.add(cursor);
 
+      boolean present_interpreterContext = true && (isSetInterpreterContext());
+      list.add(present_interpreterContext);
+      if (present_interpreterContext)
+        list.add(interpreterContext);
+
       return list.hashCode();
     }
 
@@ -10249,6 +10317,16 @@ public class RemoteInterpreterService {
           return lastComparison;
         }
       }
+      lastComparison = Boolean.valueOf(isSetInterpreterContext()).compareTo(other.isSetInterpreterContext());
+      if (lastComparison != 0) {
+        return lastComparison;
+      }
+      if (isSetInterpreterContext()) {
+        lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.interpreterContext, other.interpreterContext);
+        if (lastComparison != 0) {
+          return lastComparison;
+        }
+      }
       return 0;
     }
 
@@ -10296,6 +10374,14 @@ public class RemoteInterpreterService {
       sb.append("cursor:");
       sb.append(this.cursor);
       first = false;
+      if (!first) sb.append(", ");
+      sb.append("interpreterContext:");
+      if (this.interpreterContext == null) {
+        sb.append("null");
+      } else {
+        sb.append(this.interpreterContext);
+      }
+      first = false;
       sb.append(")");
       return sb.toString();
     }
@@ -10303,6 +10389,9 @@ public class RemoteInterpreterService {
     public void validate() throws org.apache.thrift.TException {
       // check for required fields
       // check for sub-struct validity
+      if (interpreterContext != null) {
+        interpreterContext.validate();
+      }
     }
 
     private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
@@ -10373,6 +10462,15 @@ public class RemoteInterpreterService {
                 org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
               }
               break;
+            case 5: // INTERPRETER_CONTEXT
+              if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
+                struct.interpreterContext = new RemoteInterpreterContext();
+                struct.interpreterContext.read(iprot);
+                struct.setInterpreterContextIsSet(true);
+              } else { 
+                org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+              }
+              break;
             default:
               org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
           }
@@ -10406,6 +10504,11 @@ public class RemoteInterpreterService {
         oprot.writeFieldBegin(CURSOR_FIELD_DESC);
         oprot.writeI32(struct.cursor);
         oprot.writeFieldEnd();
+        if (struct.interpreterContext != null) {
+          oprot.writeFieldBegin(INTERPRETER_CONTEXT_FIELD_DESC);
+          struct.interpreterContext.write(oprot);
+          oprot.writeFieldEnd();
+        }
         oprot.writeFieldStop();
         oprot.writeStructEnd();
       }
@@ -10436,7 +10539,10 @@ public class RemoteInterpreterService {
         if (struct.isSetCursor()) {
           optionals.set(3);
         }
-        oprot.writeBitSet(optionals, 4);
+        if (struct.isSetInterpreterContext()) {
+          optionals.set(4);
+        }
+        oprot.writeBitSet(optionals, 5);
         if (struct.isSetSessionKey()) {
           oprot.writeString(struct.sessionKey);
         }
@@ -10449,12 +10555,15 @@ public class RemoteInterpreterService {
         if (struct.isSetCursor()) {
           oprot.writeI32(struct.cursor);
         }
+        if (struct.isSetInterpreterContext()) {
+          struct.interpreterContext.write(oprot);
+        }
       }
 
       @Override
       public void read(org.apache.thrift.protocol.TProtocol prot, completion_args struct) throws org.apache.thrift.TException {
         TTupleProtocol iprot = (TTupleProtocol) prot;
-        BitSet incoming = iprot.readBitSet(4);
+        BitSet incoming = iprot.readBitSet(5);
         if (incoming.get(0)) {
           struct.sessionKey = iprot.readString();
           struct.setSessionKeyIsSet(true);
@@ -10471,6 +10580,11 @@ public class RemoteInterpreterService {
           struct.cursor = iprot.readI32();
           struct.setCursorIsSet(true);
         }
+        if (incoming.get(4)) {
+          struct.interpreterContext = new RemoteInterpreterContext();
+          struct.interpreterContext.read(iprot);
+          struct.setInterpreterContextIsSet(true);
+        }
       }
     }
 

zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/ZeppelinServerResourceParagraphRunner.java

@@ -51,7 +51,7 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 @SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"})
-@Generated(value = "Autogenerated by Thrift Compiler (0.9.2)", date = "2017-1-25")
+@Generated(value = "Autogenerated by Thrift Compiler (0.9.2)", date = "2017-3-27")
 public class ZeppelinServerResourceParagraphRunner implements org.apache.thrift.TBase<ZeppelinServerResourceParagraphRunner, ZeppelinServerResourceParagraphRunner._Fields>, java.io.Serializable, Cloneable, Comparable<ZeppelinServerResourceParagraphRunner> {
   private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("ZeppelinServerResourceParagraphRunner");
 

zeppelin-interpreter/src/main/thrift/RemoteInterpreterService.thrift

@@ -84,7 +84,8 @@ struct ZeppelinServerResourceParagraphRunner {
  */
 struct InterpreterCompletion {
   1: string name,
-  2: string value
+  2: string value,
+  3: string meta
 }
 
 service RemoteInterpreterService {
@@ -96,7 +97,7 @@ service RemoteInterpreterService {
   void cancel(1: string sessionKey, 2: string className, 3: RemoteInterpreterContext interpreterContext);
   i32 getProgress(1: string sessionKey, 2: string className, 3: RemoteInterpreterContext interpreterContext);
   string getFormType(1: string sessionKey, 2: string className);
-  list<InterpreterCompletion> completion(1: string sessionKey, 2: string className, 3: string buf, 4: i32 cursor);
+  list<InterpreterCompletion> completion(1: string sessionKey, 2: string className, 3: string buf, 4: i32 cursor, 5: RemoteInterpreterContext interpreterContext);
   void shutdown();
 
   string getStatus(1: string sessionKey, 2:string jobId);

zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/mock/MockInterpreterA.java

@@ -78,7 +78,8 @@ public class MockInterpreterA extends Interpreter {
   }
 
   @Override
-  public List<InterpreterCompletion> completion(String buf, int cursor) {
+  public List<InterpreterCompletion> completion(String buf, int cursor,
+      InterpreterContext interpreterContext) {
     return null;
   }
 

zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/mock/MockInterpreterAngular.java

@@ -106,7 +106,8 @@ public class MockInterpreterAngular extends Interpreter {
   }
 
   @Override
-  public List<InterpreterCompletion> completion(String buf, int cursor) {
+  public List<InterpreterCompletion> completion(String buf, int cursor,
+      InterpreterContext interpreterContext) {
     return null;
   }
 }

zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/mock/MockInterpreterB.java

@@ -78,7 +78,8 @@ public class MockInterpreterB extends Interpreter {
   }
 
   @Override
-  public List<InterpreterCompletion> completion(String buf, int cursor) {
+  public List<InterpreterCompletion> completion(String buf, int cursor,
+      InterpreterContext interpreterContext) {
     return null;
   }
 

zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/mock/MockInterpreterEnv.java

@@ -67,7 +67,8 @@ public class MockInterpreterEnv extends Interpreter {
   }
 
   @Override
-  public List<InterpreterCompletion> completion(String buf, int cursor) {
+  public List<InterpreterCompletion> completion(String buf, int cursor,
+      InterpreterContext interpreterContext) {
     return null;
   }
 

zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/mock/MockInterpreterOutputStream.java

@@ -78,7 +78,8 @@ public class MockInterpreterOutputStream extends Interpreter {
   }
 
   @Override
-  public List<InterpreterCompletion> completion(String buf, int cursor) {
+  public List<InterpreterCompletion> completion(String buf, int cursor,
+      InterpreterContext interpreterContext) {
     return null;
   }
 

zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/mock/MockInterpreterResourcePool.java

@@ -121,7 +121,8 @@ public class MockInterpreterResourcePool extends Interpreter {
   }
 
   @Override
-  public List<InterpreterCompletion> completion(String buf, int cursor) {
+  public List<InterpreterCompletion> completion(String buf, int cursor,
+      InterpreterContext interpreterContext) {
     return null;
   }
 }

zeppelin-server/src/test/java/org/apache/zeppelin/interpreter/mock/MockInterpreter1.java

@@ -68,7 +68,8 @@ public class MockInterpreter1 extends Interpreter{
   }
 
   @Override
-  public List<InterpreterCompletion> completion(String buf, int cursor) {
+  public List<InterpreterCompletion> completion(String buf, int cursor,
+      InterpreterContext interpreterContext) {
     return null;
   }
 }

zeppelin-web/karma.conf.js

@@ -37,6 +37,9 @@ module.exports = function(config) {
 
     // list of files / patterns to load in the browser
     files: [
+      // for polyfill
+      'node_modules/babel-polyfill/dist/polyfill.js',
+
       // bower:js
       'bower_components/jquery/dist/jquery.js',
       'bower_components/es5-shim/es5-shim.js',
@@ -124,8 +127,7 @@ module.exports = function(config) {
 
     preprocessors: {
       'src/*/{*.js,!(test)/**/*.js}': 'coverage',
-      'src/index.js': ['webpack', 'sourcemap',],
-      'src/**/*.test.js': ['webpack', 'sourcemap',],
+      'src/**/*.js': ['webpack', 'sourcemap',],
     },
 
     coverageReporter: {

zeppelin-web/package.json

@@ -16,7 +16,7 @@
     "dev:watch": "grunt watch-webpack-dev",
     "dev": "npm-run-all --parallel dev:server dev:watch",
     "visdev": "npm-run-all --parallel visdev:server dev:watch",
-    "pretest": "npm install karma-phantomjs-launcher",
+    "pretest": "npm install karma-phantomjs-launcher babel-polyfill",
     "test": "karma start karma.conf.js"
   },
   "dependencies": {

zeppelin-web/src/app/home/notebook-template.html

@@ -59,8 +59,16 @@ limitations under the License.
         <i style="font-size: 10px;" ng-class="node.hidden ? 'icon-folder' : 'icon-folder-alt'"></i> {{getNoteName(node)}}
       </a>
       <a ng-if="!node.isTrash" style="text-decoration: none;">
-        <i style="margin-left: 10px;"
-           class="fa fa-pencil notebook-list-btn" ng-show="showFolderButton" ng-click="renameFolder(node)"
+        <a href="" data-toggle="modal" data-target="#noteNameModal" style="text-decoration: none;"
+           ng-controller="NotenameCtrl as notenamectrl" ng-click="notenamectrl.getInterpreterSettings()" data-path="{{node.id}}">
+          <i style="margin-left: 10px;"
+             class="fa fa-plus notebook-list-btn" ng-show="showFolderButton"
+             tooltip-placement="bottom" uib-tooltip="Create new note">
+          </i>
+        </a>
+      </a>
+      <a ng-if="!node.isTrash" style="text-decoration: none;">
+        <i class="fa fa-pencil notebook-list-btn" ng-show="showFolderButton" ng-click="renameFolder(node)"
            tooltip-placement="bottom" uib-tooltip="Rename folder">
         </i>
       </a>

zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js

@@ -640,6 +640,19 @@ function ParagraphCtrl($scope, $rootScope, $route, $window, $routeParams, $locat
 
       var remoteCompleter = {
         getCompletions: function(editor, session, pos, prefix, callback) {
+          var langTools = ace.require('ace/ext/language_tools');
+          var defaultKeywords = new Set();
+          var getDefaultKeywords = function(err, completions) {
+              if (completions !== undefined) {
+                  completions.forEach(function(c) {
+                      defaultKeywords.add(c.value);
+                  });
+              }
+          }
+          if (langTools.keyWordCompleter !== undefined) {
+              langTools.keyWordCompleter.getCompletions(editor, session, pos, prefix, getDefaultKeywords);
+          }
+
           if (!editor.isFocused()) {
             return;
           }
@@ -650,13 +663,29 @@ function ParagraphCtrl($scope, $rootScope, $route, $window, $routeParams, $locat
           websocketMsgSrv.completion($scope.paragraph.id, buf, pos);
 
           $scope.$on('completionList', function(event, data) {
+            var computeCaption = function(value, meta) {
+              var metaLength = meta !== undefined ? meta.length : 0;
+              var length = 42;
+              var whitespaceLength = 3;
+              var ellipses = '...';
+              var maxLengthCaption = length - metaLength - whitespaceLength - ellipses.length;
+              if (value !== undefined && value.length > maxLengthCaption) {
+                  return value.substr(0, maxLengthCaption) + ellipses;
+              }
+              return value;
+            }
             if (data.completions) {
               var completions = [];
               for (var c in data.completions) {
                 var v = data.completions[c];
+                if (v.meta !== undefined && v.meta === 'keyword' && defaultKeywords.has(v.value.trim())) {
+                    continue;
+                }
                 completions.push({
                   name: v.name,
                   value: v.value,
+                  meta: v.meta,
+                  caption: computeCaption(v.value, v.meta),
                   score: 300
                 });
               }
@@ -709,19 +738,27 @@ function ParagraphCtrl($scope, $rootScope, $route, $window, $routeParams, $locat
        */
 
       // remove binding
-      $scope.editor.commands.bindKey('ctrl-alt-n.', null);
       $scope.editor.commands.removeCommand('showSettingsMenu');
 
-      $scope.editor.commands.bindKey('ctrl-alt-l', null);
-      $scope.editor.commands.bindKey('ctrl-alt-w', null);
-      $scope.editor.commands.bindKey('ctrl-alt-a', null);
-      $scope.editor.commands.bindKey('ctrl-alt-k', null);
-      $scope.editor.commands.bindKey('ctrl-alt-e', null);
-      $scope.editor.commands.bindKey('ctrl-alt-t', null);
+      var isOption = $rootScope.isMac? 'option' : 'alt';
+
+      $scope.editor.commands.bindKey('ctrl-' + isOption + '-n.', null);
+      $scope.editor.commands.bindKey('ctrl-' + isOption + '-l', null);
+      $scope.editor.commands.bindKey('ctrl-' + isOption + '-w', null);
+      $scope.editor.commands.bindKey('ctrl-' + isOption + '-a', null);
+      $scope.editor.commands.bindKey('ctrl-' + isOption + '-k', null);
+      $scope.editor.commands.bindKey('ctrl-' + isOption + '-e', null);
+      $scope.editor.commands.bindKey('ctrl-' + isOption + '-t', null);
+      $scope.editor.commands.bindKey('ctrl-space', null);
+
+      if ($rootScope.isMac) {
+        $scope.editor.commands.bindKey('command-l', null);
+      } else {
+        $scope.editor.commands.bindKey('ctrl-l', null);
+      }
 
       // autocomplete on 'ctrl+.'
       $scope.editor.commands.bindKey('ctrl-.', 'startAutocomplete');
-      $scope.editor.commands.bindKey('ctrl-space', null);
 
       var keyBindingEditorFocusAction = function(scrollValue) {
         var numRows = $scope.editor.getSession().getLength();
@@ -834,7 +871,7 @@ function ParagraphCtrl($scope, $rootScope, $route, $window, $routeParams, $locat
   };
 
   var getInterpreterName = function(paragraphText) {
-    var intpNameRegexp = /^\s*%(.+?)\s/g;
+    var intpNameRegexp = /^\s*%(.+?)(\s|\()/g;
     var match = intpNameRegexp.exec(paragraphText);
     if (match) {
       return match[1].trim();

zeppelin-web/src/app/tabledata/advanced-transformation-setting.html

@@ -0,0 +1,280 @@
+<!--
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+
+<div class="panel panel-default" style="margin-top: 10px; margin-bottom: 11px;">
+
+  <!-- panel: axis (configured column) information -->
+  <div class="panel-heading"
+       style="padding: 6px 12px 6px 12px; font-size: 13px;">
+    <span style="vertical-align: middle; display: inline-block; margin-top: 3px;">Charts</span>
+    <span style="float: right;">
+       <div class="btn-group" role="group" aria-label="...">
+         <div type="button" ng-click="resetAxisConfig()"
+              ng-if="config.panel.columnPanelOpened"
+              class="btn btn-default" style="padding: 2px 5px 2px 5px;">
+           <i class="fa fa-trash-o" aria-hidden="true"></i>
+         </div>
+         <div type="button" ng-if="config.panel.columnPanelOpened"
+              ng-click="toggleColumnPanel()"
+              class="btn btn-default" style="padding: 2px 5px 2px 5px;">
+           <i class="fa fa-minus" style="font-size: 12px;" aria-hidden="true"></i>
+         </div>
+         <div type="button" ng-if="!config.panel.columnPanelOpened"
+              ng-click="toggleColumnPanel()"
+              class="btn btn-default" style="padding: 2px 5px 2px 5px;">
+           <i class="fa fa-expand" style="font-size: 11px;" aria-hidden="true"></i>
+         </div>
+       </div>
+    </span>
+    <div style="clear: both;"></div> <!-- to fix previous span which has float: right -->
+  </div>
+  <div class="panel-body" ng-if="config.panel.columnPanelOpened"
+       style="padding: 8px; margin-top: 3px;">
+    <ul class="noDot">
+      <li class="liVertical" ng-repeat="chart in config.chart.available">
+        <label class="radio-inline">
+          <input type="radio" style="margin-top: 1px; margin-left: -17px;"
+                 ng-checked="config.chart.current === chart"
+                 ng-click="chartChanged(chart)" value="{{chart}}" />
+          <span style="vertical-align: middle;">
+            {{chart}} {{useSharedAxis(chart) ? '(shared)' : ''}}
+          </span>
+        </label>
+      </li>
+    </ul>
+  </div>
+
+  <!-- panel: available columns -->
+  <div class="panel-heading" ng-if="config.panel.columnPanelOpened"
+       style="padding: 6px 12px 6px 12px; font-size: 13px; border-top: 1px solid #ddd; border-top-left-radius: 0px; border-top-right-radius: 0px;">
+    <span>Available Columns</span>
+  </div>
+  <div class="panel-body" ng-if="config.panel.columnPanelOpened"
+       style="padding: 8px; margin-top: 3px;">
+    <ul class="noDot">
+      <li class="liVertical" ng-repeat="column in columns">
+        <div class="btn btn-default btn-xs"
+             style="background-color: #EFEFEF;"
+             data-drag="true"
+             data-jqyoui-options="{revert: 'invalid', helper: 'clone'}"
+             ng-model="columns"
+             jqyoui-draggable="{index: {{$index}}, placeholder: 'keep'}">
+          {{column.name | limitTo: 30}}{{column.name.length > 30 ? '...' : ''}}
+        </div>
+      </li>
+    </ul>
+  </div>
+
+  <!-- panel: axis (configured columns) -->
+  <hr style="margin: 1px;" ng-if="config.panel.columnPanelOpened" />
+  <div class="panel-body" ng-if="config.panel.columnPanelOpened"
+       style="margin-top: 7px; padding-top: 9px; padding-bottom: 4px;">
+    <div class="row">
+      <div class="col-sm-4 col-md-3"
+           ng-repeat="axisSpec in config.axisSpecs[config.chart.current]">
+        <div class="columns lightBold">
+          <!-- axis name -->
+          <span class="label label-default"
+                uib-tooltip="{{axisSpec.description ? axisSpec.description : ''}}"
+                style="font-weight: 300; font-size: 13px; margin-left: 1px;">
+            {{getAxisAnnotation(axisSpec)}}
+          </span>
+          <span class="label label-default"
+                ng-style="getAxisTypeAnnotationColor(axisSpec)"
+                style="font-weight: 300; font-size: 13px; margin-left: 3px;">
+            {{getAxisTypeAnnotation(axisSpec)}}
+          </span>
+
+          <!-- axis box: in case of single dimension -->
+          <ul data-drop="true"
+              ng-if="isSingleDimensionAxis(axisSpec)"
+              ng-model="config.axis[config.chart.current][axisSpec.name]"
+              jqyoui-droppable="{onDrop:'axisChanged(axisSpec)'}"
+              class="list-unstyled"
+              style="height:36px; border-radius: 6px; margin-top: 7px; overflow: visible !important;">
+            <li ng-if="config.axis[config.chart.current][axisSpec.name]">
+
+              <!-- in case of axis is single dimension and not aggregator -->
+              <div ng-if="!isAggregatorAxis(axisSpec)"
+                   class="btn btn-default btn-xs"
+                   style="background-color: #EFEFEF;">
+                {{ getSingleDimensionAxis(axisSpec).name }}
+                <span class="fa fa-close" ng-click="removeFromAxis(null, axisSpec)"></span>
+              </div>
+
+              <!-- in case of axis is single dimension and aggregator -->
+              <div class="btn-group">
+                <div ng-if="isAggregatorAxis(axisSpec)"
+                     class="btn btn-default btn-xs dropdown-toggle"
+                     style="background-color: #EFEFEF; "
+                     type="button" data-toggle="dropdown">
+                  {{getSingleDimensionAxis(axisSpec).name | limitTo: 30}}{{getSingleDimensionAxis(axisSpec).name > 30 ? '...' : ''}}
+                  <span style="color:#717171;">
+                    <span class="lightBold" style="text-transform: uppercase;">{{getSingleDimensionAxis(axisSpec).aggr}}</span>
+                  </span>
+                  <span class="fa fa-close" ng-click="removeFromAxis(null, axisSpec)"></span>
+                </div>
+                <ul class="dropdown-menu" role="menu">
+                  <li ng-click="aggregatorChanged(null, axisSpec, 'sum')"><a>sum</a></li>
+                  <li ng-click="aggregatorChanged(null, axisSpec, 'count')"><a>count</a></li>
+                  <li ng-click="aggregatorChanged(null, axisSpec, 'avg')"><a>avg</a></li>
+                  <li ng-click="aggregatorChanged(null, axisSpec, 'min')"><a>min</a></li>
+                  <li ng-click="aggregatorChanged(null, axisSpec, 'max')"><a>max</a></li>
+                </ul>
+              </div>
+
+            </li>
+          </ul>
+
+          <!-- axis box: in case of multiple dimensions -->
+          <ul data-drop="true"
+              ng-if="!isSingleDimensionAxis(axisSpec) "
+              ng-model="config.axis[config.chart.current][axisSpec.name]"
+              jqyoui-droppable="{multiple: true, onDrop:'axisChanged(axisSpec)'}"
+              class="list-unstyled"
+              style="height: 108px; border-radius: 6px; margin-top: 7px; overflow: auto !important;">
+
+            <span ng-repeat="col in config.axis[config.chart.current][axisSpec.name]">
+
+              <!-- in case of axis is multiple dimensions and not aggregator -->
+              <span ng-if="!isAggregatorAxis(axisSpec)"
+                    class="btn btn-default btn-xs"
+                    style="background-color: #EFEFEF; margin: 2px 0px 0px 2px;">
+                {{col.name}}
+                <span class="fa fa-close" ng-click="removeFromAxis($index, axisSpec)"></span>
+              </span>
+
+              <!-- in case of axis is multiple dimension and aggregator -->
+              <span class="btn-group">
+                <span ng-if="isAggregatorAxis(axisSpec)"
+                      class="btn btn-default btn-xs dropdown-toggle"
+                      style="background-color: #EFEFEF; margin: 2px 0px 0px 2px;"
+                      type="button" data-toggle="dropdown">
+                  {{col.name | limitTo: 30}}{{col.name.length > 30 ? '...' : ''}}
+                  <span style="color:#717171; margin: 0px;">
+                    <span class="lightBold"
+                          style="text-transform: uppercase; margin: 0px;">{{col.aggr}}
+                    </span>
+                  </span>
+                  <span class="fa fa-close" style="margin: 0px;" ng-click="removeFromAxis($index, axisSpec)"></span>
+                </span>
+                <ul class="dropdown-menu" role="menu">
+                  <li ng-click="aggregatorChanged($index, axisSpec, 'sum')"><a>sum</a></li>
+                  <li ng-click="aggregatorChanged($index, axisSpec, 'count')"><a>count</a></li>
+                  <li ng-click="aggregatorChanged($index, axisSpec, 'avg')"><a>avg</a></li>
+                  <li ng-click="aggregatorChanged($index, axisSpec, 'min')"><a>min</a></li>
+                  <li ng-click="aggregatorChanged($index, axisSpec, 'max')"><a>max</a></li>
+                </ul>
+              </span>
+
+            </span>
+          </ul>
+
+        </div>
+      </div>
+    </div>
+  </div>
+
+</div>
+
+<!-- panel: parameter information -->
+<div class="panel panel-default">
+
+  <div class="panel-heading" style="padding: 6px 12px 6px 12px; font-size: 13px;">
+    <span style="vertical-align: middle; display: inline-block; margin-top: 3px;">Parameters</span>
+    <span style="float: right;">
+      <div class="btn-group" role="group" aria-label="...">
+        <div type="button" ng-click="parameterChanged()"
+             ng-if="config.panel.parameterPanelOpened"
+             class="btn btn-default" style="padding: 2px 5px 2px 5px;">
+          <i class="fa fa-floppy-o" aria-hidden="true"></i>
+        </div>
+        <div type="button" ng-click="resetParameterConfig()"
+             ng-if="config.panel.parameterPanelOpened"
+             class="btn btn-default" style="padding: 2px 5px 2px 5px;">
+          <i class="fa fa-trash-o" aria-hidden="true"></i>
+        </div>
+        <div type="button" ng-if="config.panel.parameterPanelOpened"
+             ng-click="toggleParameterPanel()"
+             class="btn btn-default" style="padding: 2px 5px 2px 5px;">
+          <i class="fa fa-minus" style="font-size: 12px;" aria-hidden="true"></i>
+        </div>
+        <div type="button" ng-if="!config.panel.parameterPanelOpened"
+             ng-click="toggleParameterPanel()"
+             class="btn btn-default" style="padding: 2px 5px 2px 5px;">
+          <i class="fa fa-expand" style="font-size: 11px;" aria-hidden="true"></i>
+        </div>
+      </div>
+    </span>
+    <div style="clear: both;"></div> <!-- to fix previous span which has float: right -->
+  </div>
+  <div class="panel-body"
+       ng-if="config.panel.parameterPanelOpened"
+       style="padding-top: 13px; padding-bottom: 13px; height: 400px; overflow: auto;">
+    <table class="table table-striped">
+      <tr>
+        <th style="font-size: 12px; font-style: italic">Name</th>
+        <th style="font-size: 12px; font-style: italic">Type</th>
+        <th style="font-size: 12px; font-style: italic">Description</th>
+        <th style="font-size: 12px; font-style: italic">Value</th>
+      </tr>
+      <tr>
+      </tr>
+
+      <tr data-ng-repeat="paramSpec in config.paramSpecs[config.chart.current]">
+        <td style="font-weight: 400; vertical-align: middle;">{{paramSpec.name}}</td>
+        <td style="font-weight: 400; vertical-align: middle;">{{paramSpec.valueType}}</td>
+        <td ng-bind-html="paramSpec.description"
+          style="font-weight: 400; vertical-align: middle;"></td>
+        <td>
+          <div ng-if="isInputWidget(paramSpec)"
+               class="input-group">
+            <input type="text" class="form-control input-sm"
+                   style="font-weight: 400; font-size: 12px; vertical-align:middle; border-radius: 5px;"
+                   ng-keydown="parameterOnKeyDown($event, paramSpec)"
+                   data-ng-model="config.parameter[config.chart.current][paramSpec.name]" />
+          </div>
+          <div class="btn-group"
+               ng-if="isOptionWidget(paramSpec)">
+            <select class="form-control input-sm"
+                    ng-keydown="parameterOnKeyDown($event, paramSpec)"
+                    ng-change="parameterChanged(paramSpec)"
+                    data-ng-model="config.parameter[config.chart.current][paramSpec.name]"
+                    data-ng-options="optionValue for optionValue in paramSpec.optionValues"
+                    style="font-weight: 400; font-size: 12px;">
+            </select>
+          </div>
+
+          <div ng-if="isCheckboxWidget(paramSpec)">
+            <input type="checkbox"
+                   ng-keydown="parameterOnKeyDown($event, paramSpec)"
+                   ng-click="parameterChanged(paramSpec)"
+                   data-ng-model="config.parameter[config.chart.current][paramSpec.name]"
+                   data-ng-checked="config.parameter[config.chart.current][paramSpec.name]" />
+          </div>
+
+          <div ng-if="isTextareaWidget(paramSpec)">
+            <textarea class="form-control" rows="3"
+                      ng-keydown="parameterOnKeyDown($event, paramSpec)"
+                      data-ng-model="config.parameter[config.chart.current][paramSpec.name]"
+                      style="font-weight: 400; font-size: 12px;">
+            </textarea>
+          </div>
+
+        </td>
+      </tr>
+    </table>
+  </div>
+
+</div>

zeppelin-web/src/app/tabledata/advanced-transformation.js

@@ -0,0 +1,230 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import Transformation from './transformation';
+
+import {
+  getCurrentChart, getCurrentChartAxis, getCurrentChartParam,
+  serializeSharedAxes, useSharedAxis,
+  getCurrentChartAxisSpecs, getCurrentChartParamSpecs,
+  initializeConfig, resetAxisConfig, resetParameterConfig,
+  isAggregatorAxis, isGroupAxis, isKeyAxis, isSingleDimensionAxis,
+  removeDuplicatedColumnsInMultiDimensionAxis, applyMaxAxisCount,
+  isInputWidget, isOptionWidget, isCheckboxWidget, isTextareaWidget, parseParameter,
+  getTransformer,
+} from './advanced-transformation-util';
+
+const SETTING_TEMPLATE = 'app/tabledata/advanced-transformation-setting.html';
+
+export default class AdvancedTransformation extends Transformation {
+  constructor(config, spec) {
+    super(config);
+
+    this.columns = []; /** [{ name, index, comment }] */
+    this.props = {};
+    this.spec = spec
+
+    initializeConfig(config, spec);
+  }
+
+  emitConfigChange(conf) {
+    conf.chartChanged = false
+    conf.parameterChanged = false
+    this.emitConfig(conf)
+  }
+
+  emitChartChange(conf) {
+    conf.chartChanged = true
+    conf.parameterChanged = false
+    this.emitConfig(conf)
+  }
+
+  emitParameterChange(conf) {
+    conf.chartChanged = false
+    conf.parameterChanged = true
+    this.emitConfig(conf)
+  }
+
+  getSetting() {
+    const self = this; /** for closure */
+    const configInstance = self.config; /** for closure */
+
+    if (self.spec.initialized) {
+      self.spec.initialized = false
+      self.emitConfig(configInstance)
+    }
+
+    return {
+      template: SETTING_TEMPLATE,
+      scope: {
+        config: configInstance,
+        columns: self.columns,
+        resetAxisConfig: () => {
+          resetAxisConfig(configInstance)
+          self.emitChartChange(configInstance)
+        },
+
+        resetParameterConfig: () => {
+          resetParameterConfig(configInstance)
+          self.emitParameterChange(configInstance)
+        },
+
+        toggleColumnPanel: () => {
+          configInstance.panel.columnPanelOpened = !configInstance.panel.columnPanelOpened
+          self.emitConfigChange(configInstance)
+        },
+
+        toggleParameterPanel: () => {
+          configInstance.panel.parameterPanelOpened = !configInstance.panel.parameterPanelOpened
+          self.emitConfigChange(configInstance)
+        },
+
+        getAxisAnnotation: (axisSpec) => {
+          let anno = `${axisSpec.name}`
+          if (axisSpec.valueType) {
+            anno = `${anno} (${axisSpec.valueType})`
+          }
+
+          return anno
+        },
+
+        getAxisTypeAnnotation: (axisSpec) => {
+          let anno = `${axisSpec.axisType}`
+
+          let minAxisCount = axisSpec.minAxisCount
+          let maxAxisCount = axisSpec.maxAxisCount
+
+          if (isSingleDimensionAxis(axisSpec)) {
+            maxAxisCount = 1
+          }
+
+          let comment = ''
+          if (minAxisCount) { comment = `min: ${minAxisCount}` }
+          if (minAxisCount && maxAxisCount) { comment = `${comment}, `}
+          if (maxAxisCount) { comment = `${comment}max: ${maxAxisCount}` }
+
+          if (comment !== '') {
+            anno = `${anno} (${comment})`
+          }
+
+          return anno
+        },
+
+        getAxisTypeAnnotationColor: (axisSpec) => {
+          if (isAggregatorAxis(axisSpec)) {
+            return { 'background-color': '#5782bd' };
+          } else if (isGroupAxis(axisSpec)) {
+            return { 'background-color': '#cd5c5c' };
+          } else if (isKeyAxis(axisSpec)) {
+            return { 'background-color': '#906ebd' };
+          } else {
+            return { 'background-color': '#62bda9' };
+          }
+        },
+
+        useSharedAxis: (chartName) => { return useSharedAxis(configInstance, chartName) },
+        isGroupAxis: (axisSpec) => { return isGroupAxis(axisSpec) },
+        isKeyAxis: (axisSpec) => { return isKeyAxis(axisSpec) },
+        isAggregatorAxis: (axisSpec) => { return isAggregatorAxis(axisSpec) },
+        isSingleDimensionAxis: (axisSpec) => { return isSingleDimensionAxis(axisSpec) },
+        getSingleDimensionAxis: (axisSpec) => { return getCurrentChartAxis(configInstance)[axisSpec.name] },
+
+        chartChanged: (selected) => {
+          configInstance.chart.current = selected
+          self.emitChartChange(configInstance)
+        },
+
+        axisChanged: function(e, ui, axisSpec) {
+          removeDuplicatedColumnsInMultiDimensionAxis(configInstance, axisSpec)
+          applyMaxAxisCount(configInstance, axisSpec)
+
+          self.emitChartChange(configInstance)
+        },
+
+        aggregatorChanged: (colIndex, axisSpec, aggregator) => {
+          if (isSingleDimensionAxis(axisSpec)) {
+            getCurrentChartAxis(configInstance)[axisSpec.name].aggr = aggregator
+          } else {
+            getCurrentChartAxis(configInstance)[axisSpec.name][colIndex].aggr = aggregator
+            removeDuplicatedColumnsInMultiDimensionAxis(configInstance, axisSpec)
+          }
+
+          self.emitChartChange(configInstance)
+        },
+
+        removeFromAxis: function(colIndex, axisSpec) {
+          if (isSingleDimensionAxis(axisSpec)) {
+            getCurrentChartAxis(configInstance)[axisSpec.name] = null
+          } else {
+            getCurrentChartAxis(configInstance)[axisSpec.name].splice(colIndex, 1)
+          }
+
+          self.emitChartChange(configInstance)
+        },
+
+        isInputWidget: function(paramSpec) { return isInputWidget(paramSpec) },
+        isCheckboxWidget: function(paramSpec) { return isCheckboxWidget(paramSpec) },
+        isOptionWidget: function(paramSpec) { return isOptionWidget(paramSpec) },
+        isTextareaWidget: function(paramSpec) { return isTextareaWidget(paramSpec) },
+
+        parameterChanged: (paramSpec) => {
+
+          configInstance.chartChanged = false
+          configInstance.parameterChanged = true
+          self.emitParameterChange(configInstance)
+        },
+
+        parameterOnKeyDown: function(event, paramSpec) {
+          const code = event.keyCode || event.which;
+          if (code === 13 && isInputWidget(paramSpec)) {
+            self.emitParameterChange(configInstance)
+          } else if (code === 13 && event.shiftKey && isTextareaWidget(paramSpec)) {
+            self.emitParameterChange(configInstance)
+          }
+
+          event.stopPropagation() /** avoid to conflict with paragraph shortcuts */
+        },
+
+      }
+    }
+  }
+
+  transform(tableData) {
+    this.columns = tableData.columns; /** used in `getSetting` */
+    /** initialize in `transform` instead of `getSetting` because this method is called before */
+    serializeSharedAxes(this.config)
+
+    const conf = this.config
+    const chart = getCurrentChart(conf)
+    const axis = getCurrentChartAxis(conf)
+    const axisSpecs = getCurrentChartAxisSpecs(conf)
+    const param = getCurrentChartParam(conf)
+    const paramSpecs = getCurrentChartParamSpecs(conf)
+    const parsedParam = parseParameter(paramSpecs, param)
+
+   let { transformer, column, }  = getTransformer(conf, tableData.rows, axisSpecs, axis)
+
+    return {
+      chartChanged: conf.chartChanged,
+      parameterChanged: conf.parameterChanged,
+
+      chart: chart, /** current chart */
+      axis: axis, /** persisted axis */
+      parameter: parsedParam, /** persisted parameter */
+      column: column,
+
+      transformer: transformer,
+    }
+  }
+}

zeppelin-web/src/components/noteName-create/notename.controller.js

@@ -44,14 +44,14 @@ function NotenameCtrl($scope, noteListDataFactory, $routeParams, websocketMsgSrv
     vm.createNote();
   };
 
-  vm.preVisible = function(clone, sourceNoteName) {
+  vm.preVisible = function(clone, sourceNoteName, path) {
     vm.clone = clone;
     vm.sourceNoteName = sourceNoteName;
-    $scope.note.notename = vm.clone ? vm.cloneNoteName() : vm.newNoteName();
+    $scope.note.notename = vm.clone ? vm.cloneNoteName() : vm.newNoteName(path);
     $scope.$apply();
   };
 
-  vm.newNoteName = function() {
+  vm.newNoteName = function(path) {
     var newCount = 1;
     angular.forEach(vm.notes.flatList, function(noteName) {
       noteName = noteName.name;
@@ -62,7 +62,7 @@ function NotenameCtrl($scope, noteListDataFactory, $routeParams, websocketMsgSrv
         }
       }
     });
-    return 'Untitled Note ' + newCount;
+    return (path ? path + '/' : '') +'Untitled Note ' + newCount;
   };
 
   vm.cloneNoteName = function() {

zeppelin-web/src/components/noteName-create/visible.directive.js

@@ -30,8 +30,9 @@ function modalvisible() {
         var relatedTarget = angular.element(e.relatedTarget);
         var clone = relatedTarget.data('clone');
         var sourceNoteName = relatedTarget.data('source-note-name');
+        var path = relatedTarget.data('path');
         var cloneNote = clone ? true : false;
-        previsibleMethod()(cloneNote, sourceNoteName);
+        previsibleMethod()(cloneNote, sourceNoteName, path);
       });
       element.on('shown.bs.modal', function(e) {
         if (scope.targetinput) {

zeppelin-web/src/index.js

@@ -23,6 +23,7 @@ import './app/tabledata/transformation.js';
 import './app/tabledata/pivot.js';
 import './app/tabledata/passthrough.js';
 import './app/tabledata/columnselector.js';
+import './app/tabledata/advanced-transformation.js';
 import './app/visualization/visualization.js';
 import './app/visualization/builtins/visualization-table.js';
 import './app/visualization/builtins/visualization-nvd3chart.js';

zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumBundleFactory.java

@@ -55,6 +55,7 @@ public class HeliumBundleFactory {
   public static final String HELIUM_LOCAL_MODULE_DIR = "local_modules";
   public static final String HELIUM_BUNDLES_SRC_DIR = "src";
   public static final String HELIUM_BUNDLES_SRC = "load.js";
+  public static final String YARN_CACHE_DIR = "yarn-cache";
   public static final String PACKAGE_JSON = "package.json";
   public static final String HELIUM_BUNDLE_CACHE = "helium.bundle.cache.js";
   public static final String HELIUM_BUNDLE = "helium.bundle.js";
@@ -68,6 +69,7 @@ public class HeliumBundleFactory {
   private final File heliumLocalRepoDirectory;
   private final File heliumBundleDirectory;
   private final File heliumLocalModuleDirectory;
+  private final File yarnCacheDir;
   private ZeppelinConfiguration conf;
   private File tabledataModulePath;
   private File visualizationModulePath;
@@ -98,6 +100,7 @@ public class HeliumBundleFactory {
     this.heliumLocalRepoDirectory = new File(moduleDownloadPath, HELIUM_LOCAL_REPO);
     this.heliumBundleDirectory = new File(heliumLocalRepoDirectory, HELIUM_BUNDLES_DIR);
     this.heliumLocalModuleDirectory = new File(heliumLocalRepoDirectory, HELIUM_LOCAL_MODULE_DIR);
+    this.yarnCacheDir = new File(heliumLocalRepoDirectory, YARN_CACHE_DIR);
     this.conf = conf;
     this.defaultNpmRegistryUrl = conf.getHeliumNpmRegistry();
 
@@ -110,7 +113,7 @@ public class HeliumBundleFactory {
     gson = new Gson();
   }
 
-  void installNodeAndNpm() {
+  void installNodeAndNpm() throws TaskRunnerException {
     if (nodeAndNpmInstalled) {
       return;
     }
@@ -126,6 +129,8 @@ public class HeliumBundleFactory {
       YarnInstaller yarnInstaller = frontEndPluginFactory.getYarnInstaller(getProxyConfig());
       yarnInstaller.setYarnVersion(YARN_VERSION);
       yarnInstaller.install();
+      String yarnCacheDirPath = yarnCacheDir.getAbsolutePath();
+      yarnCommand(frontEndPluginFactory, "config set cache-folder " + yarnCacheDirPath);
 
       configureLogger();
       nodeAndNpmInstalled = true;
@@ -362,7 +367,11 @@ public class HeliumBundleFactory {
     }
 
     // 0. install node, npm (should be called before `downloadPackage`
-    installNodeAndNpm();
+    try {
+      installNodeAndNpm();
+    } catch (TaskRunnerException e) {
+      throw new IOException(e);
+    }
 
     // 1. prepare directories
     if (!heliumLocalRepoDirectory.exists() || !heliumLocalRepoDirectory.isDirectory()) {
@@ -391,7 +400,7 @@ public class HeliumBundleFactory {
     prepareSource(pkg, moduleNameVersion, mainFileName);
 
     // 4. install node and local modules for a bundle
-    copyFrameworkModuleToInstallPath(recopyLocalModule); // should copy local modules first
+    copyFrameworkModulesToInstallPath(recopyLocalModule); // should copy local modules first
     installNodeModules(fpf);
 
     // 5. let's bundle and update cache
@@ -421,7 +430,44 @@ public class HeliumBundleFactory {
     }
   }
 
-  void copyFrameworkModuleToInstallPath(boolean recopy)
+  void copyFrameworkModule(boolean recopy, FileFilter filter,
+                           File src, File dest) throws IOException {
+    if (src != null) {
+      if (recopy && dest.exists()) {
+        FileUtils.deleteDirectory(dest);
+      }
+
+      if (!dest.exists()) {
+        FileUtils.copyDirectory(
+            src,
+            dest,
+            filter);
+      }
+    }
+  }
+
+  void deleteYarnCache() {
+    FilenameFilter filter = new FilenameFilter() {
+      @Override
+      public boolean accept(File dir, String name) {
+        if ((name.startsWith("npm-zeppelin-vis-") ||
+            name.startsWith("npm-zeppelin-tabledata-") ||
+            name.startsWith("npm-zeppelin-spell-")) &&
+            dir.isDirectory()) {
+          return true;
+        }
+
+        return false;
+      }
+    };
+
+    File[] localModuleCaches = yarnCacheDir.listFiles(filter);
+    for (File f : localModuleCaches) {
+      FileUtils.deleteQuietly(f);
+    }
+  }
+
+  void copyFrameworkModulesToInstallPath(boolean recopy)
       throws IOException {
 
     FileFilter npmPackageCopyFilter = new FileFilter() {
@@ -436,56 +482,27 @@ public class HeliumBundleFactory {
       }
     };
 
-    // install tabledata module
     FileUtils.forceMkdir(heliumLocalModuleDirectory);
+    // should delete yarn caches for local modules since they might be updated
+    deleteYarnCache();
 
+    // install tabledata module
     File tabledataModuleInstallPath = new File(heliumLocalModuleDirectory,
         "zeppelin-tabledata");
-    if (tabledataModulePath != null) {
-      if (recopy && tabledataModuleInstallPath.exists()) {
-        FileUtils.deleteDirectory(tabledataModuleInstallPath);
-
-      }
-
-      if (!tabledataModuleInstallPath.exists()) {
-        FileUtils.copyDirectory(
-            tabledataModulePath,
-            tabledataModuleInstallPath,
-            npmPackageCopyFilter);
-      }
-    }
+    copyFrameworkModule(recopy, npmPackageCopyFilter,
+        tabledataModulePath, tabledataModuleInstallPath);
 
     // install visualization module
     File visModuleInstallPath = new File(heliumLocalModuleDirectory,
         "zeppelin-vis");
-    if (visualizationModulePath != null) {
-      if (recopy && visModuleInstallPath.exists()) {
-        FileUtils.deleteDirectory(visModuleInstallPath);
-      }
-
-      if (!visModuleInstallPath.exists()) {
-        FileUtils.copyDirectory(
-            visualizationModulePath,
-            visModuleInstallPath,
-            npmPackageCopyFilter);
-      }
-    }
+    copyFrameworkModule(recopy, npmPackageCopyFilter,
+        visualizationModulePath, visModuleInstallPath);
 
     // install spell module
     File spellModuleInstallPath = new File(heliumLocalModuleDirectory,
         "zeppelin-spell");
-    if (spellModulePath != null) {
-      if (recopy && spellModuleInstallPath.exists()) {
-        FileUtils.deleteDirectory(spellModuleInstallPath);
-      }
-
-      if (!spellModuleInstallPath.exists()) {
-        FileUtils.copyDirectory(
-            spellModulePath,
-            spellModuleInstallPath,
-            npmPackageCopyFilter);
-      }
-    }
+    copyFrameworkModule(recopy, npmPackageCopyFilter,
+        spellModulePath, spellModuleInstallPath);
   }
 
   private WebpackResult getWebpackResultFromOutput(String output) {

zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java

@@ -20,6 +20,7 @@ package org.apache.zeppelin.notebook;
 import com.google.common.collect.Maps;
 import com.google.common.base.Strings;
 import org.apache.commons.lang.StringUtils;
+import org.apache.zeppelin.completer.CompletionType;
 import org.apache.zeppelin.display.AngularObject;
 import org.apache.zeppelin.display.AngularObjectRegistry;
 import org.apache.zeppelin.helium.HeliumPackage;
@@ -267,10 +268,11 @@ public class Paragraph extends Job implements Serializable, Cloneable {
       if (intInfo.size() > 1) {
         for (InterpreterInfo info : intInfo) {
           String name = intp.getName() + "." + info.getName();
-          completion.add(new InterpreterCompletion(name, name));
+          completion.add(new InterpreterCompletion(name, name, CompletionType.setting.name()));
         }
       } else {
-        completion.add(new InterpreterCompletion(intp.getName(), intp.getName()));
+        completion.add(new InterpreterCompletion(intp.getName(), intp.getName(),
+            CompletionType.setting.name()));
       }
     }
     return completion;
@@ -297,7 +299,9 @@ public class Paragraph extends Job implements Serializable, Cloneable {
       return null;
     }
 
-    List completion = repl.completion(body, cursor);
+    InterpreterContext interpreterContext = getInterpreterContextWithoutRunner(null);
+
+    List completion = repl.completion(body, cursor, interpreterContext);
     return completion;
   }
 
@@ -533,7 +537,9 @@ public class Paragraph extends Job implements Serializable, Cloneable {
     final Paragraph self = this;
 
     Credentials credentials = note.getCredentials();
-    if (authenticationInfo != null) {
+    setAuthenticationInfo(new AuthenticationInfo(getUser()));
+
+    if (authenticationInfo.getUser() != null) {
       UserCredentials userCredentials =
           credentials.getUserCredentials(authenticationInfo.getUser());
       authenticationInfo.setUserCredentials(userCredentials);

zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumBundleFactoryTest.java

@@ -66,7 +66,7 @@ public class HeliumBundleFactoryTest {
         new File(moduleDir, "visualization"),
         new File(moduleDir, "spell"));
     hbf.installNodeAndNpm();
-    hbf.copyFrameworkModuleToInstallPath(true);
+    hbf.copyFrameworkModulesToInstallPath(true);
   }
 
   @After

zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/mock/MockInterpreter1.java

@@ -98,7 +98,8 @@ Map<String, Object> vars = new HashMap<>();
 	}
 
 	@Override
-	public List<InterpreterCompletion> completion(String buf, int cursor) {
+	public List<InterpreterCompletion> completion(String buf, int cursor,
+			InterpreterContext interpreterContext) {
 		return null;
 	}
 }

zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/mock/MockInterpreter11.java

@@ -76,7 +76,8 @@ public class MockInterpreter11 extends Interpreter{
   }
 
   @Override
-  public List<InterpreterCompletion> completion(String buf, int cursor) {
+  public List<InterpreterCompletion> completion(String buf, int cursor,
+      InterpreterContext interpreterContext) {
     return null;
   }
 }

zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/mock/MockInterpreter2.java

@@ -97,7 +97,8 @@ public class MockInterpreter2 extends Interpreter{
 	}
 
 	@Override
-	public List<InterpreterCompletion> completion(String buf, int cursor) {
+	public List<InterpreterCompletion> completion(String buf, int cursor,
+			InterpreterContext interpreterContext) {
 		return null;
 	}
 }